diff --git a/cmd/verdande/main.go b/cmd/verdande/main.go index acb4e5f..874a342 100644 --- a/cmd/verdande/main.go +++ b/cmd/verdande/main.go @@ -74,6 +74,7 @@ func run() error { defer stopJobs() runner := jobs.New(cfg, db, log, api.Mail(), api.Hub()) runner.SyncGmail = api.SyncGmail + runner.SyncMailbox = api.SyncMailbox runner.Start(jobCtx) srv := &http.Server{ diff --git a/internal/httpapi/errors.go b/internal/httpapi/errors.go index 1be61b5..1bee44d 100644 --- a/internal/httpapi/errors.go +++ b/internal/httpapi/errors.go @@ -58,6 +58,11 @@ const ( // read "something went wrong", and had nowhere to go. The interface wraps the // message in one that says where it came from. CodeGmailFailed = "gmail_failed" + + // And the same for a mailbox read over IMAP. A host's refusal is the whole + // diagnosis — a wrong app password, a host that is not listening, a folder that + // is not there — and it says it in its own words. + CodeMailboxFailed = "mailbox_failed" ) // errLastAdmin is returned to the caller that already wrote the response, so it diff --git a/internal/httpapi/mailbox_handlers.go b/internal/httpapi/mailbox_handlers.go new file mode 100644 index 0000000..9b05b6a --- /dev/null +++ b/internal/httpapi/mailbox_handlers.go @@ -0,0 +1,211 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-chi/chi/v5" + "github.com/kristianwind/verdande/internal/imap" + "github.com/kristianwind/verdande/internal/store" +) + +// defaultMailboxSyncBudget bounds one read of one mailbox when somebody is waiting. +// The same reasoning as Gmail's: a mailbox that has stopped answering must not +// hold a request open until whatever sits in front of this server answers for it. +const defaultMailboxSyncBudget = 25 * time.Second + +func (s *Server) handleListMailboxes(w http.ResponseWriter, r *http.Request) { + user := userFrom(r.Context()) + list, err := s.db.Mailboxes(r.Context(), user.ID) + if err != nil { + s.internal(w, r, "list mailboxes", err) + return + } + if list == nil { + list = []store.Mailbox{} + } + writeJSON(w, http.StatusOK, map[string]any{"mailboxes": list}) +} + +// handleAddMailbox connects one, after proving it can actually be read. +// +// Tested before it is saved, not after: a mailbox that was accepted and then +// silently fails every ten minutes is worse than one that refused at the door, +// where the person is still holding the password they just typed. +func (s *Server) handleAddMailbox(w http.ResponseWriter, r *http.Request) { + var body struct { + Name string `json:"name"` + Host string `json:"host"` + Username string `json:"username"` + Password string `json:"password"` + Folder string `json:"folder"` + } + if err := decodeJSON(w, r, &body); err != nil { + return + } + + body.Host = strings.TrimSpace(body.Host) + body.Username = strings.TrimSpace(body.Username) + if body.Host == "" || body.Username == "" || body.Password == "" { + writeError(w, http.StatusBadRequest, CodeValidation, + "a mailbox needs a host, a username and a password") + return + } + // A host with no port is the mistake everyone makes once, and 993 is the answer + // for every host worth naming. + if !strings.Contains(body.Host, ":") { + body.Host += ":993" + } + + account := imap.Account{ + Host: body.Host, Username: body.Username, + Password: body.Password, Folder: body.Folder, + } + client, err := imap.Dial(account) + if err != nil { + s.upstream(w, r, CodeMailboxFailed, "connect mailbox", err) + return + } + client.Close() + + user := userFrom(r.Context()) + m := &store.Mailbox{ + UserID: user.ID, Kind: "imap", Name: strings.TrimSpace(body.Name), + Host: body.Host, Username: body.Username, + Password: body.Password, Folder: body.Folder, + } + if m.Name == "" { + m.Name = body.Username + } + if err := s.db.SaveMailbox(r.Context(), m); err != nil { + // The unique index. Saying which mailbox is already here beats "constraint + // failed", which is true and useless. + if strings.Contains(err.Error(), "UNIQUE") { + writeError(w, http.StatusConflict, CodeValidation, + fmt.Sprintf("%s on %s is already connected", body.Username, body.Host)) + return + } + s.internal(w, r, "save mailbox", err) + return + } + writeJSON(w, http.StatusCreated, m) +} + +func (s *Server) handleDeleteMailbox(w http.ResponseWriter, r *http.Request) { + user := userFrom(r.Context()) + if err := s.db.DeleteMailbox(r.Context(), user.ID, chi.URLParam(r, "mailboxID")); err != nil { + s.internal(w, r, "delete mailbox", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleSyncMailbox reads one mailbox now, within a budget of its own. +func (s *Server) handleSyncMailbox(w http.ResponseWriter, r *http.Request) { + user := userFrom(r.Context()) + m, err := s.db.Mailbox(r.Context(), user.ID, chi.URLParam(r, "mailboxID")) + if err != nil { + s.internal(w, r, "read mailbox", err) + return + } + if m == nil { + writeError(w, http.StatusNotFound, CodeNotFound, "no such mailbox") + return + } + + ctx, cancel := context.WithTimeout(r.Context(), defaultMailboxSyncBudget) + defer cancel() + + created, err := s.SyncMailbox(ctx, user, m) + if err != nil { + // Whatever was made by the deadline is kept: each task is committed as it + // is made, so a short budget is a shorter run rather than a lost one. + if errors.Is(err, context.DeadlineExceeded) { + writeJSON(w, http.StatusOK, map[string]any{"created": created, "partial": true}) + return + } + s.upstream(w, r, CodeMailboxFailed, "sync mailbox", err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"created": created}) +} + +// SyncMailbox turns the flagged mail newer than the last run into tasks. +// +// Exported because the background job calls it too, on the same terms and without +// a budget: nobody is waiting there. +func (s *Server) SyncMailbox(ctx context.Context, user *store.User, m *store.Mailbox) (int, error) { + client, err := imap.Dial(imap.Account{ + Host: m.Host, Username: m.Username, Password: m.Password, Folder: m.Folder, + }) + if err != nil { + return 0, err + } + defer client.Close() + + messages, err := client.Since(m.LastUID, 25) + if err != nil { + return 0, err + } + if len(messages) == 0 { + return 0, nil + } + + projectID, err := s.db.InboxID(ctx, user.ID) + if err != nil { + return 0, err + } + + created := 0 + highest := m.LastUID + for _, msg := range messages { + if ctx.Err() != nil { + // What was made so far is already committed and counted, and the marker + // below has to be written for it: a run that created tasks and forgot how + // far it got would make them all again on the next pass. + break + } + + subject := msg.Subject + if subject == "" { + subject = "(uden emne)" + } + content := subject + if msg.From != "" { + content = msg.From + ": " + subject + } + + task := &store.Task{ + ProjectID: projectID, + Content: content, + Description: msg.Snippet, + Priority: 4, + CreatedBy: user.ID, + } + if err := s.db.CreateTask(ctx, task, nil); err != nil { + s.log.Warn("mailbox create task", "err", err, "mailbox", m.ID) + continue + } + created++ + if msg.UID > highest { + highest = msg.UID + } + s.hub.Publish(projectID, "task.created", toTaskJSON(*task)) + } + + // Written even when the loop was cut short, and outside the request's context + // so a cancelled request still records what it did. + if highest > m.LastUID { + if err := s.db.MarkMailboxRead(context.WithoutCancel(ctx), m.ID, highest, time.Now()); err != nil { + s.log.Warn("mark mailbox read", "err", err, "mailbox", m.ID) + } + } + if ctx.Err() != nil { + return created, ctx.Err() + } + return created, nil +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 3be111d..d3913cf 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -260,6 +260,16 @@ func New(cfg *config.Config, db *store.DB, log *slog.Logger, web fs.FS) *Server r.Get("/version", s.handleVersion) + // Mailboxes read over IMAP — iCloud, Fastmail, any ordinary host. Each + // belongs to the person who connected it, so there is no admin gate + // here and no shared registration: the credential is theirs. + r.Route("/mailboxes", func(r chi.Router) { + r.Get("/", s.handleListMailboxes) + r.Post("/", s.handleAddMailbox) + r.Delete("/{mailboxID}", s.handleDeleteMailbox) + r.Post("/{mailboxID}/sync", s.handleSyncMailbox) + }) + r.Route("/gmail", func(r chi.Router) { r.Get("/", s.handleGetGmail) r.Put("/", s.handleSetGmail) diff --git a/internal/imap/imap.go b/internal/imap/imap.go index 2aa2515..ba4b6e8 100644 --- a/internal/imap/imap.go +++ b/internal/imap/imap.go @@ -17,6 +17,7 @@ package imap import ( "crypto/tls" + "crypto/x509" "fmt" "net" "strings" @@ -41,6 +42,13 @@ type Account struct { Username string Password string // an app-specific password on every host worth naming Folder string // "INBOX" when empty + + // RootCAs adds a certificate authority to trust, on top of the system's. It + // exists so a test can put a real IMAP server behind a self-signed certificate + // and still be refused by everything else — the field can only widen trust, + // never switch verification off, which is the difference between a seam and a + // hole. Nil in production, where the system's roots are the answer. + RootCAs *x509.CertPool } // Dial opens a TLS connection and logs in. The caller closes it. @@ -55,7 +63,11 @@ func Dial(a Account) (*Client, error) { } c, err := imapclient.DialTLS(a.Host, &imapclient.Options{ - TLSConfig: &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}, + TLSConfig: &tls.Config{ + ServerName: host, + MinVersion: tls.VersionTLS12, + RootCAs: a.RootCAs, + }, }) if err != nil { return nil, fmt.Errorf("connect to %s: %w", a.Host, err) diff --git a/internal/imap/roundtrip_test.go b/internal/imap/roundtrip_test.go new file mode 100644 index 0000000..5eea409 --- /dev/null +++ b/internal/imap/roundtrip_test.go @@ -0,0 +1,183 @@ +package imap + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "strings" + "testing" + "time" + + "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapserver" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" +) + +// The round trip, against a real IMAP server rather than a stub of one. The +// earlier tests cover the pure parts — what a sender renders as, where a snippet +// is cut — and would all have passed with the fetch itself broken. +func TestFlaggedMailComesBackAsMessages(t *testing.T) { + addr, pool := serveMailbox(t, []mail{ + {subject: "Faktura 4711", from: "Anders Bo ", body: "Vedhæftet.", flagged: true}, + {subject: "Nyhedsbrev", from: "noreply@example.com", body: "Læs mere", flagged: false}, + {subject: "Kontrakt", from: "jura@example.dk", body: "Til underskrift", flagged: true}, + }) + + client, err := Dial(Account{ + Host: addr, Username: "kw", Password: "hemmelig", RootCAs: pool, + }) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + msgs, err := client.Since(0, 25) + if err != nil { + t.Fatal(err) + } + + // Two of the three. The newsletter is unflagged, and flagging is the whole + // signal: unread is a state the owner is still using. + if len(msgs) != 2 { + t.Fatalf("got %d messages, want the 2 flagged ones", len(msgs)) + } + if msgs[0].Subject != "Faktura 4711" { + t.Errorf("first subject is %q", msgs[0].Subject) + } + if !strings.Contains(msgs[0].From, "anders@example.dk") { + t.Errorf("first sender is %q", msgs[0].From) + } + if msgs[0].UID == 0 { + t.Error("no uid came back, so a second run would fetch everything again") + } + + // And reading on from a marker returns only what is newer, which is the whole + // of the deduplication. + rest, err := client.Since(msgs[0].UID, 25) + if err != nil { + t.Fatal(err) + } + if len(rest) != 1 || rest[0].Subject != "Kontrakt" { + t.Fatalf("reading on from uid %d gave %d messages", msgs[0].UID, len(rest)) + } +} + +func TestAWrongPasswordIsRefusedClearly(t *testing.T) { + addr, pool := serveMailbox(t, nil) + _, err := Dial(Account{Host: addr, Username: "kw", Password: "forkert", RootCAs: pool}) + if err == nil { + t.Fatal("a wrong password was accepted") + } + // Hosts say everything from AUTHENTICATIONFAILED to a sentence about the web, + // so the reply names the host and the user rather than repeating their words. + if !strings.Contains(err.Error(), "kw") { + t.Errorf("the error does not say whose login was refused: %v", err) + } +} + +// A certificate this process signed itself must not be enough on its own. +func TestAnUntrustedCertificateIsRefused(t *testing.T) { + addr, _ := serveMailbox(t, nil) + if _, err := Dial(Account{Host: addr, Username: "kw", Password: "hemmelig"}); err == nil { + t.Fatal("a self-signed certificate was accepted without being trusted") + } +} + +type mail struct { + subject, from, body string + flagged bool +} + +// serveMailbox starts an in-process IMAP server over TLS and returns its address +// and the pool that trusts it. +func serveMailbox(t *testing.T, mails []mail) (string, *x509.CertPool) { + t.Helper() + + cert, pool := selfSigned(t) + memServer := imapmemserver.New() + user := imapmemserver.NewUser("kw", "hemmelig") + if err := user.Create("INBOX", nil); err != nil { + t.Fatal(err) + } + memServer.AddUser(user) + + for _, m := range mails { + flags := []imap.Flag{} + if m.flagged { + flags = append(flags, imap.FlagFlagged) + } + raw := "From: " + m.from + "\r\nSubject: " + m.subject + + "\r\nDate: Tue, 18 Aug 2026 10:00:00 +0200\r\n\r\n" + m.body + "\r\n" + if _, err := user.Append("INBOX", strings.NewReader(raw), &imap.AppendOptions{ + Flags: flags, + }); err != nil { + t.Fatal(err) + } + } + + srv := imapserver.New(&imapserver.Options{ + NewSession: func(_ *imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return memServer.NewSession(), nil, nil + }, + TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}}, + InsecureAuth: false, + }) + t.Cleanup(func() { srv.Close() }) + + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}}) + if err != nil { + t.Fatal(err) + } + go srv.Serve(ln) + + // "localhost", not the loopback address: the certificate names a host, and the + // client checks that name — as it should. + _, port, _ := net.SplitHostPort(ln.Addr().String()) + return net.JoinHostPort("localhost", port), pool +} + +func selfSigned(t *testing.T) (tls.Certificate, *x509.CertPool) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "localhost"}, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(certPEM) + return cert, pool +} diff --git a/internal/jobs/jobs.go b/internal/jobs/jobs.go index 9038ed8..1261ef8 100644 --- a/internal/jobs/jobs.go +++ b/internal/jobs/jobs.go @@ -37,6 +37,10 @@ type Runner struct { // startable from main. SyncGmail func(ctx context.Context, user *store.User) (int, error) + // SyncMailbox reads one IMAP mailbox. Supplied by the HTTP layer for the same + // reason as SyncGmail: it owns turning a message into a task. + SyncMailbox func(ctx context.Context, user *store.User, m *store.Mailbox) (int, error) + wg sync.WaitGroup } @@ -61,6 +65,7 @@ func (r *Runner) Start(ctx context.Context) { // account for a to-do app. Ten minutes is fast enough for something somebody // starred and slow enough not to look like abuse. r.every(ctx, "gmail", 10*time.Minute, r.syncGmailAccounts) + r.every(ctx, "mailboxes", 10*time.Minute, r.syncMailboxes) } func (r *Runner) Wait() { r.wg.Wait() } @@ -272,6 +277,44 @@ func (r *Runner) syncGmailAccounts(ctx context.Context) error { return nil } +// syncMailboxes polls every connected IMAP mailbox. +// +// Nobody is waiting, so no budget of its own — the request handler has one because +// a person is watching a spinner. A host that has gone quiet costs this run its +// own dial timeout and nothing else: the loop moves on, per mailbox and per +// person, so one bad password does not stop everybody else's mail. +func (r *Runner) syncMailboxes(ctx context.Context) error { + if r.SyncMailbox == nil { + return nil + } + userIDs, err := r.db.UsersWithMailboxes(ctx) + if err != nil { + return err + } + for _, id := range userIDs { + user, err := r.db.UserByID(ctx, id) + if err != nil || user == nil { + continue + } + boxes, err := r.db.Mailboxes(ctx, id) + if err != nil { + r.log.Warn("list mailboxes", "err", err, "user", id) + continue + } + for i := range boxes { + created, err := r.SyncMailbox(ctx, user, &boxes[i]) + if err != nil { + r.log.Warn("mailbox sync", "err", err, "user", id, "mailbox", boxes[i].ID) + continue + } + if created > 0 { + r.log.Info("tasks created from a mailbox", "user", id, "count", created) + } + } + } + return nil +} + // --- housekeeping ----------------------------------------------------------------- func (r *Runner) emptyTrash(ctx context.Context) error {