verdande/internal/imap/imap.go

// Package imap reads a mailbox over IMAP and hands back the messages that should
// become tasks.
//
// This exists next to internal/gmail rather than inside it because the two agree
// on almost nothing. Gmail is an HTTP API with OAuth, labels and stars; IMAP is a
// connection with a password, folders and flags. What they do share is the shape
// of the answer — a handful of messages, each with a sender, a subject and enough
// of a body to read — and that shape is Message here, deliberately identical to
// what the Gmail client returns, so the caller can be told where to look and not
// how to look.
//
// iCloud, Fastmail and any ordinary host speak this. Office 365 no longer accepts
// a password here at all — it wants OAuth of its own — so it is not a host you can
// reach with this package, and pretending otherwise would only fail later and
// less clearly.
package imap

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"net"
	"strings"
	"time"

	"github.com/emersion/go-imap/v2"
	"github.com/emersion/go-imap/v2/imapclient"
	"github.com/emersion/go-message"
)

// Message is one mail, reduced to what a task needs.
type Message struct {
	UID     uint32
	From    string
	Subject string
	Snippet string
	Date    time.Time
}

// Account is one mailbox to read.
type Account struct {
	Host     string // "imap.mail.me.com:993"
	Username string
	Password string // an app-specific password on every host worth naming
	Folder   string // "INBOX" when empty

	// RootCAs adds a certificate authority to trust, on top of the system's. It
	// exists so a test can put a real IMAP server behind a self-signed certificate
	// and still be refused by everything else — the field can only widen trust,
	// never switch verification off, which is the difference between a seam and a
	// hole. Nil in production, where the system's roots are the answer.
	RootCAs *x509.CertPool
}

// Dial opens a TLS connection and logs in. The caller closes it.
//
// TLS is not optional and there is no flag to make it so: this carries a password
// that unlocks a person's whole mailbox, and a switch for turning that protection
// off is a switch that will one day be found already flipped.
func Dial(a Account) (*Client, error) {
	host, _, err := net.SplitHostPort(a.Host)
	if err != nil {
		return nil, fmt.Errorf("host must include a port, like imap.mail.me.com:993: %w", err)
	}

	c, err := imapclient.DialTLS(a.Host, &imapclient.Options{
		TLSConfig: &tls.Config{
			ServerName: host,
			MinVersion: tls.VersionTLS12,
			RootCAs:    a.RootCAs,
		},
	})
	if err != nil {
		return nil, fmt.Errorf("connect to %s: %w", a.Host, err)
	}

	if err := c.Login(a.Username, a.Password).Wait(); err != nil {
		c.Close()
		// Deliberately not repeating the server's own words here. A bad password is
		// the overwhelmingly likely cause, and hosts answer it with everything from
		// "AUTHENTICATIONFAILED" to a sentence about web login being required.
		return nil, fmt.Errorf("%s refused the login for %s: %w", a.Host, a.Username, err)
	}

	folder := a.Folder
	if folder == "" {
		folder = "INBOX"
	}
	return &Client{c: c, folder: folder}, nil
}

// Client is a logged-in connection to one mailbox.
type Client struct {
	c      *imapclient.Client
	folder string
}

// Close ends the session politely, then hangs up regardless.
func (c *Client) Close() error {
	if err := c.c.Logout().Wait(); err != nil {
		return c.c.Close()
	}
	return c.c.Close()
}

// Since returns the flagged messages in the folder newer than uid.
//
// Flagged, not unread: unread is a state the person is still using, and a sync
// that consumed it would be taking something away. Flagging a mail is a deliberate
// act with no other meaning, which is exactly what is wanted — the same reason the
// Gmail side reads stars.
// The second return is the highest uid the search considered. A caller that got
// through every message can move its marker to that, and then it does not matter
// whether the server put a uid in each fetch reply — which some do not, and which
// is how the same mail became a task again on every run.
func (c *Client) Since(uid uint32, limit int) ([]Message, uint32, error) {
	if _, err := c.c.Select(c.folder, &imap.SelectOptions{ReadOnly: true}).Wait(); err != nil {
		return nil, 0, fmt.Errorf("open %s: %w", c.folder, err)
	}

	// From uid+1 to the end. A mailbox with nothing new answers with no uids
	// rather than an error, and that is not a failure worth reporting upwards.
	set := imap.UIDSetNum()
	set.AddRange(imap.UID(uid+1), 0)

	criteria := &imap.SearchCriteria{
		UID:  []imap.UIDSet{set},
		Flag: []imap.Flag{imap.FlagFlagged},
	}
	found, err := c.c.UIDSearch(criteria, nil).Wait()
	if err != nil {
		return nil, 0, fmt.Errorf("search %s: %w", c.folder, err)
	}

	uids := found.AllUIDs()

	// Anything at or below the marker is dropped here rather than trusted to the
	// range in the query. A server that reads `1272:*` as `0:1272` — and one does,
	// which is how the same mail became a task every ten minutes with the marker
	// standing still at 1271 — hands back the whole mailbox. The marker is ours;
	// enforcing it here costs one pass over a list of at most a few hundred
	// numbers and does not depend on anybody else's reading of the spec.
	kept := make([]imap.UID, 0, len(uids))
	for _, u := range uids {
		if uint32(u) > uid {
			kept = append(kept, u)
		}
	}
	uids = kept

	if len(uids) == 0 {
		return nil, uid, nil
	}
	if limit > 0 && len(uids) > limit {
		// The newest ones. A mailbox flagged over years would otherwise make the
		// first sync the largest one, which is the run most likely to be watched.
		uids = uids[len(uids)-limit:]
	}

	want := imap.UIDSetNum(uids...)
	fetch := &imap.FetchOptions{
		Envelope: true,
		UID:      true,
		// The whole message, headers included, so the MIME structure can be walked.
		// TEXT alone gives the body as bytes — boundaries and part headers and all —
		// which is what put "--=_be71cb39… Content-Transfer-Encoding: 8bit" at the
		// front of every task made from a multipart mail.
		BodySection: []*imap.FetchItemBodySection{{Peek: true}},
	}
	var highest uint32
	for _, u := range uids {
		if uint32(u) > highest {
			highest = uint32(u)
		}
	}

	msgs, err := c.c.Fetch(want, fetch).Collect()
	if err != nil {
		return nil, 0, fmt.Errorf("fetch from %s: %w", c.folder, err)
	}

	out := make([]Message, 0, len(msgs))
	for i, m := range msgs {
		if m.Envelope == nil {
			continue
		}

		// The uid comes from the search when the fetch reply does not carry one.
		// A missing uid is not cosmetic: it is the marker the next run starts from,
		// and a marker stuck at zero makes every message arrive again on every run.
		// That is what happened against a real server while the tests stayed green,
		// because the server they run against always answers with it.
		msgUID := uint32(m.UID)
		if msgUID == 0 && i < len(uids) {
			msgUID = uint32(uids[i])
		}

		out = append(out, Message{
			UID:     msgUID,
			From:    address(m.Envelope.From),
			Subject: strings.TrimSpace(m.Envelope.Subject),
			Snippet: snippet(m),
			Date:    m.Envelope.Date,
		})
	}
	return out, highest, nil
}

// address renders the first sender as "Name <addr>", or just the address.
func address(list []imap.Address) string {
	if len(list) == 0 {
		return ""
	}
	a := list[0]
	addr := a.Addr()
	if a.Name == "" {
		return addr
	}
	return a.Name + " <" + addr + ">"
}

// snippet is the first readable text in the mail.
//
// It walks the MIME structure rather than taking the body as bytes. text/plain is
// preferred and HTML is the fallback with its tags stripped — crudely, because
// this is the line under a task's title and not a rendering engine. The mail
// itself stays where it is.
const snippetLimit = 400

func snippet(m *imapclient.FetchMessageBuffer) string {
	for _, body := range m.BodySection {
		if text := readable(body.Bytes); text != "" {
			return cut(text)
		}
	}
	return ""
}

func readable(raw []byte) string {
	entity, err := message.Read(bytes.NewReader(raw))
	if err != nil {
		// Not something this parser recognises. The raw bytes read badly but read,
		// and a mail is not worth losing over its own headers.
		return strings.TrimSpace(string(raw))
	}
	return strings.TrimSpace(fromEntity(entity, 0))
}

// fromEntity descends into a multipart mail. The depth guard is for the mail that
// nests parts inside parts inside parts — rare, and cheaper to bound than to trust.
func fromEntity(entity *message.Entity, depth int) string {
	if depth > 4 {
		return ""
	}

	if parts := entity.MultipartReader(); parts != nil {
		var html string
		for {
			part, err := parts.NextPart()
			if err != nil {
				break
			}
			if text := fromEntity(part, depth+1); text != "" {
				kind, _, _ := part.Header.ContentType()
				if kind != "text/html" {
					return text
				}
				if html == "" {
					html = text
				}
			}
		}
		return html
	}

	body, _ := io.ReadAll(entity.Body)
	kind, _, _ := entity.Header.ContentType()
	if kind == "text/html" {
		return stripTags(string(body))
	}
	return strings.TrimSpace(string(body))
}

// stripTags removes markup without pretending to understand it. Enough for one
// line of preview, and honest about being no more than that.
func stripTags(html string) string {
	var b strings.Builder
	depth := 0
	for _, r := range html {
		switch {
		case r == '<':
			depth++
		case r == '>':
			if depth > 0 {
				depth--
			}
		case depth == 0:
			b.WriteRune(r)
		}
	}
	return strings.Join(strings.Fields(b.String()), " ")
}

func cut(text string) string {
	runes := []rune(text)
	if len(runes) > snippetLimit {
		return string(runes[:snippetLimit]) + "…"
	}
	return text
}

// Unflag takes the flag off the messages that have become tasks.
//
// This is what stops the same mail arriving twice, and it replaces a marker that
// could not be made to hold. The marker was a uid: remember the highest one seen,
// start above it next time. That is correct only if uids are what the server says
// they are, and it broke in every way such a scheme can — a fetch reply without a
// uid, a UIDVALIDITY that changed under a rebuilt mailbox, two mailbox rows
// pointing at the same folder and each keeping its own marker. Every one of those
// produced the same symptom: the same mail became a task again, and again.
//
// Unflagging moves the record out of our database and into the mailbox, where the
// question is asked. "Flagged" means "turn this into a task"; once it is a task,
// the flag has done its job and is not true any more. Nothing has to be
// remembered, so there is nothing to get out of step — a second sync, a second
// mailbox row, or a database restored from a backup all find the same answer,
// because the answer is in the mailbox.
//
// It is destructive in the small way that matters: it changes something in
// somebody's mail. That is the trade, and it is the right way round — a flag is a
// note to self that this needs doing, and it has been done.
//
// The folder is re-selected writable. Since() opens it read-only on purpose, and
// a search should not need write access just because a later step might.
func (c *Client) Unflag(uids []uint32) error {
	if len(uids) == 0 {
		return nil
	}
	if _, err := c.c.Select(c.folder, nil).Wait(); err != nil {
		return fmt.Errorf("open %s for writing: %w", c.folder, err)
	}

	set := imap.UIDSetNum()
	for _, u := range uids {
		if u == 0 {
			// No uid means nothing to address. The message keeps its flag and will
			// be seen again — which is the failure this whole function exists to
			// prevent, so it is worth saying rather than skipping quietly.
			continue
		}
		set.AddNum(imap.UID(u))
	}
	if len(set) == 0 {
		return nil
	}

	// Silent: the server has nothing to tell us that we would act on, and the
	// untagged replies would only have to be collected and thrown away.
	cmd := c.c.Store(set, &imap.StoreFlags{
		Op:     imap.StoreFlagsDel,
		Silent: true,
		Flags:  []imap.Flag{imap.FlagFlagged},
	}, nil)
	if err := cmd.Close(); err != nil {
		return fmt.Errorf("unflag in %s: %w", c.folder, err)
	}
	return nil
}