verdande/internal/store/search.go

package store

import (
	"strings"
	"unicode"
)

// FoldDanish transliterates the Danish letters that Unicode does not treat as
// accented forms of anything, so they survive into a form FTS5 can fold further.
//
// This has to be applied to the query with exactly the same mapping the `fold`
// column on `tasks` uses. The two are a pair: change one and searching quietly
// stops finding half the words in the user's own language.
func FoldDanish(s string) string {
	r := strings.NewReplacer(
		"ø", "o", "Ø", "o",
		"æ", "ae", "Æ", "ae",
		"å", "aa", "Å", "aa",
	)
	return r.Replace(s)
}

// The ceilings on a search. See MatchExprOver for why they exist; both are far
// above anything a person types and far below anything that costs.
const (
	maxSearchInput = 512
	maxSearchTerms = 16
)

// MatchExpr turns what someone typed into the search box into an FTS5 MATCH
// expression.
//
// User input is never passed to FTS5 as-is. The MATCH grammar has its own operators
// (`AND`, `NEAR`, `*`, `^`, `:`, `-`), so a perfectly ordinary task title like
// "budget: q3 - final" parses as a query and fails with a syntax error rather than
// finding anything. Every term is quoted, which makes it a literal.
//
// Each term must match somewhere in the row, and may match either the text as
// written or its folded form — so "gron" finds "grøn" and "grøn" still finds
// itself. Terms are prefix-matched, which is what makes the Cmd+K box feel like it
// is searching as you type.
//
// Returns "" when there is nothing searchable in the input; callers should treat
// that as "no query" rather than as "match everything".
func MatchExpr(query string) string {
	return MatchExprOver(query, "content", "description")
}

// MatchExprOver is MatchExpr against a named set of columns.
//
// The columns have to be named because the index they are searched in decides
// them: tasks carry content and description, notes carry title and body. Passing
// one table's column names to the other's index is not a wrong answer but an
// error — "no such column" — which is at least loud.
func MatchExprOver(query string, columns ...string) string {
	terms := tokenize(query)
	if len(terms) == 0 || len(columns) == 0 {
		return ""
	}

	// Capped, because the expression is built from the input and then run inside
	// SQLite on one of a handful of pooled connections.
	//
	// Every term becomes roughly fifty bytes of FTS5, and every one of them is a
	// prefix query — which is what makes ⌘K feel like it searches as you type, and
	// also what makes each one expensive. A quarter of a megabyte pasted into the
	// box became a two-megabyte MATCH expression and half a minute inside SQLite,
	// holding a connection the whole time. A few of those at once and the database
	// is unavailable to everybody on the instance, logins included.
	//
	// Sixteen terms is far past anything a person types into a search box and
	// nowhere near enough to hurt. Silently, because a message about a limit
	// nobody has reached on purpose is noise.
	if len(terms) > maxSearchTerms {
		terms = terms[:maxSearchTerms]
	}

	var b strings.Builder
	for i, term := range terms {
		if i > 0 {
			b.WriteString(" AND ")
		}
		folded := FoldDanish(term)
		b.WriteString(`({` + strings.Join(columns, " ") + `} : `)
		b.WriteString(quote(term))
		b.WriteString(`* OR fold : `)
		b.WriteString(quote(folded))
		b.WriteString(`*)`)
	}
	return b.String()
}

// SearchTerms is the words a query asks about, capped the same way the MATCH
// expression is.
//
// Exported because two searches need the same list: the FTS expression is built
// from it, and so is the substring arm that finds a word living inside another
// word. Two tokenizers would be two answers to "what did they type".
func SearchTerms(query string) []string {
	terms := tokenize(query)
	if len(terms) > maxSearchTerms {
		terms = terms[:maxSearchTerms]
	}
	return terms
}

// tokenize splits on everything that is not a letter or a digit. Splitting on
// punctuation rather than whitespace alone means "q3/2026" searches for "q3" and
// "2026" instead of for a term FTS5's own tokenizer would never have indexed.
func tokenize(query string) []string {
	// Trimmed before it is split, so a megabyte of text is not walked rune by rune
	// only to have all but sixteen of the results thrown away.
	if len(query) > maxSearchInput {
		query = query[:maxSearchInput]
	}
	fields := strings.FieldsFunc(query, func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
	})
	terms := make([]string, 0, len(fields))
	for _, f := range fields {
		if f != "" {
			terms = append(terms, f)
		}
	}
	return terms
}

// quote wraps a term as an FTS5 string literal. Inside one, the only character
// with meaning is the double quote, escaped by doubling it.
func quote(term string) string {
	return `"` + strings.ReplaceAll(term, `"`, `""`) + `"`
}