package store import ( "strings" "unicode" ) // FoldDanish transliterates the Danish letters that Unicode does not treat as // accented forms of anything, so they survive into a form FTS5 can fold further. // // This has to be applied to the query with exactly the same mapping the `fold` // column on `tasks` uses. The two are a pair: change one and searching quietly // stops finding half the words in the user's own language. func FoldDanish(s string) string { r := strings.NewReplacer( "ø", "o", "Ø", "o", "æ", "ae", "Æ", "ae", "å", "aa", "Å", "aa", ) return r.Replace(s) } // The ceilings on a search. See MatchExprOver for why they exist; both are far // above anything a person types and far below anything that costs. const ( maxSearchInput = 512 maxSearchTerms = 16 ) // MatchExpr turns what someone typed into the search box into an FTS5 MATCH // expression. // // User input is never passed to FTS5 as-is. The MATCH grammar has its own operators // (`AND`, `NEAR`, `*`, `^`, `:`, `-`), so a perfectly ordinary task title like // "budget: q3 - final" parses as a query and fails with a syntax error rather than // finding anything. Every term is quoted, which makes it a literal. // // Each term must match somewhere in the row, and may match either the text as // written or its folded form — so "gron" finds "grøn" and "grøn" still finds // itself. Terms are prefix-matched, which is what makes the Cmd+K box feel like it // is searching as you type. // // Returns "" when there is nothing searchable in the input; callers should treat // that as "no query" rather than as "match everything". func MatchExpr(query string) string { return MatchExprOver(query, "content", "description") } // MatchExprOver is MatchExpr against a named set of columns. // // The columns have to be named because the index they are searched in decides // them: tasks carry content and description, notes carry title and body. Passing // one table's column names to the other's index is not a wrong answer but an // error — "no such column" — which is at least loud. func MatchExprOver(query string, columns ...string) string { terms := tokenize(query) if len(terms) == 0 || len(columns) == 0 { return "" } // Capped, because the expression is built from the input and then run inside // SQLite on one of a handful of pooled connections. // // Every term becomes roughly fifty bytes of FTS5, and every one of them is a // prefix query — which is what makes ⌘K feel like it searches as you type, and // also what makes each one expensive. A quarter of a megabyte pasted into the // box became a two-megabyte MATCH expression and half a minute inside SQLite, // holding a connection the whole time. A few of those at once and the database // is unavailable to everybody on the instance, logins included. // // Sixteen terms is far past anything a person types into a search box and // nowhere near enough to hurt. Silently, because a message about a limit // nobody has reached on purpose is noise. if len(terms) > maxSearchTerms { terms = terms[:maxSearchTerms] } var b strings.Builder for i, term := range terms { if i > 0 { b.WriteString(" AND ") } folded := FoldDanish(term) b.WriteString(`({` + strings.Join(columns, " ") + `} : `) b.WriteString(quote(term)) b.WriteString(`* OR fold : `) b.WriteString(quote(folded)) b.WriteString(`*)`) } return b.String() } // SearchTerms is the words a query asks about, capped the same way the MATCH // expression is. // // Exported because two searches need the same list: the FTS expression is built // from it, and so is the substring arm that finds a word living inside another // word. Two tokenizers would be two answers to "what did they type". func SearchTerms(query string) []string { terms := tokenize(query) if len(terms) > maxSearchTerms { terms = terms[:maxSearchTerms] } return terms } // tokenize splits on everything that is not a letter or a digit. Splitting on // punctuation rather than whitespace alone means "q3/2026" searches for "q3" and // "2026" instead of for a term FTS5's own tokenizer would never have indexed. func tokenize(query string) []string { // Trimmed before it is split, so a megabyte of text is not walked rune by rune // only to have all but sixteen of the results thrown away. if len(query) > maxSearchInput { query = query[:maxSearchInput] } fields := strings.FieldsFunc(query, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) terms := make([]string, 0, len(fields)) for _, f := range fields { if f != "" { terms = append(terms, f) } } return terms } // quote wraps a term as an FTS5 string literal. Inside one, the only character // with meaning is the double quote, escaped by doubling it. func quote(term string) string { return `"` + strings.ReplaceAll(term, `"`, `""`) + `"` }