diff --git a/internal/httpapi/tasks_test.go b/internal/httpapi/tasks_test.go index c92babd..81cdd5a 100644 --- a/internal/httpapi/tasks_test.go +++ b/internal/httpapi/tasks_test.go @@ -6,6 +6,7 @@ import ( "io" "mime/multipart" "net/http" + "net/url" "strings" "testing" "time" @@ -1777,3 +1778,42 @@ func TestQuickAddSectionIDIsCheckedAgainstTheProject(t *testing.T) { t.Errorf("content = %v", moved["content"]) } } + +// A word inside a word, on the task side. +// +// The notes search grew this first — a note holding `keepInventory` could not be +// found by typing "keep inventory" — and a task is the same sentence with a +// different table behind it. `SpeedAdmin` is one token, so "speed admin" asks for +// two words that are nowhere. +// +// Having it in one of the two searches would be worse than having it in neither: +// "it finds it in my notes but not in my tasks" is the next report. +func TestATaskIsFoundByAWordInsideAWord(t *testing.T) { + ts := newTestServer(t) + ts.bootstrap(t) + + ts.do(t, "POST", "/api/v1/tasks", map[string]any{ + "content": "Opret kodeord til Kulturskolen (SpeedAdmin)", + }) + ts.do(t, "POST", "/api/v1/tasks", map[string]any{"content": "Køb kaffe"}) + + for _, tc := range []struct { + query string + want int + }{ + {"speed admin", 1}, + {"admin", 1}, + {"speedadmin", 1}, + {"kulturskolen", 1}, + {"vandhane", 0}, + } { + resp, body := ts.do(t, "GET", "/api/v1/search?q="+url.QueryEscape(tc.query), nil) + if resp.StatusCode != http.StatusOK { + t.Fatalf("%q: status %d", tc.query, resp.StatusCode) + } + tasks, _ := body["tasks"].([]any) + if len(tasks) != tc.want { + t.Errorf("%q found %d tasks, want %d", tc.query, len(tasks), tc.want) + } + } +} diff --git a/internal/store/notes.go b/internal/store/notes.go index 080c34d..f78a737 100644 --- a/internal/store/notes.go +++ b/internal/store/notes.go @@ -289,12 +289,73 @@ func (db *DB) SearchNotes(ctx context.Context, query string, limit int) ([]Note, // // Joined rather than `rowid IN (…)`: the rank belongs to the match, and a // subquery throws it away. - return db.notesJoin(ctx, noteColumnsQualified, ` + found, err := db.notesJoin(ctx, noteColumnsQualified, ` JOIN notes_fts ON notes_fts.rowid = notes.rowid WHERE notes.deleted_at IS NULL AND notes_fts MATCH ? ORDER BY bm25(notes_fts, 10.0, 1.0, 1.0) LIMIT ?`, expr, limit) + if err != nil || len(found) > 0 { + return found, err + } + return db.searchNotesInside(ctx, query, limit) +} + +// searchNotesInside finds a word that lives *inside* another word. +// +// An index is built of tokens, and a token is what a tokenizer decided was a +// word. `gamerule keepInventory true` is three of them, and the middle one is +// `keepinventory` — so "keep inventory" asks for two words that are nowhere, and +// gets nothing. Reported as "søgning har ikke det store held", which from where +// the person is sitting is exactly what it is: the note is right there, and the +// search says there is nothing. +// +// A prefix query gets half of it. `keep*` does match `keepinventory`; `inventory*` +// cannot, because it is not a prefix of anything — and the two are joined with AND, +// so the half that works is thrown away by the half that cannot. +// +// This is not an exotic case in a notes app. A decade of notes is full of +// commands, identifiers and file names, and every one of them is several words the +// index thinks is one: `keepInventory`, `VACUUM INTO`, `apple-notes-til-markdown`. +// +// So: the index answers first, and when it has nothing, the text is read. Every +// term has to appear somewhere in the note, anywhere, in any word. +// +// # What it costs, and when to stop doing it this way +// +// A LIKE with a leading % cannot use an index; this reads every note. At twelve +// hundred notes that is four megabytes and a millisecond or two, and it only runs +// when the fast path already came back empty — which is the search that was about +// to disappoint somebody. At a hundred thousand notes it would be the wrong shape, +// and the answer then is an FTS5 `trigram` index, not a bigger scan. +// +// It does not rank. bm25 needs a match to rank, and there is no match here — that +// is the whole situation. The rows are the most recently touched first, and the +// set is small, because the index already said there was nothing in it. +func (db *DB) searchNotesInside(ctx context.Context, query string, limit int) ([]Note, error) { + terms := SearchTerms(query) + if len(terms) == 0 { + return nil, nil + } + + // `fold` is `title || ' ' || body` with the Danish letters transliterated, so + // one column covers the title, the body and "gron" for "grøn" at once. LIKE is + // case-insensitive for ASCII in SQLite, which is what makes `keepInventory` + // answer to "inventory". + // + // The needle is safe to interpolate into a pattern because `tokenize` splits on + // everything that is not a letter or a digit: a term cannot contain % or _, so + // there is no wildcard for somebody to smuggle in. It is still a bound + // parameter — that is about SQL, and this is about LIKE. + where := "WHERE deleted_at IS NULL" + args := make([]any, 0, len(terms)+1) + for _, term := range terms { + where += " AND fold LIKE ?" + args = append(args, "%"+FoldDanish(term)+"%") + } + args = append(args, limit) + + return db.notesWhere(ctx, where+" ORDER BY updated_at DESC LIMIT ?", args...) } // SetNoteTimes writes the dates a note had somewhere else. diff --git a/internal/store/notes_test.go b/internal/store/notes_test.go index 577d743..7de6163 100644 --- a/internal/store/notes_test.go +++ b/internal/store/notes_test.go @@ -369,3 +369,67 @@ func TestSearchAnswersWithTheBestMatchFirst(t *testing.T) { t.Errorf("øverst stod %q; den note, der hedder det, man søgte efter, skal stå først", found[0].Title) } } + +// A word inside a word. +// +// Reported as "søgning har ikke det store held": a note holding +// `gamerule keepInventory true` could not be found by typing "keep inventory". +// The index is built of tokens, and `keepInventory` is one token — so the two +// words are nowhere, and the search says there is nothing while the note is right +// there. +// +// A decade of notes is full of these: commands, identifiers, file names. Every one +// of them is several words the index thinks is one. +func TestAWordInsideAWordIsStillFound(t *testing.T) { + db, userID := sealedStore(t) + ctx := context.Background() + + if err := db.SaveNote(ctx, &Note{ + CreatedBy: userID, + Body: "Minecraft Bedrock\n\ngamerule keepInventory true", + }); err != nil { + t.Fatal(err) + } + // A second note, so a fallback that simply returns everything would be caught. + if err := db.SaveNote(ctx, &Note{ + CreatedBy: userID, + Body: "Indkøb\n\nKaffe og filtre", + }); err != nil { + t.Fatal(err) + } + + for _, q := range []string{ + "keep inventory", // the two words, which is how somebody looks for it + "inventory", // the tail on its own — no prefix query can reach this + "keepinventory", // written as one, which the index does hold + "KEEP INVENTORY", // and case is not a question anybody is asking + "vacuum", // not in either note + } { + found, err := db.SearchNotes(ctx, q, 10) + if err != nil { + t.Fatalf("%q: %v", q, err) + } + want := 1 + if q == "vacuum" { + want = 0 + } + if len(found) != want { + t.Errorf("%q found %d notes, want %d", q, len(found), want) + continue + } + if want == 1 && found[0].Title != "Minecraft Bedrock" { + t.Errorf("%q found %q", q, found[0].Title) + } + } + + // The fast path is still the fast path: a word the index does hold must not be + // answered by the scan, or the ranking that was built for twelve hundred notes + // is quietly gone. + found, err := db.SearchNotes(ctx, "Bedrock", 10) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 { + t.Fatalf("an ordinary word found %d notes", len(found)) + } +} diff --git a/internal/store/search.go b/internal/store/search.go index 4f28e28..9eb9b49 100644 --- a/internal/store/search.go +++ b/internal/store/search.go @@ -90,6 +90,20 @@ func MatchExprOver(query string, columns ...string) string { return b.String() } +// SearchTerms is the words a query asks about, capped the same way the MATCH +// expression is. +// +// Exported because two searches need the same list: the FTS expression is built +// from it, and so is the substring arm that finds a word living inside another +// word. Two tokenizers would be two answers to "what did they type". +func SearchTerms(query string) []string { + terms := tokenize(query) + if len(terms) > maxSearchTerms { + terms = terms[:maxSearchTerms] + } + return terms +} + // tokenize splits on everything that is not a letter or a digit. Splitting on // punctuation rather than whitespace alone means "q3/2026" searches for "q3" and // "2026" instead of for a term FTS5's own tokenizer would never have indexed. diff --git a/internal/store/tasks.go b/internal/store/tasks.go index 9d86156..fb96a69 100644 --- a/internal/store/tasks.go +++ b/internal/store/tasks.go @@ -166,15 +166,42 @@ func (db *DB) ListTasks(ctx context.Context, userID string, f TaskFilter) ([]Tas // labels are short, and their names are typed by the person searching for // them, so there is no spelling to be generous about. The diacritic // folding that matters for task text does not earn a second index here. + // + // The third arm is a word inside a word. An index is built of tokens, and + // `SpeedAdmin` is one of them — so "speed admin" asks for two words that are + // nowhere. A prefix query gets half of it (`speed*` matches, `admin*` cannot) + // and the two are joined with AND, so the half that works is thrown away. + // + // Notes got the same arm, and for the same reason. Having it in one of the + // two would be worse than having it in neither: "it finds it in my notes but + // not in my tasks" is a bug report waiting, and the answer to it would be + // this line. + // + // It scans, and it only ever runs when somebody typed something in the box — + // Today and Kommende do not set `Search`. Every term has to appear, so the + // arms are ANDed with each other while the whole thing is ORed with the two + // above. + inside := make([]string, 0, maxSearchTerms) + insideArgs := make([]any, 0, maxSearchTerms) + for _, term := range SearchTerms(f.Search) { + inside = append(inside, "t.fold LIKE ?") + insideArgs = append(insideArgs, "%"+FoldDanish(term)+"%") + } + text := "" + if len(inside) > 0 { + text = " OR (" + strings.Join(inside, " AND ") + ")" + } + where = append(where, `( t.rowid IN (SELECT rowid FROM tasks_fts WHERE tasks_fts MATCH ?) OR t.id IN ( SELECT tl.task_id FROM task_labels tl JOIN labels l ON l.id = tl.label_id WHERE lower(l.name) LIKE ? - ) + )`+text+` )`) args = append(args, expr, "%"+strings.ToLower(strings.TrimSpace(f.Search))+"%") + args = append(args, insideArgs...) } if f.FilterSQL != "" { where = append(where, "("+f.FilterSQL+")")