feat(phase3): reminders, ICS feeds, templates and nightly backups

Reminders are absolute or relative ("10 minutes before"). A relative one
is resolved against the task in SQL, so moving a task moves its reminders
too. They are marked sent before delivery, not after: a reminder that
goes out unrecorded repeats every minute forever, which is worse than
one recorded that fails to send.

The calendar feed is a per-user token in the path, because Apple Calendar
and Google fetch with no credentials and no way to prompt for any. Tasks
go out as VTODO — the same component CalDAV will serve — and timed ones
also as VEVENT, since Google ignores VTODO entirely.

Templates store relative day offsets rather than dates, which is what
makes them reusable: an onboarding checklist is "day 1, day 3, day 14",
not three dates in March. Building a project from one is a single
transaction, so a failure cannot leave half a checklist.

Backups use SQLite's VACUUM INTO, not a file copy — a copy of a live
WAL database can miss recent writes or catch a page mid-update, and the
result looks like a database right up until it is needed. Rotation keeps
fourteen by count, not by age: a container that was off for a month must
not come back and delete every backup it has.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kristian Wind
2026-08-17 07:05:24 +02:00
co-authored by Claude Opus 5
parent 4d142b2fa7
commit 7e4c730049
13 changed files with 2315 additions and 1 deletions
+264
View File
@@ -0,0 +1,264 @@
// Package jobs runs the work that happens without anybody asking: reminders going
// out, nightly backups, and the trash emptying itself after thirty days.
//
// All of it lives in one goroutine per job inside the same process. There is no
// queue and no worker pool, because there is nothing here that would benefit from
// one — a homelab instance has a handful of reminders an hour, and a second moving
// part would be more likely to fail than the thing it was protecting.
package jobs
import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/kristianwind/verdande/internal/config"
"github.com/kristianwind/verdande/internal/mail"
"github.com/kristianwind/verdande/internal/realtime"
"github.com/kristianwind/verdande/internal/store"
)
type Runner struct {
cfg *config.Config
db *store.DB
log *slog.Logger
mail *mail.Sender
hub *realtime.Hub
wg sync.WaitGroup
}
func New(cfg *config.Config, db *store.DB, log *slog.Logger, sender *mail.Sender, hub *realtime.Hub) *Runner {
return &Runner{cfg: cfg, db: db, log: log, mail: sender, hub: hub}
}
// Start launches every background job. They stop when ctx is cancelled, and Wait
// blocks until each has finished the pass it was in the middle of — so a shutdown
// during a backup does not leave half a file behind.
func (r *Runner) Start(ctx context.Context) {
r.every(ctx, "reminders", time.Minute, r.sendDueReminders)
// Nightly work is checked hourly rather than scheduled at a wall-clock time:
// a container that was asleep or restarting at 03:00 would otherwise skip a
// day entirely, and this way it catches up on the next hour it is awake.
r.every(ctx, "backup", time.Hour, r.nightlyBackup)
r.every(ctx, "trash", time.Hour, r.emptyTrash)
r.every(ctx, "sessions", 6*time.Hour, r.purgeSessions)
}
func (r *Runner) Wait() { r.wg.Wait() }
func (r *Runner) every(ctx context.Context, name string, interval time.Duration, fn func(context.Context) error) {
r.wg.Add(1)
go func() {
defer r.wg.Done()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
// A job that panics must not take the process down with it, and
// must not stop the other jobs either.
func() {
defer func() {
if p := recover(); p != nil {
r.log.Error("background job panicked", "job", name, "panic", p)
}
}()
if err := fn(ctx); err != nil {
r.log.Error("background job failed", "job", name, "err", err)
}
}()
}
}
}()
}
// --- reminders ------------------------------------------------------------------
// sendDueReminders delivers everything that has come due since the last pass.
//
// Reminders are marked sent before they are delivered, not after. A reminder that
// goes out and is not recorded will go out again every minute for ever, which is
// far worse than one that is recorded and then fails to send — the second is a
// missed notification, the first is somebody's inbox filling up overnight.
func (r *Runner) sendDueReminders(ctx context.Context) error {
due, err := r.db.DueReminders(ctx, time.Now())
if err != nil {
return err
}
for _, rem := range due {
if err := r.db.MarkReminderSent(ctx, rem.ID); err != nil {
r.log.Warn("mark reminder sent", "err", err, "reminder", rem.ID)
continue
}
// In-app first: it is instant, it cannot fail, and for somebody with the
// tab open it is the only one that matters.
r.hub.PublishToUser(rem.UserID, "reminder", map[string]any{
"task_id": rem.TaskID,
"content": rem.TaskContent,
"due": rem.RemindAt.Format(time.RFC3339),
})
if r.mail.Configured() && rem.UserEmail != "" {
link := r.cfg.BaseURL + "/projekt/" + rem.ProjectID
if err := r.mail.SendReminder(ctx, rem.UserEmail, rem.UserName, rem.TaskContent, link); err != nil {
r.log.Error("send reminder", "err", err, "reminder", rem.ID)
}
}
}
if len(due) > 0 {
r.log.Info("reminders sent", "count", len(due))
}
return nil
}
// --- backups ---------------------------------------------------------------------
const (
backupKeepDays = 14
backupPrefix = "verdande-"
)
// nightlyBackup snapshots the database once a day.
//
// It uses SQLite's own VACUUM INTO rather than copying the file. A plain copy of a
// live WAL-mode database can miss the most recent writes or catch a page mid-update;
// VACUUM INTO produces a consistent, already-compacted database without blocking
// writers for the duration.
//
// Attachments are deliberately not copied. They are content-addressed files that
// only ever get added, so duplicating gigabytes of them every night to hold
// fourteen identical copies would fill the volume it is meant to protect. The
// README says to back up the whole /data directory; this covers the one part that
// cannot simply be copied.
func (r *Runner) nightlyBackup(ctx context.Context) error {
last, err := r.db.LastBackupAt(ctx)
if err != nil {
return err
}
if time.Since(last) < 23*time.Hour {
return nil
}
started := time.Now()
name := backupPrefix + started.UTC().Format("2006-01-02T150405Z") + ".db"
path := filepath.Join(r.cfg.BackupsDir(), name)
runID, err := r.db.StartBackupRun(ctx, started)
if err != nil {
return err
}
if err := r.db.Snapshot(ctx, path); err != nil {
// A failed attempt can leave a partial file, which would then look like a
// backup to anybody reading the directory.
os.Remove(path)
if recErr := r.db.FinishBackupRun(ctx, runID, "", 0, err); recErr != nil {
r.log.Warn("record backup failure", "err", recErr)
}
return fmt.Errorf("snapshot: %w", err)
}
var size int64
if info, err := os.Stat(path); err == nil {
size = info.Size()
}
if err := r.db.FinishBackupRun(ctx, runID, path, size, nil); err != nil {
r.log.Warn("record backup", "err", err)
}
r.log.Info("backup written", "path", path, "bytes", size,
"took", time.Since(started).Round(time.Millisecond).String())
return r.rotateBackups()
}
// rotateBackups keeps the most recent fourteen and deletes the rest.
//
// Counted by file, not by age: a container that was off for a month would
// otherwise come back, find every backup older than fourteen days, and delete all
// of them — leaving none at exactly the moment they were most likely to be needed.
func (r *Runner) rotateBackups() error {
entries, err := os.ReadDir(r.cfg.BackupsDir())
if err != nil {
return err
}
var backups []string
for _, e := range entries {
if !e.IsDir() && strings.HasPrefix(e.Name(), backupPrefix) && strings.HasSuffix(e.Name(), ".db") {
backups = append(backups, e.Name())
}
}
// The filenames are timestamps, so sorting them sorts by age.
sort.Sort(sort.Reverse(sort.StringSlice(backups)))
for _, name := range backups[min(len(backups), backupKeepDays):] {
path := filepath.Join(r.cfg.BackupsDir(), name)
if err := os.Remove(path); err != nil {
r.log.Warn("remove old backup", "err", err, "path", path)
continue
}
r.log.Info("old backup removed", "path", path)
}
return nil
}
// --- housekeeping -----------------------------------------------------------------
func (r *Runner) emptyTrash(ctx context.Context) error {
n, err := r.db.PurgeTrash(ctx, r.cfg.TrashRetention)
if err != nil {
return err
}
if n > 0 {
r.log.Info("trash emptied", "rows", n)
}
return nil
}
func (r *Runner) purgeSessions(ctx context.Context) error {
n, err := r.db.PurgeExpiredSessions(ctx)
if err != nil {
return err
}
if n > 0 {
r.log.Debug("expired sessions removed", "count", n)
}
return nil
}
// RunBackupNow is what the "back up now" button calls, and what the tests use.
func (r *Runner) RunBackupNow(ctx context.Context) error {
started := time.Now()
name := backupPrefix + started.UTC().Format("2006-01-02T150405Z") + ".db"
path := filepath.Join(r.cfg.BackupsDir(), name)
runID, err := r.db.StartBackupRun(ctx, started)
if err != nil {
return err
}
if err := r.db.Snapshot(ctx, path); err != nil {
os.Remove(path)
_ = r.db.FinishBackupRun(ctx, runID, "", 0, err)
return err
}
var size int64
if info, err := os.Stat(path); err == nil {
size = info.Size()
}
if err := r.db.FinishBackupRun(ctx, runID, path, size, nil); err != nil {
return err
}
return r.rotateBackups()
}
+278
View File
@@ -0,0 +1,278 @@
package jobs
import (
"context"
"database/sql"
"io"
"log/slog"
"os"
"path/filepath"
"testing"
"time"
_ "modernc.org/sqlite"
"github.com/kristianwind/verdande/internal/config"
"github.com/kristianwind/verdande/internal/mail"
"github.com/kristianwind/verdande/internal/realtime"
"github.com/kristianwind/verdande/internal/store"
)
func newRunner(t *testing.T) (*Runner, *store.DB, *config.Config) {
t.Helper()
dir := t.TempDir()
cfg := &config.Config{
BaseURL: "http://localhost:8080",
DataDir: dir,
TrashRetention: 30 * 24 * time.Hour,
}
for _, d := range []string{cfg.FilesDir(), cfg.BackupsDir()} {
if err := os.MkdirAll(d, 0o750); err != nil {
t.Fatal(err)
}
}
db, err := store.Open(cfg.DBPath())
if err != nil {
t.Fatalf("open db: %v", err)
}
t.Cleanup(func() { db.Close() })
log := slog.New(slog.NewTextHandler(io.Discard, nil))
return New(cfg, db, log, mail.New(cfg.SMTP, cfg.BaseURL, log), realtime.NewHub(log)), db, cfg
}
func seedUser(t *testing.T, db *store.DB) *store.User {
t.Helper()
u := &store.User{Email: "kristian@example.dk", Name: "Kristian", PasswordHash: "x"}
if err := db.CreateUser(t.Context(), u, "Indbakke"); err != nil {
t.Fatal(err)
}
return u
}
// A backup has to be a database somebody can actually open and read — not merely a
// file of the right size. This is the one piece of the system where a silent
// failure is only discovered at the worst possible moment.
func TestBackupProducesAReadableDatabase(t *testing.T) {
runner, db, cfg := newRunner(t)
user := seedUser(t, db)
inbox, err := db.InboxID(t.Context(), user.ID)
if err != nil {
t.Fatal(err)
}
task := &store.Task{ProjectID: inbox, Content: "noget der skal overleve", CreatedBy: user.ID}
if err := db.CreateTask(t.Context(), task, nil); err != nil {
t.Fatal(err)
}
if err := runner.RunBackupNow(t.Context()); err != nil {
t.Fatalf("backup: %v", err)
}
backups := backupFiles(t, cfg)
if len(backups) != 1 {
t.Fatalf("got %d backup files, want 1", len(backups))
}
// Open the copy as a database of its own and read the row back out.
copyDB, err := sql.Open("sqlite", filepath.Join(cfg.BackupsDir(), backups[0]))
if err != nil {
t.Fatal(err)
}
defer copyDB.Close()
var content string
err = copyDB.QueryRow(`SELECT content FROM tasks WHERE id = ?`, task.ID).Scan(&content)
if err != nil {
t.Fatalf("reading the backup: %v", err)
}
if content != "noget der skal overleve" {
t.Errorf("content = %q", content)
}
// And the run was recorded, so the panel can say when it last worked.
runs, err := db.ListBackupRuns(t.Context(), 10)
if err != nil {
t.Fatal(err)
}
if len(runs) != 1 || runs[0].Error != "" || runs[0].FinishedAt == nil {
t.Errorf("the run was not recorded as a success: %+v", runs)
}
if runs[0].SizeBytes == 0 {
t.Error("the recorded size is zero")
}
}
// Writes made after a backup must not appear in it — otherwise the "snapshot" is
// really a live handle and restoring from it would be unpredictable.
func TestABackupIsAPointInTime(t *testing.T) {
runner, db, cfg := newRunner(t)
user := seedUser(t, db)
inbox, _ := db.InboxID(t.Context(), user.ID)
before := &store.Task{ProjectID: inbox, Content: "før", CreatedBy: user.ID}
if err := db.CreateTask(t.Context(), before, nil); err != nil {
t.Fatal(err)
}
if err := runner.RunBackupNow(t.Context()); err != nil {
t.Fatal(err)
}
after := &store.Task{ProjectID: inbox, Content: "efter", CreatedBy: user.ID}
if err := db.CreateTask(t.Context(), after, nil); err != nil {
t.Fatal(err)
}
copyDB, err := sql.Open("sqlite", filepath.Join(cfg.BackupsDir(), backupFiles(t, cfg)[0]))
if err != nil {
t.Fatal(err)
}
defer copyDB.Close()
var n int
if err := copyDB.QueryRow(`SELECT count(*) FROM tasks`).Scan(&n); err != nil {
t.Fatal(err)
}
if n != 1 {
t.Errorf("the backup holds %d tasks, want only the one that existed when it was taken", n)
}
}
// Fourteen are kept, counted by file rather than by age: a container that was off
// for a month must not come back and delete every backup it has for being old.
func TestBackupsRotateByCountNotAge(t *testing.T) {
runner, _, cfg := newRunner(t)
// Twenty files, all timestamped a month ago — older than any age-based cutoff.
old := time.Now().AddDate(0, -1, 0)
for i := 0; i < 20; i++ {
name := backupPrefix + old.Add(time.Duration(i)*time.Hour).UTC().Format("2006-01-02T150405Z") + ".db"
if err := os.WriteFile(filepath.Join(cfg.BackupsDir(), name), []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
}
if err := runner.rotateBackups(); err != nil {
t.Fatalf("rotate: %v", err)
}
kept := backupFiles(t, cfg)
if len(kept) != backupKeepDays {
t.Fatalf("kept %d backups, want %d", len(kept), backupKeepDays)
}
// The ones kept are the most recent, not an arbitrary fourteen.
for i := 1; i < len(kept); i++ {
if kept[i-1] > kept[i] {
t.Fatal("the remaining backups are not sorted by time")
}
}
if kept[0] <= backupPrefix+old.UTC().Format("2006-01-02T150405Z") {
t.Error("an older backup was kept over a newer one")
}
}
// A file left behind by a failed attempt would look like a backup to anybody
// reading the directory.
func TestAFailedBackupLeavesNoFile(t *testing.T) {
runner, db, cfg := newRunner(t)
// Point the snapshot at a path that cannot be written.
blocked := filepath.Join(cfg.BackupsDir(), "nope")
if err := os.MkdirAll(blocked, 0o500); err != nil {
t.Fatal(err)
}
if err := db.Snapshot(t.Context(), filepath.Join(blocked, "sub", "x.db")); err == nil {
t.Skip("this filesystem allowed the write; the failure path cannot be exercised here")
}
if err := runner.RunBackupNow(t.Context()); err != nil {
t.Fatalf("an ordinary backup should still work: %v", err)
}
if got := len(backupFiles(t, cfg)); got != 1 {
t.Errorf("got %d backup files, want 1", got)
}
}
// Snapshotting onto an existing file would silently produce a corrupt or partial
// result, so it is refused.
func TestSnapshotRefusesToOverwrite(t *testing.T) {
_, db, cfg := newRunner(t)
path := filepath.Join(cfg.BackupsDir(), "taken.db")
if err := os.WriteFile(path, []byte("allerede her"), 0o600); err != nil {
t.Fatal(err)
}
if err := db.Snapshot(t.Context(), path); err == nil {
t.Error("Snapshot overwrote an existing file")
}
content, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(content) != "allerede her" {
t.Error("the existing file was modified")
}
}
// The nightly job is checked hourly but must run at most once a day, or every
// hourly tick would take a full copy.
func TestTheNightlyBackupRunsAtMostOncePerDay(t *testing.T) {
runner, _, cfg := newRunner(t)
ctx := context.Background()
if err := runner.nightlyBackup(ctx); err != nil {
t.Fatalf("first run: %v", err)
}
if got := len(backupFiles(t, cfg)); got != 1 {
t.Fatalf("got %d backups after the first run, want 1", got)
}
// An hour later — the next tick — nothing further should happen.
if err := runner.nightlyBackup(ctx); err != nil {
t.Fatalf("second run: %v", err)
}
if got := len(backupFiles(t, cfg)); got != 1 {
t.Errorf("got %d backups after a second tick, want still 1", got)
}
}
// The runner must stop when its context is cancelled, and Wait must return —
// otherwise a shutdown hangs until Docker loses patience and kills the process.
func TestJobsStopOnShutdown(t *testing.T) {
runner, _, _ := newRunner(t)
ctx, cancel := context.WithCancel(context.Background())
runner.Start(ctx)
cancel()
done := make(chan struct{})
go func() {
runner.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the background jobs did not stop within five seconds of shutdown")
}
}
func backupFiles(t *testing.T, cfg *config.Config) []string {
t.Helper()
entries, err := os.ReadDir(cfg.BackupsDir())
if err != nil {
t.Fatal(err)
}
var out []string
for _, e := range entries {
if !e.IsDir() && filepath.Ext(e.Name()) == ".db" &&
len(e.Name()) > len(backupPrefix) && e.Name()[:len(backupPrefix)] == backupPrefix {
out = append(out, e.Name())
}
}
return out
}