diff --git a/internal/httpapi/backup_handlers.go b/internal/httpapi/backup_handlers.go new file mode 100644 index 0000000..de96514 --- /dev/null +++ b/internal/httpapi/backup_handlers.go @@ -0,0 +1,158 @@ +package httpapi + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/kristianwind/verdande/internal/jobs" +) + +// The nightly backup, made visible. +// +// It has run since the beginning and nothing showed it. A backup nobody can see is +// a backup nobody knows has been failing — and the failure mode is quiet: a volume +// that went read-only, a disk that filled, a container that has not been up at +// midnight for a week. The first time anybody finds out is the day they need one. +// +// Administrators only, and behind a session as well as the admin check. A backup +// file is a complete copy of the database, so downloading one is downloading +// everybody's data; a leaked API token must not be able to. + +type backupRunJSON struct { + ID string `json:"id"` + Started string `json:"started_at"` + // Finished is absent while a backup is still running, and on one that was + // interrupted — which is the same shape from outside and honestly so: a run + // the process died in the middle of never finished. + Finished string `json:"finished_at,omitempty"` + Size int64 `json:"size_bytes"` + Error string `json:"error,omitempty"` + // Present says the file is still on disk. Rotation keeps the most recent + // fourteen, so an older row is a true record of a backup that was made and a + // file that is no longer there — and a download link for it would 404. + Present bool `json:"present"` +} + +func (s *Server) handleListBackups(w http.ResponseWriter, r *http.Request) { + runs, err := s.db.ListBackupRuns(r.Context(), parseLimit(r.URL.Query().Get("limit"), 30, 100)) + if err != nil { + s.internal(w, r, "list backups", err) + return + } + + out := make([]backupRunJSON, 0, len(runs)) + for _, b := range runs { + j := backupRunJSON{ + ID: b.ID, Started: b.StartedAt.Format(time.RFC3339), + Size: b.SizeBytes, Error: b.Error, + } + if b.FinishedAt != nil { + j.Finished = b.FinishedAt.Format(time.RFC3339) + } + if b.Path != "" { + if _, err := os.Stat(b.Path); err == nil { + j.Present = true + } + } + out = append(out, j) + } + writeJSON(w, http.StatusOK, map[string]any{"backups": out}) +} + +// handleRunBackup takes one now. +// +// The same function the nightly job runs, without the "has one run today" check. +// Waiting until tonight to learn whether backups work at all is how somebody finds +// out on the day they need one — and it is the only way to get a snapshot from +// *before* something you are about to do. +// +// Synchronous. A backup of a homelab database is a second or two, and a button +// that returns immediately and tells you nothing is a button you press twice. +func (s *Server) handleRunBackup(w http.ResponseWriter, r *http.Request) { + if err := jobs.Backup(r.Context(), s.cfg, s.db, s.log); err != nil { + s.internal(w, r, "run backup", err) + return + } + runs, err := s.db.ListBackupRuns(r.Context(), 1) + if err != nil || len(runs) == 0 { + w.WriteHeader(http.StatusNoContent) + return + } + b := runs[0] + j := backupRunJSON{ + ID: b.ID, Started: b.StartedAt.Format(time.RFC3339), + Size: b.SizeBytes, Error: b.Error, Present: true, + } + if b.FinishedAt != nil { + j.Finished = b.FinishedAt.Format(time.RFC3339) + } + writeJSON(w, http.StatusCreated, j) +} + +// handleDownloadBackup sends one backup file. +// +// The path comes from the row rather than from the request, and is then checked to +// be inside the backups directory anyway. The row is written by this process and +// should always be safe; the check is there because "should always" is how a path +// traversal gets shipped, and the thing on the other side of it is every account's +// data in one file. +func (s *Server) handleDownloadBackup(w http.ResponseWriter, r *http.Request) { + runs, err := s.db.ListBackupRuns(r.Context(), 100) + if err != nil { + s.internal(w, r, "list backups", err) + return + } + + id := chi.URLParam(r, "backupID") + var path string + for _, b := range runs { + if b.ID == id { + path = b.Path + break + } + } + if path == "" { + writeError(w, http.StatusNotFound, CodeNotFound, "not found") + return + } + + dir, err := filepath.Abs(s.cfg.BackupsDir()) + if err != nil { + s.internal(w, r, "backups dir", err) + return + } + full, err := filepath.Abs(path) + if err != nil || !strings.HasPrefix(full, dir+string(os.PathSeparator)) { + writeError(w, http.StatusNotFound, CodeNotFound, "not found") + return + } + file, err := os.Open(full) + if err != nil { + // Rotation keeps the most recent fourteen, so a row older than that is a + // true record with no file behind it. Not an error — a 404 is the honest + // answer to "send me that one". + writeError(w, http.StatusNotFound, CodeNotFound, "not found") + return + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + s.internal(w, r, "stat backup", err) + return + } + + // The same treatment attachments get, and for a stronger reason: this is a + // SQLite database, and nothing should ever be tempted to render it inline. + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", + fmt.Sprintf("attachment; filename=%q", filepath.Base(full))) + w.Header().Set("X-Content-Type-Options", "nosniff") + http.ServeContent(w, r, "", info.ModTime(), file) +} diff --git a/internal/httpapi/backup_test.go b/internal/httpapi/backup_test.go new file mode 100644 index 0000000..72e0c0f --- /dev/null +++ b/internal/httpapi/backup_test.go @@ -0,0 +1,116 @@ +package httpapi + +import ( + "net/http" + "testing" +) + +// The backup panel exists because the job was invisible, so the thing to check is +// that it can be seen and asked for — and that the file it hands out is guarded +// like what it is, which is every account's data in one download. +func TestABackupCanBeTakenListedAndDownloaded(t *testing.T) { + ts := newTestServer(t) + ts.bootstrap(t) + + // Nothing yet, and that is a state the list has to survive rather than a case + // nobody hits: it is what a fresh instance looks like. + _, empty := ts.do(t, "GET", "/api/v1/backups", nil) + if got, _ := empty["backups"].([]any); len(got) != 0 { + t.Fatalf("a fresh instance has %d backups", len(got)) + } + + resp, made := ts.do(t, "POST", "/api/v1/backups", nil) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("run backup: status %d, body %v", resp.StatusCode, made) + } + id, _ := made["id"].(string) + if id == "" { + t.Fatal("the new run has no id") + } + if size, _ := made["size_bytes"].(float64); size <= 0 { + t.Errorf("size_bytes = %v — a backup of a database with rows in it is not empty", made["size_bytes"]) + } + if made["error"] != nil { + t.Errorf("the backup reported an error: %v", made["error"]) + } + + _, listed := ts.do(t, "GET", "/api/v1/backups", nil) + rows, _ := listed["backups"].([]any) + if len(rows) != 1 { + t.Fatalf("listed %d backups, want 1", len(rows)) + } + row := rows[0].(map[string]any) + if row["id"] != id { + t.Errorf("the listed run is not the one that was made") + } + if row["present"] != true { + t.Error("present = false on a backup that was just written") + } + + // And the file itself, which must arrive as a download rather than as + // something a browser might try to render. + req, _ := http.NewRequest("GET", ts.URL+"/api/v1/backups/"+id, nil) + req.Header.Set("Sec-Fetch-Site", "same-origin") + dl, err := ts.client.Do(req) + if err != nil { + t.Fatal(err) + } + defer dl.Body.Close() + if dl.StatusCode != http.StatusOK { + t.Fatalf("download: status %d", dl.StatusCode) + } + if ct := dl.Header.Get("Content-Type"); ct != "application/octet-stream" { + t.Errorf("Content-Type = %q, want application/octet-stream", ct) + } + if cd := dl.Header.Get("Content-Disposition"); cd == "" || cd[:10] != "attachment" { + t.Errorf("Content-Disposition = %q, want an attachment", cd) + } + + // An id that is not a backup is a 404, not a path to try opening. + if resp, _ := ts.do(t, "GET", "/api/v1/backups/../../etc/passwd", nil); resp.StatusCode == http.StatusOK { + t.Error("a traversal in the id was served") + } + if resp, _ := ts.do(t, "GET", "/api/v1/backups/nogetsomhelst", nil); resp.StatusCode != http.StatusNotFound { + t.Errorf("an unknown id: status %d, want 404", resp.StatusCode) + } +} + +// Administrators only, and sessions only. A backup is a complete copy of the +// database — a leaked token that could download one turns a stolen credential into +// a copy of everybody's data. +func TestBackupsAreAdminsOnlyAndSessionsOnly(t *testing.T) { + ts := newTestServer(t) + ts.bootstrap(t) + other := ts.newUser(t, "anden@example.dk", "Anden") + + for _, c := range []struct{ method, path string }{ + {"GET", "/api/v1/backups"}, + {"POST", "/api/v1/backups"}, + {"GET", "/api/v1/backups/whichever"}, + } { + if resp, _ := other.do(t, c.method, c.path, nil); resp.StatusCode != http.StatusForbidden { + t.Errorf("%s %s as an ordinary user: status %d, want 403", c.method, c.path, resp.StatusCode) + } + } + + admin, err := ts.db.UserByEmail(t.Context(), "kristian@example.dk") + if err != nil { + t.Fatal(err) + } + token := ts.apiToken(t, admin.ID) + for _, c := range []struct{ method, path string }{ + {"GET", "/api/v1/backups"}, + {"POST", "/api/v1/backups"}, + } { + req, _ := http.NewRequest(c.method, ts.URL+c.path, nil) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := (&http.Client{}).Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Errorf("%s %s with an admin's API token: status %d, want 403", c.method, c.path, resp.StatusCode) + } + } +} diff --git a/internal/jobs/jobs.go b/internal/jobs/jobs.go index e83ab87..9038ed8 100644 --- a/internal/jobs/jobs.go +++ b/internal/jobs/jobs.go @@ -162,22 +162,41 @@ func (r *Runner) nightlyBackup(ctx context.Context) error { if time.Since(last) < 23*time.Hour { return nil } + return Backup(ctx, r.cfg, r.db, r.log) +} +// Backup takes one snapshot now, records the run and rotates the old files. +// +// Exported and free of the Runner, because the settings page can ask for one too: +// waiting until tonight to find out whether backups work at all is how somebody +// discovers on the day they need one that the volume has been read-only for a +// month. The nightly job is this function behind a "has one run today" check; the +// button is this function without it. +// +// It uses SQLite's own VACUUM INTO rather than copying the file. A plain copy of a +// live WAL-mode database can miss the most recent writes or catch a page +// mid-update; VACUUM INTO produces a consistent, already-compacted database +// without blocking writers for the duration. +func Backup(ctx context.Context, cfg *config.Config, db *store.DB, log *slog.Logger) error { started := time.Now() name := backupPrefix + started.UTC().Format("2006-01-02T150405Z") + ".db" - path := filepath.Join(r.cfg.BackupsDir(), name) + path := filepath.Join(cfg.BackupsDir(), name) - runID, err := r.db.StartBackupRun(ctx, started) + if err := os.MkdirAll(cfg.BackupsDir(), 0o755); err != nil { + return fmt.Errorf("backups dir: %w", err) + } + + runID, err := db.StartBackupRun(ctx, started) if err != nil { return err } - if err := r.db.Snapshot(ctx, path); err != nil { + if err := db.Snapshot(ctx, path); err != nil { // A failed attempt can leave a partial file, which would then look like a // backup to anybody reading the directory. os.Remove(path) - if recErr := r.db.FinishBackupRun(ctx, runID, "", 0, err); recErr != nil { - r.log.Warn("record backup failure", "err", recErr) + if recErr := db.FinishBackupRun(ctx, runID, "", 0, err); recErr != nil { + log.Warn("record backup failure", "err", recErr) } return fmt.Errorf("snapshot: %w", err) } @@ -186,13 +205,13 @@ func (r *Runner) nightlyBackup(ctx context.Context) error { if info, err := os.Stat(path); err == nil { size = info.Size() } - if err := r.db.FinishBackupRun(ctx, runID, path, size, nil); err != nil { - r.log.Warn("record backup", "err", err) + if err := db.FinishBackupRun(ctx, runID, path, size, nil); err != nil { + log.Warn("record backup", "err", err) } - r.log.Info("backup written", "path", path, "bytes", size, + log.Info("backup written", "path", path, "bytes", size, "took", time.Since(started).Round(time.Millisecond).String()) - return r.rotateBackups() + return rotateBackups(cfg, log) } // rotateBackups keeps the most recent fourteen and deletes the rest. @@ -200,8 +219,8 @@ func (r *Runner) nightlyBackup(ctx context.Context) error { // Counted by file, not by age: a container that was off for a month would // otherwise come back, find every backup older than fourteen days, and delete all // of them — leaving none at exactly the moment they were most likely to be needed. -func (r *Runner) rotateBackups() error { - entries, err := os.ReadDir(r.cfg.BackupsDir()) +func rotateBackups(cfg *config.Config, log *slog.Logger) error { + entries, err := os.ReadDir(cfg.BackupsDir()) if err != nil { return err } @@ -216,12 +235,12 @@ func (r *Runner) rotateBackups() error { sort.Sort(sort.Reverse(sort.StringSlice(backups))) for _, name := range backups[min(len(backups), backupKeepDays):] { - path := filepath.Join(r.cfg.BackupsDir(), name) + path := filepath.Join(cfg.BackupsDir(), name) if err := os.Remove(path); err != nil { - r.log.Warn("remove old backup", "err", err, "path", path) + log.Warn("remove old backup", "err", err, "path", path) continue } - r.log.Info("old backup removed", "path", path) + log.Info("old backup removed", "path", path) } return nil } @@ -291,27 +310,12 @@ func (r *Runner) purgeSessions(ctx context.Context) error { return nil } -// RunBackupNow is what the "back up now" button calls, and what the tests use. +// RunBackupNow is the Runner's way in, for the tests that already hold one. +// +// It was a second copy of Backup — written when the button that would call it did +// not exist yet — and the two had already drifted: this one never created the +// backups directory, and logged nothing. One implementation, so a fix to the +// nightly path is a fix to the button as well. func (r *Runner) RunBackupNow(ctx context.Context) error { - started := time.Now() - name := backupPrefix + started.UTC().Format("2006-01-02T150405Z") + ".db" - path := filepath.Join(r.cfg.BackupsDir(), name) - - runID, err := r.db.StartBackupRun(ctx, started) - if err != nil { - return err - } - if err := r.db.Snapshot(ctx, path); err != nil { - os.Remove(path) - _ = r.db.FinishBackupRun(ctx, runID, "", 0, err) - return err - } - var size int64 - if info, err := os.Stat(path); err == nil { - size = info.Size() - } - if err := r.db.FinishBackupRun(ctx, runID, path, size, nil); err != nil { - return err - } - return r.rotateBackups() + return Backup(ctx, r.cfg, r.db, r.log) } diff --git a/internal/jobs/jobs_test.go b/internal/jobs/jobs_test.go index df882e2..175c941 100644 --- a/internal/jobs/jobs_test.go +++ b/internal/jobs/jobs_test.go @@ -154,7 +154,7 @@ func TestBackupsRotateByCountNotAge(t *testing.T) { } } - if err := runner.rotateBackups(); err != nil { + if err := rotateBackups(cfg, runner.log); err != nil { t.Fatalf("rotate: %v", err) } diff --git a/web/src/routes/indstillinger/data/+page.svelte b/web/src/routes/indstillinger/data/+page.svelte index 06e1975..37b8add 100644 --- a/web/src/routes/indstillinger/data/+page.svelte +++ b/web/src/routes/indstillinger/data/+page.svelte @@ -4,6 +4,67 @@ import { app } from '$lib/stores.svelte.js'; import { goto } from '$app/navigation'; + // --- backups --------------------------------------------------------------------- + + let backups = $state([]); + let backupsLoaded = $state(false); + let backingUp = $state(false); + let lastBackup = $state(''); + + $effect(() => { + if (!app.user?.is_admin) return; + api + .listBackups() + .then((r) => { + backups = r.backups; + backupsLoaded = true; + }) + .catch((e) => app.toast(humanMessage(e))); + }); + + /** + * Takes one now. + * + * Worth a button of its own rather than trusting the nightly job: waiting until + * tonight to find out whether backups work at all is how somebody finds out on + * the day they need one. It is also the only way to get a snapshot from *before* + * something you are about to do. + */ + async function backupNow() { + backingUp = true; + lastBackup = ''; + try { + const made = await api.runBackup(); + backups = [made, ...backups]; + lastBackup = `Kopi taget — ${size(made.size_bytes)}`; + } catch (e) { + app.toast(humanMessage(e)); + } finally { + backingUp = false; + } + } + + function size(bytes) { + if (!bytes) return 'tom'; + const mb = bytes / 1024 / 1024; + if (mb >= 1) return `${mb.toFixed(1)} MB`; + return `${Math.max(1, Math.round(bytes / 1024))} KB`; + } + + function when(iso) { + const then = new Date(iso); + const seconds = Math.round((Date.now() - then) / 1000); + if (seconds < 60) return 'lige nu'; + if (seconds < 3600) return `for ${Math.floor(seconds / 60)} min. siden`; + if (seconds < 86400) return `for ${Math.floor(seconds / 3600)} timer siden`; + return then.toLocaleString('da-DK', { + day: 'numeric', + month: 'short', + hour: '2-digit', + minute: '2-digit' + }); + } + // --- Todoist import ------------------------------------------------------------ let importing = $state(false); @@ -208,6 +269,68 @@ {/if} + +{#if app.user?.is_admin} +
+
+

Sikkerhedskopier

+

+ Databasen kopieres hver nat og de seneste fjorten gemmes. Kopierne er + talt, ikke dateret: en container, der har stået slukket en måned, må ikke + komme tilbage og slette dem alle for at være for gamle. +

+

+ Vedhæftede filer er ikke med. De er indholdsadresserede og bliver kun lagt + til, så fjorten ens kopier af dem ville fylde det drev op, de skulle + beskytte — tag hele /data, hvis du vil have det hele. +

+
+ +
+ + {#if lastBackup}{lastBackup}{/if} +
+ + {#if backups.length} + + {:else if backupsLoaded} +

Der er ikke taget nogen kopi endnu.

+ {/if} + +

+ At lægge en kopi tilbage er ikke en knap her, og det er med vilje: en database + kan ikke skiftes ud under en server, der skriver i den, og en halvt gennemført + udskiftning ødelægger netop det, den skulle redde. Stop containeren, læg filen + i stedet for verdande.db i datamappen, slet -wal og + -shm ved siden af, og start den igen. +

+
+{/if} +

Eksport

@@ -323,6 +446,39 @@