From 78891bae6fc29aa7c28f66d09917c5ff2f27862f Mon Sep 17 00:00:00 2001 From: Kristian Wind Date: Thu, 20 Aug 2026 06:59:50 +0200 Subject: [PATCH] =?UTF-8?q?feat(beacon):=20t=C3=A6l=20installationer,=20si?= =?UTF-8?q?g=20pr=C3=A6cis=20hvad=20der=20sendes,=20og=20lad=20folk=20sige?= =?UTF-8?q?=20nej?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spørgsmålet er ét: hvor mange bruger det her. Svaret er to værdier én gang i døgnet — et tilfældigt id, installationen selv fandt på, og versionen. Ikke andet. Beskeden er lille nok til at stå i fuld længde på indstillingssiden, og den gør det, fordi et løfte om telemetri er præcis så meget værd som læserens mulighed for at efterprøve det. **Ikke obligatorisk, og grunden er praktisk frem for principiel:** det kan ikke håndhæves. Domænet kan blokeres, binæren kan patches, og en fork fjerner den i første commit — så obligatorisk køber intet over slået-til-som-udgangspunkt og koster den tillid, resten af programmet arbejder for. Hemmeligheder forsegles her, nøglen holdes uden for databasen, og sikkerhedskopisiden advarer om, hvad en kopi indeholder. Telemetri, man ikke må sige nej til, er den ene sætning i den historie, der ikke passer. Uenigheden med `config.UpdateCheck`, som er slået fra med den modsatte begrundelse, står skrevet i pakkens egen dokumentation frem for at blive glattet ud. Den er reel, og læseren fortjener at se begge halvdele. Id'et findes på og gemmes ved første brug frem for at blive udledt af værtsnavn eller adresse. En hash af noget om maskinen ville være stabil, bekvem og samtidig en værdi, modtageren kunne holde op mod et gæt — og det er forskellen mellem anonym og pseudonym. Modtagerens side er den samme binary med en kontakt: `POST /api/v1/beacon` uden login, fordi den, der ringer, er en fremmed installation. Den svarer 404, når instansen ikke er tælleren, så en almindelig installation ikke er en spamspand, der svarer 200 på alt. Ratebegrænset som login-ruterne, og tabellen har et loft: endepunktet indsætter en række på et id, den, der ringer, har fundet på, og uden et loft kunne enhver fylde disken. Tallet er "set inden for tredive dage" og ikke et løbende total. Et total stiger kun og holder op med at betyde "hvor mange bruger det" og begynder at betyde "hvor mange har prøvet det". `internal/safedial` kom til undervejs: modtagerens adresse kan skrives i fladen, og en adresse, en bruger angiver, må ikke kunne pege indad. Kontrollen ligger på den opslåede adresse og ikke på URL'en, fordi et navn kan svare 127.0.0.1 — og svare noget andet anden gang, efter en kontrol ved parsning er gået godt. Co-Authored-By: Claude Opus 5 --- cmd/verdande/main.go | 1 + docs/openapi.yaml | 108 +++++++ internal/beacon/beacon.go | 139 ++++++++ internal/beacon/beacon_test.go | 114 +++++++ internal/httpapi/beacon_handlers.go | 297 ++++++++++++++++++ internal/httpapi/server.go | 18 ++ internal/jobs/jobs.go | 20 ++ internal/safedial/safedial.go | 106 +++++++ internal/safedial/safedial_test.go | 55 ++++ internal/store/beacon.go | 138 ++++++++ internal/store/migrations/0022_beacon.sql | 26 ++ web/src/lib/api.js | 3 + web/src/lib/locales/da.js | 15 + web/src/lib/locales/en.js | 15 + web/src/lib/richtext.js | 78 +++-- .../routes/indstillinger/data/+page.svelte | 157 ++++++++- 16 files changed, 1260 insertions(+), 30 deletions(-) create mode 100644 internal/beacon/beacon.go create mode 100644 internal/beacon/beacon_test.go create mode 100644 internal/httpapi/beacon_handlers.go create mode 100644 internal/safedial/safedial.go create mode 100644 internal/safedial/safedial_test.go create mode 100644 internal/store/beacon.go create mode 100644 internal/store/migrations/0022_beacon.sql diff --git a/cmd/verdande/main.go b/cmd/verdande/main.go index edbdea6..b17f785 100644 --- a/cmd/verdande/main.go +++ b/cmd/verdande/main.go @@ -76,6 +76,7 @@ func run() error { runner.SyncGmail = api.SyncGmail runner.SyncMailbox = api.SyncMailbox runner.Push = api.PushToUser + runner.SendBeacon = api.SendBeacon runner.Start(jobCtx) srv := &http.Server{ diff --git a/docs/openapi.yaml b/docs/openapi.yaml index ade28d3..fe2f645 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1396,6 +1396,114 @@ paths: description: The environment variables that are not set. "403": { $ref: "#/components/responses/Forbidden" } + /api/v1/beacon: + post: + tags: [auth] + summary: Report that an installation exists. Unauthenticated. + description: | + The receiving end of the beacon. Open on purpose: the caller is a + stranger's installation, which is the whole feature. + + Answers 404 unless this instance has been told it is the collector, so an + ordinary installation is not a spam sink that answers 200 to anything. + Rate limited like the login routes, and the table behind it has a ceiling: + the endpoint inserts a row keyed on an id the caller invented, so without + one anybody could grow it until the volume is full. + + Nothing about the request is recorded but the two values in the body. No + source address is read or written here. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [instance_id] + properties: + instance_id: + type: string + description: A random id the installation generated for itself. + version: + type: string + responses: + "204": + description: Recorded. + "400": + description: The id or the version is not the expected shape. + "404": + description: This instance is not a collector. + + /api/v1/beacon/count: + get: + tags: [auth] + summary: How many installations reported in the last thirty days. + description: | + Unauthenticated and cached for a minute, and only answered when the + collector has been told to publish it. + + A window rather than a running total: a total only climbs, so it stops + meaning "how many people use this" and starts meaning "how many ever + tried it". + responses: + "200": + description: The count. + content: + application/json: + schema: + type: object + properties: + installs: { type: integer } + window_days: { type: integer } + "404": + description: Not a collector, or not published. + + /api/v1/beacon/settings: + get: + tags: [auth] + summary: What this instance reports, and to whom. Administrators only. + description: | + Includes the exact payload that will be sent, so the settings page can + print it in full rather than describe it. + responses: + "200": + description: The current configuration, and the counts if this is the collector. + content: + application/json: + schema: + type: object + properties: + enabled: { type: boolean } + collector_url: { type: string } + instance_id: { type: string } + version: { type: string } + is_collector: { type: boolean } + publish_count: { type: boolean } + last_ping_at: { type: string, format: date-time } + put: + tags: [auth] + summary: Turn the beacon off, or point it somewhere else. Administrators only. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: { type: boolean } + collector_url: + type: string + description: | + Where reports go. Fetched through a dialer that refuses + loopback and private address space, so this cannot be used to + make the server reach inside its own network. + is_collector: { type: boolean } + publish_count: { type: boolean } + responses: + "200": + description: The configuration as it now stands. + "422": + description: The collector address is not an http or https URL. + /api/v1/panel/restart: post: tags: [auth] diff --git a/internal/beacon/beacon.go b/internal/beacon/beacon.go new file mode 100644 index 0000000..8b0c6a7 --- /dev/null +++ b/internal/beacon/beacon.go @@ -0,0 +1,139 @@ +// Package beacon answers one question for the project, and refuses to answer any +// other: how many installations are out there. +// +// It sends two values once a day — an anonymous id the installation made up for +// itself, and the version it is running — and nothing else. No IP is stored by +// the collector, no hostname, no domain, no account, no usage. The payload is +// small enough to be printed in full on the settings page, and it is, because a +// promise about telemetry is worth exactly as much as the reader's ability to +// check it. +// +// # On by default, and off in one click +// +// The alternative that was asked for first was mandatory, and the argument +// against it is practical rather than principled: it cannot be enforced. The +// domain can be blocked, the binary can be patched, and a fork removes it in its +// first commit — so mandatory buys nothing over on-by-default, while costing the +// trust the rest of this program works for. Secrets are sealed here, the key is +// kept out of the database, and the backup page warns you what a copy contains. +// Telemetry you are not allowed to refuse is the one sentence in that story that +// does not fit. +// +// So: on by default, said plainly, one switch to turn it off. That collects +// essentially the same number and leaves nobody feeling tricked. +// +// # Where this disagrees with the rest of the program +// +// `config.UpdateCheck` is off unless asked for, and its comment gives the reason: +// "a self-hosted app that reaches out without being told to has broken the deal +// its operator made by self-hosting." That is the same situation as this one, and +// the two defaults do not agree. +// +// The disagreement is written down here rather than smoothed over, because it is +// a real one and the reader deserves to see both halves. The case for treating +// them differently is that the update check reveals an instance to a third party +// (GitHub) and returns something the operator can act on, while this returns +// nothing and goes to the project itself — and that a count nobody sends is a +// count that does not exist, whereas an update nobody checks for is one you can +// still go and look up. Whether that is a distinction or an excuse is a judgement +// call, and it was made deliberately by the person whose instance this is. +// +// If it is ever decided the other way, the change is one word: the default in +// `beaconSettings`. Nothing else in this package assumes which way it points. +// +// # What the collector can still see +// +// A request has a source address whether or not anybody writes it down. The +// collector does not store or log it, and the code is arranged so that it never +// reaches a function that could — but it passes through a reverse proxy and a +// tunnel on the way, and those keep their own logs. Saying "we send no IP" would +// be a half-truth; what is true is that nothing here records one. +package beacon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "time" +) + +// DefaultCollector is where a fresh installation reports, unless it is told +// otherwise. Somebody running their own fleet points it at their own instance; +// the field is on the settings page for exactly that. +const DefaultCollector = "https://verdande.yggdrasilpanel.com/api/v1/beacon" + +// Payload is the entire message. Two fields, and adding a third is a change to +// what this program promises, not a change to a struct. +type Payload struct { + InstanceID string `json:"instance_id"` + Version string `json:"version"` +} + +// InstanceID and Version are checked on the way in as well as written on the way +// out, because the collector's endpoint is unauthenticated: anything that reaches +// it is a stranger's bytes, and it should be able to store only the two shapes it +// expects. +var ( + idShape = regexp.MustCompile(`^[0-9a-fA-F-]{16,64}$`) + versionShape = regexp.MustCompile(`^v?[0-9]{1,4}(\.[0-9]{1,5}){0,3}(-[0-9A-Za-z.]{1,20})?$`) +) + +// ValidID reports whether an id is the shape this program writes. +func ValidID(s string) bool { return idShape.MatchString(s) } + +// ValidVersion allows a version number and the word this binary uses when it was +// built outside a release. +func ValidVersion(s string) bool { return s == "dev" || versionShape.MatchString(s) } + +// Send delivers one ping. It is deliberately dull: a short timeout, no retry, no +// queue, and a failure that is returned rather than made anybody's problem. +// +// No retry, because the next pass is tomorrow and a missed day changes a count of +// installations by nothing. A telemetry ping that retries is a telemetry ping +// that can hammer somebody's server on the day it is down. +func Send(ctx context.Context, client *http.Client, collector string, p Payload) error { + if !ValidID(p.InstanceID) { + return fmt.Errorf("beacon: refusing to send a malformed id") + } + // Versionen kontrolleres også her, ikke kun hos modtageren. Kommentaren ved + // mønstrene lovede begge veje, og gjorde det ikke — og en installation, der + // sender noget mærkeligt, skal opdage det hos sig selv frem for at være noget, + // en fremmed collector skal filtrere fra. + if !ValidVersion(p.Version) { + return fmt.Errorf("beacon: refusing to send a malformed version") + } + body, err := json.Marshal(p) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, collector, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + // A plain user agent with the version already in the payload. Nothing is + // learned from it that the body does not already say, and it makes the request + // legible in a log rather than anonymous-looking. + req.Header.Set("User-Agent", "verdande-beacon/1") + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + // Read and discard, so the connection can be reused rather than dropped. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) + + if resp.StatusCode >= 400 { + return fmt.Errorf("beacon: collector answered %d", resp.StatusCode) + } + return nil +} diff --git a/internal/beacon/beacon_test.go b/internal/beacon/beacon_test.go new file mode 100644 index 0000000..3d19423 --- /dev/null +++ b/internal/beacon/beacon_test.go @@ -0,0 +1,114 @@ +package beacon + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// Hvad der bliver sendt, tegn for tegn. +// +// Prøven er skrevet imod *hele* kroppen og ikke imod enkelte felter, fordi det er +// den, der er løftet: siden siger "præcis det her og ikke mere", og et felt lagt +// til senere skal vælte noget. En prøve, der kun så efter, at instance_id var med, +// ville lade en ny nøgle glide igennem uden at nogen opdagede det. +func TestSendsExactlyTwoFields(t *testing.T) { + var got map[string]any + var contentType string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentType = r.Header.Get("Content-Type") + body, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(body, &got); err != nil { + t.Errorf("kroppen er ikke JSON: %v", err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + err := Send(context.Background(), srv.Client(), srv.URL, Payload{ + InstanceID: "01a01b16-4e54-7000-9eaa-2f8eb0629c3a", + Version: "v0.26.3", + }) + if err != nil { + t.Fatalf("Send: %v", err) + } + + if contentType != "application/json" { + t.Errorf("Content-Type = %q", contentType) + } + if len(got) != 2 { + t.Fatalf("der blev sendt %d felter, ikke to: %v", len(got), got) + } + if got["instance_id"] != "01a01b16-4e54-7000-9eaa-2f8eb0629c3a" { + t.Errorf("instance_id = %v", got["instance_id"]) + } + if got["version"] != "v0.26.3" { + t.Errorf("version = %v", got["version"]) + } +} + +// En collector, der er nede, er ikke denne installations problem. +func TestSendReportsAFailureRatherThanRetrying(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + err := Send(context.Background(), srv.Client(), srv.URL, Payload{ + InstanceID: "01a01b16-4e54-7000-9eaa-2f8eb0629c3a", Version: "v1.0.0", + }) + if err == nil { + t.Fatal("en 500 skal give en fejl") + } + if calls != 1 { + t.Errorf("der blev prøvet %d gange; en telemetri-ping må ikke hamre på en server, der er nede", calls) + } +} + +// Et id, der ikke er id-formet, sendes slet ikke — så en fejl her ikke bliver til +// noget, en fremmed collector skal filtrere fra. +func TestSendRefusesAMalformedID(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("der blev sendt noget, som ikke skulle sendes") + })) + defer srv.Close() + + if err := Send(context.Background(), srv.Client(), srv.URL, Payload{ + InstanceID: "; DROP TABLE beacon_installs", Version: "v1.0.0", + }); err == nil { + t.Fatal("et misformet id skal afvises før det forlader maskinen") + } +} + +func TestShapes(t *testing.T) { + ok := []string{"01a01b16-4e54-7000-9eaa-2f8eb0629c3a", "89df44d7-3742-4611-9a43-db0b07733c6d"} + for _, s := range ok { + if !ValidID(s) { + t.Errorf("ValidID(%q) = false", s) + } + } + bad := []string{"", "kort", "../../etc/passwd", "01a01b16-4e54-7000-9eaa-2f8eb0629c3a'; --", + ""} + for _, s := range bad { + if ValidID(s) { + t.Errorf("ValidID(%q) = true", s) + } + } + + for _, s := range []string{"v0.26.3", "0.26.3", "v1.2.3-rc.1", "dev"} { + if !ValidVersion(s) { + t.Errorf("ValidVersion(%q) = false", s) + } + } + for _, s := range []string{"", "v