diff --git a/cmd/verdande/main.go b/cmd/verdande/main.go index edbdea6..b17f785 100644 --- a/cmd/verdande/main.go +++ b/cmd/verdande/main.go @@ -76,6 +76,7 @@ func run() error { runner.SyncGmail = api.SyncGmail runner.SyncMailbox = api.SyncMailbox runner.Push = api.PushToUser + runner.SendBeacon = api.SendBeacon runner.Start(jobCtx) srv := &http.Server{ diff --git a/docs/openapi.yaml b/docs/openapi.yaml index ade28d3..fe2f645 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1396,6 +1396,114 @@ paths: description: The environment variables that are not set. "403": { $ref: "#/components/responses/Forbidden" } + /api/v1/beacon: + post: + tags: [auth] + summary: Report that an installation exists. Unauthenticated. + description: | + The receiving end of the beacon. Open on purpose: the caller is a + stranger's installation, which is the whole feature. + + Answers 404 unless this instance has been told it is the collector, so an + ordinary installation is not a spam sink that answers 200 to anything. + Rate limited like the login routes, and the table behind it has a ceiling: + the endpoint inserts a row keyed on an id the caller invented, so without + one anybody could grow it until the volume is full. + + Nothing about the request is recorded but the two values in the body. No + source address is read or written here. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [instance_id] + properties: + instance_id: + type: string + description: A random id the installation generated for itself. + version: + type: string + responses: + "204": + description: Recorded. + "400": + description: The id or the version is not the expected shape. + "404": + description: This instance is not a collector. + + /api/v1/beacon/count: + get: + tags: [auth] + summary: How many installations reported in the last thirty days. + description: | + Unauthenticated and cached for a minute, and only answered when the + collector has been told to publish it. + + A window rather than a running total: a total only climbs, so it stops + meaning "how many people use this" and starts meaning "how many ever + tried it". + responses: + "200": + description: The count. + content: + application/json: + schema: + type: object + properties: + installs: { type: integer } + window_days: { type: integer } + "404": + description: Not a collector, or not published. + + /api/v1/beacon/settings: + get: + tags: [auth] + summary: What this instance reports, and to whom. Administrators only. + description: | + Includes the exact payload that will be sent, so the settings page can + print it in full rather than describe it. + responses: + "200": + description: The current configuration, and the counts if this is the collector. + content: + application/json: + schema: + type: object + properties: + enabled: { type: boolean } + collector_url: { type: string } + instance_id: { type: string } + version: { type: string } + is_collector: { type: boolean } + publish_count: { type: boolean } + last_ping_at: { type: string, format: date-time } + put: + tags: [auth] + summary: Turn the beacon off, or point it somewhere else. Administrators only. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: { type: boolean } + collector_url: + type: string + description: | + Where reports go. Fetched through a dialer that refuses + loopback and private address space, so this cannot be used to + make the server reach inside its own network. + is_collector: { type: boolean } + publish_count: { type: boolean } + responses: + "200": + description: The configuration as it now stands. + "422": + description: The collector address is not an http or https URL. + /api/v1/panel/restart: post: tags: [auth] diff --git a/internal/beacon/beacon.go b/internal/beacon/beacon.go new file mode 100644 index 0000000..8b0c6a7 --- /dev/null +++ b/internal/beacon/beacon.go @@ -0,0 +1,139 @@ +// Package beacon answers one question for the project, and refuses to answer any +// other: how many installations are out there. +// +// It sends two values once a day — an anonymous id the installation made up for +// itself, and the version it is running — and nothing else. No IP is stored by +// the collector, no hostname, no domain, no account, no usage. The payload is +// small enough to be printed in full on the settings page, and it is, because a +// promise about telemetry is worth exactly as much as the reader's ability to +// check it. +// +// # On by default, and off in one click +// +// The alternative that was asked for first was mandatory, and the argument +// against it is practical rather than principled: it cannot be enforced. The +// domain can be blocked, the binary can be patched, and a fork removes it in its +// first commit — so mandatory buys nothing over on-by-default, while costing the +// trust the rest of this program works for. Secrets are sealed here, the key is +// kept out of the database, and the backup page warns you what a copy contains. +// Telemetry you are not allowed to refuse is the one sentence in that story that +// does not fit. +// +// So: on by default, said plainly, one switch to turn it off. That collects +// essentially the same number and leaves nobody feeling tricked. +// +// # Where this disagrees with the rest of the program +// +// `config.UpdateCheck` is off unless asked for, and its comment gives the reason: +// "a self-hosted app that reaches out without being told to has broken the deal +// its operator made by self-hosting." That is the same situation as this one, and +// the two defaults do not agree. +// +// The disagreement is written down here rather than smoothed over, because it is +// a real one and the reader deserves to see both halves. The case for treating +// them differently is that the update check reveals an instance to a third party +// (GitHub) and returns something the operator can act on, while this returns +// nothing and goes to the project itself — and that a count nobody sends is a +// count that does not exist, whereas an update nobody checks for is one you can +// still go and look up. Whether that is a distinction or an excuse is a judgement +// call, and it was made deliberately by the person whose instance this is. +// +// If it is ever decided the other way, the change is one word: the default in +// `beaconSettings`. Nothing else in this package assumes which way it points. +// +// # What the collector can still see +// +// A request has a source address whether or not anybody writes it down. The +// collector does not store or log it, and the code is arranged so that it never +// reaches a function that could — but it passes through a reverse proxy and a +// tunnel on the way, and those keep their own logs. Saying "we send no IP" would +// be a half-truth; what is true is that nothing here records one. +package beacon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "time" +) + +// DefaultCollector is where a fresh installation reports, unless it is told +// otherwise. Somebody running their own fleet points it at their own instance; +// the field is on the settings page for exactly that. +const DefaultCollector = "https://verdande.yggdrasilpanel.com/api/v1/beacon" + +// Payload is the entire message. Two fields, and adding a third is a change to +// what this program promises, not a change to a struct. +type Payload struct { + InstanceID string `json:"instance_id"` + Version string `json:"version"` +} + +// InstanceID and Version are checked on the way in as well as written on the way +// out, because the collector's endpoint is unauthenticated: anything that reaches +// it is a stranger's bytes, and it should be able to store only the two shapes it +// expects. +var ( + idShape = regexp.MustCompile(`^[0-9a-fA-F-]{16,64}$`) + versionShape = regexp.MustCompile(`^v?[0-9]{1,4}(\.[0-9]{1,5}){0,3}(-[0-9A-Za-z.]{1,20})?$`) +) + +// ValidID reports whether an id is the shape this program writes. +func ValidID(s string) bool { return idShape.MatchString(s) } + +// ValidVersion allows a version number and the word this binary uses when it was +// built outside a release. +func ValidVersion(s string) bool { return s == "dev" || versionShape.MatchString(s) } + +// Send delivers one ping. It is deliberately dull: a short timeout, no retry, no +// queue, and a failure that is returned rather than made anybody's problem. +// +// No retry, because the next pass is tomorrow and a missed day changes a count of +// installations by nothing. A telemetry ping that retries is a telemetry ping +// that can hammer somebody's server on the day it is down. +func Send(ctx context.Context, client *http.Client, collector string, p Payload) error { + if !ValidID(p.InstanceID) { + return fmt.Errorf("beacon: refusing to send a malformed id") + } + // Versionen kontrolleres også her, ikke kun hos modtageren. Kommentaren ved + // mønstrene lovede begge veje, og gjorde det ikke — og en installation, der + // sender noget mærkeligt, skal opdage det hos sig selv frem for at være noget, + // en fremmed collector skal filtrere fra. + if !ValidVersion(p.Version) { + return fmt.Errorf("beacon: refusing to send a malformed version") + } + body, err := json.Marshal(p) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, collector, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + // A plain user agent with the version already in the payload. Nothing is + // learned from it that the body does not already say, and it makes the request + // legible in a log rather than anonymous-looking. + req.Header.Set("User-Agent", "verdande-beacon/1") + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + // Read and discard, so the connection can be reused rather than dropped. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) + + if resp.StatusCode >= 400 { + return fmt.Errorf("beacon: collector answered %d", resp.StatusCode) + } + return nil +} diff --git a/internal/beacon/beacon_test.go b/internal/beacon/beacon_test.go new file mode 100644 index 0000000..3d19423 --- /dev/null +++ b/internal/beacon/beacon_test.go @@ -0,0 +1,114 @@ +package beacon + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// Hvad der bliver sendt, tegn for tegn. +// +// Prøven er skrevet imod *hele* kroppen og ikke imod enkelte felter, fordi det er +// den, der er løftet: siden siger "præcis det her og ikke mere", og et felt lagt +// til senere skal vælte noget. En prøve, der kun så efter, at instance_id var med, +// ville lade en ny nøgle glide igennem uden at nogen opdagede det. +func TestSendsExactlyTwoFields(t *testing.T) { + var got map[string]any + var contentType string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentType = r.Header.Get("Content-Type") + body, _ := io.ReadAll(r.Body) + if err := json.Unmarshal(body, &got); err != nil { + t.Errorf("kroppen er ikke JSON: %v", err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + err := Send(context.Background(), srv.Client(), srv.URL, Payload{ + InstanceID: "01a01b16-4e54-7000-9eaa-2f8eb0629c3a", + Version: "v0.26.3", + }) + if err != nil { + t.Fatalf("Send: %v", err) + } + + if contentType != "application/json" { + t.Errorf("Content-Type = %q", contentType) + } + if len(got) != 2 { + t.Fatalf("der blev sendt %d felter, ikke to: %v", len(got), got) + } + if got["instance_id"] != "01a01b16-4e54-7000-9eaa-2f8eb0629c3a" { + t.Errorf("instance_id = %v", got["instance_id"]) + } + if got["version"] != "v0.26.3" { + t.Errorf("version = %v", got["version"]) + } +} + +// En collector, der er nede, er ikke denne installations problem. +func TestSendReportsAFailureRatherThanRetrying(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + err := Send(context.Background(), srv.Client(), srv.URL, Payload{ + InstanceID: "01a01b16-4e54-7000-9eaa-2f8eb0629c3a", Version: "v1.0.0", + }) + if err == nil { + t.Fatal("en 500 skal give en fejl") + } + if calls != 1 { + t.Errorf("der blev prøvet %d gange; en telemetri-ping må ikke hamre på en server, der er nede", calls) + } +} + +// Et id, der ikke er id-formet, sendes slet ikke — så en fejl her ikke bliver til +// noget, en fremmed collector skal filtrere fra. +func TestSendRefusesAMalformedID(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("der blev sendt noget, som ikke skulle sendes") + })) + defer srv.Close() + + if err := Send(context.Background(), srv.Client(), srv.URL, Payload{ + InstanceID: "; DROP TABLE beacon_installs", Version: "v1.0.0", + }); err == nil { + t.Fatal("et misformet id skal afvises før det forlader maskinen") + } +} + +func TestShapes(t *testing.T) { + ok := []string{"01a01b16-4e54-7000-9eaa-2f8eb0629c3a", "89df44d7-3742-4611-9a43-db0b07733c6d"} + for _, s := range ok { + if !ValidID(s) { + t.Errorf("ValidID(%q) = false", s) + } + } + bad := []string{"", "kort", "../../etc/passwd", "01a01b16-4e54-7000-9eaa-2f8eb0629c3a'; --", + ""} + for _, s := range bad { + if ValidID(s) { + t.Errorf("ValidID(%q) = true", s) + } + } + + for _, s := range []string{"v0.26.3", "0.26.3", "v1.2.3-rc.1", "dev"} { + if !ValidVersion(s) { + t.Errorf("ValidVersion(%q) = false", s) + } + } + for _, s := range []string{"", "v