diff --git a/.gitea/workflows/compat.yml b/.gitea/workflows/compat.yml index b05a07a..8f201a1 100644 --- a/.gitea/workflows/compat.yml +++ b/.gitea/workflows/compat.yml @@ -39,7 +39,14 @@ jobs: # work again on a single self-hosted runner for no extra information. # GitHub's runners take the scheduled load; Gitea keeps workflow_dispatch, # so the run is still one click away there. - if: github.event_name != 'schedule' || github.server_url == 'https://github.com' + # + # 🔴 This gates EVERY event except workflow_dispatch, not just the + # schedule. The earlier `!= 'schedule'` form let `pull_request` through, + # and this workflow triggers on any PR touching ci/ or tests/ — so five + # cells landed on the one self-hosted runner together. A sibling plugin + # measured six jobs dying at once with exit code 137: SIGKILL, out of + # memory, two seconds in, with not one error message in the log. + if: github.event_name == 'workflow_dispatch' || github.server_url == 'https://github.com' runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index b05a07a..8f201a1 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -39,7 +39,14 @@ jobs: # work again on a single self-hosted runner for no extra information. # GitHub's runners take the scheduled load; Gitea keeps workflow_dispatch, # so the run is still one click away there. - if: github.event_name != 'schedule' || github.server_url == 'https://github.com' + # + # 🔴 This gates EVERY event except workflow_dispatch, not just the + # schedule. The earlier `!= 'schedule'` form let `pull_request` through, + # and this workflow triggers on any PR touching ci/ or tests/ — so five + # cells landed on the one self-hosted runner together. A sibling plugin + # measured six jobs dying at once with exit code 137: SIGKILL, out of + # memory, two seconds in, with not one error message in the log. + if: github.event_name == 'workflow_dispatch' || github.server_url == 'https://github.com' runs-on: ubuntu-latest strategy: diff --git a/ci/check-workflows.py b/ci/check-workflows.py new file mode 100755 index 0000000..9f02995 --- /dev/null +++ b/ci/check-workflows.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Parse every workflow file and check the rules that only bite on Gitea. + +Two things this catches, both of which are silent where they happen: + +1. A workflow that does not parse produces NO RUN on Gitea — not a failed run, + not a cancelled one. It looks exactly like a runner that has not picked the + job up, so you wait, and waiting never fixes it. + +2. A job with a multi-cell matrix that is not gated to GitHub asks a single + self-hosted runner to do all of it at once. A sibling plugin measured six + jobs dying together with exit code 137 — SIGKILL, out of memory, two + seconds in, with not one error message in the log. + +It reports how many files and jobs it examined, and it is fatal when that is +zero: "all workflows are fine" is also what this prints having parsed nothing, +which is the failure mode the whole ci/ kit has now been bitten by four times. +""" +import sys +import yaml + +# 🔴 Exact conditions, not a substring. The first version of this check asked +# whether the `if:` CONTAINED the server_url test, and passed the very +# condition it was written to catch: +# +# github.event_name != 'schedule' || github.server_url == '...' +# +# — which gates the schedule and lets `pull_request` through, so five cells +# still land on the Gitea runner on every PR that touches ci/. A substring +# match is "adjacent to the signal" rather than the signal, which is the +# failure mode this whole ci/ kit keeps being bitten by. +ACCEPTED = { + # Gitea runs it only when a human clicks it. + "github.event_name == 'workflow_dispatch' || " + "github.server_url == 'https://github.com'", + # Or never on Gitea at all. + "github.server_url == 'https://github.com'", +} + +def cells(job): + """Number of matrix cells, 1 when the job has no matrix.""" + matrix = (job.get('strategy') or {}).get('matrix') + if not isinstance(matrix, dict): + return 1 + if 'include' in matrix and isinstance(matrix['include'], list): + return max(len(matrix['include']), 1) + total = 1 + for key, values in matrix.items(): + if key in ('include', 'exclude') or not isinstance(values, list): + continue + total *= max(len(values), 1) + return total + +def main(paths): + problems = 0 + files = 0 + jobs = 0 + + for path in paths: + try: + with open(path) as handle: + doc = yaml.safe_load(handle) + except yaml.YAMLError as err: + print(f"::error file={path}::Does not parse, so on Gitea it will " + f"simply never run: {err}") + problems += 1 + continue + + files += 1 + if not isinstance(doc, dict): + continue + + # Only .gitea/ is checked for the gate: .github/ workflows run on + # GitHub's runners, which are not the scarce resource here. + if not path.startswith('.gitea/'): + for _ in (doc.get('jobs') or {}): + jobs += 1 + continue + + for name, job in (doc.get('jobs') or {}).items(): + jobs += 1 + if not isinstance(job, dict): + continue + n = cells(job) + if n <= 1: + continue + condition = ' '.join(str(job.get('if', '')).split()) + if condition not in ACCEPTED: + print(f"::error file={path}::Job '{name}' runs {n} matrix " + f"cells on Gitea, and its `if:` is not one of the " + f"accepted gates. One self-hosted runner is asked to do " + f"all of it at once, and the symptom is exit code 137 " + f"with an empty log. Found: {condition or ''}") + problems += 1 + + # Not caller-configurable: the smallest repo that has workflows has one. + if files == 0: + print("No workflow file was parsed at all. The paths are wrong, so " + "this check proved nothing.", file=sys.stderr) + return 1 + + if problems: + print(f"\n{problems} workflow problem(s) across {files} file(s).") + return 1 + + print(f" {files} workflow files parsed, {jobs} jobs checked") + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/ci/lib.sh b/ci/lib.sh index 4e0c7cc..a723a88 100644 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -96,6 +96,27 @@ ci_php() { ci_in_container "php:8.4-cli" "exec php $(printf '%q ' "$@")" } +# Run one of this repository's python scripts, with PyYAML available. +# +# Same shape and the same reason as ci_php above, and it is not hypothetical: +# the Gitea runner image has NO PyYAML (measured — `import yaml` raises +# ModuleNotFoundError there), while every laptop and GitHub's hosted runners +# have it. So a check that merely skips when the import fails is a check that +# is switched off on exactly one forge — and Gitea is the forge where a +# workflow that does not parse produces no run and no error to click on. +# +# alpine + py3-yaml rather than a python image + pip: no network install, one +# apk call, the same pinned answer every time. +ci_python_yaml() { + if python3 -c 'import yaml' 2>/dev/null; then + python3 "$@" + return + fi + + ci_in_container "alpine:3.20" \ + "apk add --no-cache python3 py3-yaml >/dev/null && exec python3 $(printf '%q ' "$@")" +} + ci_step() { printf '\n\033[1m→ %s\033[0m\n' "$*"; } ci_fail() { printf '\033[31mFAIL\033[0m %s\n' "$*" >&2; } diff --git a/ci/selftest.sh b/ci/selftest.sh index d17111f..b514a63 100755 --- a/ci/selftest.sh +++ b/ci/selftest.sh @@ -178,6 +178,53 @@ else ok "a plugin.conf with no floors fails to load" fi +# ------------------------------------------------------------------ workflows +# Both of these are silent where they happen. An unparseable workflow produces +# NO RUN on Gitea, and a multi-cell matrix that is not gated lands on one +# self-hosted runner as exit code 137 with an empty log. Neither shows up on +# GitHub, which is where anyone would be looking. +echo +echo "→ ci/workflows-in-sync.sh must catch what only breaks on Gitea" +cd "$work/repo" +cp -a .github/workflows "$work/wf-github-backup" +cp -a .gitea/workflows "$work/wf-gitea-backup" +restore_workflows() { + rm -rf .github/workflows .gitea/workflows + cp -a "$work/wf-github-backup" .github/workflows + cp -a "$work/wf-gitea-backup" .gitea/workflows +} + +# Corrupt BOTH roots identically, so this fails on the PARSE rather than on the +# byte-identity rule that would otherwise fire first. +printf '\n this: [is not\n' >> .github/workflows/ci.yml +printf '\n this: [is not\n' >> .gitea/workflows/ci.yml +./ci/workflows-in-sync.sh >/dev/null 2>&1 \ + && bad "an unparseable workflow passed — on Gitea it would simply never run" \ + || ok "an unparseable workflow fails the build" +restore_workflows + +# The gate that keeps a five-cell matrix off the single self-hosted runner. +# @ as the delimiter, not |: the gate itself contains `||`, and with | as the +# delimiter sed parses the fixture into nonsense and silently changes nothing — +# which is how this assertion first ran against an unmodified file. It reported +# "could not stage the fixture" rather than passing, which is the whole reason +# the staging is checked before the assertion is made. +sed -i "s@^ if: github.event_name == 'workflow_dispatch'.*@ if: always()@" \ + .github/workflows/compat.yml .gitea/workflows/compat.yml +if grep -q 'if: always()' .gitea/workflows/compat.yml; then + ./ci/workflows-in-sync.sh >/dev/null 2>&1 \ + && bad "an ungated multi-cell matrix passed — it lands on the Gitea runner as exit 137" \ + || ok "an ungated multi-cell matrix fails the build" +else + bad "could not stage the ungated-matrix fixture — the gate line did not match" +fi +restore_workflows + +# Parsing nothing is never a legitimate result. +ci_python_yaml ci/check-workflows.py >/dev/null 2>&1 \ + && bad "check-workflows.py passed having parsed no file at all" \ + || ok "parsing zero workflow files is fatal" + echo if [ "$fail" != 0 ]; then echo "Self-test: $fail of $((pass + fail)) assertions FAILED." diff --git a/ci/workflows-in-sync.sh b/ci/workflows-in-sync.sh index 1bdb471..ec3e65b 100755 --- a/ci/workflows-in-sync.sh +++ b/ci/workflows-in-sync.sh @@ -12,6 +12,7 @@ # one appears in only one root without a stated reason. set -euo pipefail cd "$(dirname "$0")/.." +. ci/lib.sh # Workflows that are deliberately single-forge, and why. declare -A only_on=( @@ -70,15 +71,18 @@ fi # A workflow that does not parse produces no run — the same symptom as every # other trap in this file. ci.yml cannot catch its own syntax error this way, # but release.yml and compat.yml can be caught here rather than on the tag. -if python3 -c 'import yaml' 2>/dev/null; then - for f in .github/workflows/*.yml .gitea/workflows/*.yml; do - [ -e "$f" ] || continue - python3 -c 'import sys,yaml; yaml.safe_load(open(sys.argv[1]))' "$f" \ - || { echo "$f is not valid YAML, so it will simply never run."; fail=1; } - done -else - echo "(PyYAML is not installed here, so the workflow files were not parsed.)" -fi +# +# 🔴 This used to skip with a note when PyYAML was missing, and still printed +# "Workflows: in sync." at the end. PyYAML is missing in exactly one place — +# the GITEA RUNNER (measured: `import yaml` raises ModuleNotFoundError on +# gitea/runner-images:ubuntu-latest) — which is the one forge where an +# unparseable workflow produces no run and no error to click on. So the check +# was switched off precisely where it was needed, and said "in sync" anyway. +# +# ci_python_yaml falls back to a container, so there is no skip path left: if +# the workflows cannot be parsed, that is a failure, not a note. +ci_python_yaml ci/check-workflows.py .github/workflows/*.yml .gitea/workflows/*.yml \ + || fail=1 [ "$fail" = 0 ] || exit 1 echo "Workflows: in sync."