From 0a586618647bca3dc1f63ffa5fb99b1e7da531ae Mon Sep 17 00:00:00 2001 From: kristianwind Date: Thu, 17 Sep 2026 00:30:52 +0200 Subject: [PATCH] Add ci/selftest.sh: prove the checks can still fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three rounds of fixes to this kit, and each round found the previous round's blind spot: a check that examined nothing, then a check that examined everything with a pattern that could not match, then a flag that made one rule skip binary files without a word. None of them were visible from a green run, and each was found by a sibling plugin rather than by anything here. So the battery stops being a shell loop somebody retypes and becomes a script CI runs. 26 assertions: 13 credential shapes that must be flagged, 6 lookalikes that must stay quiet, the direct-access count, the four text-domain verdicts, the annotated path being a real file, and a plugin.conf with its floors stripped failing to load. 🔴 It asserts on the thing that has CONSEQUENCES — the exit status of the script CI runs, and the specific failure message. Never a filename, a count, or a log line that merely sits near the signal. The previous ad-hoc battery asserted "the fixture's filename appears in the output", which passes on grep's own `binary file matches` diagnostic, because that diagnostic prints the filename in exactly the place a real failure would. A sibling plugin recorded two binary cases as PASS on that basis. The probe had the same defect as the check it was testing. The self-test was then checked to go red: making the AKIA pattern inert fails exactly one assertion, naming the shape. Its first run failed an assertion, and the fixture was at fault rather than the product. This script sources ci/lib.sh, so the floors are in the environment and a plain subshell inherits them — ci_load_conf then passed over a plugin.conf that does not define them. Fixed with an explicit unset, and the reason is inline so nobody weakens the assertion instead. A failing probe is evidence about the probe until you have looked at it. Wired into the static job in BOTH workflow roots, byte-identical: .gitea hides .github completely rather than merging it, so a step added to one root only would not run on Gitea and would look exactly like a runner that had not picked the job up. ci/workflows-in-sync.sh passes. Excluded from the release zip, and it plants nothing in the real tree — the fixtures go into a throwaway copy under mktemp, several being credential-shaped by definition. Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/ci.yml | 8 ++ .github/workflows/ci.yml | 8 ++ ci/selftest.sh | 163 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100755 ci/selftest.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 856cde1..feb5c82 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -30,6 +30,14 @@ jobs: - name: Lint, hygiene, versions and compatibility headers run: ci/static.sh + # Runs static.sh again against planted fixtures and asserts it goes RED. + # Every check in ci/ has been silently hollow at least once — reporting + # success having examined nothing, then examining everything with a + # pattern that could not match. Neither is visible from a green run, so + # the only defence is a thing that must be found. + - name: The checks can actually fail + run: ci/selftest.sh + - name: The release zip is buildable and complete run: ci/build-zip.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 856cde1..feb5c82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,14 @@ jobs: - name: Lint, hygiene, versions and compatibility headers run: ci/static.sh + # Runs static.sh again against planted fixtures and asserts it goes RED. + # Every check in ci/ has been silently hollow at least once — reporting + # success having examined nothing, then examining everything with a + # pattern that could not match. Neither is visible from a green run, so + # the only defence is a thing that must be found. + - name: The checks can actually fail + run: ci/selftest.sh + - name: The release zip is buildable and complete run: ci/build-zip.sh diff --git a/ci/selftest.sh b/ci/selftest.sh new file mode 100755 index 0000000..91ce0ef --- /dev/null +++ b/ci/selftest.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Does ci/static.sh actually FAIL when it should? +# +# 🔴 Why this exists, and why it asserts what it asserts. +# +# Every check in this kit has now been wrong twice in the same way: it printed +# a reassuring line having examined nothing, and later it examined everything +# with a pattern that could not match. Both are invisible from the passing +# side, because a check that finds nothing and a check that CANNOT find +# anything produce identical output. The only defence is to plant a thing that +# must be found and watch the build go red. +# +# 🔴 And the probe itself has the same failure mode. An earlier version of this +# battery asserted "the fixture's filename appears somewhere in the output" — +# which passes on grep's own `binary file matches` diagnostic, because that +# diagnostic prints the filename in exactly the place a real failure would. +# A sibling plugin recorded two binary cases as PASS on that basis and was one +# message away from reporting the shape covered. +# +# So: assert on the thing that has CONSEQUENCES. The exit status of the script +# CI runs, and the specific failure message — never a filename, a count, or a +# log line that merely sits near the signal. +set -uo pipefail +cd "$(dirname "$0")/.." + +# Loaded from the REAL tree, before the throwaway copy has its floors stripped +# by the last assertion below. +. ci/lib.sh +ci_load_conf || { echo "ci/plugin.conf will not load — fix that before running the self-test." >&2; exit 1; } + +pass=0; fail=0 +ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; pass=$((pass + 1)); } +bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; fail=$((fail + 1)); } + +# A throwaway copy: these fixtures must never be committable to the real tree, +# and several of them are credential-shaped by definition. +work=$(mktemp -d -t wpci-selftest-XXXXXX) +trap 'rm -rf "$work"' EXIT +cp -a . "$work/repo" +cd "$work/repo" + +# ---------------------------------------------------------------- credentials +# One fixture per shape the scan CLAIMS to catch, plus lookalikes that must +# stay quiet. An unfilled placeholder is not optional decoration: it is the one +# string a settings screen is guaranteed to contain, so a rule without that +# exclusion cries wolf on day one and gets commented out. +printf 'aws = AKIAIOSFODNN7EXAMPLE\n' > probe-aws.txt +printf -- '-----BEGIN RSA PRIVATE KEY-----\nMIIEfake\n' > probe-pem.txt +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > probe-skant.txt +printf 'k = ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n' > probe-ghp.txt +printf 'k = gho_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\n' > probe-gho.txt +printf 'k = github_pat_%s\n' "$(printf 'C%.0s' $(seq 1 60))" > probe-pat.txt +printf 'k = sk-DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD\n' > probe-sk.txt +printf "'api_key' => 'h8Kd92MsLp01QwErTyUiOp',\n" > probe-array.txt +printf "define( 'WC_EDM_SECRET', 'h8Kd92MsLp01QwErTyUi' );\n" > probe-define.txt + +# Directory coverage. .distignore removes all three from the zip, and a scan +# reading the SHIPPED list instead of the tracked one therefore misses every CI +# script, workflow and test fixture — which is exactly where a key gets pasted +# to test it. A sibling plugin was missing 36 of 73 files this way. +mkdir -p ci tests .github/workflows +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > ci/probe-dir.txt +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > tests/probe-dir.txt +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > .github/workflows/probe-dir.txt + +# A key inside a file grep calls binary. `grep -I` skips these SILENTLY, so the +# value rule must not use it; the build going red is the right direction to be +# wrong in. This fixture exists because that flag was added by accident and +# nothing went red to announce it. +printf 'PNG\x00\x00\x00fake-----BEGIN RSA PRIVATE KEY-----\nMIIE\n' > probe-binary.bin + +printf 'sha = 9f2b7c1e4a8d6b3f0e5c2a9d7b4f1e8c\nAKIA\n' > control-hash.txt +printf "'api_key' => 'REPLACE_WITH_YOUR_API_KEY',\n" > control-placeholder.txt +printf "'password' => 'xxxxxxxxxxxxxxxxxxxxxxxx',\n" > control-masked.txt +printf "'api_key' => '',\n'token' => 'short',\n" > control-empty.txt +printf "wp_die( 'token', 'a long sentence that follows a comma' );\n" > control-comma.txt +printf "define( 'WC_EDM_TOKEN', 'TODO_FILL_THIS_IN_LATER_XX' );\n" > control-tododefine.txt + +# -N: an UNTRACKED fixture is invisible to a scan reading `git ls-files`, so it +# passes clean and looks exactly like the scan working. +git add -N probe-*.txt probe-binary.bin ci/probe-dir.txt tests/probe-dir.txt \ + .github/workflows/probe-dir.txt control-*.txt >/dev/null + +echo "→ ci/static.sh must FAIL with credentials planted" +out=$(./ci/static.sh 2>&1); code=$? +[ "$code" != 0 ] && ok "exit status is $code" || bad "exit status is 0 — the build would have gone green" + +# The message, not the filename. grep's own "binary file matches" diagnostic +# names the file too, in the same stream, which is what made the weak version +# of this assertion pass on a check that never fired. +fired() { echo "$out" | grep -q "^.*$1 \(looks like it contains a credential\|assigns a long opaque value\)"; } + +for f in probe-aws.txt probe-pem.txt probe-skant.txt probe-ghp.txt probe-gho.txt \ + probe-pat.txt probe-sk.txt probe-array.txt probe-define.txt \ + ci/probe-dir.txt tests/probe-dir.txt .github/workflows/probe-dir.txt \ + probe-binary.bin; do + fired "$f" && ok "flagged $f" || bad "NOT FLAGGED: $f — the pattern is inert for this shape" +done + +for f in control-hash.txt control-placeholder.txt control-masked.txt \ + control-empty.txt control-comma.txt control-tododefine.txt; do + fired "$f" && bad "FALSE POSITIVE on $f — a check that cries wolf gets commented out" \ + || ok "ignored $f" +done + +# The guard check must report a plausible number rather than a bare adjective. +echo "$out" | grep -qE '^ [0-9]+ shipped PHP files examined' \ + && ok "the direct-access check reports a count" \ + || bad "the direct-access check printed no count — it cannot tell 13 files from 0" + +# ---------------------------------------------------------------- text domain +# No docker, so these are cheap. Three zeros, three verdicts. +cd "$work/repo" +echo +echo "→ ci/check-textdomain.php verdicts" +mkdir -p "$work/empty" "$work/nostrings" +printf ' "$work/nostrings/plugin.php" + +php ci/check-textdomain.php --min=0 "$TEXT_DOMAIN" "$work/empty" >/dev/null 2>&1 \ + && bad "an empty scan root passed — the root verdict must not be configurable away" \ + || ok "an empty scan root is fatal even with --min=0" + +php ci/check-textdomain.php --min=999999 "$TEXT_DOMAIN" . >/dev/null 2>&1 \ + && bad "a --min above the real count passed" \ + || ok "a --min above the real count is fatal" + +php ci/check-textdomain.php "$TEXT_DOMAIN" "$work/nostrings" >/dev/null 2>&1 \ + && ok "zero calls without --min warns rather than failing" \ + || bad "zero calls without --min failed — a small plugin may legitimately have none" + +# The annotated path must be a file that exists. str_replace( $root, '', $path ) +# with the default root of "." strips every dot and yields "class-dbphp". +mkdir -p "$work/dotted/includes" +printf " "$work/dotted/includes/class-db.php" +ann=$(cd "$work/dotted" && php "$work/repo/ci/check-textdomain.php" "$TEXT_DOMAIN" . 2>&1 | sed -n 's/.*file=\([^,]*\),.*/\1/p' | head -1) +[ -n "$ann" ] && [ -f "$work/dotted/$ann" ] \ + && ok "the annotated path exists on disk: $ann" \ + || bad "the annotated path is not a file: '$ann'" + +# ------------------------------------------------------------------- plugin.conf +echo +echo "→ the floors must be required, not defaulted" +cd "$work/repo" +grep -v '^MIN_' ci/plugin.conf > ci/plugin.conf.tmp && mv ci/plugin.conf.tmp ci/plugin.conf +# The `unset` is load-bearing and this assertion failed without it. This script +# sources ci/lib.sh itself, so the floors are already in the environment and a +# plain subshell INHERITS them — ci_load_conf then finds them set and passes +# over a plugin.conf that does not define them. The fixture was contaminated, +# not the check: a failing probe is evidence about the probe until you have +# looked at it. The real static.sh runs in a fresh shell where they are unset. +if ( unset MIN_SHIPPED_PHP_FILES MIN_TRANSLATION_CALLS + . ci/lib.sh && ci_load_conf ) >/dev/null 2>&1; then + bad "a plugin.conf with no floors loaded — an unset floor is a no-op comparison" +else + ok "a plugin.conf with no floors fails to load" +fi + +echo +if [ "$fail" != 0 ]; then + echo "Self-test: $fail of $((pass + fail)) assertions FAILED." + exit 1 +fi +echo "Self-test: all $pass assertions passed."