#!/usr/bin/env bash # Does ci/static.sh actually FAIL when it should? # # 🔴 Why this exists, and why it asserts what it asserts. # # Every check in this kit has now been wrong twice in the same way: it printed # a reassuring line having examined nothing, and later it examined everything # with a pattern that could not match. Both are invisible from the passing # side, because a check that finds nothing and a check that CANNOT find # anything produce identical output. The only defence is to plant a thing that # must be found and watch the build go red. # # 🔴 And the probe itself has the same failure mode. An earlier version of this # battery asserted "the fixture's filename appears somewhere in the output" — # which passes on grep's own `binary file matches` diagnostic, because that # diagnostic prints the filename in exactly the place a real failure would. # A sibling plugin recorded two binary cases as PASS on that basis and was one # message away from reporting the shape covered. # # So: assert on the thing that has CONSEQUENCES. The exit status of the script # CI runs, and the specific failure message — never a filename, a count, or a # log line that merely sits near the signal. set -uo pipefail cd "$(dirname "$0")/.." # Loaded from the REAL tree, before the throwaway copy has its floors stripped # by the last assertion below. . ci/lib.sh ci_load_conf || { echo "ci/plugin.conf will not load — fix that before running the self-test." >&2; exit 1; } pass=0; fail=0 ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; pass=$((pass + 1)); } bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; fail=$((fail + 1)); } # A throwaway copy: these fixtures must never be committable to the real tree, # and several of them are credential-shaped by definition. work=$(mktemp -d -t wpci-selftest-XXXXXX) trap 'rm -rf "$work"' EXIT cp -a . "$work/repo" cd "$work/repo" # ---------------------------------------------------------------- credentials # One fixture per shape the scan CLAIMS to catch, plus lookalikes that must # stay quiet. An unfilled placeholder is not optional decoration: it is the one # string a settings screen is guaranteed to contain, so a rule without that # exclusion cries wolf on day one and gets commented out. # 🔴 Every fixture value below is ASSEMBLED from pieces, never written as a # literal. This script is itself a tracked file, so ci/static.sh scans it — and # a credential-shaped literal here makes the real tree go red. It did: CI # flagged ci/selftest.sh on this file's first push. # # Assembling is better than any exemption. A filename exemption would switch # the scan off in ci/ and tests/, which is exactly where a key gets pasted to # test it. A placeholder marker in the value would stop the fixture triggering # the check it exists to trigger. Split literals leave the written file fully # credential-shaped while the source contains no matching string at all, so # nothing has to be excused. akia="AKIA""IOSFODNN7EXAMPLE" pem="-----BEGIN RSA PRIVATE"" KEY-----"; pem="${pem/ / }" skant="sk-ant-""api03-AAAAAAAAAAAAAAAAAAAA" ghp="ghp_""AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" gho="gho_""BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" pat="github_pat_""$(printf 'C%.0s' $(seq 1 60))" sk="sk-""DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" opaque="h8Kd92MsLp01QwErTyUiOp" printf 'aws = %s\n' "$akia" > probe-aws.txt printf -- '%s\nMIIEfake\n' "$pem" > probe-pem.txt printf 'k = %s\n' "$skant" > probe-skant.txt printf 'k = %s\n' "$ghp" > probe-ghp.txt printf 'k = %s\n' "$gho" > probe-gho.txt printf 'k = %s\n' "$pat" > probe-pat.txt printf 'k = %s\n' "$sk" > probe-sk.txt printf "'api_key' => '%s',\n" "$opaque" > probe-array.txt printf "define( 'WC_EDM_SECRET', '%s' );\n" "$opaque" > probe-define.txt # Directory coverage. .distignore removes all three from the zip, and a scan # reading the SHIPPED list instead of the tracked one therefore misses every CI # script, workflow and test fixture — which is exactly where a key gets pasted # to test it. A sibling plugin was missing 36 of 73 files this way. mkdir -p ci tests .github/workflows printf 'k = %s\n' "$skant" > ci/probe-dir.txt printf 'k = %s\n' "$skant" > tests/probe-dir.txt printf 'k = %s\n' "$skant" > .github/workflows/probe-dir.txt # A key inside a file grep calls binary. `grep -I` skips these SILENTLY, so the # value rule must not use it; the build going red is the right direction to be # wrong in. This fixture exists because that flag was added by accident and # nothing went red to announce it. printf 'PNG\x00\x00\x00fake%s\nMIIE\n' "$pem" > probe-binary.bin # Controls need no assembling: each is either a placeholder the scan excludes # by design, or ordinary code that was never credential-shaped. If one of these # ever makes the real tree go red, that is a false positive worth knowing about. printf 'sha = 9f2b7c1e4a8d6b3f0e5c2a9d7b4f1e8c\nAKIA\n' > control-hash.txt printf "'api_key' => 'REPLACE_WITH_YOUR_API_KEY',\n" > control-placeholder.txt printf "'password' => 'xxxxxxxxxxxxxxxxxxxxxxxx',\n" > control-masked.txt printf "'api_key' => '',\n'token' => 'short',\n" > control-empty.txt printf "wp_die( 'token', 'a long sentence that follows a comma' );\n" > control-comma.txt printf "define( 'WC_EDM_TOKEN', 'TODO_FILL_THIS_IN_LATER_XX' );\n" > control-tododefine.txt # -N: an UNTRACKED fixture is invisible to a scan reading `git ls-files`, so it # passes clean and looks exactly like the scan working. git add -N probe-*.txt probe-binary.bin ci/probe-dir.txt tests/probe-dir.txt \ .github/workflows/probe-dir.txt control-*.txt >/dev/null echo "→ ci/static.sh must FAIL with credentials planted" out=$(./ci/static.sh 2>&1); code=$? [ "$code" != 0 ] && ok "exit status is $code" || bad "exit status is 0 — the build would have gone green" # The message, not the filename. grep's own "binary file matches" diagnostic # names the file too, in the same stream, which is what made the weak version # of this assertion pass on a check that never fired. fired() { echo "$out" | grep -q "^.*$1 \(looks like it contains a credential\|assigns a long opaque value\)"; } for f in probe-aws.txt probe-pem.txt probe-skant.txt probe-ghp.txt probe-gho.txt \ probe-pat.txt probe-sk.txt probe-array.txt probe-define.txt \ ci/probe-dir.txt tests/probe-dir.txt .github/workflows/probe-dir.txt \ probe-binary.bin; do fired "$f" && ok "flagged $f" || bad "NOT FLAGGED: $f — the pattern is inert for this shape" done for f in control-hash.txt control-placeholder.txt control-masked.txt \ control-empty.txt control-comma.txt control-tododefine.txt; do fired "$f" && bad "FALSE POSITIVE on $f — a check that cries wolf gets commented out" \ || ok "ignored $f" done # The guard check must report a plausible number rather than a bare adjective. echo "$out" | grep -qE '^ [0-9]+ shipped PHP files parsed' \ && ok "the direct-access check reports a count" \ || bad "the direct-access check printed no count — it cannot tell 13 files from 0" # --------------------------------------------------- direct-access guard # 🔴 Grade 2 evidence: a planted instance of exactly the failure the check # describes, not an arbitrary trigger. The grep version of this check passed # this fixture — the words are present, in a comment, and there is no guard. # It reported "every one guarded" over a file that runs for anyone who fetches # it directly. echo echo "→ the direct-access check reads structure, not text" cat > includes/class-selftest-unguarded.php <<'FIXTURE' /dev/null # Captured first, NOT piped into grep. This script runs under `set -o # pipefail`, and static.sh is expected to exit non-zero here — so # `static.sh | grep -q` would report STATIC.SH's status, not grep's, and the # assertion would read "not found" however the check behaved. The mirror image # of the `go test | grep` trap, and it cost this assertion one red run. guard_out=$(./ci/static.sh 2>&1 || true) if printf '%s' "$guard_out" | grep -q 'class-selftest-unguarded.php,line=1::No guard'; then ok "a file mentioning the guard only in a comment is still flagged" else bad "an unguarded file passed because the words appear in a comment" fi rm -f includes/class-selftest-unguarded.php git rm -q --cached includes/class-selftest-unguarded.php >/dev/null 2>&1 || true # ---------------------------------------------------------------- text domain # 🔴 ci_php, never a bare `php`. The Gitea runner image has NO php — that is # why ci_php exists and says so — and this section originally called `php` # directly. Two of its four assertions then failed there, and the other two # PASSED FOR THE WRONG REASON: they expect a non-zero exit, and `php: command # not found` is a non-zero exit. A green tick for a check that never ran is the # exact thing this whole script exists to prevent, reproduced inside it. # # ci_php falls back to a container whose filesystem comes from `git ls-files`, # so the fixtures live INSIDE this copy of the repo and are staged. A fixture # outside the repo, or an unstaged one, is simply not there when it matters. cd "$work/repo" echo echo "→ ci/check-textdomain.php verdicts" # Not a literally empty directory: git cannot track one, so it would not reach # the container at all and the check would fail for a different reason than the # one being asserted. mkdir -p selftest-empty selftest-nostrings selftest-dotted/includes printf 'no php in here\n' > selftest-empty/.keep printf ' selftest-nostrings/plugin.php printf " selftest-dotted/includes/class-db.php git add -N selftest-empty/.keep selftest-nostrings/plugin.php \ selftest-dotted/includes/class-db.php >/dev/null ci_php ci/check-textdomain.php --min=0 "$TEXT_DOMAIN" selftest-empty >/dev/null 2>&1 \ && bad "an empty scan root passed — the root verdict must not be configurable away" \ || ok "an empty scan root is fatal even with --min=0" ci_php ci/check-textdomain.php --min=999999 "$TEXT_DOMAIN" . >/dev/null 2>&1 \ && bad "a --min above the real count passed" \ || ok "a --min above the real count is fatal" ci_php ci/check-textdomain.php "$TEXT_DOMAIN" selftest-nostrings >/dev/null 2>&1 \ && ok "zero calls without --min warns rather than failing" \ || bad "zero calls without --min failed — a small plugin may legitimately have none" # The annotated path must be a file that exists. str_replace( $root, '', $path ) # with a root of "." strips every dot and yields "class-dbphp". ann=$(ci_php ci/check-textdomain.php "$TEXT_DOMAIN" selftest-dotted 2>&1 \ | sed -n 's/.*file=\([^,]*\),.*/\1/p' | head -1) [ -n "$ann" ] && [ -f "selftest-dotted/$ann" ] \ && ok "the annotated path exists on disk: $ann" \ || bad "the annotated path is not a file: '$ann'" # ------------------------------------------------------- the clone's git identity # 🔴 Grade 2 evidence: row 1 is the exact line ci/bump-tested-up-to.sh carried # until 2026-09-25, not an arbitrary `git config`. That line writes .git/config, # so every later commit in the clone — a release, by a person — is authored by # the robot with no trailer. Six such commits in one sibling plugin, two in # another. git warns about an identity it cannot FIND and never about a wrong # one, so nothing in the repository ever says so. # # The controls are the larger half here, and three of them were live faults in # earlier versions of this check: `--[a-z-]+` as the scope group reads `--unset` # as a scope and flags the REPAIR, and without a trailing-value requirement a # `$(git config user.name)` READ is a violation. A build that fails on its own # remedy is how a correct check gets deleted — the failure accuses the check, # and the next person believes it. # # 🔴 The fixtures are assembled through $gc rather than written out, because # THIS file is a ci/*.sh file and the check reads every one of them. Spelled in # full, its own rows match: the first version of this block reported six false # positives and no count, while the same rows measured standalone were 16/16 # correct. A failing probe is evidence about the probe until you have looked at # it — and the labels have to dodge it too, which is why they name the shape # instead of quoting the line. Identical to the credential fixtures three # hundred lines up, one file over. # # The plant goes into ci/build-zip.sh because nothing here SOURCES it: static.sh # and this script source ci/lib.sh only. A plant in a sourced file is not inert # text — it runs, sets the clone's identity for real, and the commit fixing the # fault is authored by the plant. nolimit-seo lost a run to exactly that. So the # last assertion checks the clone rather than the output. echo echo "→ ci/static.sh must catch a written identity and ignore reads and repairs" cd "$work/repo" cp ci/build-zip.sh "$work/build-zip.orig" identity_row() { # $1 = RED|green (want), $2 = label, $3 = line to plant cp "$work/build-zip.orig" ci/build-zip.sh printf '%s\n' "$3" >> ci/build-zip.sh # The plant must BE there before any verdict is believed. A printf that went # somewhere else leaves the control state, and then every row passes while # proving nothing — and the all-green rows are exactly the ones whose zeroes # you want to believe. if [ "$(grep -cF "$3" ci/build-zip.sh)" -lt 1 ]; then bad "could not stage the git-identity fixture: $2" return fi row_out=$(./ci/static.sh 2>&1 || true) # The stable half of the message. The wider phrasing moved from "A ci/ script # sets …" to "A tracked file writes …" when the scan was widened, and these # rows all reported NOT FLAGGED while the check was working perfectly — an # assertion on a log line is coupled to its wording. if printf '%s' "$row_out" | grep -q 'git identity that outlives it'; then got=RED else got=green fi if [ "$got" = "$1" ]; then ok "$2 -> $got" elif [ "$1" = RED ]; then bad "NOT FLAGGED: $2 — the clone keeps the robot's name" else bad "FALSE POSITIVE: $2 — the check goes red on something it must allow" fi } gc="git config" # never spelled out below: see the note above identity_row RED 'WRITE no flag, quoted value' "$gc user.name \"compatibility check\"" identity_row RED 'WRITE --local scope' "$gc --local user.name \"x\"" identity_row RED 'WRITE --global scope' "$gc --global user.email \"x@y\"" identity_row RED 'WRITE --system scope' "$gc --system user.name \"x\"" identity_row RED 'WRITE unquoted value' "$gc user.email noreply@localhost" identity_row green 'REPAIR --local --unset' "$gc --local --unset user.name" identity_row green 'REPAIR --unset, no scope' "$gc --unset user.email" identity_row green 'READ inside a command substitution' "echo \"\$($gc user.name)\"" identity_row green 'FIX git -c ... commit' 'git -c user.name="x" -c user.email=y commit -m z' identity_row green 'other a non-identity key' "$gc core.autocrlf false" identity_row green 'prose a commented-out mention' "# $gc user.name \"x\" is the fault" cp "$work/build-zip.orig" ci/build-zip.sh # A violation OUTSIDE ci/*.sh. This is the row the old scope missed entirely: # a workflow step writes an identity into the runner's clone just as # permanently, and the glob the check started with could not see it. cp "$work/build-zip.orig" ci/build-zip.sh wf=.github/workflows/selftest-identity.yml printf 'name: x\njobs:\n y:\n steps:\n - run: %s user.email "x@y"\n' "$gc" > "$wf" git add -N "$wf" >/dev/null # untracked files are invisible to git ls-files wf_out=$(./ci/static.sh 2>&1 || true) printf '%s' "$wf_out" | grep -q 'writes a git identity that outlives it' \ && ok "flagged a write outside ci/ (a workflow step)" \ || bad "NOT FLAGGED: a workflow step writing an identity — the scan is still scoped to ci/" rm -f "$wf" git rm --cached -q --ignore-unmatch "$wf" >/dev/null 2>&1 # 🔴 A drift check on the FORM, and only that: `if … | xargs -0 grep …; then` # has a false green in it, because xargs exits 123 when any batch found nothing # and `if` reads 123 as no-match. It is invisible while the list fits one exec, # which is why nothing here can measure it — this tree's 48 files are one batch. # So this asserts the shape is gone rather than that the bug is absent, and it # cannot tell you the replacement is correct. scan_tracked greps per file and # reads its verdict from the output, which is what makes batching irrelevant. # ^[^#]* — the first version of this matched the COMMENT in static.sh that # quotes the bad form in order to explain it, which is the stale-docblock trap # in its own right: the paragraph documenting a rule is text the rule applies to. if grep -nE '^[^#]*(if|elif).*\| *xargs.*grep' ci/static.sh; then bad "a check above consumes xargs' exit status — 123 on a partial match reads as clean" else ok "no check in ci/static.sh consumes an xargs exit status" fi # And the passing line must carry a number: "every script is clean" reads the # same over zero files as over eleven. clean_out=$(./ci/static.sh 2>&1 || true) printf '%s' "$clean_out" | grep -qE "^ [0-9]+ tracked files leave the clone's git identity alone" \ && ok "the git-identity check reports how many files it read" \ || bad "the git-identity check printed no count — it cannot tell 48 files from 0" # The fixtures are lines that SET an identity. If any of them executed rather # than being read, this clone now has one — which would also mean every verdict # above was measured against a tree nobody intended. if git config --local --get-regexp '^user\.' >/dev/null 2>&1; then bad "a git-identity fixture RAN — the clone has a local identity and the rows above are void" else ok "no fixture executed: the clone still has no local identity" fi # ------------------------------------------------------------------- plugin.conf echo echo "→ the floors must be required, not defaulted" cd "$work/repo" grep -v '^MIN_' ci/plugin.conf > ci/plugin.conf.tmp && mv ci/plugin.conf.tmp ci/plugin.conf # The `unset` is load-bearing and this assertion failed without it. This script # sources ci/lib.sh itself, so the floors are already in the environment and a # plain subshell INHERITS them — ci_load_conf then finds them set and passes # over a plugin.conf that does not define them. The fixture was contaminated, # not the check: a failing probe is evidence about the probe until you have # looked at it. The real static.sh runs in a fresh shell where they are unset. if ( unset MIN_SHIPPED_PHP_FILES MIN_TRANSLATION_CALLS . ci/lib.sh && ci_load_conf ) >/dev/null 2>&1; then bad "a plugin.conf with no floors loaded — an unset floor is a no-op comparison" else ok "a plugin.conf with no floors fails to load" fi # ------------------------------------------------------------------ workflows # Both of these are silent where they happen. An unparseable workflow produces # NO RUN on Gitea, and a multi-cell matrix that is not gated lands on one # self-hosted runner as exit code 137 with an empty log. Neither shows up on # GitHub, which is where anyone would be looking. echo echo "→ ci/workflows-in-sync.sh must catch what only breaks on Gitea" cd "$work/repo" cp -a .github/workflows "$work/wf-github-backup" cp -a .gitea/workflows "$work/wf-gitea-backup" restore_workflows() { rm -rf .github/workflows .gitea/workflows cp -a "$work/wf-github-backup" .github/workflows cp -a "$work/wf-gitea-backup" .gitea/workflows } # Corrupt BOTH roots identically, so this fails on the PARSE rather than on the # byte-identity rule that would otherwise fire first. printf '\n this: [is not\n' >> .github/workflows/ci.yml printf '\n this: [is not\n' >> .gitea/workflows/ci.yml ./ci/workflows-in-sync.sh >/dev/null 2>&1 \ && bad "an unparseable workflow passed — on Gitea it would simply never run" \ || ok "an unparseable workflow fails the build" restore_workflows # The gate that keeps a five-cell matrix off the single self-hosted runner. # @ as the delimiter, not |: the gate itself contains `||`, and with | as the # delimiter sed parses the fixture into nonsense and silently changes nothing — # which is how this assertion first ran against an unmodified file. It reported # "could not stage the fixture" rather than passing, which is the whole reason # the staging is checked before the assertion is made. sed -i "s@^ if: github.event_name == 'workflow_dispatch'.*@ if: always()@" \ .github/workflows/compat.yml .gitea/workflows/compat.yml if grep -q 'if: always()' .gitea/workflows/compat.yml; then ./ci/workflows-in-sync.sh >/dev/null 2>&1 \ && bad "an ungated multi-cell matrix passed — it lands on the Gitea runner as exit 137" \ || ok "an ungated multi-cell matrix fails the build" else bad "could not stage the ungated-matrix fixture — the gate line did not match" fi restore_workflows # Parsing nothing is never a legitimate result. ci_python_yaml ci/check-workflows.py >/dev/null 2>&1 \ && bad "check-workflows.py passed having parsed no file at all" \ || ok "parsing zero workflow files is fatal" # ------------------------------------------------- no bare interpreter calls # The bug that cost a green Gitea build: this script called `php` directly. # The Gitea runner image has no php and no PyYAML, so a bare call there is # "command not found" — which is a NON-ZERO exit, and therefore indistinguishable # from the failure two of these assertions were written to expect. They went # green having run nothing. # # ci_php and ci_python_yaml exist for this and say so in their comments. lib.sh # is excluded because it is where the real calls live. # # Measured, so the rule is not uniform: the runner HAS python3 and has NO php, # and PyYAML is the only missing module. A bare python3 over the stdlib is # therefore correct and is not flagged. echo echo "→ no ci/ script may call php or python3 directly" # Narrow on purpose. A broad "the word php appears" fires on every string # literal that mentions php, and on the calls INSIDE the docker scripts in # static.sh and site-test.sh, where php is exactly right because the container # is a php image. A check that cries wolf gets commented out, which is worse # than not having it. These two shapes are what the real bug looked like: the # interpreter as the first word of a command, or inside $( ). bare=$(cd "$work/repo" && for f in ci/*.sh; do [ "$f" = ci/lib.sh ] && continue sed 's/#.*//' "$f" \ | grep -nE '^[[:space:]]*php[[:space:]]|\$\((cd [^)]*&&[[:space:]]*)?php[[:space:]]' \ | sed "s|^|$f:|" # python3 itself IS on the Gitea runner; only PyYAML is missing. So a bare # python3 is fine for anything stdlib (build-zip.sh's zipfile, the JSON # readers in publish-release.sh) and wrong only when yaml is involved. sed 's/#.*//' "$f" \ | grep -nE '^[[:space:]]*python3[[:space:]].*(yaml|check-workflows)' \ | sed "s|^|$f:|" done) if [ -n "$bare" ]; then printf '%s\n' "$bare" | sed 's/^/ /' bad "a ci/ script calls an interpreter directly instead of ci_php / ci_python_yaml" else ok "every interpreter call goes through the container-capable wrapper" fi echo if [ "$fail" != 0 ]; then echo "Self-test: $fail of $((pass + fail)) assertions FAILED." exit 1 fi echo "Self-test: all $pass assertions passed."