diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 856cde1..feb5c82 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -30,6 +30,14 @@ jobs: - name: Lint, hygiene, versions and compatibility headers run: ci/static.sh + # Runs static.sh again against planted fixtures and asserts it goes RED. + # Every check in ci/ has been silently hollow at least once — reporting + # success having examined nothing, then examining everything with a + # pattern that could not match. Neither is visible from a green run, so + # the only defence is a thing that must be found. + - name: The checks can actually fail + run: ci/selftest.sh + - name: The release zip is buildable and complete run: ci/build-zip.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 856cde1..feb5c82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,14 @@ jobs: - name: Lint, hygiene, versions and compatibility headers run: ci/static.sh + # Runs static.sh again against planted fixtures and asserts it goes RED. + # Every check in ci/ has been silently hollow at least once — reporting + # success having examined nothing, then examining everything with a + # pattern that could not match. Neither is visible from a green run, so + # the only defence is a thing that must be found. + - name: The checks can actually fail + run: ci/selftest.sh + - name: The release zip is buildable and complete run: ci/build-zip.sh diff --git a/ci/selftest.sh b/ci/selftest.sh new file mode 100755 index 0000000..91ce0ef --- /dev/null +++ b/ci/selftest.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Does ci/static.sh actually FAIL when it should? +# +# 🔴 Why this exists, and why it asserts what it asserts. +# +# Every check in this kit has now been wrong twice in the same way: it printed +# a reassuring line having examined nothing, and later it examined everything +# with a pattern that could not match. Both are invisible from the passing +# side, because a check that finds nothing and a check that CANNOT find +# anything produce identical output. The only defence is to plant a thing that +# must be found and watch the build go red. +# +# 🔴 And the probe itself has the same failure mode. An earlier version of this +# battery asserted "the fixture's filename appears somewhere in the output" — +# which passes on grep's own `binary file matches` diagnostic, because that +# diagnostic prints the filename in exactly the place a real failure would. +# A sibling plugin recorded two binary cases as PASS on that basis and was one +# message away from reporting the shape covered. +# +# So: assert on the thing that has CONSEQUENCES. The exit status of the script +# CI runs, and the specific failure message — never a filename, a count, or a +# log line that merely sits near the signal. +set -uo pipefail +cd "$(dirname "$0")/.." + +# Loaded from the REAL tree, before the throwaway copy has its floors stripped +# by the last assertion below. +. ci/lib.sh +ci_load_conf || { echo "ci/plugin.conf will not load — fix that before running the self-test." >&2; exit 1; } + +pass=0; fail=0 +ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; pass=$((pass + 1)); } +bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; fail=$((fail + 1)); } + +# A throwaway copy: these fixtures must never be committable to the real tree, +# and several of them are credential-shaped by definition. +work=$(mktemp -d -t wpci-selftest-XXXXXX) +trap 'rm -rf "$work"' EXIT +cp -a . "$work/repo" +cd "$work/repo" + +# ---------------------------------------------------------------- credentials +# One fixture per shape the scan CLAIMS to catch, plus lookalikes that must +# stay quiet. An unfilled placeholder is not optional decoration: it is the one +# string a settings screen is guaranteed to contain, so a rule without that +# exclusion cries wolf on day one and gets commented out. +printf 'aws = AKIAIOSFODNN7EXAMPLE\n' > probe-aws.txt +printf -- '-----BEGIN RSA PRIVATE KEY-----\nMIIEfake\n' > probe-pem.txt +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > probe-skant.txt +printf 'k = ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n' > probe-ghp.txt +printf 'k = gho_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\n' > probe-gho.txt +printf 'k = github_pat_%s\n' "$(printf 'C%.0s' $(seq 1 60))" > probe-pat.txt +printf 'k = sk-DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD\n' > probe-sk.txt +printf "'api_key' => 'h8Kd92MsLp01QwErTyUiOp',\n" > probe-array.txt +printf "define( 'WC_EDM_SECRET', 'h8Kd92MsLp01QwErTyUi' );\n" > probe-define.txt + +# Directory coverage. .distignore removes all three from the zip, and a scan +# reading the SHIPPED list instead of the tracked one therefore misses every CI +# script, workflow and test fixture — which is exactly where a key gets pasted +# to test it. A sibling plugin was missing 36 of 73 files this way. +mkdir -p ci tests .github/workflows +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > ci/probe-dir.txt +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > tests/probe-dir.txt +printf 'k = sk-ant-api03-AAAAAAAAAAAAAAAAAAAA\n' > .github/workflows/probe-dir.txt + +# A key inside a file grep calls binary. `grep -I` skips these SILENTLY, so the +# value rule must not use it; the build going red is the right direction to be +# wrong in. This fixture exists because that flag was added by accident and +# nothing went red to announce it. +printf 'PNG\x00\x00\x00fake-----BEGIN RSA PRIVATE KEY-----\nMIIE\n' > probe-binary.bin + +printf 'sha = 9f2b7c1e4a8d6b3f0e5c2a9d7b4f1e8c\nAKIA\n' > control-hash.txt +printf "'api_key' => 'REPLACE_WITH_YOUR_API_KEY',\n" > control-placeholder.txt +printf "'password' => 'xxxxxxxxxxxxxxxxxxxxxxxx',\n" > control-masked.txt +printf "'api_key' => '',\n'token' => 'short',\n" > control-empty.txt +printf "wp_die( 'token', 'a long sentence that follows a comma' );\n" > control-comma.txt +printf "define( 'WC_EDM_TOKEN', 'TODO_FILL_THIS_IN_LATER_XX' );\n" > control-tododefine.txt + +# -N: an UNTRACKED fixture is invisible to a scan reading `git ls-files`, so it +# passes clean and looks exactly like the scan working. +git add -N probe-*.txt probe-binary.bin ci/probe-dir.txt tests/probe-dir.txt \ + .github/workflows/probe-dir.txt control-*.txt >/dev/null + +echo "→ ci/static.sh must FAIL with credentials planted" +out=$(./ci/static.sh 2>&1); code=$? +[ "$code" != 0 ] && ok "exit status is $code" || bad "exit status is 0 — the build would have gone green" + +# The message, not the filename. grep's own "binary file matches" diagnostic +# names the file too, in the same stream, which is what made the weak version +# of this assertion pass on a check that never fired. +fired() { echo "$out" | grep -q "^.*$1 \(looks like it contains a credential\|assigns a long opaque value\)"; } + +for f in probe-aws.txt probe-pem.txt probe-skant.txt probe-ghp.txt probe-gho.txt \ + probe-pat.txt probe-sk.txt probe-array.txt probe-define.txt \ + ci/probe-dir.txt tests/probe-dir.txt .github/workflows/probe-dir.txt \ + probe-binary.bin; do + fired "$f" && ok "flagged $f" || bad "NOT FLAGGED: $f — the pattern is inert for this shape" +done + +for f in control-hash.txt control-placeholder.txt control-masked.txt \ + control-empty.txt control-comma.txt control-tododefine.txt; do + fired "$f" && bad "FALSE POSITIVE on $f — a check that cries wolf gets commented out" \ + || ok "ignored $f" +done + +# The guard check must report a plausible number rather than a bare adjective. +echo "$out" | grep -qE '^ [0-9]+ shipped PHP files examined' \ + && ok "the direct-access check reports a count" \ + || bad "the direct-access check printed no count — it cannot tell 13 files from 0" + +# ---------------------------------------------------------------- text domain +# No docker, so these are cheap. Three zeros, three verdicts. +cd "$work/repo" +echo +echo "→ ci/check-textdomain.php verdicts" +mkdir -p "$work/empty" "$work/nostrings" +printf ' "$work/nostrings/plugin.php" + +php ci/check-textdomain.php --min=0 "$TEXT_DOMAIN" "$work/empty" >/dev/null 2>&1 \ + && bad "an empty scan root passed — the root verdict must not be configurable away" \ + || ok "an empty scan root is fatal even with --min=0" + +php ci/check-textdomain.php --min=999999 "$TEXT_DOMAIN" . >/dev/null 2>&1 \ + && bad "a --min above the real count passed" \ + || ok "a --min above the real count is fatal" + +php ci/check-textdomain.php "$TEXT_DOMAIN" "$work/nostrings" >/dev/null 2>&1 \ + && ok "zero calls without --min warns rather than failing" \ + || bad "zero calls without --min failed — a small plugin may legitimately have none" + +# The annotated path must be a file that exists. str_replace( $root, '', $path ) +# with the default root of "." strips every dot and yields "class-dbphp". +mkdir -p "$work/dotted/includes" +printf " "$work/dotted/includes/class-db.php" +ann=$(cd "$work/dotted" && php "$work/repo/ci/check-textdomain.php" "$TEXT_DOMAIN" . 2>&1 | sed -n 's/.*file=\([^,]*\),.*/\1/p' | head -1) +[ -n "$ann" ] && [ -f "$work/dotted/$ann" ] \ + && ok "the annotated path exists on disk: $ann" \ + || bad "the annotated path is not a file: '$ann'" + +# ------------------------------------------------------------------- plugin.conf +echo +echo "→ the floors must be required, not defaulted" +cd "$work/repo" +grep -v '^MIN_' ci/plugin.conf > ci/plugin.conf.tmp && mv ci/plugin.conf.tmp ci/plugin.conf +# The `unset` is load-bearing and this assertion failed without it. This script +# sources ci/lib.sh itself, so the floors are already in the environment and a +# plain subshell INHERITS them — ci_load_conf then finds them set and passes +# over a plugin.conf that does not define them. The fixture was contaminated, +# not the check: a failing probe is evidence about the probe until you have +# looked at it. The real static.sh runs in a fresh shell where they are unset. +if ( unset MIN_SHIPPED_PHP_FILES MIN_TRANSLATION_CALLS + . ci/lib.sh && ci_load_conf ) >/dev/null 2>&1; then + bad "a plugin.conf with no floors loaded — an unset floor is a no-op comparison" +else + ok "a plugin.conf with no floors fails to load" +fi + +echo +if [ "$fail" != 0 ]; then + echo "Self-test: $fail of $((pass + fail)) assertions FAILED." + exit 1 +fi +echo "Self-test: all $pass assertions passed."