Merge pull request #19 from kristianwind/ci-selftest
Add ci/selftest.sh: prove the checks can still fail
This commit is contained in:
Kristian Wind
@@ -30,6 +30,14 @@ jobs:
|
||||
- name: Lint, hygiene, versions and compatibility headers
|
||||
run: ci/static.sh
|
||||
|
||||
# Runs static.sh again against planted fixtures and asserts it goes RED.
|
||||
# Every check in ci/ has been silently hollow at least once — reporting
|
||||
# success having examined nothing, then examining everything with a
|
||||
# pattern that could not match. Neither is visible from a green run, so
|
||||
# the only defence is a thing that must be found.
|
||||
- name: The checks can actually fail
|
||||
run: ci/selftest.sh
|
||||
|
||||
- name: The release zip is buildable and complete
|
||||
run: ci/build-zip.sh
|
||||
|
||||
|
||||
@@ -30,6 +30,14 @@ jobs:
|
||||
- name: Lint, hygiene, versions and compatibility headers
|
||||
run: ci/static.sh
|
||||
|
||||
# Runs static.sh again against planted fixtures and asserts it goes RED.
|
||||
# Every check in ci/ has been silently hollow at least once — reporting
|
||||
# success having examined nothing, then examining everything with a
|
||||
# pattern that could not match. Neither is visible from a green run, so
|
||||
# the only defence is a thing that must be found.
|
||||
- name: The checks can actually fail
|
||||
run: ci/selftest.sh
|
||||
|
||||
- name: The release zip is buildable and complete
|
||||
run: ci/build-zip.sh
|
||||
|
||||
|
||||
Executable
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
# Does ci/static.sh actually FAIL when it should?
|
||||
#
|
||||
# 🔴 Why this exists, and why it asserts what it asserts.
|
||||
#
|
||||
# Every check in this kit has now been wrong twice in the same way: it printed
|
||||
# a reassuring line having examined nothing, and later it examined everything
|
||||
# with a pattern that could not match. Both are invisible from the passing
|
||||
# side, because a check that finds nothing and a check that CANNOT find
|
||||
# anything produce identical output. The only defence is to plant a thing that
|
||||
# must be found and watch the build go red.
|
||||
#
|
||||
# 🔴 And the probe itself has the same failure mode. An earlier version of this
|
||||
# battery asserted "the fixture's filename appears somewhere in the output" —
|
||||
# which passes on grep's own `binary file matches` diagnostic, because that
|
||||
# diagnostic prints the filename in exactly the place a real failure would.
|
||||
# A sibling plugin recorded two binary cases as PASS on that basis and was one
|
||||
# message away from reporting the shape covered.
|
||||
#
|
||||
# So: assert on the thing that has CONSEQUENCES. The exit status of the script
|
||||
# CI runs, and the specific failure message — never a filename, a count, or a
|
||||
# log line that merely sits near the signal.
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Loaded from the REAL tree, before the throwaway copy has its floors stripped
|
||||
# by the last assertion below.
|
||||
. ci/lib.sh
|
||||
ci_load_conf || { echo "ci/plugin.conf will not load — fix that before running the self-test." >&2; exit 1; }
|
||||
|
||||
pass=0; fail=0
|
||||
ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; pass=$((pass + 1)); }
|
||||
bad() { printf ' \033[31m✗\033[0m %s\n' "$1"; fail=$((fail + 1)); }
|
||||
|
||||
# A throwaway copy: these fixtures must never be committable to the real tree,
|
||||
# and several of them are credential-shaped by definition.
|
||||
work=$(mktemp -d -t wpci-selftest-XXXXXX)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
cp -a . "$work/repo"
|
||||
cd "$work/repo"
|
||||
|
||||
# ---------------------------------------------------------------- credentials
|
||||
# One fixture per shape the scan CLAIMS to catch, plus lookalikes that must
|
||||
# stay quiet. An unfilled placeholder is not optional decoration: it is the one
|
||||
# string a settings screen is guaranteed to contain, so a rule without that
|
||||
# exclusion cries wolf on day one and gets commented out.
|
||||
# 🔴 Every fixture value below is ASSEMBLED from pieces, never written as a
|
||||
# literal. This script is itself a tracked file, so ci/static.sh scans it — and
|
||||
# a credential-shaped literal here makes the real tree go red. It did: CI
|
||||
# flagged ci/selftest.sh on this file's first push.
|
||||
#
|
||||
# Assembling is better than any exemption. A filename exemption would switch
|
||||
# the scan off in ci/ and tests/, which is exactly where a key gets pasted to
|
||||
# test it. A placeholder marker in the value would stop the fixture triggering
|
||||
# the check it exists to trigger. Split literals leave the written file fully
|
||||
# credential-shaped while the source contains no matching string at all, so
|
||||
# nothing has to be excused.
|
||||
akia="AKIA""IOSFODNN7EXAMPLE"
|
||||
pem="-----BEGIN RSA PRIVATE"" KEY-----"; pem="${pem/ / }"
|
||||
skant="sk-ant-""api03-AAAAAAAAAAAAAAAAAAAA"
|
||||
ghp="ghp_""AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
gho="gho_""BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
|
||||
pat="github_pat_""$(printf 'C%.0s' $(seq 1 60))"
|
||||
sk="sk-""DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"
|
||||
opaque="h8Kd92MsLp01QwErTyUiOp"
|
||||
|
||||
printf 'aws = %s\n' "$akia" > probe-aws.txt
|
||||
printf -- '%s\nMIIEfake\n' "$pem" > probe-pem.txt
|
||||
printf 'k = %s\n' "$skant" > probe-skant.txt
|
||||
printf 'k = %s\n' "$ghp" > probe-ghp.txt
|
||||
printf 'k = %s\n' "$gho" > probe-gho.txt
|
||||
printf 'k = %s\n' "$pat" > probe-pat.txt
|
||||
printf 'k = %s\n' "$sk" > probe-sk.txt
|
||||
printf "'api_key' => '%s',\n" "$opaque" > probe-array.txt
|
||||
printf "define( 'WC_EDM_SECRET', '%s' );\n" "$opaque" > probe-define.txt
|
||||
|
||||
# Directory coverage. .distignore removes all three from the zip, and a scan
|
||||
# reading the SHIPPED list instead of the tracked one therefore misses every CI
|
||||
# script, workflow and test fixture — which is exactly where a key gets pasted
|
||||
# to test it. A sibling plugin was missing 36 of 73 files this way.
|
||||
mkdir -p ci tests .github/workflows
|
||||
printf 'k = %s\n' "$skant" > ci/probe-dir.txt
|
||||
printf 'k = %s\n' "$skant" > tests/probe-dir.txt
|
||||
printf 'k = %s\n' "$skant" > .github/workflows/probe-dir.txt
|
||||
|
||||
# A key inside a file grep calls binary. `grep -I` skips these SILENTLY, so the
|
||||
# value rule must not use it; the build going red is the right direction to be
|
||||
# wrong in. This fixture exists because that flag was added by accident and
|
||||
# nothing went red to announce it.
|
||||
printf 'PNG\x00\x00\x00fake%s\nMIIE\n' "$pem" > probe-binary.bin
|
||||
|
||||
# Controls need no assembling: each is either a placeholder the scan excludes
|
||||
# by design, or ordinary code that was never credential-shaped. If one of these
|
||||
# ever makes the real tree go red, that is a false positive worth knowing about.
|
||||
printf 'sha = 9f2b7c1e4a8d6b3f0e5c2a9d7b4f1e8c\nAKIA\n' > control-hash.txt
|
||||
printf "'api_key' => 'REPLACE_WITH_YOUR_API_KEY',\n" > control-placeholder.txt
|
||||
printf "'password' => 'xxxxxxxxxxxxxxxxxxxxxxxx',\n" > control-masked.txt
|
||||
printf "'api_key' => '',\n'token' => 'short',\n" > control-empty.txt
|
||||
printf "wp_die( 'token', 'a long sentence that follows a comma' );\n" > control-comma.txt
|
||||
printf "define( 'WC_EDM_TOKEN', 'TODO_FILL_THIS_IN_LATER_XX' );\n" > control-tododefine.txt
|
||||
|
||||
# -N: an UNTRACKED fixture is invisible to a scan reading `git ls-files`, so it
|
||||
# passes clean and looks exactly like the scan working.
|
||||
git add -N probe-*.txt probe-binary.bin ci/probe-dir.txt tests/probe-dir.txt \
|
||||
.github/workflows/probe-dir.txt control-*.txt >/dev/null
|
||||
|
||||
echo "→ ci/static.sh must FAIL with credentials planted"
|
||||
out=$(./ci/static.sh 2>&1); code=$?
|
||||
[ "$code" != 0 ] && ok "exit status is $code" || bad "exit status is 0 — the build would have gone green"
|
||||
|
||||
# The message, not the filename. grep's own "binary file matches" diagnostic
|
||||
# names the file too, in the same stream, which is what made the weak version
|
||||
# of this assertion pass on a check that never fired.
|
||||
fired() { echo "$out" | grep -q "^.*$1 \(looks like it contains a credential\|assigns a long opaque value\)"; }
|
||||
|
||||
for f in probe-aws.txt probe-pem.txt probe-skant.txt probe-ghp.txt probe-gho.txt \
|
||||
probe-pat.txt probe-sk.txt probe-array.txt probe-define.txt \
|
||||
ci/probe-dir.txt tests/probe-dir.txt .github/workflows/probe-dir.txt \
|
||||
probe-binary.bin; do
|
||||
fired "$f" && ok "flagged $f" || bad "NOT FLAGGED: $f — the pattern is inert for this shape"
|
||||
done
|
||||
|
||||
for f in control-hash.txt control-placeholder.txt control-masked.txt \
|
||||
control-empty.txt control-comma.txt control-tododefine.txt; do
|
||||
fired "$f" && bad "FALSE POSITIVE on $f — a check that cries wolf gets commented out" \
|
||||
|| ok "ignored $f"
|
||||
done
|
||||
|
||||
# The guard check must report a plausible number rather than a bare adjective.
|
||||
echo "$out" | grep -qE '^ [0-9]+ shipped PHP files examined' \
|
||||
&& ok "the direct-access check reports a count" \
|
||||
|| bad "the direct-access check printed no count — it cannot tell 13 files from 0"
|
||||
|
||||
# ---------------------------------------------------------------- text domain
|
||||
# No docker, so these are cheap. Three zeros, three verdicts.
|
||||
cd "$work/repo"
|
||||
echo
|
||||
echo "→ ci/check-textdomain.php verdicts"
|
||||
mkdir -p "$work/empty" "$work/nostrings"
|
||||
printf '<?php\n$x = 1;\n' > "$work/nostrings/plugin.php"
|
||||
|
||||
php ci/check-textdomain.php --min=0 "$TEXT_DOMAIN" "$work/empty" >/dev/null 2>&1 \
|
||||
&& bad "an empty scan root passed — the root verdict must not be configurable away" \
|
||||
|| ok "an empty scan root is fatal even with --min=0"
|
||||
|
||||
php ci/check-textdomain.php --min=999999 "$TEXT_DOMAIN" . >/dev/null 2>&1 \
|
||||
&& bad "a --min above the real count passed" \
|
||||
|| ok "a --min above the real count is fatal"
|
||||
|
||||
php ci/check-textdomain.php "$TEXT_DOMAIN" "$work/nostrings" >/dev/null 2>&1 \
|
||||
&& ok "zero calls without --min warns rather than failing" \
|
||||
|| bad "zero calls without --min failed — a small plugin may legitimately have none"
|
||||
|
||||
# The annotated path must be a file that exists. str_replace( $root, '', $path )
|
||||
# with the default root of "." strips every dot and yields "class-dbphp".
|
||||
mkdir -p "$work/dotted/includes"
|
||||
printf "<?php\necho __( 'x' );\n" > "$work/dotted/includes/class-db.php"
|
||||
ann=$(cd "$work/dotted" && php "$work/repo/ci/check-textdomain.php" "$TEXT_DOMAIN" . 2>&1 | sed -n 's/.*file=\([^,]*\),.*/\1/p' | head -1)
|
||||
[ -n "$ann" ] && [ -f "$work/dotted/$ann" ] \
|
||||
&& ok "the annotated path exists on disk: $ann" \
|
||||
|| bad "the annotated path is not a file: '$ann'"
|
||||
|
||||
# ------------------------------------------------------------------- plugin.conf
|
||||
echo
|
||||
echo "→ the floors must be required, not defaulted"
|
||||
cd "$work/repo"
|
||||
grep -v '^MIN_' ci/plugin.conf > ci/plugin.conf.tmp && mv ci/plugin.conf.tmp ci/plugin.conf
|
||||
# The `unset` is load-bearing and this assertion failed without it. This script
|
||||
# sources ci/lib.sh itself, so the floors are already in the environment and a
|
||||
# plain subshell INHERITS them — ci_load_conf then finds them set and passes
|
||||
# over a plugin.conf that does not define them. The fixture was contaminated,
|
||||
# not the check: a failing probe is evidence about the probe until you have
|
||||
# looked at it. The real static.sh runs in a fresh shell where they are unset.
|
||||
if ( unset MIN_SHIPPED_PHP_FILES MIN_TRANSLATION_CALLS
|
||||
. ci/lib.sh && ci_load_conf ) >/dev/null 2>&1; then
|
||||
bad "a plugin.conf with no floors loaded — an unset floor is a no-op comparison"
|
||||
else
|
||||
ok "a plugin.conf with no floors fails to load"
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" != 0 ]; then
|
||||
echo "Self-test: $fail of $((pass + fail)) assertions FAILED."
|
||||
exit 1
|
||||
fi
|
||||
echo "Self-test: all $pass assertions passed."
|
||||
Reference in New Issue
Block a user