nolimit-discounts/ci/selftest.sh

#!/usr/bin/env bash
# Does ci/static.sh actually FAIL when it should?
#
# 🔴 Why this exists, and why it asserts what it asserts.
#
# Every check in this kit has now been wrong twice in the same way: it printed
# a reassuring line having examined nothing, and later it examined everything
# with a pattern that could not match. Both are invisible from the passing
# side, because a check that finds nothing and a check that CANNOT find
# anything produce identical output. The only defence is to plant a thing that
# must be found and watch the build go red.
#
# 🔴 And the probe itself has the same failure mode. An earlier version of this
# battery asserted "the fixture's filename appears somewhere in the output" —
# which passes on grep's own `binary file matches` diagnostic, because that
# diagnostic prints the filename in exactly the place a real failure would.
# A sibling plugin recorded two binary cases as PASS on that basis and was one
# message away from reporting the shape covered.
#
# So: assert on the thing that has CONSEQUENCES. The exit status of the script
# CI runs, and the specific failure message — never a filename, a count, or a
# log line that merely sits near the signal.
set -uo pipefail
cd "$(dirname "$0")/.."

# Loaded from the REAL tree, before the throwaway copy has its floors stripped
# by the last assertion below.
. ci/lib.sh
ci_load_conf || { echo "ci/plugin.conf will not load — fix that before running the self-test." >&2; exit 1; }

pass=0; fail=0
ok()  { printf '  \033[32m✓\033[0m %s\n' "$1"; pass=$((pass + 1)); }
bad() { printf '  \033[31m✗\033[0m %s\n' "$1"; fail=$((fail + 1)); }

# A throwaway copy: these fixtures must never be committable to the real tree,
# and several of them are credential-shaped by definition.
work=$(mktemp -d -t wpci-selftest-XXXXXX)
trap 'rm -rf "$work"' EXIT
cp -a . "$work/repo"
cd "$work/repo"

# ---------------------------------------------------------------- credentials
# One fixture per shape the scan CLAIMS to catch, plus lookalikes that must
# stay quiet. An unfilled placeholder is not optional decoration: it is the one
# string a settings screen is guaranteed to contain, so a rule without that
# exclusion cries wolf on day one and gets commented out.
# 🔴 Every fixture value below is ASSEMBLED from pieces, never written as a
# literal. This script is itself a tracked file, so ci/static.sh scans it — and
# a credential-shaped literal here makes the real tree go red. It did: CI
# flagged ci/selftest.sh on this file's first push.
#
# Assembling is better than any exemption. A filename exemption would switch
# the scan off in ci/ and tests/, which is exactly where a key gets pasted to
# test it. A placeholder marker in the value would stop the fixture triggering
# the check it exists to trigger. Split literals leave the written file fully
# credential-shaped while the source contains no matching string at all, so
# nothing has to be excused.
akia="AKIA""IOSFODNN7EXAMPLE"
pem="-----BEGIN RSA PRIVATE""  KEY-----"; pem="${pem/  / }"
skant="sk-ant-""api03-AAAAAAAAAAAAAAAAAAAA"
ghp="ghp_""AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
gho="gho_""BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
pat="github_pat_""$(printf 'C%.0s' $(seq 1 60))"
sk="sk-""DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"
opaque="h8Kd92MsLp01QwErTyUiOp"

printf 'aws = %s\n'  "$akia"                        > probe-aws.txt
printf -- '%s\nMIIEfake\n' "$pem"                   > probe-pem.txt
printf 'k = %s\n'    "$skant"                       > probe-skant.txt
printf 'k = %s\n'    "$ghp"                         > probe-ghp.txt
printf 'k = %s\n'    "$gho"                         > probe-gho.txt
printf 'k = %s\n'    "$pat"                         > probe-pat.txt
printf 'k = %s\n'    "$sk"                          > probe-sk.txt
printf "'api_key' => '%s',\n"              "$opaque" > probe-array.txt
printf "define( 'WC_EDM_SECRET', '%s' );\n" "$opaque" > probe-define.txt

# Directory coverage. .distignore removes all three from the zip, and a scan
# reading the SHIPPED list instead of the tracked one therefore misses every CI
# script, workflow and test fixture — which is exactly where a key gets pasted
# to test it. A sibling plugin was missing 36 of 73 files this way.
mkdir -p ci tests .github/workflows
printf 'k = %s\n' "$skant" > ci/probe-dir.txt
printf 'k = %s\n' "$skant" > tests/probe-dir.txt
printf 'k = %s\n' "$skant" > .github/workflows/probe-dir.txt

# A key inside a file grep calls binary. `grep -I` skips these SILENTLY, so the
# value rule must not use it; the build going red is the right direction to be
# wrong in. This fixture exists because that flag was added by accident and
# nothing went red to announce it.
printf 'PNG\x00\x00\x00fake%s\nMIIE\n' "$pem"       > probe-binary.bin

# Controls need no assembling: each is either a placeholder the scan excludes
# by design, or ordinary code that was never credential-shaped. If one of these
# ever makes the real tree go red, that is a false positive worth knowing about.
printf 'sha = 9f2b7c1e4a8d6b3f0e5c2a9d7b4f1e8c\nAKIA\n'                 > control-hash.txt
printf "'api_key' => 'REPLACE_WITH_YOUR_API_KEY',\n"                    > control-placeholder.txt
printf "'password' => 'xxxxxxxxxxxxxxxxxxxxxxxx',\n"                    > control-masked.txt
printf "'api_key' => '',\n'token' => 'short',\n"                        > control-empty.txt
printf "wp_die( 'token', 'a long sentence that follows a comma' );\n"   > control-comma.txt
printf "define( 'WC_EDM_TOKEN', 'TODO_FILL_THIS_IN_LATER_XX' );\n"      > control-tododefine.txt

# -N: an UNTRACKED fixture is invisible to a scan reading `git ls-files`, so it
# passes clean and looks exactly like the scan working.
git add -N probe-*.txt probe-binary.bin ci/probe-dir.txt tests/probe-dir.txt \
           .github/workflows/probe-dir.txt control-*.txt >/dev/null

echo "→ ci/static.sh must FAIL with credentials planted"
out=$(./ci/static.sh 2>&1); code=$?
[ "$code" != 0 ] && ok "exit status is $code" || bad "exit status is 0 — the build would have gone green"

# The message, not the filename. grep's own "binary file matches" diagnostic
# names the file too, in the same stream, which is what made the weak version
# of this assertion pass on a check that never fired.
fired() { echo "$out" | grep -q "^.*$1 \(looks like it contains a credential\|assigns a long opaque value\)"; }

for f in probe-aws.txt probe-pem.txt probe-skant.txt probe-ghp.txt probe-gho.txt \
         probe-pat.txt probe-sk.txt probe-array.txt probe-define.txt \
         ci/probe-dir.txt tests/probe-dir.txt .github/workflows/probe-dir.txt \
         probe-binary.bin; do
  fired "$f" && ok "flagged $f" || bad "NOT FLAGGED: $f — the pattern is inert for this shape"
done

for f in control-hash.txt control-placeholder.txt control-masked.txt \
         control-empty.txt control-comma.txt control-tododefine.txt; do
  fired "$f" && bad "FALSE POSITIVE on $f — a check that cries wolf gets commented out" \
             || ok "ignored $f"
done

# The guard check must report a plausible number rather than a bare adjective.
echo "$out" | grep -qE '^  [0-9]+ shipped PHP files examined' \
  && ok "the direct-access check reports a count" \
  || bad "the direct-access check printed no count — it cannot tell 13 files from 0"

# ---------------------------------------------------------------- text domain
# 🔴 ci_php, never a bare `php`. The Gitea runner image has NO php — that is
# why ci_php exists and says so — and this section originally called `php`
# directly. Two of its four assertions then failed there, and the other two
# PASSED FOR THE WRONG REASON: they expect a non-zero exit, and `php: command
# not found` is a non-zero exit. A green tick for a check that never ran is the
# exact thing this whole script exists to prevent, reproduced inside it.
#
# ci_php falls back to a container whose filesystem comes from `git ls-files`,
# so the fixtures live INSIDE this copy of the repo and are staged. A fixture
# outside the repo, or an unstaged one, is simply not there when it matters.
cd "$work/repo"
echo
echo "→ ci/check-textdomain.php verdicts"

# Not a literally empty directory: git cannot track one, so it would not reach
# the container at all and the check would fail for a different reason than the
# one being asserted.
mkdir -p selftest-empty selftest-nostrings selftest-dotted/includes
printf 'no php in here\n'                  > selftest-empty/.keep
printf '<?php\n$x = 1;\n'                  > selftest-nostrings/plugin.php
printf "<?php\necho __( 'x' );\n"          > selftest-dotted/includes/class-db.php
git add -N selftest-empty/.keep selftest-nostrings/plugin.php \
           selftest-dotted/includes/class-db.php >/dev/null

ci_php ci/check-textdomain.php --min=0 "$TEXT_DOMAIN" selftest-empty >/dev/null 2>&1 \
  && bad "an empty scan root passed — the root verdict must not be configurable away" \
  || ok "an empty scan root is fatal even with --min=0"

ci_php ci/check-textdomain.php --min=999999 "$TEXT_DOMAIN" . >/dev/null 2>&1 \
  && bad "a --min above the real count passed" \
  || ok "a --min above the real count is fatal"

ci_php ci/check-textdomain.php "$TEXT_DOMAIN" selftest-nostrings >/dev/null 2>&1 \
  && ok "zero calls without --min warns rather than failing" \
  || bad "zero calls without --min failed — a small plugin may legitimately have none"

# The annotated path must be a file that exists. str_replace( $root, '', $path )
# with a root of "." strips every dot and yields "class-dbphp".
ann=$(ci_php ci/check-textdomain.php "$TEXT_DOMAIN" selftest-dotted 2>&1 \
      | sed -n 's/.*file=\([^,]*\),.*/\1/p' | head -1)
[ -n "$ann" ] && [ -f "selftest-dotted/$ann" ] \
  && ok "the annotated path exists on disk: $ann" \
  || bad "the annotated path is not a file: '$ann'"

# ------------------------------------------------------------------- plugin.conf
echo
echo "→ the floors must be required, not defaulted"
cd "$work/repo"
grep -v '^MIN_' ci/plugin.conf > ci/plugin.conf.tmp && mv ci/plugin.conf.tmp ci/plugin.conf
# The `unset` is load-bearing and this assertion failed without it. This script
# sources ci/lib.sh itself, so the floors are already in the environment and a
# plain subshell INHERITS them — ci_load_conf then finds them set and passes
# over a plugin.conf that does not define them. The fixture was contaminated,
# not the check: a failing probe is evidence about the probe until you have
# looked at it. The real static.sh runs in a fresh shell where they are unset.
if ( unset MIN_SHIPPED_PHP_FILES MIN_TRANSLATION_CALLS
     . ci/lib.sh && ci_load_conf ) >/dev/null 2>&1; then
  bad "a plugin.conf with no floors loaded — an unset floor is a no-op comparison"
else
  ok "a plugin.conf with no floors fails to load"
fi

# ------------------------------------------------------------------ workflows
# Both of these are silent where they happen. An unparseable workflow produces
# NO RUN on Gitea, and a multi-cell matrix that is not gated lands on one
# self-hosted runner as exit code 137 with an empty log. Neither shows up on
# GitHub, which is where anyone would be looking.
echo
echo "→ ci/workflows-in-sync.sh must catch what only breaks on Gitea"
cd "$work/repo"
cp -a .github/workflows "$work/wf-github-backup"
cp -a .gitea/workflows  "$work/wf-gitea-backup"
restore_workflows() {
  rm -rf .github/workflows .gitea/workflows
  cp -a "$work/wf-github-backup" .github/workflows
  cp -a "$work/wf-gitea-backup"  .gitea/workflows
}

# Corrupt BOTH roots identically, so this fails on the PARSE rather than on the
# byte-identity rule that would otherwise fire first.
printf '\n  this: [is not\n' >> .github/workflows/ci.yml
printf '\n  this: [is not\n' >> .gitea/workflows/ci.yml
./ci/workflows-in-sync.sh >/dev/null 2>&1 \
  && bad "an unparseable workflow passed — on Gitea it would simply never run" \
  || ok "an unparseable workflow fails the build"
restore_workflows

# The gate that keeps a five-cell matrix off the single self-hosted runner.
# @ as the delimiter, not |: the gate itself contains `||`, and with | as the
# delimiter sed parses the fixture into nonsense and silently changes nothing —
# which is how this assertion first ran against an unmodified file. It reported
# "could not stage the fixture" rather than passing, which is the whole reason
# the staging is checked before the assertion is made.
sed -i "s@^    if: github.event_name == 'workflow_dispatch'.*@    if: always()@" \
  .github/workflows/compat.yml .gitea/workflows/compat.yml
if grep -q 'if: always()' .gitea/workflows/compat.yml; then
  ./ci/workflows-in-sync.sh >/dev/null 2>&1 \
    && bad "an ungated multi-cell matrix passed — it lands on the Gitea runner as exit 137" \
    || ok "an ungated multi-cell matrix fails the build"
else
  bad "could not stage the ungated-matrix fixture — the gate line did not match"
fi
restore_workflows

# Parsing nothing is never a legitimate result.
ci_python_yaml ci/check-workflows.py >/dev/null 2>&1 \
  && bad "check-workflows.py passed having parsed no file at all" \
  || ok "parsing zero workflow files is fatal"

# ------------------------------------------------- no bare interpreter calls
# The bug that cost a green Gitea build: this script called `php` directly.
# The Gitea runner image has no php and no PyYAML, so a bare call there is
# "command not found" — which is a NON-ZERO exit, and therefore indistinguishable
# from the failure two of these assertions were written to expect. They went
# green having run nothing.
#
# ci_php and ci_python_yaml exist for this and say so in their comments. lib.sh
# is excluded because it is where the real calls live.
#
# Measured, so the rule is not uniform: the runner HAS python3 and has NO php,
# and PyYAML is the only missing module. A bare python3 over the stdlib is
# therefore correct and is not flagged.
echo
echo "→ no ci/ script may call php or python3 directly"
# Narrow on purpose. A broad "the word php appears" fires on every string
# literal that mentions php, and on the calls INSIDE the docker scripts in
# static.sh and site-test.sh, where php is exactly right because the container
# is a php image. A check that cries wolf gets commented out, which is worse
# than not having it. These two shapes are what the real bug looked like: the
# interpreter as the first word of a command, or inside $( ).
bare=$(cd "$work/repo" && for f in ci/*.sh; do
  [ "$f" = ci/lib.sh ] && continue
  sed 's/#.*//' "$f" \
    | grep -nE '^[[:space:]]*php[[:space:]]|\$\((cd [^)]*&&[[:space:]]*)?php[[:space:]]' \
    | sed "s|^|$f:|"
  # python3 itself IS on the Gitea runner; only PyYAML is missing. So a bare
  # python3 is fine for anything stdlib (build-zip.sh's zipfile, the JSON
  # readers in publish-release.sh) and wrong only when yaml is involved.
  sed 's/#.*//' "$f" \
    | grep -nE '^[[:space:]]*python3[[:space:]].*(yaml|check-workflows)' \
    | sed "s|^|$f:|"
done)
if [ -n "$bare" ]; then
  printf '%s\n' "$bare" | sed 's/^/      /'
  bad "a ci/ script calls an interpreter directly instead of ci_php / ci_python_yaml"
else
  ok "every interpreter call goes through the container-capable wrapper"
fi

echo
if [ "$fail" != 0 ]; then
  echo "Self-test: $fail of $((pass + fail)) assertions FAILED."
  exit 1
fi
echo "Self-test: all $pass assertions passed."