From d64d0fd1a8de0c42792a412285f0941fbca7c347 Mon Sep 17 00:00:00 2001 From: kristianwind Date: Thu, 17 Sep 2026 00:47:01 +0200 Subject: [PATCH] Stop ci/site-test.sh leaking a MariaDB volume on every run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mariadb:lts` declares VOLUME /var/lib/mysql, so every run creates an anonymous volume — and `docker rm -f` does not remove anonymous volumes. Only `docker rm -f -v` does. What hid it: cleanup() removed the containers, the network and the NAMED $id-wp volume correctly. Three of the four things the run creates. Removing three of four is indistinguishable from removing all four unless you count, which is the same shape as every other bug found in this kit today. Found by the subscriptions session, which traced the Gitea runner's disk being 100% full — 26 GB in 95 volumes with 1 in use — and by woo-cache-manager from the other end, whose push failed with `unable to create temporary object directory`, three layers away from the cause. Measured here rather than taken on report: a mariadb:lts container creates exactly one anonymous volume, it survives `docker rm -f`, and it is removed by `docker rm -f -v`. This machine was holding 133 volumes, 127 dangling, 15.2 GB reclaimable. The fix is the -v. The rest is so it cannot come back silently: cleanup() now asks the containers which volumes they own, via docker inspect, BEFORE removing them, and checks afterwards that none survive. Attribution rather than a global before/after count on purpose — several sessions share this box and run these container pairs concurrently, so a neighbour's volume would read as this run's leak. The check is advisory on a run that already failed, and fatal on one that otherwise passed: site_test_ok is set at the single point the run is known to have succeeded. And it calls exit rather than return, because `return` from an EXIT trap does not change the script's exit status — the warning would have printed while the build went green, which is the failure mode this whole day has been about. Verified: the real site test now runs green with a volume delta of zero. Co-Authored-By: Claude Opus 5 (1M context) --- ci/site-test.sh | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/ci/site-test.sh b/ci/site-test.sh index 0f6e5b2..f42c4fe 100755 --- a/ci/site-test.sh +++ b/ci/site-test.sh @@ -27,10 +27,49 @@ PHP_VERSION="${PHP_VERSION:-8.3}" id="wpci-$$" net="$id-net" +# 🔴 -v, and it is the whole bug. `mariadb:lts` declares VOLUME +# /var/lib/mysql, so every run creates an ANONYMOUS volume, and `docker rm -f` +# does not remove anonymous volumes — only `docker rm -f -v` does. The named +# $id-wp below is removed correctly, which is exactly what made this function +# look complete: removing three of the four things you created is +# indistinguishable from removing all four unless you count. +# +# Measured 2026-09-17 after a sibling plugin traced it: the Gitea runner's disk +# was 100% full, 26 GB in 95 volumes with 1 in use, and a push there failed +# with `unable to create temporary object directory` — three layers from the +# cause. This machine had 127 dangling volumes and 15.2 GB reclaimable. cleanup() { - docker rm -f "$id-db" "$id-cli" >/dev/null 2>&1 || true + # Ask the containers which volumes they own BEFORE removing them. A global + # before/after count would be wrong here: several sessions share this box and + # run these container pairs concurrently, so a neighbour's volume would read + # as this run's leak. + local owned="" + owned=$(docker inspect -f \ + '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' \ + "$id-db" "$id-cli" 2>/dev/null || true) + + docker rm -f -v "$id-db" "$id-cli" >/dev/null 2>&1 || true docker volume rm -f "$id-wp" >/dev/null 2>&1 || true docker network rm "$net" >/dev/null 2>&1 || true + + # Count rather than trust the flag. This is advisory on a failed run — a test + # that died halfway has more urgent news — but on a successful run a leftover + # volume is a real regression and says so. + local leaked="" + for v in $owned; do + docker volume inspect "$v" >/dev/null 2>&1 && leaked="$leaked $v" + done + if [ -n "$leaked" ]; then + echo "::warning::This run left docker volume(s) behind:$leaked — 95 of these filled the Gitea runner's disk." + # `return` from an EXIT trap does NOT change the script's exit status; + # only `exit` does. Without this the warning would print and the build + # would still go green, which is the failure mode of every other check + # this kit has had to fix tonight. + if [ "${site_test_ok:-0}" = 1 ]; then + ci_fail "The site test leaked the volume(s) above. Left alone this fills the runner's disk, and the symptom is a push failing with 'unable to create temporary object directory'." + exit 1 + fi + fi } trap cleanup EXIT @@ -144,5 +183,9 @@ else echo " nothing logged at all" fi +# Set only here, so the leak check above is advisory on a run that failed for +# some other reason and fatal on one that otherwise passed. +site_test_ok=1 + ci_step "done" echo "WordPress $(wp core version)$( [ "$NEEDS_WOOCOMMERCE" = yes ] && echo " + WooCommerce $(wp plugin get woocommerce --field=version)" ) on PHP $PHP_VERSION: green"