mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
feat(docker-nuke): add three-stage kill escalation
This commit is contained in:
David Ibia
+165
-3
@@ -15,10 +15,14 @@ set -euo pipefail
|
||||
# directives should remain.
|
||||
|
||||
HEALTH_TIMEOUT=6 # seconds to wait on `docker info` before calling it wedged
|
||||
# shellcheck disable=SC2034
|
||||
GRACEFUL_WAIT=8 # seconds to let Docker quit politely
|
||||
# shellcheck disable=SC2034
|
||||
TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL
|
||||
# Bounded number of "fresh docker_pids -> SIGKILL -> settle" rounds run if a
|
||||
# post-SIGKILL scan still finds live Docker processes. A supervisor such as
|
||||
# com.docker.backend can respawn a killed helper under a brand new pid the
|
||||
# entry snapshot never knew about, so this must be a real cap, never an
|
||||
# unbounded loop, or a respawning Docker could hang this tool forever.
|
||||
KILL_RETRY_ROUNDS=3
|
||||
# shellcheck disable=SC2034
|
||||
RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back
|
||||
# shellcheck disable=SC2034
|
||||
@@ -31,6 +35,12 @@ LOG_CAPTURE_TIMEOUT=20 # hard cap on the whole capture_logs step
|
||||
CRASH_CAPTURE_TIMEOUT=10 # hard cap on the whole capture_crashes step
|
||||
|
||||
DOCKER_APP="/Applications/Docker.app"
|
||||
# The bundle path above cannot substitute for the AppleScript app name below:
|
||||
# the running GUI application is the nested "Contents/MacOS/Docker
|
||||
# Desktop.app", whose CFBundleName is "Docker Desktop", not the top-level
|
||||
# bundle path. Both are genuinely needed; this constant exists only so the
|
||||
# two literal identities for the same app cannot drift apart silently.
|
||||
DOCKER_APP_NAME="Docker Desktop"
|
||||
DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log"
|
||||
DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw"
|
||||
CRASH_DIR="$HOME/Library/Logs/DiagnosticReports"
|
||||
@@ -308,6 +318,158 @@ prune_dumps() {
|
||||
return 0
|
||||
}
|
||||
|
||||
pids_alive() {
|
||||
local pid alive="" stat
|
||||
for pid in "$@"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
# kill -0 succeeds against a zombie (state Z, <defunct>): the process
|
||||
# table entry is still there but the process has already exited and is
|
||||
# only waiting for its parent to reap it. Treat that as dead so a slow
|
||||
# reaper can never manufacture a spurious "survived SIGKILL" report
|
||||
# (Fix 7). ps -o stat= can print extra flag letters after the primary
|
||||
# state letter (e.g. Z+), so match a leading Z rather than equality.
|
||||
stat="$(ps -o stat= -p "$pid" 2>/dev/null)" || true
|
||||
case "$stat" in
|
||||
Z*) : ;;
|
||||
*) alive="$alive $pid" ;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
printf '%s' "${alive# }"
|
||||
}
|
||||
|
||||
kill_pids() {
|
||||
local sig="$1"
|
||||
shift
|
||||
local pid
|
||||
for pid in "$@"; do
|
||||
kill "-$sig" "$pid" 2>/dev/null || true
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# Returns 0 once every PID is gone, 1 if SECONDS elapses first.
|
||||
wait_for_exit() {
|
||||
local secs="$1"
|
||||
shift
|
||||
local waited=0
|
||||
while [ "$waited" -lt "$secs" ]; do
|
||||
if [ -z "$(pids_alive "$@")" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
if [ -z "$(pids_alive "$@")" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# Stop root launchd helpers. Only called with --deep; needs a cached sudo
|
||||
# credential.
|
||||
stop_privileged_helpers() {
|
||||
log "docker-nuke: stopping privileged helpers (sudo required)"
|
||||
# -n (non-interactive) never prompts: sudo writes "Password:" to stderr but
|
||||
# reads the answer from /dev/tty, so with stderr discarded and no cached
|
||||
# credential a plain `sudo` blocks forever with no visible prompt -- and by
|
||||
# the time this runs, Docker has already been SIGKILLed (Fix 3). -n instead
|
||||
# fails fast into the warning below.
|
||||
if ! sudo -n launchctl stop com.docker.vmnetd 2>/dev/null; then
|
||||
warn "docker-nuke: sudo -n failed for com.docker.vmnetd (no cached credential, or not permitted); skipping"
|
||||
fi
|
||||
if ! sudo -n launchctl stop com.docker.socket 2>/dev/null; then
|
||||
warn "docker-nuke: sudo -n failed for com.docker.socket (no cached credential, or not permitted); skipping"
|
||||
fi
|
||||
}
|
||||
|
||||
# Three-stage escalation: polite quit, SIGTERM, SIGKILL.
|
||||
#
|
||||
# Success at every stage is decided from a FRESH docker_pids scan, never from
|
||||
# the entry snapshot below or from wait_for_exit's own return value (Fix 1).
|
||||
# wait_for_exit still runs against the snapshot at each stage for TIMING --
|
||||
# it correctly answers "did the ones we signalled exit" -- but it cannot see
|
||||
# a Docker process that appeared after the snapshot was taken, so it can
|
||||
# never be trusted alone to answer "is Docker gone". Concretely:
|
||||
# com.docker.backend supervises helpers; SIGKILL order is numeric (sort -un
|
||||
# in docker_pids), unrelated to the supervision tree. A helper can be killed,
|
||||
# the still-alive backend respawns it under a NEW pid, then the backend
|
||||
# itself is killed. Every snapshot pid is now gone, but the respawned helper
|
||||
# is still running -- only a fresh docker_pids call can see it.
|
||||
nuke_docker() {
|
||||
local pids fresh
|
||||
pids="$(docker_pids | tr '\n' ' ')"
|
||||
if [ -z "$pids" ]; then
|
||||
log "docker-nuke: no Docker processes found"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "docker-nuke: asking Docker Desktop to quit"
|
||||
osascript -e 'quit app "'"$DOCKER_APP_NAME"'"' >/dev/null 2>&1 || true
|
||||
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
||||
wait_for_exit "$GRACEFUL_WAIT" $pids || true
|
||||
fresh="$(docker_pids | tr '\n' ' ')"
|
||||
if [ -z "$fresh" ]; then
|
||||
log "docker-nuke: Docker exited cleanly"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Narrow to the subset still alive from the entry snapshot before
|
||||
# signalling (Fix 8): a pid that already exited during the graceful-quit
|
||||
# wait must never be re-signalled, at this stage or the next.
|
||||
local snapshot_alive
|
||||
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
||||
snapshot_alive="$(pids_alive $pids)"
|
||||
|
||||
log "docker-nuke: sending SIGTERM"
|
||||
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
||||
kill_pids TERM $snapshot_alive
|
||||
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
||||
wait_for_exit "$TERM_WAIT" $snapshot_alive || true
|
||||
fresh="$(docker_pids | tr '\n' ' ')"
|
||||
if [ -z "$fresh" ]; then
|
||||
log "docker-nuke: Docker exited after SIGTERM"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "docker-nuke: sending SIGKILL"
|
||||
local remaining
|
||||
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
||||
remaining="$(pids_alive $snapshot_alive)"
|
||||
# shellcheck disable=SC2086 # $remaining must word-split into separate PID arguments here
|
||||
kill_pids KILL $remaining
|
||||
sleep 1
|
||||
|
||||
if [ "${OPT_DEEP:-0}" -eq 1 ]; then
|
||||
stop_privileged_helpers
|
||||
fi
|
||||
|
||||
# Bounded respawn retry (Fix 1): if a fresh scan still finds live Docker
|
||||
# processes, SIGKILL exactly that fresh set, let it settle, and scan again.
|
||||
# Capped at KILL_RETRY_ROUNDS -- never an unbounded loop -- so a Docker
|
||||
# that keeps respawning helpers cannot hang this tool forever.
|
||||
local round=0
|
||||
fresh="$(docker_pids | tr '\n' ' ')"
|
||||
while [ -n "$fresh" ]; do
|
||||
round=$((round + 1))
|
||||
if [ "$round" -gt "$KILL_RETRY_ROUNDS" ]; then
|
||||
break
|
||||
fi
|
||||
warn "docker-nuke: Docker processes still present after SIGKILL, retrying (round $round of $KILL_RETRY_ROUNDS): $fresh"
|
||||
# shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here
|
||||
kill_pids KILL $fresh
|
||||
sleep 1
|
||||
fresh="$(docker_pids | tr '\n' ' ')"
|
||||
done
|
||||
|
||||
if [ -n "$fresh" ]; then
|
||||
warn "docker-nuke: processes survived SIGKILL: $fresh"
|
||||
return 1
|
||||
fi
|
||||
log "docker-nuke: all Docker processes terminated"
|
||||
return 0
|
||||
}
|
||||
|
||||
log() { printf '%s\n' "$*"; }
|
||||
warn() { printf '%s\n' "$*" >&2; }
|
||||
|
||||
@@ -341,7 +503,7 @@ parse_args() {
|
||||
OPT_DRY_RUN=0
|
||||
OPT_DEEP=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
# shellcheck disable=SC2034 # OPT_* are read by main once it is wired up
|
||||
# shellcheck disable=SC2034 # OPT_FORCE, OPT_NO_RESTART, OPT_DRY_RUN are read by main once it is wired up. OPT_DEEP is already read by nuke_docker but stays under this same directive since it covers the whole case statement.
|
||||
case "$1" in
|
||||
-f|--force) OPT_FORCE=1 ;;
|
||||
-n|--no-restart) OPT_NO_RESTART=1 ;;
|
||||
|
||||
@@ -403,6 +403,388 @@ test_dump() {
|
||||
}
|
||||
test_dump
|
||||
|
||||
printf '\n=== kill escalation ===\n'
|
||||
|
||||
test_kill() {
|
||||
local a b c d rc alive
|
||||
|
||||
sleep 60 & a=$!
|
||||
sleep 60 & b=$!
|
||||
sleep 1
|
||||
|
||||
alive="$(pids_alive "$a" "$b")"
|
||||
assert_contains "pids_alive reports first live pid" " $alive " " $a "
|
||||
assert_contains "pids_alive reports second live pid" " $alive " " $b "
|
||||
|
||||
kill_pids TERM "$a"
|
||||
sleep 1
|
||||
alive="$(pids_alive "$a" "$b")"
|
||||
assert_not_contains "SIGTERM kills the target" " $alive " " $a "
|
||||
assert_contains "SIGTERM leaves others alone" " $alive " " $b "
|
||||
|
||||
rc=0; wait_for_exit 3 "$a" || rc=$?
|
||||
assert_eq "wait_for_exit returns 0 for a dead pid" "0" "$rc"
|
||||
|
||||
rc=0; wait_for_exit 2 "$b" || rc=$?
|
||||
assert_eq "wait_for_exit returns 1 while pid lives" "1" "$rc"
|
||||
|
||||
# A third pid, still alive at this point, pairs with the emptiness check
|
||||
# below (Fix 5): a pids_alive that always prints nothing would make the
|
||||
# emptiness assertion pass for the wrong reason, but this positive
|
||||
# assertion over the same call would then fail, since c is genuinely
|
||||
# alive right now.
|
||||
sleep 60 & c=$!
|
||||
|
||||
kill_pids KILL "$b"
|
||||
sleep 1
|
||||
assert_eq "SIGKILL clears the last pid" "" "$(pids_alive "$a" "$b")"
|
||||
assert_contains "pids_alive still detects a live pid alongside dead ones" \
|
||||
" $(pids_alive "$a" "$b" "$c") " " $c "
|
||||
|
||||
# Fix 4: invoking kill_pids inside a `||` list (as this assertion previously
|
||||
# did) suppresses set -e throughout the whole function body, so it can
|
||||
# never prove the `|| true` guard on the real kill call inside kill_pids
|
||||
# does anything at all. A bare invocation under genuinely active set -e is
|
||||
# required. Pairing it with a real target in the same call (Fix 5) proves
|
||||
# kill_pids is not simply a no-op that trivially "succeeds": if it were, d
|
||||
# would still be alive afterward.
|
||||
sleep 60 & d=$!
|
||||
rc=0
|
||||
( set -e; kill_pids TERM 999999 "$d" ); rc=$?
|
||||
assert_eq "killing a nonexistent pid alongside a real one is not an error" "0" "$rc"
|
||||
sleep 1
|
||||
assert_not_contains "kill_pids still kills the real target in the same call" \
|
||||
" $(pids_alive "$d") " " $d "
|
||||
|
||||
wait "$a" 2>/dev/null || true
|
||||
wait "$b" 2>/dev/null || true
|
||||
kill -9 "$c" 2>/dev/null || true
|
||||
wait "$c" 2>/dev/null || true
|
||||
kill -9 "$d" 2>/dev/null || true
|
||||
wait "$d" 2>/dev/null || true
|
||||
}
|
||||
test_kill
|
||||
|
||||
printf '\n=== pids_alive zombie handling (Fix 7) ===\n'
|
||||
|
||||
test_pids_alive_zombie() {
|
||||
# A real zombie is too transient to depend on in a portable test harness
|
||||
# (it is reaped almost immediately once bash notices SIGCHLD), so this
|
||||
# exercises the same branch deterministically by shadowing `kill` and `ps`
|
||||
# inside a subshell rather than relying on a real defunct process. `kill`
|
||||
# is an ordinary builtin here (not a special one), so a same-named shell
|
||||
# function shadows it for calls made from inside this subshell only.
|
||||
local alive
|
||||
|
||||
alive="$(
|
||||
# shellcheck disable=SC2329 # invoked indirectly by pids_alive name lookup
|
||||
kill() {
|
||||
if [ "$1" = "-0" ]; then
|
||||
return 0
|
||||
fi
|
||||
command kill "$@"
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by pids_alive name lookup
|
||||
ps() { printf 'Z+\n'; }
|
||||
pids_alive 424242
|
||||
)"
|
||||
assert_eq "pids_alive treats a zombie as dead" "" "$alive"
|
||||
|
||||
# Pairs with the assertion above (Fix 5): a pids_alive that always prints
|
||||
# nothing would make the zombie assertion pass for the wrong reason, but
|
||||
# this positive assertion over the same shadowed setup, differing only in
|
||||
# the reported process state, would then fail.
|
||||
alive="$(
|
||||
# shellcheck disable=SC2329 # invoked indirectly by pids_alive name lookup
|
||||
kill() {
|
||||
if [ "$1" = "-0" ]; then
|
||||
return 0
|
||||
fi
|
||||
command kill "$@"
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by pids_alive name lookup
|
||||
ps() { printf 'R\n'; }
|
||||
pids_alive 424242
|
||||
)"
|
||||
assert_contains "pids_alive still reports a genuinely running process" " $alive " " 424242 "
|
||||
}
|
||||
test_pids_alive_zombie
|
||||
|
||||
printf '\n=== OPT_DEEP unbound safety (Fix 2) ===\n'
|
||||
|
||||
test_opt_deep_unbound() {
|
||||
# [ "$OPT_DEEP" -eq 1 ] aborts under set -u the moment OPT_DEEP is unset --
|
||||
# which nuke_docker reaches only AFTER SIGKILL has already been sent. Under
|
||||
# the old code this scenario would kill the throwaway process below, then
|
||||
# blow up on the OPT_DEEP read before ever logging completion. Prove the
|
||||
# fixed read (${OPT_DEEP:-0}) lets nuke_docker run to its normal finish
|
||||
# with OPT_DEEP genuinely unset, not merely zero.
|
||||
local out rc
|
||||
|
||||
out="$(
|
||||
set -u
|
||||
unset OPT_DEEP
|
||||
GRACEFUL_WAIT=1
|
||||
TERM_WAIT=1
|
||||
p1=""
|
||||
bash -c 'trap "" TERM; sleep 60' & p1=$!
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() {
|
||||
if kill -0 "$p1" 2>/dev/null; then printf '%s\n' "$p1"; fi
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() { return 0; }
|
||||
nuke_docker
|
||||
rc_inner=$?
|
||||
kill -9 "$p1" 2>/dev/null || true
|
||||
wait "$p1" 2>/dev/null || true
|
||||
exit "$rc_inner"
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "nuke_docker survives OPT_DEEP unset under set -u" "0" "$rc"
|
||||
assert_contains "nuke_docker still logs completion with OPT_DEEP unset" "$out" "all Docker processes terminated"
|
||||
}
|
||||
test_opt_deep_unbound
|
||||
|
||||
printf '\n=== sudo non-interactive guard (Fix 3) ===\n'
|
||||
|
||||
test_sudo_noninteractive() {
|
||||
# Simulating an actual hung, non-cached sudo prompt is not something this
|
||||
# harness can safely or deterministically drive. This is a static
|
||||
# regression guard instead: stop_privileged_helpers must call `sudo -n`
|
||||
# (never a plain `sudo`), so a missing credential fails fast into the
|
||||
# existing `|| true`-style guard rather than blocking forever on a hidden
|
||||
# /dev/tty prompt after Docker has already been SIGKILLed.
|
||||
local src
|
||||
src="$(cat "$SCRIPT_DIR/docker-nuke")"
|
||||
assert_contains "stop_privileged_helpers uses sudo -n for vmnetd" "$src" "sudo -n launchctl stop com.docker.vmnetd"
|
||||
assert_contains "stop_privileged_helpers uses sudo -n for socket" "$src" "sudo -n launchctl stop com.docker.socket"
|
||||
assert_not_contains "stop_privileged_helpers never calls a plain sudo" "$src" $'\nsudo launchctl'
|
||||
}
|
||||
test_sudo_noninteractive
|
||||
|
||||
printf '\n=== nuke_docker escalation orchestration (Fix 6) ===\n'
|
||||
|
||||
test_nuke_docker() {
|
||||
local out rc
|
||||
|
||||
# 1. No processes found -> returns 0, logs accordingly. Nothing spawned,
|
||||
# nothing to reap.
|
||||
out="$(
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() { :; }
|
||||
nuke_docker
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "no processes: returns 0" "0" "$rc"
|
||||
assert_contains "no processes: logs the no-processes message" "$out" "no Docker processes found"
|
||||
|
||||
# 2. All processes exit at the polite-quit stage -> returns 0. The
|
||||
# shadowed osascript stands in for Docker actually honoring the quit
|
||||
# request; docker_pids re-checks real liveness on every call, just like
|
||||
# the real pgrep-backed implementation.
|
||||
out="$(
|
||||
GRACEFUL_WAIT=2
|
||||
p1=""; p2=""
|
||||
sleep 60 & p1=$!
|
||||
sleep 60 & p2=$!
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() {
|
||||
local pid
|
||||
for pid in "$p1" "$p2"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then printf '%s\n' "$pid"; fi
|
||||
done
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() {
|
||||
kill -TERM "$p1" "$p2" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
nuke_docker
|
||||
rc_inner=$?
|
||||
wait "$p1" 2>/dev/null || true
|
||||
wait "$p2" 2>/dev/null || true
|
||||
exit "$rc_inner"
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "polite quit: returns 0" "0" "$rc"
|
||||
assert_contains "polite quit: logs clean exit" "$out" "Docker exited cleanly"
|
||||
|
||||
# 3. Survivors reach SIGTERM -> returns 0. osascript no-ops (Docker ignores
|
||||
# the polite quit); the throwaway sleeps have no TERM trap, so nuke_docker's
|
||||
# own SIGTERM kills them for real.
|
||||
out="$(
|
||||
GRACEFUL_WAIT=1
|
||||
TERM_WAIT=2
|
||||
p1=""; p2=""
|
||||
sleep 60 & p1=$!
|
||||
sleep 60 & p2=$!
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() {
|
||||
local pid
|
||||
for pid in "$p1" "$p2"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then printf '%s\n' "$pid"; fi
|
||||
done
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() { return 0; }
|
||||
nuke_docker
|
||||
rc_inner=$?
|
||||
wait "$p1" 2>/dev/null || true
|
||||
wait "$p2" 2>/dev/null || true
|
||||
exit "$rc_inner"
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "SIGTERM stage: returns 0" "0" "$rc"
|
||||
assert_contains "SIGTERM stage: logs post-SIGTERM exit" "$out" "Docker exited after SIGTERM"
|
||||
|
||||
# 4. Survivors reach SIGKILL -> returns 0, and stop_privileged_helpers is
|
||||
# NOT called when OPT_DEEP=0 (Fix 6's last bullet). Each throwaway process
|
||||
# traps and ignores SIGTERM so only SIGKILL can end it.
|
||||
out="$(
|
||||
OPT_DEEP=0
|
||||
GRACEFUL_WAIT=1
|
||||
TERM_WAIT=1
|
||||
p1=""; p2=""
|
||||
bash -c 'trap "" TERM; sleep 60' & p1=$!
|
||||
bash -c 'trap "" TERM; sleep 60' & p2=$!
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() {
|
||||
local pid
|
||||
for pid in "$p1" "$p2"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then printf '%s\n' "$pid"; fi
|
||||
done
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() { return 0; }
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
stop_privileged_helpers() { printf 'STOP_PRIVILEGED_CALLED\n'; }
|
||||
nuke_docker
|
||||
rc_inner=$?
|
||||
kill -9 "$p1" "$p2" 2>/dev/null || true
|
||||
wait "$p1" 2>/dev/null || true
|
||||
wait "$p2" 2>/dev/null || true
|
||||
exit "$rc_inner"
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "SIGKILL stage: returns 0" "0" "$rc"
|
||||
assert_contains "SIGKILL stage: logs full termination" "$out" "all Docker processes terminated"
|
||||
assert_not_contains "SIGKILL stage: stop_privileged_helpers not called when OPT_DEEP=0" "$out" "STOP_PRIVILEGED_CALLED"
|
||||
|
||||
# 4b. Same shape, but OPT_DEEP=1 -- pairs with 4's negative assertion (Fix
|
||||
# 5): a stop_privileged_helpers call that is unconditionally skipped (or
|
||||
# unconditionally run) would fail one of these two, never both.
|
||||
out="$(
|
||||
OPT_DEEP=1
|
||||
GRACEFUL_WAIT=1
|
||||
TERM_WAIT=1
|
||||
p1=""; p2=""
|
||||
bash -c 'trap "" TERM; sleep 60' & p1=$!
|
||||
bash -c 'trap "" TERM; sleep 60' & p2=$!
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() {
|
||||
local pid
|
||||
for pid in "$p1" "$p2"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then printf '%s\n' "$pid"; fi
|
||||
done
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() { return 0; }
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
stop_privileged_helpers() { printf 'STOP_PRIVILEGED_CALLED\n'; }
|
||||
nuke_docker
|
||||
rc_inner=$?
|
||||
kill -9 "$p1" "$p2" 2>/dev/null || true
|
||||
wait "$p1" 2>/dev/null || true
|
||||
wait "$p2" 2>/dev/null || true
|
||||
exit "$rc_inner"
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "SIGKILL stage with --deep: returns 0" "0" "$rc"
|
||||
assert_contains "SIGKILL stage with --deep: stop_privileged_helpers is called" "$out" "STOP_PRIVILEGED_CALLED"
|
||||
|
||||
# 5. Something survives everything -> returns 1, and does NOT log success.
|
||||
# docker_pids is stubbed to unconditionally keep reporting a fixed,
|
||||
# nonexistent pid as live, modeling a process nothing here can actually
|
||||
# kill, so the bounded retry loop is guaranteed to exhaust its rounds.
|
||||
out="$(
|
||||
OPT_DEEP=0
|
||||
GRACEFUL_WAIT=1
|
||||
TERM_WAIT=1
|
||||
KILL_RETRY_ROUNDS=2
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() { printf '%s\n' 555555; }
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() { return 0; }
|
||||
nuke_docker 2>&1
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "unkillable survivor: returns 1" "1" "$rc"
|
||||
assert_not_contains "unkillable survivor: never logs Docker exited cleanly" "$out" "Docker exited cleanly"
|
||||
assert_not_contains "unkillable survivor: never logs post-SIGTERM exit" "$out" "Docker exited after SIGTERM"
|
||||
assert_not_contains "unkillable survivor: never logs full termination" "$out" "all Docker processes terminated"
|
||||
assert_contains "unkillable survivor: warns that processes survived" "$out" "processes survived SIGKILL"
|
||||
|
||||
# 6. Fix 1's respawn scenario. docker_pids returns the real entry-snapshot
|
||||
# pids on its FIRST call only; every call after that returns a different,
|
||||
# fixed, nonexistent pid, modeling a helper that respawned under a brand
|
||||
# new pid after the snapshot pids died. The snapshot pids are killed
|
||||
# BEFORE nuke_docker even runs, so wait_for_exit sees them as gone
|
||||
# immediately at the very first stage -- exactly the shape that fooled the
|
||||
# old stale-snapshot success check. Assert nuke_docker does not report
|
||||
# success at any stage.
|
||||
out="$(
|
||||
OPT_DEEP=0
|
||||
# shellcheck disable=SC2034 # read by wait_for_exit (sourced from docker-nuke); the source=/dev/null directive above hides that cross-file read from this analysis
|
||||
GRACEFUL_WAIT=1
|
||||
# shellcheck disable=SC2034 # read by wait_for_exit (sourced from docker-nuke); the source=/dev/null directive above hides that cross-file read from this analysis
|
||||
TERM_WAIT=1
|
||||
# shellcheck disable=SC2034 # read by the nuke_docker retry loop (sourced from docker-nuke); the source=/dev/null directive above hides that cross-file read from this analysis
|
||||
KILL_RETRY_ROUNDS=2
|
||||
p1=""; p2=""
|
||||
sleep 60 & p1=$!
|
||||
sleep 60 & p2=$!
|
||||
kill -9 "$p1" "$p2" 2>/dev/null || true
|
||||
wait "$p1" 2>/dev/null || true
|
||||
wait "$p2" 2>/dev/null || true
|
||||
# docker_pids is always invoked as the first stage of a pipeline inside
|
||||
# nuke_docker (`docker_pids | tr ...`), and bash runs every non-last
|
||||
# pipeline stage in a forked subshell. A plain shell-variable call
|
||||
# counter would therefore reset to its parent value on every single
|
||||
# call and never actually advance, silently defeating this stub. A
|
||||
# counter file survives across those forks since it lives on disk, not
|
||||
# in process memory.
|
||||
dp_call_marker="$TMPROOT/respawn-dp-calls"
|
||||
printf '0' >"$dp_call_marker"
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
docker_pids() {
|
||||
local n
|
||||
n="$(cat "$dp_call_marker" 2>/dev/null)"
|
||||
n=$((n + 1))
|
||||
printf '%s' "$n" >"$dp_call_marker"
|
||||
if [ "$n" -eq 1 ]; then
|
||||
printf '%s\n%s\n' "$p1" "$p2"
|
||||
else
|
||||
printf '%s\n' 666666
|
||||
fi
|
||||
}
|
||||
# shellcheck disable=SC2329 # invoked indirectly by nuke_docker name lookup
|
||||
osascript() { return 0; }
|
||||
nuke_docker 2>&1
|
||||
)"
|
||||
rc=$?
|
||||
assert_eq "respawn scenario: returns 1, not success" "1" "$rc"
|
||||
assert_not_contains "respawn scenario: never logs Docker exited cleanly" "$out" "Docker exited cleanly"
|
||||
assert_not_contains "respawn scenario: never logs post-SIGTERM exit" "$out" "Docker exited after SIGTERM"
|
||||
assert_not_contains "respawn scenario: never logs full termination" "$out" "all Docker processes terminated"
|
||||
# Positive counterpart (Fix 5): proves the stub genuinely produced a
|
||||
# DIFFERENT pid after the snapshot died, rather than this test vacuously
|
||||
# passing because the entry-snapshot pids simply stayed non-empty text for
|
||||
# an unrelated reason.
|
||||
assert_contains "respawn scenario: nuke_docker discovers the NEW respawned pid" "$out" "666666"
|
||||
}
|
||||
test_nuke_docker
|
||||
|
||||
printf '\n=== summary ===\n'
|
||||
teardown_tmp
|
||||
printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"
|
||||
|
||||
Reference in New Issue
Block a user