mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
fix(docker-nuke): make dump failure non-fatal and surface failed kills
This commit is contained in:
+152
-40
@@ -413,8 +413,8 @@ nuke_docker() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Narrow to the subset still alive from the entry snapshot before
|
# Narrow to the subset still alive from the entry snapshot before
|
||||||
# signalling (Fix 8): a pid that already exited during the graceful-quit
|
# signalling SIGTERM (Fix 8): a pid that already exited during the
|
||||||
# wait must never be re-signalled, at this stage or the next.
|
# graceful-quit wait must never be re-signalled.
|
||||||
local snapshot_alive
|
local snapshot_alive
|
||||||
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
||||||
snapshot_alive="$(pids_alive $pids)"
|
snapshot_alive="$(pids_alive $pids)"
|
||||||
@@ -430,12 +430,18 @@ nuke_docker() {
|
|||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# SIGKILL targets the fresh scan just taken above, not a set re-derived
|
||||||
|
# from the stale SIGTERM-stage snapshot (final review fix): a process that
|
||||||
|
# first appeared after snapshot_alive was captured -- e.g. a helper
|
||||||
|
# respawned by com.docker.backend during the SIGTERM wait -- is present in
|
||||||
|
# $fresh but was never a member of snapshot_alive, so filtering through
|
||||||
|
# snapshot_alive here silently skipped it for this entire round. It would
|
||||||
|
# only be caught a round later via the bounded respawn retry below,
|
||||||
|
# emitting a spurious "still present after SIGKILL, retrying" warning for a
|
||||||
|
# process that this round never actually SIGKILLed in the first place.
|
||||||
log "docker-nuke: sending SIGKILL"
|
log "docker-nuke: sending SIGKILL"
|
||||||
local remaining
|
# shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here
|
||||||
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
kill_pids KILL $fresh
|
||||||
remaining="$(pids_alive $snapshot_alive)"
|
|
||||||
# shellcheck disable=SC2086 # $remaining must word-split into separate PID arguments here
|
|
||||||
kill_pids KILL $remaining
|
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
||||||
if [ "${OPT_DEEP:-0}" -eq 1 ]; then
|
if [ "${OPT_DEEP:-0}" -eq 1 ]; then
|
||||||
@@ -535,6 +541,7 @@ write_summary() {
|
|||||||
printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n'
|
printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n'
|
||||||
printf '* sample-* user-space stacks showing where the daemon was stuck\n'
|
printf '* sample-* user-space stacks showing where the daemon was stuck\n'
|
||||||
printf '* logs/ Docker host and vm logs from just before the kill\n'
|
printf '* logs/ Docker host and vm logs from just before the kill\n'
|
||||||
|
printf '* restart docker-nuke restarts Docker Desktop itself, not your containers -- only containers started with a restart policy (e.g. unless-stopped) come back automatically; run docker ps -a to check the rest\n'
|
||||||
} >"$DUMP_DIR/summary.txt"
|
} >"$DUMP_DIR/summary.txt"
|
||||||
} 2>/dev/null || rc=$?
|
} 2>/dev/null || rc=$?
|
||||||
if [ "$rc" -ne 0 ]; then
|
if [ "$rc" -ne 0 ]; then
|
||||||
@@ -547,7 +554,7 @@ log() { printf '%s\n' "$*"; }
|
|||||||
warn() { printf '%s\n' "$*" >&2; }
|
warn() { printf '%s\n' "$*" >&2; }
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<EOF
|
||||||
docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first
|
docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
@@ -557,15 +564,18 @@ Options:
|
|||||||
-f, --force Skip the confirmation shown when Docker looks healthy
|
-f, --force Skip the confirmation shown when Docker looks healthy
|
||||||
-n, --no-restart Kill Docker but do not relaunch it
|
-n, --no-restart Kill Docker but do not relaunch it
|
||||||
--dry-run Report what would be killed and dumped; change nothing
|
--dry-run Report what would be killed and dumped; change nothing
|
||||||
--deep Also stop root launchd helpers (com.docker.vmnetd); needs sudo
|
--deep Also stop root launchd helpers (com.docker.vmnetd);
|
||||||
|
requires a cached sudo credential -- run "sudo -v" first
|
||||||
-h, --help Show this help
|
-h, --help Show this help
|
||||||
|
|
||||||
Diagnostics are written to ~/.local/state/docker-nuke/<timestamp>/ before
|
Diagnostics are written to $DUMP_ROOT/<timestamp>/ before anything is
|
||||||
anything is killed. The most recent 10 dumps are kept.
|
killed (override the root with DOCKER_NUKE_DUMP_ROOT). The most recent
|
||||||
|
$DUMP_RETENTION dumps are kept.
|
||||||
|
|
||||||
Exit codes:
|
Exit codes:
|
||||||
0 Docker killed and confirmed responsive again
|
0 Docker killed and confirmed responsive again
|
||||||
1 Daemon did not come back within the restart timeout
|
1 Daemon did not come back within the restart timeout, or the kill itself
|
||||||
|
did not complete
|
||||||
2 Usage error, or aborted at the healthy-daemon prompt
|
2 Usage error, or aborted at the healthy-daemon prompt
|
||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
@@ -595,10 +605,20 @@ parse_args() {
|
|||||||
|
|
||||||
# --- entrypoint -------------------------------------------------------------
|
# --- entrypoint -------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
# docker_version_summary — one-line "<version> (<N> running)" snapshot, or
|
||||||
|
# empty if `docker info` fails or hangs. Shared by confirm_healthy_nuke's
|
||||||
|
# healthy-daemon prompt and main's post-recovery report (final review fix:
|
||||||
|
# previously the same `docker info --format ...` call was duplicated verbatim
|
||||||
|
# in both places).
|
||||||
|
docker_version_summary() {
|
||||||
|
run_timeout "$HEALTH_TIMEOUT" docker info \
|
||||||
|
--format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
confirm_healthy_nuke() {
|
confirm_healthy_nuke() {
|
||||||
local info
|
local info
|
||||||
info="$(run_timeout "$HEALTH_TIMEOUT" docker info \
|
info="$(docker_version_summary)"
|
||||||
--format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true)"
|
|
||||||
log "docker-nuke: Docker looks healthy: ${info:-unknown}"
|
log "docker-nuke: Docker looks healthy: ${info:-unknown}"
|
||||||
log "Nuking will stop running containers."
|
log "Nuking will stop running containers."
|
||||||
printf 'Proceed anyway? [y/N] '
|
printf 'Proceed anyway? [y/N] '
|
||||||
@@ -606,7 +626,18 @@ confirm_healthy_nuke() {
|
|||||||
read -r reply || true
|
read -r reply || true
|
||||||
case "$reply" in
|
case "$reply" in
|
||||||
y|Y|yes|YES) return 0 ;;
|
y|Y|yes|YES) return 0 ;;
|
||||||
*) log "docker-nuke: aborted"; return 2 ;;
|
*)
|
||||||
|
# A non-TTY stdin (piped, redirected from /dev/null, closed) can never
|
||||||
|
# answer this prompt interactively even on a re-run, so the default
|
||||||
|
# deny should point at the actual way past it (final review fix): a
|
||||||
|
# bare "aborted" here previously gave no hint that --force exists.
|
||||||
|
if [ -t 0 ]; then
|
||||||
|
log "docker-nuke: aborted"
|
||||||
|
else
|
||||||
|
log "docker-nuke: aborted (no interactive input on stdin; use --force to skip this prompt)"
|
||||||
|
fi
|
||||||
|
return 2
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -621,12 +652,54 @@ dry_run_report() {
|
|||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
ps -o pid,stat,%cpu,%mem,etime,comm -p $pids 2>/dev/null || true
|
ps -o pid,stat,%cpu,%mem,etime,comm -p $pids 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
|
if [ "$OPT_DEEP" -eq 1 ]; then
|
||||||
|
log " would also stop privileged helpers (--deep): com.docker.vmnetd, com.docker.socket"
|
||||||
|
fi
|
||||||
log " would dump to: $DUMP_ROOT/<timestamp>"
|
log " would dump to: $DUMP_ROOT/<timestamp>"
|
||||||
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
||||||
log " would not restart Docker (--no-restart)"
|
log " would not restart Docker (--no-restart)"
|
||||||
else
|
else
|
||||||
log " would restart Docker and wait up to ${RESTART_TIMEOUT}s"
|
log " would restart Docker and wait up to ${RESTART_TIMEOUT}s"
|
||||||
fi
|
fi
|
||||||
|
if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then
|
||||||
|
log " NOTE: Docker currently looks healthy; a real run would prompt for confirmation unless --force is given"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# finish VERDICT RC ELAPSED [EXTRA_LINE ...] — shared tail for every terminal
|
||||||
|
# path in main() (final review fix). Writes the summary, prunes old dumps,
|
||||||
|
# then logs the verdict, any path-specific EXTRA_LINEs, and finally the dump
|
||||||
|
# location -- guarding the case where run_dump itself failed and DUMP_DIR is
|
||||||
|
# empty (Fix C1), so the tool can never print "diagnostics in " with nothing
|
||||||
|
# after it. Success (RC 0) logs via log(); any other RC logs every line via
|
||||||
|
# warn() instead, so a failed kill (Fix C2) can never look identical to a
|
||||||
|
# clean run in the terminal output. Collapsing the three near-identical
|
||||||
|
# terminal blocks that used to live in main() into this one helper makes it
|
||||||
|
# structurally impossible for a future fourth path to forget to thread the
|
||||||
|
# kill status into the verdict/exit code, or to forget the empty-DUMP_DIR
|
||||||
|
# guard.
|
||||||
|
finish() {
|
||||||
|
local verdict="$1" rc="$2" elapsed="$3"
|
||||||
|
shift 3
|
||||||
|
|
||||||
|
write_summary "$verdict" "$elapsed"
|
||||||
|
prune_dumps
|
||||||
|
|
||||||
|
local logfn=log
|
||||||
|
if [ "$rc" -ne 0 ]; then logfn=warn; fi
|
||||||
|
|
||||||
|
"$logfn" "docker-nuke: $verdict"
|
||||||
|
local line
|
||||||
|
for line in "$@"; do
|
||||||
|
"$logfn" "$line"
|
||||||
|
done
|
||||||
|
if [ -n "${DUMP_DIR:-}" ]; then
|
||||||
|
"$logfn" "docker-nuke: diagnostics in $DUMP_DIR"
|
||||||
|
else
|
||||||
|
"$logfn" "docker-nuke: diagnostics were not captured this run"
|
||||||
|
fi
|
||||||
|
|
||||||
|
return "$rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
@@ -640,48 +713,87 @@ main() {
|
|||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if docker_healthy && [ "$OPT_FORCE" -eq 0 ]; then
|
# Fix I2: checked up front, before anything is killed, so the user can fix
|
||||||
|
# a missing cached credential while Docker is still alive. Never fatal --
|
||||||
|
# stop_privileged_helpers already degrades gracefully (Fix 3) if this was
|
||||||
|
# ignored; this just surfaces the problem while it is still actionable
|
||||||
|
# instead of two silent `sudo -n` failures after the kill.
|
||||||
|
if [ "$OPT_DEEP" -eq 1 ] && ! sudo -n true 2>/dev/null; then
|
||||||
|
warn "docker-nuke: --deep requires a cached sudo credential; run \"sudo -v\" first. Continuing without it -- privileged helpers will not be stopped."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# OPT_FORCE checked first (final review fix): docker_healthy can cost up to
|
||||||
|
# HEALTH_TIMEOUT (6s) before answering, and --force is exactly the flag
|
||||||
|
# reached for when Docker is already known to be wedged -- paying that
|
||||||
|
# latency to compute an answer that is about to be skipped anyway serves no
|
||||||
|
# purpose.
|
||||||
|
if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then
|
||||||
confirm_healthy_nuke || return 2
|
confirm_healthy_nuke || return 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local started elapsed verdict
|
local started elapsed
|
||||||
started="$(date +%s)"
|
started="$(date +%s)"
|
||||||
|
|
||||||
run_dump
|
# Fix C1: run_dump is never called bare. It returns 1 when new_dump_dir
|
||||||
nuke_docker || true
|
# fails (e.g. dump root unwritable or full) -- precisely the condition it
|
||||||
|
# exists to detect -- and under set -e a bare call would abort the entire
|
||||||
|
# script right here: Docker is never killed, never restarted. The design
|
||||||
|
# requirement is explicit: the dump can never block recovery. A failure is
|
||||||
|
# recorded and recovery proceeds with no dump saved, exactly like every
|
||||||
|
# individual capture failure inside run_dump already does.
|
||||||
|
local dumped=1
|
||||||
|
run_dump || { dumped=0; warn "docker-nuke: diagnostics capture failed; continuing with recovery (no dump will be saved)"; }
|
||||||
|
local dump_note=""
|
||||||
|
if [ "$dumped" -eq 0 ]; then
|
||||||
|
dump_note=" (no dump captured)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Fix C2: nuke_docker's return status -- 1 only after KILL_RETRY_ROUNDS is
|
||||||
|
# exhausted with Docker processes still alive -- was previously discarded
|
||||||
|
# with `|| true` and never consulted again. It is captured here and
|
||||||
|
# threaded into every terminal path's verdict (so it lands in summary.txt,
|
||||||
|
# the file actually read during triage) and exit code, so a failed kill can
|
||||||
|
# never be reported as success.
|
||||||
|
local killed=1 kill_note=""
|
||||||
|
nuke_docker || killed=0
|
||||||
|
if [ "$killed" -eq 0 ]; then
|
||||||
|
local survivors
|
||||||
|
survivors="$(docker_pids | tr '\n' ' ')"
|
||||||
|
kill_note=" -- WARNING: kill incomplete, processes survived: ${survivors:-unknown}"
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
||||||
verdict="killed, not restarted (--no-restart)"
|
local verdict="killed, not restarted (--no-restart)${dump_note}${kill_note}"
|
||||||
|
local finish_rc=0
|
||||||
|
if [ "$killed" -eq 0 ]; then finish_rc=1; fi
|
||||||
elapsed=$(( $(date +%s) - started ))
|
elapsed=$(( $(date +%s) - started ))
|
||||||
write_summary "$verdict" "$elapsed"
|
finish "$verdict" "$finish_rc" "$elapsed"
|
||||||
prune_dumps
|
return "$finish_rc"
|
||||||
log "docker-nuke: $verdict"
|
|
||||||
log "docker-nuke: diagnostics in $DUMP_DIR"
|
|
||||||
return 0
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
start_docker
|
start_docker
|
||||||
if wait_for_daemon "$RESTART_TIMEOUT"; then
|
if wait_for_daemon "$RESTART_TIMEOUT"; then
|
||||||
verdict="recovered"
|
|
||||||
elapsed=$(( $(date +%s) - started ))
|
elapsed=$(( $(date +%s) - started ))
|
||||||
write_summary "$verdict" "$elapsed"
|
|
||||||
prune_dumps
|
|
||||||
local info
|
local info
|
||||||
info="$(run_timeout "$HEALTH_TIMEOUT" docker info \
|
info="$(docker_version_summary)"
|
||||||
--format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true)"
|
local verdict="recovered after ${elapsed}s — ${info:-ok}${dump_note}${kill_note}"
|
||||||
log "docker-nuke: Docker is back after ${elapsed}s — ${info:-ok}"
|
local finish_rc=0
|
||||||
log "docker-nuke: diagnostics in $DUMP_DIR"
|
if [ "$killed" -eq 0 ]; then finish_rc=1; fi
|
||||||
return 0
|
finish "$verdict" "$finish_rc" "$elapsed"
|
||||||
|
return "$finish_rc"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
verdict="daemon did not return within ${RESTART_TIMEOUT}s"
|
local verdict="daemon did not return within ${RESTART_TIMEOUT}s${dump_note}${kill_note}"
|
||||||
elapsed=$(( $(date +%s) - started ))
|
elapsed=$(( $(date +%s) - started ))
|
||||||
write_summary "$verdict" "$elapsed"
|
local still_running
|
||||||
prune_dumps
|
still_running="docker-nuke: still running: $(docker_pids | tr '\n' ' ')"
|
||||||
warn "docker-nuke: $verdict"
|
local deep_hint
|
||||||
warn "docker-nuke: still running: $(docker_pids | tr '\n' ' ')"
|
if [ -n "${DUMP_DIR:-}" ]; then
|
||||||
warn "docker-nuke: diagnostics in $DUMP_DIR"
|
deep_hint="docker-nuke: try 'docker-nuke --deep', or check disk space in $DUMP_DIR/disk.txt"
|
||||||
warn "docker-nuke: try 'docker-nuke --deep', or check disk space in $DUMP_DIR/disk.txt"
|
else
|
||||||
|
deep_hint="docker-nuke: try 'docker-nuke --deep'; no dump was captured this run to check disk space in"
|
||||||
|
fi
|
||||||
|
finish "$verdict" 1 "$elapsed" "$still_running" "$deep_hint"
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -348,6 +348,12 @@ printf '\n=== dump ===\n'
|
|||||||
|
|
||||||
test_dump() {
|
test_dump() {
|
||||||
local d rc
|
local d rc
|
||||||
|
# DUMP_DIR and CAPTURE_FAILURES are mutated below with no restore below
|
||||||
|
# this point in the original code -- the identical leak Task 7 already
|
||||||
|
# fixed one block below in test_restart_and_summary (Fix I3, final
|
||||||
|
# review). The stale path then propagates through every later block in
|
||||||
|
# this file. Save/restore the same way test_restart_and_summary does.
|
||||||
|
local old_dump_dir="$DUMP_DIR" old_capture_failures="$CAPTURE_FAILURES"
|
||||||
|
|
||||||
d="$(new_dump_dir)"
|
d="$(new_dump_dir)"
|
||||||
assert_eq "dump dir is created" "yes" "$([ -d "$d" ] && printf 'yes' || printf 'no')"
|
assert_eq "dump dir is created" "yes" "$([ -d "$d" ] && printf 'yes' || printf 'no')"
|
||||||
@@ -400,6 +406,9 @@ test_dump() {
|
|||||||
"$([ -d "$DUMP_ROOT/20260101T000000Z" ] && printf 'yes' || printf 'no')"
|
"$([ -d "$DUMP_ROOT/20260101T000000Z" ] && printf 'yes' || printf 'no')"
|
||||||
assert_eq "keeps the newest" "yes" \
|
assert_eq "keeps the newest" "yes" \
|
||||||
"$([ -d "$DUMP_ROOT/20260112T000000Z" ] && printf 'yes' || printf 'no')"
|
"$([ -d "$DUMP_ROOT/20260112T000000Z" ] && printf 'yes' || printf 'no')"
|
||||||
|
|
||||||
|
DUMP_DIR="$old_dump_dir"
|
||||||
|
CAPTURE_FAILURES="$old_capture_failures"
|
||||||
}
|
}
|
||||||
test_dump
|
test_dump
|
||||||
|
|
||||||
@@ -563,6 +572,28 @@ test_sudo_noninteractive() {
|
|||||||
}
|
}
|
||||||
test_sudo_noninteractive
|
test_sudo_noninteractive
|
||||||
|
|
||||||
|
printf '\n=== --deep help text and upfront credential check (Fix I2) ===\n'
|
||||||
|
|
||||||
|
test_deep_upfront_check() {
|
||||||
|
# Task 6 correctly moved stop_privileged_helpers to `sudo -n` so a missing
|
||||||
|
# credential fails fast instead of hanging on a hidden prompt after Docker
|
||||||
|
# is already dead (Fix 3, above). But until this fix, usage() still claimed
|
||||||
|
# "needs sudo" (never prompts) and main() never checked for a cached
|
||||||
|
# credential until AFTER the kill, when it is too late for the user to do
|
||||||
|
# anything about it. This is a static regression guard: usage() text must
|
||||||
|
# be corrected, and main() must check `sudo -n true` before anything is
|
||||||
|
# killed.
|
||||||
|
local help_text src
|
||||||
|
help_text="$(usage)"
|
||||||
|
assert_not_contains "usage no longer claims --deep just needs sudo" "$help_text" "needs sudo"
|
||||||
|
assert_contains "usage explains --deep requires a cached credential" "$help_text" "cached sudo credential"
|
||||||
|
assert_contains "usage tells the user how to cache it" "$help_text" "sudo -v"
|
||||||
|
|
||||||
|
src="$(cat "$SCRIPT_DIR/docker-nuke")"
|
||||||
|
assert_contains "main checks sudo -n true up front" "$src" "sudo -n true"
|
||||||
|
}
|
||||||
|
test_deep_upfront_check
|
||||||
|
|
||||||
printf '\n=== nuke_docker escalation orchestration (Fix 6) ===\n'
|
printf '\n=== nuke_docker escalation orchestration (Fix 6) ===\n'
|
||||||
|
|
||||||
test_nuke_docker() {
|
test_nuke_docker() {
|
||||||
@@ -958,6 +989,266 @@ test_dry_run() {
|
|||||||
}
|
}
|
||||||
test_dry_run
|
test_dry_run
|
||||||
|
|
||||||
|
printf '\n=== main() integration (Fix I1) ===\n'
|
||||||
|
|
||||||
|
# Before this block, the only entrypoint coverage was --dry-run / --help /
|
||||||
|
# --bogus (test_dry_run above), all of which return before run_dump is ever
|
||||||
|
# called -- so main()'s own composition of run_dump, nuke_docker,
|
||||||
|
# start_docker, and wait_for_daemon had zero automated coverage. That is
|
||||||
|
# exactly why the C1 (bare `run_dump` under set -e aborting recovery) and C2
|
||||||
|
# (a failed nuke_docker silently reported as success) review findings were
|
||||||
|
# never caught by any earlier per-task test.
|
||||||
|
#
|
||||||
|
# Every case below shadows run_dump/nuke_docker/start_docker/wait_for_daemon/
|
||||||
|
# docker_healthy (never the real Docker) per the standing safety rule for
|
||||||
|
# this file, plus docker_pids for deterministic survivor lists. Each
|
||||||
|
# subshell explicitly re-enables `set -euo pipefail` to reproduce the real
|
||||||
|
# script's own top-of-file directive -- the harness's `set +e` (line 5) would
|
||||||
|
# otherwise mask the exact class of bug C1 fixes (a bare failing command
|
||||||
|
# aborting the caller under set -e). stdin is always either piped or
|
||||||
|
# redirected from /dev/null so a wrong turn in the code under test can never
|
||||||
|
# block on a real terminal prompt.
|
||||||
|
|
||||||
|
printf '\n--- healthy daemon, declines the prompt ---\n'
|
||||||
|
|
||||||
|
test_main_healthy_decline() {
|
||||||
|
local out rc=0
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { printf 'RUN_DUMP_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { printf 'NUKE_DOCKER_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
printf 'n\n' | main 2>&1
|
||||||
|
)"
|
||||||
|
# Deliberately NOT `)" || rc=$?` (final review fix): per the bash manual,
|
||||||
|
# "If a compound command or shell function executes in a context where -e
|
||||||
|
# is being ignored, none of the commands executed within [it] will be
|
||||||
|
# affected by the -e setting, even if -e is set" -- and a command
|
||||||
|
# substitution used as the left operand of `||` is exactly such a context.
|
||||||
|
# Attaching `|| rc=$?` directly to this assignment would silently defeat
|
||||||
|
# the `set -euo pipefail` above for the ENTIRE subshell, no matter how
|
||||||
|
# explicitly it is set inside. Capturing $? on its own line next avoids
|
||||||
|
# that trap; see test_nuke_docker above for the same established pattern.
|
||||||
|
rc=$?
|
||||||
|
assert_eq "healthy+decline: rc 2" "2" "$rc"
|
||||||
|
assert_not_contains "healthy+decline: run_dump never called" "$out" "RUN_DUMP_CALLED"
|
||||||
|
assert_not_contains "healthy+decline: nuke_docker never called" "$out" "NUKE_DOCKER_CALLED"
|
||||||
|
assert_not_contains "healthy+decline: start_docker never called" "$out" "START_DOCKER_CALLED"
|
||||||
|
}
|
||||||
|
test_main_healthy_decline
|
||||||
|
|
||||||
|
printf '\n--- healthy daemon with --force: proceeds without prompting ---\n'
|
||||||
|
|
||||||
|
test_main_healthy_force() {
|
||||||
|
local out rc=0 dump_dir="$TMPROOT/main-force-dump"
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { :; }
|
||||||
|
main --force </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above
|
||||||
|
assert_eq "healthy+force: rc 0" "0" "$rc"
|
||||||
|
assert_not_contains "healthy+force: does not print the confirmation prompt" "$out" "Proceed anyway"
|
||||||
|
}
|
||||||
|
test_main_healthy_force
|
||||||
|
|
||||||
|
printf '\n--- wedged daemon: proceeds with no prompt ---\n'
|
||||||
|
|
||||||
|
test_main_wedged_no_prompt() {
|
||||||
|
local out rc=0 dump_dir="$TMPROOT/main-wedged-dump"
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { :; }
|
||||||
|
main </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above
|
||||||
|
assert_eq "wedged: rc 0" "0" "$rc"
|
||||||
|
assert_not_contains "wedged: does not print the confirmation prompt" "$out" "Proceed anyway"
|
||||||
|
}
|
||||||
|
test_main_wedged_no_prompt
|
||||||
|
|
||||||
|
printf '\n--- C1: run_dump failure never aborts recovery ---\n'
|
||||||
|
|
||||||
|
test_main_c1_dump_failure_does_not_abort() {
|
||||||
|
# THE headline regression test for Fix C1. Under the pre-fix code, `run_dump`
|
||||||
|
# was called bare on its own line under set -euo pipefail; run_dump returns
|
||||||
|
# 1 when new_dump_dir fails (dump root unwritable or full), and a bare
|
||||||
|
# failing command aborts the whole script right there via set -e -- Docker
|
||||||
|
# is never killed, never restarted. Proving this requires `set -e` to
|
||||||
|
# genuinely be active in this subshell (see the block-level comment above),
|
||||||
|
# not just the harness's relaxed `set +e`.
|
||||||
|
local out rc=0
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { printf 'NUKE_DOCKER_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { :; }
|
||||||
|
main </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above; this
|
||||||
|
# one matters most of all, since it is what proves C1
|
||||||
|
assert_eq "C1: main still completes (rc 0) after a dump failure" "0" "$rc"
|
||||||
|
assert_contains "C1: nuke_docker is still called after a dump failure" "$out" "NUKE_DOCKER_CALLED"
|
||||||
|
assert_contains "C1: start_docker is still called after a dump failure" "$out" "START_DOCKER_CALLED"
|
||||||
|
assert_contains "C1: warns that diagnostics capture failed" "$out" "diagnostics capture failed"
|
||||||
|
}
|
||||||
|
test_main_c1_dump_failure_does_not_abort
|
||||||
|
|
||||||
|
printf '\n--- C2: a failed kill is never reported as success ---\n'
|
||||||
|
|
||||||
|
test_main_c2_failed_kill_no_restart() {
|
||||||
|
# THE headline regression test for Fix C2, --no-restart path. nuke_docker
|
||||||
|
# returns 1 only after KILL_RETRY_ROUNDS is exhausted with Docker processes
|
||||||
|
# still alive. Under the pre-fix code this was discarded with `|| true` and
|
||||||
|
# never consulted again: --no-restart's verdict said "killed, not
|
||||||
|
# restarted" and returned 0 even though nothing was actually killed.
|
||||||
|
local out rc=0 dump_dir="$TMPROOT/main-c2-norestart-dump"
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { printf '%s\n' 777777; }
|
||||||
|
main --no-restart </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above
|
||||||
|
assert_eq "C2 --no-restart: exit code is non-zero when the kill fails" "1" "$rc"
|
||||||
|
assert_not_contains "C2 --no-restart: start_docker is never called" "$out" "START_DOCKER_CALLED"
|
||||||
|
assert_contains "C2 --no-restart: the survivor pid reaches the log" "$out" "777777"
|
||||||
|
assert_contains "C2 --no-restart: summary.txt records the failed kill" \
|
||||||
|
"$(cat "$dump_dir/summary.txt" 2>/dev/null)" "777777"
|
||||||
|
}
|
||||||
|
test_main_c2_failed_kill_no_restart
|
||||||
|
|
||||||
|
test_main_c2_failed_kill_restart() {
|
||||||
|
# C2's restart-path counterpart: even when wait_for_daemon subsequently
|
||||||
|
# reports the daemon healthy again (a genuinely possible outcome -- a fresh
|
||||||
|
# instance can start responding while old zombies from the failed kill
|
||||||
|
# still linger), a kill that did not complete must still surface as a
|
||||||
|
# failure, not get overwritten by the later "recovered" success.
|
||||||
|
local out rc=0 dump_dir="$TMPROOT/main-c2-restart-dump"
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { printf '%s\n' 888888; }
|
||||||
|
main </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above
|
||||||
|
assert_eq "C2 restart path: exit code is non-zero despite the daemon coming back" "1" "$rc"
|
||||||
|
assert_contains "C2 restart path: the survivor pid reaches the log" "$out" "888888"
|
||||||
|
assert_contains "C2 restart path: summary.txt records the failed kill" \
|
||||||
|
"$(cat "$dump_dir/summary.txt" 2>/dev/null)" "888888"
|
||||||
|
}
|
||||||
|
test_main_c2_failed_kill_restart
|
||||||
|
|
||||||
|
printf '\n--- --no-restart: start_docker never called (success path) ---\n'
|
||||||
|
|
||||||
|
test_main_no_restart_success() {
|
||||||
|
local out rc=0 dump_dir="$TMPROOT/main-norestart-ok-dump"
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { :; }
|
||||||
|
main --no-restart </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above
|
||||||
|
assert_eq "no-restart success: rc 0" "0" "$rc"
|
||||||
|
assert_not_contains "no-restart success: start_docker never called" "$out" "START_DOCKER_CALLED"
|
||||||
|
}
|
||||||
|
test_main_no_restart_success
|
||||||
|
|
||||||
|
printf '\n--- daemon never returns: rc 1 ---\n'
|
||||||
|
|
||||||
|
test_main_daemon_never_returns() {
|
||||||
|
local out rc=0 dump_dir="$TMPROOT/main-timeout-dump"
|
||||||
|
out="$(
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup
|
||||||
|
docker_healthy() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup
|
||||||
|
run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup
|
||||||
|
nuke_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup
|
||||||
|
start_docker() { return 0; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup
|
||||||
|
wait_for_daemon() { return 1; }
|
||||||
|
# shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup
|
||||||
|
docker_pids() { :; }
|
||||||
|
main </dev/null 2>&1
|
||||||
|
)"
|
||||||
|
rc=$? # not `|| rc=$?` -- see the note on the first main() test above
|
||||||
|
assert_eq "daemon never returns: rc 1" "1" "$rc"
|
||||||
|
assert_contains "daemon never returns: warns it did not return" "$out" "did not return"
|
||||||
|
}
|
||||||
|
test_main_daemon_never_returns
|
||||||
|
|
||||||
printf '\n=== summary ===\n'
|
printf '\n=== summary ===\n'
|
||||||
teardown_tmp
|
teardown_tmp
|
||||||
printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"
|
printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"
|
||||||
|
|||||||
Reference in New Issue
Block a user