From 7831f0c7992e3805665da86ad12f0328f972869b Mon Sep 17 00:00:00 2001 From: David Ibia Date: Mon, 20 Jul 2026 02:40:47 +0100 Subject: [PATCH] fix(docker-nuke): make dump failure non-fatal and surface failed kills --- .local/bin/docker-nuke | 192 +++++++++++++++++----- .local/bin/test-docker-nuke.sh | 291 +++++++++++++++++++++++++++++++++ 2 files changed, 443 insertions(+), 40 deletions(-) diff --git a/.local/bin/docker-nuke b/.local/bin/docker-nuke index 744df1f..b7296e2 100755 --- a/.local/bin/docker-nuke +++ b/.local/bin/docker-nuke @@ -413,8 +413,8 @@ nuke_docker() { fi # Narrow to the subset still alive from the entry snapshot before - # signalling (Fix 8): a pid that already exited during the graceful-quit - # wait must never be re-signalled, at this stage or the next. + # signalling SIGTERM (Fix 8): a pid that already exited during the + # graceful-quit wait must never be re-signalled. local snapshot_alive # shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here snapshot_alive="$(pids_alive $pids)" @@ -430,12 +430,18 @@ nuke_docker() { return 0 fi + # SIGKILL targets the fresh scan just taken above, not a set re-derived + # from the stale SIGTERM-stage snapshot (final review fix): a process that + # first appeared after snapshot_alive was captured -- e.g. a helper + # respawned by com.docker.backend during the SIGTERM wait -- is present in + # $fresh but was never a member of snapshot_alive, so filtering through + # snapshot_alive here silently skipped it for this entire round. It would + # only be caught a round later via the bounded respawn retry below, + # emitting a spurious "still present after SIGKILL, retrying" warning for a + # process that this round never actually SIGKILLed in the first place. log "docker-nuke: sending SIGKILL" - local remaining - # shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here - remaining="$(pids_alive $snapshot_alive)" - # shellcheck disable=SC2086 # $remaining must word-split into separate PID arguments here - kill_pids KILL $remaining + # shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here + kill_pids KILL $fresh sleep 1 if [ "${OPT_DEEP:-0}" -eq 1 ]; then @@ -535,6 +541,7 @@ write_summary() { printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n' printf '* sample-* user-space stacks showing where the daemon was stuck\n' printf '* logs/ Docker host and vm logs from just before the kill\n' + printf '* restart docker-nuke restarts Docker Desktop itself, not your containers -- only containers started with a restart policy (e.g. unless-stopped) come back automatically; run docker ps -a to check the rest\n' } >"$DUMP_DIR/summary.txt" } 2>/dev/null || rc=$? if [ "$rc" -ne 0 ]; then @@ -547,7 +554,7 @@ log() { printf '%s\n' "$*"; } warn() { printf '%s\n' "$*" >&2; } usage() { - cat <<'EOF' + cat </ before -anything is killed. The most recent 10 dumps are kept. +Diagnostics are written to $DUMP_ROOT// before anything is +killed (override the root with DOCKER_NUKE_DUMP_ROOT). The most recent +$DUMP_RETENTION dumps are kept. Exit codes: 0 Docker killed and confirmed responsive again - 1 Daemon did not come back within the restart timeout + 1 Daemon did not come back within the restart timeout, or the kill itself + did not complete 2 Usage error, or aborted at the healthy-daemon prompt EOF } @@ -595,10 +605,20 @@ parse_args() { # --- entrypoint ------------------------------------------------------------- + +# docker_version_summary — one-line " ( running)" snapshot, or +# empty if `docker info` fails or hangs. Shared by confirm_healthy_nuke's +# healthy-daemon prompt and main's post-recovery report (final review fix: +# previously the same `docker info --format ...` call was duplicated verbatim +# in both places). +docker_version_summary() { + run_timeout "$HEALTH_TIMEOUT" docker info \ + --format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true +} + confirm_healthy_nuke() { local info - info="$(run_timeout "$HEALTH_TIMEOUT" docker info \ - --format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true)" + info="$(docker_version_summary)" log "docker-nuke: Docker looks healthy: ${info:-unknown}" log "Nuking will stop running containers." printf 'Proceed anyway? [y/N] ' @@ -606,7 +626,18 @@ confirm_healthy_nuke() { read -r reply || true case "$reply" in y|Y|yes|YES) return 0 ;; - *) log "docker-nuke: aborted"; return 2 ;; + *) + # A non-TTY stdin (piped, redirected from /dev/null, closed) can never + # answer this prompt interactively even on a re-run, so the default + # deny should point at the actual way past it (final review fix): a + # bare "aborted" here previously gave no hint that --force exists. + if [ -t 0 ]; then + log "docker-nuke: aborted" + else + log "docker-nuke: aborted (no interactive input on stdin; use --force to skip this prompt)" + fi + return 2 + ;; esac } @@ -621,12 +652,54 @@ dry_run_report() { # shellcheck disable=SC2086 ps -o pid,stat,%cpu,%mem,etime,comm -p $pids 2>/dev/null || true fi + if [ "$OPT_DEEP" -eq 1 ]; then + log " would also stop privileged helpers (--deep): com.docker.vmnetd, com.docker.socket" + fi log " would dump to: $DUMP_ROOT/" if [ "$OPT_NO_RESTART" -eq 1 ]; then log " would not restart Docker (--no-restart)" else log " would restart Docker and wait up to ${RESTART_TIMEOUT}s" fi + if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then + log " NOTE: Docker currently looks healthy; a real run would prompt for confirmation unless --force is given" + fi +} + +# finish VERDICT RC ELAPSED [EXTRA_LINE ...] — shared tail for every terminal +# path in main() (final review fix). Writes the summary, prunes old dumps, +# then logs the verdict, any path-specific EXTRA_LINEs, and finally the dump +# location -- guarding the case where run_dump itself failed and DUMP_DIR is +# empty (Fix C1), so the tool can never print "diagnostics in " with nothing +# after it. Success (RC 0) logs via log(); any other RC logs every line via +# warn() instead, so a failed kill (Fix C2) can never look identical to a +# clean run in the terminal output. Collapsing the three near-identical +# terminal blocks that used to live in main() into this one helper makes it +# structurally impossible for a future fourth path to forget to thread the +# kill status into the verdict/exit code, or to forget the empty-DUMP_DIR +# guard. +finish() { + local verdict="$1" rc="$2" elapsed="$3" + shift 3 + + write_summary "$verdict" "$elapsed" + prune_dumps + + local logfn=log + if [ "$rc" -ne 0 ]; then logfn=warn; fi + + "$logfn" "docker-nuke: $verdict" + local line + for line in "$@"; do + "$logfn" "$line" + done + if [ -n "${DUMP_DIR:-}" ]; then + "$logfn" "docker-nuke: diagnostics in $DUMP_DIR" + else + "$logfn" "docker-nuke: diagnostics were not captured this run" + fi + + return "$rc" } main() { @@ -640,48 +713,87 @@ main() { return 0 fi - if docker_healthy && [ "$OPT_FORCE" -eq 0 ]; then + # Fix I2: checked up front, before anything is killed, so the user can fix + # a missing cached credential while Docker is still alive. Never fatal -- + # stop_privileged_helpers already degrades gracefully (Fix 3) if this was + # ignored; this just surfaces the problem while it is still actionable + # instead of two silent `sudo -n` failures after the kill. + if [ "$OPT_DEEP" -eq 1 ] && ! sudo -n true 2>/dev/null; then + warn "docker-nuke: --deep requires a cached sudo credential; run \"sudo -v\" first. Continuing without it -- privileged helpers will not be stopped." + fi + + # OPT_FORCE checked first (final review fix): docker_healthy can cost up to + # HEALTH_TIMEOUT (6s) before answering, and --force is exactly the flag + # reached for when Docker is already known to be wedged -- paying that + # latency to compute an answer that is about to be skipped anyway serves no + # purpose. + if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then confirm_healthy_nuke || return 2 fi - local started elapsed verdict + local started elapsed started="$(date +%s)" - run_dump - nuke_docker || true + # Fix C1: run_dump is never called bare. It returns 1 when new_dump_dir + # fails (e.g. dump root unwritable or full) -- precisely the condition it + # exists to detect -- and under set -e a bare call would abort the entire + # script right here: Docker is never killed, never restarted. The design + # requirement is explicit: the dump can never block recovery. A failure is + # recorded and recovery proceeds with no dump saved, exactly like every + # individual capture failure inside run_dump already does. + local dumped=1 + run_dump || { dumped=0; warn "docker-nuke: diagnostics capture failed; continuing with recovery (no dump will be saved)"; } + local dump_note="" + if [ "$dumped" -eq 0 ]; then + dump_note=" (no dump captured)" + fi + + # Fix C2: nuke_docker's return status -- 1 only after KILL_RETRY_ROUNDS is + # exhausted with Docker processes still alive -- was previously discarded + # with `|| true` and never consulted again. It is captured here and + # threaded into every terminal path's verdict (so it lands in summary.txt, + # the file actually read during triage) and exit code, so a failed kill can + # never be reported as success. + local killed=1 kill_note="" + nuke_docker || killed=0 + if [ "$killed" -eq 0 ]; then + local survivors + survivors="$(docker_pids | tr '\n' ' ')" + kill_note=" -- WARNING: kill incomplete, processes survived: ${survivors:-unknown}" + fi if [ "$OPT_NO_RESTART" -eq 1 ]; then - verdict="killed, not restarted (--no-restart)" + local verdict="killed, not restarted (--no-restart)${dump_note}${kill_note}" + local finish_rc=0 + if [ "$killed" -eq 0 ]; then finish_rc=1; fi elapsed=$(( $(date +%s) - started )) - write_summary "$verdict" "$elapsed" - prune_dumps - log "docker-nuke: $verdict" - log "docker-nuke: diagnostics in $DUMP_DIR" - return 0 + finish "$verdict" "$finish_rc" "$elapsed" + return "$finish_rc" fi start_docker if wait_for_daemon "$RESTART_TIMEOUT"; then - verdict="recovered" elapsed=$(( $(date +%s) - started )) - write_summary "$verdict" "$elapsed" - prune_dumps local info - info="$(run_timeout "$HEALTH_TIMEOUT" docker info \ - --format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true)" - log "docker-nuke: Docker is back after ${elapsed}s — ${info:-ok}" - log "docker-nuke: diagnostics in $DUMP_DIR" - return 0 + info="$(docker_version_summary)" + local verdict="recovered after ${elapsed}s — ${info:-ok}${dump_note}${kill_note}" + local finish_rc=0 + if [ "$killed" -eq 0 ]; then finish_rc=1; fi + finish "$verdict" "$finish_rc" "$elapsed" + return "$finish_rc" fi - verdict="daemon did not return within ${RESTART_TIMEOUT}s" + local verdict="daemon did not return within ${RESTART_TIMEOUT}s${dump_note}${kill_note}" elapsed=$(( $(date +%s) - started )) - write_summary "$verdict" "$elapsed" - prune_dumps - warn "docker-nuke: $verdict" - warn "docker-nuke: still running: $(docker_pids | tr '\n' ' ')" - warn "docker-nuke: diagnostics in $DUMP_DIR" - warn "docker-nuke: try 'docker-nuke --deep', or check disk space in $DUMP_DIR/disk.txt" + local still_running + still_running="docker-nuke: still running: $(docker_pids | tr '\n' ' ')" + local deep_hint + if [ -n "${DUMP_DIR:-}" ]; then + deep_hint="docker-nuke: try 'docker-nuke --deep', or check disk space in $DUMP_DIR/disk.txt" + else + deep_hint="docker-nuke: try 'docker-nuke --deep'; no dump was captured this run to check disk space in" + fi + finish "$verdict" 1 "$elapsed" "$still_running" "$deep_hint" return 1 } diff --git a/.local/bin/test-docker-nuke.sh b/.local/bin/test-docker-nuke.sh index 066b66a..4e8a1fe 100755 --- a/.local/bin/test-docker-nuke.sh +++ b/.local/bin/test-docker-nuke.sh @@ -348,6 +348,12 @@ printf '\n=== dump ===\n' test_dump() { local d rc + # DUMP_DIR and CAPTURE_FAILURES are mutated below with no restore below + # this point in the original code -- the identical leak Task 7 already + # fixed one block below in test_restart_and_summary (Fix I3, final + # review). The stale path then propagates through every later block in + # this file. Save/restore the same way test_restart_and_summary does. + local old_dump_dir="$DUMP_DIR" old_capture_failures="$CAPTURE_FAILURES" d="$(new_dump_dir)" assert_eq "dump dir is created" "yes" "$([ -d "$d" ] && printf 'yes' || printf 'no')" @@ -400,6 +406,9 @@ test_dump() { "$([ -d "$DUMP_ROOT/20260101T000000Z" ] && printf 'yes' || printf 'no')" assert_eq "keeps the newest" "yes" \ "$([ -d "$DUMP_ROOT/20260112T000000Z" ] && printf 'yes' || printf 'no')" + + DUMP_DIR="$old_dump_dir" + CAPTURE_FAILURES="$old_capture_failures" } test_dump @@ -563,6 +572,28 @@ test_sudo_noninteractive() { } test_sudo_noninteractive +printf '\n=== --deep help text and upfront credential check (Fix I2) ===\n' + +test_deep_upfront_check() { + # Task 6 correctly moved stop_privileged_helpers to `sudo -n` so a missing + # credential fails fast instead of hanging on a hidden prompt after Docker + # is already dead (Fix 3, above). But until this fix, usage() still claimed + # "needs sudo" (never prompts) and main() never checked for a cached + # credential until AFTER the kill, when it is too late for the user to do + # anything about it. This is a static regression guard: usage() text must + # be corrected, and main() must check `sudo -n true` before anything is + # killed. + local help_text src + help_text="$(usage)" + assert_not_contains "usage no longer claims --deep just needs sudo" "$help_text" "needs sudo" + assert_contains "usage explains --deep requires a cached credential" "$help_text" "cached sudo credential" + assert_contains "usage tells the user how to cache it" "$help_text" "sudo -v" + + src="$(cat "$SCRIPT_DIR/docker-nuke")" + assert_contains "main checks sudo -n true up front" "$src" "sudo -n true" +} +test_deep_upfront_check + printf '\n=== nuke_docker escalation orchestration (Fix 6) ===\n' test_nuke_docker() { @@ -958,6 +989,266 @@ test_dry_run() { } test_dry_run +printf '\n=== main() integration (Fix I1) ===\n' + +# Before this block, the only entrypoint coverage was --dry-run / --help / +# --bogus (test_dry_run above), all of which return before run_dump is ever +# called -- so main()'s own composition of run_dump, nuke_docker, +# start_docker, and wait_for_daemon had zero automated coverage. That is +# exactly why the C1 (bare `run_dump` under set -e aborting recovery) and C2 +# (a failed nuke_docker silently reported as success) review findings were +# never caught by any earlier per-task test. +# +# Every case below shadows run_dump/nuke_docker/start_docker/wait_for_daemon/ +# docker_healthy (never the real Docker) per the standing safety rule for +# this file, plus docker_pids for deterministic survivor lists. Each +# subshell explicitly re-enables `set -euo pipefail` to reproduce the real +# script's own top-of-file directive -- the harness's `set +e` (line 5) would +# otherwise mask the exact class of bug C1 fixes (a bare failing command +# aborting the caller under set -e). stdin is always either piped or +# redirected from /dev/null so a wrong turn in the code under test can never +# block on a real terminal prompt. + +printf '\n--- healthy daemon, declines the prompt ---\n' + +test_main_healthy_decline() { + local out rc=0 + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { printf 'RUN_DUMP_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { printf 'NUKE_DOCKER_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + printf 'n\n' | main 2>&1 + )" + # Deliberately NOT `)" || rc=$?` (final review fix): per the bash manual, + # "If a compound command or shell function executes in a context where -e + # is being ignored, none of the commands executed within [it] will be + # affected by the -e setting, even if -e is set" -- and a command + # substitution used as the left operand of `||` is exactly such a context. + # Attaching `|| rc=$?` directly to this assignment would silently defeat + # the `set -euo pipefail` above for the ENTIRE subshell, no matter how + # explicitly it is set inside. Capturing $? on its own line next avoids + # that trap; see test_nuke_docker above for the same established pattern. + rc=$? + assert_eq "healthy+decline: rc 2" "2" "$rc" + assert_not_contains "healthy+decline: run_dump never called" "$out" "RUN_DUMP_CALLED" + assert_not_contains "healthy+decline: nuke_docker never called" "$out" "NUKE_DOCKER_CALLED" + assert_not_contains "healthy+decline: start_docker never called" "$out" "START_DOCKER_CALLED" +} +test_main_healthy_decline + +printf '\n--- healthy daemon with --force: proceeds without prompting ---\n' + +test_main_healthy_force() { + local out rc=0 dump_dir="$TMPROOT/main-force-dump" + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { :; } + main --force &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above + assert_eq "healthy+force: rc 0" "0" "$rc" + assert_not_contains "healthy+force: does not print the confirmation prompt" "$out" "Proceed anyway" +} +test_main_healthy_force + +printf '\n--- wedged daemon: proceeds with no prompt ---\n' + +test_main_wedged_no_prompt() { + local out rc=0 dump_dir="$TMPROOT/main-wedged-dump" + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { :; } + main &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above + assert_eq "wedged: rc 0" "0" "$rc" + assert_not_contains "wedged: does not print the confirmation prompt" "$out" "Proceed anyway" +} +test_main_wedged_no_prompt + +printf '\n--- C1: run_dump failure never aborts recovery ---\n' + +test_main_c1_dump_failure_does_not_abort() { + # THE headline regression test for Fix C1. Under the pre-fix code, `run_dump` + # was called bare on its own line under set -euo pipefail; run_dump returns + # 1 when new_dump_dir fails (dump root unwritable or full), and a bare + # failing command aborts the whole script right there via set -e -- Docker + # is never killed, never restarted. Proving this requires `set -e` to + # genuinely be active in this subshell (see the block-level comment above), + # not just the harness's relaxed `set +e`. + local out rc=0 + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { printf 'NUKE_DOCKER_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { :; } + main &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above; this + # one matters most of all, since it is what proves C1 + assert_eq "C1: main still completes (rc 0) after a dump failure" "0" "$rc" + assert_contains "C1: nuke_docker is still called after a dump failure" "$out" "NUKE_DOCKER_CALLED" + assert_contains "C1: start_docker is still called after a dump failure" "$out" "START_DOCKER_CALLED" + assert_contains "C1: warns that diagnostics capture failed" "$out" "diagnostics capture failed" +} +test_main_c1_dump_failure_does_not_abort + +printf '\n--- C2: a failed kill is never reported as success ---\n' + +test_main_c2_failed_kill_no_restart() { + # THE headline regression test for Fix C2, --no-restart path. nuke_docker + # returns 1 only after KILL_RETRY_ROUNDS is exhausted with Docker processes + # still alive. Under the pre-fix code this was discarded with `|| true` and + # never consulted again: --no-restart's verdict said "killed, not + # restarted" and returned 0 even though nothing was actually killed. + local out rc=0 dump_dir="$TMPROOT/main-c2-norestart-dump" + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { printf '%s\n' 777777; } + main --no-restart &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above + assert_eq "C2 --no-restart: exit code is non-zero when the kill fails" "1" "$rc" + assert_not_contains "C2 --no-restart: start_docker is never called" "$out" "START_DOCKER_CALLED" + assert_contains "C2 --no-restart: the survivor pid reaches the log" "$out" "777777" + assert_contains "C2 --no-restart: summary.txt records the failed kill" \ + "$(cat "$dump_dir/summary.txt" 2>/dev/null)" "777777" +} +test_main_c2_failed_kill_no_restart + +test_main_c2_failed_kill_restart() { + # C2's restart-path counterpart: even when wait_for_daemon subsequently + # reports the daemon healthy again (a genuinely possible outcome -- a fresh + # instance can start responding while old zombies from the failed kill + # still linger), a kill that did not complete must still surface as a + # failure, not get overwritten by the later "recovered" success. + local out rc=0 dump_dir="$TMPROOT/main-c2-restart-dump" + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { printf '%s\n' 888888; } + main &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above + assert_eq "C2 restart path: exit code is non-zero despite the daemon coming back" "1" "$rc" + assert_contains "C2 restart path: the survivor pid reaches the log" "$out" "888888" + assert_contains "C2 restart path: summary.txt records the failed kill" \ + "$(cat "$dump_dir/summary.txt" 2>/dev/null)" "888888" +} +test_main_c2_failed_kill_restart + +printf '\n--- --no-restart: start_docker never called (success path) ---\n' + +test_main_no_restart_success() { + local out rc=0 dump_dir="$TMPROOT/main-norestart-ok-dump" + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { printf 'START_DOCKER_CALLED\n'; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { :; } + main --no-restart &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above + assert_eq "no-restart success: rc 0" "0" "$rc" + assert_not_contains "no-restart success: start_docker never called" "$out" "START_DOCKER_CALLED" +} +test_main_no_restart_success + +printf '\n--- daemon never returns: rc 1 ---\n' + +test_main_daemon_never_returns() { + local out rc=0 dump_dir="$TMPROOT/main-timeout-dump" + out="$( + set -euo pipefail + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_healthy name lookup + docker_healthy() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via run_dump name lookup + run_dump() { DUMP_DIR="$dump_dir"; mkdir -p "$DUMP_DIR"; return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via nuke_docker name lookup + nuke_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via start_docker name lookup + start_docker() { return 0; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via wait_for_daemon name lookup + wait_for_daemon() { return 1; } + # shellcheck disable=SC2329 # invoked indirectly by main (sourced from docker-nuke) via docker_pids name lookup + docker_pids() { :; } + main &1 + )" + rc=$? # not `|| rc=$?` -- see the note on the first main() test above + assert_eq "daemon never returns: rc 1" "1" "$rc" + assert_contains "daemon never returns: warns it did not return" "$out" "did not return" +} +test_main_daemon_never_returns + printf '\n=== summary ===\n' teardown_tmp printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"