mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
feat(docker-nuke): add restart, daemon polling, and triage summary
This commit is contained in:
David Ibia
+76
-2
@@ -23,9 +23,8 @@ TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL
|
||||
# entry snapshot never knew about, so this must be a real cap, never an
|
||||
# unbounded loop, or a respawning Docker could hang this tool forever.
|
||||
KILL_RETRY_ROUNDS=3
|
||||
# shellcheck disable=SC2034
|
||||
# shellcheck disable=SC2034 # consumed by Task 8's main() when it calls wait_for_daemon "$RESTART_TIMEOUT"; not yet referenced by this task's code
|
||||
RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back
|
||||
# shellcheck disable=SC2034
|
||||
POLL_INTERVAL=2 # seconds between daemon health polls
|
||||
SAMPLE_DURATION=2 # seconds of stack sampling per process
|
||||
SAMPLE_TIMEOUT=10 # hard cap on a single `sample` invocation
|
||||
@@ -470,6 +469,81 @@ nuke_docker() {
|
||||
return 0
|
||||
}
|
||||
|
||||
start_docker() {
|
||||
log "docker-nuke: relaunching Docker Desktop"
|
||||
open -a Docker >/dev/null 2>&1 || open "$DOCKER_APP" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# Returns 0 once the daemon answers, 1 if SECONDS elapses first.
|
||||
#
|
||||
# The deadline is wall-clock, not a count of sleeps (Fix 2, review round 1).
|
||||
# docker_healthy can itself cost up to HEALTH_TIMEOUT per call (it runs
|
||||
# run_timeout "$HEALTH_TIMEOUT" docker info internally), but the old version
|
||||
# of this loop only ever accumulated POLL_INTERVAL in `waited`, silently
|
||||
# discounting docker_healthy's own cost every single iteration. Against a
|
||||
# genuinely wedged daemon that undercount compounds per iteration: with real
|
||||
# constants (HEALTH_TIMEOUT=6, POLL_INTERVAL=2) an 8s request measured 32s
|
||||
# wall clock (4x), and the RESTART_TIMEOUT=90 caller could overshoot to
|
||||
# roughly 6 minutes. Capturing a start time and checking `now - start`
|
||||
# before each docker_healthy call instead bounds the overshoot to at most
|
||||
# one in-flight docker_healthy call, never a per-iteration multiplier.
|
||||
wait_for_daemon() {
|
||||
local secs="$1"
|
||||
local start now
|
||||
start="$(date +%s)"
|
||||
while :; do
|
||||
now="$(date +%s)"
|
||||
if [ "$((now - start))" -ge "$secs" ]; then
|
||||
return 1
|
||||
fi
|
||||
if docker_healthy; then
|
||||
return 0
|
||||
fi
|
||||
sleep "$POLL_INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
# write_summary VERDICT ELAPSED — record the triage payload the user reads
|
||||
# after recovery. Must never abort the caller, on ANY path (Fix 1, review
|
||||
# round 1): a no-op when DUMP_DIR is unset, empty, or not a directory (e.g.
|
||||
# after new_dump_dir failed on a full disk) -- and the write itself is
|
||||
# guarded too, since the disk can fill AFTER new_dump_dir already created a
|
||||
# valid DUMP_DIR but BEFORE this function runs; an unwritable or full
|
||||
# DUMP_DIR at write time is exactly the scenario this tool exists to
|
||||
# survive. The redirection is nested inside an outer group so the outer
|
||||
# group's own 2>/dev/null is already in effect before the inner redirection
|
||||
# is attempted -- a single-level `{ ... } >file 2>/dev/null` does NOT
|
||||
# suppress bash's own "cannot create" diagnostic, since bash reports that
|
||||
# failure before the trailing 2>/dev/null on the same command takes effect.
|
||||
# A write failure is logged via warn and swallowed, never propagated.
|
||||
write_summary() {
|
||||
local verdict="$1" elapsed="$2" rc=0
|
||||
local cf="${CAPTURE_FAILURES:-}"
|
||||
if [ -z "$cf" ]; then cf=" none"; fi
|
||||
if [ -z "${DUMP_DIR:-}" ] || [ ! -d "${DUMP_DIR:-}" ]; then
|
||||
return 0
|
||||
fi
|
||||
{
|
||||
{
|
||||
printf 'docker-nuke summary\n'
|
||||
printf '===================\n\n'
|
||||
printf 'verdict: %s\n' "$verdict"
|
||||
printf 'elapsed seconds: %s\n' "$elapsed"
|
||||
printf 'dump directory: %s\n' "$DUMP_DIR"
|
||||
printf 'capture failures:%s\n' "$cf"
|
||||
printf '\nTriage hints\n------------\n'
|
||||
printf '* ps.txt STAT column: U or D means stuck in the kernel, not user space\n'
|
||||
printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n'
|
||||
printf '* sample-* user-space stacks showing where the daemon was stuck\n'
|
||||
printf '* logs/ Docker host and vm logs from just before the kill\n'
|
||||
} >"$DUMP_DIR/summary.txt"
|
||||
} 2>/dev/null || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
warn "docker-nuke: could not write summary to $DUMP_DIR/summary.txt (dump directory unwritable or full)"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
log() { printf '%s\n' "$*"; }
|
||||
warn() { printf '%s\n' "$*" >&2; }
|
||||
|
||||
|
||||
@@ -785,6 +785,152 @@ test_nuke_docker() {
|
||||
}
|
||||
test_nuke_docker
|
||||
|
||||
printf '\n=== restart and summary ===\n'
|
||||
|
||||
test_restart_and_summary() {
|
||||
local stub="$TMPROOT/stub2" oldpath="$PATH" rc
|
||||
# DUMP_DIR and CAPTURE_FAILURES are mutated below like PATH is above.
|
||||
# Restore all three on the way out (Fix 4, review round 1): this is the
|
||||
# last test block in the file today, so nothing currently leaks, but
|
||||
# Task 8 appends its own block immediately after this one and would
|
||||
# silently inherit whatever non-empty DUMP_DIR/CAPTURE_FAILURES this test
|
||||
# left behind if they were not restored, exactly like PATH and
|
||||
# HEALTH_TIMEOUT already are elsewhere in this file.
|
||||
local old_dump_dir="$DUMP_DIR" old_capture_failures="$CAPTURE_FAILURES"
|
||||
|
||||
make_docker_stub "$stub" healthy
|
||||
PATH="$stub:$oldpath"
|
||||
rc=0; wait_for_daemon 6 || rc=$?
|
||||
assert_eq "wait_for_daemon returns 0 when healthy" "0" "$rc"
|
||||
|
||||
make_docker_stub "$stub" broken
|
||||
rc=0; wait_for_daemon 4 || rc=$?
|
||||
assert_eq "wait_for_daemon returns 1 when never healthy" "1" "$rc"
|
||||
PATH="$oldpath"
|
||||
|
||||
DUMP_DIR="$(new_dump_dir)"
|
||||
CAPTURE_FAILURES=" slow.txt"
|
||||
write_summary "recovered" "42"
|
||||
local s
|
||||
s="$(cat "$DUMP_DIR/summary.txt")"
|
||||
assert_contains "summary records the verdict" "$s" "recovered"
|
||||
assert_contains "summary records elapsed time" "$s" "42"
|
||||
assert_contains "summary records capture failures" "$s" "slow.txt"
|
||||
assert_not_contains "capture failures line has no double space (Fix 5)" "$s" "failures: "
|
||||
|
||||
DUMP_DIR="$old_dump_dir"
|
||||
CAPTURE_FAILURES="$old_capture_failures"
|
||||
}
|
||||
test_restart_and_summary
|
||||
|
||||
printf '\n=== wait_for_daemon wall-clock deadline (Fix 2) ===\n'
|
||||
|
||||
test_wait_for_daemon_deadline() {
|
||||
# Regression coverage for the headline wait_for_daemon fix: the old loop
|
||||
# counted only sleeps, never docker_healthy's own cost, so against a
|
||||
# daemon that never answers it silently overshoots the requested timeout
|
||||
# by roughly (HEALTH_TIMEOUT / POLL_INTERVAL)x. HEALTH_TIMEOUT and
|
||||
# POLL_INTERVAL are scaled down here (2s / 1s instead of the real 6s / 2s)
|
||||
# purely so this test stays fast; the ratio between them -- the thing that
|
||||
# actually exposes the bug -- is preserved.
|
||||
local stub="$TMPROOT/stub3" oldpath="$PATH" rc start elapsed
|
||||
local old_health="$HEALTH_TIMEOUT" old_poll="$POLL_INTERVAL"
|
||||
|
||||
make_docker_stub "$stub" hang
|
||||
PATH="$stub:$oldpath"
|
||||
HEALTH_TIMEOUT=2
|
||||
POLL_INTERVAL=1
|
||||
|
||||
start="$(date +%s)"
|
||||
rc=0; wait_for_daemon 4 || rc=$?
|
||||
elapsed=$(( $(date +%s) - start ))
|
||||
|
||||
assert_eq "wait_for_daemon returns 1 against a hung daemon" "1" "$rc"
|
||||
# Requested 4s. New deadline-based code bounds the overshoot to at most one
|
||||
# in-flight docker_healthy call (~2s here), landing around 6s. The old
|
||||
# counting-sleeps code discounted every docker_healthy call and would land
|
||||
# around 12s for these constants (see the manual revert-and-measure proof
|
||||
# run alongside this fix). 8s cleanly separates "new" from "old" while
|
||||
# leaving headroom for process-spawn/date-granularity jitter.
|
||||
if [ "$elapsed" -le 8 ]; then
|
||||
pass "wait_for_daemon honors the wall-clock deadline against a hung daemon (elapsed=${elapsed}s, requested=4s)"
|
||||
else
|
||||
fail "wait_for_daemon honors the wall-clock deadline against a hung daemon" \
|
||||
"elapsed=${elapsed}s, requested=4s, expected <=8s"
|
||||
fi
|
||||
|
||||
HEALTH_TIMEOUT="$old_health"
|
||||
POLL_INTERVAL="$old_poll"
|
||||
PATH="$oldpath"
|
||||
}
|
||||
test_wait_for_daemon_deadline
|
||||
|
||||
printf '\n=== write_summary survives an unwritable DUMP_DIR (Fix 1) ===\n'
|
||||
|
||||
test_write_summary_survives() {
|
||||
# write_summary was previously only ever called with a fresh, valid,
|
||||
# writable DUMP_DIR (see test_restart_and_summary above), so the
|
||||
# "unset/missing -> no-op" branch -- the entire reason the guard at the
|
||||
# top of write_summary exists -- was never exercised, and the write
|
||||
# itself (the actual Fix 1 bug: an existing-but-now-unwritable-or-full
|
||||
# DUMP_DIR) had zero coverage at all. Each case below runs write_summary
|
||||
# in a subshell so a mutated DUMP_DIR/CAPTURE_FAILURES never escapes into
|
||||
# the rest of the suite.
|
||||
local rc out
|
||||
|
||||
rc=0
|
||||
out="$(
|
||||
unset DUMP_DIR CAPTURE_FAILURES
|
||||
set -u
|
||||
write_summary "recovered" "1"
|
||||
printf 'survived\n'
|
||||
)" || rc=$?
|
||||
assert_eq "write_summary no-ops when DUMP_DIR is unset (under set -u)" "0" "$rc"
|
||||
assert_contains "caller survives write_summary with DUMP_DIR unset" "$out" "survived"
|
||||
|
||||
rc=0
|
||||
out="$(
|
||||
DUMP_DIR=""
|
||||
write_summary "recovered" "1"
|
||||
printf 'survived\n'
|
||||
)" || rc=$?
|
||||
assert_eq "write_summary no-ops when DUMP_DIR is empty" "0" "$rc"
|
||||
assert_contains "caller survives write_summary with DUMP_DIR empty" "$out" "survived"
|
||||
|
||||
rc=0
|
||||
out="$(
|
||||
DUMP_DIR="$TMPROOT/write-summary-does-not-exist"
|
||||
write_summary "recovered" "1"
|
||||
printf 'survived\n'
|
||||
)" || rc=$?
|
||||
assert_eq "write_summary no-ops when DUMP_DIR does not exist" "0" "$rc"
|
||||
assert_contains "caller survives write_summary with DUMP_DIR missing" "$out" "survived"
|
||||
|
||||
# The actual Fix 1 bug: DUMP_DIR is a real, valid, existing directory (the
|
||||
# guard's "unset or not a directory" check passes it straight through) but
|
||||
# is unwritable -- isomorphic to the disk filling between new_dump_dir
|
||||
# succeeding and write_summary running. set -e is turned on explicitly
|
||||
# inside the subshell so this reproduces the real script's own
|
||||
# set -euo pipefail context, not just the harness's relaxed `set +e`.
|
||||
local ro_dir="$TMPROOT/write-summary-ro"
|
||||
mkdir -p "$ro_dir"
|
||||
chmod 555 "$ro_dir"
|
||||
rc=0
|
||||
out="$(
|
||||
set -e
|
||||
DUMP_DIR="$ro_dir"
|
||||
CAPTURE_FAILURES=""
|
||||
write_summary "recovered" "1"
|
||||
printf 'survived\n'
|
||||
)" || rc=$?
|
||||
chmod 755 "$ro_dir"
|
||||
assert_eq "write_summary returns 0 on an unwritable DUMP_DIR" "0" "$rc"
|
||||
assert_contains "caller survives write_summary on an unwritable DUMP_DIR (Fix 1)" "$out" "survived"
|
||||
assert_eq "unwritable DUMP_DIR: no summary.txt was created" "no" \
|
||||
"$([ -f "$ro_dir/summary.txt" ] && printf 'yes' || printf 'no')"
|
||||
}
|
||||
test_write_summary_survives
|
||||
|
||||
printf '\n=== summary ===\n'
|
||||
teardown_tmp
|
||||
printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"
|
||||
|
||||
Reference in New Issue
Block a user