feat(docker-nuke): add pre-kill diagnostic dump with retention

2026-10-24 08:23:19 +00:00 · 2026-07-19 23:50:13 +01:00
parent 49380dc87d
commit a49eb5adef
2 changed files with 237 additions and 10 deletions
@@ -23,25 +23,19 @@ TERM_WAIT=5             # seconds to wait after SIGTERM before SIGKILL
 RESTART_TIMEOUT=90      # seconds to wait for the daemon to come back
 # shellcheck disable=SC2034
 POLL_INTERVAL=2         # seconds between daemon health polls
-# shellcheck disable=SC2034
 SAMPLE_DURATION=2       # seconds of stack sampling per process
-# shellcheck disable=SC2034
 SAMPLE_TIMEOUT=10       # hard cap on a single `sample` invocation
-# shellcheck disable=SC2034
 DUMP_RETENTION=10       # how many dump directories to keep
-# shellcheck disable=SC2034
 LOG_TAIL_LINES=2000     # lines of Docker log to copy (keeps dumps small)
+LOG_CAPTURE_TIMEOUT=20  # hard cap on the whole capture_logs step
+CRASH_CAPTURE_TIMEOUT=10 # hard cap on the whole capture_crashes step

 DOCKER_APP="/Applications/Docker.app"
-# shellcheck disable=SC2034
 DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log"
-# shellcheck disable=SC2034
 DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw"
-# shellcheck disable=SC2034
 CRASH_DIR="$HOME/Library/Logs/DiagnosticReports"

 # Overridable so tests never touch real state.
-# shellcheck disable=SC2034
 DUMP_ROOT="${DOCKER_NUKE_DUMP_ROOT:-$HOME/.local/state/docker-nuke}"

 OPT_FORCE=0
@@ -58,11 +52,15 @@ bin_exists() { command -v "$1" >/dev/null 2>&1; }
 run_timeout() {
  local secs="$1"
  shift
-  if bin_exists timeout; then
+  # A shell function (e.g. disk_report) has no PATH-resolvable executable, so
+  # an external timeout/gtimeout binary can never exec it directly (execvp
+  # fails with rc=127). Route those straight to the pure-bash poller below,
+  # which runs the command in this same shell instead of a child process.
+  if bin_exists timeout && ! declare -F "$1" >/dev/null 2>&1; then
    timeout "$secs" "$@"
    return $?
  fi
-  if bin_exists gtimeout; then
+  if bin_exists gtimeout && ! declare -F "$1" >/dev/null 2>&1; then
    gtimeout "$secs" "$@"
    return $?
  fi
@@ -140,6 +138,176 @@ docker_healthy() {
  return "$rc"
 }

+DUMP_DIR=""
+CAPTURE_FAILURES=""
+
+# new_dump_dir — create and print a fresh timestamped directory under
+# DUMP_ROOT. This is the one place in the dump path that fails loudly: if the
+# directory cannot be created there is nowhere left to record a failure, so
+# the caller must find out immediately rather than have every later capture
+# silently write into a path that never existed.
+new_dump_dir() {
+  local stamp dir
+  stamp="$(date -u '+%Y%m%dT%H%M%SZ')"
+  dir="$DUMP_ROOT/$stamp"
+  if ! mkdir -p "$dir" 2>/dev/null; then
+    warn "docker-nuke: could not create dump directory $dir (dump root unwritable or full)"
+    return 1
+  fi
+  printf '%s\n' "$dir"
+}
+
+# capture NAME SECONDS CMD... — write CMD output to $DUMP_DIR/NAME.
+# Never fails: a timeout or error is recorded and execution continues, so a
+# broken capture can never block recovery.
+capture() {
+  local name="$1" secs="$2"
+  shift 2
+  local out="$DUMP_DIR/$name"
+  local rc=0
+  run_timeout "$secs" "$@" >"$out" 2>&1 || rc=$?
+  if [ "$rc" -ne 0 ]; then
+    # The banner append itself must never be able to trip set -e: on an
+    # unwritable or full dump root this redirection fails too, and without
+    # the guard that failure -- not the original capture failure -- would be
+    # what aborts the entire dump at the very first capture.
+    printf '\n[docker-nuke] capture "%s" failed or timed out (rc=%s)\n' "$name" "$rc" >>"$out" 2>/dev/null || true
+    CAPTURE_FAILURES="$CAPTURE_FAILURES $name"
+  fi
+  return 0
+}
+
+# Sample user-space stacks of the daemon-side processes. This is the one
+# signal that cannot be reconstructed after the processes are killed.
+capture_samples() {
+  if ! bin_exists sample; then
+    printf 'sample(1) not available\n' >"$DUMP_DIR/samples-skipped.txt"
+    return 0
+  fi
+  local name pid
+  for name in com.docker.backend com.docker.krun; do
+    for pid in $(pgrep -x "$name" 2>/dev/null || true); do
+      capture "sample-$name-$pid.txt" "$SAMPLE_TIMEOUT" \
+        sample "$pid" "$SAMPLE_DURATION" -mayDie
+    done
+  done
+}
+
+capture_logs() {
+  mkdir -p "$DUMP_DIR/logs"
+  local f base
+  for f in "$DOCKER_LOG_DIR"/host/*.log "$DOCKER_LOG_DIR"/vm/*.log; do
+    if [ -f "$f" ]; then
+      base="$(basename "$f")"
+      tail -n "$LOG_TAIL_LINES" "$f" >"$DUMP_DIR/logs/$base" 2>/dev/null || true
+    fi
+  done
+}
+
+capture_crashes() {
+  mkdir -p "$DUMP_DIR/crashes"
+  if [ ! -d "$CRASH_DIR" ]; then
+    return 0
+  fi
+  local f
+  # grep exits 1 when no crash report matches (the common case: this dump
+  # feature is expected to run against a healthy history most of the time).
+  # Under `set -o pipefail` that would fail the whole pipeline and abort
+  # run_dump via `set -e`. Group it with `|| true` so a clean "no matches"
+  # never propagates as a pipeline failure.
+  #
+  # Grouping only the grep is not enough on its own: `head -5` closes its
+  # read end once it has 5 lines, so with more than 5 matches upstream
+  # `ls -1t` is later killed by SIGPIPE (rc=141) instead of exiting 0; an
+  # unreadable CRASH_DIR gives `ls` rc=2 the same way. Either propagates as
+  # this pipeline's exit status under pipefail and would abort the dump via
+  # `set -e`, so the trailing `|| true` guards the pipeline as a whole, not
+  # just the grep.
+  # shellcheck disable=SC2012  # crash report names are macOS-generated, never contain newlines; ls -t gives the mtime order find cannot without extra plumbing
+  ls -1t "$CRASH_DIR" 2>/dev/null | { grep -i docker || true; } | head -5 | while IFS= read -r f; do
+    cp "$CRASH_DIR/$f" "$DUMP_DIR/crashes/" 2>/dev/null || true
+  done || true
+}
+
+disk_report() {
+  df -h
+  printf '\n--- Docker.raw ---\n'
+  ls -lh "$DOCKER_RAW" 2>/dev/null || printf 'Docker.raw not found\n'
+  du -h -d0 "$DOCKER_RAW" 2>/dev/null || true
+}
+
+socket_report() {
+  local sock="$HOME/.docker/run/docker.sock"
+  printf 'socket: %s\n' "$sock"
+  ls -l "$sock" 2>/dev/null || printf 'socket does not exist\n'
+  printf '\n--- listeners ---\n'
+  lsof "$sock" 2>/dev/null || printf 'no processes hold the socket\n'
+}
+
+run_dump() {
+  local rc
+  # DUMP_DIR is reset to empty first so a failed new_dump_dir never leaves it
+  # pointing at a directory that does not exist (Fix 1). new_dump_dir is the
+  # one place in this path that is allowed to fail loudly: on an unwritable
+  # or full dump root it has already logged why via `warn` and returned
+  # non-zero, and there is nowhere left to record a capture failure, so
+  # run_dump stops here -- before any capture runs -- rather than limping
+  # into the first capture and dying mid-dump.
+  DUMP_DIR=""
+  DUMP_DIR="$(new_dump_dir)" || return 1
+  CAPTURE_FAILURES=""
+  log "docker-nuke: capturing diagnostics to $DUMP_DIR"
+
+  capture "ps.txt" 10 ps -eo pid,ppid,stat,%cpu,%mem,etime,command
+  capture "docker-info.txt" "$HEALTH_TIMEOUT" docker info
+  capture "docker-version.txt" "$HEALTH_TIMEOUT" docker version
+  capture "disk.txt" 15 disk_report
+  capture "memory.txt" 10 vm_stat
+  capture "socket.txt" 10 socket_report
+
+  # capture_logs and capture_crashes write files directly rather than
+  # emitting stdout, so they are not routed through capture() (which would
+  # redirect stdout to a single named artifact). They still must be
+  # time-bounded like every other step, and run_timeout's `declare -F` guard
+  # (see run_timeout above) routes them to the pure-bash poller since neither
+  # is a PATH-resolvable executable. A timeout or error is recorded into
+  # CAPTURE_FAILURES the same way capture() records one, so a wedged
+  # filesystem here cannot hang the dump either.
+  rc=0
+  run_timeout "$LOG_CAPTURE_TIMEOUT" capture_logs || rc=$?
+  if [ "$rc" -ne 0 ]; then
+    CAPTURE_FAILURES="$CAPTURE_FAILURES logs"
+  fi
+
+  rc=0
+  run_timeout "$CRASH_CAPTURE_TIMEOUT" capture_crashes || rc=$?
+  if [ "$rc" -ne 0 ]; then
+    CAPTURE_FAILURES="$CAPTURE_FAILURES crashes"
+  fi
+
+  capture_samples
+}
+
+prune_dumps() {
+  local keep="${1:-$DUMP_RETENTION}"
+  if [ ! -d "$DUMP_ROOT" ]; then
+    return 0
+  fi
+  local d
+  # An unreadable DUMP_ROOT gives `ls` a non-zero exit, which -- since this
+  # pipeline is prune_dumps' last statement and therefore its return value --
+  # would otherwise abort a `set -e` caller. The trailing `|| true` matches
+  # the guard used in capture_crashes for the same class of problem; the
+  # normal-path behavior (prune the oldest, keep $keep newest) is unchanged.
+  # shellcheck disable=SC2012  # dump dir names are our own generated timestamps, never contain newlines; sort -r needs plain lines, not find -print0
+  ls -1 "$DUMP_ROOT" 2>/dev/null | sort -r | tail -n "+$((keep + 1))" | while IFS= read -r d; do
+    if [ -n "$d" ]; then
+      rm -rf "${DUMP_ROOT:?}/$d"
+    fi
+  done || true
+  return 0
+}
+
 log()  { printf '%s\n' "$*"; }
 warn() { printf '%s\n' "$*" >&2; }

@@ -344,6 +344,65 @@ test_docker_healthy() {
 }
 test_docker_healthy

+printf '\n=== dump ===\n'
+
+test_dump() {
+  local d rc
+
+  d="$(new_dump_dir)"
+  assert_eq "dump dir is created" "yes" "$([ -d "$d" ] && printf 'yes' || printf 'no')"
+  assert_contains "dump dir lives under DUMP_ROOT" "$d" "$DUMP_ROOT"
+
+  # shellcheck disable=SC2034  # DUMP_DIR is read by capture() (sourced from docker-nuke); the source=/dev/null directive above hides that cross-file read from this analysis
+  DUMP_DIR="$d"
+  CAPTURE_FAILURES=""
+
+  capture "ok.txt" 5 printf 'captured'
+  assert_eq "successful capture writes its file" "captured" "$(cat "$d/ok.txt")"
+  assert_eq "successful capture records no failure" "" "$CAPTURE_FAILURES"
+
+  rc=0
+  capture "slow.txt" 1 sleep 5 || rc=$?
+  assert_eq "timed-out capture still returns 0" "0" "$rc"
+  assert_contains "timed-out capture is recorded" "$CAPTURE_FAILURES" "slow.txt"
+  assert_contains "timed-out capture notes failure in file" "$(cat "$d/slow.txt")" "failed or timed out"
+
+  rc=0
+  capture "missing.txt" 5 this-command-does-not-exist || rc=$?
+  assert_eq "failed capture still returns 0" "0" "$rc"
+  assert_contains "failed capture is recorded" "$CAPTURE_FAILURES" "missing.txt"
+
+  # Regression coverage for the headline run_timeout fix: an external
+  # timeout/gtimeout binary can never exec a shell function directly (execvp
+  # fails with rc=127), so run_timeout must route shell functions to its
+  # pure-bash poller instead. Every capture test above passes an external
+  # command (printf, sleep, this-command-does-not-exist), so without this
+  # assertion that routing has zero coverage and could regress silently.
+  # shellcheck disable=SC2329  # invoked indirectly by capture -> run_timeout name lookup
+  that_function() { printf 'shell-function-output\n'; }
+  capture "fn.txt" 5 that_function
+  assert_contains "capture of a shell function produces its real output" \
+    "$(cat "$d/fn.txt")" "shell-function-output"
+  assert_not_contains "capture of a shell function is not a failure banner" \
+    "$(cat "$d/fn.txt")" "failed or timed out"
+
+  # Retention: 12 dumps in, 10 newest survive.
+  rm -rf "${DUMP_ROOT:?}"
+  mkdir -p "$DUMP_ROOT"
+  local i
+  for i in 01 02 03 04 05 06 07 08 09 10 11 12; do
+    mkdir -p "$DUMP_ROOT/202601${i}T000000Z"
+  done
+  prune_dumps 10
+  # shellcheck disable=SC2012  # dump dir names are our own generated timestamps, never contain newlines; a plain count needs plain lines, not find -print0
+  assert_eq "keeps exactly 10 dumps" "10" "$(ls -1 "$DUMP_ROOT" | wc -l | tr -d ' ')"
+  assert_eq "prunes the oldest" "no" \
+    "$([ -d "$DUMP_ROOT/20260101T000000Z" ] && printf 'yes' || printf 'no')"
+  assert_eq "keeps the newest" "yes" \
+    "$([ -d "$DUMP_ROOT/20260112T000000Z" ] && printf 'yes' || printf 'no')"
+}
+test_dump
+
 printf '\n=== summary ===\n'
 teardown_tmp
 printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"