#!/usr/bin/env bash # docker-nuke — capture diagnostics from a wedged Docker Desktop, then # force-kill and restart it. # # Written for bash 3.2 compatibility (macOS /bin/bash). set -euo pipefail # --- configuration ---------------------------------------------------------- # # The SC2034 directives below are per-line on purpose, not file-level: each # constant is consumed by a later task, and its directive is deleted at that # point. A file-level disable would silence SC2034 for the whole file forever, # hiding a genuinely dead variable added later. By the final task none of these # directives should remain. HEALTH_TIMEOUT=6 # seconds to wait on `docker info` before calling it wedged GRACEFUL_WAIT=8 # seconds to let Docker quit politely TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL # Bounded number of "fresh docker_pids -> SIGKILL -> settle" rounds run if a # post-SIGKILL scan still finds live Docker processes. A supervisor such as # com.docker.backend can respawn a killed helper under a brand new pid the # entry snapshot never knew about, so this must be a real cap, never an # unbounded loop, or a respawning Docker could hang this tool forever. KILL_RETRY_ROUNDS=3 # shellcheck disable=SC2034 # consumed by Task 8's main() when it calls wait_for_daemon "$RESTART_TIMEOUT"; not yet referenced by this task's code RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back POLL_INTERVAL=2 # seconds between daemon health polls SAMPLE_DURATION=2 # seconds of stack sampling per process SAMPLE_TIMEOUT=10 # hard cap on a single `sample` invocation DUMP_RETENTION=10 # how many dump directories to keep LOG_TAIL_LINES=2000 # lines of Docker log to copy (keeps dumps small) LOG_CAPTURE_TIMEOUT=20 # hard cap on the whole capture_logs step CRASH_CAPTURE_TIMEOUT=10 # hard cap on the whole capture_crashes step DOCKER_APP="/Applications/Docker.app" # The bundle path above cannot substitute for the AppleScript app name below: # the running GUI application is the nested "Contents/MacOS/Docker # Desktop.app", whose CFBundleName is "Docker Desktop", not the top-level # bundle path. Both are genuinely needed; this constant exists only so the # two literal identities for the same app cannot drift apart silently. DOCKER_APP_NAME="Docker Desktop" DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log" DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw" CRASH_DIR="$HOME/Library/Logs/DiagnosticReports" # Overridable so tests never touch real state. DUMP_ROOT="${DOCKER_NUKE_DUMP_ROOT:-$HOME/.local/state/docker-nuke}" OPT_FORCE=0 OPT_NO_RESTART=0 OPT_DRY_RUN=0 OPT_DEEP=0 # --- helpers ---------------------------------------------------------------- bin_exists() { command -v "$1" >/dev/null 2>&1; } # Run a command with a hard time limit. Returns 124 if it timed out. # Call inside a conditional so `set -e` does not abort on a non-zero return. run_timeout() { local secs="$1" shift # A shell function (e.g. disk_report) has no PATH-resolvable executable, so # an external timeout/gtimeout binary can never exec it directly (execvp # fails with rc=127). Route those straight to the pure-bash poller below, # which runs the command in this same shell instead of a child process. if bin_exists timeout && ! declare -F "$1" >/dev/null 2>&1; then timeout "$secs" "$@" return $? fi if bin_exists gtimeout && ! declare -F "$1" >/dev/null 2>&1; then gtimeout "$secs" "$@" return $? fi # Pure-bash fallback: run in background and poll. "$@" & local pid=$! local waited=0 while kill -0 "$pid" 2>/dev/null; do if [ "$waited" -ge "$secs" ]; then kill -9 "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true return 124 fi sleep 1 waited=$((waited + 1)) done local rc=0 wait "$pid" || rc=$? return "$rc" } # Exact process names to target, in addition to anything under Docker.app. # NOTE: never use `pgrep -f docker` here — it matches this script's own # command line and the tool would kill itself. DOCKER_PROC_NAMES="com.docker.backend com.docker.build com.docker.krun \ com.docker.vpnkit com.docker.virtualization vpnkit docker" docker_pids() { local self="$$" local parent="${PPID:-0}" local name pid comm { # `pgrep -f "$DOCKER_APP"` matches any process whose *command line* # mentions the bundle path — e.g. `tail -f /Applications/Docker.app/...` # matches even though that process is plainly not Docker. Post-filter # each candidate by its actual executable so only processes truly # launched from the bundle survive. Wrapped in `|| true`: under # `set -o pipefail`, an empty `pgrep -f` match (exit 1) would otherwise # make this whole statement fail and abort the function under `set -e`. pgrep -f "$DOCKER_APP" 2>/dev/null | while read -r pid; do if [ -z "$pid" ]; then continue; fi # A dead PID (process exited between pgrep and ps) or a `ps` failure # must yield an empty comm, which matches no case branch below — # never a crash, never a false positive. # NOTE: `ps -o comm=` reports the executable path as invoked, not # canonicalized — a Docker helper ever exec'd by relative path or bare # name would report a non-bundle comm here and be silently dropped. comm="$(ps -o comm= -p "$pid" 2>/dev/null)" || true case "$comm" in "$DOCKER_APP"/*) printf '%s\n' "$pid" ;; esac done || true # Exact-name matches are already precise — do NOT apply the comm # filter here. One job of this branch is catching a hung # /usr/local/bin/docker CLI, which legitimately lives outside the # bundle; filtering by $DOCKER_APP prefix would break that. for name in $DOCKER_PROC_NAMES; do pgrep -x "$name" 2>/dev/null || true done } | sort -un | while read -r pid; do if [ -z "$pid" ]; then continue; fi if [ "$pid" = "$self" ]; then continue; fi if [ "$pid" = "$parent" ]; then continue; fi printf '%s\n' "$pid" done } # Returns 0 if the daemon answers within HEALTH_TIMEOUT, else non-zero # (124 specifically means it hung rather than errored). docker_healthy() { local rc=0 run_timeout "$HEALTH_TIMEOUT" docker info >/dev/null 2>&1 || rc=$? return "$rc" } DUMP_DIR="" CAPTURE_FAILURES="" # new_dump_dir — create and print a fresh timestamped directory under # DUMP_ROOT. This is the one place in the dump path that fails loudly: if the # directory cannot be created there is nowhere left to record a failure, so # the caller must find out immediately rather than have every later capture # silently write into a path that never existed. new_dump_dir() { local stamp dir stamp="$(date -u '+%Y%m%dT%H%M%SZ')" dir="$DUMP_ROOT/$stamp" if ! mkdir -p "$dir" 2>/dev/null; then warn "docker-nuke: could not create dump directory $dir (dump root unwritable or full)" return 1 fi printf '%s\n' "$dir" } # capture NAME SECONDS CMD... — write CMD output to $DUMP_DIR/NAME. # Never fails: a timeout or error is recorded and execution continues, so a # broken capture can never block recovery. capture() { local name="$1" secs="$2" shift 2 local out="$DUMP_DIR/$name" local rc=0 run_timeout "$secs" "$@" >"$out" 2>&1 || rc=$? if [ "$rc" -ne 0 ]; then # The banner append itself must never be able to trip set -e: on an # unwritable or full dump root this redirection fails too, and without # the guard that failure -- not the original capture failure -- would be # what aborts the entire dump at the very first capture. printf '\n[docker-nuke] capture "%s" failed or timed out (rc=%s)\n' "$name" "$rc" >>"$out" 2>/dev/null || true CAPTURE_FAILURES="$CAPTURE_FAILURES $name" fi return 0 } # Sample user-space stacks of the daemon-side processes. This is the one # signal that cannot be reconstructed after the processes are killed. capture_samples() { if ! bin_exists sample; then printf 'sample(1) not available\n' >"$DUMP_DIR/samples-skipped.txt" return 0 fi local name pid for name in com.docker.backend com.docker.krun; do for pid in $(pgrep -x "$name" 2>/dev/null || true); do capture "sample-$name-$pid.txt" "$SAMPLE_TIMEOUT" \ sample "$pid" "$SAMPLE_DURATION" -mayDie done done } capture_logs() { mkdir -p "$DUMP_DIR/logs" local f base for f in "$DOCKER_LOG_DIR"/host/*.log "$DOCKER_LOG_DIR"/vm/*.log; do if [ -f "$f" ]; then base="$(basename "$f")" tail -n "$LOG_TAIL_LINES" "$f" >"$DUMP_DIR/logs/$base" 2>/dev/null || true fi done } capture_crashes() { mkdir -p "$DUMP_DIR/crashes" if [ ! -d "$CRASH_DIR" ]; then return 0 fi local f # grep exits 1 when no crash report matches (the common case: this dump # feature is expected to run against a healthy history most of the time). # Under `set -o pipefail` that would fail the whole pipeline and abort # run_dump via `set -e`. Group it with `|| true` so a clean "no matches" # never propagates as a pipeline failure. # # Grouping only the grep is not enough on its own: `head -5` closes its # read end once it has 5 lines, so with more than 5 matches upstream # `ls -1t` is later killed by SIGPIPE (rc=141) instead of exiting 0; an # unreadable CRASH_DIR gives `ls` rc=2 the same way. Either propagates as # this pipeline's exit status under pipefail and would abort the dump via # `set -e`, so the trailing `|| true` guards the pipeline as a whole, not # just the grep. # shellcheck disable=SC2012 # crash report names are macOS-generated, never contain newlines; ls -t gives the mtime order find cannot without extra plumbing ls -1t "$CRASH_DIR" 2>/dev/null | { grep -i docker || true; } | head -5 | while IFS= read -r f; do cp "$CRASH_DIR/$f" "$DUMP_DIR/crashes/" 2>/dev/null || true done || true } disk_report() { df -h printf '\n--- Docker.raw ---\n' ls -lh "$DOCKER_RAW" 2>/dev/null || printf 'Docker.raw not found\n' du -h -d0 "$DOCKER_RAW" 2>/dev/null || true } socket_report() { local sock="$HOME/.docker/run/docker.sock" printf 'socket: %s\n' "$sock" ls -l "$sock" 2>/dev/null || printf 'socket does not exist\n' printf '\n--- listeners ---\n' lsof "$sock" 2>/dev/null || printf 'no processes hold the socket\n' } run_dump() { local rc # DUMP_DIR is reset to empty first so a failed new_dump_dir never leaves it # pointing at a directory that does not exist (Fix 1). new_dump_dir is the # one place in this path that is allowed to fail loudly: on an unwritable # or full dump root it has already logged why via `warn` and returned # non-zero, and there is nowhere left to record a capture failure, so # run_dump stops here -- before any capture runs -- rather than limping # into the first capture and dying mid-dump. DUMP_DIR="" DUMP_DIR="$(new_dump_dir)" || return 1 CAPTURE_FAILURES="" log "docker-nuke: capturing diagnostics to $DUMP_DIR" capture "ps.txt" 10 ps -eo pid,ppid,stat,%cpu,%mem,etime,command capture "docker-info.txt" "$HEALTH_TIMEOUT" docker info capture "docker-version.txt" "$HEALTH_TIMEOUT" docker version capture "disk.txt" 15 disk_report capture "memory.txt" 10 vm_stat capture "socket.txt" 10 socket_report # capture_logs and capture_crashes write files directly rather than # emitting stdout, so they are not routed through capture() (which would # redirect stdout to a single named artifact). They still must be # time-bounded like every other step, and run_timeout's `declare -F` guard # (see run_timeout above) routes them to the pure-bash poller since neither # is a PATH-resolvable executable. A timeout or error is recorded into # CAPTURE_FAILURES the same way capture() records one, so a wedged # filesystem here cannot hang the dump either. rc=0 run_timeout "$LOG_CAPTURE_TIMEOUT" capture_logs || rc=$? if [ "$rc" -ne 0 ]; then CAPTURE_FAILURES="$CAPTURE_FAILURES logs" fi rc=0 run_timeout "$CRASH_CAPTURE_TIMEOUT" capture_crashes || rc=$? if [ "$rc" -ne 0 ]; then CAPTURE_FAILURES="$CAPTURE_FAILURES crashes" fi capture_samples } prune_dumps() { local keep="${1:-$DUMP_RETENTION}" if [ ! -d "$DUMP_ROOT" ]; then return 0 fi local d # An unreadable DUMP_ROOT gives `ls` a non-zero exit, which -- since this # pipeline is prune_dumps' last statement and therefore its return value -- # would otherwise abort a `set -e` caller. The trailing `|| true` matches # the guard used in capture_crashes for the same class of problem; the # normal-path behavior (prune the oldest, keep $keep newest) is unchanged. # shellcheck disable=SC2012 # dump dir names are our own generated timestamps, never contain newlines; sort -r needs plain lines, not find -print0 ls -1 "$DUMP_ROOT" 2>/dev/null | sort -r | tail -n "+$((keep + 1))" | while IFS= read -r d; do if [ -n "$d" ]; then rm -rf "${DUMP_ROOT:?}/$d" fi done || true return 0 } pids_alive() { local pid alive="" stat for pid in "$@"; do if kill -0 "$pid" 2>/dev/null; then # kill -0 succeeds against a zombie (state Z, ): the process # table entry is still there but the process has already exited and is # only waiting for its parent to reap it. Treat that as dead so a slow # reaper can never manufacture a spurious "survived SIGKILL" report # (Fix 7). ps -o stat= can print extra flag letters after the primary # state letter (e.g. Z+), so match a leading Z rather than equality. stat="$(ps -o stat= -p "$pid" 2>/dev/null)" || true case "$stat" in Z*) : ;; *) alive="$alive $pid" ;; esac fi done printf '%s' "${alive# }" } kill_pids() { local sig="$1" shift local pid for pid in "$@"; do kill "-$sig" "$pid" 2>/dev/null || true done return 0 } # Returns 0 once every PID is gone, 1 if SECONDS elapses first. wait_for_exit() { local secs="$1" shift local waited=0 while [ "$waited" -lt "$secs" ]; do if [ -z "$(pids_alive "$@")" ]; then return 0 fi sleep 1 waited=$((waited + 1)) done if [ -z "$(pids_alive "$@")" ]; then return 0 fi return 1 } # Stop root launchd helpers. Only called with --deep; needs a cached sudo # credential. stop_privileged_helpers() { log "docker-nuke: stopping privileged helpers (sudo required)" # -n (non-interactive) never prompts: sudo writes "Password:" to stderr but # reads the answer from /dev/tty, so with stderr discarded and no cached # credential a plain `sudo` blocks forever with no visible prompt -- and by # the time this runs, Docker has already been SIGKILLed (Fix 3). -n instead # fails fast into the warning below. if ! sudo -n launchctl stop com.docker.vmnetd 2>/dev/null; then warn "docker-nuke: sudo -n failed for com.docker.vmnetd (no cached credential, or not permitted); skipping" fi if ! sudo -n launchctl stop com.docker.socket 2>/dev/null; then warn "docker-nuke: sudo -n failed for com.docker.socket (no cached credential, or not permitted); skipping" fi } # Three-stage escalation: polite quit, SIGTERM, SIGKILL. # # Success at every stage is decided from a FRESH docker_pids scan, never from # the entry snapshot below or from wait_for_exit's own return value (Fix 1). # wait_for_exit still runs against the snapshot at each stage for TIMING -- # it correctly answers "did the ones we signalled exit" -- but it cannot see # a Docker process that appeared after the snapshot was taken, so it can # never be trusted alone to answer "is Docker gone". Concretely: # com.docker.backend supervises helpers; SIGKILL order is numeric (sort -un # in docker_pids), unrelated to the supervision tree. A helper can be killed, # the still-alive backend respawns it under a NEW pid, then the backend # itself is killed. Every snapshot pid is now gone, but the respawned helper # is still running -- only a fresh docker_pids call can see it. nuke_docker() { local pids fresh pids="$(docker_pids | tr '\n' ' ')" if [ -z "$pids" ]; then log "docker-nuke: no Docker processes found" return 0 fi log "docker-nuke: asking Docker Desktop to quit" osascript -e 'quit app "'"$DOCKER_APP_NAME"'"' >/dev/null 2>&1 || true # shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here wait_for_exit "$GRACEFUL_WAIT" $pids || true fresh="$(docker_pids | tr '\n' ' ')" if [ -z "$fresh" ]; then log "docker-nuke: Docker exited cleanly" return 0 fi # Narrow to the subset still alive from the entry snapshot before # signalling (Fix 8): a pid that already exited during the graceful-quit # wait must never be re-signalled, at this stage or the next. local snapshot_alive # shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here snapshot_alive="$(pids_alive $pids)" log "docker-nuke: sending SIGTERM" # shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here kill_pids TERM $snapshot_alive # shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here wait_for_exit "$TERM_WAIT" $snapshot_alive || true fresh="$(docker_pids | tr '\n' ' ')" if [ -z "$fresh" ]; then log "docker-nuke: Docker exited after SIGTERM" return 0 fi log "docker-nuke: sending SIGKILL" local remaining # shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here remaining="$(pids_alive $snapshot_alive)" # shellcheck disable=SC2086 # $remaining must word-split into separate PID arguments here kill_pids KILL $remaining sleep 1 if [ "${OPT_DEEP:-0}" -eq 1 ]; then stop_privileged_helpers fi # Bounded respawn retry (Fix 1): if a fresh scan still finds live Docker # processes, SIGKILL exactly that fresh set, let it settle, and scan again. # Capped at KILL_RETRY_ROUNDS -- never an unbounded loop -- so a Docker # that keeps respawning helpers cannot hang this tool forever. local round=0 fresh="$(docker_pids | tr '\n' ' ')" while [ -n "$fresh" ]; do round=$((round + 1)) if [ "$round" -gt "$KILL_RETRY_ROUNDS" ]; then break fi warn "docker-nuke: Docker processes still present after SIGKILL, retrying (round $round of $KILL_RETRY_ROUNDS): $fresh" # shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here kill_pids KILL $fresh sleep 1 fresh="$(docker_pids | tr '\n' ' ')" done if [ -n "$fresh" ]; then warn "docker-nuke: processes survived SIGKILL: $fresh" return 1 fi log "docker-nuke: all Docker processes terminated" return 0 } start_docker() { log "docker-nuke: relaunching Docker Desktop" open -a Docker >/dev/null 2>&1 || open "$DOCKER_APP" >/dev/null 2>&1 || true } # Returns 0 once the daemon answers, 1 if SECONDS elapses first. # # The deadline is wall-clock, not a count of sleeps (Fix 2, review round 1). # docker_healthy can itself cost up to HEALTH_TIMEOUT per call (it runs # run_timeout "$HEALTH_TIMEOUT" docker info internally), but the old version # of this loop only ever accumulated POLL_INTERVAL in `waited`, silently # discounting docker_healthy's own cost every single iteration. Against a # genuinely wedged daemon that undercount compounds per iteration: with real # constants (HEALTH_TIMEOUT=6, POLL_INTERVAL=2) an 8s request measured 32s # wall clock (4x), and the RESTART_TIMEOUT=90 caller could overshoot to # roughly 6 minutes. Capturing a start time and checking `now - start` # before each docker_healthy call instead bounds the overshoot to at most # one in-flight docker_healthy call, never a per-iteration multiplier. wait_for_daemon() { local secs="$1" local start now start="$(date +%s)" while :; do now="$(date +%s)" if [ "$((now - start))" -ge "$secs" ]; then return 1 fi if docker_healthy; then return 0 fi sleep "$POLL_INTERVAL" done } # write_summary VERDICT ELAPSED — record the triage payload the user reads # after recovery. Must never abort the caller, on ANY path (Fix 1, review # round 1): a no-op when DUMP_DIR is unset, empty, or not a directory (e.g. # after new_dump_dir failed on a full disk) -- and the write itself is # guarded too, since the disk can fill AFTER new_dump_dir already created a # valid DUMP_DIR but BEFORE this function runs; an unwritable or full # DUMP_DIR at write time is exactly the scenario this tool exists to # survive. The redirection is nested inside an outer group so the outer # group's own 2>/dev/null is already in effect before the inner redirection # is attempted -- a single-level `{ ... } >file 2>/dev/null` does NOT # suppress bash's own "cannot create" diagnostic, since bash reports that # failure before the trailing 2>/dev/null on the same command takes effect. # A write failure is logged via warn and swallowed, never propagated. write_summary() { local verdict="$1" elapsed="$2" rc=0 local cf="${CAPTURE_FAILURES:-}" if [ -z "$cf" ]; then cf=" none"; fi if [ -z "${DUMP_DIR:-}" ] || [ ! -d "${DUMP_DIR:-}" ]; then return 0 fi { { printf 'docker-nuke summary\n' printf '===================\n\n' printf 'verdict: %s\n' "$verdict" printf 'elapsed seconds: %s\n' "$elapsed" printf 'dump directory: %s\n' "$DUMP_DIR" printf 'capture failures:%s\n' "$cf" printf '\nTriage hints\n------------\n' printf '* ps.txt STAT column: U or D means stuck in the kernel, not user space\n' printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n' printf '* sample-* user-space stacks showing where the daemon was stuck\n' printf '* logs/ Docker host and vm logs from just before the kill\n' } >"$DUMP_DIR/summary.txt" } 2>/dev/null || rc=$? if [ "$rc" -ne 0 ]; then warn "docker-nuke: could not write summary to $DUMP_DIR/summary.txt (dump directory unwritable or full)" fi return 0 } log() { printf '%s\n' "$*"; } warn() { printf '%s\n' "$*" >&2; } usage() { cat <<'EOF' docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first Usage: docker-nuke [options] Options: -f, --force Skip the confirmation shown when Docker looks healthy -n, --no-restart Kill Docker but do not relaunch it --dry-run Report what would be killed and dumped; change nothing --deep Also stop root launchd helpers (com.docker.vmnetd); needs sudo -h, --help Show this help Diagnostics are written to ~/.local/state/docker-nuke// before anything is killed. The most recent 10 dumps are kept. Exit codes: 0 Docker killed and confirmed responsive again 1 Daemon did not come back within the restart timeout 2 Usage error, or aborted at the healthy-daemon prompt EOF } parse_args() { OPT_FORCE=0 OPT_NO_RESTART=0 OPT_DRY_RUN=0 OPT_DEEP=0 while [ "$#" -gt 0 ]; do # shellcheck disable=SC2034 # OPT_FORCE, OPT_NO_RESTART, OPT_DRY_RUN are read by main once it is wired up. OPT_DEEP is already read by nuke_docker but stays under this same directive since it covers the whole case statement. case "$1" in -f|--force) OPT_FORCE=1 ;; -n|--no-restart) OPT_NO_RESTART=1 ;; --dry-run) OPT_DRY_RUN=1 ;; --deep) OPT_DEEP=1 ;; -h|--help) usage; return 3 ;; *) warn "docker-nuke: unknown option '$1'" usage >&2 return 2 ;; esac shift done return 0 } # --- entrypoint ------------------------------------------------------------- main() { local rc=0 parse_args "$@" || rc=$? if [ "$rc" -eq 3 ]; then return 0; fi if [ "$rc" -ne 0 ]; then return "$rc"; fi log "docker-nuke: not yet implemented" return 0 } # Only run when executed, not when sourced by the test harness. if [ "${BASH_SOURCE[0]}" = "${0}" ]; then main "$@" fi