mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
David Ibia
74a718efa8
The failure path advised --deep and a disk-space check regardless of cause. On the first live run that was wrong on both counts: 101 GiB were free and --deep was irrelevant. Docker Desktop had crashed on startup, and the actual fix was launching it from the Dock. Branch the advice on whether any Docker process survived the relaunch. No survivors means the app did not stay up, which no programmatic relaunch can work around; survivors mean it is up but not answering yet.
821 lines
32 KiB
Bash
Executable File
821 lines
32 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# docker-nuke — capture diagnostics from a wedged Docker Desktop, then
|
|
# force-kill and restart it.
|
|
#
|
|
# Written for bash 3.2 compatibility (macOS /bin/bash).
|
|
|
|
set -euo pipefail
|
|
|
|
# --- configuration ----------------------------------------------------------
|
|
#
|
|
# The SC2034 directives below are per-line on purpose, not file-level: each
|
|
# constant is consumed by a later task, and its directive is deleted at that
|
|
# point. A file-level disable would silence SC2034 for the whole file forever,
|
|
# hiding a genuinely dead variable added later. By the final task none of these
|
|
# directives should remain.
|
|
|
|
HEALTH_TIMEOUT=6 # seconds to wait on `docker info` before calling it wedged
|
|
GRACEFUL_WAIT=8 # seconds to let Docker quit politely
|
|
TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL
|
|
# Bounded number of "fresh docker_pids -> SIGKILL -> settle" rounds run if a
|
|
# post-SIGKILL scan still finds live Docker processes. A supervisor such as
|
|
# com.docker.backend can respawn a killed helper under a brand new pid the
|
|
# entry snapshot never knew about, so this must be a real cap, never an
|
|
# unbounded loop, or a respawning Docker could hang this tool forever.
|
|
KILL_RETRY_ROUNDS=3
|
|
RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back
|
|
POLL_INTERVAL=2 # seconds between daemon health polls
|
|
SAMPLE_DURATION=2 # seconds of stack sampling per process
|
|
SAMPLE_TIMEOUT=10 # hard cap on a single `sample` invocation
|
|
DUMP_RETENTION=10 # how many dump directories to keep
|
|
LOG_TAIL_LINES=2000 # lines of Docker log to copy (keeps dumps small)
|
|
LOG_CAPTURE_TIMEOUT=20 # hard cap on the whole capture_logs step
|
|
CRASH_CAPTURE_TIMEOUT=10 # hard cap on the whole capture_crashes step
|
|
|
|
DOCKER_APP="/Applications/Docker.app"
|
|
# The bundle path above cannot substitute for the AppleScript app name below:
|
|
# the running GUI application is the nested "Contents/MacOS/Docker
|
|
# Desktop.app", whose CFBundleName is "Docker Desktop", not the top-level
|
|
# bundle path. Both are genuinely needed; this constant exists only so the
|
|
# two literal identities for the same app cannot drift apart silently.
|
|
DOCKER_APP_NAME="Docker Desktop"
|
|
DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log"
|
|
DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw"
|
|
CRASH_DIR="$HOME/Library/Logs/DiagnosticReports"
|
|
|
|
# Overridable so tests never touch real state.
|
|
DUMP_ROOT="${DOCKER_NUKE_DUMP_ROOT:-$HOME/.local/state/docker-nuke}"
|
|
|
|
OPT_FORCE=0
|
|
OPT_NO_RESTART=0
|
|
OPT_DRY_RUN=0
|
|
OPT_DEEP=0
|
|
|
|
# --- helpers ----------------------------------------------------------------
|
|
|
|
bin_exists() { command -v "$1" >/dev/null 2>&1; }
|
|
|
|
# Run a command with a hard time limit. Returns 124 if it timed out.
|
|
# Call inside a conditional so `set -e` does not abort on a non-zero return.
|
|
run_timeout() {
|
|
local secs="$1"
|
|
shift
|
|
# A shell function (e.g. disk_report) has no PATH-resolvable executable, so
|
|
# an external timeout/gtimeout binary can never exec it directly (execvp
|
|
# fails with rc=127). Route those straight to the pure-bash poller below,
|
|
# which runs the command in this same shell instead of a child process.
|
|
if bin_exists timeout && ! declare -F "$1" >/dev/null 2>&1; then
|
|
timeout "$secs" "$@"
|
|
return $?
|
|
fi
|
|
if bin_exists gtimeout && ! declare -F "$1" >/dev/null 2>&1; then
|
|
gtimeout "$secs" "$@"
|
|
return $?
|
|
fi
|
|
|
|
# Pure-bash fallback: run in background and poll.
|
|
"$@" &
|
|
local pid=$!
|
|
local waited=0
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
if [ "$waited" -ge "$secs" ]; then
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
return 124
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
local rc=0
|
|
wait "$pid" || rc=$?
|
|
return "$rc"
|
|
}
|
|
|
|
# Exact process names to target, in addition to anything under Docker.app.
|
|
# NOTE: never use `pgrep -f docker` here — it matches this script's own
|
|
# command line and the tool would kill itself.
|
|
DOCKER_PROC_NAMES="com.docker.backend com.docker.build com.docker.krun \
|
|
com.docker.vpnkit com.docker.virtualization vpnkit docker"
|
|
|
|
docker_pids() {
|
|
local self="$$"
|
|
local parent="${PPID:-0}"
|
|
local name pid comm
|
|
{
|
|
# `pgrep -f "$DOCKER_APP"` matches any process whose *command line*
|
|
# mentions the bundle path — e.g. `tail -f /Applications/Docker.app/...`
|
|
# matches even though that process is plainly not Docker. Post-filter
|
|
# each candidate by its actual executable so only processes truly
|
|
# launched from the bundle survive. Wrapped in `|| true`: under
|
|
# `set -o pipefail`, an empty `pgrep -f` match (exit 1) would otherwise
|
|
# make this whole statement fail and abort the function under `set -e`.
|
|
pgrep -f "$DOCKER_APP" 2>/dev/null | while read -r pid; do
|
|
if [ -z "$pid" ]; then continue; fi
|
|
# A dead PID (process exited between pgrep and ps) or a `ps` failure
|
|
# must yield an empty comm, which matches no case branch below —
|
|
# never a crash, never a false positive.
|
|
# NOTE: `ps -o comm=` reports the executable path as invoked, not
|
|
# canonicalized — a Docker helper ever exec'd by relative path or bare
|
|
# name would report a non-bundle comm here and be silently dropped.
|
|
comm="$(ps -o comm= -p "$pid" 2>/dev/null)" || true
|
|
case "$comm" in
|
|
"$DOCKER_APP"/*) printf '%s\n' "$pid" ;;
|
|
esac
|
|
done || true
|
|
|
|
# Exact-name matches are already precise — do NOT apply the comm
|
|
# filter here. One job of this branch is catching a hung
|
|
# /usr/local/bin/docker CLI, which legitimately lives outside the
|
|
# bundle; filtering by $DOCKER_APP prefix would break that.
|
|
for name in $DOCKER_PROC_NAMES; do
|
|
pgrep -x "$name" 2>/dev/null || true
|
|
done
|
|
} | sort -un | while read -r pid; do
|
|
if [ -z "$pid" ]; then continue; fi
|
|
if [ "$pid" = "$self" ]; then continue; fi
|
|
if [ "$pid" = "$parent" ]; then continue; fi
|
|
printf '%s\n' "$pid"
|
|
done
|
|
}
|
|
|
|
# Returns 0 if the daemon answers within HEALTH_TIMEOUT, else non-zero
|
|
# (124 specifically means it hung rather than errored).
|
|
docker_healthy() {
|
|
local rc=0
|
|
run_timeout "$HEALTH_TIMEOUT" docker info >/dev/null 2>&1 || rc=$?
|
|
return "$rc"
|
|
}
|
|
|
|
DUMP_DIR=""
|
|
CAPTURE_FAILURES=""
|
|
|
|
# new_dump_dir — create and print a fresh timestamped directory under
|
|
# DUMP_ROOT. This is the one place in the dump path that fails loudly: if the
|
|
# directory cannot be created there is nowhere left to record a failure, so
|
|
# the caller must find out immediately rather than have every later capture
|
|
# silently write into a path that never existed.
|
|
new_dump_dir() {
|
|
local stamp dir
|
|
stamp="$(date -u '+%Y%m%dT%H%M%SZ')"
|
|
dir="$DUMP_ROOT/$stamp"
|
|
if ! mkdir -p "$dir" 2>/dev/null; then
|
|
warn "docker-nuke: could not create dump directory $dir (dump root unwritable or full)"
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$dir"
|
|
}
|
|
|
|
# capture NAME SECONDS CMD... — write CMD output to $DUMP_DIR/NAME.
|
|
# Never fails: a timeout or error is recorded and execution continues, so a
|
|
# broken capture can never block recovery.
|
|
capture() {
|
|
local name="$1" secs="$2"
|
|
shift 2
|
|
local out="$DUMP_DIR/$name"
|
|
local rc=0
|
|
run_timeout "$secs" "$@" >"$out" 2>&1 || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
# The banner append itself must never be able to trip set -e: on an
|
|
# unwritable or full dump root this redirection fails too, and without
|
|
# the guard that failure -- not the original capture failure -- would be
|
|
# what aborts the entire dump at the very first capture.
|
|
printf '\n[docker-nuke] capture "%s" failed or timed out (rc=%s)\n' "$name" "$rc" >>"$out" 2>/dev/null || true
|
|
CAPTURE_FAILURES="$CAPTURE_FAILURES $name"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Sample user-space stacks of the daemon-side processes. This is the one
|
|
# signal that cannot be reconstructed after the processes are killed.
|
|
capture_samples() {
|
|
if ! bin_exists sample; then
|
|
printf 'sample(1) not available\n' >"$DUMP_DIR/samples-skipped.txt"
|
|
return 0
|
|
fi
|
|
local name pid
|
|
for name in com.docker.backend com.docker.krun; do
|
|
for pid in $(pgrep -x "$name" 2>/dev/null || true); do
|
|
capture "sample-$name-$pid.txt" "$SAMPLE_TIMEOUT" \
|
|
sample "$pid" "$SAMPLE_DURATION" -mayDie
|
|
done
|
|
done
|
|
}
|
|
|
|
capture_logs() {
|
|
mkdir -p "$DUMP_DIR/logs"
|
|
local f base
|
|
for f in "$DOCKER_LOG_DIR"/host/*.log "$DOCKER_LOG_DIR"/vm/*.log; do
|
|
if [ -f "$f" ]; then
|
|
base="$(basename "$f")"
|
|
tail -n "$LOG_TAIL_LINES" "$f" >"$DUMP_DIR/logs/$base" 2>/dev/null || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
capture_crashes() {
|
|
mkdir -p "$DUMP_DIR/crashes"
|
|
if [ ! -d "$CRASH_DIR" ]; then
|
|
return 0
|
|
fi
|
|
local f
|
|
# grep exits 1 when no crash report matches (the common case: this dump
|
|
# feature is expected to run against a healthy history most of the time).
|
|
# Under `set -o pipefail` that would fail the whole pipeline and abort
|
|
# run_dump via `set -e`. Group it with `|| true` so a clean "no matches"
|
|
# never propagates as a pipeline failure.
|
|
#
|
|
# Grouping only the grep is not enough on its own: `head -5` closes its
|
|
# read end once it has 5 lines, so with more than 5 matches upstream
|
|
# `ls -1t` is later killed by SIGPIPE (rc=141) instead of exiting 0; an
|
|
# unreadable CRASH_DIR gives `ls` rc=2 the same way. Either propagates as
|
|
# this pipeline's exit status under pipefail and would abort the dump via
|
|
# `set -e`, so the trailing `|| true` guards the pipeline as a whole, not
|
|
# just the grep.
|
|
# shellcheck disable=SC2012 # crash report names are macOS-generated, never contain newlines; ls -t gives the mtime order find cannot without extra plumbing
|
|
ls -1t "$CRASH_DIR" 2>/dev/null | { grep -i docker || true; } | head -5 | while IFS= read -r f; do
|
|
cp "$CRASH_DIR/$f" "$DUMP_DIR/crashes/" 2>/dev/null || true
|
|
done || true
|
|
}
|
|
|
|
disk_report() {
|
|
df -h
|
|
printf '\n--- Docker.raw ---\n'
|
|
ls -lh "$DOCKER_RAW" 2>/dev/null || printf 'Docker.raw not found\n'
|
|
du -h -d0 "$DOCKER_RAW" 2>/dev/null || true
|
|
}
|
|
|
|
socket_report() {
|
|
local sock="$HOME/.docker/run/docker.sock"
|
|
printf 'socket: %s\n' "$sock"
|
|
ls -l "$sock" 2>/dev/null || printf 'socket does not exist\n'
|
|
printf '\n--- listeners ---\n'
|
|
lsof "$sock" 2>/dev/null || printf 'no processes hold the socket\n'
|
|
}
|
|
|
|
run_dump() {
|
|
local rc
|
|
# DUMP_DIR is reset to empty first so a failed new_dump_dir never leaves it
|
|
# pointing at a directory that does not exist (Fix 1). new_dump_dir is the
|
|
# one place in this path that is allowed to fail loudly: on an unwritable
|
|
# or full dump root it has already logged why via `warn` and returned
|
|
# non-zero, and there is nowhere left to record a capture failure, so
|
|
# run_dump stops here -- before any capture runs -- rather than limping
|
|
# into the first capture and dying mid-dump.
|
|
DUMP_DIR=""
|
|
DUMP_DIR="$(new_dump_dir)" || return 1
|
|
CAPTURE_FAILURES=""
|
|
log "docker-nuke: capturing diagnostics to $DUMP_DIR"
|
|
|
|
capture "ps.txt" 10 ps -eo pid,ppid,stat,%cpu,%mem,etime,command
|
|
capture "docker-info.txt" "$HEALTH_TIMEOUT" docker info
|
|
capture "docker-version.txt" "$HEALTH_TIMEOUT" docker version
|
|
capture "disk.txt" 15 disk_report
|
|
capture "memory.txt" 10 vm_stat
|
|
capture "socket.txt" 10 socket_report
|
|
|
|
# capture_logs and capture_crashes write files directly rather than
|
|
# emitting stdout, so they are not routed through capture() (which would
|
|
# redirect stdout to a single named artifact). They still must be
|
|
# time-bounded like every other step, and run_timeout's `declare -F` guard
|
|
# (see run_timeout above) routes them to the pure-bash poller since neither
|
|
# is a PATH-resolvable executable. A timeout or error is recorded into
|
|
# CAPTURE_FAILURES the same way capture() records one, so a wedged
|
|
# filesystem here cannot hang the dump either.
|
|
rc=0
|
|
run_timeout "$LOG_CAPTURE_TIMEOUT" capture_logs || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
CAPTURE_FAILURES="$CAPTURE_FAILURES logs"
|
|
fi
|
|
|
|
rc=0
|
|
run_timeout "$CRASH_CAPTURE_TIMEOUT" capture_crashes || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
CAPTURE_FAILURES="$CAPTURE_FAILURES crashes"
|
|
fi
|
|
|
|
capture_samples
|
|
}
|
|
|
|
prune_dumps() {
|
|
local keep="${1:-$DUMP_RETENTION}"
|
|
if [ ! -d "$DUMP_ROOT" ]; then
|
|
return 0
|
|
fi
|
|
local d
|
|
# An unreadable DUMP_ROOT gives `ls` a non-zero exit, which -- since this
|
|
# pipeline is prune_dumps' last statement and therefore its return value --
|
|
# would otherwise abort a `set -e` caller. The trailing `|| true` matches
|
|
# the guard used in capture_crashes for the same class of problem; the
|
|
# normal-path behavior (prune the oldest, keep $keep newest) is unchanged.
|
|
# shellcheck disable=SC2012 # dump dir names are our own generated timestamps, never contain newlines; sort -r needs plain lines, not find -print0
|
|
ls -1 "$DUMP_ROOT" 2>/dev/null | sort -r | tail -n "+$((keep + 1))" | while IFS= read -r d; do
|
|
if [ -n "$d" ]; then
|
|
rm -rf "${DUMP_ROOT:?}/$d"
|
|
fi
|
|
done || true
|
|
return 0
|
|
}
|
|
|
|
pids_alive() {
|
|
local pid alive="" stat
|
|
for pid in "$@"; do
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
# kill -0 succeeds against a zombie (state Z, <defunct>): the process
|
|
# table entry is still there but the process has already exited and is
|
|
# only waiting for its parent to reap it. Treat that as dead so a slow
|
|
# reaper can never manufacture a spurious "survived SIGKILL" report
|
|
# (Fix 7). ps -o stat= can print extra flag letters after the primary
|
|
# state letter (e.g. Z+), so match a leading Z rather than equality.
|
|
stat="$(ps -o stat= -p "$pid" 2>/dev/null)" || true
|
|
case "$stat" in
|
|
Z*) : ;;
|
|
*) alive="$alive $pid" ;;
|
|
esac
|
|
fi
|
|
done
|
|
printf '%s' "${alive# }"
|
|
}
|
|
|
|
kill_pids() {
|
|
local sig="$1"
|
|
shift
|
|
local pid
|
|
for pid in "$@"; do
|
|
kill "-$sig" "$pid" 2>/dev/null || true
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# Returns 0 once every PID is gone, 1 if SECONDS elapses first.
|
|
wait_for_exit() {
|
|
local secs="$1"
|
|
shift
|
|
local waited=0
|
|
while [ "$waited" -lt "$secs" ]; do
|
|
if [ -z "$(pids_alive "$@")" ]; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
if [ -z "$(pids_alive "$@")" ]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Stop root launchd helpers. Only called with --deep; needs a cached sudo
|
|
# credential.
|
|
stop_privileged_helpers() {
|
|
log "docker-nuke: stopping privileged helpers (sudo required)"
|
|
# -n (non-interactive) never prompts: sudo writes "Password:" to stderr but
|
|
# reads the answer from /dev/tty, so with stderr discarded and no cached
|
|
# credential a plain `sudo` blocks forever with no visible prompt -- and by
|
|
# the time this runs, Docker has already been SIGKILLed (Fix 3). -n instead
|
|
# fails fast into the warning below.
|
|
if ! sudo -n launchctl stop com.docker.vmnetd 2>/dev/null; then
|
|
warn "docker-nuke: sudo -n failed for com.docker.vmnetd (no cached credential, or not permitted); skipping"
|
|
fi
|
|
if ! sudo -n launchctl stop com.docker.socket 2>/dev/null; then
|
|
warn "docker-nuke: sudo -n failed for com.docker.socket (no cached credential, or not permitted); skipping"
|
|
fi
|
|
}
|
|
|
|
# Three-stage escalation: polite quit, SIGTERM, SIGKILL.
|
|
#
|
|
# Success at every stage is decided from a FRESH docker_pids scan, never from
|
|
# the entry snapshot below or from wait_for_exit's own return value (Fix 1).
|
|
# wait_for_exit still runs against the snapshot at each stage for TIMING --
|
|
# it correctly answers "did the ones we signalled exit" -- but it cannot see
|
|
# a Docker process that appeared after the snapshot was taken, so it can
|
|
# never be trusted alone to answer "is Docker gone". Concretely:
|
|
# com.docker.backend supervises helpers; SIGKILL order is numeric (sort -un
|
|
# in docker_pids), unrelated to the supervision tree. A helper can be killed,
|
|
# the still-alive backend respawns it under a NEW pid, then the backend
|
|
# itself is killed. Every snapshot pid is now gone, but the respawned helper
|
|
# is still running -- only a fresh docker_pids call can see it.
|
|
nuke_docker() {
|
|
local pids fresh
|
|
pids="$(docker_pids | tr '\n' ' ')"
|
|
if [ -z "$pids" ]; then
|
|
log "docker-nuke: no Docker processes found"
|
|
return 0
|
|
fi
|
|
|
|
log "docker-nuke: asking Docker Desktop to quit"
|
|
osascript -e 'quit app "'"$DOCKER_APP_NAME"'"' >/dev/null 2>&1 || true
|
|
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
|
wait_for_exit "$GRACEFUL_WAIT" $pids || true
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
if [ -z "$fresh" ]; then
|
|
log "docker-nuke: Docker exited cleanly"
|
|
return 0
|
|
fi
|
|
|
|
# Narrow to the subset still alive from the entry snapshot before
|
|
# signalling SIGTERM (Fix 8): a pid that already exited during the
|
|
# graceful-quit wait must never be re-signalled.
|
|
local snapshot_alive
|
|
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
|
snapshot_alive="$(pids_alive $pids)"
|
|
|
|
log "docker-nuke: sending SIGTERM"
|
|
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
|
kill_pids TERM $snapshot_alive
|
|
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
|
wait_for_exit "$TERM_WAIT" $snapshot_alive || true
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
if [ -z "$fresh" ]; then
|
|
log "docker-nuke: Docker exited after SIGTERM"
|
|
return 0
|
|
fi
|
|
|
|
# SIGKILL targets the fresh scan just taken above, not a set re-derived
|
|
# from the stale SIGTERM-stage snapshot (final review fix): a process that
|
|
# first appeared after snapshot_alive was captured -- e.g. a helper
|
|
# respawned by com.docker.backend during the SIGTERM wait -- is present in
|
|
# $fresh but was never a member of snapshot_alive, so filtering through
|
|
# snapshot_alive here silently skipped it for this entire round. It would
|
|
# only be caught a round later via the bounded respawn retry below,
|
|
# emitting a spurious "still present after SIGKILL, retrying" warning for a
|
|
# process that this round never actually SIGKILLed in the first place.
|
|
log "docker-nuke: sending SIGKILL"
|
|
# shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here
|
|
kill_pids KILL $fresh
|
|
sleep 1
|
|
|
|
if [ "${OPT_DEEP:-0}" -eq 1 ]; then
|
|
stop_privileged_helpers
|
|
fi
|
|
|
|
# Bounded respawn retry (Fix 1): if a fresh scan still finds live Docker
|
|
# processes, SIGKILL exactly that fresh set, let it settle, and scan again.
|
|
# Capped at KILL_RETRY_ROUNDS -- never an unbounded loop -- so a Docker
|
|
# that keeps respawning helpers cannot hang this tool forever.
|
|
local round=0
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
while [ -n "$fresh" ]; do
|
|
round=$((round + 1))
|
|
if [ "$round" -gt "$KILL_RETRY_ROUNDS" ]; then
|
|
break
|
|
fi
|
|
warn "docker-nuke: Docker processes still present after SIGKILL, retrying (round $round of $KILL_RETRY_ROUNDS): $fresh"
|
|
# shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here
|
|
kill_pids KILL $fresh
|
|
sleep 1
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
done
|
|
|
|
if [ -n "$fresh" ]; then
|
|
warn "docker-nuke: processes survived SIGKILL: $fresh"
|
|
return 1
|
|
fi
|
|
log "docker-nuke: all Docker processes terminated"
|
|
return 0
|
|
}
|
|
|
|
start_docker() {
|
|
log "docker-nuke: relaunching Docker Desktop"
|
|
open -a Docker >/dev/null 2>&1 || open "$DOCKER_APP" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Returns 0 once the daemon answers, 1 if SECONDS elapses first.
|
|
#
|
|
# The deadline is wall-clock, not a count of sleeps (Fix 2, review round 1).
|
|
# docker_healthy can itself cost up to HEALTH_TIMEOUT per call (it runs
|
|
# run_timeout "$HEALTH_TIMEOUT" docker info internally), but the old version
|
|
# of this loop only ever accumulated POLL_INTERVAL in `waited`, silently
|
|
# discounting docker_healthy's own cost every single iteration. Against a
|
|
# genuinely wedged daemon that undercount compounds per iteration: with real
|
|
# constants (HEALTH_TIMEOUT=6, POLL_INTERVAL=2) an 8s request measured 32s
|
|
# wall clock (4x), and the RESTART_TIMEOUT=90 caller could overshoot to
|
|
# roughly 6 minutes. Capturing a start time and checking `now - start`
|
|
# before each docker_healthy call instead bounds the overshoot to at most
|
|
# one in-flight docker_healthy call, never a per-iteration multiplier.
|
|
wait_for_daemon() {
|
|
local secs="$1"
|
|
local start now
|
|
start="$(date +%s)"
|
|
while :; do
|
|
now="$(date +%s)"
|
|
if [ "$((now - start))" -ge "$secs" ]; then
|
|
return 1
|
|
fi
|
|
if docker_healthy; then
|
|
return 0
|
|
fi
|
|
sleep "$POLL_INTERVAL"
|
|
done
|
|
}
|
|
|
|
# write_summary VERDICT ELAPSED — record the triage payload the user reads
|
|
# after recovery. Must never abort the caller, on ANY path (Fix 1, review
|
|
# round 1): a no-op when DUMP_DIR is unset, empty, or not a directory (e.g.
|
|
# after new_dump_dir failed on a full disk) -- and the write itself is
|
|
# guarded too, since the disk can fill AFTER new_dump_dir already created a
|
|
# valid DUMP_DIR but BEFORE this function runs; an unwritable or full
|
|
# DUMP_DIR at write time is exactly the scenario this tool exists to
|
|
# survive. The redirection is nested inside an outer group so the outer
|
|
# group's own 2>/dev/null is already in effect before the inner redirection
|
|
# is attempted -- a single-level `{ ... } >file 2>/dev/null` does NOT
|
|
# suppress bash's own "cannot create" diagnostic, since bash reports that
|
|
# failure before the trailing 2>/dev/null on the same command takes effect.
|
|
# A write failure is logged via warn and swallowed, never propagated.
|
|
write_summary() {
|
|
local verdict="$1" elapsed="$2" rc=0
|
|
local cf="${CAPTURE_FAILURES:-}"
|
|
if [ -z "$cf" ]; then cf=" none"; fi
|
|
if [ -z "${DUMP_DIR:-}" ] || [ ! -d "${DUMP_DIR:-}" ]; then
|
|
return 0
|
|
fi
|
|
{
|
|
{
|
|
printf 'docker-nuke summary\n'
|
|
printf '===================\n\n'
|
|
printf 'verdict: %s\n' "$verdict"
|
|
printf 'elapsed seconds: %s\n' "$elapsed"
|
|
printf 'dump directory: %s\n' "$DUMP_DIR"
|
|
printf 'capture failures:%s\n' "$cf"
|
|
printf '\nTriage hints\n------------\n'
|
|
printf '* ps.txt STAT column: U or D means stuck in the kernel, not user space\n'
|
|
printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n'
|
|
printf '* sample-* user-space stacks showing where the daemon was stuck\n'
|
|
printf '* logs/ Docker host and vm logs from just before the kill\n'
|
|
printf '* restart docker-nuke restarts Docker Desktop itself, not your containers -- only containers started with a restart policy (e.g. unless-stopped) come back automatically; run docker ps -a to check the rest\n'
|
|
} >"$DUMP_DIR/summary.txt"
|
|
} 2>/dev/null || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
warn "docker-nuke: could not write summary to $DUMP_DIR/summary.txt (dump directory unwritable or full)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
log() { printf '%s\n' "$*"; }
|
|
warn() { printf '%s\n' "$*" >&2; }
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first
|
|
|
|
Usage:
|
|
docker-nuke [options]
|
|
|
|
Options:
|
|
-f, --force Skip the confirmation shown when Docker looks healthy
|
|
-n, --no-restart Kill Docker but do not relaunch it
|
|
--dry-run Report what would be killed and dumped; change nothing
|
|
--deep Also stop root launchd helpers (com.docker.vmnetd);
|
|
requires a cached sudo credential -- run "sudo -v" first
|
|
-h, --help Show this help
|
|
|
|
Diagnostics are written to $DUMP_ROOT/<timestamp>/ before anything is
|
|
killed (override the root with DOCKER_NUKE_DUMP_ROOT). The most recent
|
|
$DUMP_RETENTION dumps are kept.
|
|
|
|
Exit codes:
|
|
0 Docker killed and confirmed responsive again
|
|
1 Daemon did not come back within the restart timeout, or the kill itself
|
|
did not complete
|
|
2 Usage error, or aborted at the healthy-daemon prompt
|
|
EOF
|
|
}
|
|
|
|
parse_args() {
|
|
OPT_FORCE=0
|
|
OPT_NO_RESTART=0
|
|
OPT_DRY_RUN=0
|
|
OPT_DEEP=0
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
-f|--force) OPT_FORCE=1 ;;
|
|
-n|--no-restart) OPT_NO_RESTART=1 ;;
|
|
--dry-run) OPT_DRY_RUN=1 ;;
|
|
--deep) OPT_DEEP=1 ;;
|
|
-h|--help) usage; return 3 ;;
|
|
*)
|
|
warn "docker-nuke: unknown option '$1'"
|
|
usage >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# --- entrypoint -------------------------------------------------------------
|
|
|
|
|
|
# docker_version_summary — one-line "<version> (<N> running)" snapshot, or
|
|
# empty if `docker info` fails or hangs. Shared by confirm_healthy_nuke's
|
|
# healthy-daemon prompt and main's post-recovery report (final review fix:
|
|
# previously the same `docker info --format ...` call was duplicated verbatim
|
|
# in both places).
|
|
docker_version_summary() {
|
|
run_timeout "$HEALTH_TIMEOUT" docker info \
|
|
--format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true
|
|
}
|
|
|
|
confirm_healthy_nuke() {
|
|
local info
|
|
info="$(docker_version_summary)"
|
|
log "docker-nuke: Docker looks healthy: ${info:-unknown}"
|
|
log "Nuking will stop running containers."
|
|
printf 'Proceed anyway? [y/N] '
|
|
local reply=""
|
|
read -r reply || true
|
|
case "$reply" in
|
|
y|Y|yes|YES) return 0 ;;
|
|
*)
|
|
# A non-TTY stdin (piped, redirected from /dev/null, closed) can never
|
|
# answer this prompt interactively even on a re-run, so the default
|
|
# deny should point at the actual way past it (final review fix): a
|
|
# bare "aborted" here previously gave no hint that --force exists.
|
|
if [ -t 0 ]; then
|
|
log "docker-nuke: aborted"
|
|
else
|
|
log "docker-nuke: aborted (no interactive input on stdin; use --force to skip this prompt)"
|
|
fi
|
|
return 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
dry_run_report() {
|
|
local pids
|
|
pids="$(docker_pids | tr '\n' ' ')"
|
|
log "docker-nuke: DRY RUN — no changes made"
|
|
if [ -z "$pids" ]; then
|
|
log " would kill: (no Docker processes found)"
|
|
else
|
|
log " would kill: $pids"
|
|
# shellcheck disable=SC2086
|
|
ps -o pid,stat,%cpu,%mem,etime,comm -p $pids 2>/dev/null || true
|
|
fi
|
|
if [ "$OPT_DEEP" -eq 1 ]; then
|
|
log " would also stop privileged helpers (--deep): com.docker.vmnetd, com.docker.socket"
|
|
fi
|
|
log " would dump to: $DUMP_ROOT/<timestamp>"
|
|
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
|
log " would not restart Docker (--no-restart)"
|
|
else
|
|
log " would restart Docker and wait up to ${RESTART_TIMEOUT}s"
|
|
fi
|
|
if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then
|
|
log " NOTE: Docker currently looks healthy; a real run would prompt for confirmation unless --force is given"
|
|
fi
|
|
}
|
|
|
|
# finish VERDICT RC ELAPSED [EXTRA_LINE ...] — shared tail for every terminal
|
|
# path in main() (final review fix). Writes the summary, prunes old dumps,
|
|
# then logs the verdict, any path-specific EXTRA_LINEs, and finally the dump
|
|
# location -- guarding the case where run_dump itself failed and DUMP_DIR is
|
|
# empty (Fix C1), so the tool can never print "diagnostics in " with nothing
|
|
# after it. Success (RC 0) logs via log(); any other RC logs every line via
|
|
# warn() instead, so a failed kill (Fix C2) can never look identical to a
|
|
# clean run in the terminal output. Collapsing the three near-identical
|
|
# terminal blocks that used to live in main() into this one helper makes it
|
|
# structurally impossible for a future fourth path to forget to thread the
|
|
# kill status into the verdict/exit code, or to forget the empty-DUMP_DIR
|
|
# guard.
|
|
finish() {
|
|
local verdict="$1" rc="$2" elapsed="$3"
|
|
shift 3
|
|
|
|
write_summary "$verdict" "$elapsed"
|
|
prune_dumps
|
|
|
|
local logfn=log
|
|
if [ "$rc" -ne 0 ]; then logfn=warn; fi
|
|
|
|
"$logfn" "docker-nuke: $verdict"
|
|
local line
|
|
for line in "$@"; do
|
|
"$logfn" "$line"
|
|
done
|
|
if [ -n "${DUMP_DIR:-}" ]; then
|
|
"$logfn" "docker-nuke: diagnostics in $DUMP_DIR"
|
|
else
|
|
"$logfn" "docker-nuke: diagnostics were not captured this run"
|
|
fi
|
|
|
|
return "$rc"
|
|
}
|
|
|
|
main() {
|
|
local rc=0
|
|
parse_args "$@" || rc=$?
|
|
if [ "$rc" -eq 3 ]; then return 0; fi
|
|
if [ "$rc" -ne 0 ]; then return "$rc"; fi
|
|
|
|
if [ "$OPT_DRY_RUN" -eq 1 ]; then
|
|
dry_run_report
|
|
return 0
|
|
fi
|
|
|
|
# Fix I2: checked up front, before anything is killed, so the user can fix
|
|
# a missing cached credential while Docker is still alive. Never fatal --
|
|
# stop_privileged_helpers already degrades gracefully (Fix 3) if this was
|
|
# ignored; this just surfaces the problem while it is still actionable
|
|
# instead of two silent `sudo -n` failures after the kill.
|
|
if [ "$OPT_DEEP" -eq 1 ] && ! sudo -n true 2>/dev/null; then
|
|
warn "docker-nuke: --deep requires a cached sudo credential; run \"sudo -v\" first. Continuing without it -- privileged helpers will not be stopped."
|
|
fi
|
|
|
|
# OPT_FORCE checked first (final review fix): docker_healthy can cost up to
|
|
# HEALTH_TIMEOUT (6s) before answering, and --force is exactly the flag
|
|
# reached for when Docker is already known to be wedged -- paying that
|
|
# latency to compute an answer that is about to be skipped anyway serves no
|
|
# purpose.
|
|
if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then
|
|
confirm_healthy_nuke || return 2
|
|
fi
|
|
|
|
local started elapsed
|
|
started="$(date +%s)"
|
|
|
|
# Fix C1: run_dump is never called bare. It returns 1 when new_dump_dir
|
|
# fails (e.g. dump root unwritable or full) -- precisely the condition it
|
|
# exists to detect -- and under set -e a bare call would abort the entire
|
|
# script right here: Docker is never killed, never restarted. The design
|
|
# requirement is explicit: the dump can never block recovery. A failure is
|
|
# recorded and recovery proceeds with no dump saved, exactly like every
|
|
# individual capture failure inside run_dump already does.
|
|
local dumped=1
|
|
run_dump || { dumped=0; warn "docker-nuke: diagnostics capture failed; continuing with recovery (no dump will be saved)"; }
|
|
local dump_note=""
|
|
if [ "$dumped" -eq 0 ]; then
|
|
dump_note=" (no dump captured)"
|
|
fi
|
|
|
|
# Fix C2: nuke_docker's return status -- 1 only after KILL_RETRY_ROUNDS is
|
|
# exhausted with Docker processes still alive -- was previously discarded
|
|
# with `|| true` and never consulted again. It is captured here and
|
|
# threaded into every terminal path's verdict (so it lands in summary.txt,
|
|
# the file actually read during triage) and exit code, so a failed kill can
|
|
# never be reported as success.
|
|
local killed=1 kill_note=""
|
|
nuke_docker || killed=0
|
|
if [ "$killed" -eq 0 ]; then
|
|
local survivors
|
|
survivors="$(docker_pids | tr '\n' ' ')"
|
|
kill_note=" -- WARNING: kill incomplete, processes survived: ${survivors:-unknown}"
|
|
fi
|
|
|
|
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
|
local verdict="killed, not restarted (--no-restart)${dump_note}${kill_note}"
|
|
local finish_rc=0
|
|
if [ "$killed" -eq 0 ]; then finish_rc=1; fi
|
|
elapsed=$(( $(date +%s) - started ))
|
|
finish "$verdict" "$finish_rc" "$elapsed"
|
|
return "$finish_rc"
|
|
fi
|
|
|
|
start_docker
|
|
if wait_for_daemon "$RESTART_TIMEOUT"; then
|
|
elapsed=$(( $(date +%s) - started ))
|
|
local info
|
|
info="$(docker_version_summary)"
|
|
local verdict="recovered after ${elapsed}s — ${info:-ok}${dump_note}${kill_note}"
|
|
local finish_rc=0
|
|
if [ "$killed" -eq 0 ]; then finish_rc=1; fi
|
|
finish "$verdict" "$finish_rc" "$elapsed"
|
|
return "$finish_rc"
|
|
fi
|
|
|
|
local verdict="daemon did not return within ${RESTART_TIMEOUT}s${dump_note}${kill_note}"
|
|
elapsed=$(( $(date +%s) - started ))
|
|
|
|
# The advice depends on WHY the daemon is absent, and the tell is whether any
|
|
# Docker process survived the relaunch. No survivors means Docker Desktop was
|
|
# started but did not stay up -- an app crashing on startup, which no
|
|
# programmatic relaunch can work around. Survivors mean it is up but the
|
|
# daemon is not answering yet: still starting, or wedged.
|
|
local survivors dump_hint
|
|
survivors="$(docker_pids | tr '\n' ' ')"
|
|
if [ -n "${DUMP_DIR:-}" ]; then
|
|
dump_hint="docker-nuke: see $DUMP_DIR/summary.txt for triage, and disk.txt for free space"
|
|
else
|
|
dump_hint="docker-nuke: no dump was captured this run"
|
|
fi
|
|
|
|
if [ -z "$survivors" ]; then
|
|
finish "$verdict" 1 "$elapsed" \
|
|
"docker-nuke: no Docker processes are running -- Docker Desktop was relaunched but did not stay up" \
|
|
"docker-nuke: launch Docker Desktop from the Dock or Spotlight; a programmatic relaunch cannot recover an app that crashes on startup" \
|
|
"docker-nuke: if it keeps crashing, check Console.app for com.docker.backend, then update or reinstall Docker Desktop" \
|
|
"$dump_hint"
|
|
else
|
|
finish "$verdict" 1 "$elapsed" \
|
|
"docker-nuke: still running: $survivors" \
|
|
"docker-nuke: Docker is up but the daemon is not answering -- it may still be starting; retry, or use 'docker-nuke --deep' to also stop root helpers" \
|
|
"$dump_hint"
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Only run when executed, not when sourced by the test harness.
|
|
if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
|
|
main "$@"
|
|
fi
|