mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
David Ibia
804 lines
31 KiB
Bash
Executable File
804 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# docker-nuke — capture diagnostics from a wedged Docker Desktop, then
|
|
# force-kill and restart it.
|
|
#
|
|
# Written for bash 3.2 compatibility (macOS /bin/bash).
|
|
|
|
set -euo pipefail
|
|
|
|
# --- configuration ----------------------------------------------------------
|
|
#
|
|
# The SC2034 directives below are per-line on purpose, not file-level: each
|
|
# constant is consumed by a later task, and its directive is deleted at that
|
|
# point. A file-level disable would silence SC2034 for the whole file forever,
|
|
# hiding a genuinely dead variable added later. By the final task none of these
|
|
# directives should remain.
|
|
|
|
HEALTH_TIMEOUT=6 # seconds to wait on `docker info` before calling it wedged
|
|
GRACEFUL_WAIT=8 # seconds to let Docker quit politely
|
|
TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL
|
|
# Bounded number of "fresh docker_pids -> SIGKILL -> settle" rounds run if a
|
|
# post-SIGKILL scan still finds live Docker processes. A supervisor such as
|
|
# com.docker.backend can respawn a killed helper under a brand new pid the
|
|
# entry snapshot never knew about, so this must be a real cap, never an
|
|
# unbounded loop, or a respawning Docker could hang this tool forever.
|
|
KILL_RETRY_ROUNDS=3
|
|
RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back
|
|
POLL_INTERVAL=2 # seconds between daemon health polls
|
|
SAMPLE_DURATION=2 # seconds of stack sampling per process
|
|
SAMPLE_TIMEOUT=10 # hard cap on a single `sample` invocation
|
|
DUMP_RETENTION=10 # how many dump directories to keep
|
|
LOG_TAIL_LINES=2000 # lines of Docker log to copy (keeps dumps small)
|
|
LOG_CAPTURE_TIMEOUT=20 # hard cap on the whole capture_logs step
|
|
CRASH_CAPTURE_TIMEOUT=10 # hard cap on the whole capture_crashes step
|
|
|
|
DOCKER_APP="/Applications/Docker.app"
|
|
# The bundle path above cannot substitute for the AppleScript app name below:
|
|
# the running GUI application is the nested "Contents/MacOS/Docker
|
|
# Desktop.app", whose CFBundleName is "Docker Desktop", not the top-level
|
|
# bundle path. Both are genuinely needed; this constant exists only so the
|
|
# two literal identities for the same app cannot drift apart silently.
|
|
DOCKER_APP_NAME="Docker Desktop"
|
|
DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log"
|
|
DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw"
|
|
CRASH_DIR="$HOME/Library/Logs/DiagnosticReports"
|
|
|
|
# Overridable so tests never touch real state.
|
|
DUMP_ROOT="${DOCKER_NUKE_DUMP_ROOT:-$HOME/.local/state/docker-nuke}"
|
|
|
|
OPT_FORCE=0
|
|
OPT_NO_RESTART=0
|
|
OPT_DRY_RUN=0
|
|
OPT_DEEP=0
|
|
|
|
# --- helpers ----------------------------------------------------------------
|
|
|
|
bin_exists() { command -v "$1" >/dev/null 2>&1; }
|
|
|
|
# Run a command with a hard time limit. Returns 124 if it timed out.
|
|
# Call inside a conditional so `set -e` does not abort on a non-zero return.
|
|
run_timeout() {
|
|
local secs="$1"
|
|
shift
|
|
# A shell function (e.g. disk_report) has no PATH-resolvable executable, so
|
|
# an external timeout/gtimeout binary can never exec it directly (execvp
|
|
# fails with rc=127). Route those straight to the pure-bash poller below,
|
|
# which runs the command in this same shell instead of a child process.
|
|
if bin_exists timeout && ! declare -F "$1" >/dev/null 2>&1; then
|
|
timeout "$secs" "$@"
|
|
return $?
|
|
fi
|
|
if bin_exists gtimeout && ! declare -F "$1" >/dev/null 2>&1; then
|
|
gtimeout "$secs" "$@"
|
|
return $?
|
|
fi
|
|
|
|
# Pure-bash fallback: run in background and poll.
|
|
"$@" &
|
|
local pid=$!
|
|
local waited=0
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
if [ "$waited" -ge "$secs" ]; then
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
wait "$pid" 2>/dev/null || true
|
|
return 124
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
local rc=0
|
|
wait "$pid" || rc=$?
|
|
return "$rc"
|
|
}
|
|
|
|
# Exact process names to target, in addition to anything under Docker.app.
|
|
# NOTE: never use `pgrep -f docker` here — it matches this script's own
|
|
# command line and the tool would kill itself.
|
|
DOCKER_PROC_NAMES="com.docker.backend com.docker.build com.docker.krun \
|
|
com.docker.vpnkit com.docker.virtualization vpnkit docker"
|
|
|
|
docker_pids() {
|
|
local self="$$"
|
|
local parent="${PPID:-0}"
|
|
local name pid comm
|
|
{
|
|
# `pgrep -f "$DOCKER_APP"` matches any process whose *command line*
|
|
# mentions the bundle path — e.g. `tail -f /Applications/Docker.app/...`
|
|
# matches even though that process is plainly not Docker. Post-filter
|
|
# each candidate by its actual executable so only processes truly
|
|
# launched from the bundle survive. Wrapped in `|| true`: under
|
|
# `set -o pipefail`, an empty `pgrep -f` match (exit 1) would otherwise
|
|
# make this whole statement fail and abort the function under `set -e`.
|
|
pgrep -f "$DOCKER_APP" 2>/dev/null | while read -r pid; do
|
|
if [ -z "$pid" ]; then continue; fi
|
|
# A dead PID (process exited between pgrep and ps) or a `ps` failure
|
|
# must yield an empty comm, which matches no case branch below —
|
|
# never a crash, never a false positive.
|
|
# NOTE: `ps -o comm=` reports the executable path as invoked, not
|
|
# canonicalized — a Docker helper ever exec'd by relative path or bare
|
|
# name would report a non-bundle comm here and be silently dropped.
|
|
comm="$(ps -o comm= -p "$pid" 2>/dev/null)" || true
|
|
case "$comm" in
|
|
"$DOCKER_APP"/*) printf '%s\n' "$pid" ;;
|
|
esac
|
|
done || true
|
|
|
|
# Exact-name matches are already precise — do NOT apply the comm
|
|
# filter here. One job of this branch is catching a hung
|
|
# /usr/local/bin/docker CLI, which legitimately lives outside the
|
|
# bundle; filtering by $DOCKER_APP prefix would break that.
|
|
for name in $DOCKER_PROC_NAMES; do
|
|
pgrep -x "$name" 2>/dev/null || true
|
|
done
|
|
} | sort -un | while read -r pid; do
|
|
if [ -z "$pid" ]; then continue; fi
|
|
if [ "$pid" = "$self" ]; then continue; fi
|
|
if [ "$pid" = "$parent" ]; then continue; fi
|
|
printf '%s\n' "$pid"
|
|
done
|
|
}
|
|
|
|
# Returns 0 if the daemon answers within HEALTH_TIMEOUT, else non-zero
|
|
# (124 specifically means it hung rather than errored).
|
|
docker_healthy() {
|
|
local rc=0
|
|
run_timeout "$HEALTH_TIMEOUT" docker info >/dev/null 2>&1 || rc=$?
|
|
return "$rc"
|
|
}
|
|
|
|
DUMP_DIR=""
|
|
CAPTURE_FAILURES=""
|
|
|
|
# new_dump_dir — create and print a fresh timestamped directory under
|
|
# DUMP_ROOT. This is the one place in the dump path that fails loudly: if the
|
|
# directory cannot be created there is nowhere left to record a failure, so
|
|
# the caller must find out immediately rather than have every later capture
|
|
# silently write into a path that never existed.
|
|
new_dump_dir() {
|
|
local stamp dir
|
|
stamp="$(date -u '+%Y%m%dT%H%M%SZ')"
|
|
dir="$DUMP_ROOT/$stamp"
|
|
if ! mkdir -p "$dir" 2>/dev/null; then
|
|
warn "docker-nuke: could not create dump directory $dir (dump root unwritable or full)"
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$dir"
|
|
}
|
|
|
|
# capture NAME SECONDS CMD... — write CMD output to $DUMP_DIR/NAME.
|
|
# Never fails: a timeout or error is recorded and execution continues, so a
|
|
# broken capture can never block recovery.
|
|
capture() {
|
|
local name="$1" secs="$2"
|
|
shift 2
|
|
local out="$DUMP_DIR/$name"
|
|
local rc=0
|
|
run_timeout "$secs" "$@" >"$out" 2>&1 || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
# The banner append itself must never be able to trip set -e: on an
|
|
# unwritable or full dump root this redirection fails too, and without
|
|
# the guard that failure -- not the original capture failure -- would be
|
|
# what aborts the entire dump at the very first capture.
|
|
printf '\n[docker-nuke] capture "%s" failed or timed out (rc=%s)\n' "$name" "$rc" >>"$out" 2>/dev/null || true
|
|
CAPTURE_FAILURES="$CAPTURE_FAILURES $name"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Sample user-space stacks of the daemon-side processes. This is the one
|
|
# signal that cannot be reconstructed after the processes are killed.
|
|
capture_samples() {
|
|
if ! bin_exists sample; then
|
|
printf 'sample(1) not available\n' >"$DUMP_DIR/samples-skipped.txt"
|
|
return 0
|
|
fi
|
|
local name pid
|
|
for name in com.docker.backend com.docker.krun; do
|
|
for pid in $(pgrep -x "$name" 2>/dev/null || true); do
|
|
capture "sample-$name-$pid.txt" "$SAMPLE_TIMEOUT" \
|
|
sample "$pid" "$SAMPLE_DURATION" -mayDie
|
|
done
|
|
done
|
|
}
|
|
|
|
capture_logs() {
|
|
mkdir -p "$DUMP_DIR/logs"
|
|
local f base
|
|
for f in "$DOCKER_LOG_DIR"/host/*.log "$DOCKER_LOG_DIR"/vm/*.log; do
|
|
if [ -f "$f" ]; then
|
|
base="$(basename "$f")"
|
|
tail -n "$LOG_TAIL_LINES" "$f" >"$DUMP_DIR/logs/$base" 2>/dev/null || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
capture_crashes() {
|
|
mkdir -p "$DUMP_DIR/crashes"
|
|
if [ ! -d "$CRASH_DIR" ]; then
|
|
return 0
|
|
fi
|
|
local f
|
|
# grep exits 1 when no crash report matches (the common case: this dump
|
|
# feature is expected to run against a healthy history most of the time).
|
|
# Under `set -o pipefail` that would fail the whole pipeline and abort
|
|
# run_dump via `set -e`. Group it with `|| true` so a clean "no matches"
|
|
# never propagates as a pipeline failure.
|
|
#
|
|
# Grouping only the grep is not enough on its own: `head -5` closes its
|
|
# read end once it has 5 lines, so with more than 5 matches upstream
|
|
# `ls -1t` is later killed by SIGPIPE (rc=141) instead of exiting 0; an
|
|
# unreadable CRASH_DIR gives `ls` rc=2 the same way. Either propagates as
|
|
# this pipeline's exit status under pipefail and would abort the dump via
|
|
# `set -e`, so the trailing `|| true` guards the pipeline as a whole, not
|
|
# just the grep.
|
|
# shellcheck disable=SC2012 # crash report names are macOS-generated, never contain newlines; ls -t gives the mtime order find cannot without extra plumbing
|
|
ls -1t "$CRASH_DIR" 2>/dev/null | { grep -i docker || true; } | head -5 | while IFS= read -r f; do
|
|
cp "$CRASH_DIR/$f" "$DUMP_DIR/crashes/" 2>/dev/null || true
|
|
done || true
|
|
}
|
|
|
|
disk_report() {
|
|
df -h
|
|
printf '\n--- Docker.raw ---\n'
|
|
ls -lh "$DOCKER_RAW" 2>/dev/null || printf 'Docker.raw not found\n'
|
|
du -h -d0 "$DOCKER_RAW" 2>/dev/null || true
|
|
}
|
|
|
|
socket_report() {
|
|
local sock="$HOME/.docker/run/docker.sock"
|
|
printf 'socket: %s\n' "$sock"
|
|
ls -l "$sock" 2>/dev/null || printf 'socket does not exist\n'
|
|
printf '\n--- listeners ---\n'
|
|
lsof "$sock" 2>/dev/null || printf 'no processes hold the socket\n'
|
|
}
|
|
|
|
run_dump() {
|
|
local rc
|
|
# DUMP_DIR is reset to empty first so a failed new_dump_dir never leaves it
|
|
# pointing at a directory that does not exist (Fix 1). new_dump_dir is the
|
|
# one place in this path that is allowed to fail loudly: on an unwritable
|
|
# or full dump root it has already logged why via `warn` and returned
|
|
# non-zero, and there is nowhere left to record a capture failure, so
|
|
# run_dump stops here -- before any capture runs -- rather than limping
|
|
# into the first capture and dying mid-dump.
|
|
DUMP_DIR=""
|
|
DUMP_DIR="$(new_dump_dir)" || return 1
|
|
CAPTURE_FAILURES=""
|
|
log "docker-nuke: capturing diagnostics to $DUMP_DIR"
|
|
|
|
capture "ps.txt" 10 ps -eo pid,ppid,stat,%cpu,%mem,etime,command
|
|
capture "docker-info.txt" "$HEALTH_TIMEOUT" docker info
|
|
capture "docker-version.txt" "$HEALTH_TIMEOUT" docker version
|
|
capture "disk.txt" 15 disk_report
|
|
capture "memory.txt" 10 vm_stat
|
|
capture "socket.txt" 10 socket_report
|
|
|
|
# capture_logs and capture_crashes write files directly rather than
|
|
# emitting stdout, so they are not routed through capture() (which would
|
|
# redirect stdout to a single named artifact). They still must be
|
|
# time-bounded like every other step, and run_timeout's `declare -F` guard
|
|
# (see run_timeout above) routes them to the pure-bash poller since neither
|
|
# is a PATH-resolvable executable. A timeout or error is recorded into
|
|
# CAPTURE_FAILURES the same way capture() records one, so a wedged
|
|
# filesystem here cannot hang the dump either.
|
|
rc=0
|
|
run_timeout "$LOG_CAPTURE_TIMEOUT" capture_logs || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
CAPTURE_FAILURES="$CAPTURE_FAILURES logs"
|
|
fi
|
|
|
|
rc=0
|
|
run_timeout "$CRASH_CAPTURE_TIMEOUT" capture_crashes || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
CAPTURE_FAILURES="$CAPTURE_FAILURES crashes"
|
|
fi
|
|
|
|
capture_samples
|
|
}
|
|
|
|
prune_dumps() {
|
|
local keep="${1:-$DUMP_RETENTION}"
|
|
if [ ! -d "$DUMP_ROOT" ]; then
|
|
return 0
|
|
fi
|
|
local d
|
|
# An unreadable DUMP_ROOT gives `ls` a non-zero exit, which -- since this
|
|
# pipeline is prune_dumps' last statement and therefore its return value --
|
|
# would otherwise abort a `set -e` caller. The trailing `|| true` matches
|
|
# the guard used in capture_crashes for the same class of problem; the
|
|
# normal-path behavior (prune the oldest, keep $keep newest) is unchanged.
|
|
# shellcheck disable=SC2012 # dump dir names are our own generated timestamps, never contain newlines; sort -r needs plain lines, not find -print0
|
|
ls -1 "$DUMP_ROOT" 2>/dev/null | sort -r | tail -n "+$((keep + 1))" | while IFS= read -r d; do
|
|
if [ -n "$d" ]; then
|
|
rm -rf "${DUMP_ROOT:?}/$d"
|
|
fi
|
|
done || true
|
|
return 0
|
|
}
|
|
|
|
pids_alive() {
|
|
local pid alive="" stat
|
|
for pid in "$@"; do
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
# kill -0 succeeds against a zombie (state Z, <defunct>): the process
|
|
# table entry is still there but the process has already exited and is
|
|
# only waiting for its parent to reap it. Treat that as dead so a slow
|
|
# reaper can never manufacture a spurious "survived SIGKILL" report
|
|
# (Fix 7). ps -o stat= can print extra flag letters after the primary
|
|
# state letter (e.g. Z+), so match a leading Z rather than equality.
|
|
stat="$(ps -o stat= -p "$pid" 2>/dev/null)" || true
|
|
case "$stat" in
|
|
Z*) : ;;
|
|
*) alive="$alive $pid" ;;
|
|
esac
|
|
fi
|
|
done
|
|
printf '%s' "${alive# }"
|
|
}
|
|
|
|
kill_pids() {
|
|
local sig="$1"
|
|
shift
|
|
local pid
|
|
for pid in "$@"; do
|
|
kill "-$sig" "$pid" 2>/dev/null || true
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# Returns 0 once every PID is gone, 1 if SECONDS elapses first.
|
|
wait_for_exit() {
|
|
local secs="$1"
|
|
shift
|
|
local waited=0
|
|
while [ "$waited" -lt "$secs" ]; do
|
|
if [ -z "$(pids_alive "$@")" ]; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
waited=$((waited + 1))
|
|
done
|
|
if [ -z "$(pids_alive "$@")" ]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Stop root launchd helpers. Only called with --deep; needs a cached sudo
|
|
# credential.
|
|
stop_privileged_helpers() {
|
|
log "docker-nuke: stopping privileged helpers (sudo required)"
|
|
# -n (non-interactive) never prompts: sudo writes "Password:" to stderr but
|
|
# reads the answer from /dev/tty, so with stderr discarded and no cached
|
|
# credential a plain `sudo` blocks forever with no visible prompt -- and by
|
|
# the time this runs, Docker has already been SIGKILLed (Fix 3). -n instead
|
|
# fails fast into the warning below.
|
|
if ! sudo -n launchctl stop com.docker.vmnetd 2>/dev/null; then
|
|
warn "docker-nuke: sudo -n failed for com.docker.vmnetd (no cached credential, or not permitted); skipping"
|
|
fi
|
|
if ! sudo -n launchctl stop com.docker.socket 2>/dev/null; then
|
|
warn "docker-nuke: sudo -n failed for com.docker.socket (no cached credential, or not permitted); skipping"
|
|
fi
|
|
}
|
|
|
|
# Three-stage escalation: polite quit, SIGTERM, SIGKILL.
|
|
#
|
|
# Success at every stage is decided from a FRESH docker_pids scan, never from
|
|
# the entry snapshot below or from wait_for_exit's own return value (Fix 1).
|
|
# wait_for_exit still runs against the snapshot at each stage for TIMING --
|
|
# it correctly answers "did the ones we signalled exit" -- but it cannot see
|
|
# a Docker process that appeared after the snapshot was taken, so it can
|
|
# never be trusted alone to answer "is Docker gone". Concretely:
|
|
# com.docker.backend supervises helpers; SIGKILL order is numeric (sort -un
|
|
# in docker_pids), unrelated to the supervision tree. A helper can be killed,
|
|
# the still-alive backend respawns it under a NEW pid, then the backend
|
|
# itself is killed. Every snapshot pid is now gone, but the respawned helper
|
|
# is still running -- only a fresh docker_pids call can see it.
|
|
nuke_docker() {
|
|
local pids fresh
|
|
pids="$(docker_pids | tr '\n' ' ')"
|
|
if [ -z "$pids" ]; then
|
|
log "docker-nuke: no Docker processes found"
|
|
return 0
|
|
fi
|
|
|
|
log "docker-nuke: asking Docker Desktop to quit"
|
|
osascript -e 'quit app "'"$DOCKER_APP_NAME"'"' >/dev/null 2>&1 || true
|
|
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
|
wait_for_exit "$GRACEFUL_WAIT" $pids || true
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
if [ -z "$fresh" ]; then
|
|
log "docker-nuke: Docker exited cleanly"
|
|
return 0
|
|
fi
|
|
|
|
# Narrow to the subset still alive from the entry snapshot before
|
|
# signalling SIGTERM (Fix 8): a pid that already exited during the
|
|
# graceful-quit wait must never be re-signalled.
|
|
local snapshot_alive
|
|
# shellcheck disable=SC2086 # $pids must word-split into separate PID arguments here
|
|
snapshot_alive="$(pids_alive $pids)"
|
|
|
|
log "docker-nuke: sending SIGTERM"
|
|
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
|
kill_pids TERM $snapshot_alive
|
|
# shellcheck disable=SC2086 # $snapshot_alive must word-split into separate PID arguments here
|
|
wait_for_exit "$TERM_WAIT" $snapshot_alive || true
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
if [ -z "$fresh" ]; then
|
|
log "docker-nuke: Docker exited after SIGTERM"
|
|
return 0
|
|
fi
|
|
|
|
# SIGKILL targets the fresh scan just taken above, not a set re-derived
|
|
# from the stale SIGTERM-stage snapshot (final review fix): a process that
|
|
# first appeared after snapshot_alive was captured -- e.g. a helper
|
|
# respawned by com.docker.backend during the SIGTERM wait -- is present in
|
|
# $fresh but was never a member of snapshot_alive, so filtering through
|
|
# snapshot_alive here silently skipped it for this entire round. It would
|
|
# only be caught a round later via the bounded respawn retry below,
|
|
# emitting a spurious "still present after SIGKILL, retrying" warning for a
|
|
# process that this round never actually SIGKILLed in the first place.
|
|
log "docker-nuke: sending SIGKILL"
|
|
# shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here
|
|
kill_pids KILL $fresh
|
|
sleep 1
|
|
|
|
if [ "${OPT_DEEP:-0}" -eq 1 ]; then
|
|
stop_privileged_helpers
|
|
fi
|
|
|
|
# Bounded respawn retry (Fix 1): if a fresh scan still finds live Docker
|
|
# processes, SIGKILL exactly that fresh set, let it settle, and scan again.
|
|
# Capped at KILL_RETRY_ROUNDS -- never an unbounded loop -- so a Docker
|
|
# that keeps respawning helpers cannot hang this tool forever.
|
|
local round=0
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
while [ -n "$fresh" ]; do
|
|
round=$((round + 1))
|
|
if [ "$round" -gt "$KILL_RETRY_ROUNDS" ]; then
|
|
break
|
|
fi
|
|
warn "docker-nuke: Docker processes still present after SIGKILL, retrying (round $round of $KILL_RETRY_ROUNDS): $fresh"
|
|
# shellcheck disable=SC2086 # $fresh must word-split into separate PID arguments here
|
|
kill_pids KILL $fresh
|
|
sleep 1
|
|
fresh="$(docker_pids | tr '\n' ' ')"
|
|
done
|
|
|
|
if [ -n "$fresh" ]; then
|
|
warn "docker-nuke: processes survived SIGKILL: $fresh"
|
|
return 1
|
|
fi
|
|
log "docker-nuke: all Docker processes terminated"
|
|
return 0
|
|
}
|
|
|
|
start_docker() {
|
|
log "docker-nuke: relaunching Docker Desktop"
|
|
open -a Docker >/dev/null 2>&1 || open "$DOCKER_APP" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
# Returns 0 once the daemon answers, 1 if SECONDS elapses first.
|
|
#
|
|
# The deadline is wall-clock, not a count of sleeps (Fix 2, review round 1).
|
|
# docker_healthy can itself cost up to HEALTH_TIMEOUT per call (it runs
|
|
# run_timeout "$HEALTH_TIMEOUT" docker info internally), but the old version
|
|
# of this loop only ever accumulated POLL_INTERVAL in `waited`, silently
|
|
# discounting docker_healthy's own cost every single iteration. Against a
|
|
# genuinely wedged daemon that undercount compounds per iteration: with real
|
|
# constants (HEALTH_TIMEOUT=6, POLL_INTERVAL=2) an 8s request measured 32s
|
|
# wall clock (4x), and the RESTART_TIMEOUT=90 caller could overshoot to
|
|
# roughly 6 minutes. Capturing a start time and checking `now - start`
|
|
# before each docker_healthy call instead bounds the overshoot to at most
|
|
# one in-flight docker_healthy call, never a per-iteration multiplier.
|
|
wait_for_daemon() {
|
|
local secs="$1"
|
|
local start now
|
|
start="$(date +%s)"
|
|
while :; do
|
|
now="$(date +%s)"
|
|
if [ "$((now - start))" -ge "$secs" ]; then
|
|
return 1
|
|
fi
|
|
if docker_healthy; then
|
|
return 0
|
|
fi
|
|
sleep "$POLL_INTERVAL"
|
|
done
|
|
}
|
|
|
|
# write_summary VERDICT ELAPSED — record the triage payload the user reads
|
|
# after recovery. Must never abort the caller, on ANY path (Fix 1, review
|
|
# round 1): a no-op when DUMP_DIR is unset, empty, or not a directory (e.g.
|
|
# after new_dump_dir failed on a full disk) -- and the write itself is
|
|
# guarded too, since the disk can fill AFTER new_dump_dir already created a
|
|
# valid DUMP_DIR but BEFORE this function runs; an unwritable or full
|
|
# DUMP_DIR at write time is exactly the scenario this tool exists to
|
|
# survive. The redirection is nested inside an outer group so the outer
|
|
# group's own 2>/dev/null is already in effect before the inner redirection
|
|
# is attempted -- a single-level `{ ... } >file 2>/dev/null` does NOT
|
|
# suppress bash's own "cannot create" diagnostic, since bash reports that
|
|
# failure before the trailing 2>/dev/null on the same command takes effect.
|
|
# A write failure is logged via warn and swallowed, never propagated.
|
|
write_summary() {
|
|
local verdict="$1" elapsed="$2" rc=0
|
|
local cf="${CAPTURE_FAILURES:-}"
|
|
if [ -z "$cf" ]; then cf=" none"; fi
|
|
if [ -z "${DUMP_DIR:-}" ] || [ ! -d "${DUMP_DIR:-}" ]; then
|
|
return 0
|
|
fi
|
|
{
|
|
{
|
|
printf 'docker-nuke summary\n'
|
|
printf '===================\n\n'
|
|
printf 'verdict: %s\n' "$verdict"
|
|
printf 'elapsed seconds: %s\n' "$elapsed"
|
|
printf 'dump directory: %s\n' "$DUMP_DIR"
|
|
printf 'capture failures:%s\n' "$cf"
|
|
printf '\nTriage hints\n------------\n'
|
|
printf '* ps.txt STAT column: U or D means stuck in the kernel, not user space\n'
|
|
printf '* disk.txt a full or near-full data volume is a common cause of a wedged daemon -- check free space on the volume backing Docker.raw (df -h) and clear it, or raise the disk image limit in Docker Desktop Settings, before relaunching\n'
|
|
printf '* sample-* user-space stacks showing where the daemon was stuck\n'
|
|
printf '* logs/ Docker host and vm logs from just before the kill\n'
|
|
printf '* restart docker-nuke restarts Docker Desktop itself, not your containers -- only containers started with a restart policy (e.g. unless-stopped) come back automatically; run docker ps -a to check the rest\n'
|
|
} >"$DUMP_DIR/summary.txt"
|
|
} 2>/dev/null || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
warn "docker-nuke: could not write summary to $DUMP_DIR/summary.txt (dump directory unwritable or full)"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
log() { printf '%s\n' "$*"; }
|
|
warn() { printf '%s\n' "$*" >&2; }
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first
|
|
|
|
Usage:
|
|
docker-nuke [options]
|
|
|
|
Options:
|
|
-f, --force Skip the confirmation shown when Docker looks healthy
|
|
-n, --no-restart Kill Docker but do not relaunch it
|
|
--dry-run Report what would be killed and dumped; change nothing
|
|
--deep Also stop root launchd helpers (com.docker.vmnetd);
|
|
requires a cached sudo credential -- run "sudo -v" first
|
|
-h, --help Show this help
|
|
|
|
Diagnostics are written to $DUMP_ROOT/<timestamp>/ before anything is
|
|
killed (override the root with DOCKER_NUKE_DUMP_ROOT). The most recent
|
|
$DUMP_RETENTION dumps are kept.
|
|
|
|
Exit codes:
|
|
0 Docker killed and confirmed responsive again
|
|
1 Daemon did not come back within the restart timeout, or the kill itself
|
|
did not complete
|
|
2 Usage error, or aborted at the healthy-daemon prompt
|
|
EOF
|
|
}
|
|
|
|
parse_args() {
|
|
OPT_FORCE=0
|
|
OPT_NO_RESTART=0
|
|
OPT_DRY_RUN=0
|
|
OPT_DEEP=0
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
-f|--force) OPT_FORCE=1 ;;
|
|
-n|--no-restart) OPT_NO_RESTART=1 ;;
|
|
--dry-run) OPT_DRY_RUN=1 ;;
|
|
--deep) OPT_DEEP=1 ;;
|
|
-h|--help) usage; return 3 ;;
|
|
*)
|
|
warn "docker-nuke: unknown option '$1'"
|
|
usage >&2
|
|
return 2
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# --- entrypoint -------------------------------------------------------------
|
|
|
|
|
|
# docker_version_summary — one-line "<version> (<N> running)" snapshot, or
|
|
# empty if `docker info` fails or hangs. Shared by confirm_healthy_nuke's
|
|
# healthy-daemon prompt and main's post-recovery report (final review fix:
|
|
# previously the same `docker info --format ...` call was duplicated verbatim
|
|
# in both places).
|
|
docker_version_summary() {
|
|
run_timeout "$HEALTH_TIMEOUT" docker info \
|
|
--format '{{.ServerVersion}} ({{.ContainersRunning}} running)' 2>/dev/null || true
|
|
}
|
|
|
|
confirm_healthy_nuke() {
|
|
local info
|
|
info="$(docker_version_summary)"
|
|
log "docker-nuke: Docker looks healthy: ${info:-unknown}"
|
|
log "Nuking will stop running containers."
|
|
printf 'Proceed anyway? [y/N] '
|
|
local reply=""
|
|
read -r reply || true
|
|
case "$reply" in
|
|
y|Y|yes|YES) return 0 ;;
|
|
*)
|
|
# A non-TTY stdin (piped, redirected from /dev/null, closed) can never
|
|
# answer this prompt interactively even on a re-run, so the default
|
|
# deny should point at the actual way past it (final review fix): a
|
|
# bare "aborted" here previously gave no hint that --force exists.
|
|
if [ -t 0 ]; then
|
|
log "docker-nuke: aborted"
|
|
else
|
|
log "docker-nuke: aborted (no interactive input on stdin; use --force to skip this prompt)"
|
|
fi
|
|
return 2
|
|
;;
|
|
esac
|
|
}
|
|
|
|
dry_run_report() {
|
|
local pids
|
|
pids="$(docker_pids | tr '\n' ' ')"
|
|
log "docker-nuke: DRY RUN — no changes made"
|
|
if [ -z "$pids" ]; then
|
|
log " would kill: (no Docker processes found)"
|
|
else
|
|
log " would kill: $pids"
|
|
# shellcheck disable=SC2086
|
|
ps -o pid,stat,%cpu,%mem,etime,comm -p $pids 2>/dev/null || true
|
|
fi
|
|
if [ "$OPT_DEEP" -eq 1 ]; then
|
|
log " would also stop privileged helpers (--deep): com.docker.vmnetd, com.docker.socket"
|
|
fi
|
|
log " would dump to: $DUMP_ROOT/<timestamp>"
|
|
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
|
log " would not restart Docker (--no-restart)"
|
|
else
|
|
log " would restart Docker and wait up to ${RESTART_TIMEOUT}s"
|
|
fi
|
|
if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then
|
|
log " NOTE: Docker currently looks healthy; a real run would prompt for confirmation unless --force is given"
|
|
fi
|
|
}
|
|
|
|
# finish VERDICT RC ELAPSED [EXTRA_LINE ...] — shared tail for every terminal
|
|
# path in main() (final review fix). Writes the summary, prunes old dumps,
|
|
# then logs the verdict, any path-specific EXTRA_LINEs, and finally the dump
|
|
# location -- guarding the case where run_dump itself failed and DUMP_DIR is
|
|
# empty (Fix C1), so the tool can never print "diagnostics in " with nothing
|
|
# after it. Success (RC 0) logs via log(); any other RC logs every line via
|
|
# warn() instead, so a failed kill (Fix C2) can never look identical to a
|
|
# clean run in the terminal output. Collapsing the three near-identical
|
|
# terminal blocks that used to live in main() into this one helper makes it
|
|
# structurally impossible for a future fourth path to forget to thread the
|
|
# kill status into the verdict/exit code, or to forget the empty-DUMP_DIR
|
|
# guard.
|
|
finish() {
|
|
local verdict="$1" rc="$2" elapsed="$3"
|
|
shift 3
|
|
|
|
write_summary "$verdict" "$elapsed"
|
|
prune_dumps
|
|
|
|
local logfn=log
|
|
if [ "$rc" -ne 0 ]; then logfn=warn; fi
|
|
|
|
"$logfn" "docker-nuke: $verdict"
|
|
local line
|
|
for line in "$@"; do
|
|
"$logfn" "$line"
|
|
done
|
|
if [ -n "${DUMP_DIR:-}" ]; then
|
|
"$logfn" "docker-nuke: diagnostics in $DUMP_DIR"
|
|
else
|
|
"$logfn" "docker-nuke: diagnostics were not captured this run"
|
|
fi
|
|
|
|
return "$rc"
|
|
}
|
|
|
|
main() {
|
|
local rc=0
|
|
parse_args "$@" || rc=$?
|
|
if [ "$rc" -eq 3 ]; then return 0; fi
|
|
if [ "$rc" -ne 0 ]; then return "$rc"; fi
|
|
|
|
if [ "$OPT_DRY_RUN" -eq 1 ]; then
|
|
dry_run_report
|
|
return 0
|
|
fi
|
|
|
|
# Fix I2: checked up front, before anything is killed, so the user can fix
|
|
# a missing cached credential while Docker is still alive. Never fatal --
|
|
# stop_privileged_helpers already degrades gracefully (Fix 3) if this was
|
|
# ignored; this just surfaces the problem while it is still actionable
|
|
# instead of two silent `sudo -n` failures after the kill.
|
|
if [ "$OPT_DEEP" -eq 1 ] && ! sudo -n true 2>/dev/null; then
|
|
warn "docker-nuke: --deep requires a cached sudo credential; run \"sudo -v\" first. Continuing without it -- privileged helpers will not be stopped."
|
|
fi
|
|
|
|
# OPT_FORCE checked first (final review fix): docker_healthy can cost up to
|
|
# HEALTH_TIMEOUT (6s) before answering, and --force is exactly the flag
|
|
# reached for when Docker is already known to be wedged -- paying that
|
|
# latency to compute an answer that is about to be skipped anyway serves no
|
|
# purpose.
|
|
if [ "$OPT_FORCE" -eq 0 ] && docker_healthy; then
|
|
confirm_healthy_nuke || return 2
|
|
fi
|
|
|
|
local started elapsed
|
|
started="$(date +%s)"
|
|
|
|
# Fix C1: run_dump is never called bare. It returns 1 when new_dump_dir
|
|
# fails (e.g. dump root unwritable or full) -- precisely the condition it
|
|
# exists to detect -- and under set -e a bare call would abort the entire
|
|
# script right here: Docker is never killed, never restarted. The design
|
|
# requirement is explicit: the dump can never block recovery. A failure is
|
|
# recorded and recovery proceeds with no dump saved, exactly like every
|
|
# individual capture failure inside run_dump already does.
|
|
local dumped=1
|
|
run_dump || { dumped=0; warn "docker-nuke: diagnostics capture failed; continuing with recovery (no dump will be saved)"; }
|
|
local dump_note=""
|
|
if [ "$dumped" -eq 0 ]; then
|
|
dump_note=" (no dump captured)"
|
|
fi
|
|
|
|
# Fix C2: nuke_docker's return status -- 1 only after KILL_RETRY_ROUNDS is
|
|
# exhausted with Docker processes still alive -- was previously discarded
|
|
# with `|| true` and never consulted again. It is captured here and
|
|
# threaded into every terminal path's verdict (so it lands in summary.txt,
|
|
# the file actually read during triage) and exit code, so a failed kill can
|
|
# never be reported as success.
|
|
local killed=1 kill_note=""
|
|
nuke_docker || killed=0
|
|
if [ "$killed" -eq 0 ]; then
|
|
local survivors
|
|
survivors="$(docker_pids | tr '\n' ' ')"
|
|
kill_note=" -- WARNING: kill incomplete, processes survived: ${survivors:-unknown}"
|
|
fi
|
|
|
|
if [ "$OPT_NO_RESTART" -eq 1 ]; then
|
|
local verdict="killed, not restarted (--no-restart)${dump_note}${kill_note}"
|
|
local finish_rc=0
|
|
if [ "$killed" -eq 0 ]; then finish_rc=1; fi
|
|
elapsed=$(( $(date +%s) - started ))
|
|
finish "$verdict" "$finish_rc" "$elapsed"
|
|
return "$finish_rc"
|
|
fi
|
|
|
|
start_docker
|
|
if wait_for_daemon "$RESTART_TIMEOUT"; then
|
|
elapsed=$(( $(date +%s) - started ))
|
|
local info
|
|
info="$(docker_version_summary)"
|
|
local verdict="recovered after ${elapsed}s — ${info:-ok}${dump_note}${kill_note}"
|
|
local finish_rc=0
|
|
if [ "$killed" -eq 0 ]; then finish_rc=1; fi
|
|
finish "$verdict" "$finish_rc" "$elapsed"
|
|
return "$finish_rc"
|
|
fi
|
|
|
|
local verdict="daemon did not return within ${RESTART_TIMEOUT}s${dump_note}${kill_note}"
|
|
elapsed=$(( $(date +%s) - started ))
|
|
local still_running
|
|
still_running="docker-nuke: still running: $(docker_pids | tr '\n' ' ')"
|
|
local deep_hint
|
|
if [ -n "${DUMP_DIR:-}" ]; then
|
|
deep_hint="docker-nuke: try 'docker-nuke --deep', or check disk space in $DUMP_DIR/disk.txt"
|
|
else
|
|
deep_hint="docker-nuke: try 'docker-nuke --deep'; no dump was captured this run to check disk space in"
|
|
fi
|
|
finish "$verdict" 1 "$elapsed" "$still_running" "$deep_hint"
|
|
return 1
|
|
}
|
|
|
|
# Only run when executed, not when sourced by the test harness.
|
|
if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
|
|
main "$@"
|
|
fi
|