mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
feat(docker-nuke): add script scaffold, arg parsing, and test harness
This commit is contained in:
David Ibia
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
# docker-nuke — capture diagnostics from a wedged Docker Desktop, then
|
||||
# force-kill and restart it.
|
||||
#
|
||||
# Written for bash 3.2 compatibility (macOS /bin/bash).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --- configuration ----------------------------------------------------------
|
||||
#
|
||||
# The SC2034 directives below are per-line on purpose, not file-level: each
|
||||
# constant is consumed by a later task, and its directive is deleted at that
|
||||
# point. A file-level disable would silence SC2034 for the whole file forever,
|
||||
# hiding a genuinely dead variable added later. By the final task none of these
|
||||
# directives should remain.
|
||||
|
||||
# shellcheck disable=SC2034
|
||||
HEALTH_TIMEOUT=6 # seconds to wait on `docker info` before calling it wedged
|
||||
# shellcheck disable=SC2034
|
||||
GRACEFUL_WAIT=8 # seconds to let Docker quit politely
|
||||
# shellcheck disable=SC2034
|
||||
TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL
|
||||
# shellcheck disable=SC2034
|
||||
RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back
|
||||
# shellcheck disable=SC2034
|
||||
POLL_INTERVAL=2 # seconds between daemon health polls
|
||||
# shellcheck disable=SC2034
|
||||
SAMPLE_DURATION=2 # seconds of stack sampling per process
|
||||
# shellcheck disable=SC2034
|
||||
SAMPLE_TIMEOUT=10 # hard cap on a single `sample` invocation
|
||||
# shellcheck disable=SC2034
|
||||
DUMP_RETENTION=10 # how many dump directories to keep
|
||||
# shellcheck disable=SC2034
|
||||
LOG_TAIL_LINES=2000 # lines of Docker log to copy (keeps dumps small)
|
||||
|
||||
# shellcheck disable=SC2034
|
||||
DOCKER_APP="/Applications/Docker.app"
|
||||
# shellcheck disable=SC2034
|
||||
DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log"
|
||||
# shellcheck disable=SC2034
|
||||
DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw"
|
||||
# shellcheck disable=SC2034
|
||||
CRASH_DIR="$HOME/Library/Logs/DiagnosticReports"
|
||||
|
||||
# Overridable so tests never touch real state.
|
||||
# shellcheck disable=SC2034
|
||||
DUMP_ROOT="${DOCKER_NUKE_DUMP_ROOT:-$HOME/.local/state/docker-nuke}"
|
||||
|
||||
OPT_FORCE=0
|
||||
OPT_NO_RESTART=0
|
||||
OPT_DRY_RUN=0
|
||||
OPT_DEEP=0
|
||||
|
||||
# --- helpers ----------------------------------------------------------------
|
||||
|
||||
bin_exists() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
log() { printf '%s\n' "$*"; }
|
||||
warn() { printf '%s\n' "$*" >&2; }
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first
|
||||
|
||||
Usage:
|
||||
docker-nuke [options]
|
||||
|
||||
Options:
|
||||
-f, --force Skip the confirmation shown when Docker looks healthy
|
||||
-n, --no-restart Kill Docker but do not relaunch it
|
||||
--dry-run Report what would be killed and dumped; change nothing
|
||||
--deep Also stop root launchd helpers (com.docker.vmnetd); needs sudo
|
||||
-h, --help Show this help
|
||||
|
||||
Diagnostics are written to ~/.local/state/docker-nuke/<timestamp>/ before
|
||||
anything is killed. The most recent 10 dumps are kept.
|
||||
|
||||
Exit codes:
|
||||
0 Docker killed and confirmed responsive again
|
||||
1 Daemon did not come back within the restart timeout
|
||||
2 Usage error, or aborted at the healthy-daemon prompt
|
||||
EOF
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
OPT_FORCE=0
|
||||
OPT_NO_RESTART=0
|
||||
OPT_DRY_RUN=0
|
||||
OPT_DEEP=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
# shellcheck disable=SC2034 # OPT_* are read by main once it is wired up
|
||||
case "$1" in
|
||||
-f|--force) OPT_FORCE=1 ;;
|
||||
-n|--no-restart) OPT_NO_RESTART=1 ;;
|
||||
--dry-run) OPT_DRY_RUN=1 ;;
|
||||
--deep) OPT_DEEP=1 ;;
|
||||
-h|--help) usage; return 3 ;;
|
||||
*)
|
||||
warn "docker-nuke: unknown option '$1'"
|
||||
usage >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- entrypoint -------------------------------------------------------------
|
||||
|
||||
main() {
|
||||
local rc=0
|
||||
parse_args "$@" || rc=$?
|
||||
if [ "$rc" -eq 3 ]; then return 0; fi
|
||||
if [ "$rc" -ne 0 ]; then return "$rc"; fi
|
||||
log "docker-nuke: not yet implemented"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Only run when executed, not when sourced by the test harness.
|
||||
if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
|
||||
main "$@"
|
||||
fi
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test harness for docker-nuke. No external dependencies.
|
||||
# Run: ./test-docker-nuke.sh
|
||||
|
||||
set -uo pipefail # deliberately not -e: a failing assertion must not abort the run
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TESTS_RUN=0
|
||||
TESTS_FAILED=0
|
||||
TMPROOT=""
|
||||
|
||||
setup_tmp() {
|
||||
TMPROOT="$(mktemp -d)"
|
||||
export DOCKER_NUKE_DUMP_ROOT="$TMPROOT/dumps"
|
||||
}
|
||||
|
||||
teardown_tmp() {
|
||||
if [ -n "$TMPROOT" ] && [ -d "$TMPROOT" ]; then
|
||||
rm -rf "$TMPROOT"
|
||||
fi
|
||||
TMPROOT=""
|
||||
}
|
||||
|
||||
pass() {
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
printf ' ok %s\n' "$1"
|
||||
}
|
||||
|
||||
fail() {
|
||||
TESTS_RUN=$((TESTS_RUN + 1))
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
printf ' FAIL %s\n %s\n' "$1" "$2"
|
||||
}
|
||||
|
||||
assert_eq() {
|
||||
local desc="$1" expected="$2" actual="$3"
|
||||
if [ "$expected" = "$actual" ]; then
|
||||
pass "$desc"
|
||||
else
|
||||
fail "$desc" "expected [$expected] got [$actual]"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local desc="$1" haystack="$2" needle="$3"
|
||||
case "$haystack" in
|
||||
*"$needle"*) pass "$desc" ;;
|
||||
*) fail "$desc" "expected to contain [$needle] got [$haystack]" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
assert_not_contains() {
|
||||
local desc="$1" haystack="$2" needle="$3"
|
||||
case "$haystack" in
|
||||
*"$needle"*) fail "$desc" "expected NOT to contain [$needle] got [$haystack]" ;;
|
||||
*) pass "$desc" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Source the script under test. Its trailing BASH_SOURCE/$0 guard means main
|
||||
# runs only when the script is executed directly, so sourcing it here defines
|
||||
# the functions without triggering a real nuke.
|
||||
setup_tmp
|
||||
# shellcheck source=/dev/null
|
||||
source "$SCRIPT_DIR/docker-nuke"
|
||||
set +e # the sourced script sets -e; undo it for the harness
|
||||
|
||||
printf '\n=== argument parsing ===\n'
|
||||
|
||||
test_parse_args() {
|
||||
parse_args --force
|
||||
assert_eq "--force sets OPT_FORCE" "1" "$OPT_FORCE"
|
||||
|
||||
parse_args -n
|
||||
assert_eq "-n sets OPT_NO_RESTART" "1" "$OPT_NO_RESTART"
|
||||
|
||||
parse_args --dry-run
|
||||
assert_eq "--dry-run sets OPT_DRY_RUN" "1" "$OPT_DRY_RUN"
|
||||
|
||||
parse_args --deep
|
||||
assert_eq "--deep sets OPT_DEEP" "1" "$OPT_DEEP"
|
||||
|
||||
parse_args
|
||||
assert_eq "no flags leaves OPT_FORCE off" "0" "$OPT_FORCE"
|
||||
|
||||
parse_args --bogus >/dev/null 2>&1
|
||||
assert_eq "unknown flag returns 2" "2" "$?"
|
||||
|
||||
local help_text
|
||||
help_text="$(usage)"
|
||||
assert_contains "usage mentions --dry-run" "$help_text" "--dry-run"
|
||||
assert_contains "usage mentions --deep" "$help_text" "--deep"
|
||||
}
|
||||
test_parse_args
|
||||
|
||||
printf '\n=== summary ===\n'
|
||||
teardown_tmp
|
||||
printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"
|
||||
[ "$TESTS_FAILED" -eq 0 ]
|
||||
@@ -0,0 +1,162 @@
|
||||
# docker-nuke — Design
|
||||
|
||||
**Date:** 2026-07-19
|
||||
**Status:** Approved
|
||||
**Target:** `~/.local/bin/docker-nuke`
|
||||
|
||||
## Purpose
|
||||
|
||||
Recover a wedged Docker Desktop on macOS with a single command, and capture enough
|
||||
forensic state beforehand to triage *why* it wedged.
|
||||
|
||||
The script exists because Docker Desktop periodically becomes unresponsive on this
|
||||
machine and the manual recovery (quit app, hunt stray processes, kill, relaunch,
|
||||
wait) is slow and easy to get wrong.
|
||||
|
||||
## Context
|
||||
|
||||
Observed environment at design time:
|
||||
|
||||
- Docker Desktop is the only runtime. Daemon 29.6.1, 27 running containers.
|
||||
- `/usr/local/bin/docker` is the real CLI.
|
||||
- `~/bin` contains **dangling** OrbStack symlinks (`docker`, `docker-compose`,
|
||||
`kubectl`, `orb`, `orbctl`) pointing into a gutted `~/.orbstack/bin/`.
|
||||
Out of scope here; noted for separate cleanup.
|
||||
- `sample`, `spindump`, `lsof`, `vm_stat`, `fs_usage` all available.
|
||||
- Docker logs: `~/Library/Containers/com.docker.docker/Data/log/{host,vm}`.
|
||||
- **The data volume is 100% full — 653 MB free of 926 GB. `Docker.raw` is 140 GB
|
||||
on disk (215 GB apparent).** This is the likely root cause of the recurring
|
||||
hangs. Out of scope for this script, but it is why disk capture is a
|
||||
first-class part of the dump.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **No container-state snapshot/replay.** Containers without a restart policy stay
|
||||
stopped after a nuke. Recording running state requires querying the daemon —
|
||||
precisely what is broken when the script is needed — so it would be unreliable
|
||||
exactly when it matters. Docker's own restart policies cover what matters.
|
||||
- **No sudo on the default path.** Root launchd helper cleanup is opt-in via `--deep`
|
||||
so the normal path never blocks on a password prompt.
|
||||
- **No disk cleanup or pruning.** The script never deletes Docker data.
|
||||
- **No fix for the full disk.** Surfaced in the dump, handled separately.
|
||||
|
||||
## Behavior
|
||||
|
||||
### 1. Health check
|
||||
|
||||
Run `docker info` under a 6s hard timeout.
|
||||
|
||||
- **Times out or fails** → Docker is wedged. Proceed immediately, no prompt.
|
||||
- **Succeeds** → print daemon version and container count, then require `y/N`
|
||||
confirmation. `--force` skips the prompt.
|
||||
|
||||
Rationale: zero friction in the case the tool exists for, while a healthy daemon
|
||||
cannot be bounced by an accidental invocation.
|
||||
|
||||
### 2. Pre-kill dump
|
||||
|
||||
Capture happens **before** any kill, because killing destroys the state being
|
||||
triaged. Written to `~/.local/state/docker-nuke/<UTC-timestamp>/`.
|
||||
|
||||
The full dump always runs — there is no flag to skip or reduce it. The only run
|
||||
that produces no dump is `--dry-run`. Stack sampling is included unconditionally,
|
||||
accepting roughly 10-15s of added latency before the kill begins, because stack
|
||||
traces are the one signal that cannot be reconstructed after the processes die.
|
||||
`--no-restart` still produces a complete dump.
|
||||
|
||||
| File | Contents | Diagnostic value |
|
||||
|---|---|---|
|
||||
| `ps.txt` | Docker procs with `STAT`, `%cpu`, `%mem`, elapsed, full command | `U`/`D` state means stuck in kernel, not user space |
|
||||
| `sample-<pid>.txt` | 2s stack samples of `com.docker.backend`, `com.docker.krun` | Shows where it is hung |
|
||||
| `disk.txt` | `df -h` all volumes, `Docker.raw` apparent + on-disk size | Would have caught the current full-disk issue |
|
||||
| `memory.txt` | `vm_stat`, memory pressure | Detects pressure-induced stalls |
|
||||
| `socket.txt` | `docker.sock` existence, permissions, listener | Distinguishes dead daemon from dead socket |
|
||||
| `docker-info.txt` | `docker info` / `docker version` output + whether each timed out | |
|
||||
| `logs/` | Last 2000 lines of Docker host + vm logs | Restart rotates these away |
|
||||
| `crashes/` | Recent Docker entries from `~/Library/Logs/DiagnosticReports` | |
|
||||
| `summary.txt` | Verdict, timings, recovery outcome — written at end of run | The file actually read during triage |
|
||||
|
||||
**Robustness requirement:** every capture step runs under its own timeout. `sample`
|
||||
can itself hang on a truly wedged process. A failed or timed-out step records the
|
||||
failure in `summary.txt` and continues. **The dump can never block recovery.**
|
||||
|
||||
Log tails are capped at 2000 lines to keep dumps small — relevant given only
|
||||
653 MB free.
|
||||
|
||||
Retention: keep the 10 most recent dump directories, prune older ones at the end
|
||||
of a run.
|
||||
|
||||
### 3. Kill, escalating
|
||||
|
||||
Three stages, re-checking liveness between each:
|
||||
|
||||
1. `osascript` quit of Docker Desktop, wait up to 8s — lets it clean up if merely slow
|
||||
2. `SIGTERM` everything still alive, wait up to 5s
|
||||
3. `SIGKILL` the remainder
|
||||
|
||||
### 4. Restart
|
||||
|
||||
`open -a Docker`, then poll `docker info` every 2s up to 90s. Skipped with
|
||||
`--no-restart`.
|
||||
|
||||
### 5. Report
|
||||
|
||||
On success: daemon version, container count, elapsed recovery time, dump path.
|
||||
On failure: what is still running, the dump path, and suggested next steps.
|
||||
|
||||
## Process targeting
|
||||
|
||||
The most correctness-sensitive part. Matched two ways:
|
||||
|
||||
- Anything under `/Applications/Docker.app` (`pgrep -f '/Applications/Docker.app'`)
|
||||
- Exact process names: `com.docker.backend`, `com.docker.build`, `com.docker.krun`,
|
||||
`com.docker.vpnkit`, `com.docker.virtualization`, `vpnkit`
|
||||
- Hung CLI clients: `pgrep -x docker`
|
||||
|
||||
**Critical hazard:** a naive `pgrep -f docker` matches the string `docker` in this
|
||||
script's own name and command line, so `docker-nuke` would kill itself mid-run.
|
||||
Matching is therefore anchored to the app bundle path and exact process names, and
|
||||
`$$` / `$PPID` are always filtered from the result set. `pgrep -f docker` must never
|
||||
be used.
|
||||
|
||||
## Interface
|
||||
|
||||
| Flag | Effect |
|
||||
|---|---|
|
||||
| `-f, --force` | Skip the confirmation when Docker looks healthy |
|
||||
| `-n, --no-restart` | Kill only, leave Docker down |
|
||||
| `--dry-run` | Report what would be killed and dumped; change nothing |
|
||||
| `--deep` | Also stop root launchd helpers (`com.docker.vmnetd`); prompts for sudo |
|
||||
| `-h, --help` | Usage |
|
||||
|
||||
Exit codes:
|
||||
|
||||
- `0` — Docker killed and (unless `--no-restart`) confirmed responsive again
|
||||
- `1` — daemon did not come back within the restart timeout
|
||||
- `2` — aborted by user at the healthy-daemon confirmation
|
||||
|
||||
## Constants
|
||||
|
||||
```
|
||||
HEALTH_TIMEOUT=6 GRACEFUL_WAIT=8 TERM_WAIT=5
|
||||
RESTART_TIMEOUT=90 POLL_INTERVAL=2 SAMPLE_DURATION=2
|
||||
SAMPLE_TIMEOUT=10 DUMP_RETENTION=10 LOG_TAIL_LINES=2000
|
||||
DUMP_ROOT="$HOME/.local/state/docker-nuke"
|
||||
```
|
||||
|
||||
## Implementation notes
|
||||
|
||||
- Bash. `timeout` is available via coreutils but must not be assumed — provide a
|
||||
`run_timeout` helper that uses `timeout`/`gtimeout` when present and falls back
|
||||
to a background-process-and-poll implementation.
|
||||
- Care with `set -euo pipefail`: `pgrep` exits non-zero when it matches nothing,
|
||||
which is a normal condition here, not an error.
|
||||
|
||||
## Verification plan
|
||||
|
||||
Docker is healthy with 27 running containers, so testing is staged:
|
||||
|
||||
1. Safe, non-disruptive: `--help`, `--dry-run`, health check on a healthy daemon,
|
||||
the confirmation guard, dump creation and contents, retention pruning.
|
||||
2. Disruptive: a real end-to-end nuke. Bounces all 27 containers. Requires explicit
|
||||
user go-ahead before running.
|
||||
Reference in New Issue
Block a user