mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
feat(docker-nuke): add script scaffold, arg parsing, and test harness
This commit is contained in:
Executable
+123
@@ -0,0 +1,123 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# docker-nuke — capture diagnostics from a wedged Docker Desktop, then
|
||||||
|
# force-kill and restart it.
|
||||||
|
#
|
||||||
|
# Written for bash 3.2 compatibility (macOS /bin/bash).
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# --- configuration ----------------------------------------------------------
|
||||||
|
#
|
||||||
|
# The SC2034 directives below are per-line on purpose, not file-level: each
|
||||||
|
# constant is consumed by a later task, and its directive is deleted at that
|
||||||
|
# point. A file-level disable would silence SC2034 for the whole file forever,
|
||||||
|
# hiding a genuinely dead variable added later. By the final task none of these
|
||||||
|
# directives should remain.
|
||||||
|
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
HEALTH_TIMEOUT=6 # seconds to wait on `docker info` before calling it wedged
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
GRACEFUL_WAIT=8 # seconds to let Docker quit politely
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
TERM_WAIT=5 # seconds to wait after SIGTERM before SIGKILL
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
RESTART_TIMEOUT=90 # seconds to wait for the daemon to come back
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
POLL_INTERVAL=2 # seconds between daemon health polls
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
SAMPLE_DURATION=2 # seconds of stack sampling per process
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
SAMPLE_TIMEOUT=10 # hard cap on a single `sample` invocation
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
DUMP_RETENTION=10 # how many dump directories to keep
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
LOG_TAIL_LINES=2000 # lines of Docker log to copy (keeps dumps small)
|
||||||
|
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
DOCKER_APP="/Applications/Docker.app"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
DOCKER_LOG_DIR="$HOME/Library/Containers/com.docker.docker/Data/log"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
DOCKER_RAW="$HOME/Library/Containers/com.docker.docker/Data/vms/0/data/Docker.raw"
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
CRASH_DIR="$HOME/Library/Logs/DiagnosticReports"
|
||||||
|
|
||||||
|
# Overridable so tests never touch real state.
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
DUMP_ROOT="${DOCKER_NUKE_DUMP_ROOT:-$HOME/.local/state/docker-nuke}"
|
||||||
|
|
||||||
|
OPT_FORCE=0
|
||||||
|
OPT_NO_RESTART=0
|
||||||
|
OPT_DRY_RUN=0
|
||||||
|
OPT_DEEP=0
|
||||||
|
|
||||||
|
# --- helpers ----------------------------------------------------------------
|
||||||
|
|
||||||
|
bin_exists() { command -v "$1" >/dev/null 2>&1; }
|
||||||
|
|
||||||
|
log() { printf '%s\n' "$*"; }
|
||||||
|
warn() { printf '%s\n' "$*" >&2; }
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
docker-nuke: force-restart a wedged Docker Desktop, capturing diagnostics first
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
docker-nuke [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-f, --force Skip the confirmation shown when Docker looks healthy
|
||||||
|
-n, --no-restart Kill Docker but do not relaunch it
|
||||||
|
--dry-run Report what would be killed and dumped; change nothing
|
||||||
|
--deep Also stop root launchd helpers (com.docker.vmnetd); needs sudo
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
Diagnostics are written to ~/.local/state/docker-nuke/<timestamp>/ before
|
||||||
|
anything is killed. The most recent 10 dumps are kept.
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
0 Docker killed and confirmed responsive again
|
||||||
|
1 Daemon did not come back within the restart timeout
|
||||||
|
2 Usage error, or aborted at the healthy-daemon prompt
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
OPT_FORCE=0
|
||||||
|
OPT_NO_RESTART=0
|
||||||
|
OPT_DRY_RUN=0
|
||||||
|
OPT_DEEP=0
|
||||||
|
while [ "$#" -gt 0 ]; do
|
||||||
|
# shellcheck disable=SC2034 # OPT_* are read by main once it is wired up
|
||||||
|
case "$1" in
|
||||||
|
-f|--force) OPT_FORCE=1 ;;
|
||||||
|
-n|--no-restart) OPT_NO_RESTART=1 ;;
|
||||||
|
--dry-run) OPT_DRY_RUN=1 ;;
|
||||||
|
--deep) OPT_DEEP=1 ;;
|
||||||
|
-h|--help) usage; return 3 ;;
|
||||||
|
*)
|
||||||
|
warn "docker-nuke: unknown option '$1'"
|
||||||
|
usage >&2
|
||||||
|
return 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- entrypoint -------------------------------------------------------------
|
||||||
|
|
||||||
|
main() {
|
||||||
|
local rc=0
|
||||||
|
parse_args "$@" || rc=$?
|
||||||
|
if [ "$rc" -eq 3 ]; then return 0; fi
|
||||||
|
if [ "$rc" -ne 0 ]; then return "$rc"; fi
|
||||||
|
log "docker-nuke: not yet implemented"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Only run when executed, not when sourced by the test harness.
|
||||||
|
if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
|
||||||
|
main "$@"
|
||||||
|
fi
|
||||||
Executable
+99
@@ -0,0 +1,99 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Test harness for docker-nuke. No external dependencies.
|
||||||
|
# Run: ./test-docker-nuke.sh
|
||||||
|
|
||||||
|
set -uo pipefail # deliberately not -e: a failing assertion must not abort the run
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
TESTS_RUN=0
|
||||||
|
TESTS_FAILED=0
|
||||||
|
TMPROOT=""
|
||||||
|
|
||||||
|
setup_tmp() {
|
||||||
|
TMPROOT="$(mktemp -d)"
|
||||||
|
export DOCKER_NUKE_DUMP_ROOT="$TMPROOT/dumps"
|
||||||
|
}
|
||||||
|
|
||||||
|
teardown_tmp() {
|
||||||
|
if [ -n "$TMPROOT" ] && [ -d "$TMPROOT" ]; then
|
||||||
|
rm -rf "$TMPROOT"
|
||||||
|
fi
|
||||||
|
TMPROOT=""
|
||||||
|
}
|
||||||
|
|
||||||
|
pass() {
|
||||||
|
TESTS_RUN=$((TESTS_RUN + 1))
|
||||||
|
printf ' ok %s\n' "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
fail() {
|
||||||
|
TESTS_RUN=$((TESTS_RUN + 1))
|
||||||
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||||
|
printf ' FAIL %s\n %s\n' "$1" "$2"
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq() {
|
||||||
|
local desc="$1" expected="$2" actual="$3"
|
||||||
|
if [ "$expected" = "$actual" ]; then
|
||||||
|
pass "$desc"
|
||||||
|
else
|
||||||
|
fail "$desc" "expected [$expected] got [$actual]"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_contains() {
|
||||||
|
local desc="$1" haystack="$2" needle="$3"
|
||||||
|
case "$haystack" in
|
||||||
|
*"$needle"*) pass "$desc" ;;
|
||||||
|
*) fail "$desc" "expected to contain [$needle] got [$haystack]" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_not_contains() {
|
||||||
|
local desc="$1" haystack="$2" needle="$3"
|
||||||
|
case "$haystack" in
|
||||||
|
*"$needle"*) fail "$desc" "expected NOT to contain [$needle] got [$haystack]" ;;
|
||||||
|
*) pass "$desc" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# Source the script under test. Its trailing BASH_SOURCE/$0 guard means main
|
||||||
|
# runs only when the script is executed directly, so sourcing it here defines
|
||||||
|
# the functions without triggering a real nuke.
|
||||||
|
setup_tmp
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "$SCRIPT_DIR/docker-nuke"
|
||||||
|
set +e # the sourced script sets -e; undo it for the harness
|
||||||
|
|
||||||
|
printf '\n=== argument parsing ===\n'
|
||||||
|
|
||||||
|
test_parse_args() {
|
||||||
|
parse_args --force
|
||||||
|
assert_eq "--force sets OPT_FORCE" "1" "$OPT_FORCE"
|
||||||
|
|
||||||
|
parse_args -n
|
||||||
|
assert_eq "-n sets OPT_NO_RESTART" "1" "$OPT_NO_RESTART"
|
||||||
|
|
||||||
|
parse_args --dry-run
|
||||||
|
assert_eq "--dry-run sets OPT_DRY_RUN" "1" "$OPT_DRY_RUN"
|
||||||
|
|
||||||
|
parse_args --deep
|
||||||
|
assert_eq "--deep sets OPT_DEEP" "1" "$OPT_DEEP"
|
||||||
|
|
||||||
|
parse_args
|
||||||
|
assert_eq "no flags leaves OPT_FORCE off" "0" "$OPT_FORCE"
|
||||||
|
|
||||||
|
parse_args --bogus >/dev/null 2>&1
|
||||||
|
assert_eq "unknown flag returns 2" "2" "$?"
|
||||||
|
|
||||||
|
local help_text
|
||||||
|
help_text="$(usage)"
|
||||||
|
assert_contains "usage mentions --dry-run" "$help_text" "--dry-run"
|
||||||
|
assert_contains "usage mentions --deep" "$help_text" "--deep"
|
||||||
|
}
|
||||||
|
test_parse_args
|
||||||
|
|
||||||
|
printf '\n=== summary ===\n'
|
||||||
|
teardown_tmp
|
||||||
|
printf 'ran %d, failed %d\n' "$TESTS_RUN" "$TESTS_FAILED"
|
||||||
|
[ "$TESTS_FAILED" -eq 0 ]
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
# docker-nuke — Design
|
||||||
|
|
||||||
|
**Date:** 2026-07-19
|
||||||
|
**Status:** Approved
|
||||||
|
**Target:** `~/.local/bin/docker-nuke`
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Recover a wedged Docker Desktop on macOS with a single command, and capture enough
|
||||||
|
forensic state beforehand to triage *why* it wedged.
|
||||||
|
|
||||||
|
The script exists because Docker Desktop periodically becomes unresponsive on this
|
||||||
|
machine and the manual recovery (quit app, hunt stray processes, kill, relaunch,
|
||||||
|
wait) is slow and easy to get wrong.
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Observed environment at design time:
|
||||||
|
|
||||||
|
- Docker Desktop is the only runtime. Daemon 29.6.1, 27 running containers.
|
||||||
|
- `/usr/local/bin/docker` is the real CLI.
|
||||||
|
- `~/bin` contains **dangling** OrbStack symlinks (`docker`, `docker-compose`,
|
||||||
|
`kubectl`, `orb`, `orbctl`) pointing into a gutted `~/.orbstack/bin/`.
|
||||||
|
Out of scope here; noted for separate cleanup.
|
||||||
|
- `sample`, `spindump`, `lsof`, `vm_stat`, `fs_usage` all available.
|
||||||
|
- Docker logs: `~/Library/Containers/com.docker.docker/Data/log/{host,vm}`.
|
||||||
|
- **The data volume is 100% full — 653 MB free of 926 GB. `Docker.raw` is 140 GB
|
||||||
|
on disk (215 GB apparent).** This is the likely root cause of the recurring
|
||||||
|
hangs. Out of scope for this script, but it is why disk capture is a
|
||||||
|
first-class part of the dump.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- **No container-state snapshot/replay.** Containers without a restart policy stay
|
||||||
|
stopped after a nuke. Recording running state requires querying the daemon —
|
||||||
|
precisely what is broken when the script is needed — so it would be unreliable
|
||||||
|
exactly when it matters. Docker's own restart policies cover what matters.
|
||||||
|
- **No sudo on the default path.** Root launchd helper cleanup is opt-in via `--deep`
|
||||||
|
so the normal path never blocks on a password prompt.
|
||||||
|
- **No disk cleanup or pruning.** The script never deletes Docker data.
|
||||||
|
- **No fix for the full disk.** Surfaced in the dump, handled separately.
|
||||||
|
|
||||||
|
## Behavior
|
||||||
|
|
||||||
|
### 1. Health check
|
||||||
|
|
||||||
|
Run `docker info` under a 6s hard timeout.
|
||||||
|
|
||||||
|
- **Times out or fails** → Docker is wedged. Proceed immediately, no prompt.
|
||||||
|
- **Succeeds** → print daemon version and container count, then require `y/N`
|
||||||
|
confirmation. `--force` skips the prompt.
|
||||||
|
|
||||||
|
Rationale: zero friction in the case the tool exists for, while a healthy daemon
|
||||||
|
cannot be bounced by an accidental invocation.
|
||||||
|
|
||||||
|
### 2. Pre-kill dump
|
||||||
|
|
||||||
|
Capture happens **before** any kill, because killing destroys the state being
|
||||||
|
triaged. Written to `~/.local/state/docker-nuke/<UTC-timestamp>/`.
|
||||||
|
|
||||||
|
The full dump always runs — there is no flag to skip or reduce it. The only run
|
||||||
|
that produces no dump is `--dry-run`. Stack sampling is included unconditionally,
|
||||||
|
accepting roughly 10-15s of added latency before the kill begins, because stack
|
||||||
|
traces are the one signal that cannot be reconstructed after the processes die.
|
||||||
|
`--no-restart` still produces a complete dump.
|
||||||
|
|
||||||
|
| File | Contents | Diagnostic value |
|
||||||
|
|---|---|---|
|
||||||
|
| `ps.txt` | Docker procs with `STAT`, `%cpu`, `%mem`, elapsed, full command | `U`/`D` state means stuck in kernel, not user space |
|
||||||
|
| `sample-<pid>.txt` | 2s stack samples of `com.docker.backend`, `com.docker.krun` | Shows where it is hung |
|
||||||
|
| `disk.txt` | `df -h` all volumes, `Docker.raw` apparent + on-disk size | Would have caught the current full-disk issue |
|
||||||
|
| `memory.txt` | `vm_stat`, memory pressure | Detects pressure-induced stalls |
|
||||||
|
| `socket.txt` | `docker.sock` existence, permissions, listener | Distinguishes dead daemon from dead socket |
|
||||||
|
| `docker-info.txt` | `docker info` / `docker version` output + whether each timed out | |
|
||||||
|
| `logs/` | Last 2000 lines of Docker host + vm logs | Restart rotates these away |
|
||||||
|
| `crashes/` | Recent Docker entries from `~/Library/Logs/DiagnosticReports` | |
|
||||||
|
| `summary.txt` | Verdict, timings, recovery outcome — written at end of run | The file actually read during triage |
|
||||||
|
|
||||||
|
**Robustness requirement:** every capture step runs under its own timeout. `sample`
|
||||||
|
can itself hang on a truly wedged process. A failed or timed-out step records the
|
||||||
|
failure in `summary.txt` and continues. **The dump can never block recovery.**
|
||||||
|
|
||||||
|
Log tails are capped at 2000 lines to keep dumps small — relevant given only
|
||||||
|
653 MB free.
|
||||||
|
|
||||||
|
Retention: keep the 10 most recent dump directories, prune older ones at the end
|
||||||
|
of a run.
|
||||||
|
|
||||||
|
### 3. Kill, escalating
|
||||||
|
|
||||||
|
Three stages, re-checking liveness between each:
|
||||||
|
|
||||||
|
1. `osascript` quit of Docker Desktop, wait up to 8s — lets it clean up if merely slow
|
||||||
|
2. `SIGTERM` everything still alive, wait up to 5s
|
||||||
|
3. `SIGKILL` the remainder
|
||||||
|
|
||||||
|
### 4. Restart
|
||||||
|
|
||||||
|
`open -a Docker`, then poll `docker info` every 2s up to 90s. Skipped with
|
||||||
|
`--no-restart`.
|
||||||
|
|
||||||
|
### 5. Report
|
||||||
|
|
||||||
|
On success: daemon version, container count, elapsed recovery time, dump path.
|
||||||
|
On failure: what is still running, the dump path, and suggested next steps.
|
||||||
|
|
||||||
|
## Process targeting
|
||||||
|
|
||||||
|
The most correctness-sensitive part. Matched two ways:
|
||||||
|
|
||||||
|
- Anything under `/Applications/Docker.app` (`pgrep -f '/Applications/Docker.app'`)
|
||||||
|
- Exact process names: `com.docker.backend`, `com.docker.build`, `com.docker.krun`,
|
||||||
|
`com.docker.vpnkit`, `com.docker.virtualization`, `vpnkit`
|
||||||
|
- Hung CLI clients: `pgrep -x docker`
|
||||||
|
|
||||||
|
**Critical hazard:** a naive `pgrep -f docker` matches the string `docker` in this
|
||||||
|
script's own name and command line, so `docker-nuke` would kill itself mid-run.
|
||||||
|
Matching is therefore anchored to the app bundle path and exact process names, and
|
||||||
|
`$$` / `$PPID` are always filtered from the result set. `pgrep -f docker` must never
|
||||||
|
be used.
|
||||||
|
|
||||||
|
## Interface
|
||||||
|
|
||||||
|
| Flag | Effect |
|
||||||
|
|---|---|
|
||||||
|
| `-f, --force` | Skip the confirmation when Docker looks healthy |
|
||||||
|
| `-n, --no-restart` | Kill only, leave Docker down |
|
||||||
|
| `--dry-run` | Report what would be killed and dumped; change nothing |
|
||||||
|
| `--deep` | Also stop root launchd helpers (`com.docker.vmnetd`); prompts for sudo |
|
||||||
|
| `-h, --help` | Usage |
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
|
||||||
|
- `0` — Docker killed and (unless `--no-restart`) confirmed responsive again
|
||||||
|
- `1` — daemon did not come back within the restart timeout
|
||||||
|
- `2` — aborted by user at the healthy-daemon confirmation
|
||||||
|
|
||||||
|
## Constants
|
||||||
|
|
||||||
|
```
|
||||||
|
HEALTH_TIMEOUT=6 GRACEFUL_WAIT=8 TERM_WAIT=5
|
||||||
|
RESTART_TIMEOUT=90 POLL_INTERVAL=2 SAMPLE_DURATION=2
|
||||||
|
SAMPLE_TIMEOUT=10 DUMP_RETENTION=10 LOG_TAIL_LINES=2000
|
||||||
|
DUMP_ROOT="$HOME/.local/state/docker-nuke"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Implementation notes
|
||||||
|
|
||||||
|
- Bash. `timeout` is available via coreutils but must not be assumed — provide a
|
||||||
|
`run_timeout` helper that uses `timeout`/`gtimeout` when present and falls back
|
||||||
|
to a background-process-and-poll implementation.
|
||||||
|
- Care with `set -euo pipefail`: `pgrep` exits non-zero when it matches nothing,
|
||||||
|
which is a normal condition here, not an error.
|
||||||
|
|
||||||
|
## Verification plan
|
||||||
|
|
||||||
|
Docker is healthy with 27 running containers, so testing is staged:
|
||||||
|
|
||||||
|
1. Safe, non-disruptive: `--help`, `--dry-run`, health check on a healthy daemon,
|
||||||
|
the confirmation guard, dump creation and contents, retention pruning.
|
||||||
|
2. Disruptive: a real end-to-end nuke. Bounces all 27 containers. Requires explicit
|
||||||
|
user go-ahead before running.
|
||||||
Reference in New Issue
Block a user