mirror of
https://github.com/boxpositron/absolute-dotfiles.git
synced 2026-10-24 08:23:19 +00:00
feat(bin): run impeccable URL scans on Browserless on flowstate-remote
impeccable-browser stands in for Chrome via IMPECCABLE_BROWSER. Public URLs render on Browserless with the key from the Reoclo project flowstate-tools; localhost and private origins reach it through a cloudflared quick tunnel that lives only for the scan, with the CDP traffic rewritten so reports name the local URL. Anything else, or a failed tunnel or Browserless connection, falls back to local headless Chrome with --no-sandbox. It is Linux only, so setup-mac.sh does not link it.
This commit is contained in:
David Ibia
Executable
+287
@@ -0,0 +1,287 @@
|
||||
#!/bin/sh
|
||||
// 2>/dev/null; b="$HOME/.bun/bin/bun"; [ -x "$b" ] || b=bun; exec "$b" "$0" "$@"
|
||||
// (sh runs the line above and execs Bun on this file; Bun reads it as a comment.)
|
||||
//
|
||||
// Chrome stand-in for the impeccable engine, wired in via IMPECCABLE_BROWSER. Linux only: it reads
|
||||
// the engine's argv from /proc and finds Chrome in the Linux Puppeteer and Playwright caches, so
|
||||
// setup-mac.sh does not link it. On flowstate-remote, ~/.zshenv exports
|
||||
// IMPECCABLE_BROWSER="$HOME/.local/bin/impeccable-browser".
|
||||
//
|
||||
// The engine launches its browser the way Puppeteer does (--remote-debugging-port=0, then reads
|
||||
// "DevTools listening on ws://..." from stderr). Here, URL scans run on Browserless (key
|
||||
// BROWSERLESS_API_KEY in the Reoclo project flowstate-tools) through a local CDP proxy. Local and
|
||||
// private origins (localhost, LAN, Tailscale) are exposed to Browserless through a cloudflared
|
||||
// quick tunnel for the length of the scan; the proxy swaps the local origin for the tunnel origin
|
||||
// on the way out and back, so the engine and its report only ever see the local URL.
|
||||
// Anything else (non-detect launches, or Browserless or the tunnel failing) runs on local Chrome.
|
||||
//
|
||||
// Env overrides:
|
||||
// IMPECCABLE_BROWSER_MODE auto (default) | remote | local
|
||||
// IMPECCABLE_TUNNEL 1 (default) tunnels private origins; 0 sends them to local Chrome
|
||||
// BROWSERLESS_TOKEN use this key instead of the Reoclo secret
|
||||
// BROWSERLESS_URL default wss://production-sfo.browserless.io
|
||||
// BROWSERLESS_TIMEOUT_MS Browserless session cap, default 600000
|
||||
// IMPECCABLE_LOCAL_CHROME local Chrome binary, default newest cached headless shell
|
||||
// Decisions are appended to ~/.impeccable/browser.log.
|
||||
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import type { Subprocess } from "bun";
|
||||
|
||||
const HOME = process.env.HOME ?? "/home/ubuntu";
|
||||
const STATE_DIR = `${HOME}/.impeccable`;
|
||||
const TOKEN_CACHE = `${STATE_DIR}/browserless.token`;
|
||||
const TOKEN_TTL_MS = 12 * 60 * 60 * 1000;
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
mkdirSync(STATE_DIR, { recursive: true });
|
||||
const note = (msg: string) =>
|
||||
appendFileSync(`${STATE_DIR}/browser.log`, `${new Date().toISOString()} pid=${process.pid} ${msg}\n`);
|
||||
const bin = (name: string) => (existsSync(`${HOME}/.local/bin/${name}`) ? `${HOME}/.local/bin/${name}` : name);
|
||||
|
||||
function isPrivateHost(host: string): boolean {
|
||||
const h = host.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
return (
|
||||
h === "localhost" ||
|
||||
h === "0.0.0.0" ||
|
||||
h === "::1" ||
|
||||
/\.(localhost|local|internal|lan)$/.test(h) ||
|
||||
/^127\./.test(h) ||
|
||||
/^10\./.test(h) ||
|
||||
/^192\.168\./.test(h) ||
|
||||
/^169\.254\./.test(h) ||
|
||||
/^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
|
||||
/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(h) ||
|
||||
/^f[cd][0-9a-f]{2}:/.test(h)
|
||||
);
|
||||
}
|
||||
|
||||
type Plan = { mode: "remote" | "local"; why: string; privateOrigins: string[] };
|
||||
|
||||
// The engine binary is our parent; its argv says what is being scanned.
|
||||
function plan(): Plan {
|
||||
const forced = process.env.IMPECCABLE_BROWSER_MODE;
|
||||
if (forced === "local") return { mode: "local", why: "IMPECCABLE_BROWSER_MODE", privateOrigins: [] };
|
||||
let argv: string[];
|
||||
try {
|
||||
argv = readFileSync(`/proc/${process.ppid}/cmdline`, "utf8").split("\0").filter(Boolean);
|
||||
} catch {
|
||||
return { mode: "local", why: "parent argv unreadable", privateOrigins: [] };
|
||||
}
|
||||
const at = argv.indexOf("detect");
|
||||
if (at < 0) return { mode: "local", why: `not a detect run (${argv.slice(1, 3).join(" ")})`, privateOrigins: [] };
|
||||
const valueFlags = new Set(["--scope", "--viewport", "--target"]);
|
||||
const targets: string[] = [];
|
||||
for (let i = at + 1; i < argv.length; i++) {
|
||||
if (valueFlags.has(argv[i])) i++;
|
||||
else if (!argv[i].startsWith("-")) targets.push(argv[i]);
|
||||
}
|
||||
if (targets.length === 0) return { mode: "local", why: "no targets", privateOrigins: [] };
|
||||
const privateOrigins = new Set<string>();
|
||||
for (const t of targets) {
|
||||
if (!/^https?:\/\//i.test(t)) return { mode: "local", why: `file target ${t}`, privateOrigins: [] };
|
||||
const url = new URL(t);
|
||||
if (isPrivateHost(url.hostname)) {
|
||||
if (process.env.IMPECCABLE_TUNNEL === "0") return { mode: "local", why: `private host ${t}, tunnels off`, privateOrigins: [] };
|
||||
privateOrigins.add(url.origin);
|
||||
}
|
||||
}
|
||||
return { mode: "remote", why: targets.join(" "), privateOrigins: [...privateOrigins] };
|
||||
}
|
||||
|
||||
function localChrome(): string {
|
||||
const pinned = process.env.IMPECCABLE_LOCAL_CHROME;
|
||||
if (pinned) return pinned;
|
||||
const scan = (pattern: string, cwd: string) =>
|
||||
existsSync(cwd) ? [...new Bun.Glob(pattern).scanSync({ cwd, absolute: true, onlyFiles: true })] : [];
|
||||
const newestFirst = (paths: string[]) => paths.sort((a, b) => b.localeCompare(a, undefined, { numeric: true }));
|
||||
const found = [
|
||||
...newestFirst(scan("*/chrome-headless-shell-linux64/chrome-headless-shell", `${HOME}/.cache/puppeteer/chrome-headless-shell`)),
|
||||
...newestFirst(scan("chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell", `${HOME}/.cache/ms-playwright`)),
|
||||
];
|
||||
if (found.length === 0) throw new Error("no local Chrome found; set IMPECCABLE_LOCAL_CHROME");
|
||||
return found[0];
|
||||
}
|
||||
|
||||
type Tunnel = { local: string; remote: string; proc: Subprocess };
|
||||
const tunnels: Tunnel[] = [];
|
||||
process.on("exit", () => tunnels.forEach((t) => t.proc.kill()));
|
||||
|
||||
async function runLocal(why: string): Promise<never> {
|
||||
tunnels.forEach((t) => t.proc.kill());
|
||||
const chrome = localChrome();
|
||||
note(`local ${chrome} (${why})`);
|
||||
// This host's AppArmor blocks unprivileged user namespaces, so Chrome's sandbox cannot start.
|
||||
const flags = args.includes("--no-sandbox") ? args : [...args, "--no-sandbox"];
|
||||
const child = Bun.spawn([chrome, ...flags], { stdin: "inherit", stdout: "inherit", stderr: "inherit" });
|
||||
for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(sig, () => child.kill(sig));
|
||||
process.exit(await child.exited);
|
||||
}
|
||||
|
||||
function withTimeout<T>(p: Promise<T>, ms: number, what: string): Promise<T> {
|
||||
return Promise.race([p, new Promise<never>((_, reject) => setTimeout(() => reject(new Error(`${what} timed out after ${ms / 1000}s`)), ms))]);
|
||||
}
|
||||
|
||||
// A cloudflared quick tunnel to one local origin, alive until this shim exits. The sh wrapper is a
|
||||
// watchdog: it stops cloudflared when this process dies, even by SIGKILL, so no tunnel outlives a scan.
|
||||
async function openTunnel(local: string): Promise<Tunnel> {
|
||||
const watchdog =
|
||||
'trap \'kill $c 2>/dev/null; exit 0\' TERM; "$0" tunnel --no-autoupdate --url "$1" --http-host-header "$2" & c=$!; ' +
|
||||
'while kill -0 "$3" 2>/dev/null && kill -0 $c 2>/dev/null; do sleep 1; done; kill $c 2>/dev/null';
|
||||
const proc = Bun.spawn(["sh", "-c", watchdog, bin("cloudflared"), local, new URL(local).host, String(process.pid)], {
|
||||
stdin: "ignore",
|
||||
stdout: "ignore",
|
||||
stderr: "pipe",
|
||||
});
|
||||
const tunnel: Tunnel = { local, remote: "", proc };
|
||||
tunnels.push(tunnel);
|
||||
const reader = proc.stderr.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let log = "";
|
||||
await withTimeout(
|
||||
(async () => {
|
||||
while (!(tunnel.remote && /Registered tunnel connection/.test(log))) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) throw new Error(`cloudflared exited: ${log.trim().split("\n").slice(-2).join(" | ")}`);
|
||||
log += decoder.decode(value);
|
||||
tunnel.remote ||= log.match(/https:\/\/[a-z0-9-]+\.trycloudflare\.com/)?.[0] ?? "";
|
||||
}
|
||||
})(),
|
||||
30_000,
|
||||
"cloudflared quick tunnel",
|
||||
);
|
||||
// Keep draining cloudflared's log so a full pipe never stalls it.
|
||||
(async () => {
|
||||
while (!(await reader.read()).done);
|
||||
})();
|
||||
// The edge can answer 530 for a moment after registration; wait until it routes to the origin.
|
||||
await withTimeout(
|
||||
(async () => {
|
||||
for (;;) {
|
||||
const status = await fetch(tunnel.remote, { redirect: "manual" }).then((r) => r.status, () => 0);
|
||||
if (status && status !== 530 && status !== 502) return;
|
||||
await Bun.sleep(500);
|
||||
}
|
||||
})(),
|
||||
20_000,
|
||||
`tunnel ${tunnel.remote} readiness`,
|
||||
);
|
||||
return tunnel;
|
||||
}
|
||||
|
||||
async function browserlessKey(fresh: boolean): Promise<string> {
|
||||
const fromEnv = process.env.BROWSERLESS_TOKEN;
|
||||
if (fromEnv && !fresh) return fromEnv;
|
||||
if (!fresh && existsSync(TOKEN_CACHE) && Date.now() - statSync(TOKEN_CACHE).mtimeMs < TOKEN_TTL_MS) {
|
||||
return readFileSync(TOKEN_CACHE, "utf8").trim();
|
||||
}
|
||||
const proc = Bun.spawn([bin("reoclo"), "secrets", "get", "BROWSERLESS_API_KEY", "--project", "flowstate-tools"], {
|
||||
stdin: "ignore",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
const [out, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
|
||||
const key = out.trim();
|
||||
if (code !== 0 || !key) throw new Error(`reoclo secrets get exited ${code}: ${err.trim().slice(0, 200)}`);
|
||||
writeFileSync(TOKEN_CACHE, key, { mode: 0o600 });
|
||||
return key;
|
||||
}
|
||||
|
||||
function openUpstream(key: string): Promise<WebSocket> {
|
||||
const url = new URL(process.env.BROWSERLESS_URL ?? "wss://production-sfo.browserless.io");
|
||||
url.searchParams.set("token", key);
|
||||
url.searchParams.set("timeout", process.env.BROWSERLESS_TIMEOUT_MS ?? "600000");
|
||||
return new Promise((resolve, reject) => {
|
||||
const ws = new WebSocket(url.toString());
|
||||
ws.binaryType = "arraybuffer";
|
||||
const timer = setTimeout(() => {
|
||||
ws.close();
|
||||
reject(new Error("connect timed out after 20s"));
|
||||
}, 20_000);
|
||||
ws.onopen = () => {
|
||||
clearTimeout(timer);
|
||||
resolve(ws);
|
||||
};
|
||||
ws.onerror = () => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error("connect refused (bad key, quota, or network)"));
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function connectBrowserless(): Promise<WebSocket> {
|
||||
try {
|
||||
return await openUpstream(await browserlessKey(false));
|
||||
} catch (first) {
|
||||
try {
|
||||
return await openUpstream(await browserlessKey(true));
|
||||
} catch (second) {
|
||||
throw new Error(`browserless unavailable: ${(first as Error).message}; retry: ${(second as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const decided = plan();
|
||||
if (decided.mode === "local") await runLocal(decided.why);
|
||||
|
||||
let upstream: WebSocket;
|
||||
try {
|
||||
[upstream] = await Promise.all([connectBrowserless(), ...decided.privateOrigins.map(openTunnel)]);
|
||||
} catch (error) {
|
||||
await runLocal((error as Error).message);
|
||||
}
|
||||
const routes = tunnels.map((t) => `${t.local} via ${t.remote}`).join(", ");
|
||||
note(`browserless (${decided.why})${routes ? ` tunnels: ${routes}` : ""}`);
|
||||
|
||||
// Longest origin first, so http://127.0.0.1:80 never rewrites inside http://127.0.0.1:8080.
|
||||
const byLength = [...tunnels].sort((a, b) => b.local.length - a.local.length);
|
||||
const outbound = (msg: string) => byLength.reduce((text, t) => text.replaceAll(t.local, t.remote), msg);
|
||||
const inbound = (msg: string) => byLength.reduce((text, t) => text.replaceAll(t.remote, t.local), msg);
|
||||
|
||||
const exit = () => {
|
||||
try {
|
||||
upstream.close();
|
||||
} catch {}
|
||||
process.exit(0);
|
||||
};
|
||||
for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"] as const) process.on(sig, exit);
|
||||
|
||||
const browserId = crypto.randomUUID();
|
||||
let client: import("bun").ServerWebSocket<unknown> | null = null;
|
||||
const backlog: (string | ArrayBuffer)[] = [];
|
||||
upstream.onmessage = (event) => {
|
||||
const data = typeof event.data === "string" ? inbound(event.data) : event.data;
|
||||
if (client) client.send(data);
|
||||
else backlog.push(data);
|
||||
};
|
||||
upstream.onclose = () => {
|
||||
client?.close();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
const server = Bun.serve({
|
||||
hostname: "127.0.0.1",
|
||||
port: 0,
|
||||
fetch(req, srv) {
|
||||
if (new URL(req.url).pathname === "/json/version") {
|
||||
return Response.json({
|
||||
Browser: "Browserless",
|
||||
webSocketDebuggerUrl: `ws://127.0.0.1:${srv.port}/devtools/browser/${browserId}`,
|
||||
});
|
||||
}
|
||||
if (srv.upgrade(req)) return;
|
||||
return new Response("not found", { status: 404 });
|
||||
},
|
||||
websocket: {
|
||||
maxPayloadLength: 512 * 1024 * 1024,
|
||||
open(ws) {
|
||||
client = ws;
|
||||
for (const msg of backlog.splice(0)) ws.send(msg);
|
||||
},
|
||||
message(_ws, msg) {
|
||||
upstream.send(typeof msg === "string" ? outbound(msg) : msg);
|
||||
},
|
||||
close: exit,
|
||||
},
|
||||
});
|
||||
|
||||
process.stderr.write(`\nDevTools listening on ws://127.0.0.1:${server.port}/devtools/browser/${browserId}\n`);
|
||||
@@ -234,7 +234,9 @@ Custom scripts and binaries are stored in `.local/bin/`:
|
||||
- `omo-remote`: The same sound, tab marker and notification for omo sessions on a remote server (flowstate-remote). The server's `omo-notify` (kept on the server in `~/.local/bin`, not in this repository) writes the session's state into its terminal title, marker first (`🟢 studio · omo:done:...`). mosh drops notifications but carries titles, and so does ssh, so tmux's `pane-title-changed` hook passes the event to `omo-notify` for the pane running mosh or ssh. Everything else reaches you through ntfy: `omo-remote install` starts a launchd agent that subscribes to the topic and plays the sound and posts the notification for omo pushes, unless a tmux pane here already handled that event through its title, so nothing arrives twice. That covers sessions no terminal is showing and `fs` in a plain Ghostty tab, whose title shows the marker and which posts the notification itself (the coloured tab marker needs tmux). The agent reads `NTFY_URL`, `NTFY_TOPIC` and `NTFY_TOKEN` from `~/.config/ntfy/omo-remote.env` (chmod 600, not in this repository) and needs `brew "ntfy"`
|
||||
- `docker-nuke`: Force-restart a wedged Docker Desktop, capturing diagnostics to `~/.local/state/docker-nuke/` first (`--dry-run` to preview, `--no-restart` to leave it down, `--deep` for root helpers)
|
||||
|
||||
The `links` step of `setup-mac.sh` links each script into `~/.local/bin`, which `.zshrc` puts on your PATH. `pbcopy` and `pbpaste` are OSC 52 clipboard shims for headless servers, so they are not linked on macOS.
|
||||
- `impeccable-browser`: The browser for Impeccable's URL scans (`impeccable detect <url>`) on flowstate-remote, set as `IMPECCABLE_BROWSER` in the server's `~/.zshenv`. Public URLs render on Browserless. Its key, `BROWSERLESS_API_KEY` from the Reoclo project `flowstate-tools`, is cached for 12 hours in `~/.impeccable/browserless.token`. Localhost and private addresses reach Browserless through a cloudflared quick tunnel that lasts only for the scan, and reports still name the local URL. Other launches, and any scan whose tunnel or Browserless connection fails, run on the headless Chrome in the Puppeteer cache with `--no-sandbox`, because the server's AppArmor blocks Chrome's sandbox. `IMPECCABLE_TUNNEL=0` keeps private addresses on local Chrome, which pages that call a second local port (a separate API, say) need. Every decision is logged to `~/.impeccable/browser.log`
|
||||
|
||||
The `links` step of `setup-mac.sh` links each script into `~/.local/bin`, which `.zshrc` puts on your PATH. `pbcopy` and `pbpaste` (OSC 52 clipboard shims for headless servers) and `impeccable-browser` (Linux only) are not linked on macOS.
|
||||
|
||||
### Local Servers
|
||||
|
||||
|
||||
@@ -198,6 +198,7 @@ step_links() {
|
||||
name=${script##*/}
|
||||
case $name in
|
||||
pbcopy | pbpaste) continue ;; # OSC 52 clipboard shims for headless servers
|
||||
impeccable-browser) continue ;; # Linux-only browser shim for flowstate-remote
|
||||
esac
|
||||
if [[ -x $script ]]; then link ".local/bin/$name" ".local/bin/$name"; fi
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user