rechrome 1.20.0 → 1.22.0

package/serve.ts

@@ -1,6 +1,6 @@
 import { file } from "bun";
 import { createHash, X509Certificate } from "crypto";
-import { mkdirSync, unlinkSync, accessSync, constants as fsConstants } from "fs";
+import { mkdirSync, unlinkSync, accessSync, readdirSync, constants as fsConstants } from "fs";
 import { join, resolve, relative, isAbsolute } from "path";
 import {
   log,
@@ -16,25 +16,121 @@ import {
 const TAILSCALE_BIN = process.env.TAILSCALE_BIN || "/Applications/Tailscale.app/Contents/MacOS/Tailscale";
 const CERT_RENEW_THRESHOLD_DAYS = 7;
 
-// Short label for a client identity, used as the Chrome tab-group name (the tab
-// strip is space-constrained, so cap at 7 chars). gitUrl ".../owner/repo/tree/branch"
-// -> "rep:bra" (3+3); "host:/path/to/dir" -> "dir"; bare host/IP -> as-is. Strips a
-// trailing "@profile" suffix first.
-const MAX_GROUP_LABEL_LEN = 7;
-function shortClientLabel(raw: string): string {
+// Short label for a client identity, used as the Chrome tab-group name (the tab strip is
+// space-constrained, so cap at 7 chars). Handles the current label shape and the legacy gitUrl:
+//   "host/owner/repo#<basename>@<branch>" -> "bas:bra" (3+3)   (current)
+//   "host/owner/repo/tree/branch"         -> "rep:bra" (3+3)   (legacy gitUrl)
+//   "host:/path/to/dir"                   -> "dir"             (non-git)
+//   bare host/IP                          -> as-is
+export const MAX_GROUP_LABEL_LEN = 7;
+export function shortClientLabel(raw: string): string {
   if (!raw) return raw;
-  const baseId = raw.includes("@") ? raw.slice(0, raw.indexOf("@")) : raw;
-  const git = baseId.match(/^https?:\/\/[^/]+\/[^/]+\/([^/]+?)(?:\/tree\/(.+))?$/);
-  let label: string;
-  if (git)
-    label = git[2] ? `${git[1].slice(0, 3)}:${git[2].slice(0, 3)}` : git[1];
-  else {
-    const hostCwd = baseId.match(/^[^:]+:(.+)$/);
-    label = hostCwd ? (hostCwd[1].split("/").filter(Boolean).pop() || baseId) : baseId;
+  const join3 = (a: string, b?: string) => (b ? `${a.slice(0, 3)}:${b.slice(0, 3)}` : a);
+  // Current label: "[remote#]<worktree-basename>[@<branch>]"
+  if (raw.includes("#") || (raw.includes("@") && !raw.startsWith("http"))) {
+    const afterHash = raw.includes("#") ? raw.slice(raw.indexOf("#") + 1) : raw;
+    const [base, branch] = afterHash.split("@");
+    return join3(base, branch).slice(0, MAX_GROUP_LABEL_LEN);
   }
+  // Legacy gitUrl: ".../owner/repo/tree/branch"
+  const git = raw.match(/^https?:\/\/[^/]+\/[^/]+\/([^/]+?)(?:\/tree\/(.+))?$/);
+  if (git) return join3(git[1], git[2]).slice(0, MAX_GROUP_LABEL_LEN);
+  // "host:/path/to/dir" -> basename
+  const hostCwd = raw.match(/^[^:]+:(.+)$/);
+  const label = hostCwd ? (hostCwd[1].split("/").filter(Boolean).pop() || raw) : raw;
   return label.slice(0, MAX_GROUP_LABEL_LEN);
 }
 
+// --isolate sessions (`rech --isolate ...` -> `-s=iso-<rand>`) are throwaway, single-flow
+// buckets. We reap them on an idle TTL so an OAuth/login drive can't leak an orphaned browser
+// context. A TTL (not close-on-exit-per-command) is required because the flows are multi-step
+// (open -> click -> consent): closing after each command would break them. The reaper runs the
+// CLI's `close` for the specific iso session only — it never touches the user's other sessions
+// or quits Chrome.
+const ISO_SESSION_TTL_MS = Number(process.env.RECH_ISOLATE_TTL_MS) || 15 * 60_000;
+const ISO_REAP_INTERVAL_MS = 60_000;
+const isoLastUsed = new Map<string, number>();
+
+export function isIsoSession(namespacedSession: string): boolean {
+  return /(?:^|-)iso-[0-9a-f]+$/.test(namespacedSession);
+}
+
+// --- Relay health / self-heal ---------------------------------------------------------
+// The daemon spawns a short-lived CLI child per command, but all commands share a long-lived
+// per-session cliDaemon and a single extension↔relay path. Under sustained multi-session load
+// the relay can wedge: navigations (`open`) hang to the 60s cap while cheap calls still return,
+// and — critically — the wedge PERSISTS (every later command reuses the same broken cliDaemon),
+// so historically only a manual `oxmgr restart rechrome` cleared it. We heal automatically on
+// two layers, driven by real command timeouts (no synthetic probe → Chrome is never touched):
+//   - per-session: after SESSION_CLOSE_TIMEOUTS consecutive timeouts on ONE session, run the
+//     CLI `close` for it so the next command respawns a clean cliDaemon.
+//   - global: after WATCHDOG_TIMEOUTS consecutive timeouts across ALL sessions with no success
+//     in between (= the shared relay is dead), exit(1); oxmgr's `--restart always` respawns us
+//     clean and the extension WS reconnects. Each timeout burns 60s of wall time, so this can't
+//     spin faster than ~once/minute even under concurrent load.
+const SESSION_CLOSE_TIMEOUTS = Number(process.env.RECH_SESSION_CLOSE_TIMEOUTS) || 2;
+const WATCHDOG_TIMEOUTS = Number(process.env.RECH_WATCHDOG_TIMEOUTS) || 3;
+let consecutiveTimeouts = 0;                       // global; reset on any non-timeout /run
+const sessionTimeouts = new Map<string, number>(); // per-session consecutive-timeout streak
+// Most-recently-used non-iso sessions, so the deep health probe can target something real
+// instead of spawning a fresh session (which could open a browser window).
+const recentSessions = new Map<string, number>();
+function noteSession(sess: string, now: number): void {
+  recentSessions.set(sess, now);
+  if (recentSessions.size > 64) {
+    let oldestKey: string | undefined, oldestAt = Infinity;
+    for (const [k, t] of recentSessions) if (t < oldestAt) { oldestAt = t; oldestKey = k; }
+    if (oldestKey) recentSessions.delete(oldestKey);
+  }
+}
+
+function tmpSocketRoot(): string {
+  return `${(process.env.TMPDIR || "/tmp").replace(/\/$/, "")}/playwright-cli`;
+}
+
+// On startup, adopt any iso-* sessions a previous daemon left behind so they still get reaped
+// (in-memory tracking alone would miss pre-restart orphans). Best-effort: a mis-derived session
+// just yields a harmless no-op `close`. Socket files are named with the session as their prefix.
+function adoptOrphanedIsoSessions(): void {
+  try {
+    const root = tmpSocketRoot();
+    for (const sub of readdirSync(root)) {
+      let entries: string[];
+      try { entries = readdirSync(`${root}/${sub}`); } catch { continue; }
+      for (const f of entries) {
+        const m = f.match(/^([0-9a-f]+-iso-[0-9a-f]+)/) || f.match(/^(iso-[0-9a-f]+)/);
+        if (m && !isoLastUsed.has(m[1])) {
+          isoLastUsed.set(m[1], Date.now());
+          log(`adopted orphaned isolated session for reaping: ${m[1]}`);
+        }
+      }
+    }
+  } catch {
+    // socket dir may not exist yet — nothing to adopt
+  }
+}
+
+function reapIdleIsoSessions(bin: string, binArgs: string[], workDir: string): void {
+  const now = Date.now();
+  for (const [sess, last] of isoLastUsed) {
+    if (now - last < ISO_SESSION_TTL_MS) continue;
+    isoLastUsed.delete(sess);
+    try {
+      Bun.spawn([bin, ...binArgs, "close", `-s=${sess}`], {
+        cwd: workDir,
+        stdin: "ignore",
+        stdout: "ignore",
+        stderr: "ignore",
+        windowsHide: true, // hide the CLI child's console; the user's Chrome (a GUI grandchild) stays visible
+        env: { PATH: process.env.PATH, HOME: HOME, USERPROFILE: process.env.USERPROFILE },
+      });
+      log(`reaped idle isolated session (idle ${Math.round((now - last) / 1000)}s): ${sess}`);
+    } catch (e) {
+      log(`reap failed for ${sess}: ${e}`);
+    }
+  }
+}
+
 async function renewCertIfNeeded(certPath: string, keyPath: string): Promise<boolean> {
   const certContent = await file(certPath).text().catch(() => null);
   if (!certContent) return false;
@@ -46,7 +142,7 @@ async function renewCertIfNeeded(certPath: string, keyPath: string): Promise<boo
     if (!domain) { log("TLS cert renewal: could not determine domain"); return false; }
     log(`TLS cert expires in ${Math.floor(daysLeft)} days, renewing ${domain}...`);
     const proc = Bun.spawn([TAILSCALE_BIN, "cert", "--cert-file", certPath, "--key-file", keyPath, domain], {
-      stdout: "pipe", stderr: "pipe",
+      stdout: "pipe", stderr: "pipe", windowsHide: true,
     });
     const [status, stderr] = await Promise.all([proc.exited, new Response(proc.stderr).text()]);
     if (status !== 0) { log(`TLS cert renewal failed: ${stderr.trim()}`); return false; }
@@ -123,7 +219,7 @@ async function freeStalePort(port: number): Promise<void> {
         "  $h | ForEach-Object { Stop-Process -Id $_.ProcessId -Force }",
         "}",
       ].join(" ");
-      const r = Bun.spawnSync(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps]);
+      const r = Bun.spawnSync(["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], { windowsHide: true });
       const out = r.stdout?.toString().trim();
       if (out) log(out);
     } else {
@@ -142,6 +238,13 @@ export async function serve() {
   const workDir = join(RECH_DIR, "output");
   mkdirSync(workDir, { recursive: true });
 
+  // Reap idle --isolate sessions so single-shot OAuth/login drives don't leak browser contexts.
+  adoptOrphanedIsoSessions();
+  setInterval(() => {
+    const [bin, ...binArgs] = splitCommand(resolvePlaywrightCli());
+    reapIdleIsoSessions(bin, binArgs, workDir);
+  }, ISO_REAP_INTERVAL_MS);
+
   const listenHost = process.env.RECH_HOST || "127.0.0.1";
   const canRead = (p?: string) => { try { accessSync(p!, fsConstants.R_OK); return true; } catch { return false; } };
   const certPath = canRead(process.env.RECH_TLS_CERT) ? process.env.RECH_TLS_CERT : undefined;
@@ -155,9 +258,13 @@ export async function serve() {
     }, 86_400_000);
   }
   const tls = certPath && keyPath ? { cert: Bun.file(certPath), key: Bun.file(keyPath) } : undefined;
-  const startServer = () => Bun.serve({
+  const startServer = (reusePort = false) => Bun.serve({
     hostname: listenHost,
     port,
+    // reusePort is used only as a last-resort fallback (see the bind loop below): if an orphaned
+    // holder can't be killed, binding with SO_REUSEADDR keeps serve up (degraded, port-shared)
+    // instead of crash-looping on EADDRINUSE. The normal path binds a clean, exclusive socket.
+    reusePort,
     tls,
     error(err) {
       log(`unhandled error: ${err.message}`);
@@ -181,7 +288,32 @@ export async function serve() {
       if (reqUrl.pathname === "/ping") {
         const denied = authCheck(req, key);
         if (denied) return denied;
-        return Response.json({ ok: true, bind: listenHost });
+        // Shallow ping proves only that the HTTP listener is up (it stayed up through past
+        // relay wedges). `degraded` surfaces the passive timeout streak from real traffic so
+        // clients / `rech status` can see trouble without an active probe.
+        const degraded = consecutiveTimeouts > 0;
+        if (!reqUrl.searchParams.get("deep"))
+          return Response.json({ ok: true, bind: listenHost, consecutiveTimeouts, degraded });
+        // Deep probe: exercise the relay read-only via `tab-list` (opens nothing) against the
+        // most-recently-used session — never a fresh one, which could spawn a browser window.
+        const target = [...recentSessions.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
+        if (!target)
+          return Response.json({ ok: true, bind: listenHost, relay: "idle", consecutiveTimeouts, degraded });
+        const [pbin, ...pbinArgs] = splitCommand(resolvePlaywrightCli());
+        const probe = Bun.spawn([pbin, ...pbinArgs, "tab-list", `-s=${target}`], {
+          cwd: workDir, stdin: "ignore", stdout: "ignore", stderr: "ignore",
+          windowsHide: true,
+          env: { PATH: process.env.PATH, HOME: HOME, USERPROFILE: process.env.USERPROFILE },
+        });
+        const probeStatus = await Promise.race([
+          probe.exited,
+          new Promise<number>((r) => setTimeout(() => { probe.kill(); r(-1); }, 5000)),
+        ]);
+        const healthy = probeStatus !== -1;
+        return Response.json({
+          ok: healthy, bind: listenHost, relay: healthy ? "healthy" : "degraded",
+          consecutiveTimeouts, degraded: degraded || !healthy,
+        });
       }
       if (reqUrl.pathname !== "/run") return new Response("rech server\n");
       const denied = authCheck(req, key);
@@ -201,14 +333,19 @@ export async function serve() {
       } else {
         args = body.args;
         const id = body.identity as
-          | { gitUrl?: string; hostname?: string; cwd?: string; profile?: string }
+          | { key?: string; label?: string; gitUrl?: string; hostname?: string; cwd?: string; profile?: string }
           | undefined;
-        const baseId = id?.gitUrl || (id?.hostname && id?.cwd ? `${id.hostname}:${id.cwd}` : null);
-        const raw = baseId && id?.profile ? `${baseId}@${id.profile}` : baseId;
-        if (raw) {
-          sessionId = createHash("sha256").update(raw).digest("hex").slice(0, 8);
-          clientName = raw;
-          log(`session from identity: ${raw} -> ${sessionId}`);
+        // New clients send {key,label} (key = worktree-root-based, decoupled from the pretty
+        // label). Fall back to the legacy {gitUrl|hostname:cwd} shape for older clients.
+        const legacy = id?.gitUrl || (id?.hostname && id?.cwd ? `${id.hostname}:${id.cwd}` : null);
+        const keyBase = id?.key || legacy;
+        const labelBase = id?.label || legacy || keyBase;
+        if (keyBase) {
+          // Hash the key (+ profile via a NUL separator that never appears in a label).
+          const hashInput = id?.profile ? `${keyBase}\u0000${id.profile}` : keyBase;
+          sessionId = createHash("sha256").update(hashInput).digest("hex").slice(0, 8);
+          clientName = labelBase || keyBase;
+          log(`session from identity: ${clientName}${id?.profile ? ` profile:${id.profile}` : ""} [${keyBase}] -> ${sessionId}`);
         } else {
           const clientAddr = `${req.headers.get("x-forwarded-for") || server.requestIP(req)?.address || "unknown"}`;
           sessionId = createHash("sha256").update(clientAddr).digest("hex").slice(0, 8);
@@ -233,6 +370,10 @@ export async function serve() {
         return true;
       });
       const namespacedSession = clientSession ? `${sessionId}-${clientSession}` : sessionId;
+      // Track --isolate sessions so the idle-TTL reaper can close them later.
+      const nowMs = Date.now();
+      if (isIsoSession(namespacedSession)) isoLastUsed.set(namespacedSession, nowMs);
+      else noteSession(namespacedSession, nowMs); // for the deep health probe to target
 
       // daemonInstall bakes PLAYWRIGHT_CLI into the daemon env; resolvePlaywrightCli() is the
       // fallback for a standalone `serve` (it re-runs the same env > fork > @playwright/cli > legacy chain).
@@ -259,6 +400,7 @@ export async function serve() {
             stdin: "ignore",
             stdout: "pipe",
             stderr: "pipe",
+            windowsHide: true, // hide the CLI child's console; the user's Chrome (a GUI grandchild) stays visible
             env: { PATH: process.env.PATH, HOME: HOME, USERPROFILE: process.env.USERPROFILE },
           });
           const [listStatus, listOut] = await Promise.race([
@@ -341,16 +483,20 @@ export async function serve() {
         stdin: "ignore",
         stdout: "pipe",
         stderr: "pipe",
+        windowsHide: true, // hide the CLI child's console; the user's Chrome (a GUI grandchild) stays visible
         env: childEnv,
       });
 
       const TIMEOUT = 60_000;
-      const timeout = new Promise<never>((_, reject) =>
-        setTimeout(() => {
+      let timedOut = false;
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const timeout = new Promise<never>((_, reject) => {
+        timer = setTimeout(() => {
+          timedOut = true;
           proc.kill();
           reject(new Error("timeout"));
-        }, TIMEOUT),
-      );
+        }, TIMEOUT);
+      });
       const [status, stdout, stderr] = await Promise.race([
         Promise.all([
           proc.exited,
@@ -361,9 +507,37 @@ export async function serve() {
       ]).catch(
         () => [1, "", `Command timed out after ${TIMEOUT / 1000}s\n`] as [number, string, string],
       ) as [number, string, string];
+      clearTimeout(timer);
 
       log(`exit: ${status}${stdout.trim() ? ` | ${stdout.trim().slice(0, 200)}` : ""}`);
 
+      // Relay self-heal (see notes at SESSION_CLOSE_TIMEOUTS): a command that RETURNS — even
+      // non-zero — proves the relay answered, so clear the streaks; a TIMEOUT means it didn't.
+      if (timedOut) {
+        consecutiveTimeouts++;
+        const streak = (sessionTimeouts.get(namespacedSession) ?? 0) + 1;
+        sessionTimeouts.set(namespacedSession, streak);
+        log(`timeout: session=${namespacedSession} sessionStreak=${streak} globalStreak=${consecutiveTimeouts}`);
+        if (streak >= SESSION_CLOSE_TIMEOUTS) {
+          log(`session ${namespacedSession} wedged (${streak} consecutive timeouts) — closing so the next command respawns a clean cliDaemon`);
+          try {
+            Bun.spawn([bin, ...binArgs, "close", `-s=${namespacedSession}`], {
+              cwd: workDir, stdin: "ignore", stdout: "ignore", stderr: "ignore",
+              windowsHide: true,
+              env: { PATH: process.env.PATH, HOME: HOME, USERPROFILE: process.env.USERPROFILE },
+            });
+          } catch {}
+          sessionTimeouts.delete(namespacedSession);
+        }
+        if (consecutiveTimeouts >= WATCHDOG_TIMEOUTS) {
+          log(`relay wedged: ${consecutiveTimeouts} consecutive timeouts, no success between — exiting for oxmgr (--restart always) to respawn clean; extension WS will reconnect (Chrome untouched)`);
+          setTimeout(() => process.exit(1), 100); // brief delay to flush this response
+        }
+      } else {
+        consecutiveTimeouts = 0;
+        sessionTimeouts.delete(namespacedSession);
+      }
+
       // Detect files mentioned in output
       const filePattern = /[\w./-]+\.(?:png|jpe?g|pdf|json|yml)\b/gi;
       const mentionedFiles = [
@@ -402,17 +576,34 @@ export async function serve() {
     },
   });
 
-  // A leaked listening-socket handle in an orphaned cliDaemon can keep the port held after a
-  // prior serve exits; on EADDRINUSE, clear stale holders once and retry rather than crash-loop.
-  let server: ReturnType<typeof startServer>;
-  try {
-    server = startServer();
-  } catch (e: any) {
-    if (!String(e?.code ?? e?.message ?? "").includes("EADDRINUSE")) throw e;
-    log(`port ${port} in use — clearing stale daemon holders and retrying`);
-    await freeStalePort(port);
-    server = startServer();
+  // A leaked listening-socket handle in an orphaned cliDaemon can keep the port in LISTEN after a
+  // prior serve exits: Bun.serve creates the socket inheritable and Bun.spawn sweeps it into the
+  // detached daemon grandchild via bInheritHandles, so the socket outlives its creating serve.
+  // netstat then attributes the port to the now-dead *creator*, not the live holder, so we can't
+  // map port -> killable PID — freeStalePort kills the orphan by its cliDaemon signature instead.
+  // Retry a bounded number of times: the old single retry crash-looped whenever the OS hadn't yet
+  // released the socket after the kill (pm2 then restarts serve into the same race). As an absolute
+  // last resort, bind with reusePort so a holder we genuinely can't kill degrades to "up but sharing
+  // the port" rather than a permanent EADDRINUSE crash-loop.
+  const isEaddrInUse = (e: any) => String(e?.code ?? e?.message ?? "").includes("EADDRINUSE");
+  const MAX_BIND_ATTEMPTS = 4;
+  let server: ReturnType<typeof startServer> | undefined;
+  for (let attempt = 1; attempt <= MAX_BIND_ATTEMPTS; attempt++) {
+    try {
+      server = startServer();
+      break;
+    } catch (e: any) {
+      if (!isEaddrInUse(e)) throw e;
+      if (attempt === MAX_BIND_ATTEMPTS) {
+        log(`port ${port} still held after ${attempt - 1} cleanup attempts — binding with reusePort (last resort)`);
+        server = startServer(true);
+        break;
+      }
+      log(`port ${port} in use — clearing stale daemon holders and retrying (attempt ${attempt}/${MAX_BIND_ATTEMPTS - 1})`);
+      await freeStalePort(port);
+    }
   }
+  if (!server) throw new Error(`failed to bind port ${port}`);
 
   log(`serving on ${tls ? "https" : "http"}://${server.hostname}:${server.port}`);
   log(`Connection URL set (use .env.local to view)`);