sensorium-mcp 3.1.0-refactor.17 → 3.1.0-refactor.18

package/dist/services/process.service.js

@@ -1,3 +1,4 @@
+import { execFile, spawn } from "node:child_process";
 import { mkdirSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
@@ -5,7 +6,6 @@ import { log } from "../lib/logger.js";
 import { errorMessage } from "../lib/utils.js";
 import { getAllThreads } from "../data/memory/thread-registry.js";
 import { getDashboardSessions } from "../sessions/dashboard-registry.js";
-import { brokerAdopt, brokerCheckAlive, brokerKillTree } from "./broker-client.js";
 const BASE_DIR = join(homedir(), ".remote-copilot-mcp");
 const LOGS_DIR = join(BASE_DIR, "logs");
 const PIDS_DIR = join(BASE_DIR, "pids");
@@ -59,39 +59,294 @@ export function deletePendingTaskMailbox(threadId) {
         }
     }
 }
-/**
- * Liveness + identity in one question: is `pid` alive AND (when `spawnedAtMs`
- * is given) still the process we spawned?
- *
- * Two-step check, neither of which ever spawns a child process:
- *   1. `process.kill(pid, 0)` — a plain syscall. ESRCH (any non-EPERM error) is
- *      a definitive "dead" and needs no broker (this also keeps dev mode
- *      without a supervisor fully functional for the common case).
- *   2. Otherwise the answer is NOT trustworthy on Windows — kill(0) succeeds
- *      for kernel-zombie PIDs held open by a lingering handle, and EPERM is
- *      what a parent sees for its own terminated child. Ask the supervisor's
- *      process broker, which answers from a native process snapshot and also
- *      verifies creation-time identity (PID-reuse guard).
- *
- * Broker unreachable → "cannot confirm → treat alive", exactly the audited-safe
- * degradation the deleted tasklist/Get-Process breakers had.
- */
-export async function isProcessAlive(pid, spawnedAtMs) {
+/** @param spawnedAtMs when the caller knows the process's spawn time (pid-file
+ *  startedAt), pass it: absence from a pid-table snapshot taken BEFORE the
+ *  spawn proves nothing, so such a process is treated as alive until a fresher
+ *  snapshot can actually testify. Without it, a freshly spawned child could be
+ *  declared dead against a stale snapshot → duplicate spawn. */
+export function isProcessAlive(pid, spawnedAtMs) {
     try {
         process.kill(pid, 0);
     }
     catch (err) {
+        // ESRCH (or any non-EPERM error) = no such process → dead. EPERM = kill(0)
+        // was denied; do NOT trust that as alive — fall through to the win32 table
+        // check below (see next comment for why). On POSIX, EPERM does mean a live
+        // process owned by another user, so the fall-through returns true there.
         if (err.code !== "EPERM")
             return false;
     }
-    if (process.platform !== "win32") {
-        // POSIX: kill(0) success or EPERM both mean a live process.
+    // Reached when kill(0) either SUCCEEDED or threw EPERM. On Windows neither is
+    // trustworthy on its own:
+    //   - kill(0) succeeds for a process that already TERMINATED but whose PID is
+    //     still held open by a lingering handle — a kernel "zombie".
+    //   - OpenProcess from the PARENT of such a dead child returns ACCESS_DENIED
+    //     (→EPERM) for that same terminated PID (a non-parent instead sees ESRCH).
+    // Both cases wedged the keeper into treating an unkillable zombie as alive
+    // forever, never restarting the thread. The process table excludes terminated
+    // PIDs even while their handle lingers, so make it authoritative for both
+    // paths. (tasklist unavailable → isPidInWindowsTable conservatively returns
+    // true, so we never declare a genuinely live process dead.)
+    if (process.platform === "win32")
+        return isPidInWindowsTable(pid, spawnedAtMs);
+    return true;
+}
+const WIN_PID_SNAPSHOT_TTL_MS = 2000;
+// A snapshot older than this is discarded rather than trusted: a very stale pid
+// set would mark freshly-spawned processes dead (they're absent from it), and a
+// stale start-time map is merely incomplete. Normal refreshes complete in well
+// under a second, so this bound is only reached when refreshes keep failing —
+// at which point the getters degrade to null (= "can't confirm" → alive).
+const WIN_SNAPSHOT_MAX_STALE_MS = 15_000;
+// Circuit breaker for the snapshot commands. When the OS throttles external
+// process spawns (battery / Energy Saver / EDR), tasklist and powershell hang
+// their full 5s timeout on EVERY call. The former synchronous snapshots then
+// froze the event loop for 5s at a time, so newly-spawned agent children could
+// not complete the MCP initialize/tools-list handshake and started TOOLLESS —
+// then exited cleanly and were respawned forever (the 2026-07-17 respawn storm,
+// see implementation_plans/2026-07-18_toolless-respawn-storm.md). Refreshes are
+// now async (never on the event loop) and, after repeated failures, paused
+// entirely for a cooldown instead of hammering a throttled OS every sweep.
+const SNAPSHOT_BREAKER_THRESHOLD = 3;
+const SNAPSHOT_BREAKER_COOLDOWN_MS = 60_000;
+function makeSnapshotBreaker(label) {
+    let consecutiveFailures = 0;
+    let openUntil = 0;
+    return {
+        isOpen: () => Date.now() < openUntil,
+        recordSuccess: () => {
+            consecutiveFailures = 0;
+        },
+        recordFailure: (err) => {
+            consecutiveFailures++;
+            if (consecutiveFailures >= SNAPSHOT_BREAKER_THRESHOLD) {
+                openUntil = Date.now() + SNAPSHOT_BREAKER_COOLDOWN_MS;
+                // Reset on open so re-opening after the cooldown requires THRESHOLD
+                // fresh failures — one lingering failure must not instantly re-trip.
+                consecutiveFailures = 0;
+                log.warn(`[process] ${label} snapshot failed ${SNAPSHOT_BREAKER_THRESHOLD}x consecutively (${errorMessage(err)}) — pausing refreshes for ${SNAPSHOT_BREAKER_COOLDOWN_MS / 1000}s`);
+            }
+            else {
+                log.debug(`[process] ${label} snapshot failed: ${errorMessage(err)}`);
+            }
+        },
+    };
+}
+/** Authoritative Windows liveness check via the process table, which — unlike
+ *  OpenProcess/process.kill(0) — does NOT list a terminated process whose PID is
+ *  still held by a lingering handle. One `tasklist` snapshot of ALL pids serves
+ *  every pid in a sweep, and it is refreshed asynchronously in the background:
+ *  the getter reads the cached snapshot and NEVER blocks the event loop (a 5s
+ *  synchronous hang here is what starved MCP handshakes and caused the toolless
+ *  respawn storm). Callers treat null as "can't confirm dead" → alive. */
+let winPidSnapshot = null;
+let winPidRefreshInFlight = false;
+const pidSnapshotBreaker = makeSnapshotBreaker("tasklist");
+function refreshWindowsPidSet() {
+    if (winPidRefreshInFlight || pidSnapshotBreaker.isOpen())
+        return;
+    winPidRefreshInFlight = true;
+    // The snapshot's ts is the REFRESH START: the process table is read at (or
+    // after) this instant, so any pid spawned later can legitimately be absent.
+    const refreshStartedAt = Date.now();
+    // /NH = no header, /FO CSV = quoted columns; PID is column 2: "image","PID",...
+    execFile("tasklist", ["/NH", "/FO", "CSV"], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
+        winPidRefreshInFlight = false;
+        if (err) {
+            pidSnapshotBreaker.recordFailure(err);
+            return;
+        }
+        pidSnapshotBreaker.recordSuccess();
+        const pids = new Set();
+        for (const m of out.matchAll(/^"[^"]*","(\d+)"/gm))
+            pids.add(Number(m[1]));
+        winPidSnapshot = { pids, ts: refreshStartedAt };
+    });
+}
+function getWindowsPidSet() {
+    const now = Date.now();
+    if (!winPidSnapshot || now - winPidSnapshot.ts >= WIN_PID_SNAPSHOT_TTL_MS)
+        refreshWindowsPidSet();
+    if (winPidSnapshot && now - winPidSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
+        return winPidSnapshot.pids;
+    return null; // no usable snapshot — conservative: never declare a real process dead
+}
+function isPidInWindowsTable(pid, spawnedAtMs) {
+    const pids = getWindowsPidSet();
+    if (pids === null)
+        return true; // tasklist unavailable — never declare a real process dead
+    if (pids.has(pid))
         return true;
+    // Absent from the table — but if the process was spawned AFTER the snapshot
+    // was taken, the absence proves nothing (the table simply predates it).
+    // Treat as alive; the refresh already triggered above will testify shortly.
+    return spawnedAtMs !== undefined && spawnedAtMs >= winPidSnapshot.ts;
+}
+// Tolerance between the pid-file's recorded startedAt (Date.now() at spawn) and
+// the OS process creation time. Wide enough to absorb spawn latency and a wall-
+// clock step (NTP/VM resume) between the two readings, yet far below the minutes-
+// to-hours gap a recycled PID produces — so PID reuse is still caught.
+const PID_IDENTITY_TOLERANCE_MS = 60_000;
+const WIN_START_SNAPSHOT_TTL_MS = 2000;
+/** One powershell snapshot of every pid's creation time (Unix ms), reused for
+ *  all pids in a keeper sweep and refreshed asynchronously in the background —
+ *  same non-blocking pattern as the tasklist snapshot above. Staleness is safe
+ *  here: a pid's start time never changes, so an old map is merely missing new
+ *  pids, and a missing entry already means "can't tell" → trust PID. StartTime
+ *  is unreadable for some protected processes; those are simply omitted. */
+let winStartSnapshot = null;
+let winStartRefreshInFlight = false;
+const startSnapshotBreaker = makeSnapshotBreaker("start-time");
+function refreshWindowsStartTimeMap() {
+    if (winStartRefreshInFlight || startSnapshotBreaker.isOpen())
+        return;
+    winStartRefreshInFlight = true;
+    const refreshStartedAt = Date.now();
+    execFile("powershell", [
+        "-NoProfile",
+        "-Command",
+        'Get-Process | ForEach-Object { try { "$($_.Id) $($_.StartTime.ToUniversalTime().Ticks)" } catch {} }',
+    ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
+        winStartRefreshInFlight = false;
+        if (err) {
+            startSnapshotBreaker.recordFailure(err);
+            return;
+        }
+        startSnapshotBreaker.recordSuccess();
+        const starts = new Map();
+        for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
+            starts.set(Number(m[1]), Math.round(Number(m[2]) / 10000 - 62135596800000)); // .NET ticks (100ns from year 1) → Unix ms
+        }
+        winStartSnapshot = { starts, ts: refreshStartedAt };
+    });
+}
+function getWindowsStartTimeMap() {
+    const now = Date.now();
+    if (!winStartSnapshot || now - winStartSnapshot.ts >= WIN_START_SNAPSHOT_TTL_MS)
+        refreshWindowsStartTimeMap();
+    if (winStartSnapshot && now - winStartSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
+        return winStartSnapshot.starts;
+    return null; // no usable snapshot — conservative: caller trusts the PID
+}
+/** Windows process creation time as Unix ms, or undefined if unavailable.
+ *  Lets us tell OUR process apart from an unrelated one the OS assigned the
+ *  same (recycled) PID after ours exited. */
+function processStartMs(pid) {
+    if (process.platform !== "win32")
+        return undefined;
+    return getWindowsStartTimeMap()?.get(pid);
+}
+/** True if the live PID is (still) the process we spawned, judged by start time.
+ *  Guards against PID reuse: a recycled PID now owned by a different process has
+ *  a start time far from what we recorded, so we stop treating it as ours (which
+ *  otherwise wedges the keeper into an endless "unkillable" kill loop). */
+function isSameProcess(pid, startedAt) {
+    if (startedAt === undefined)
+        return true; // nothing recorded to compare — trust PID
+    // Our process was spawned AFTER the current start-time map was taken: any
+    // entry the map holds for this pid describes a PREVIOUS owner of the (since
+    // recycled) pid, not our child — comparing against it would wrongly reject
+    // our own fresh process. Trust the PID until a fresher map exists.
+    if (process.platform === "win32" &&
+        winStartSnapshot !== null &&
+        startedAt >= winStartSnapshot.ts) {
+        return true;
+    }
+    const actual = processStartMs(pid);
+    if (actual === undefined)
+        return true; // can't read start time — keep prior behavior
+    return Math.abs(actual - startedAt) < PID_IDENTITY_TOLERANCE_MS;
+}
+/** Descendants of rootPid (including root), ordered leaves-first (post-order),
+ *  from one process-table snapshot, delivered via callback (async so the kill
+ *  escalation path never blocks the event loop either). Fallback for when
+ *  `taskkill /T` fails to tear the whole tree down: a single wedged descendant
+ *  can make taskkill abort the tree, so killing each PID individually lets the
+ *  still-killable relatives (often including the root) go down anyway. */
+function getWindowsProcessTreeLeafFirst(rootPid, threadId, done) {
+    execFile("powershell", [
+        "-NoProfile",
+        "-Command",
+        'Get-CimInstance Win32_Process | ForEach-Object { "$($_.ProcessId) $($_.ParentProcessId)" }',
+    ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
+        if (err) {
+            // "Could not look" — distinct from "found nothing" below: the snapshot
+            // itself failed (the corporate-agent CIM failure seen on this machine),
+            // so we cannot enumerate the tree at all and fall back to the root pid.
+            log.warn(`[process] leaf-first walk pid=${rootPid} thread=${threadId}: process-tree snapshot FAILED (${errorMessage(err)}) — could NOT enumerate the tree; falling back to root pid only`);
+            done([rootPid]);
+            return;
+        }
+        const children = new Map();
+        for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
+            const pid = Number(m[1]);
+            const ppid = Number(m[2]);
+            const list = children.get(ppid);
+            if (list)
+                list.push(pid);
+            else
+                children.set(ppid, [pid]);
+        }
+        const ordered = [];
+        const seen = new Set();
+        const visit = (pid) => {
+            if (seen.has(pid))
+                return; // guard against cycles / PID-reuse loops
+            seen.add(pid);
+            for (const child of children.get(pid) ?? [])
+                visit(child);
+            ordered.push(pid); // post-order → leaves first, root last
+        };
+        visit(rootPid);
+        // "Found nothing" reads as "root + 0 descendant(s)" — clearly not the same
+        // as the snapshot-failed warning above.
+        log.info(`[process] leaf-first walk pid=${rootPid} thread=${threadId} enumerated ${ordered.length} process(es): root + ${ordered.length - 1} descendant(s)`);
+        done(ordered);
+    });
+}
+/** Force-kill each PID in order, one at a time, tolerating any that refuse (a
+ *  wedged leaf must not block the ones after it). Async so a sweep over a deep
+ *  tree never blocks the server event loop. */
+function killPidsSequentiallyWin(pids, done) {
+    let i = 0;
+    let killed = 0; // taskkill invocations that returned no error (reported killed)
+    const next = () => {
+        if (i >= pids.length) {
+            done(killed);
+            return;
+        }
+        const pid = pids[i++];
+        execFile("taskkill", ["/F", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
+            if (!err)
+                killed++;
+            next();
+        });
+    };
+    next();
+}
+/** Last-resort escalation for EPERM-wedged trees: spawn taskkill as a DETACHED,
+ *  FRESH process so it does NOT inherit the server's wedged tree handles. When
+ *  the server's own process tree is job-object-constrained / I/O-wedged, an
+ *  in-process `taskkill /F /T` returns EPERM — but a sibling process spawned
+ *  with its own handles (detached, own stdio) killed the exact same zombie trees
+ *  instantly in this session's experiment. Nested securevault contexts (which
+ *  once made detached killers cross-context Access-Denied, commit 1b7ded6) are
+ *  gone — there is a single context now — so a detached killer in the server's
+ *  own context is safe. Fire-and-forget: unref'd so the server never waits. */
+function spawnDetachedKillerWin(pid) {
+    try {
+        const child = spawn("taskkill", ["/F", "/T", "/PID", String(pid)], {
+            detached: true,
+            windowsHide: true,
+            stdio: "ignore",
+        });
+        child.on("error", (err) => log.warn(`[process] detached killer for PID ${pid} errored: ${errorMessage(err)}`));
+        child.unref();
+        log.warn(`[process] ESCALATION: spawned DETACHED fresh taskkill /F /T /PID ${pid} (bypasses server's EPERM-wedged tree)`);
+    }
+    catch (err) {
+        log.warn(`[process] detached killer spawn for PID ${pid} failed: ${errorMessage(err)}`);
     }
-    const result = await brokerCheckAlive(pid, spawnedAtMs);
-    if (result === null)
-        return true; // cannot confirm → never declare a live process dead
-    return result.alive;
 }
 export function readPidFiles() {
     const entries = [];
@@ -126,42 +381,40 @@ export function readPidFiles() {
     }
     return entries;
 }
-export async function findAliveThread(threadId) {
+export function findAliveThread(threadId) {
     for (let i = spawnedThreads.length - 1; i >= 0; i--) {
         const thread = spawnedThreads[i];
         if (thread.threadId !== threadId)
             continue;
-        if (await isProcessAlive(thread.pid, thread.startedAt))
+        if (isProcessAlive(thread.pid, thread.startedAt) && isSameProcess(thread.pid, thread.startedAt))
             return thread;
         log.warn(`[findAliveThread] Thread ${threadId} PID ${thread.pid} stale in spawnedThreads - removing`);
         spawnedThreads.splice(i, 1);
     }
-    for (const entry of readPidFiles()) {
-        if (entry.threadId !== threadId)
-            continue;
-        if (!(await isProcessAlive(entry.pid, entry.startedAt)))
-            continue;
-        const restored = {
-            pid: entry.pid,
-            threadId,
-            name: entry.name ?? `Thread ${threadId}`,
-            startedAt: entry.startedAt ?? Date.now(),
-            createdAt: entry.startedAt ?? Date.now(),
-            logFile: "",
-            ...(entry.threadType ? { threadType: entry.threadType } : {}),
-        };
-        spawnedThreads.push(restored);
-        log.info(`[findAliveThread] Restored thread ${threadId} PID=${entry.pid} from PID file`);
-        return restored;
-    }
-    return undefined;
+    const pidEntry = readPidFiles().find((entry) => entry.threadId === threadId &&
+        isProcessAlive(entry.pid, entry.startedAt) &&
+        isSameProcess(entry.pid, entry.startedAt));
+    if (!pidEntry)
+        return undefined;
+    const restored = {
+        pid: pidEntry.pid,
+        threadId,
+        name: pidEntry.name ?? `Thread ${threadId}`,
+        startedAt: pidEntry.startedAt ?? Date.now(),
+        createdAt: pidEntry.startedAt ?? Date.now(),
+        logFile: "",
+        ...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
+    };
+    spawnedThreads.push(restored);
+    log.info(`[findAliveThread] Restored thread ${threadId} PID=${pidEntry.pid} from PID file`);
+    return restored;
 }
-export const isThreadRunning = async (threadId) => (await findAliveThread(threadId)) !== undefined;
-export async function getActiveThreadIds() {
+export const isThreadRunning = (threadId) => findAliveThread(threadId) !== undefined;
+export function getActiveThreadIds() {
     // Workers are disposable and not eligible for reconnect — exclude them.
-    const candidates = spawnedThreads.filter((t) => t.threadType !== "worker");
-    const aliveFlags = await Promise.all(candidates.map((t) => isProcessAlive(t.pid, t.startedAt)));
-    const spawned = candidates.filter((_, i) => aliveFlags[i]).map((t) => t.threadId);
+    const spawned = spawnedThreads
+        .filter((t) => t.threadType !== "worker" && isProcessAlive(t.pid, t.startedAt))
+        .map((t) => t.threadId);
     // Also include threads from active MCP sessions (root threads that aren't
     // in spawnedThreads because they connected externally, not spawned by us).
     const sessionThreadIds = getDashboardSessions()
@@ -211,16 +464,14 @@ export function pruneOldThreadLogs() {
 }
 /**
  * Check liveness of a thread directly via its PID file, bypassing the
- * spawnedThreads[] in-memory cache. Returns the live PID together with its
- * recorded spawn identity (startedAt) so callers can make identity-checked
- * kill/alive calls, or undefined if no PID file exists or the process is dead
- * (stale file is deleted).
+ * spawnedThreads[] in-memory cache.  Returns the live PID, or undefined
+ * if no PID file exists or the process is dead (stale file is deleted).
  */
-export async function findAliveThreadViaPidFile(threadId) {
+export function findAliveThreadViaPidFile(threadId) {
     const entry = readPidFiles().find((e) => e.threadId === threadId);
     if (!entry)
         return undefined;
-    if (!(await isProcessAlive(entry.pid, entry.startedAt))) {
+    if (!isProcessAlive(entry.pid, entry.startedAt) || !isSameProcess(entry.pid, entry.startedAt)) {
         try {
             unlinkSync(entry.filePath);
         }
@@ -229,7 +480,7 @@ export async function findAliveThreadViaPidFile(threadId) {
         }
         return undefined;
     }
-    return { pid: entry.pid, startedAt: entry.startedAt };
+    return entry.pid;
 }
 /**
  * Reconcile in-memory state (spawnedThreads[]) with the two authoritative
@@ -239,7 +490,7 @@ export async function findAliveThreadViaPidFile(threadId) {
  *
  * Replaces the old restoreFromPidFiles() + cleanupStalePidFiles() pair.
  */
-export async function reconcileState(db, threadLifecycle) {
+export function reconcileState(db, threadLifecycle) {
     const pidEntries = readPidFiles();
     const dbThreadMap = new Map(getAllThreads(db)
         .filter((t) => ["active", "exited", "exiting", "spawning", "stuck"].includes(t.status))
@@ -252,7 +503,7 @@ export async function reconcileState(db, threadLifecycle) {
             // If the process is still alive it's an orphan that must be reaped — never
             // just drop the PID file, or the live process becomes untrackable.
             // killProcessTree removes the PID file itself once the process is confirmed dead.
-            if (await isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
+            if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
                 log.warn(`[reconcileState] Live process for thread ${pidEntry.threadId} (PID=${pidEntry.pid}) has no active DB record — killing orphan`);
                 void killProcessTree(pidEntry.pid, pidEntry.threadId, "reconcile-orphan", pidEntry.startedAt);
             }
@@ -267,7 +518,7 @@ export async function reconcileState(db, threadLifecycle) {
             }
             continue;
         }
-        if (await isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
+        if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
             // Alive: populate spawnedThreads[] if not already present
             if (!spawnedThreads.some((t) => t.threadId === pidEntry.threadId)) {
                 spawnedThreads.push({
@@ -280,9 +531,6 @@ export async function reconcileState(db, threadLifecycle) {
                     ...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
                 });
                 log.info(`[reconcileState] Restored thread ${pidEntry.threadId} PID=${pidEntry.pid} from PID file`);
-                // Safe cutover (Phase 3): pull pre-broker processes into jobs+registry
-                // now rather than waiting for a supervisor restart. Idempotent.
-                void brokerAdopt(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt, pidEntry.threadType ?? "agent");
             }
         }
         else {
@@ -343,71 +591,318 @@ export async function reconcileState(db, threadLifecycle) {
         log.warn(`[reconcileState] Failed to sweep orphan MCP configs: ${errorMessage(err)}`);
     }
 }
+// descendantPid → the tracked thread it was witnessed under (in-memory: an
+// orphan spanning a server restart is intentionally not tracked — see header).
+const orphanLineage = new Map();
+const orphanSnapshotBreaker = makeSnapshotBreaker("win32-process");
+/** One `Get-CimInstance Win32_Process` snapshot giving ppid AND creation time for
+ *  every pid — the only single call that yields both, which the sweep needs to
+ *  walk lineage and enforce the pid-reuse guards. Async (never blocks the event
+ *  loop) and behind the shared circuit breaker: on the corporate-agent failure
+ *  observed on this machine it resolves null, and the sweep degrades to a no-op. */
+function takeWindowsProcessSnapshot() {
+    return new Promise((resolve) => {
+        if (process.platform !== "win32" || orphanSnapshotBreaker.isOpen()) {
+            resolve(null);
+            return;
+        }
+        execFile("powershell", [
+            "-NoProfile",
+            "-Command",
+            'Get-CimInstance Win32_Process | ForEach-Object { try { "$($_.ProcessId) $($_.ParentProcessId) $($_.CreationDate.ToUniversalTime().Ticks)" } catch {} }',
+        ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
+            if (err) {
+                orphanSnapshotBreaker.recordFailure(err);
+                resolve(null);
+                return;
+            }
+            orphanSnapshotBreaker.recordSuccess();
+            const snap = new Map();
+            for (const m of out.matchAll(/^(\d+) (\d+) (\d+)$/gm)) {
+                snap.set(Number(m[1]), {
+                    ppid: Number(m[2]),
+                    startMs: Math.round(Number(m[3]) / 10000 - 62135596800000), // .NET ticks → Unix ms
+                });
+            }
+            resolve(snap);
+        });
+    });
+}
 /**
- * Kill the process tree rooted at `pid` via the supervisor's broker (native
- * TerminateProcess over the whole tree, identity-checked when `spawnedAtMs` is
- * given, outcome recorded in the broker's on-disk registry). Never spawns a
- * child process and never blocks the event loop.
- *
- * Broker unreachable → degraded best-effort: one native `process.kill(pid)`
- * (root only — Node cannot enumerate a tree without shelling out, and that
- * machinery is gone by design) followed by a conservative liveness re-check.
- * A survived kill keeps the PID file so the keeper/cleanup policy loops retry
- * and escalate — same contract as before.
+ * Pure core of the orphan sweep — no IO, no clock, fully deterministic so the
+ * pid-reuse guard and the snapshot-unavailable path can be tested directly.
+ * Mutates `lineage` in place (records new descendants, drops dead/recycled ones).
  */
-export async function killProcessTree(pid, threadId, caller, spawnedAtMs) {
-    // Self-protection: NEVER tree-kill our own process or the authoritative
-    // server PID (a recycled worker PID could point at us — killing it would be
-    // the traceless server death this layer exists to prevent). The broker has
-    // the same guard server-side; this is defense in depth.
-    if (pid === process.pid) {
-        log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
-        return;
+export function computeOrphanSweep(params) {
+    const { snapshot, trackedThreads, lineage, protectedPids, toleranceMs } = params;
+    const result = {
+        orphans: [],
+        reusedPidsSkipped: [],
+        olderThanParentSkipped: [],
+        recorded: 0,
+    };
+    // Snapshot unavailable → never guess, never kill, never mutate lineage.
+    if (snapshot === null)
+        return result;
+    const aliveOwners = new Set();
+    for (const t of trackedThreads)
+        if (t.alive)
+            aliveOwners.add(t.threadId);
+    // Children adjacency, built once from the snapshot.
+    const children = new Map();
+    for (const [pid, e] of snapshot) {
+        const list = children.get(e.ppid);
+        if (list)
+            list.push(pid);
+        else
+            children.set(e.ppid, [pid]);
     }
-    const serverPid = readServerPid();
-    if (serverPid !== undefined && pid === serverPid) {
-        log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
-        return;
+    // Step A — witness lineage: record every descendant of each ALIVE tracked
+    // thread while its subtree chain is still intact. Excludes the root pid itself
+    // (that thread is managed by the worker-cleanup path, not here).
+    for (const t of trackedThreads) {
+        if (!t.alive || !snapshot.has(t.pid))
+            continue;
+        const seen = new Set([t.pid]);
+        const stack = [...(children.get(t.pid) ?? [])];
+        while (stack.length) {
+            const d = stack.pop();
+            if (seen.has(d))
+                continue; // cycle / pid-reuse loop guard
+            seen.add(d);
+            const e = snapshot.get(d);
+            if (!e)
+                continue;
+            lineage.set(d, {
+                ownerThreadId: t.threadId,
+                ownerPid: t.pid,
+                ownerStartedAt: t.startedAt,
+                descendantStartMs: e.startMs,
+            });
+            result.recorded++;
+            for (const c of children.get(d) ?? [])
+                if (!seen.has(c))
+                    stack.push(c);
+        }
     }
-    log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
-    const result = await brokerKillTree(pid, {
-        threadId,
-        killedBy: caller ?? "mcp",
-        reason: caller,
-        startedAtMs: spawnedAtMs,
-    });
-    let stillAlive;
-    if (result !== null) {
-        if (result.refused)
-            log.error(`[process] broker REFUSED kill of pid ${pid}: ${result.error ?? "protected pid"}`);
-        else if (result.identityMismatch)
-            log.warn(`[process] broker: pid ${pid} was recycled (identity mismatch) — our process is already gone, nothing killed`);
-        stillAlive = result.alive;
+    // Step B — evaluate: a recorded descendant whose owner is no longer alive is an
+    // orphan candidate, subject to both pid-reuse guards.
+    for (const [pid, entry] of [...lineage]) {
+        // Prune a descendant that has left the process table FIRST, before the
+        // owner-liveness check — otherwise a dead descendant of a still-live owner is
+        // never evaluated here (its owner short-circuits below) and never re-recorded
+        // by Step A (it is gone), so it lingers in `lineage` for the owner's entire
+        // lifetime. A long-lived thread that churns many short-lived children (e.g. a
+        // claude-agent spawning a bash.exe per hook) would grow the map without bound.
+        const cur = snapshot.get(pid);
+        if (!cur) {
+            lineage.delete(pid); // descendant already gone — nothing to reap
+            continue;
+        }
+        if (aliveOwners.has(entry.ownerThreadId))
+            continue; // owner still alive → not orphaned
+        // Never touch a pid managed elsewhere (a live tracked thread, the server, us).
+        if (protectedPids.has(pid))
+            continue;
+        // GUARD 1 — same-process: a recycled pid now hosts a different process whose
+        // creation time differs from the one we recorded. Drop it; do NOT kill.
+        if (Math.abs(cur.startMs - entry.descendantStartMs) > toleranceMs) {
+            lineage.delete(pid);
+            result.reusedPidsSkipped.push(pid);
+            continue;
+        }
+        // GUARD 2 — a child cannot predate its parent. If this pid started before the
+        // owning thread did, it is not really our descendant (recorded owner is wrong
+        // or the pid was reused across the boundary). Keep it for logging, never kill.
+        if (entry.ownerStartedAt !== undefined && cur.startMs < entry.ownerStartedAt - toleranceMs) {
+            result.olderThanParentSkipped.push(pid);
+            continue;
+        }
+        result.orphans.push({ pid, ownerThreadId: entry.ownerThreadId, startMs: cur.startMs });
     }
-    else {
-        // Degraded path (no supervisor): best-effort native root kill, then re-check.
-        log.warn(`[process] broker unreachable for kill of pid ${pid} (thread ${threadId}) — degraded native root-only kill`);
+    return result;
+}
+/** True when the sweep is allowed to actually kill confirmed orphans. Default is
+ *  log-only (see header): a week of real candidate logs comes first. */
+function orphanSweepWillKill() {
+    const v = process.env.ORPHAN_SWEEP_KILL;
+    return v === "1" || v === "true";
+}
+/**
+ * Run one orphan-descendant sweep on the existing worker-cleanup cadence. Takes a
+ * single process snapshot (no new timer, no second snapshot on the hot liveness
+ * path), records lineage for live threads, and logs — or, when ORPHAN_SWEEP_KILL
+ * is set, reaps — confirmed orphans. Best-effort and self-contained; any failure
+ * degrades to doing nothing.
+ */
+export async function sweepOrphanDescendants() {
+    const snapshot = await takeWindowsProcessSnapshot();
+    if (snapshot === null) {
+        log.debug("[orphan-sweep] process snapshot unavailable — skipping (never kill on partial data)");
+        return { orphans: [], reusedPidsSkipped: [], olderThanParentSkipped: [], recorded: 0 };
+    }
+    const trackedThreads = readPidFiles().map((e) => ({
+        threadId: e.threadId,
+        pid: e.pid,
+        startedAt: e.startedAt,
+        alive: isProcessAlive(e.pid, e.startedAt) && isSameProcess(e.pid, e.startedAt),
+    }));
+    // Protected: every live tracked-thread pid (managed by worker-cleanup), the
+    // authoritative server pid, and our own pid — these are never orphan targets.
+    const protectedPids = new Set([process.pid]);
+    const serverPid = readServerPid();
+    if (serverPid !== undefined)
+        protectedPids.add(serverPid);
+    for (const t of trackedThreads)
+        if (t.alive)
+            protectedPids.add(t.pid);
+    const result = computeOrphanSweep({
+        snapshot,
+        trackedThreads,
+        lineage: orphanLineage,
+        protectedPids,
+        toleranceMs: PID_IDENTITY_TOLERANCE_MS,
+    });
+    for (const pid of result.reusedPidsSkipped)
+        log.debug(`[orphan-sweep] pid ${pid} recycled since we recorded it — dropped, not killed`);
+    for (const pid of result.olderThanParentSkipped)
+        log.warn(`[orphan-sweep] pid ${pid} is older than its recorded owner — refusing to kill`);
+    const willKill = orphanSweepWillKill();
+    for (const orphan of result.orphans) {
+        if (!willKill) {
+            log.warn(`[orphan-sweep] CANDIDATE (log-only): pid ${orphan.pid} orphaned from dead thread ${orphan.ownerThreadId} — set ORPHAN_SWEEP_KILL=1 to reap`);
+            continue;
+        }
+        log.warn(`[orphan-sweep] REAPING orphan pid ${orphan.pid} (owner thread ${orphan.ownerThreadId} is dead)`);
         try {
-            process.kill(pid);
+            await killProcessTree(orphan.pid, orphan.ownerThreadId, "orphan-sweep", orphan.startMs);
         }
         catch (err) {
-            log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
+            log.warn(`[orphan-sweep] kill of orphan pid ${orphan.pid} failed: ${errorMessage(err)}`);
+        }
+        finally {
+            orphanLineage.delete(orphan.pid);
         }
-        await new Promise((resolve) => setTimeout(resolve, 500));
-        stillAlive = await isProcessAlive(pid, spawnedAtMs);
     }
-    if (!stillAlive) {
-        log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
-        try {
-            unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
+    return result;
+}
+// taskkill on a WEDGED process tree (e.g. a quota-stuck agent whose child
+// node/bash/git processes sit in blocking IO) can take well over 10s to force
+// the whole /T tree down. A short execFile timeout aborts taskkill mid-kill and
+// surfaces as "Command failed" with no stderr, so the process survives and the
+// keeper wrongly flags it UNKILLABLE. Give taskkill headroom to finish; the
+// caller still verifies liveness afterwards, so a fast kill returns immediately.
+const TASKKILL_TIMEOUT_MS = 60_000;
+export function killProcessTree(pid, threadId, caller, spawnedAtMs) {
+    return new Promise((resolve) => {
+        // Self-protection: NEVER tree-kill our own process or the authoritative
+        // server PID. The leading hypothesis for the traceless server death is a
+        // keeper/worker-reaper tree-killing a RECYCLED worker PID that the OS
+        // reassigned to the node server (or one of its ancestors). `taskkill /T`
+        // would then take the whole server tree down with no signal, no [fatal] and
+        // no crash dump — exactly the observed symptom. Refuse and log loudly.
+        if (pid === process.pid) {
+            log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
+            resolve();
+            return;
         }
-        catch {
-            /* PID file already gone (exit handler may have raced us) */
+        const serverPid = readServerPid();
+        if (serverPid !== undefined && pid === serverPid) {
+            log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
+            resolve();
+            return;
         }
-        return;
-    }
-    // Conservative: a survived kill keeps the PID file so a later sweep retries
-    // against fresh evidence (keeper/worker-cleanup own the escalation policy).
-    log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt — NOT removing PID file`);
+        log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
+        const cleanupPidFile = () => {
+            const pidFile = join(PROCESS_PIDS_DIR, `${threadId}.pid`);
+            try {
+                unlinkSync(pidFile);
+            }
+            catch {
+                /* PID file already gone (exit handler may have raced us) */
+            }
+        };
+        const verifyDead = (attempt) => {
+            // spawnedAtMs matters here for orphan-kill callers reaping FRESH pids: a
+            // failed kill must not let a pre-spawn snapshot falsely confirm death and
+            // unlink a live process's pid file. A genuinely dead child is usually
+            // caught by the kill(0) ESRCH probe or a post-spawn snapshot refresh; if
+            // neither lands within the 3 attempts (refresh in flight / breaker open),
+            // we resolve as "still alive" and KEEP the pid file — conservative, and a
+            // later sweep finishes the cleanup against a fresher snapshot.
+            if (!isProcessAlive(pid, spawnedAtMs)) {
+                log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
+                cleanupPidFile();
+                resolve();
+                return;
+            }
+            if (attempt >= 3) {
+                log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after ${attempt} kill attempts — NOT removing PID file`);
+                resolve();
+                return;
+            }
+            // Retry kill after a short delay
+            log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt ${attempt} — retrying`);
+            if (process.platform === "win32") {
+                // Escalation, retry-path only: the happy-path in-process `taskkill /F /T`
+                // already failed to verify-dead. If the failure was EPERM because the
+                // server's own tree is job-object-constrained / I/O-wedged, no in-process
+                // kill will ever succeed — so first spawn a DETACHED, fresh sibling
+                // taskkill that does not inherit our wedged handles.
+                spawnDetachedKillerWin(pid);
+                // Native TerminateProcess (via libuv) as well as taskkill — they take
+                // different paths (taskkill is an external process with its own token;
+                // process.kill issues the syscall directly from this process), so one
+                // can succeed where the other fails on a stubborn process.
+                try {
+                    process.kill(pid);
+                }
+                catch (err) {
+                    log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
+                }
+                // `taskkill /T` walks the tree top-down and can stall when a hung child
+                // blocks the parent's termination — the exact case where the root sat
+                // alive for hours. Kill each descendant leaves-first so a wedged child is
+                // reaped before its parent, freeing the root to die. Logged at info (from
+                // entry through kill count) so a week of logs can answer whether this
+                // escalation path ever actually runs, or is dead weight.
+                log.info(`[process] leaf-first kill ESCALATION entered for pid=${pid} thread=${threadId} (attempt ${attempt}) — taskkill /F /T did not verify-dead`);
+                getWindowsProcessTreeLeafFirst(pid, threadId, (tree) => {
+                    killPidsSequentiallyWin(tree, (killed) => {
+                        log.info(`[process] leaf-first escalation for pid=${pid} thread=${threadId}: issued taskkill to ${tree.length} process(es), ${killed} reported killed`);
+                        setTimeout(() => verifyDead(attempt + 1), 1000);
+                    });
+                });
+            }
+            else {
+                try {
+                    process.kill(pid, "SIGKILL");
+                }
+                catch {
+                    /* process already gone (ESRCH) — verifyDead confirms */
+                }
+                setTimeout(() => verifyDead(attempt + 1), 1000);
+            }
+        };
+        if (process.platform === "win32") {
+            execFile("taskkill", ["/F", "/T", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
+                if (err) {
+                    log.warn(`[process] Kill process ${pid} (thread ${threadId}) failed: ${errorMessage(err)}`);
+                }
+                // Always verify — taskkill can report success but process lingers,
+                // or report failure but process is actually dead.
+                setTimeout(() => verifyDead(1), 500);
+            });
+        }
+        else {
+            try {
+                process.kill(pid, "SIGTERM");
+            }
+            catch (err) {
+                log.debug(`[process] Kill process ${pid} (thread ${threadId}): ${errorMessage(err)}`);
+            }
+            setTimeout(() => verifyDead(1), 500);
+        }
+    });
 }
 //# sourceMappingURL=process.service.js.map