sensorium-mcp 3.1.0-refactor.8 → 3.1.0

package/dist/services/flight-recorder.service.js

@@ -0,0 +1,210 @@
+/**
+ * Flight-recorder heartbeat — the "progress" liveness signal from the
+ * 2026-07-26 process-lifecycle plan (§4.4).
+ *
+ * A dedicated worker thread (immune to main-loop stalls) writes a small
+ * heartbeat file every ~5s containing `{ts, eventLoopLagMs}`. The main
+ * thread pings the worker every second with its own measured timer drift;
+ * the worker combines that with how overdue the last ping is (its own
+ * monotonic clock). The result: when the main event loop freezes, the file
+ * KEEPS updating with a rising lag number — so the supervisor can tell
+ * "alive but stalled" from "dead", and the lag at kill time is on record.
+ *
+ * Clocks are never compared across threads: the main thread measures its
+ * lag with its own performance.now() delta, the worker measures beat
+ * staleness with its own. Both are monotonic within their thread.
+ */
+import { Worker } from "node:worker_threads";
+import { log } from "../lib/logger.js";
+import { dataPath } from "../lib/paths.js";
+/** Where the supervisor looks for the heartbeat (keep in sync with SupervisorWorker.cs). */
+export const HEARTBEAT_FILE_PATH = dataPath("server-heartbeat.json");
+const DEFAULT_BEAT_INTERVAL_MS = 1_000;
+const DEFAULT_WRITE_INTERVAL_MS = 5_000;
+/**
+ * Stall persistence (2026-07-28 forensics): the heartbeat file is overwritten
+ * every write interval, so a lag SPIKE leaves no durable record — the 15:11
+ * freeze had to be reconstructed from telemetry gaps. Spikes above this
+ * threshold are appended to a dedicated spike log by the WORKER (immune to the
+ * stall, survives a mid-stall kill) and WARN-logged by the main thread once the
+ * loop unblocks (records the full stall duration).
+ */
+const LAG_SPIKE_THRESHOLD_MS = 1_000;
+/** Min gap between spike-log appends so a long stall writes a line per write
+ *  interval, not an unbounded stream. */
+const SPIKE_APPEND_MIN_GAP_MS = 5_000;
+/**
+ * How many times a dead heartbeat worker is respawned before giving up. The
+ * heartbeat is the supervisor's stalled-vs-dead evidence — losing it silently
+ * would put every future probe failure back on the kill-blind hung path.
+ */
+const MAX_WORKER_RESPAWNS = 3;
+// Worker body as a CommonJS eval string: avoids a separate compiled worker
+// file, so it works identically under tsx (dev) and dist (production).
+const WORKER_SOURCE = `
+const { parentPort, workerData } = require("node:worker_threads");
+const fs = require("node:fs");
+const { filePath, beatIntervalMs, writeIntervalMs, spikeFilePath, spikeThresholdMs, spikeMinGapMs } = workerData;
+
+let lastBeatAt = performance.now();
+let lastMainLagMs = 0;
+let lastSpikeAppendAt = 0;
+
+parentPort.on("message", (m) => {
+  // Test hook: lets the respawn path be exercised through a real worker crash.
+  if (m.crashForTest) throw new Error("flight-recorder test crash");
+  lastBeatAt = performance.now();
+  lastMainLagMs = m.lagMs;
+});
+
+setInterval(() => {
+  // How overdue is the main thread's ping, by the worker's own clock?
+  const msSinceLastBeat = performance.now() - lastBeatAt;
+  const overdueMs = Math.max(0, msSinceLastBeat - beatIntervalMs);
+  const eventLoopLagMs = Math.round(Math.max(lastMainLagMs, overdueMs));
+  // Durable stall evidence: the heartbeat file below is overwritten each tick,
+  // so append spikes to their own log — written from THIS thread, which keeps
+  // running (and keeps appending a rising number) while the main loop is
+  // frozen, and survives the process being killed mid-stall.
+  if (eventLoopLagMs >= spikeThresholdMs && performance.now() - lastSpikeAppendAt >= spikeMinGapMs) {
+    lastSpikeAppendAt = performance.now();
+    try {
+      fs.appendFileSync(
+        spikeFilePath,
+        JSON.stringify({ ts: new Date().toISOString(), pid: process.pid, eventLoopLagMs }) + "\\n",
+      );
+    } catch { /* best-effort */ }
+  }
+  const payload = JSON.stringify({
+    ts: Date.now(),
+    pid: process.pid,
+    eventLoopLagMs,
+    mainThreadLagMs: Math.round(lastMainLagMs),
+    msSinceLastBeat: Math.round(msSinceLastBeat),
+  });
+  // Prefer atomic tmp+rename; fall back to a direct write if the destination
+  // is transiently locked (Windows AV/indexer). Readers tolerate a rare torn
+  // read by treating unparseable content as "no heartbeat".
+  const tmp = filePath + ".tmp";
+  try {
+    fs.writeFileSync(tmp, payload);
+    fs.renameSync(tmp, filePath);
+  } catch {
+    try {
+      fs.writeFileSync(filePath, payload);
+    } catch {
+      /* heartbeat is best-effort; never throw from the recorder */
+    }
+  }
+}, writeIntervalMs);
+`;
+/**
+ * Start the flight recorder. Never throws — a broken recorder must not take
+ * down the server it instruments. Both the worker and the main-thread timer
+ * are unref'd, so they never keep the process alive.
+ */
+export function startFlightRecorder(options = {}) {
+    const filePath = options.filePath ?? HEARTBEAT_FILE_PATH;
+    const beatIntervalMs = options.beatIntervalMs ?? DEFAULT_BEAT_INTERVAL_MS;
+    const writeIntervalMs = options.writeIntervalMs ?? DEFAULT_WRITE_INTERVAL_MS;
+    let worker = null;
+    let beatTimer = null;
+    let stopped = false;
+    let respawns = 0;
+    function stopBeat() {
+        if (beatTimer)
+            clearInterval(beatTimer);
+        beatTimer = null;
+    }
+    function startBeat(w) {
+        let lastFire = performance.now();
+        beatTimer = setInterval(() => {
+            const now = performance.now();
+            // Timer drift = how late this 1s tick fired — the event-loop lag.
+            const lagMs = Math.max(0, now - lastFire - beatIntervalMs);
+            lastFire = now;
+            // This code runs AFTER the loop unblocked, so lagMs here is the FULL
+            // stall duration — record it where post-mortems will look (server.log),
+            // with what the loop was late for. The worker already streamed the
+            // rising lag to the spike file while the stall was in progress.
+            if (lagMs >= LAG_SPIKE_THRESHOLD_MS)
+                log.warn(`[flight-recorder] event-loop stall ended: the loop was blocked for ~${Math.round(lagMs)}ms`);
+            w.postMessage({ lagMs });
+        }, beatIntervalMs);
+        beatTimer.unref();
+    }
+    // A dead worker means the supervisor loses its stalled-vs-dead evidence for
+    // the rest of the process's life — never let that happen silently. Stop the
+    // beat timer (no more postMessage into a corpse) and respawn a bounded
+    // number of times; past the cap, say loudly that the signal is down.
+    function onWorkerDown(cause) {
+        stopBeat();
+        worker = null;
+        if (stopped)
+            return;
+        if (respawns >= MAX_WORKER_RESPAWNS) {
+            log.error(`[flight-recorder] heartbeat worker down (${cause}) after ${respawns} respawns — ` +
+                `giving up: the progress signal is DOWN for the rest of this process`);
+            return;
+        }
+        respawns++;
+        log.error(`[flight-recorder] heartbeat worker down (${cause}) — respawning (${respawns}/${MAX_WORKER_RESPAWNS})`);
+        try {
+            spawnWorker();
+        }
+        catch (err) {
+            // Never let the recorder take down the server it instruments — this
+            // runs inside a worker event handler, where a throw would be uncaught.
+            worker = null;
+            log.error(`[flight-recorder] respawn failed (progress signal is DOWN): ${err}`);
+        }
+    }
+    function spawnWorker() {
+        const w = new Worker(WORKER_SOURCE, {
+            eval: true,
+            workerData: {
+                filePath,
+                beatIntervalMs,
+                writeIntervalMs,
+                spikeFilePath: filePath.replace(/\.json$/, "") + "-spikes.log",
+                spikeThresholdMs: LAG_SPIKE_THRESHOLD_MS,
+                spikeMinGapMs: SPIKE_APPEND_MIN_GAP_MS,
+            },
+        });
+        worker = w;
+        w.unref();
+        // "error" is always followed by "exit"; the worker===w guard makes the
+        // second event (or a stale exit after a respawn) a no-op.
+        w.on("error", (err) => {
+            if (worker === w)
+                onWorkerDown(`error: ${err}`);
+        });
+        w.on("exit", (code) => {
+            if (worker === w)
+                onWorkerDown(`exit code ${code}`);
+        });
+        startBeat(w);
+    }
+    try {
+        spawnWorker();
+        log.info(`[flight-recorder] heartbeat → ${filePath} every ${writeIntervalMs}ms`);
+    }
+    catch (err) {
+        worker = null;
+        stopBeat();
+        log.warn(`[flight-recorder] failed to start (continuing without heartbeat): ${err}`);
+    }
+    return {
+        async stop() {
+            stopped = true;
+            stopBeat();
+            if (worker)
+                await worker.terminate().catch(() => { });
+            worker = null;
+        },
+        crashWorkerForTest() {
+            worker?.postMessage({ crashForTest: true });
+        },
+    };
+}
+//# sourceMappingURL=flight-recorder.service.js.map

package/dist/services/flight-recorder.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flight-recorder.service.js","sourceRoot":"","sources":["../../src/services/flight-recorder.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,4FAA4F;AAC5F,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAC,uBAAuB,CAAC,CAAC;AAErE,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,KAAK,CAAC;AACrC;yCACyC;AACzC,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAgBtC;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,2EAA2E;AAC3E,uEAAuE;AACvE,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwDrB,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAAiC,EAAE;IACrE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,mBAAmB,CAAC;IACzD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;IAE7E,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAA0B,IAAI,CAAC;IAC5C,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,SAAS,QAAQ;QACf,IAAI,SAAS;YAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QACxC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,SAAS,SAAS,CAAC,CAAS;QAC1B,IAAI,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACjC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YAC3B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC9B,kEAAkE;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,QAAQ,GAAG,cAAc,CAAC,CAAC;YAC3D,QAAQ,GAAG,GAAG,CAAC;YACf,qEAAqE;YACrE,wEAAwE;YACxE,mEAAmE;YACnE,gEAAgE;YAChE,IAAI,KAAK,IAAI,sBAAsB;gBACjC,GAAG,CAAC,IAAI,CACN,uEAAuE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAC7F,CAAC;YACJ,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3B,CAAC,EAAE,cAAc,CAAC,CAAC;QACnB,SAAS,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED,4EAA4E;IAC5E,4EAA4E;IAC5E,uEAAuE;IACvE,qEAAqE;IACrE,SAAS,YAAY,CAAC,KAAa;QACjC,QAAQ,EAAE,CAAC;QACX,MAAM,GAAG,IAAI,CAAC;QACd,IAAI,OAAO;YAAE,OAAO;QACpB,IAAI,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACpC,GAAG,CAAC,KAAK,CACP,4CAA4C,KAAK,WAAW,QAAQ,cAAc;gBAChF,qEAAqE,CACxE,CAAC;YACF,OAAO;QACT,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,GAAG,CAAC,KAAK,CACP,4CAA4C,KAAK,mBAAmB,QAAQ,IAAI,mBAAmB,GAAG,CACvG,CAAC;QACF,IAAI,CAAC;YACH,WAAW,EAAE,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,oEAAoE;YACpE,uEAAuE;YACvE,MAAM,GAAG,IAAI,CAAC;YACd,GAAG,CAAC,KAAK,CAAC,+DAA+D,GAAG,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,SAAS,WAAW;QAClB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE;YAClC,IAAI,EAAE,IAAI;YACV,UAAU,EAAE;gBACV,QAAQ;gBACR,cAAc;gBACd,eAAe;gBACf,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,aAAa;gBAC9D,gBAAgB,EAAE,sBAAsB;gBACxC,aAAa,EAAE,uBAAuB;aACvC;SACF,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,CAAC;QACX,CAAC,CAAC,KAAK,EAAE,CAAC;QACV,uEAAuE;QACvE,0DAA0D;QAC1D,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACpB,IAAI,MAAM,KAAK,CAAC;gBAAE,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACpB,IAAI,MAAM,KAAK,CAAC;gBAAE,YAAY,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,SAAS,CAAC,CAAC,CAAC,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,WAAW,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,iCAAiC,QAAQ,UAAU,eAAe,IAAI,CAAC,CAAC;IACnF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,IAAI,CAAC;QACd,QAAQ,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,qEAAqE,GAAG,EAAE,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,KAAK,CAAC,IAAI;YACR,OAAO,GAAG,IAAI,CAAC;YACf,QAAQ,EAAE,CAAC;YACX,IAAI,MAAM;gBAAE,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACrD,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,kBAAkB;YAChB,MAAM,EAAE,WAAW,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/services/keeper.service.d.ts

@@ -15,6 +15,15 @@ export declare class KeeperService {
     start(): void;
     stop(): void;
     private syncKeepers;
+    /** One keeper cycle with persistence: whenever a cycle changed any durable
+     *  policy counter, push the new state to the broker's registry so it survives
+     *  an MCP restart (plan §4.3). Single choke point — the cycle body has many
+     *  early returns and each mutation site would otherwise need its own save. */
+    private runKeeperCycle;
+    private static durableStateOf;
+    /** Restore persisted counters when a keeper starts (max-merge so counters that
+     *  already advanced in this session are never rolled back). */
+    private hydrateFromRegistry;
     private checkAndRestart;
     private notifyDeath;
 }

package/dist/services/keeper.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"keeper.service.d.ts","sourceRoot":"","sources":["../../src/services/keeper.service.ts"],"names":[],"mappings":"AAcA,OAAO,EAAe,KAAK,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAGzF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AA2I5D,qBAAa,aAAa;IAKtB,OAAO,CAAC,IAAI;IAJd,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAA+C;gBAGtD,IAAI,EAAE;QACZ,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;QACnD,eAAe,EAAE,sBAAsB,CAAC;QACxC,QAAQ,EAAE;YACR,WAAW,CACT,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;SAClB,CAAC;QACF,MAAM,EAAE,MAAM,CAAC;KAChB;IAGH,KAAK,IAAI,IAAI;IAMb,IAAI,IAAI,IAAI;IAWZ,OAAO,CAAC,WAAW;YAgEL,eAAe;YA4Of,WAAW;CAkB1B"}
+{"version":3,"file":"keeper.service.d.ts","sourceRoot":"","sources":["../../src/services/keeper.service.ts"],"names":[],"mappings":"AAeA,OAAO,EAAe,KAAK,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAGzF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAmK5D,qBAAa,aAAa;IAKtB,OAAO,CAAC,IAAI;IAJd,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAA+C;gBAGtD,IAAI,EAAE;QACZ,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;QACnD,eAAe,EAAE,sBAAsB,CAAC;QACxC,QAAQ,EAAE;YACR,WAAW,CACT,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;SAClB,CAAC;QACF,MAAM,EAAE,MAAM,CAAC;KAChB;IAGH,KAAK,IAAI,IAAI;IAMb,IAAI,IAAI,IAAI;IAWZ,OAAO,CAAC,WAAW;IAwEnB;;;kFAG8E;YAChE,cAAc;IAc5B,OAAO,CAAC,MAAM,CAAC,cAAc;IAY7B;mEAC+D;YACjD,mBAAmB;YAkCnB,eAAe;YA2Uf,WAAW;CAkB1B"}

package/dist/services/keeper.service.js

@@ -1,13 +1,14 @@
 import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { log } from "../lib/logger.js";
-import { findAliveThreadViaPidFile, isProcessAlive, killProcessTree, PROCESS_LOGS_DIR, THREAD_LOGS_DIR, } from "./process.service.js";
+import { findAliveThreadViaPidFile, isProcessAlive, isThreadAlive, killThreadTree, PROCESS_LOGS_DIR, THREAD_LOGS_DIR, } from "./process.service.js";
 import { getThreadSessionCount } from "../sessions/mcp-registry.js";
 import { readThreadHeartbeat } from "../data/file-storage.js";
 import { dispatchSpawn } from "./agent-spawn.service.js";
 import { getKeepAliveThreads, getThread } from "../data/memory/thread-registry.js";
 import { ThreadState } from "./thread-lifecycle.service.js";
 import { errorMessage } from "../lib/utils.js";
+import { brokerGetKeeperState, brokerSaveKeeperState } from "./broker-client.js";
 const KEEPER_CHECK_INTERVAL_MS = 120_000; // 2 min
 const KEEPER_MAX_RETRIES = 5;
 const KEEPER_COOLDOWN_MS = 300_000; // 5 min
@@ -26,8 +27,8 @@ const UNKILLABLE_RENOTIFY_MS = 3_600_000; // 1 hour — re-ping the operator at
 // tool calls) after startup. Used to avoid trusting a frozen pre-restart
 // heartbeat as an "untracked live worker" and needlessly delaying restart.
 const SERVER_START_TIME = Date.now();
-/** DIAGNOSTIC (read-only): newest stream-json stdout log file for a thread, or null. */
-function findLatestThreadLogFile(threadId) {
+/** DIAGNOSTIC (read-only): newest stream-json stdout log for a thread (path + mtime + size), or null. */
+function findLatestThreadLog(threadId) {
     let files;
     try {
         files = readdirSync(THREAD_LOGS_DIR);
@@ -42,15 +43,19 @@ function findLatestThreadLogFile(threadId) {
             continue;
         const full = join(THREAD_LOGS_DIR, file);
         try {
-            const mtimeMs = statSync(full).mtimeMs;
-            if (!best || mtimeMs > best.mtimeMs)
-                best = { path: full, mtimeMs };
+            const st = statSync(full);
+            if (!best || st.mtimeMs > best.mtimeMs)
+                best = { path: full, mtimeMs: st.mtimeMs, sizeBytes: st.size };
         }
         catch {
             /* file vanished between readdir and stat */
         }
     }
-    return best?.path ?? null;
+    return best;
+}
+/** DIAGNOSTIC (read-only): newest stream-json stdout log path for a thread, or null. */
+function findLatestThreadLogFile(threadId) {
+    return findLatestThreadLog(threadId)?.path ?? null;
 }
 /**
  * DIAGNOSTIC (logging-only): capture what node knows about a STUCK thread at the
@@ -61,7 +66,7 @@ function findLatestThreadLogFile(threadId) {
  * NEVER affect the kill. This runs only when the keeper has ALREADY decided the
  * thread is stuck; it triggers no new kills.
  */
-function writeWedgeSnapshot(threadId, pid, reason, heartbeat, now) {
+async function writeWedgeSnapshot(threadId, pid, reason, heartbeat, now) {
     try {
         const dir = join(PROCESS_LOGS_DIR, "wedge-snapshots");
         mkdirSync(dir, { recursive: true });
@@ -72,7 +77,7 @@ function writeWedgeSnapshot(threadId, pid, reason, heartbeat, now) {
         lines.push(`timestamp: ${new Date(now).toISOString()}`);
         let alive;
         try {
-            alive = isProcessAlive(pid);
+            alive = await isProcessAlive(pid);
         }
         catch (err) {
             alive = `error: ${errorMessage(err)}`;
@@ -163,7 +168,7 @@ export class KeeperService {
         // Add new keepers
         for (const thread of threads) {
             if (!this.keepers.has(thread.threadId)) {
-                this.keepers.set(thread.threadId, {
+                const entry = {
                     threadId: thread.threadId,
                     retryCount: 0,
                     fastExitCount: 0,
@@ -175,15 +180,23 @@ export class KeeperService {
                     limboWarned: false,
                     survivedKillCount: 0,
                     lastUnkillableNotifyMs: 0,
-                });
+                    lastLogSizeBytes: 0,
+                    lastHeartbeatMs: 0,
+                };
+                this.keepers.set(thread.threadId, entry);
                 log.info(`[keeper] Started keeper for thread ${thread.threadId} ('${thread.name}')`);
+                // Hydrate backoff counters from the broker's registry (plan §4.3): they
+                // used to reset to zero on every MCP restart — during the Jul 25 wave 15
+                // resets in 2h19m meant no backoff ever matured. Fire-and-forget; until
+                // it lands the entry runs on defaults (today's behavior).
+                void this.hydrateFromRegistry(entry);
             }
         }
         // Check all active keepers (skip if already in-flight to prevent duplicate spawns)
         for (const entry of this.keepers.values()) {
             if (!entry.stopped && !entry.checking) {
                 entry.checking = true;
-                this.checkAndRestart(entry)
+                this.runKeeperCycle(entry)
                     .catch((err) => {
                     log.warn(`[keeper] checkAndRestart error for thread ${entry.threadId}: ${errorMessage(err)}`);
                 })
@@ -193,6 +206,64 @@ export class KeeperService {
             }
         }
     }
+    /** One keeper cycle with persistence: whenever a cycle changed any durable
+     *  policy counter, push the new state to the broker's registry so it survives
+     *  an MCP restart (plan §4.3). Single choke point — the cycle body has many
+     *  early returns and each mutation site would otherwise need its own save. */
+    async runKeeperCycle(entry) {
+        const before = KeeperService.durableStateOf(entry);
+        try {
+            await this.checkAndRestart(entry);
+        }
+        finally {
+            const after = KeeperService.durableStateOf(entry);
+            if (JSON.stringify(before) !== JSON.stringify(after)) {
+                void brokerSaveKeeperState(after).catch(() => {
+                    /* unreachable broker already WARN-logged by the client */
+                });
+            }
+        }
+    }
+    static durableStateOf(entry) {
+        return {
+            threadId: entry.threadId,
+            retryCount: entry.retryCount,
+            fastExitCount: entry.fastExitCount,
+            fastExitEscalation: entry.fastExitEscalation,
+            cooldownUntilMs: entry.cooldownUntil > 0 ? entry.cooldownUntil : null,
+            survivedKillCount: entry.survivedKillCount,
+            lastStartAtMs: entry.lastStartTime > 0 ? entry.lastStartTime : null,
+        };
+    }
+    /** Restore persisted counters when a keeper starts (max-merge so counters that
+     *  already advanced in this session are never rolled back). */
+    async hydrateFromRegistry(entry) {
+        try {
+            const state = await brokerGetKeeperState(entry.threadId);
+            if (!state || entry.stopped)
+                return;
+            entry.retryCount = Math.max(entry.retryCount, state.retryCount);
+            entry.fastExitCount = Math.max(entry.fastExitCount, state.fastExitCount);
+            entry.fastExitEscalation = Math.max(entry.fastExitEscalation, state.fastExitEscalation);
+            entry.survivedKillCount = Math.max(entry.survivedKillCount, state.survivedKillCount);
+            if (state.cooldownUntilMs !== null && state.cooldownUntilMs > entry.cooldownUntil)
+                entry.cooldownUntil = state.cooldownUntilMs;
+            if (state.lastStartAtMs !== null && entry.lastStartTime === 0)
+                entry.lastStartTime = state.lastStartAtMs;
+            if (state.retryCount > 0 ||
+                state.fastExitCount > 0 ||
+                state.fastExitEscalation > 0 ||
+                state.survivedKillCount > 0 ||
+                (state.cooldownUntilMs !== null && state.cooldownUntilMs > Date.now())) {
+                log.info(`[keeper] Thread ${entry.threadId}: restored persisted keeper state (retry=${state.retryCount}, fastExit=${state.fastExitCount}, escalation=${state.fastExitEscalation}, survivedKill=${state.survivedKillCount}${state.cooldownUntilMs !== null && state.cooldownUntilMs > Date.now()
+                    ? `, cooldown until ${new Date(state.cooldownUntilMs).toISOString()}`
+                    : ""})`);
+            }
+        }
+        catch (err) {
+            log.debug(`[keeper] keeper-state hydration failed for thread ${entry.threadId}: ${errorMessage(err)}`);
+        }
+    }
     async checkAndRestart(entry) {
         const now = Date.now();
         // 1. Check cooldown
@@ -201,33 +272,80 @@ export class KeeperService {
             return;
         }
         // 2. Check if thread is running — reads PID file directly (authoritative OS evidence)
-        const alivePid = findAliveThreadViaPidFile(entry.threadId);
-        // 2a. PID file shows no live tracked process — but a fresh heartbeat is the
-        // authoritative "someone is actively serving this thread" signal: the poll
-        // loop rewrites it every ~2s (and every tool call). If it is fresh, a live
-        // Claude worker is STILL serving the thread under a PID we failed to track
-        // (e.g. on Windows a process-tree/launcher death leaves the real worker
-        // running while the tracked PID is gone). Spawning now would create a
-        // DUPLICATE process → the operator gets answered twice. Skip the restart;
-        // when the real worker finally exits, its heartbeat goes stale and the
-        // normal restart path takes over.
+        const aliveProc = await findAliveThreadViaPidFile(entry.threadId);
+        const alivePid = aliveProc?.pid;
+        // 2a. PID file shows no live tracked process. A fresh heartbeat USED to be
+        // trusted alone as "someone is actively serving this thread" — but with the
+        // tracked PID gone, a heartbeat < 1h old is NOT proof of life: the heartbeat
+        // file is only rewritten in the wait poll loop and on tool calls, so a long
+        // final model turn freezes it, and if the process then dies (e.g. Claude's
+        // 5-hour org limit 429s it) the frozen-but-recent heartbeat lingers for up to
+        // STUCK_THRESHOLD_MS. That is exactly how thread 1327's death was silently
+        // swallowed for ~67 minutes on 2026-07-27: the limbo guard fired every ~2-min
+        // cycle while BOTH the restart AND the operator-facing 💀 notification were
+        // suppressed, until the frozen heartbeat finally crossed 1h.
+        //
+        // The ONLY signals that prove a worker is alive RIGHT NOW are CROSS-CYCLE
+        // deltas measured between keeper cycles (~2 min apart) — symmetric to the
+        // 2026-07-26 stuck-branch fix, which likewise learned that a stale heartbeat
+        // alone does not prove/disprove a wedge because a long model turn freezes it:
+        //   • heartbeat ADVANCED since the previous cycle — a live untracked worker
+        //     idle in the long-poll rewrites it every ~2s, so it keeps advancing, OR
+        //   • stdout GREW in bytes since the previous cycle (sizeGrew) — a live worker
+        //     mid model turn emits tokens even while its heartbeat is frozen.
+        // A genuinely dead worker shows BOTH frozen: no heartbeat advance AND no
+        // stdout growth → fall through to the normal restart + notifyDeath path so the
+        // operator is warned within ~2 keeper cycles instead of ~1 hour. (mtime < 1h
+        // is deliberately NOT used here: a just-died process leaves a log whose mtime
+        // is recent for up to an hour — the same failure mode as the frozen heartbeat.)
         if (alivePid === undefined) {
             const heartbeat = readThreadHeartbeat(entry.threadId);
-            if (heartbeat !== null &&
-                heartbeat > SERVER_START_TIME &&
-                now - heartbeat < STUCK_THRESHOLD_MS) {
-                if (!entry.limboWarned) {
-                    log.warn(`[keeper] Thread ${entry.threadId}: no live tracked PID but heartbeat is fresh (${Math.round((now - heartbeat) / 1000)}s ago) — a live worker is still serving it under an untracked PID. Skipping restart to avoid a duplicate process.`);
-                    entry.limboWarned = true;
-                }
-                else {
-                    log.debug(`[keeper] Thread ${entry.threadId}: still in untracked-live-worker limbo (heartbeat ${Math.round((now - heartbeat) / 1000)}s ago) — skipping restart.`);
+            // Upper sanity bound: no heartbeat, a pre-restart heartbeat from an older
+            // server generation, or a heartbeat already older than 1h is definitely not
+            // a live untracked worker → fall through to the cleanup + restart path.
+            const heartbeatPlausiblyLive = heartbeat !== null && heartbeat > SERVER_START_TIME && now - heartbeat < STUCK_THRESHOLD_MS;
+            if (heartbeatPlausiblyLive) {
+                const streamLog = findLatestThreadLog(entry.threadId);
+                const heartbeatAdvanced = entry.lastHeartbeatMs > 0 && heartbeat > entry.lastHeartbeatMs;
+                const sizeGrew = entry.lastLogSizeBytes > 0 &&
+                    streamLog !== null &&
+                    streamLog.sizeBytes > entry.lastLogSizeBytes;
+                // First cycle of this episode: no baseline yet, so no cross-cycle delta can
+                // be measured. Record the baselines and grant exactly ONE ~2-min grace
+                // cycle (vs. the old up-to-1-hour hole) before demanding evidence of life.
+                const noBaseline = entry.lastHeartbeatMs === 0 && entry.lastLogSizeBytes === 0;
+                if (noBaseline || heartbeatAdvanced || sizeGrew) {
+                    // Still (or possibly) a live untracked worker — refresh baselines so the
+                    // next cycle measures a fresh delta, then skip the restart.
+                    entry.lastHeartbeatMs = heartbeat;
+                    if (streamLog !== null)
+                        entry.lastLogSizeBytes = streamLog.sizeBytes;
+                    const evidence = noBaseline
+                        ? "first cycle — baselining"
+                        : heartbeatAdvanced
+                            ? "heartbeat advanced"
+                            : "stdout grew";
+                    if (!entry.limboWarned) {
+                        log.warn(`[keeper] Thread ${entry.threadId}: no live tracked PID but a worker still shows life (${evidence}, heartbeat ${Math.round((now - heartbeat) / 1000)}s ago) — a live worker is still serving it under an untracked PID. Skipping restart to avoid a duplicate process.`);
+                        entry.limboWarned = true;
+                    }
+                    else {
+                        log.debug(`[keeper] Thread ${entry.threadId}: still in untracked-live-worker limbo (${evidence}, heartbeat ${Math.round((now - heartbeat) / 1000)}s ago) — skipping restart.`);
+                    }
+                    return;
                 }
-                return;
+                // Heartbeat recent but FROZEN across cycles AND stdout static → the worker
+                // is dead (the 2026-07-27 thread-1327 case). Do NOT skip — fall through to
+                // the cleanup + normal restart + notifyDeath path below.
             }
             // Process is genuinely gone — any prior survived-kill episode is over.
             entry.survivedKillCount = 0;
             entry.lastUnkillableNotifyMs = 0;
+            // A restart writes a fresh (empty) stdout log; drop the byte baseline so the
+            // next episode never compares sizes across two different log files.
+            entry.lastLogSizeBytes = 0;
+            // Drop the heartbeat liveness baseline too — the next episode measures fresh.
+            entry.lastHeartbeatMs = 0;
         }
         // Not in limbo (PID alive, or heartbeat stale/absent) — clear the flag so a
         // future episode logs afresh.
@@ -235,7 +353,7 @@ export class KeeperService {
         if (alivePid !== undefined) {
             // 3. Check if stuck via heartbeat
             const heartbeat = readThreadHeartbeat(entry.threadId);
-            const isStuck = heartbeat !== null
+            const heartbeatStale = heartbeat !== null
                 ? now - heartbeat > STUCK_THRESHOLD_MS
                 : // No parseable heartbeat but process is alive — treat as stuck unless
                     // freshly spawned by THIS keeper instance (lastStartTime within threshold).
@@ -246,6 +364,35 @@ export class KeeperService {
                         : // Restored from PID file (lastStartTime=0): no heartbeat data at all
                             // means the process never wrote one — definitely stuck.
                             true;
+            // A stale heartbeat alone does NOT prove a wedge. The heartbeat file is only
+            // rewritten on MCP tool calls and inside the wait poll loop; a single long
+            // model turn (heavy thinking / a large generation) makes no tool calls and is
+            // not in the poll loop, so the heartbeat freezes for the whole turn even
+            // though the agent is alive and productively streaming tokens. The agent's
+            // stdout stream-json log grows on every emitted line, giving an INDEPENDENT
+            // liveness signal. We read it two ways so neither can silently fail:
+            //   (a) mtime advanced within the window, and
+            //   (b) the log GREW in bytes since the previous keeper cycle — immune to
+            //       Windows buffering the inherited-fd write so mtime lags behind output.
+            // Only declare a wedge when the heartbeat is stale AND neither stream signal
+            // shows life — otherwise the keeper decapitates a working agent mid-answer
+            // (exactly what killed thread 1327 at 61m on 2026-07-26). Checked only when
+            // the heartbeat is already stale, so the healthy common case does no extra I/O.
+            const streamLog = heartbeatStale ? findLatestThreadLog(entry.threadId) : null;
+            const streamAgeMs = streamLog !== null ? now - streamLog.mtimeMs : null;
+            const mtimeFresh = streamAgeMs !== null && streamAgeMs >= 0 && streamAgeMs < STUCK_THRESHOLD_MS;
+            const sizeGrew = streamLog !== null &&
+                entry.lastLogSizeBytes > 0 &&
+                streamLog.sizeBytes > entry.lastLogSizeBytes;
+            if (streamLog !== null)
+                entry.lastLogSizeBytes = streamLog.sizeBytes;
+            const streamFresh = mtimeFresh || sizeGrew;
+            if (heartbeatStale && streamFresh) {
+                log.info(`[keeper] Thread ${entry.threadId} heartbeat is stale (${heartbeat !== null ? Math.round((now - heartbeat) / 60000) + "m" : "none"}) but its stdout stream shows life (${mtimeFresh
+                    ? `mtime ${Math.round(streamAgeMs / 1000)}s ago`
+                    : "grew in bytes"}) — agent is mid-turn (streaming), not wedged. Skipping kill.`);
+            }
+            const isStuck = heartbeatStale && !streamFresh;
             if (isStuck) {
                 const reason = heartbeat !== null
                     ? `no heartbeat for ${Math.round((now - heartbeat) / 60000)}m`
@@ -265,11 +412,21 @@ export class KeeperService {
                 // DIAGNOSTIC (logging-only): forensic snapshot BEFORE the kill so we can
                 // see whether the stuck agent was still MCP-connected or already orphaned,
                 // plus the tail of its stdout. Best-effort; never blocks/affects the kill.
-                writeWedgeSnapshot(entry.threadId, alivePid, reason, heartbeat, now);
-                await killProcessTree(alivePid, entry.threadId);
-                // Verify kill succeeded before restarting — if the process is still alive,
-                // do NOT spawn a second instance. Keeper will retry on the next cycle.
-                if (isProcessAlive(alivePid)) {
+                await writeWedgeSnapshot(entry.threadId, alivePid, reason, heartbeat, now);
+                // THREAD-scoped kill: the recorded alivePid is a launcher that has long
+                // since exited while its forked child lives on in the thread's job, so a
+                // killProcessTree(alivePid) routes to the broker's KillByPid AlreadyDead
+                // fast-path and never touches the live child — then isProcessAlive(launcherPid)
+                // reads false and the keeper falsely concludes success → duplicate spawn +
+                // orphaned wedged child. Route by threadId (broker KillByThread → job kill of
+                // the real child); pass the launcher pid + PID-file startedAt as the
+                // identity-checked degraded fallback.
+                await killThreadTree(entry.threadId, "keeper-stuck", alivePid, aliveProc?.startedAt);
+                // Verify kill succeeded before restarting — if the thread still has a live
+                // member, do NOT spawn a second instance. Keeper will retry next cycle.
+                // THREAD-scoped so a broker-unreachable/unknown verdict treats the thread as
+                // alive (never a false "killed" → never a duplicate spawn).
+                if (await isThreadAlive(entry.threadId, alivePid, aliveProc?.startedAt)) {
                     entry.survivedKillCount++;
                     log.warn(`[keeper] Thread ${entry.threadId} PID=${alivePid} survived kill (${entry.survivedKillCount}x) — skipping restart (will retry next cycle)`);
                     // Escalate: a process that repeatedly refuses to die needs operator
@@ -290,6 +447,8 @@ export class KeeperService {
                 entry.survivedKillCount = 0;
                 entry.lastUnkillableNotifyMs = 0;
                 entry.retryCount = 0;
+                // Restart writes a fresh stdout log — drop the byte baseline (see above).
+                entry.lastLogSizeBytes = 0;
                 // Fall through to restart
             }
             else {
@@ -355,7 +514,7 @@ export class KeeperService {
             ? `last keeper-start ${Math.round((now - entry.lastStartTime) / 1000)}s ago`
             : "never started by this server (inherited PID file)";
         log.info(`[keeper] Thread ${entry.threadId} not running — restarting (attempt ${entry.retryCount + 1}/${KEEPER_MAX_RETRIES}, ${lastStartLabel})`);
-        const result = dispatchSpawn((thread.client || "claude"), thread.name, entry.threadId, this.deps.threadLifecycle, thread.workingDirectory ?? undefined);
+        const result = await dispatchSpawn((thread.client || "claude"), thread.name, entry.threadId, this.deps.threadLifecycle, thread.workingDirectory ?? undefined);
         if ("pid" in result) {
             entry.lastStartTime = Date.now();
             entry.retryCount = 0;