sensorium-mcp 3.1.0-refactor.9 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/agent-settings.d.ts +6 -0
- package/dist/config/agent-settings.d.ts.map +1 -1
- package/dist/config/agent-settings.js +22 -0
- package/dist/config/agent-settings.js.map +1 -1
- package/dist/config/detail-grant.d.ts +39 -0
- package/dist/config/detail-grant.d.ts.map +1 -0
- package/dist/config/detail-grant.js +61 -0
- package/dist/config/detail-grant.js.map +1 -0
- package/dist/config/index.d.ts +2 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -3
- package/dist/config/index.js.map +1 -1
- package/dist/config/misc-settings.d.ts +2 -0
- package/dist/config/misc-settings.d.ts.map +1 -1
- package/dist/config/misc-settings.js +22 -0
- package/dist/config/misc-settings.js.map +1 -1
- package/dist/config/settings-store.d.ts +41 -2
- package/dist/config/settings-store.d.ts.map +1 -1
- package/dist/config/settings-store.js +151 -14
- package/dist/config/settings-store.js.map +1 -1
- package/dist/dashboard/routes/copilot.d.ts.map +1 -1
- package/dist/dashboard/routes/copilot.js +17 -1
- package/dist/dashboard/routes/copilot.js.map +1 -1
- package/dist/dashboard/routes/settings.d.ts +2 -0
- package/dist/dashboard/routes/settings.d.ts.map +1 -1
- package/dist/dashboard/routes/settings.js +25 -1
- package/dist/dashboard/routes/settings.js.map +1 -1
- package/dist/dashboard/routes/threads-read.d.ts.map +1 -1
- package/dist/dashboard/routes/threads-read.js +15 -3
- package/dist/dashboard/routes/threads-read.js.map +1 -1
- package/dist/dashboard/routes/threads.d.ts.map +1 -1
- package/dist/dashboard/routes/threads.js +13 -4
- package/dist/dashboard/routes/threads.js.map +1 -1
- package/dist/dashboard/routes.d.ts.map +1 -1
- package/dist/dashboard/routes.js +14 -5
- package/dist/dashboard/routes.js.map +1 -1
- package/dist/dashboard/spa.html +11 -11
- package/dist/data/daily-marker.js +3 -3
- package/dist/data/daily-marker.js.map +1 -1
- package/dist/data/file-storage.d.ts.map +1 -1
- package/dist/data/file-storage.js +3 -3
- package/dist/data/file-storage.js.map +1 -1
- package/dist/data/memory/bootstrap.d.ts.map +1 -1
- package/dist/data/memory/bootstrap.js +2 -3
- package/dist/data/memory/bootstrap.js.map +1 -1
- package/dist/data/memory/migration-runner.d.ts.map +1 -1
- package/dist/data/memory/migration-runner.js +4 -7
- package/dist/data/memory/migration-runner.js.map +1 -1
- package/dist/data/memory/schema.js +2 -2
- package/dist/data/memory/schema.js.map +1 -1
- package/dist/index.js +67 -21
- package/dist/index.js.map +1 -1
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/logger.js +21 -2
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/paths.d.ts +29 -0
- package/dist/lib/paths.d.ts.map +1 -0
- package/dist/lib/paths.js +36 -0
- package/dist/lib/paths.js.map +1 -0
- package/dist/lib/types.d.ts +1 -0
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/server/factory.js +16 -6
- package/dist/server/factory.js.map +1 -1
- package/dist/server/http-server.d.ts.map +1 -1
- package/dist/server/http-server.js +52 -20
- package/dist/server/http-server.js.map +1 -1
- package/dist/services/agent-spawn.service.d.ts +3 -3
- package/dist/services/agent-spawn.service.d.ts.map +1 -1
- package/dist/services/agent-spawn.service.js +30 -9
- package/dist/services/agent-spawn.service.js.map +1 -1
- package/dist/services/background-runner.d.ts.map +1 -1
- package/dist/services/background-runner.js +22 -12
- package/dist/services/background-runner.js.map +1 -1
- package/dist/services/broker-client.d.ts +209 -0
- package/dist/services/broker-client.d.ts.map +1 -0
- package/dist/services/broker-client.js +359 -0
- package/dist/services/broker-client.js.map +1 -0
- package/dist/services/claude-spawn.service.d.ts +1 -1
- package/dist/services/claude-spawn.service.d.ts.map +1 -1
- package/dist/services/claude-spawn.service.js +25 -25
- package/dist/services/claude-spawn.service.js.map +1 -1
- package/dist/services/codex-spawn.service.d.ts +1 -1
- package/dist/services/codex-spawn.service.d.ts.map +1 -1
- package/dist/services/codex-spawn.service.js +19 -24
- package/dist/services/codex-spawn.service.js.map +1 -1
- package/dist/services/copilot-spawn.service.d.ts +1 -1
- package/dist/services/copilot-spawn.service.d.ts.map +1 -1
- package/dist/services/copilot-spawn.service.js +86 -37
- package/dist/services/copilot-spawn.service.js.map +1 -1
- package/dist/services/dispatcher/broker.d.ts +4 -3
- package/dist/services/dispatcher/broker.d.ts.map +1 -1
- package/dist/services/dispatcher/broker.js +23 -11
- package/dist/services/dispatcher/broker.js.map +1 -1
- package/dist/services/dispatcher/lock.js +2 -2
- package/dist/services/dispatcher/lock.js.map +1 -1
- package/dist/services/dispatcher/poller.d.ts.map +1 -1
- package/dist/services/dispatcher/poller.js +24 -5
- package/dist/services/dispatcher/poller.js.map +1 -1
- package/dist/services/flight-recorder.service.d.ts +37 -0
- package/dist/services/flight-recorder.service.d.ts.map +1 -0
- package/dist/services/flight-recorder.service.js +210 -0
- package/dist/services/flight-recorder.service.js.map +1 -0
- package/dist/services/keeper.service.d.ts +9 -0
- package/dist/services/keeper.service.d.ts.map +1 -1
- package/dist/services/keeper.service.js +198 -39
- package/dist/services/keeper.service.js.map +1 -1
- package/dist/services/maintenance-signal.d.ts.map +1 -1
- package/dist/services/maintenance-signal.js +2 -3
- package/dist/services/maintenance-signal.js.map +1 -1
- package/dist/services/process.service.d.ts +76 -61
- package/dist/services/process.service.d.ts.map +1 -1
- package/dist/services/process.service.js +224 -608
- package/dist/services/process.service.js.map +1 -1
- package/dist/services/reconnect-snapshot.service.js +2 -2
- package/dist/services/reconnect-snapshot.service.js.map +1 -1
- package/dist/services/scheduler.d.ts +1 -0
- package/dist/services/scheduler.d.ts.map +1 -1
- package/dist/services/scheduler.js +69 -14
- package/dist/services/scheduler.js.map +1 -1
- package/dist/services/self-update.service.d.ts +113 -15
- package/dist/services/self-update.service.d.ts.map +1 -1
- package/dist/services/self-update.service.js +382 -109
- package/dist/services/self-update.service.js.map +1 -1
- package/dist/services/spawn-common.d.ts +49 -23
- package/dist/services/spawn-common.d.ts.map +1 -1
- package/dist/services/spawn-common.js +262 -47
- package/dist/services/spawn-common.js.map +1 -1
- package/dist/services/spawn-exit-watcher.d.ts +43 -0
- package/dist/services/spawn-exit-watcher.d.ts.map +1 -0
- package/dist/services/spawn-exit-watcher.js +127 -0
- package/dist/services/spawn-exit-watcher.js.map +1 -0
- package/dist/services/telegram.d.ts +3 -2
- package/dist/services/telegram.d.ts.map +1 -1
- package/dist/services/telegram.js +8 -5
- package/dist/services/telegram.js.map +1 -1
- package/dist/services/thread-health.service.d.ts +1 -1
- package/dist/services/thread-health.service.d.ts.map +1 -1
- package/dist/services/thread-health.service.js +11 -7
- package/dist/services/thread-health.service.js.map +1 -1
- package/dist/services/thread-lifecycle.service.js +2 -2
- package/dist/services/thread-lifecycle.service.js.map +1 -1
- package/dist/services/topic.service.d.ts +1 -1
- package/dist/services/topic.service.d.ts.map +1 -1
- package/dist/services/topic.service.js +3 -2
- package/dist/services/topic.service.js.map +1 -1
- package/dist/services/worker-cleanup.service.d.ts.map +1 -1
- package/dist/services/worker-cleanup.service.js +25 -99
- package/dist/services/worker-cleanup.service.js.map +1 -1
- package/dist/test-bootstrap.d.ts +39 -0
- package/dist/test-bootstrap.d.ts.map +1 -0
- package/dist/test-bootstrap.js +78 -0
- package/dist/test-bootstrap.js.map +1 -0
- package/dist/tools/defs/session-defs.js +1 -1
- package/dist/tools/defs/session-defs.js.map +1 -1
- package/dist/tools/defs/utility-defs.d.ts.map +1 -1
- package/dist/tools/defs/utility-defs.js +4 -3
- package/dist/tools/defs/utility-defs.js.map +1 -1
- package/dist/tools/delegate-tool.d.ts +1 -1
- package/dist/tools/delegate-tool.d.ts.map +1 -1
- package/dist/tools/delegate-tool.js +37 -6
- package/dist/tools/delegate-tool.js.map +1 -1
- package/dist/tools/session-tools.d.ts.map +1 -1
- package/dist/tools/session-tools.js +34 -20
- package/dist/tools/session-tools.js.map +1 -1
- package/dist/tools/utility-tools.js +1 -1
- package/dist/tools/utility-tools.js.map +1 -1
- package/dist/tools/wait/context.d.ts +13 -0
- package/dist/tools/wait/context.d.ts.map +1 -1
- package/dist/tools/wait/message-delivery.d.ts.map +1 -1
- package/dist/tools/wait/message-delivery.js +2 -3
- package/dist/tools/wait/message-delivery.js.map +1 -1
- package/dist/tools/wait/poll-loop.d.ts.map +1 -1
- package/dist/tools/wait/poll-loop.js +8 -1
- package/dist/tools/wait/poll-loop.js.map +1 -1
- package/package.json +2 -2
|
@@ -2,7 +2,7 @@ import { rotateAllDailySessions } from "./daily-session.js";
|
|
|
2
2
|
import { cleanupExpiredWorkers } from "./worker-cleanup.service.js";
|
|
3
3
|
import { errorMessage } from "../lib/utils.js";
|
|
4
4
|
import { getAllThreads } from "../data/memory/thread-registry.js";
|
|
5
|
-
import {
|
|
5
|
+
import { isThreadAlive, readPidFiles, spawnedThreads } from "./process.service.js";
|
|
6
6
|
import { decommissionWorker } from "./worker-cleanup.service.js";
|
|
7
7
|
import { hasRunToday } from "../data/daily-marker.js";
|
|
8
8
|
const DAILY_ROTATION_INTERVAL_MS = 5 * 60_000;
|
|
@@ -50,23 +50,33 @@ export class BackgroundJobRunner {
|
|
|
50
50
|
catch (err) {
|
|
51
51
|
this.deps.log.error(`Expired worker cleanup error: ${errorMessage(err)}`);
|
|
52
52
|
}
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
try {
|
|
58
|
-
await sweepOrphanDescendants();
|
|
59
|
-
}
|
|
60
|
-
catch (err) {
|
|
61
|
-
this.deps.log.error(`Orphan-descendant sweep error: ${errorMessage(err)}`);
|
|
62
|
-
}
|
|
53
|
+
// The old orphan-descendant sweep (powershell CIM snapshot + witnessed-lineage
|
|
54
|
+
// map) is gone with the rest of the Node enumeration machinery (plan §8
|
|
55
|
+
// Phase 2). Its replacement is job-membership reconciliation in the broker
|
|
56
|
+
// once Phase 3 spawns agents into named job objects.
|
|
63
57
|
}
|
|
64
58
|
async runDeadWorkerSweep() {
|
|
65
59
|
try {
|
|
66
60
|
const db = this.deps.getMemoryDb();
|
|
67
61
|
const workers = getAllThreads(db).filter((t) => t.type === "worker" && (t.status === "active" || t.status === "exited"));
|
|
62
|
+
const pidEntries = readPidFiles();
|
|
68
63
|
for (const w of workers) {
|
|
69
|
-
|
|
64
|
+
// THREAD-scoped liveness (mirror cleanupExpiredWorkers): a broker-spawned
|
|
65
|
+
// worker's launcher pid reads dead ~20s after spawn while its forked child
|
|
66
|
+
// lives on in the job. A pid-first isThreadRunning()/findAliveThread here
|
|
67
|
+
// would BOTH mis-declare that live worker dead AND splice it out of
|
|
68
|
+
// spawnedThreads — so the inMemory branch below would fall through to a full
|
|
69
|
+
// decommissionWorker (job-kill + topic delete + note/DB archive) of a running
|
|
70
|
+
// worker every sweep. Source the fallback pid + startedAt exactly where
|
|
71
|
+
// cleanupExpiredWorkers does, then ask the broker by threadId; the broker's
|
|
72
|
+
// job/registry verdict (or treat-alive-on-unreachable) gates the sweep, and
|
|
73
|
+
// isThreadAlive never mutates spawnedThreads so the inMemory branch stays
|
|
74
|
+
// correct for a genuinely-dead worker still tracked in memory.
|
|
75
|
+
const pidEntry = pidEntries.find((e) => e.threadId === w.threadId);
|
|
76
|
+
const inMemEntry = spawnedThreads.find((t) => t.threadId === w.threadId);
|
|
77
|
+
const fallbackPid = pidEntry?.pid ?? inMemEntry?.pid;
|
|
78
|
+
const startedAt = pidEntry?.startedAt ?? inMemEntry?.startedAt;
|
|
79
|
+
if (!(await isThreadAlive(w.threadId, fallbackPid, startedAt))) {
|
|
70
80
|
const inMemory = spawnedThreads.some((t) => t.threadId === w.threadId);
|
|
71
81
|
try {
|
|
72
82
|
if (inMemory) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"background-runner.js","sourceRoot":"","sources":["../../src/services/background-runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAEpE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"background-runner.js","sourceRoot":"","sources":["../../src/services/background-runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAEpE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,MAAM,0BAA0B,GAAG,CAAC,GAAG,MAAM,CAAC;AAC9C,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,0BAA0B,GAAG,CAAC,GAAG,MAAM,CAAC;AAc9C,MAAM,OAAO,mBAAmB;IAID;IAHrB,SAAS,GAAqC,EAAE,CAAC;IACjD,oBAAoB,GAAkB,IAAI,CAAC;IAEnD,YAA6B,IAA0B;QAA1B,SAAI,GAAJ,IAAI,CAAsB;IAAG,CAAC;IAE3D,KAAK;QACH,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QAEtC,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAE/B,IAAI,CAAC,SAAS,CAAC,IAAI,CACjB,WAAW,CAAC,GAAG,EAAE;YACf,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC/B,CAAC,EAAE,0BAA0B,CAAC,CAC/B,CAAC;QAEF,IAAI,CAAC,SAAS,CAAC,IAAI,CACjB,WAAW,CAAC,GAAG,EAAE;YACf,KAAK,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACpC,CAAC,EAAE,0BAA0B,CAAC,CAC/B,CAAC;QAEF,IAAI,CAAC,SAAS,CAAC,IAAI,CACjB,WAAW,CAAC,GAAG,EAAE;YACf,KAAK,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACjC,CAAC,EAAE,0BAA0B,CAAC,CAC/B,CAAC;IACJ,CAAC;IAED,IAAI;QACF,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,qBAAqB,CACxC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EACvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAClB,IAAI,CAAC,IAAI,CAAC,MAAM,EAChB,IAAI,CAAC,IAAI,CAAC,eAAe,CAC1B,CAAC;YACF,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,OAAO,4BAA4B,CAAC,CAAC;YAC7F,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC;QACD,+EAA+E;QAC/E,wEAAwE;QACxE,2EAA2E;QAC3E,qDAAqD;IACvD,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAC/E,CAAC;YACF,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC;YAClC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,0EAA0E;gBAC1E,2EAA2E;gBAC3E,0EAA0E;gBAC1E,oEAAoE;gBACpE,6EAA6E;gBAC7E,8EAA8E;gBAC9E,wEAAwE;gBACxE,4EAA4E;gBAC5E,4EAA4E;gBAC5E,0EAA0E;gBAC1E,+DAA+D;gBAC/D,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACnE,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACzE,MAAM,WAAW,GAAG,QAAQ,EAAE,GAAG,IAAI,UAAU,EAAE,GAAG,CAAC;gBACrD,MAAM,SAAS,GAAG,QAAQ,EAAE,SAAS,IAAI,UAAU,EAAE,SAAS,CAAC;gBAC/D,IAAI,CAAC,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC;oBAC/D,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;oBACvE,IAAI,CAAC;wBACH,IAAI,QAAQ,EAAE,CAAC;4BACb,iFAAiF;4BACjF,gFAAgF;4BAChF,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;wBAC1D,CAAC;6BAAM,CAAC;4BACN,oFAAoF;4BACpF,wEAAwE;4BACxE,MAAM,kBAAkB,CACtB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EACtC;gCACE,EAAE;gCACF,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;gCAC5B,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;gCACxB,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe;6BAC3C,CACF,CAAC;wBACJ,CAAC;wBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAChB,4CAA4C,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,GAAG,CACrE,CAAC;oBACJ,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAChB,yCAAyC,CAAC,CAAC,QAAQ,KAAK,YAAY,CAAC,GAAG,CAAC,EAAE,CAC5E,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,4BAA4B,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,2EAA2E;QAC3E,0EAA0E;QAC1E,4EAA4E;QAC5E,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,mBAAmB;YAAE,OAAO;QAEjD,oEAAoE;QACpE,yEAAyE;QACzE,2DAA2D;QAC3D,IAAI,WAAW,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,oBAAoB,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,IAAI,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACtF,OAAO;QACT,CAAC;QAED,MAAM,WAAW,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,IAAI,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9E,IAAI,IAAI,CAAC,oBAAoB,KAAK,WAAW;YAAE,OAAO;QACtD,IAAI,CAAC,oBAAoB,GAAG,WAAW,CAAC;QAExC,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,MAAM,sBAAsB,EAAE,CAAC;YAC/C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CACjB,kCAAkC,MAAM,CAAC,YAAY,KAAK,MAAM,CAAC,KAAK,EAAE,CACzE,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oCAAoC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thin HTTP client for the supervisor's process broker (plan §4.1, Phase 2).
|
|
3
|
+
*
|
|
4
|
+
* Liveness, identity and kill move OFF the Node event loop and OUT of child
|
|
5
|
+
* processes: the supervisor answers from one native NtQuerySystemInformation
|
|
6
|
+
* snapshot and kills via native TerminateProcess/TerminateJobObject. Nothing
|
|
7
|
+
* here ever spawns a process — the EDR-gated tasklist/powershell/taskkill
|
|
8
|
+
* machinery this replaces was itself the leading event-loop stall generator.
|
|
9
|
+
*
|
|
10
|
+
* Degradation contract (the audited-safe direction, same as the old breakers):
|
|
11
|
+
* every function returns `null` when the broker cannot answer (unreachable,
|
|
12
|
+
* auth-rejected, timeout) and the caller MUST treat null as "cannot confirm
|
|
13
|
+
* → assume alive / kill not performed". Communication is strictly one-way
|
|
14
|
+
* MCP → supervisor.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* How a failed broker call failed — the distinction the spawn contract pivots on:
|
|
18
|
+
* "no-connection" the TCP connection was never established (refused, host
|
|
19
|
+
* unreachable, connect timeout). The request CANNOT have
|
|
20
|
+
* reached the broker, so no process can have been created —
|
|
21
|
+
* a legacy local spawn is safe.
|
|
22
|
+
* "uncertain" the request may have been received and processed (read
|
|
23
|
+
* timeout, connection dropped mid-flight, unknown error).
|
|
24
|
+
* A process MAY exist broker-side — a blind local spawn
|
|
25
|
+
* could create a duplicate and is forbidden.
|
|
26
|
+
*/
|
|
27
|
+
export type BrokerFetchFailure = "no-connection" | "uncertain";
|
|
28
|
+
export interface BrokerAliveResult {
|
|
29
|
+
alive: boolean;
|
|
30
|
+
pidFound: boolean;
|
|
31
|
+
identityChecked: boolean;
|
|
32
|
+
identityMismatch: boolean;
|
|
33
|
+
startTimeMs?: number;
|
|
34
|
+
/** Exit code from the broker's ledger once the process is gone (spawn-side
|
|
35
|
+
* handle-wait recorded it). Undefined while alive / never managed. */
|
|
36
|
+
exitCode?: number | null;
|
|
37
|
+
/** Ledger state of the pid's latest row ("running"/"exited"/"killed"/…). */
|
|
38
|
+
registryState?: string | null;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Ask the broker whether `pid` is alive AND (when `startedAtMs` is given) still
|
|
42
|
+
* the same process we spawned (creation-time identity, ±60s tolerance — the
|
|
43
|
+
* same rule the deleted tasklist/Get-Process snapshots enforced).
|
|
44
|
+
* `startTimeTicks` (native creation FILETIME from a broker spawn) upgrades the
|
|
45
|
+
* check to exact identity. Returns null when the broker cannot answer.
|
|
46
|
+
*/
|
|
47
|
+
export declare function brokerCheckAlive(pid: number, startedAtMs?: number, startTimeTicks?: number): Promise<BrokerAliveResult | null>;
|
|
48
|
+
export interface BrokerThreadAliveResult {
|
|
49
|
+
/** False when the broker has never managed a process for this thread (HTTP 404). */
|
|
50
|
+
known: boolean;
|
|
51
|
+
/** Alive iff the thread's job has >=1 live member (kernel truth), or a live
|
|
52
|
+
* assign-failed process is still registered for it. */
|
|
53
|
+
alive: boolean;
|
|
54
|
+
/** How the verdict was reached ("job" | "registry"). */
|
|
55
|
+
source?: string;
|
|
56
|
+
/** Recorded exit code once the worker is truly gone (job drained). */
|
|
57
|
+
exitCode?: number | null;
|
|
58
|
+
/** Ledger state of the thread's latest row ("running"/"exited"/"killed"/…). */
|
|
59
|
+
registryState?: string | null;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Job-based thread liveness — the single source of truth for a broker-spawned
|
|
63
|
+
* worker. A launcher CLI (claude.exe/copilot.exe forks the real agent then exits)
|
|
64
|
+
* makes the originally-spawned pid a lie; the thread's JOB is not. Returns
|
|
65
|
+
* `known:false` when the broker has no record of the thread, and null when the
|
|
66
|
+
* broker cannot answer (unreachable → "cannot confirm → treat alive").
|
|
67
|
+
*/
|
|
68
|
+
export declare function brokerCheckAliveByThread(threadId: number): Promise<BrokerThreadAliveResult | null>;
|
|
69
|
+
export interface BrokerKillResult {
|
|
70
|
+
killed: boolean;
|
|
71
|
+
alive: boolean;
|
|
72
|
+
alreadyDead: boolean;
|
|
73
|
+
identityMismatch: boolean;
|
|
74
|
+
refused: boolean;
|
|
75
|
+
error?: string;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Ask the broker to kill the process tree rooted at `pid` (native, identity-
|
|
79
|
+
* checked when `startedAtMs` is given; the broker records the outcome in its
|
|
80
|
+
* on-disk registry). Returns null when the broker cannot answer — the caller
|
|
81
|
+
* must then fall back / re-verify, never assume the kill happened.
|
|
82
|
+
*
|
|
83
|
+
* Pass `pid` as undefined to route the kill by THREAD instead: the broker then
|
|
84
|
+
* takes the KillByThread path (job kill of every live member, or the registry-
|
|
85
|
+
* identity survivor when the job assign failed) rather than KillByPid. This is
|
|
86
|
+
* the correct call for a launcher-forked worker whose recorded pid is the
|
|
87
|
+
* already-exited launcher — KillByPid(launcher) hits the broker's AlreadyDead
|
|
88
|
+
* fast-path and never touches the live forked child.
|
|
89
|
+
*/
|
|
90
|
+
export declare function brokerKillTree(pid: number | undefined, opts?: {
|
|
91
|
+
threadId?: number;
|
|
92
|
+
killedBy?: string;
|
|
93
|
+
reason?: string;
|
|
94
|
+
startedAtMs?: number;
|
|
95
|
+
}): Promise<BrokerKillResult | null>;
|
|
96
|
+
export interface BrokerSpawnSpec {
|
|
97
|
+
threadId: number;
|
|
98
|
+
role: string;
|
|
99
|
+
exe: string;
|
|
100
|
+
args: string[];
|
|
101
|
+
env: Record<string, string>;
|
|
102
|
+
cwd?: string;
|
|
103
|
+
logFilePath: string;
|
|
104
|
+
/** Prompt-sized stdin payload for CLIs that read their prompt from stdin
|
|
105
|
+
* (Codex). Omitted → the child gets NUL stdin (Node's "ignore"). */
|
|
106
|
+
stdinData?: string;
|
|
107
|
+
}
|
|
108
|
+
export interface BrokerSpawnSuccess {
|
|
109
|
+
pid: number;
|
|
110
|
+
/** Native creation FILETIME — exact process identity for later checks/kills. */
|
|
111
|
+
startTimeTicks: number;
|
|
112
|
+
startTimeMs: number;
|
|
113
|
+
jobName: string | null;
|
|
114
|
+
/** Launched but job containment failed (Avecto downgrade §4.2) — managed via
|
|
115
|
+
* registry identity only. Already logged loudly broker-side. */
|
|
116
|
+
jobAssignFailed: boolean;
|
|
117
|
+
/** Idempotent hit: the thread ALREADY had a live managed process and pid
|
|
118
|
+
* describes that process — nothing new was spawned. */
|
|
119
|
+
existing: boolean;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* The four spawn outcomes the caller MUST keep apart:
|
|
123
|
+
* success → a process is running and registered (existing=true when it
|
|
124
|
+
* was already there — the idempotency contract);
|
|
125
|
+
* { error } → the broker answered and the spawn FAILED. No process was
|
|
126
|
+
* created (the broker's idempotency check also found none);
|
|
127
|
+
* { unavailable } → the request DEFINITIVELY never spawned anything: the
|
|
128
|
+
* connection was never established, or the broker answered
|
|
129
|
+
* 404/401 (endpoint absent on an old binary / bad secret).
|
|
130
|
+
* A legacy local spawn cannot duplicate anything;
|
|
131
|
+
* { uncertain } → timeout or connection lost mid-flight: the broker MAY
|
|
132
|
+
* have created a process. A blind local spawn could
|
|
133
|
+
* duplicate the agent — the caller must reconcile via
|
|
134
|
+
* brokerListThread / an idempotent spawn retry instead.
|
|
135
|
+
*/
|
|
136
|
+
export type BrokerSpawnResult = BrokerSpawnSuccess | {
|
|
137
|
+
error: string;
|
|
138
|
+
} | {
|
|
139
|
+
unavailable: string;
|
|
140
|
+
} | {
|
|
141
|
+
uncertain: string;
|
|
142
|
+
};
|
|
143
|
+
/**
|
|
144
|
+
* Ask the supervisor to spawn an agent process: CreateProcess(CREATE_SUSPENDED)
|
|
145
|
+
* → assign to the thread's named job → resume. The supervisor is the process
|
|
146
|
+
* owner; children (hooks, grandchildren) land in the job automatically.
|
|
147
|
+
* Spawn is idempotent by threadId server-side (create-or-return), which is what
|
|
148
|
+
* makes retrying after an uncertain outcome safe.
|
|
149
|
+
*/
|
|
150
|
+
export declare function brokerSpawn(spec: BrokerSpawnSpec): Promise<BrokerSpawnResult>;
|
|
151
|
+
export interface BrokerProcEntry {
|
|
152
|
+
pid: number;
|
|
153
|
+
startTimeTicks: number;
|
|
154
|
+
startTimeMs: number;
|
|
155
|
+
name: string;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* The broker's live process list for a thread (job members, falling back to the
|
|
159
|
+
* registry root's tree). This is the reconcile source of truth after an
|
|
160
|
+
* uncertain spawn: a live entry here means the thread HAS its process — adopt
|
|
161
|
+
* it, never spawn another. `known:false` → the broker answered and has no
|
|
162
|
+
* record of the thread. Null → broker could not answer.
|
|
163
|
+
*/
|
|
164
|
+
export declare function brokerListThread(threadId: number): Promise<{
|
|
165
|
+
known: boolean;
|
|
166
|
+
processes: BrokerProcEntry[];
|
|
167
|
+
} | null>;
|
|
168
|
+
/**
|
|
169
|
+
* Ask the broker to adopt an already-running process (spawned before the broker
|
|
170
|
+
* owned spawning, or during a broker outage) into the thread's job + registry.
|
|
171
|
+
* Idempotent server-side. Best-effort: null when the broker cannot answer.
|
|
172
|
+
*/
|
|
173
|
+
export declare function brokerAdopt(threadId: number, pid: number, startedAtMs?: number, role?: string): Promise<{
|
|
174
|
+
adopted: boolean;
|
|
175
|
+
alreadyManaged: boolean;
|
|
176
|
+
} | null>;
|
|
177
|
+
export type BrokerRestartVerdict = "accepted" | "in-progress";
|
|
178
|
+
/**
|
|
179
|
+
* Ask the supervisor — the single process-lifecycle owner — to tear down and
|
|
180
|
+
* respawn THIS server (self-update path). The supervisor answers 202 before
|
|
181
|
+
* doing anything: the requester is the process about to be killed, so it can
|
|
182
|
+
* never observe completion. 409 means a restart is already in progress, which
|
|
183
|
+
* delivers exactly what the caller wants — treat it as success.
|
|
184
|
+
*
|
|
185
|
+
* Returns null when the supervisor cannot answer (unreachable, or an old
|
|
186
|
+
* binary without /restart → 404/401). The caller decides between the flagged
|
|
187
|
+
* legacy self-spawn fallback and a loud abort — same contract as brokerSpawn.
|
|
188
|
+
*/
|
|
189
|
+
export declare function brokerRequestRestart(reason: string): Promise<BrokerRestartVerdict | null>;
|
|
190
|
+
export interface BrokerKeeperState {
|
|
191
|
+
threadId: number;
|
|
192
|
+
retryCount: number;
|
|
193
|
+
fastExitCount: number;
|
|
194
|
+
fastExitEscalation: number;
|
|
195
|
+
cooldownUntilMs: number | null;
|
|
196
|
+
survivedKillCount: number;
|
|
197
|
+
lastStartAtMs: number | null;
|
|
198
|
+
}
|
|
199
|
+
/** Load a thread's persisted keeper counters. Null → broker unreachable OR no
|
|
200
|
+
* state saved yet (both mean "start from defaults"). */
|
|
201
|
+
export declare function brokerGetKeeperState(threadId: number): Promise<BrokerKeeperState | null>;
|
|
202
|
+
/** Persist a thread's keeper counters (fire-and-forget from the keeper; a lost
|
|
203
|
+
* write degrades to today's in-memory behavior, already logged at WARN by the
|
|
204
|
+
* unreachable path). */
|
|
205
|
+
export declare function brokerSaveKeeperState(state: BrokerKeeperState): Promise<void>;
|
|
206
|
+
/** Test hook: reset the unreachable-cooldown state. */
|
|
207
|
+
export declare function resetBrokerClientState(): void;
|
|
208
|
+
export declare function brokerSpawnTimeoutForTest(ms: number | null): void;
|
|
209
|
+
//# sourceMappingURL=broker-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"broker-client.d.ts","sourceRoot":"","sources":["../../src/services/broker-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA2BH;;;;;;;;;;GAUG;AACH,MAAM,MAAM,kBAAkB,GAAG,eAAe,GAAG,WAAW,CAAC;AAmG/D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;2EACuE;IACvE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,WAAW,CAAC,EAAE,MAAM,EACpB,cAAc,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAYnC;AAED,MAAM,WAAW,uBAAuB;IACtC,oFAAoF;IACpF,KAAK,EAAE,OAAO,CAAC;IACf;4DACwD;IACxD,KAAK,EAAE,OAAO,CAAC;IACf,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,uBAAuB,GAAG,IAAI,CAAC,CAiBzC;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,GAAG,SAAS,EACvB,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACzF,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAyBlC;AAID,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB;yEACqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,gFAAgF;IAChF,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB;qEACiE;IACjE,eAAe,EAAE,OAAO,CAAC;IACzB;4DACwD;IACxD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,iBAAiB,GAC3B,kBAAkB,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3F;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAoDnF;AAID,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,eAAe,EAAE,CAAA;CAAE,GAAG,IAAI,CAAC,CAkBlE;AAID;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,EACX,WAAW,CAAC,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,OAAO,CAAA;CAAE,GAAG,IAAI,CAAC,CAsB/D;AAID,MAAM,MAAM,oBAAoB,GAAG,UAAU,GAAG,aAAa,CAAC;AAE9D;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAU/F;AAID,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;yDACyD;AACzD,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAS9F;AAED;;yBAEyB;AACzB,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAMnF;AAED,uDAAuD;AACvD,wBAAgB,sBAAsB,IAAI,IAAI,CAI7C;AAKD,wBAAgB,yBAAyB,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAEjE"}
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thin HTTP client for the supervisor's process broker (plan §4.1, Phase 2).
|
|
3
|
+
*
|
|
4
|
+
* Liveness, identity and kill move OFF the Node event loop and OUT of child
|
|
5
|
+
* processes: the supervisor answers from one native NtQuerySystemInformation
|
|
6
|
+
* snapshot and kills via native TerminateProcess/TerminateJobObject. Nothing
|
|
7
|
+
* here ever spawns a process — the EDR-gated tasklist/powershell/taskkill
|
|
8
|
+
* machinery this replaces was itself the leading event-loop stall generator.
|
|
9
|
+
*
|
|
10
|
+
* Degradation contract (the audited-safe direction, same as the old breakers):
|
|
11
|
+
* every function returns `null` when the broker cannot answer (unreachable,
|
|
12
|
+
* auth-rejected, timeout) and the caller MUST treat null as "cannot confirm
|
|
13
|
+
* → assume alive / kill not performed". Communication is strictly one-way
|
|
14
|
+
* MCP → supervisor.
|
|
15
|
+
*/
|
|
16
|
+
import { log } from "../lib/logger.js";
|
|
17
|
+
import { errorMessage } from "../lib/utils.js";
|
|
18
|
+
const REQUEST_TIMEOUT_MS = 3000;
|
|
19
|
+
// Timeouts here are pure LIVENESS backstops — correctness never depends on
|
|
20
|
+
// their values. A spawn whose request outlives the client timeout is resolved
|
|
21
|
+
// through the reconcile path (query the broker registry, adopt or retry the
|
|
22
|
+
// idempotent spawn — see launchAgent), never by a blind local spawn, so no
|
|
23
|
+
// timeout value can cause a duplicate agent. Spawns are sub-second on this
|
|
24
|
+
// machine; the generous budgets only keep a pathologically slow call from
|
|
25
|
+
// tying up its caller forever. Kill's TerminateJobObject settle loop alone can
|
|
26
|
+
// take ~2.2s server-side, hence the longer mutate budget. Reads keep 3s.
|
|
27
|
+
const SPAWN_TIMEOUT_MS = 30_000;
|
|
28
|
+
const MUTATE_TIMEOUT_MS = 10_000;
|
|
29
|
+
// After a failed broker call, skip further calls for a short window instead of
|
|
30
|
+
// re-paying a connect timeout for every pid in a sweep (the keeper checks many
|
|
31
|
+
// pids per cycle). Connection-refused fails in ~1 ms so this is only load-bearing
|
|
32
|
+
// when the supervisor itself is wedged.
|
|
33
|
+
const UNREACHABLE_COOLDOWN_MS = 5000;
|
|
34
|
+
let unreachableUntil = 0;
|
|
35
|
+
let unreachableLogged = false;
|
|
36
|
+
let lastFailureKind = "uncertain";
|
|
37
|
+
function classifyFetchError(err) {
|
|
38
|
+
const e = err;
|
|
39
|
+
// AbortSignal.timeout → the request was sent and the response never arrived.
|
|
40
|
+
if (e?.name === "TimeoutError" || e?.name === "AbortError")
|
|
41
|
+
return "uncertain";
|
|
42
|
+
const code = e?.cause?.code ?? e?.cause?.errors?.find((x) => x?.code)?.code ?? e?.code ?? undefined;
|
|
43
|
+
switch (code) {
|
|
44
|
+
case "ECONNREFUSED":
|
|
45
|
+
case "ENOTFOUND":
|
|
46
|
+
case "EAI_AGAIN":
|
|
47
|
+
case "EADDRNOTAVAIL":
|
|
48
|
+
case "EHOSTUNREACH":
|
|
49
|
+
case "ENETUNREACH":
|
|
50
|
+
case "UND_ERR_CONNECT_TIMEOUT":
|
|
51
|
+
return "no-connection";
|
|
52
|
+
}
|
|
53
|
+
// Some undici versions wrap the socket error in a bare Error with only a
|
|
54
|
+
// message ("connect ECONNREFUSED …", "bad port"). Fall back to the text for
|
|
55
|
+
// the same connection-never-established classes; anything unrecognized stays
|
|
56
|
+
// "uncertain" — the safe direction.
|
|
57
|
+
const msg = e?.cause?.message ?? "";
|
|
58
|
+
if (/ECONNREFUSED|ENOTFOUND|EHOSTUNREACH|ENETUNREACH|bad port|invalid url/i.test(msg))
|
|
59
|
+
return "no-connection";
|
|
60
|
+
return "uncertain";
|
|
61
|
+
}
|
|
62
|
+
function brokerBaseUrl() {
|
|
63
|
+
const port = Number.parseInt(process.env.SUPERVISOR_HTTP_PORT ?? "", 10) || 8848;
|
|
64
|
+
return `http://127.0.0.1:${port}`;
|
|
65
|
+
}
|
|
66
|
+
function authHeaders() {
|
|
67
|
+
const secret = process.env.MCP_HTTP_SECRET;
|
|
68
|
+
return secret ? { Authorization: `Bearer ${secret}` } : {};
|
|
69
|
+
}
|
|
70
|
+
async function brokerFetchClassified(path, init, timeoutMs = REQUEST_TIMEOUT_MS, opts) {
|
|
71
|
+
const now = Date.now();
|
|
72
|
+
// During the cooldown we answer from the LAST observed failure kind: the
|
|
73
|
+
// spawn contract needs "could the broker have received a request" and a
|
|
74
|
+
// skipped call inherits the evidence that started the cooldown. The cooldown
|
|
75
|
+
// exists to protect the keeper's many-pids-per-cycle sweeps from paying a
|
|
76
|
+
// full timeout each; spawn/reconcile calls are rare and correctness-critical,
|
|
77
|
+
// so they bypass it — a reconcile probe must never be answered from stale
|
|
78
|
+
// evidence created by the very spawn it is reconciling.
|
|
79
|
+
if (!opts?.bypassCooldown && now < unreachableUntil)
|
|
80
|
+
return { res: null, failure: lastFailureKind };
|
|
81
|
+
try {
|
|
82
|
+
const res = await fetch(`${brokerBaseUrl()}${path}`, {
|
|
83
|
+
...init,
|
|
84
|
+
headers: { ...authHeaders(), ...init?.headers },
|
|
85
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
86
|
+
});
|
|
87
|
+
unreachableLogged = false;
|
|
88
|
+
return { res };
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
const failure = classifyFetchError(err);
|
|
92
|
+
unreachableUntil = Date.now() + UNREACHABLE_COOLDOWN_MS;
|
|
93
|
+
lastFailureKind = failure;
|
|
94
|
+
if (!unreachableLogged) {
|
|
95
|
+
// WARN, deliberately: every liveness answer during this outage degrades to
|
|
96
|
+
// "cannot confirm → treat alive" and every kill falls back to best-effort —
|
|
97
|
+
// a silent fallback must never look like a healthy answer. Logged once per
|
|
98
|
+
// outage episode (reset on the first successful call) to avoid a warn storm
|
|
99
|
+
// when the keeper sweeps many pids per cycle.
|
|
100
|
+
log.warn(`[broker] unreachable (${errorMessage(err)}, ${failure}) — degrading to "cannot confirm": liveness treated as alive, kills fall back to native root-only`);
|
|
101
|
+
unreachableLogged = true;
|
|
102
|
+
}
|
|
103
|
+
return { res: null, failure };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function brokerFetch(path, init, timeoutMs = REQUEST_TIMEOUT_MS) {
|
|
107
|
+
return (await brokerFetchClassified(path, init, timeoutMs)).res;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Ask the broker whether `pid` is alive AND (when `startedAtMs` is given) still
|
|
111
|
+
* the same process we spawned (creation-time identity, ±60s tolerance — the
|
|
112
|
+
* same rule the deleted tasklist/Get-Process snapshots enforced).
|
|
113
|
+
* `startTimeTicks` (native creation FILETIME from a broker spawn) upgrades the
|
|
114
|
+
* check to exact identity. Returns null when the broker cannot answer.
|
|
115
|
+
*/
|
|
116
|
+
export async function brokerCheckAlive(pid, startedAtMs, startTimeTicks) {
|
|
117
|
+
const params = new URLSearchParams({ pid: String(pid) });
|
|
118
|
+
if (startTimeTicks !== undefined && startTimeTicks !== 0)
|
|
119
|
+
params.set("startTicks", String(startTimeTicks));
|
|
120
|
+
else if (startedAtMs !== undefined)
|
|
121
|
+
params.set("startMs", String(Math.round(startedAtMs)));
|
|
122
|
+
const res = await brokerFetch(`/proc/alive?${params}`);
|
|
123
|
+
if (!res || !res.ok)
|
|
124
|
+
return null;
|
|
125
|
+
try {
|
|
126
|
+
return (await res.json());
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Job-based thread liveness — the single source of truth for a broker-spawned
|
|
134
|
+
* worker. A launcher CLI (claude.exe/copilot.exe forks the real agent then exits)
|
|
135
|
+
* makes the originally-spawned pid a lie; the thread's JOB is not. Returns
|
|
136
|
+
* `known:false` when the broker has no record of the thread, and null when the
|
|
137
|
+
* broker cannot answer (unreachable → "cannot confirm → treat alive").
|
|
138
|
+
*/
|
|
139
|
+
export async function brokerCheckAliveByThread(threadId) {
|
|
140
|
+
const res = await brokerFetch(`/proc/alive?threadId=${threadId}`);
|
|
141
|
+
if (!res)
|
|
142
|
+
return null; // unreachable → cannot confirm
|
|
143
|
+
if (res.status === 404)
|
|
144
|
+
return { known: false, alive: false };
|
|
145
|
+
if (!res.ok)
|
|
146
|
+
return null;
|
|
147
|
+
try {
|
|
148
|
+
const b = (await res.json());
|
|
149
|
+
return {
|
|
150
|
+
known: true,
|
|
151
|
+
alive: b.alive,
|
|
152
|
+
source: b.source,
|
|
153
|
+
exitCode: b.exitCode,
|
|
154
|
+
registryState: b.registryState,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Ask the broker to kill the process tree rooted at `pid` (native, identity-
|
|
163
|
+
* checked when `startedAtMs` is given; the broker records the outcome in its
|
|
164
|
+
* on-disk registry). Returns null when the broker cannot answer — the caller
|
|
165
|
+
* must then fall back / re-verify, never assume the kill happened.
|
|
166
|
+
*
|
|
167
|
+
* Pass `pid` as undefined to route the kill by THREAD instead: the broker then
|
|
168
|
+
* takes the KillByThread path (job kill of every live member, or the registry-
|
|
169
|
+
* identity survivor when the job assign failed) rather than KillByPid. This is
|
|
170
|
+
* the correct call for a launcher-forked worker whose recorded pid is the
|
|
171
|
+
* already-exited launcher — KillByPid(launcher) hits the broker's AlreadyDead
|
|
172
|
+
* fast-path and never touches the live forked child.
|
|
173
|
+
*/
|
|
174
|
+
export async function brokerKillTree(pid, opts = {}) {
|
|
175
|
+
const res = await brokerFetch(`/proc/kill`, {
|
|
176
|
+
method: "POST",
|
|
177
|
+
headers: { "Content-Type": "application/json" },
|
|
178
|
+
body: JSON.stringify({
|
|
179
|
+
pid,
|
|
180
|
+
threadId: opts.threadId,
|
|
181
|
+
startMs: opts.startedAtMs !== undefined ? Math.round(opts.startedAtMs) : undefined,
|
|
182
|
+
killedBy: opts.killedBy ?? "mcp",
|
|
183
|
+
reason: opts.reason,
|
|
184
|
+
}),
|
|
185
|
+
}, MUTATE_TIMEOUT_MS);
|
|
186
|
+
if (!res)
|
|
187
|
+
return null;
|
|
188
|
+
// 403 = the broker REFUSED (self/server protection). That is an answer, not an
|
|
189
|
+
// outage: report it as not-killed-still-alive so callers never retry-storm.
|
|
190
|
+
if (!res.ok && res.status !== 403)
|
|
191
|
+
return null;
|
|
192
|
+
try {
|
|
193
|
+
return (await res.json());
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Ask the supervisor to spawn an agent process: CreateProcess(CREATE_SUSPENDED)
|
|
201
|
+
* → assign to the thread's named job → resume. The supervisor is the process
|
|
202
|
+
* owner; children (hooks, grandchildren) land in the job automatically.
|
|
203
|
+
* Spawn is idempotent by threadId server-side (create-or-return), which is what
|
|
204
|
+
* makes retrying after an uncertain outcome safe.
|
|
205
|
+
*/
|
|
206
|
+
export async function brokerSpawn(spec) {
|
|
207
|
+
const { res, failure } = await brokerFetchClassified(`/proc/spawn`, {
|
|
208
|
+
method: "POST",
|
|
209
|
+
headers: { "Content-Type": "application/json" },
|
|
210
|
+
body: JSON.stringify(spec),
|
|
211
|
+
}, spawnTimeoutOverride ?? SPAWN_TIMEOUT_MS, { bypassCooldown: true });
|
|
212
|
+
if (!res) {
|
|
213
|
+
return failure === "no-connection"
|
|
214
|
+
? { unavailable: "broker unreachable — connection never established" }
|
|
215
|
+
: { uncertain: "spawn request timed out or the connection was lost mid-flight" };
|
|
216
|
+
}
|
|
217
|
+
if (res.status === 404 || res.status === 401)
|
|
218
|
+
return {
|
|
219
|
+
unavailable: res.status === 404
|
|
220
|
+
? "/proc/spawn not available on this supervisor build"
|
|
221
|
+
: "broker rejected the bearer secret",
|
|
222
|
+
};
|
|
223
|
+
try {
|
|
224
|
+
const body = (await res.json());
|
|
225
|
+
if (res.ok && body.ok && typeof body.pid === "number" && body.pid > 0) {
|
|
226
|
+
return {
|
|
227
|
+
pid: body.pid,
|
|
228
|
+
startTimeTicks: body.startTimeTicks ?? 0,
|
|
229
|
+
startTimeMs: body.startTimeMs ?? Date.now(),
|
|
230
|
+
jobName: body.jobName ?? null,
|
|
231
|
+
jobAssignFailed: body.jobAssignFailed ?? false,
|
|
232
|
+
existing: body.existing ?? false,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
return { error: body.error ?? `broker answered HTTP ${res.status} without an error detail` };
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
// A 200 the broker sent but we could not read — the spawn may well have
|
|
239
|
+
// happened. Only a non-OK status is a definitive failure.
|
|
240
|
+
return res.ok
|
|
241
|
+
? { uncertain: `broker answered HTTP ${res.status} but the body was unreadable` }
|
|
242
|
+
: { error: `broker answered HTTP ${res.status} with an unparseable body` };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* The broker's live process list for a thread (job members, falling back to the
|
|
247
|
+
* registry root's tree). This is the reconcile source of truth after an
|
|
248
|
+
* uncertain spawn: a live entry here means the thread HAS its process — adopt
|
|
249
|
+
* it, never spawn another. `known:false` → the broker answered and has no
|
|
250
|
+
* record of the thread. Null → broker could not answer.
|
|
251
|
+
*/
|
|
252
|
+
export async function brokerListThread(threadId) {
|
|
253
|
+
const { res } = await brokerFetchClassified(`/proc/list?threadId=${threadId}`, undefined, REQUEST_TIMEOUT_MS, {
|
|
254
|
+
bypassCooldown: true,
|
|
255
|
+
});
|
|
256
|
+
if (!res)
|
|
257
|
+
return null;
|
|
258
|
+
if (res.status === 404)
|
|
259
|
+
return { known: false, processes: [] };
|
|
260
|
+
if (!res.ok)
|
|
261
|
+
return null;
|
|
262
|
+
try {
|
|
263
|
+
const body = (await res.json());
|
|
264
|
+
return { known: true, processes: body.processes ?? [] };
|
|
265
|
+
}
|
|
266
|
+
catch {
|
|
267
|
+
return null;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
// ── Adopt (safe-cutover migration) ──────────────────────────────────────────
|
|
271
|
+
/**
|
|
272
|
+
* Ask the broker to adopt an already-running process (spawned before the broker
|
|
273
|
+
* owned spawning, or during a broker outage) into the thread's job + registry.
|
|
274
|
+
* Idempotent server-side. Best-effort: null when the broker cannot answer.
|
|
275
|
+
*/
|
|
276
|
+
export async function brokerAdopt(threadId, pid, startedAtMs, role) {
|
|
277
|
+
const res = await brokerFetch(`/proc/adopt`, {
|
|
278
|
+
method: "POST",
|
|
279
|
+
headers: { "Content-Type": "application/json" },
|
|
280
|
+
body: JSON.stringify({
|
|
281
|
+
threadId,
|
|
282
|
+
pid,
|
|
283
|
+
startMs: startedAtMs !== undefined ? Math.round(startedAtMs) : undefined,
|
|
284
|
+
role,
|
|
285
|
+
}),
|
|
286
|
+
},
|
|
287
|
+
// Adopt walks and job-assigns a whole tree — give it the mutate budget.
|
|
288
|
+
MUTATE_TIMEOUT_MS);
|
|
289
|
+
if (!res || !res.ok)
|
|
290
|
+
return null;
|
|
291
|
+
try {
|
|
292
|
+
return (await res.json());
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return null;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Ask the supervisor — the single process-lifecycle owner — to tear down and
|
|
300
|
+
* respawn THIS server (self-update path). The supervisor answers 202 before
|
|
301
|
+
* doing anything: the requester is the process about to be killed, so it can
|
|
302
|
+
* never observe completion. 409 means a restart is already in progress, which
|
|
303
|
+
* delivers exactly what the caller wants — treat it as success.
|
|
304
|
+
*
|
|
305
|
+
* Returns null when the supervisor cannot answer (unreachable, or an old
|
|
306
|
+
* binary without /restart → 404/401). The caller decides between the flagged
|
|
307
|
+
* legacy self-spawn fallback and a loud abort — same contract as brokerSpawn.
|
|
308
|
+
*/
|
|
309
|
+
export async function brokerRequestRestart(reason) {
|
|
310
|
+
const res = await brokerFetch(`/restart`, {
|
|
311
|
+
method: "POST",
|
|
312
|
+
headers: { "Content-Type": "application/json" },
|
|
313
|
+
body: JSON.stringify({ reason }),
|
|
314
|
+
});
|
|
315
|
+
if (!res)
|
|
316
|
+
return null;
|
|
317
|
+
if (res.status === 202)
|
|
318
|
+
return "accepted";
|
|
319
|
+
if (res.status === 409)
|
|
320
|
+
return "in-progress";
|
|
321
|
+
return null; // 404 = old supervisor binary; 401 = misconfigured secret
|
|
322
|
+
}
|
|
323
|
+
/** Load a thread's persisted keeper counters. Null → broker unreachable OR no
|
|
324
|
+
* state saved yet (both mean "start from defaults"). */
|
|
325
|
+
export async function brokerGetKeeperState(threadId) {
|
|
326
|
+
const res = await brokerFetch(`/proc/keeper?threadId=${threadId}`);
|
|
327
|
+
if (!res || !res.ok)
|
|
328
|
+
return null;
|
|
329
|
+
try {
|
|
330
|
+
const body = (await res.json());
|
|
331
|
+
return body.found && body.state ? body.state : null;
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
return null;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
/** Persist a thread's keeper counters (fire-and-forget from the keeper; a lost
|
|
338
|
+
* write degrades to today's in-memory behavior, already logged at WARN by the
|
|
339
|
+
* unreachable path). */
|
|
340
|
+
export async function brokerSaveKeeperState(state) {
|
|
341
|
+
await brokerFetch(`/proc/keeper`, {
|
|
342
|
+
method: "POST",
|
|
343
|
+
headers: { "Content-Type": "application/json" },
|
|
344
|
+
body: JSON.stringify(state),
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
/** Test hook: reset the unreachable-cooldown state. */
|
|
348
|
+
export function resetBrokerClientState() {
|
|
349
|
+
unreachableUntil = 0;
|
|
350
|
+
unreachableLogged = false;
|
|
351
|
+
lastFailureKind = "uncertain";
|
|
352
|
+
}
|
|
353
|
+
/** Test hook: shrink the spawn liveness backstop so timeout-class tests don't
|
|
354
|
+
* wait 30s. Pass null to restore the default. */
|
|
355
|
+
let spawnTimeoutOverride = null;
|
|
356
|
+
export function brokerSpawnTimeoutForTest(ms) {
|
|
357
|
+
spawnTimeoutOverride = ms;
|
|
358
|
+
}
|
|
359
|
+
//# sourceMappingURL=broker-client.js.map
|