sensorium-mcp 3.1.0-refactor.15 → 3.1.0-refactor.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/config/agent-settings.d.ts +6 -0
  2. package/dist/config/agent-settings.d.ts.map +1 -1
  3. package/dist/config/agent-settings.js +22 -0
  4. package/dist/config/agent-settings.js.map +1 -1
  5. package/dist/config/detail-grant.d.ts +39 -0
  6. package/dist/config/detail-grant.d.ts.map +1 -0
  7. package/dist/config/detail-grant.js +61 -0
  8. package/dist/config/detail-grant.js.map +1 -0
  9. package/dist/config/index.d.ts +1 -0
  10. package/dist/config/index.d.ts.map +1 -1
  11. package/dist/config/index.js +1 -0
  12. package/dist/config/index.js.map +1 -1
  13. package/dist/config/misc-settings.d.ts +2 -0
  14. package/dist/config/misc-settings.d.ts.map +1 -1
  15. package/dist/config/misc-settings.js +22 -0
  16. package/dist/config/misc-settings.js.map +1 -1
  17. package/dist/dashboard/routes/copilot.d.ts.map +1 -1
  18. package/dist/dashboard/routes/copilot.js +17 -1
  19. package/dist/dashboard/routes/copilot.js.map +1 -1
  20. package/dist/dashboard/routes/settings.d.ts +2 -0
  21. package/dist/dashboard/routes/settings.d.ts.map +1 -1
  22. package/dist/dashboard/routes/settings.js +25 -1
  23. package/dist/dashboard/routes/settings.js.map +1 -1
  24. package/dist/dashboard/routes/threads-read.d.ts.map +1 -1
  25. package/dist/dashboard/routes/threads-read.js +9 -2
  26. package/dist/dashboard/routes/threads-read.js.map +1 -1
  27. package/dist/dashboard/routes/threads.js +3 -3
  28. package/dist/dashboard/routes/threads.js.map +1 -1
  29. package/dist/dashboard/routes.d.ts.map +1 -1
  30. package/dist/dashboard/routes.js +14 -5
  31. package/dist/dashboard/routes.js.map +1 -1
  32. package/dist/dashboard/spa.html +11 -11
  33. package/dist/index.js +48 -13
  34. package/dist/index.js.map +1 -1
  35. package/dist/lib/logger.d.ts.map +1 -1
  36. package/dist/lib/logger.js +21 -2
  37. package/dist/lib/logger.js.map +1 -1
  38. package/dist/server/factory.js +16 -6
  39. package/dist/server/factory.js.map +1 -1
  40. package/dist/server/http-server.d.ts.map +1 -1
  41. package/dist/server/http-server.js +10 -7
  42. package/dist/server/http-server.js.map +1 -1
  43. package/dist/services/agent-spawn.service.d.ts +3 -3
  44. package/dist/services/agent-spawn.service.d.ts.map +1 -1
  45. package/dist/services/agent-spawn.service.js +14 -8
  46. package/dist/services/agent-spawn.service.js.map +1 -1
  47. package/dist/services/background-runner.d.ts.map +1 -1
  48. package/dist/services/background-runner.js +6 -12
  49. package/dist/services/background-runner.js.map +1 -1
  50. package/dist/services/broker-client.d.ts +181 -0
  51. package/dist/services/broker-client.d.ts.map +1 -0
  52. package/dist/services/broker-client.js +323 -0
  53. package/dist/services/broker-client.js.map +1 -0
  54. package/dist/services/claude-spawn.service.d.ts +1 -1
  55. package/dist/services/claude-spawn.service.d.ts.map +1 -1
  56. package/dist/services/claude-spawn.service.js +25 -25
  57. package/dist/services/claude-spawn.service.js.map +1 -1
  58. package/dist/services/codex-spawn.service.d.ts +1 -1
  59. package/dist/services/codex-spawn.service.d.ts.map +1 -1
  60. package/dist/services/codex-spawn.service.js +19 -24
  61. package/dist/services/codex-spawn.service.js.map +1 -1
  62. package/dist/services/copilot-spawn.service.d.ts +1 -1
  63. package/dist/services/copilot-spawn.service.d.ts.map +1 -1
  64. package/dist/services/copilot-spawn.service.js +45 -36
  65. package/dist/services/copilot-spawn.service.js.map +1 -1
  66. package/dist/services/dispatcher/poller.d.ts.map +1 -1
  67. package/dist/services/dispatcher/poller.js +24 -5
  68. package/dist/services/dispatcher/poller.js.map +1 -1
  69. package/dist/services/flight-recorder.service.d.ts.map +1 -1
  70. package/dist/services/flight-recorder.service.js +41 -2
  71. package/dist/services/flight-recorder.service.js.map +1 -1
  72. package/dist/services/keeper.service.d.ts +9 -0
  73. package/dist/services/keeper.service.d.ts.map +1 -1
  74. package/dist/services/keeper.service.js +147 -31
  75. package/dist/services/keeper.service.js.map +1 -1
  76. package/dist/services/process.service.d.ts +40 -62
  77. package/dist/services/process.service.d.ts.map +1 -1
  78. package/dist/services/process.service.js +118 -613
  79. package/dist/services/process.service.js.map +1 -1
  80. package/dist/services/scheduler.d.ts +1 -0
  81. package/dist/services/scheduler.d.ts.map +1 -1
  82. package/dist/services/scheduler.js +58 -11
  83. package/dist/services/scheduler.js.map +1 -1
  84. package/dist/services/self-update.service.d.ts +14 -3
  85. package/dist/services/self-update.service.d.ts.map +1 -1
  86. package/dist/services/self-update.service.js +141 -72
  87. package/dist/services/self-update.service.js.map +1 -1
  88. package/dist/services/spawn-common.d.ts +49 -23
  89. package/dist/services/spawn-common.d.ts.map +1 -1
  90. package/dist/services/spawn-common.js +262 -47
  91. package/dist/services/spawn-common.js.map +1 -1
  92. package/dist/services/spawn-exit-watcher.d.ts +32 -0
  93. package/dist/services/spawn-exit-watcher.d.ts.map +1 -0
  94. package/dist/services/spawn-exit-watcher.js +102 -0
  95. package/dist/services/spawn-exit-watcher.js.map +1 -0
  96. package/dist/services/thread-health.service.d.ts +1 -1
  97. package/dist/services/thread-health.service.d.ts.map +1 -1
  98. package/dist/services/thread-health.service.js +6 -6
  99. package/dist/services/thread-health.service.js.map +1 -1
  100. package/dist/services/worker-cleanup.service.d.ts.map +1 -1
  101. package/dist/services/worker-cleanup.service.js +9 -92
  102. package/dist/services/worker-cleanup.service.js.map +1 -1
  103. package/dist/tools/defs/session-defs.js +1 -1
  104. package/dist/tools/defs/session-defs.js.map +1 -1
  105. package/dist/tools/defs/utility-defs.d.ts.map +1 -1
  106. package/dist/tools/defs/utility-defs.js +4 -3
  107. package/dist/tools/defs/utility-defs.js.map +1 -1
  108. package/dist/tools/delegate-tool.d.ts +1 -1
  109. package/dist/tools/delegate-tool.d.ts.map +1 -1
  110. package/dist/tools/delegate-tool.js +5 -5
  111. package/dist/tools/delegate-tool.js.map +1 -1
  112. package/dist/tools/session-tools.d.ts.map +1 -1
  113. package/dist/tools/session-tools.js +34 -20
  114. package/dist/tools/session-tools.js.map +1 -1
  115. package/dist/tools/utility-tools.js +1 -1
  116. package/dist/tools/utility-tools.js.map +1 -1
  117. package/dist/tools/wait/message-delivery.js +2 -2
  118. package/dist/tools/wait/message-delivery.js.map +1 -1
  119. package/package.json +1 -1
@@ -1,4 +1,3 @@
1
- import { execFile, spawn } from "node:child_process";
2
1
  import { mkdirSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
3
2
  import { homedir } from "node:os";
4
3
  import { join } from "node:path";
@@ -6,6 +5,7 @@ import { log } from "../lib/logger.js";
6
5
  import { errorMessage } from "../lib/utils.js";
7
6
  import { getAllThreads } from "../data/memory/thread-registry.js";
8
7
  import { getDashboardSessions } from "../sessions/dashboard-registry.js";
8
+ import { brokerAdopt, brokerCheckAlive, brokerKillTree } from "./broker-client.js";
9
9
  const BASE_DIR = join(homedir(), ".remote-copilot-mcp");
10
10
  const LOGS_DIR = join(BASE_DIR, "logs");
11
11
  const PIDS_DIR = join(BASE_DIR, "pids");
@@ -59,294 +59,39 @@ export function deletePendingTaskMailbox(threadId) {
59
59
  }
60
60
  }
61
61
  }
62
- /** @param spawnedAtMs when the caller knows the process's spawn time (pid-file
63
- * startedAt), pass it: absence from a pid-table snapshot taken BEFORE the
64
- * spawn proves nothing, so such a process is treated as alive until a fresher
65
- * snapshot can actually testify. Without it, a freshly spawned child could be
66
- * declared dead against a stale snapshot → duplicate spawn. */
67
- export function isProcessAlive(pid, spawnedAtMs) {
62
+ /**
63
+ * Liveness + identity in one question: is `pid` alive AND (when `spawnedAtMs`
64
+ * is given) still the process we spawned?
65
+ *
66
+ * Two-step check, neither of which ever spawns a child process:
67
+ * 1. `process.kill(pid, 0)` — a plain syscall. ESRCH (any non-EPERM error) is
68
+ * a definitive "dead" and needs no broker (this also keeps dev mode
69
+ * without a supervisor fully functional for the common case).
70
+ * 2. Otherwise the answer is NOT trustworthy on Windows — kill(0) succeeds
71
+ * for kernel-zombie PIDs held open by a lingering handle, and EPERM is
72
+ * what a parent sees for its own terminated child. Ask the supervisor's
73
+ * process broker, which answers from a native process snapshot and also
74
+ * verifies creation-time identity (PID-reuse guard).
75
+ *
76
+ * Broker unreachable → "cannot confirm → treat alive", exactly the audited-safe
77
+ * degradation the deleted tasklist/Get-Process breakers had.
78
+ */
79
+ export async function isProcessAlive(pid, spawnedAtMs) {
68
80
  try {
69
81
  process.kill(pid, 0);
70
82
  }
71
83
  catch (err) {
72
- // ESRCH (or any non-EPERM error) = no such process → dead. EPERM = kill(0)
73
- // was denied; do NOT trust that as alive — fall through to the win32 table
74
- // check below (see next comment for why). On POSIX, EPERM does mean a live
75
- // process owned by another user, so the fall-through returns true there.
76
84
  if (err.code !== "EPERM")
77
85
  return false;
78
86
  }
79
- // Reached when kill(0) either SUCCEEDED or threw EPERM. On Windows neither is
80
- // trustworthy on its own:
81
- // - kill(0) succeeds for a process that already TERMINATED but whose PID is
82
- // still held open by a lingering handle — a kernel "zombie".
83
- // - OpenProcess from the PARENT of such a dead child returns ACCESS_DENIED
84
- // (→EPERM) for that same terminated PID (a non-parent instead sees ESRCH).
85
- // Both cases wedged the keeper into treating an unkillable zombie as alive
86
- // forever, never restarting the thread. The process table excludes terminated
87
- // PIDs even while their handle lingers, so make it authoritative for both
88
- // paths. (tasklist unavailable → isPidInWindowsTable conservatively returns
89
- // true, so we never declare a genuinely live process dead.)
90
- if (process.platform === "win32")
91
- return isPidInWindowsTable(pid, spawnedAtMs);
92
- return true;
93
- }
94
- const WIN_PID_SNAPSHOT_TTL_MS = 2000;
95
- // A snapshot older than this is discarded rather than trusted: a very stale pid
96
- // set would mark freshly-spawned processes dead (they're absent from it), and a
97
- // stale start-time map is merely incomplete. Normal refreshes complete in well
98
- // under a second, so this bound is only reached when refreshes keep failing —
99
- // at which point the getters degrade to null (= "can't confirm" → alive).
100
- const WIN_SNAPSHOT_MAX_STALE_MS = 15_000;
101
- // Circuit breaker for the snapshot commands. When the OS throttles external
102
- // process spawns (battery / Energy Saver / EDR), tasklist and powershell hang
103
- // their full 5s timeout on EVERY call. The former synchronous snapshots then
104
- // froze the event loop for 5s at a time, so newly-spawned agent children could
105
- // not complete the MCP initialize/tools-list handshake and started TOOLLESS —
106
- // then exited cleanly and were respawned forever (the 2026-07-17 respawn storm,
107
- // see implementation_plans/2026-07-18_toolless-respawn-storm.md). Refreshes are
108
- // now async (never on the event loop) and, after repeated failures, paused
109
- // entirely for a cooldown instead of hammering a throttled OS every sweep.
110
- const SNAPSHOT_BREAKER_THRESHOLD = 3;
111
- const SNAPSHOT_BREAKER_COOLDOWN_MS = 60_000;
112
- function makeSnapshotBreaker(label) {
113
- let consecutiveFailures = 0;
114
- let openUntil = 0;
115
- return {
116
- isOpen: () => Date.now() < openUntil,
117
- recordSuccess: () => {
118
- consecutiveFailures = 0;
119
- },
120
- recordFailure: (err) => {
121
- consecutiveFailures++;
122
- if (consecutiveFailures >= SNAPSHOT_BREAKER_THRESHOLD) {
123
- openUntil = Date.now() + SNAPSHOT_BREAKER_COOLDOWN_MS;
124
- // Reset on open so re-opening after the cooldown requires THRESHOLD
125
- // fresh failures — one lingering failure must not instantly re-trip.
126
- consecutiveFailures = 0;
127
- log.warn(`[process] ${label} snapshot failed ${SNAPSHOT_BREAKER_THRESHOLD}x consecutively (${errorMessage(err)}) — pausing refreshes for ${SNAPSHOT_BREAKER_COOLDOWN_MS / 1000}s`);
128
- }
129
- else {
130
- log.debug(`[process] ${label} snapshot failed: ${errorMessage(err)}`);
131
- }
132
- },
133
- };
134
- }
135
- /** Authoritative Windows liveness check via the process table, which — unlike
136
- * OpenProcess/process.kill(0) — does NOT list a terminated process whose PID is
137
- * still held by a lingering handle. One `tasklist` snapshot of ALL pids serves
138
- * every pid in a sweep, and it is refreshed asynchronously in the background:
139
- * the getter reads the cached snapshot and NEVER blocks the event loop (a 5s
140
- * synchronous hang here is what starved MCP handshakes and caused the toolless
141
- * respawn storm). Callers treat null as "can't confirm dead" → alive. */
142
- let winPidSnapshot = null;
143
- let winPidRefreshInFlight = false;
144
- const pidSnapshotBreaker = makeSnapshotBreaker("tasklist");
145
- function refreshWindowsPidSet() {
146
- if (winPidRefreshInFlight || pidSnapshotBreaker.isOpen())
147
- return;
148
- winPidRefreshInFlight = true;
149
- // The snapshot's ts is the REFRESH START: the process table is read at (or
150
- // after) this instant, so any pid spawned later can legitimately be absent.
151
- const refreshStartedAt = Date.now();
152
- // /NH = no header, /FO CSV = quoted columns; PID is column 2: "image","PID",...
153
- execFile("tasklist", ["/NH", "/FO", "CSV"], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
154
- winPidRefreshInFlight = false;
155
- if (err) {
156
- pidSnapshotBreaker.recordFailure(err);
157
- return;
158
- }
159
- pidSnapshotBreaker.recordSuccess();
160
- const pids = new Set();
161
- for (const m of out.matchAll(/^"[^"]*","(\d+)"/gm))
162
- pids.add(Number(m[1]));
163
- winPidSnapshot = { pids, ts: refreshStartedAt };
164
- });
165
- }
166
- function getWindowsPidSet() {
167
- const now = Date.now();
168
- if (!winPidSnapshot || now - winPidSnapshot.ts >= WIN_PID_SNAPSHOT_TTL_MS)
169
- refreshWindowsPidSet();
170
- if (winPidSnapshot && now - winPidSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
171
- return winPidSnapshot.pids;
172
- return null; // no usable snapshot — conservative: never declare a real process dead
173
- }
174
- function isPidInWindowsTable(pid, spawnedAtMs) {
175
- const pids = getWindowsPidSet();
176
- if (pids === null)
177
- return true; // tasklist unavailable — never declare a real process dead
178
- if (pids.has(pid))
87
+ if (process.platform !== "win32") {
88
+ // POSIX: kill(0) success or EPERM both mean a live process.
179
89
  return true;
180
- // Absent from the table — but if the process was spawned AFTER the snapshot
181
- // was taken, the absence proves nothing (the table simply predates it).
182
- // Treat as alive; the refresh already triggered above will testify shortly.
183
- return spawnedAtMs !== undefined && spawnedAtMs >= winPidSnapshot.ts;
184
- }
185
- // Tolerance between the pid-file's recorded startedAt (Date.now() at spawn) and
186
- // the OS process creation time. Wide enough to absorb spawn latency and a wall-
187
- // clock step (NTP/VM resume) between the two readings, yet far below the minutes-
188
- // to-hours gap a recycled PID produces — so PID reuse is still caught.
189
- const PID_IDENTITY_TOLERANCE_MS = 60_000;
190
- const WIN_START_SNAPSHOT_TTL_MS = 2000;
191
- /** One powershell snapshot of every pid's creation time (Unix ms), reused for
192
- * all pids in a keeper sweep and refreshed asynchronously in the background —
193
- * same non-blocking pattern as the tasklist snapshot above. Staleness is safe
194
- * here: a pid's start time never changes, so an old map is merely missing new
195
- * pids, and a missing entry already means "can't tell" → trust PID. StartTime
196
- * is unreadable for some protected processes; those are simply omitted. */
197
- let winStartSnapshot = null;
198
- let winStartRefreshInFlight = false;
199
- const startSnapshotBreaker = makeSnapshotBreaker("start-time");
200
- function refreshWindowsStartTimeMap() {
201
- if (winStartRefreshInFlight || startSnapshotBreaker.isOpen())
202
- return;
203
- winStartRefreshInFlight = true;
204
- const refreshStartedAt = Date.now();
205
- execFile("powershell", [
206
- "-NoProfile",
207
- "-Command",
208
- 'Get-Process | ForEach-Object { try { "$($_.Id) $($_.StartTime.ToUniversalTime().Ticks)" } catch {} }',
209
- ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
210
- winStartRefreshInFlight = false;
211
- if (err) {
212
- startSnapshotBreaker.recordFailure(err);
213
- return;
214
- }
215
- startSnapshotBreaker.recordSuccess();
216
- const starts = new Map();
217
- for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
218
- starts.set(Number(m[1]), Math.round(Number(m[2]) / 10000 - 62135596800000)); // .NET ticks (100ns from year 1) → Unix ms
219
- }
220
- winStartSnapshot = { starts, ts: refreshStartedAt };
221
- });
222
- }
223
- function getWindowsStartTimeMap() {
224
- const now = Date.now();
225
- if (!winStartSnapshot || now - winStartSnapshot.ts >= WIN_START_SNAPSHOT_TTL_MS)
226
- refreshWindowsStartTimeMap();
227
- if (winStartSnapshot && now - winStartSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
228
- return winStartSnapshot.starts;
229
- return null; // no usable snapshot — conservative: caller trusts the PID
230
- }
231
- /** Windows process creation time as Unix ms, or undefined if unavailable.
232
- * Lets us tell OUR process apart from an unrelated one the OS assigned the
233
- * same (recycled) PID after ours exited. */
234
- function processStartMs(pid) {
235
- if (process.platform !== "win32")
236
- return undefined;
237
- return getWindowsStartTimeMap()?.get(pid);
238
- }
239
- /** True if the live PID is (still) the process we spawned, judged by start time.
240
- * Guards against PID reuse: a recycled PID now owned by a different process has
241
- * a start time far from what we recorded, so we stop treating it as ours (which
242
- * otherwise wedges the keeper into an endless "unkillable" kill loop). */
243
- function isSameProcess(pid, startedAt) {
244
- if (startedAt === undefined)
245
- return true; // nothing recorded to compare — trust PID
246
- // Our process was spawned AFTER the current start-time map was taken: any
247
- // entry the map holds for this pid describes a PREVIOUS owner of the (since
248
- // recycled) pid, not our child — comparing against it would wrongly reject
249
- // our own fresh process. Trust the PID until a fresher map exists.
250
- if (process.platform === "win32" &&
251
- winStartSnapshot !== null &&
252
- startedAt >= winStartSnapshot.ts) {
253
- return true;
254
- }
255
- const actual = processStartMs(pid);
256
- if (actual === undefined)
257
- return true; // can't read start time — keep prior behavior
258
- return Math.abs(actual - startedAt) < PID_IDENTITY_TOLERANCE_MS;
259
- }
260
- /** Descendants of rootPid (including root), ordered leaves-first (post-order),
261
- * from one process-table snapshot, delivered via callback (async so the kill
262
- * escalation path never blocks the event loop either). Fallback for when
263
- * `taskkill /T` fails to tear the whole tree down: a single wedged descendant
264
- * can make taskkill abort the tree, so killing each PID individually lets the
265
- * still-killable relatives (often including the root) go down anyway. */
266
- function getWindowsProcessTreeLeafFirst(rootPid, threadId, done) {
267
- execFile("powershell", [
268
- "-NoProfile",
269
- "-Command",
270
- 'Get-CimInstance Win32_Process | ForEach-Object { "$($_.ProcessId) $($_.ParentProcessId)" }',
271
- ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
272
- if (err) {
273
- // "Could not look" — distinct from "found nothing" below: the snapshot
274
- // itself failed (the corporate-agent CIM failure seen on this machine),
275
- // so we cannot enumerate the tree at all and fall back to the root pid.
276
- log.warn(`[process] leaf-first walk pid=${rootPid} thread=${threadId}: process-tree snapshot FAILED (${errorMessage(err)}) — could NOT enumerate the tree; falling back to root pid only`);
277
- done([rootPid]);
278
- return;
279
- }
280
- const children = new Map();
281
- for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
282
- const pid = Number(m[1]);
283
- const ppid = Number(m[2]);
284
- const list = children.get(ppid);
285
- if (list)
286
- list.push(pid);
287
- else
288
- children.set(ppid, [pid]);
289
- }
290
- const ordered = [];
291
- const seen = new Set();
292
- const visit = (pid) => {
293
- if (seen.has(pid))
294
- return; // guard against cycles / PID-reuse loops
295
- seen.add(pid);
296
- for (const child of children.get(pid) ?? [])
297
- visit(child);
298
- ordered.push(pid); // post-order → leaves first, root last
299
- };
300
- visit(rootPid);
301
- // "Found nothing" reads as "root + 0 descendant(s)" — clearly not the same
302
- // as the snapshot-failed warning above.
303
- log.info(`[process] leaf-first walk pid=${rootPid} thread=${threadId} enumerated ${ordered.length} process(es): root + ${ordered.length - 1} descendant(s)`);
304
- done(ordered);
305
- });
306
- }
307
- /** Force-kill each PID in order, one at a time, tolerating any that refuse (a
308
- * wedged leaf must not block the ones after it). Async so a sweep over a deep
309
- * tree never blocks the server event loop. */
310
- function killPidsSequentiallyWin(pids, done) {
311
- let i = 0;
312
- let killed = 0; // taskkill invocations that returned no error (reported killed)
313
- const next = () => {
314
- if (i >= pids.length) {
315
- done(killed);
316
- return;
317
- }
318
- const pid = pids[i++];
319
- execFile("taskkill", ["/F", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
320
- if (!err)
321
- killed++;
322
- next();
323
- });
324
- };
325
- next();
326
- }
327
- /** Last-resort escalation for EPERM-wedged trees: spawn taskkill as a DETACHED,
328
- * FRESH process so it does NOT inherit the server's wedged tree handles. When
329
- * the server's own process tree is job-object-constrained / I/O-wedged, an
330
- * in-process `taskkill /F /T` returns EPERM — but a sibling process spawned
331
- * with its own handles (detached, own stdio) killed the exact same zombie trees
332
- * instantly in this session's experiment. Nested securevault contexts (which
333
- * once made detached killers cross-context Access-Denied, commit 1b7ded6) are
334
- * gone — there is a single context now — so a detached killer in the server's
335
- * own context is safe. Fire-and-forget: unref'd so the server never waits. */
336
- function spawnDetachedKillerWin(pid) {
337
- try {
338
- const child = spawn("taskkill", ["/F", "/T", "/PID", String(pid)], {
339
- detached: true,
340
- windowsHide: true,
341
- stdio: "ignore",
342
- });
343
- child.on("error", (err) => log.warn(`[process] detached killer for PID ${pid} errored: ${errorMessage(err)}`));
344
- child.unref();
345
- log.warn(`[process] ESCALATION: spawned DETACHED fresh taskkill /F /T /PID ${pid} (bypasses server's EPERM-wedged tree)`);
346
- }
347
- catch (err) {
348
- log.warn(`[process] detached killer spawn for PID ${pid} failed: ${errorMessage(err)}`);
349
90
  }
91
+ const result = await brokerCheckAlive(pid, spawnedAtMs);
92
+ if (result === null)
93
+ return true; // cannot confirm → never declare a live process dead
94
+ return result.alive;
350
95
  }
351
96
  export function readPidFiles() {
352
97
  const entries = [];
@@ -381,40 +126,42 @@ export function readPidFiles() {
381
126
  }
382
127
  return entries;
383
128
  }
384
- export function findAliveThread(threadId) {
129
+ export async function findAliveThread(threadId) {
385
130
  for (let i = spawnedThreads.length - 1; i >= 0; i--) {
386
131
  const thread = spawnedThreads[i];
387
132
  if (thread.threadId !== threadId)
388
133
  continue;
389
- if (isProcessAlive(thread.pid, thread.startedAt) && isSameProcess(thread.pid, thread.startedAt))
134
+ if (await isProcessAlive(thread.pid, thread.startedAt))
390
135
  return thread;
391
136
  log.warn(`[findAliveThread] Thread ${threadId} PID ${thread.pid} stale in spawnedThreads - removing`);
392
137
  spawnedThreads.splice(i, 1);
393
138
  }
394
- const pidEntry = readPidFiles().find((entry) => entry.threadId === threadId &&
395
- isProcessAlive(entry.pid, entry.startedAt) &&
396
- isSameProcess(entry.pid, entry.startedAt));
397
- if (!pidEntry)
398
- return undefined;
399
- const restored = {
400
- pid: pidEntry.pid,
401
- threadId,
402
- name: pidEntry.name ?? `Thread ${threadId}`,
403
- startedAt: pidEntry.startedAt ?? Date.now(),
404
- createdAt: pidEntry.startedAt ?? Date.now(),
405
- logFile: "",
406
- ...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
407
- };
408
- spawnedThreads.push(restored);
409
- log.info(`[findAliveThread] Restored thread ${threadId} PID=${pidEntry.pid} from PID file`);
410
- return restored;
139
+ for (const entry of readPidFiles()) {
140
+ if (entry.threadId !== threadId)
141
+ continue;
142
+ if (!(await isProcessAlive(entry.pid, entry.startedAt)))
143
+ continue;
144
+ const restored = {
145
+ pid: entry.pid,
146
+ threadId,
147
+ name: entry.name ?? `Thread ${threadId}`,
148
+ startedAt: entry.startedAt ?? Date.now(),
149
+ createdAt: entry.startedAt ?? Date.now(),
150
+ logFile: "",
151
+ ...(entry.threadType ? { threadType: entry.threadType } : {}),
152
+ };
153
+ spawnedThreads.push(restored);
154
+ log.info(`[findAliveThread] Restored thread ${threadId} PID=${entry.pid} from PID file`);
155
+ return restored;
156
+ }
157
+ return undefined;
411
158
  }
412
- export const isThreadRunning = (threadId) => findAliveThread(threadId) !== undefined;
413
- export function getActiveThreadIds() {
159
+ export const isThreadRunning = async (threadId) => (await findAliveThread(threadId)) !== undefined;
160
+ export async function getActiveThreadIds() {
414
161
  // Workers are disposable and not eligible for reconnect — exclude them.
415
- const spawned = spawnedThreads
416
- .filter((t) => t.threadType !== "worker" && isProcessAlive(t.pid, t.startedAt))
417
- .map((t) => t.threadId);
162
+ const candidates = spawnedThreads.filter((t) => t.threadType !== "worker");
163
+ const aliveFlags = await Promise.all(candidates.map((t) => isProcessAlive(t.pid, t.startedAt)));
164
+ const spawned = candidates.filter((_, i) => aliveFlags[i]).map((t) => t.threadId);
418
165
  // Also include threads from active MCP sessions (root threads that aren't
419
166
  // in spawnedThreads because they connected externally, not spawned by us).
420
167
  const sessionThreadIds = getDashboardSessions()
@@ -464,14 +211,16 @@ export function pruneOldThreadLogs() {
464
211
  }
465
212
  /**
466
213
  * Check liveness of a thread directly via its PID file, bypassing the
467
- * spawnedThreads[] in-memory cache. Returns the live PID, or undefined
468
- * if no PID file exists or the process is dead (stale file is deleted).
214
+ * spawnedThreads[] in-memory cache. Returns the live PID together with its
215
+ * recorded spawn identity (startedAt) so callers can make identity-checked
216
+ * kill/alive calls, or undefined if no PID file exists or the process is dead
217
+ * (stale file is deleted).
469
218
  */
470
- export function findAliveThreadViaPidFile(threadId) {
219
+ export async function findAliveThreadViaPidFile(threadId) {
471
220
  const entry = readPidFiles().find((e) => e.threadId === threadId);
472
221
  if (!entry)
473
222
  return undefined;
474
- if (!isProcessAlive(entry.pid, entry.startedAt) || !isSameProcess(entry.pid, entry.startedAt)) {
223
+ if (!(await isProcessAlive(entry.pid, entry.startedAt))) {
475
224
  try {
476
225
  unlinkSync(entry.filePath);
477
226
  }
@@ -480,7 +229,7 @@ export function findAliveThreadViaPidFile(threadId) {
480
229
  }
481
230
  return undefined;
482
231
  }
483
- return entry.pid;
232
+ return { pid: entry.pid, startedAt: entry.startedAt };
484
233
  }
485
234
  /**
486
235
  * Reconcile in-memory state (spawnedThreads[]) with the two authoritative
@@ -490,7 +239,7 @@ export function findAliveThreadViaPidFile(threadId) {
490
239
  *
491
240
  * Replaces the old restoreFromPidFiles() + cleanupStalePidFiles() pair.
492
241
  */
493
- export function reconcileState(db, threadLifecycle) {
242
+ export async function reconcileState(db, threadLifecycle) {
494
243
  const pidEntries = readPidFiles();
495
244
  const dbThreadMap = new Map(getAllThreads(db)
496
245
  .filter((t) => ["active", "exited", "exiting", "spawning", "stuck"].includes(t.status))
@@ -503,7 +252,7 @@ export function reconcileState(db, threadLifecycle) {
503
252
  // If the process is still alive it's an orphan that must be reaped — never
504
253
  // just drop the PID file, or the live process becomes untrackable.
505
254
  // killProcessTree removes the PID file itself once the process is confirmed dead.
506
- if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
255
+ if (await isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
507
256
  log.warn(`[reconcileState] Live process for thread ${pidEntry.threadId} (PID=${pidEntry.pid}) has no active DB record — killing orphan`);
508
257
  void killProcessTree(pidEntry.pid, pidEntry.threadId, "reconcile-orphan", pidEntry.startedAt);
509
258
  }
@@ -518,7 +267,7 @@ export function reconcileState(db, threadLifecycle) {
518
267
  }
519
268
  continue;
520
269
  }
521
- if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
270
+ if (await isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
522
271
  // Alive: populate spawnedThreads[] if not already present
523
272
  if (!spawnedThreads.some((t) => t.threadId === pidEntry.threadId)) {
524
273
  spawnedThreads.push({
@@ -531,6 +280,9 @@ export function reconcileState(db, threadLifecycle) {
531
280
  ...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
532
281
  });
533
282
  log.info(`[reconcileState] Restored thread ${pidEntry.threadId} PID=${pidEntry.pid} from PID file`);
283
+ // Safe cutover (Phase 3): pull pre-broker processes into jobs+registry
284
+ // now rather than waiting for a supervisor restart. Idempotent.
285
+ void brokerAdopt(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt, pidEntry.threadType ?? "agent");
534
286
  }
535
287
  }
536
288
  else {
@@ -591,318 +343,71 @@ export function reconcileState(db, threadLifecycle) {
591
343
  log.warn(`[reconcileState] Failed to sweep orphan MCP configs: ${errorMessage(err)}`);
592
344
  }
593
345
  }
594
- // descendantPid → the tracked thread it was witnessed under (in-memory: an
595
- // orphan spanning a server restart is intentionally not tracked — see header).
596
- const orphanLineage = new Map();
597
- const orphanSnapshotBreaker = makeSnapshotBreaker("win32-process");
598
- /** One `Get-CimInstance Win32_Process` snapshot giving ppid AND creation time for
599
- * every pid — the only single call that yields both, which the sweep needs to
600
- * walk lineage and enforce the pid-reuse guards. Async (never blocks the event
601
- * loop) and behind the shared circuit breaker: on the corporate-agent failure
602
- * observed on this machine it resolves null, and the sweep degrades to a no-op. */
603
- function takeWindowsProcessSnapshot() {
604
- return new Promise((resolve) => {
605
- if (process.platform !== "win32" || orphanSnapshotBreaker.isOpen()) {
606
- resolve(null);
607
- return;
608
- }
609
- execFile("powershell", [
610
- "-NoProfile",
611
- "-Command",
612
- 'Get-CimInstance Win32_Process | ForEach-Object { try { "$($_.ProcessId) $($_.ParentProcessId) $($_.CreationDate.ToUniversalTime().Ticks)" } catch {} }',
613
- ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
614
- if (err) {
615
- orphanSnapshotBreaker.recordFailure(err);
616
- resolve(null);
617
- return;
618
- }
619
- orphanSnapshotBreaker.recordSuccess();
620
- const snap = new Map();
621
- for (const m of out.matchAll(/^(\d+) (\d+) (\d+)$/gm)) {
622
- snap.set(Number(m[1]), {
623
- ppid: Number(m[2]),
624
- startMs: Math.round(Number(m[3]) / 10000 - 62135596800000), // .NET ticks → Unix ms
625
- });
626
- }
627
- resolve(snap);
628
- });
629
- });
630
- }
631
- /**
632
- * Pure core of the orphan sweep — no IO, no clock, fully deterministic so the
633
- * pid-reuse guard and the snapshot-unavailable path can be tested directly.
634
- * Mutates `lineage` in place (records new descendants, drops dead/recycled ones).
635
- */
636
- export function computeOrphanSweep(params) {
637
- const { snapshot, trackedThreads, lineage, protectedPids, toleranceMs } = params;
638
- const result = {
639
- orphans: [],
640
- reusedPidsSkipped: [],
641
- olderThanParentSkipped: [],
642
- recorded: 0,
643
- };
644
- // Snapshot unavailable → never guess, never kill, never mutate lineage.
645
- if (snapshot === null)
646
- return result;
647
- const aliveOwners = new Set();
648
- for (const t of trackedThreads)
649
- if (t.alive)
650
- aliveOwners.add(t.threadId);
651
- // Children adjacency, built once from the snapshot.
652
- const children = new Map();
653
- for (const [pid, e] of snapshot) {
654
- const list = children.get(e.ppid);
655
- if (list)
656
- list.push(pid);
657
- else
658
- children.set(e.ppid, [pid]);
659
- }
660
- // Step A — witness lineage: record every descendant of each ALIVE tracked
661
- // thread while its subtree chain is still intact. Excludes the root pid itself
662
- // (that thread is managed by the worker-cleanup path, not here).
663
- for (const t of trackedThreads) {
664
- if (!t.alive || !snapshot.has(t.pid))
665
- continue;
666
- const seen = new Set([t.pid]);
667
- const stack = [...(children.get(t.pid) ?? [])];
668
- while (stack.length) {
669
- const d = stack.pop();
670
- if (seen.has(d))
671
- continue; // cycle / pid-reuse loop guard
672
- seen.add(d);
673
- const e = snapshot.get(d);
674
- if (!e)
675
- continue;
676
- lineage.set(d, {
677
- ownerThreadId: t.threadId,
678
- ownerPid: t.pid,
679
- ownerStartedAt: t.startedAt,
680
- descendantStartMs: e.startMs,
681
- });
682
- result.recorded++;
683
- for (const c of children.get(d) ?? [])
684
- if (!seen.has(c))
685
- stack.push(c);
686
- }
687
- }
688
- // Step B — evaluate: a recorded descendant whose owner is no longer alive is an
689
- // orphan candidate, subject to both pid-reuse guards.
690
- for (const [pid, entry] of [...lineage]) {
691
- // Prune a descendant that has left the process table FIRST, before the
692
- // owner-liveness check — otherwise a dead descendant of a still-live owner is
693
- // never evaluated here (its owner short-circuits below) and never re-recorded
694
- // by Step A (it is gone), so it lingers in `lineage` for the owner's entire
695
- // lifetime. A long-lived thread that churns many short-lived children (e.g. a
696
- // claude-agent spawning a bash.exe per hook) would grow the map without bound.
697
- const cur = snapshot.get(pid);
698
- if (!cur) {
699
- lineage.delete(pid); // descendant already gone — nothing to reap
700
- continue;
701
- }
702
- if (aliveOwners.has(entry.ownerThreadId))
703
- continue; // owner still alive → not orphaned
704
- // Never touch a pid managed elsewhere (a live tracked thread, the server, us).
705
- if (protectedPids.has(pid))
706
- continue;
707
- // GUARD 1 — same-process: a recycled pid now hosts a different process whose
708
- // creation time differs from the one we recorded. Drop it; do NOT kill.
709
- if (Math.abs(cur.startMs - entry.descendantStartMs) > toleranceMs) {
710
- lineage.delete(pid);
711
- result.reusedPidsSkipped.push(pid);
712
- continue;
713
- }
714
- // GUARD 2 — a child cannot predate its parent. If this pid started before the
715
- // owning thread did, it is not really our descendant (recorded owner is wrong
716
- // or the pid was reused across the boundary). Keep it for logging, never kill.
717
- if (entry.ownerStartedAt !== undefined && cur.startMs < entry.ownerStartedAt - toleranceMs) {
718
- result.olderThanParentSkipped.push(pid);
719
- continue;
720
- }
721
- result.orphans.push({ pid, ownerThreadId: entry.ownerThreadId, startMs: cur.startMs });
722
- }
723
- return result;
724
- }
725
- /** True when the sweep is allowed to actually kill confirmed orphans. Default is
726
- * log-only (see header): a week of real candidate logs comes first. */
727
- function orphanSweepWillKill() {
728
- const v = process.env.ORPHAN_SWEEP_KILL;
729
- return v === "1" || v === "true";
730
- }
731
346
  /**
732
- * Run one orphan-descendant sweep on the existing worker-cleanup cadence. Takes a
733
- * single process snapshot (no new timer, no second snapshot on the hot liveness
734
- * path), records lineage for live threads, and logs — or, when ORPHAN_SWEEP_KILL
735
- * is set, reaps — confirmed orphans. Best-effort and self-contained; any failure
736
- * degrades to doing nothing.
347
+ * Kill the process tree rooted at `pid` via the supervisor's broker (native
348
+ * TerminateProcess over the whole tree, identity-checked when `spawnedAtMs` is
349
+ * given, outcome recorded in the broker's on-disk registry). Never spawns a
350
+ * child process and never blocks the event loop.
351
+ *
352
+ * Broker unreachable → degraded best-effort: one native `process.kill(pid)`
353
+ * (root only — Node cannot enumerate a tree without shelling out, and that
354
+ * machinery is gone by design) followed by a conservative liveness re-check.
355
+ * A survived kill keeps the PID file so the keeper/cleanup policy loops retry
356
+ * and escalate — same contract as before.
737
357
  */
738
- export async function sweepOrphanDescendants() {
739
- const snapshot = await takeWindowsProcessSnapshot();
740
- if (snapshot === null) {
741
- log.debug("[orphan-sweep] process snapshot unavailable — skipping (never kill on partial data)");
742
- return { orphans: [], reusedPidsSkipped: [], olderThanParentSkipped: [], recorded: 0 };
358
+ export async function killProcessTree(pid, threadId, caller, spawnedAtMs) {
359
+ // Self-protection: NEVER tree-kill our own process or the authoritative
360
+ // server PID (a recycled worker PID could point at us — killing it would be
361
+ // the traceless server death this layer exists to prevent). The broker has
362
+ // the same guard server-side; this is defense in depth.
363
+ if (pid === process.pid) {
364
+ log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
365
+ return;
743
366
  }
744
- const trackedThreads = readPidFiles().map((e) => ({
745
- threadId: e.threadId,
746
- pid: e.pid,
747
- startedAt: e.startedAt,
748
- alive: isProcessAlive(e.pid, e.startedAt) && isSameProcess(e.pid, e.startedAt),
749
- }));
750
- // Protected: every live tracked-thread pid (managed by worker-cleanup), the
751
- // authoritative server pid, and our own pid — these are never orphan targets.
752
- const protectedPids = new Set([process.pid]);
753
367
  const serverPid = readServerPid();
754
- if (serverPid !== undefined)
755
- protectedPids.add(serverPid);
756
- for (const t of trackedThreads)
757
- if (t.alive)
758
- protectedPids.add(t.pid);
759
- const result = computeOrphanSweep({
760
- snapshot,
761
- trackedThreads,
762
- lineage: orphanLineage,
763
- protectedPids,
764
- toleranceMs: PID_IDENTITY_TOLERANCE_MS,
368
+ if (serverPid !== undefined && pid === serverPid) {
369
+ log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
370
+ return;
371
+ }
372
+ log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
373
+ const result = await brokerKillTree(pid, {
374
+ threadId,
375
+ killedBy: caller ?? "mcp",
376
+ reason: caller,
377
+ startedAtMs: spawnedAtMs,
765
378
  });
766
- for (const pid of result.reusedPidsSkipped)
767
- log.debug(`[orphan-sweep] pid ${pid} recycled since we recorded it — dropped, not killed`);
768
- for (const pid of result.olderThanParentSkipped)
769
- log.warn(`[orphan-sweep] pid ${pid} is older than its recorded owner — refusing to kill`);
770
- const willKill = orphanSweepWillKill();
771
- for (const orphan of result.orphans) {
772
- if (!willKill) {
773
- log.warn(`[orphan-sweep] CANDIDATE (log-only): pid ${orphan.pid} orphaned from dead thread ${orphan.ownerThreadId} — set ORPHAN_SWEEP_KILL=1 to reap`);
774
- continue;
775
- }
776
- log.warn(`[orphan-sweep] REAPING orphan pid ${orphan.pid} (owner thread ${orphan.ownerThreadId} is dead)`);
379
+ let stillAlive;
380
+ if (result !== null) {
381
+ if (result.refused)
382
+ log.error(`[process] broker REFUSED kill of pid ${pid}: ${result.error ?? "protected pid"}`);
383
+ else if (result.identityMismatch)
384
+ log.warn(`[process] broker: pid ${pid} was recycled (identity mismatch) — our process is already gone, nothing killed`);
385
+ stillAlive = result.alive;
386
+ }
387
+ else {
388
+ // Degraded path (no supervisor): best-effort native root kill, then re-check.
389
+ log.warn(`[process] broker unreachable for kill of pid ${pid} (thread ${threadId}) — degraded native root-only kill`);
777
390
  try {
778
- await killProcessTree(orphan.pid, orphan.ownerThreadId, "orphan-sweep", orphan.startMs);
391
+ process.kill(pid);
779
392
  }
780
393
  catch (err) {
781
- log.warn(`[orphan-sweep] kill of orphan pid ${orphan.pid} failed: ${errorMessage(err)}`);
782
- }
783
- finally {
784
- orphanLineage.delete(orphan.pid);
394
+ log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
785
395
  }
396
+ await new Promise((resolve) => setTimeout(resolve, 500));
397
+ stillAlive = await isProcessAlive(pid, spawnedAtMs);
786
398
  }
787
- return result;
788
- }
789
- // taskkill on a WEDGED process tree (e.g. a quota-stuck agent whose child
790
- // node/bash/git processes sit in blocking IO) can take well over 10s to force
791
- // the whole /T tree down. A short execFile timeout aborts taskkill mid-kill and
792
- // surfaces as "Command failed" with no stderr, so the process survives and the
793
- // keeper wrongly flags it UNKILLABLE. Give taskkill headroom to finish; the
794
- // caller still verifies liveness afterwards, so a fast kill returns immediately.
795
- const TASKKILL_TIMEOUT_MS = 60_000;
796
- export function killProcessTree(pid, threadId, caller, spawnedAtMs) {
797
- return new Promise((resolve) => {
798
- // Self-protection: NEVER tree-kill our own process or the authoritative
799
- // server PID. The leading hypothesis for the traceless server death is a
800
- // keeper/worker-reaper tree-killing a RECYCLED worker PID that the OS
801
- // reassigned to the node server (or one of its ancestors). `taskkill /T`
802
- // would then take the whole server tree down with no signal, no [fatal] and
803
- // no crash dump — exactly the observed symptom. Refuse and log loudly.
804
- if (pid === process.pid) {
805
- log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
806
- resolve();
807
- return;
808
- }
809
- const serverPid = readServerPid();
810
- if (serverPid !== undefined && pid === serverPid) {
811
- log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
812
- resolve();
813
- return;
814
- }
815
- log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
816
- const cleanupPidFile = () => {
817
- const pidFile = join(PROCESS_PIDS_DIR, `${threadId}.pid`);
818
- try {
819
- unlinkSync(pidFile);
820
- }
821
- catch {
822
- /* PID file already gone (exit handler may have raced us) */
823
- }
824
- };
825
- const verifyDead = (attempt) => {
826
- // spawnedAtMs matters here for orphan-kill callers reaping FRESH pids: a
827
- // failed kill must not let a pre-spawn snapshot falsely confirm death and
828
- // unlink a live process's pid file. A genuinely dead child is usually
829
- // caught by the kill(0) ESRCH probe or a post-spawn snapshot refresh; if
830
- // neither lands within the 3 attempts (refresh in flight / breaker open),
831
- // we resolve as "still alive" and KEEP the pid file — conservative, and a
832
- // later sweep finishes the cleanup against a fresher snapshot.
833
- if (!isProcessAlive(pid, spawnedAtMs)) {
834
- log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
835
- cleanupPidFile();
836
- resolve();
837
- return;
838
- }
839
- if (attempt >= 3) {
840
- log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after ${attempt} kill attempts — NOT removing PID file`);
841
- resolve();
842
- return;
843
- }
844
- // Retry kill after a short delay
845
- log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt ${attempt} — retrying`);
846
- if (process.platform === "win32") {
847
- // Escalation, retry-path only: the happy-path in-process `taskkill /F /T`
848
- // already failed to verify-dead. If the failure was EPERM because the
849
- // server's own tree is job-object-constrained / I/O-wedged, no in-process
850
- // kill will ever succeed — so first spawn a DETACHED, fresh sibling
851
- // taskkill that does not inherit our wedged handles.
852
- spawnDetachedKillerWin(pid);
853
- // Native TerminateProcess (via libuv) as well as taskkill — they take
854
- // different paths (taskkill is an external process with its own token;
855
- // process.kill issues the syscall directly from this process), so one
856
- // can succeed where the other fails on a stubborn process.
857
- try {
858
- process.kill(pid);
859
- }
860
- catch (err) {
861
- log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
862
- }
863
- // `taskkill /T` walks the tree top-down and can stall when a hung child
864
- // blocks the parent's termination — the exact case where the root sat
865
- // alive for hours. Kill each descendant leaves-first so a wedged child is
866
- // reaped before its parent, freeing the root to die. Logged at info (from
867
- // entry through kill count) so a week of logs can answer whether this
868
- // escalation path ever actually runs, or is dead weight.
869
- log.info(`[process] leaf-first kill ESCALATION entered for pid=${pid} thread=${threadId} (attempt ${attempt}) — taskkill /F /T did not verify-dead`);
870
- getWindowsProcessTreeLeafFirst(pid, threadId, (tree) => {
871
- killPidsSequentiallyWin(tree, (killed) => {
872
- log.info(`[process] leaf-first escalation for pid=${pid} thread=${threadId}: issued taskkill to ${tree.length} process(es), ${killed} reported killed`);
873
- setTimeout(() => verifyDead(attempt + 1), 1000);
874
- });
875
- });
876
- }
877
- else {
878
- try {
879
- process.kill(pid, "SIGKILL");
880
- }
881
- catch {
882
- /* process already gone (ESRCH) — verifyDead confirms */
883
- }
884
- setTimeout(() => verifyDead(attempt + 1), 1000);
885
- }
886
- };
887
- if (process.platform === "win32") {
888
- execFile("taskkill", ["/F", "/T", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
889
- if (err) {
890
- log.warn(`[process] Kill process ${pid} (thread ${threadId}) failed: ${errorMessage(err)}`);
891
- }
892
- // Always verify — taskkill can report success but process lingers,
893
- // or report failure but process is actually dead.
894
- setTimeout(() => verifyDead(1), 500);
895
- });
399
+ if (!stillAlive) {
400
+ log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
401
+ try {
402
+ unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
896
403
  }
897
- else {
898
- try {
899
- process.kill(pid, "SIGTERM");
900
- }
901
- catch (err) {
902
- log.debug(`[process] Kill process ${pid} (thread ${threadId}): ${errorMessage(err)}`);
903
- }
904
- setTimeout(() => verifyDead(1), 500);
404
+ catch {
405
+ /* PID file already gone (exit handler may have raced us) */
905
406
  }
906
- });
407
+ return;
408
+ }
409
+ // Conservative: a survived kill keeps the PID file so a later sweep retries
410
+ // against fresh evidence (keeper/worker-cleanup own the escalation policy).
411
+ log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt — NOT removing PID file`);
907
412
  }
908
413
  //# sourceMappingURL=process.service.js.map