sensorium-mcp 3.1.0-refactor.9 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config/agent-settings.d.ts +6 -0
  2. package/dist/config/agent-settings.d.ts.map +1 -1
  3. package/dist/config/agent-settings.js +22 -0
  4. package/dist/config/agent-settings.js.map +1 -1
  5. package/dist/config/detail-grant.d.ts +39 -0
  6. package/dist/config/detail-grant.d.ts.map +1 -0
  7. package/dist/config/detail-grant.js +61 -0
  8. package/dist/config/detail-grant.js.map +1 -0
  9. package/dist/config/index.d.ts +2 -0
  10. package/dist/config/index.d.ts.map +1 -1
  11. package/dist/config/index.js +5 -3
  12. package/dist/config/index.js.map +1 -1
  13. package/dist/config/misc-settings.d.ts +2 -0
  14. package/dist/config/misc-settings.d.ts.map +1 -1
  15. package/dist/config/misc-settings.js +22 -0
  16. package/dist/config/misc-settings.js.map +1 -1
  17. package/dist/config/settings-store.d.ts +41 -2
  18. package/dist/config/settings-store.d.ts.map +1 -1
  19. package/dist/config/settings-store.js +151 -14
  20. package/dist/config/settings-store.js.map +1 -1
  21. package/dist/dashboard/routes/copilot.d.ts.map +1 -1
  22. package/dist/dashboard/routes/copilot.js +17 -1
  23. package/dist/dashboard/routes/copilot.js.map +1 -1
  24. package/dist/dashboard/routes/settings.d.ts +2 -0
  25. package/dist/dashboard/routes/settings.d.ts.map +1 -1
  26. package/dist/dashboard/routes/settings.js +25 -1
  27. package/dist/dashboard/routes/settings.js.map +1 -1
  28. package/dist/dashboard/routes/threads-read.d.ts.map +1 -1
  29. package/dist/dashboard/routes/threads-read.js +15 -3
  30. package/dist/dashboard/routes/threads-read.js.map +1 -1
  31. package/dist/dashboard/routes/threads.d.ts.map +1 -1
  32. package/dist/dashboard/routes/threads.js +13 -4
  33. package/dist/dashboard/routes/threads.js.map +1 -1
  34. package/dist/dashboard/routes.d.ts.map +1 -1
  35. package/dist/dashboard/routes.js +14 -5
  36. package/dist/dashboard/routes.js.map +1 -1
  37. package/dist/dashboard/spa.html +11 -11
  38. package/dist/data/daily-marker.js +3 -3
  39. package/dist/data/daily-marker.js.map +1 -1
  40. package/dist/data/file-storage.d.ts.map +1 -1
  41. package/dist/data/file-storage.js +3 -3
  42. package/dist/data/file-storage.js.map +1 -1
  43. package/dist/data/memory/bootstrap.d.ts.map +1 -1
  44. package/dist/data/memory/bootstrap.js +2 -3
  45. package/dist/data/memory/bootstrap.js.map +1 -1
  46. package/dist/data/memory/migration-runner.d.ts.map +1 -1
  47. package/dist/data/memory/migration-runner.js +4 -7
  48. package/dist/data/memory/migration-runner.js.map +1 -1
  49. package/dist/data/memory/schema.js +2 -2
  50. package/dist/data/memory/schema.js.map +1 -1
  51. package/dist/index.js +67 -21
  52. package/dist/index.js.map +1 -1
  53. package/dist/lib/logger.d.ts.map +1 -1
  54. package/dist/lib/logger.js +21 -2
  55. package/dist/lib/logger.js.map +1 -1
  56. package/dist/lib/paths.d.ts +29 -0
  57. package/dist/lib/paths.d.ts.map +1 -0
  58. package/dist/lib/paths.js +36 -0
  59. package/dist/lib/paths.js.map +1 -0
  60. package/dist/lib/types.d.ts +1 -0
  61. package/dist/lib/types.d.ts.map +1 -1
  62. package/dist/server/factory.js +16 -6
  63. package/dist/server/factory.js.map +1 -1
  64. package/dist/server/http-server.d.ts.map +1 -1
  65. package/dist/server/http-server.js +52 -20
  66. package/dist/server/http-server.js.map +1 -1
  67. package/dist/services/agent-spawn.service.d.ts +3 -3
  68. package/dist/services/agent-spawn.service.d.ts.map +1 -1
  69. package/dist/services/agent-spawn.service.js +30 -9
  70. package/dist/services/agent-spawn.service.js.map +1 -1
  71. package/dist/services/background-runner.d.ts.map +1 -1
  72. package/dist/services/background-runner.js +22 -12
  73. package/dist/services/background-runner.js.map +1 -1
  74. package/dist/services/broker-client.d.ts +209 -0
  75. package/dist/services/broker-client.d.ts.map +1 -0
  76. package/dist/services/broker-client.js +359 -0
  77. package/dist/services/broker-client.js.map +1 -0
  78. package/dist/services/claude-spawn.service.d.ts +1 -1
  79. package/dist/services/claude-spawn.service.d.ts.map +1 -1
  80. package/dist/services/claude-spawn.service.js +25 -25
  81. package/dist/services/claude-spawn.service.js.map +1 -1
  82. package/dist/services/codex-spawn.service.d.ts +1 -1
  83. package/dist/services/codex-spawn.service.d.ts.map +1 -1
  84. package/dist/services/codex-spawn.service.js +19 -24
  85. package/dist/services/codex-spawn.service.js.map +1 -1
  86. package/dist/services/copilot-spawn.service.d.ts +1 -1
  87. package/dist/services/copilot-spawn.service.d.ts.map +1 -1
  88. package/dist/services/copilot-spawn.service.js +86 -37
  89. package/dist/services/copilot-spawn.service.js.map +1 -1
  90. package/dist/services/dispatcher/broker.d.ts +4 -3
  91. package/dist/services/dispatcher/broker.d.ts.map +1 -1
  92. package/dist/services/dispatcher/broker.js +23 -11
  93. package/dist/services/dispatcher/broker.js.map +1 -1
  94. package/dist/services/dispatcher/lock.js +2 -2
  95. package/dist/services/dispatcher/lock.js.map +1 -1
  96. package/dist/services/dispatcher/poller.d.ts.map +1 -1
  97. package/dist/services/dispatcher/poller.js +24 -5
  98. package/dist/services/dispatcher/poller.js.map +1 -1
  99. package/dist/services/flight-recorder.service.d.ts +37 -0
  100. package/dist/services/flight-recorder.service.d.ts.map +1 -0
  101. package/dist/services/flight-recorder.service.js +210 -0
  102. package/dist/services/flight-recorder.service.js.map +1 -0
  103. package/dist/services/keeper.service.d.ts +9 -0
  104. package/dist/services/keeper.service.d.ts.map +1 -1
  105. package/dist/services/keeper.service.js +198 -39
  106. package/dist/services/keeper.service.js.map +1 -1
  107. package/dist/services/maintenance-signal.d.ts.map +1 -1
  108. package/dist/services/maintenance-signal.js +2 -3
  109. package/dist/services/maintenance-signal.js.map +1 -1
  110. package/dist/services/process.service.d.ts +76 -61
  111. package/dist/services/process.service.d.ts.map +1 -1
  112. package/dist/services/process.service.js +224 -608
  113. package/dist/services/process.service.js.map +1 -1
  114. package/dist/services/reconnect-snapshot.service.js +2 -2
  115. package/dist/services/reconnect-snapshot.service.js.map +1 -1
  116. package/dist/services/scheduler.d.ts +1 -0
  117. package/dist/services/scheduler.d.ts.map +1 -1
  118. package/dist/services/scheduler.js +69 -14
  119. package/dist/services/scheduler.js.map +1 -1
  120. package/dist/services/self-update.service.d.ts +113 -15
  121. package/dist/services/self-update.service.d.ts.map +1 -1
  122. package/dist/services/self-update.service.js +382 -109
  123. package/dist/services/self-update.service.js.map +1 -1
  124. package/dist/services/spawn-common.d.ts +49 -23
  125. package/dist/services/spawn-common.d.ts.map +1 -1
  126. package/dist/services/spawn-common.js +262 -47
  127. package/dist/services/spawn-common.js.map +1 -1
  128. package/dist/services/spawn-exit-watcher.d.ts +43 -0
  129. package/dist/services/spawn-exit-watcher.d.ts.map +1 -0
  130. package/dist/services/spawn-exit-watcher.js +127 -0
  131. package/dist/services/spawn-exit-watcher.js.map +1 -0
  132. package/dist/services/telegram.d.ts +3 -2
  133. package/dist/services/telegram.d.ts.map +1 -1
  134. package/dist/services/telegram.js +8 -5
  135. package/dist/services/telegram.js.map +1 -1
  136. package/dist/services/thread-health.service.d.ts +1 -1
  137. package/dist/services/thread-health.service.d.ts.map +1 -1
  138. package/dist/services/thread-health.service.js +11 -7
  139. package/dist/services/thread-health.service.js.map +1 -1
  140. package/dist/services/thread-lifecycle.service.js +2 -2
  141. package/dist/services/thread-lifecycle.service.js.map +1 -1
  142. package/dist/services/topic.service.d.ts +1 -1
  143. package/dist/services/topic.service.d.ts.map +1 -1
  144. package/dist/services/topic.service.js +3 -2
  145. package/dist/services/topic.service.js.map +1 -1
  146. package/dist/services/worker-cleanup.service.d.ts.map +1 -1
  147. package/dist/services/worker-cleanup.service.js +25 -99
  148. package/dist/services/worker-cleanup.service.js.map +1 -1
  149. package/dist/test-bootstrap.d.ts +39 -0
  150. package/dist/test-bootstrap.d.ts.map +1 -0
  151. package/dist/test-bootstrap.js +78 -0
  152. package/dist/test-bootstrap.js.map +1 -0
  153. package/dist/tools/defs/session-defs.js +1 -1
  154. package/dist/tools/defs/session-defs.js.map +1 -1
  155. package/dist/tools/defs/utility-defs.d.ts.map +1 -1
  156. package/dist/tools/defs/utility-defs.js +4 -3
  157. package/dist/tools/defs/utility-defs.js.map +1 -1
  158. package/dist/tools/delegate-tool.d.ts +1 -1
  159. package/dist/tools/delegate-tool.d.ts.map +1 -1
  160. package/dist/tools/delegate-tool.js +37 -6
  161. package/dist/tools/delegate-tool.js.map +1 -1
  162. package/dist/tools/session-tools.d.ts.map +1 -1
  163. package/dist/tools/session-tools.js +34 -20
  164. package/dist/tools/session-tools.js.map +1 -1
  165. package/dist/tools/utility-tools.js +1 -1
  166. package/dist/tools/utility-tools.js.map +1 -1
  167. package/dist/tools/wait/context.d.ts +13 -0
  168. package/dist/tools/wait/context.d.ts.map +1 -1
  169. package/dist/tools/wait/message-delivery.d.ts.map +1 -1
  170. package/dist/tools/wait/message-delivery.js +2 -3
  171. package/dist/tools/wait/message-delivery.js.map +1 -1
  172. package/dist/tools/wait/poll-loop.d.ts.map +1 -1
  173. package/dist/tools/wait/poll-loop.js +8 -1
  174. package/dist/tools/wait/poll-loop.js.map +1 -1
  175. package/package.json +2 -2
@@ -1,12 +1,12 @@
1
- import { execFile, spawn } from "node:child_process";
2
1
  import { mkdirSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
3
- import { homedir } from "node:os";
4
2
  import { join } from "node:path";
5
3
  import { log } from "../lib/logger.js";
4
+ import { dataDir } from "../lib/paths.js";
6
5
  import { errorMessage } from "../lib/utils.js";
7
6
  import { getAllThreads } from "../data/memory/thread-registry.js";
8
7
  import { getDashboardSessions } from "../sessions/dashboard-registry.js";
9
- const BASE_DIR = join(homedir(), ".remote-copilot-mcp");
8
+ import { brokerAdopt, brokerCheckAlive, brokerCheckAliveByThread, brokerKillTree, } from "./broker-client.js";
9
+ const BASE_DIR = dataDir();
10
10
  const LOGS_DIR = join(BASE_DIR, "logs");
11
11
  const PIDS_DIR = join(BASE_DIR, "pids");
12
12
  const SERVER_PID_FILE = join(BASE_DIR, "server.pid");
@@ -59,294 +59,67 @@ export function deletePendingTaskMailbox(threadId) {
59
59
  }
60
60
  }
61
61
  }
62
- /** @param spawnedAtMs when the caller knows the process's spawn time (pid-file
63
- * startedAt), pass it: absence from a pid-table snapshot taken BEFORE the
64
- * spawn proves nothing, so such a process is treated as alive until a fresher
65
- * snapshot can actually testify. Without it, a freshly spawned child could be
66
- * declared dead against a stale snapshot → duplicate spawn. */
67
- export function isProcessAlive(pid, spawnedAtMs) {
62
+ /**
63
+ * Liveness + identity in one question: is `pid` alive AND (when `spawnedAtMs`
64
+ * is given) still the process we spawned?
65
+ *
66
+ * Two-step check, neither of which ever spawns a child process:
67
+ * 1. `process.kill(pid, 0)` — a plain syscall. ESRCH (any non-EPERM error) is
68
+ * a definitive "dead" and needs no broker (this also keeps dev mode
69
+ * without a supervisor fully functional for the common case).
70
+ * 2. Otherwise the answer is NOT trustworthy on Windows — kill(0) succeeds
71
+ * for kernel-zombie PIDs held open by a lingering handle, and EPERM is
72
+ * what a parent sees for its own terminated child. Ask the supervisor's
73
+ * process broker, which answers from a native process snapshot and also
74
+ * verifies creation-time identity (PID-reuse guard).
75
+ *
76
+ * Broker unreachable → "cannot confirm → treat alive", exactly the audited-safe
77
+ * degradation the deleted tasklist/Get-Process breakers had.
78
+ */
79
+ export async function isProcessAlive(pid, spawnedAtMs) {
68
80
  try {
69
81
  process.kill(pid, 0);
70
82
  }
71
83
  catch (err) {
72
- // ESRCH (or any non-EPERM error) = no such process → dead. EPERM = kill(0)
73
- // was denied; do NOT trust that as alive — fall through to the win32 table
74
- // check below (see next comment for why). On POSIX, EPERM does mean a live
75
- // process owned by another user, so the fall-through returns true there.
76
84
  if (err.code !== "EPERM")
77
85
  return false;
78
86
  }
79
- // Reached when kill(0) either SUCCEEDED or threw EPERM. On Windows neither is
80
- // trustworthy on its own:
81
- // - kill(0) succeeds for a process that already TERMINATED but whose PID is
82
- // still held open by a lingering handle — a kernel "zombie".
83
- // - OpenProcess from the PARENT of such a dead child returns ACCESS_DENIED
84
- // (→EPERM) for that same terminated PID (a non-parent instead sees ESRCH).
85
- // Both cases wedged the keeper into treating an unkillable zombie as alive
86
- // forever, never restarting the thread. The process table excludes terminated
87
- // PIDs even while their handle lingers, so make it authoritative for both
88
- // paths. (tasklist unavailable → isPidInWindowsTable conservatively returns
89
- // true, so we never declare a genuinely live process dead.)
90
- if (process.platform === "win32")
91
- return isPidInWindowsTable(pid, spawnedAtMs);
92
- return true;
93
- }
94
- const WIN_PID_SNAPSHOT_TTL_MS = 2000;
95
- // A snapshot older than this is discarded rather than trusted: a very stale pid
96
- // set would mark freshly-spawned processes dead (they're absent from it), and a
97
- // stale start-time map is merely incomplete. Normal refreshes complete in well
98
- // under a second, so this bound is only reached when refreshes keep failing —
99
- // at which point the getters degrade to null (= "can't confirm" → alive).
100
- const WIN_SNAPSHOT_MAX_STALE_MS = 15_000;
101
- // Circuit breaker for the snapshot commands. When the OS throttles external
102
- // process spawns (battery / Energy Saver / EDR), tasklist and powershell hang
103
- // their full 5s timeout on EVERY call. The former synchronous snapshots then
104
- // froze the event loop for 5s at a time, so newly-spawned agent children could
105
- // not complete the MCP initialize/tools-list handshake and started TOOLLESS —
106
- // then exited cleanly and were respawned forever (the 2026-07-17 respawn storm,
107
- // see implementation_plans/2026-07-18_toolless-respawn-storm.md). Refreshes are
108
- // now async (never on the event loop) and, after repeated failures, paused
109
- // entirely for a cooldown instead of hammering a throttled OS every sweep.
110
- const SNAPSHOT_BREAKER_THRESHOLD = 3;
111
- const SNAPSHOT_BREAKER_COOLDOWN_MS = 60_000;
112
- function makeSnapshotBreaker(label) {
113
- let consecutiveFailures = 0;
114
- let openUntil = 0;
115
- return {
116
- isOpen: () => Date.now() < openUntil,
117
- recordSuccess: () => {
118
- consecutiveFailures = 0;
119
- },
120
- recordFailure: (err) => {
121
- consecutiveFailures++;
122
- if (consecutiveFailures >= SNAPSHOT_BREAKER_THRESHOLD) {
123
- openUntil = Date.now() + SNAPSHOT_BREAKER_COOLDOWN_MS;
124
- // Reset on open so re-opening after the cooldown requires THRESHOLD
125
- // fresh failures — one lingering failure must not instantly re-trip.
126
- consecutiveFailures = 0;
127
- log.warn(`[process] ${label} snapshot failed ${SNAPSHOT_BREAKER_THRESHOLD}x consecutively (${errorMessage(err)}) — pausing refreshes for ${SNAPSHOT_BREAKER_COOLDOWN_MS / 1000}s`);
128
- }
129
- else {
130
- log.debug(`[process] ${label} snapshot failed: ${errorMessage(err)}`);
131
- }
132
- },
133
- };
134
- }
135
- /** Authoritative Windows liveness check via the process table, which — unlike
136
- * OpenProcess/process.kill(0) — does NOT list a terminated process whose PID is
137
- * still held by a lingering handle. One `tasklist` snapshot of ALL pids serves
138
- * every pid in a sweep, and it is refreshed asynchronously in the background:
139
- * the getter reads the cached snapshot and NEVER blocks the event loop (a 5s
140
- * synchronous hang here is what starved MCP handshakes and caused the toolless
141
- * respawn storm). Callers treat null as "can't confirm dead" → alive. */
142
- let winPidSnapshot = null;
143
- let winPidRefreshInFlight = false;
144
- const pidSnapshotBreaker = makeSnapshotBreaker("tasklist");
145
- function refreshWindowsPidSet() {
146
- if (winPidRefreshInFlight || pidSnapshotBreaker.isOpen())
147
- return;
148
- winPidRefreshInFlight = true;
149
- // The snapshot's ts is the REFRESH START: the process table is read at (or
150
- // after) this instant, so any pid spawned later can legitimately be absent.
151
- const refreshStartedAt = Date.now();
152
- // /NH = no header, /FO CSV = quoted columns; PID is column 2: "image","PID",...
153
- execFile("tasklist", ["/NH", "/FO", "CSV"], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
154
- winPidRefreshInFlight = false;
155
- if (err) {
156
- pidSnapshotBreaker.recordFailure(err);
157
- return;
158
- }
159
- pidSnapshotBreaker.recordSuccess();
160
- const pids = new Set();
161
- for (const m of out.matchAll(/^"[^"]*","(\d+)"/gm))
162
- pids.add(Number(m[1]));
163
- winPidSnapshot = { pids, ts: refreshStartedAt };
164
- });
165
- }
166
- function getWindowsPidSet() {
167
- const now = Date.now();
168
- if (!winPidSnapshot || now - winPidSnapshot.ts >= WIN_PID_SNAPSHOT_TTL_MS)
169
- refreshWindowsPidSet();
170
- if (winPidSnapshot && now - winPidSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
171
- return winPidSnapshot.pids;
172
- return null; // no usable snapshot — conservative: never declare a real process dead
173
- }
174
- function isPidInWindowsTable(pid, spawnedAtMs) {
175
- const pids = getWindowsPidSet();
176
- if (pids === null)
177
- return true; // tasklist unavailable — never declare a real process dead
178
- if (pids.has(pid))
179
- return true;
180
- // Absent from the table — but if the process was spawned AFTER the snapshot
181
- // was taken, the absence proves nothing (the table simply predates it).
182
- // Treat as alive; the refresh already triggered above will testify shortly.
183
- return spawnedAtMs !== undefined && spawnedAtMs >= winPidSnapshot.ts;
184
- }
185
- // Tolerance between the pid-file's recorded startedAt (Date.now() at spawn) and
186
- // the OS process creation time. Wide enough to absorb spawn latency and a wall-
187
- // clock step (NTP/VM resume) between the two readings, yet far below the minutes-
188
- // to-hours gap a recycled PID produces — so PID reuse is still caught.
189
- const PID_IDENTITY_TOLERANCE_MS = 60_000;
190
- const WIN_START_SNAPSHOT_TTL_MS = 2000;
191
- /** One powershell snapshot of every pid's creation time (Unix ms), reused for
192
- * all pids in a keeper sweep and refreshed asynchronously in the background —
193
- * same non-blocking pattern as the tasklist snapshot above. Staleness is safe
194
- * here: a pid's start time never changes, so an old map is merely missing new
195
- * pids, and a missing entry already means "can't tell" → trust PID. StartTime
196
- * is unreadable for some protected processes; those are simply omitted. */
197
- let winStartSnapshot = null;
198
- let winStartRefreshInFlight = false;
199
- const startSnapshotBreaker = makeSnapshotBreaker("start-time");
200
- function refreshWindowsStartTimeMap() {
201
- if (winStartRefreshInFlight || startSnapshotBreaker.isOpen())
202
- return;
203
- winStartRefreshInFlight = true;
204
- const refreshStartedAt = Date.now();
205
- execFile("powershell", [
206
- "-NoProfile",
207
- "-Command",
208
- 'Get-Process | ForEach-Object { try { "$($_.Id) $($_.StartTime.ToUniversalTime().Ticks)" } catch {} }',
209
- ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
210
- winStartRefreshInFlight = false;
211
- if (err) {
212
- startSnapshotBreaker.recordFailure(err);
213
- return;
214
- }
215
- startSnapshotBreaker.recordSuccess();
216
- const starts = new Map();
217
- for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
218
- starts.set(Number(m[1]), Math.round(Number(m[2]) / 10000 - 62135596800000)); // .NET ticks (100ns from year 1) → Unix ms
219
- }
220
- winStartSnapshot = { starts, ts: refreshStartedAt };
221
- });
222
- }
223
- function getWindowsStartTimeMap() {
224
- const now = Date.now();
225
- if (!winStartSnapshot || now - winStartSnapshot.ts >= WIN_START_SNAPSHOT_TTL_MS)
226
- refreshWindowsStartTimeMap();
227
- if (winStartSnapshot && now - winStartSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
228
- return winStartSnapshot.starts;
229
- return null; // no usable snapshot — conservative: caller trusts the PID
230
- }
231
- /** Windows process creation time as Unix ms, or undefined if unavailable.
232
- * Lets us tell OUR process apart from an unrelated one the OS assigned the
233
- * same (recycled) PID after ours exited. */
234
- function processStartMs(pid) {
235
- if (process.platform !== "win32")
236
- return undefined;
237
- return getWindowsStartTimeMap()?.get(pid);
238
- }
239
- /** True if the live PID is (still) the process we spawned, judged by start time.
240
- * Guards against PID reuse: a recycled PID now owned by a different process has
241
- * a start time far from what we recorded, so we stop treating it as ours (which
242
- * otherwise wedges the keeper into an endless "unkillable" kill loop). */
243
- function isSameProcess(pid, startedAt) {
244
- if (startedAt === undefined)
245
- return true; // nothing recorded to compare — trust PID
246
- // Our process was spawned AFTER the current start-time map was taken: any
247
- // entry the map holds for this pid describes a PREVIOUS owner of the (since
248
- // recycled) pid, not our child — comparing against it would wrongly reject
249
- // our own fresh process. Trust the PID until a fresher map exists.
250
- if (process.platform === "win32" &&
251
- winStartSnapshot !== null &&
252
- startedAt >= winStartSnapshot.ts) {
87
+ if (process.platform !== "win32") {
88
+ // POSIX: kill(0) success or EPERM both mean a live process.
253
89
  return true;
254
90
  }
255
- const actual = processStartMs(pid);
256
- if (actual === undefined)
257
- return true; // can't read start time — keep prior behavior
258
- return Math.abs(actual - startedAt) < PID_IDENTITY_TOLERANCE_MS;
259
- }
260
- /** Descendants of rootPid (including root), ordered leaves-first (post-order),
261
- * from one process-table snapshot, delivered via callback (async so the kill
262
- * escalation path never blocks the event loop either). Fallback for when
263
- * `taskkill /T` fails to tear the whole tree down: a single wedged descendant
264
- * can make taskkill abort the tree, so killing each PID individually lets the
265
- * still-killable relatives (often including the root) go down anyway. */
266
- function getWindowsProcessTreeLeafFirst(rootPid, threadId, done) {
267
- execFile("powershell", [
268
- "-NoProfile",
269
- "-Command",
270
- 'Get-CimInstance Win32_Process | ForEach-Object { "$($_.ProcessId) $($_.ParentProcessId)" }',
271
- ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
272
- if (err) {
273
- // "Could not look" — distinct from "found nothing" below: the snapshot
274
- // itself failed (the corporate-agent CIM failure seen on this machine),
275
- // so we cannot enumerate the tree at all and fall back to the root pid.
276
- log.warn(`[process] leaf-first walk pid=${rootPid} thread=${threadId}: process-tree snapshot FAILED (${errorMessage(err)}) — could NOT enumerate the tree; falling back to root pid only`);
277
- done([rootPid]);
278
- return;
279
- }
280
- const children = new Map();
281
- for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
282
- const pid = Number(m[1]);
283
- const ppid = Number(m[2]);
284
- const list = children.get(ppid);
285
- if (list)
286
- list.push(pid);
287
- else
288
- children.set(ppid, [pid]);
289
- }
290
- const ordered = [];
291
- const seen = new Set();
292
- const visit = (pid) => {
293
- if (seen.has(pid))
294
- return; // guard against cycles / PID-reuse loops
295
- seen.add(pid);
296
- for (const child of children.get(pid) ?? [])
297
- visit(child);
298
- ordered.push(pid); // post-order → leaves first, root last
299
- };
300
- visit(rootPid);
301
- // "Found nothing" reads as "root + 0 descendant(s)" — clearly not the same
302
- // as the snapshot-failed warning above.
303
- log.info(`[process] leaf-first walk pid=${rootPid} thread=${threadId} enumerated ${ordered.length} process(es): root + ${ordered.length - 1} descendant(s)`);
304
- done(ordered);
305
- });
306
- }
307
- /** Force-kill each PID in order, one at a time, tolerating any that refuse (a
308
- * wedged leaf must not block the ones after it). Async so a sweep over a deep
309
- * tree never blocks the server event loop. */
310
- function killPidsSequentiallyWin(pids, done) {
311
- let i = 0;
312
- let killed = 0; // taskkill invocations that returned no error (reported killed)
313
- const next = () => {
314
- if (i >= pids.length) {
315
- done(killed);
316
- return;
317
- }
318
- const pid = pids[i++];
319
- execFile("taskkill", ["/F", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
320
- if (!err)
321
- killed++;
322
- next();
323
- });
324
- };
325
- next();
91
+ const result = await brokerCheckAlive(pid, spawnedAtMs);
92
+ if (result === null)
93
+ return true; // cannot confirm → never declare a live process dead
94
+ return result.alive;
326
95
  }
327
- /** Last-resort escalation for EPERM-wedged trees: spawn taskkill as a DETACHED,
328
- * FRESH process so it does NOT inherit the server's wedged tree handles. When
329
- * the server's own process tree is job-object-constrained / I/O-wedged, an
330
- * in-process `taskkill /F /T` returns EPERM — but a sibling process spawned
331
- * with its own handles (detached, own stdio) killed the exact same zombie trees
332
- * instantly in this session's experiment. Nested securevault contexts (which
333
- * once made detached killers cross-context Access-Denied, commit 1b7ded6) are
334
- * gone — there is a single context now — so a detached killer in the server's
335
- * own context is safe. Fire-and-forget: unref'd so the server never waits. */
336
- function spawnDetachedKillerWin(pid) {
337
- try {
338
- const child = spawn("taskkill", ["/F", "/T", "/PID", String(pid)], {
339
- detached: true,
340
- windowsHide: true,
341
- stdio: "ignore",
342
- });
343
- child.on("error", (err) => log.warn(`[process] detached killer for PID ${pid} errored: ${errorMessage(err)}`));
344
- child.unref();
345
- log.warn(`[process] ESCALATION: spawned DETACHED fresh taskkill /F /T /PID ${pid} (bypasses server's EPERM-wedged tree)`);
346
- }
347
- catch (err) {
348
- log.warn(`[process] detached killer spawn for PID ${pid} failed: ${errorMessage(err)}`);
96
+ /**
97
+ * Thread-scoped liveness — the correct question for a broker-spawned worker.
98
+ *
99
+ * On Windows claude.exe/copilot.exe are LAUNCHERS: they fork the real agent into
100
+ * the thread's Job Object under a NEW pid, then exit. ~20s later the launcher pid
101
+ * we recorded at spawn is gone, so a pid-first `isProcessAlive(launcherPid)`
102
+ * returns false and orphans the still-working child. The thread's JOB is the
103
+ * source of truth; ask the broker by threadId:
104
+ * - null (unreachable) → cannot confirm → treat ALIVE (never kill on unknown),
105
+ * exactly the audited-safe degradation contract.
106
+ * - known:false (404) → the broker has NEVER managed this thread (dev mode /
107
+ * pre-broker process) → fall back to the pid check.
108
+ * - known:true → use the job/registry verdict (kernel truth).
109
+ *
110
+ * Off Windows there is no launcher-fork indirection (the pid IS the real child),
111
+ * so defer straight to the pid check.
112
+ */
113
+ export async function isThreadAlive(threadId, fallbackPid, spawnedAtMs) {
114
+ if (process.platform === "win32") {
115
+ const verdict = await brokerCheckAliveByThread(threadId);
116
+ if (verdict === null)
117
+ return true; // unreachable → never declare a live worker dead
118
+ if (verdict.known)
119
+ return verdict.alive; // job/registry truth
120
+ // known:false → not a broker-managed thread; fall through to the pid check.
349
121
  }
122
+ return fallbackPid !== undefined ? isProcessAlive(fallbackPid, spawnedAtMs) : false;
350
123
  }
351
124
  export function readPidFiles() {
352
125
  const entries = [];
@@ -381,40 +154,46 @@ export function readPidFiles() {
381
154
  }
382
155
  return entries;
383
156
  }
384
- export function findAliveThread(threadId) {
157
+ export async function findAliveThread(threadId) {
385
158
  for (let i = spawnedThreads.length - 1; i >= 0; i--) {
386
159
  const thread = spawnedThreads[i];
387
160
  if (thread.threadId !== threadId)
388
161
  continue;
389
- if (isProcessAlive(thread.pid, thread.startedAt) && isSameProcess(thread.pid, thread.startedAt))
162
+ if (await isProcessAlive(thread.pid, thread.startedAt))
390
163
  return thread;
391
164
  log.warn(`[findAliveThread] Thread ${threadId} PID ${thread.pid} stale in spawnedThreads - removing`);
392
165
  spawnedThreads.splice(i, 1);
393
166
  }
394
- const pidEntry = readPidFiles().find((entry) => entry.threadId === threadId &&
395
- isProcessAlive(entry.pid, entry.startedAt) &&
396
- isSameProcess(entry.pid, entry.startedAt));
397
- if (!pidEntry)
398
- return undefined;
399
- const restored = {
400
- pid: pidEntry.pid,
401
- threadId,
402
- name: pidEntry.name ?? `Thread ${threadId}`,
403
- startedAt: pidEntry.startedAt ?? Date.now(),
404
- createdAt: pidEntry.startedAt ?? Date.now(),
405
- logFile: "",
406
- ...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
407
- };
408
- spawnedThreads.push(restored);
409
- log.info(`[findAliveThread] Restored thread ${threadId} PID=${pidEntry.pid} from PID file`);
410
- return restored;
167
+ for (const entry of readPidFiles()) {
168
+ if (entry.threadId !== threadId)
169
+ continue;
170
+ if (!(await isProcessAlive(entry.pid, entry.startedAt)))
171
+ continue;
172
+ const restored = {
173
+ pid: entry.pid,
174
+ threadId,
175
+ name: entry.name ?? `Thread ${threadId}`,
176
+ startedAt: entry.startedAt ?? Date.now(),
177
+ createdAt: entry.startedAt ?? Date.now(),
178
+ logFile: "",
179
+ ...(entry.threadType ? { threadType: entry.threadType } : {}),
180
+ };
181
+ spawnedThreads.push(restored);
182
+ log.info(`[findAliveThread] Restored thread ${threadId} PID=${entry.pid} from PID file`);
183
+ return restored;
184
+ }
185
+ return undefined;
411
186
  }
412
- export const isThreadRunning = (threadId) => findAliveThread(threadId) !== undefined;
413
- export function getActiveThreadIds() {
187
+ export async function getActiveThreadIds() {
414
188
  // Workers are disposable and not eligible for reconnect — exclude them.
415
- const spawned = spawnedThreads
416
- .filter((t) => t.threadType !== "worker" && isProcessAlive(t.pid, t.startedAt))
417
- .map((t) => t.threadId);
189
+ const candidates = spawnedThreads.filter((t) => t.threadType !== "worker");
190
+ // THREAD-scoped liveness: a live broker non-worker thread's recorded pid is a
191
+ // launcher that exits ~20s post-spawn while its forked child lives on in the
192
+ // job, so a pid-first isProcessAlive(t.pid) under-reports it as dead and drops
193
+ // it from the reconnect/adopt set. Already async with every caller awaiting, so
194
+ // this is a drop-in correctness fix with no sync→async blast radius.
195
+ const aliveFlags = await Promise.all(candidates.map((t) => isThreadAlive(t.threadId, t.pid, t.startedAt)));
196
+ const spawned = candidates.filter((_, i) => aliveFlags[i]).map((t) => t.threadId);
418
197
  // Also include threads from active MCP sessions (root threads that aren't
419
198
  // in spawnedThreads because they connected externally, not spawned by us).
420
199
  const sessionThreadIds = getDashboardSessions()
@@ -464,14 +243,23 @@ export function pruneOldThreadLogs() {
464
243
  }
465
244
  /**
466
245
  * Check liveness of a thread directly via its PID file, bypassing the
467
- * spawnedThreads[] in-memory cache. Returns the live PID, or undefined
468
- * if no PID file exists or the process is dead (stale file is deleted).
246
+ * spawnedThreads[] in-memory cache. Returns the live PID together with its
247
+ * recorded spawn identity (startedAt) so callers can make identity-checked
248
+ * kill/alive calls, or undefined if no PID file exists or the process is dead
249
+ * (stale file is deleted).
469
250
  */
470
- export function findAliveThreadViaPidFile(threadId) {
251
+ export async function findAliveThreadViaPidFile(threadId) {
471
252
  const entry = readPidFiles().find((e) => e.threadId === threadId);
472
253
  if (!entry)
473
254
  return undefined;
474
- if (!isProcessAlive(entry.pid, entry.startedAt) || !isSameProcess(entry.pid, entry.startedAt)) {
255
+ // THREAD-scoped liveness before unlinking. The recorded pid is a launcher that
256
+ // exits ~20s after spawn while its forked child lives on in the job, so a
257
+ // pid-first isProcessAlive() reads dead for a LIVE agent and would delete its pid
258
+ // file — destroying the reconcile anchor a post-self-update adopt/restore needs.
259
+ // Ask the broker by threadId: unreachable → treat-alive (never unlink on unknown),
260
+ // known:false → fall back to the pid check (dev mode / pre-broker), known:true →
261
+ // job/registry truth. Only a genuinely-gone thread reaches the unlink.
262
+ if (!(await isThreadAlive(threadId, entry.pid, entry.startedAt))) {
475
263
  try {
476
264
  unlinkSync(entry.filePath);
477
265
  }
@@ -480,7 +268,7 @@ export function findAliveThreadViaPidFile(threadId) {
480
268
  }
481
269
  return undefined;
482
270
  }
483
- return entry.pid;
271
+ return { pid: entry.pid, startedAt: entry.startedAt };
484
272
  }
485
273
  /**
486
274
  * Reconcile in-memory state (spawnedThreads[]) with the two authoritative
@@ -490,7 +278,7 @@ export function findAliveThreadViaPidFile(threadId) {
490
278
  *
491
279
  * Replaces the old restoreFromPidFiles() + cleanupStalePidFiles() pair.
492
280
  */
493
- export function reconcileState(db, threadLifecycle) {
281
+ export async function reconcileState(db, threadLifecycle) {
494
282
  const pidEntries = readPidFiles();
495
283
  const dbThreadMap = new Map(getAllThreads(db)
496
284
  .filter((t) => ["active", "exited", "exiting", "spawning", "stuck"].includes(t.status))
@@ -502,10 +290,13 @@ export function reconcileState(db, threadLifecycle) {
502
290
  // PID file references a thread with no active DB record (archived/removed).
503
291
  // If the process is still alive it's an orphan that must be reaped — never
504
292
  // just drop the PID file, or the live process becomes untrackable.
505
- // killProcessTree removes the PID file itself once the process is confirmed dead.
506
- if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
293
+ // killThreadTree removes the PID file itself once the process is confirmed dead.
294
+ // Liveness/kill are THREAD-scoped: a broker-spawned worker's launcher pid is
295
+ // dead here even while its forked child lives, so a pid-first check/kill would
296
+ // both miss the live orphan and (via KillByPid AlreadyDead) fail to reap it.
297
+ if (await isThreadAlive(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt)) {
507
298
  log.warn(`[reconcileState] Live process for thread ${pidEntry.threadId} (PID=${pidEntry.pid}) has no active DB record — killing orphan`);
508
- void killProcessTree(pidEntry.pid, pidEntry.threadId, "reconcile-orphan", pidEntry.startedAt);
299
+ void killThreadTree(pidEntry.threadId, "reconcile-orphan", pidEntry.pid, pidEntry.startedAt);
509
300
  }
510
301
  else {
511
302
  log.warn(`[reconcileState] PID file for thread ${pidEntry.threadId} has no DB record — removing`);
@@ -518,8 +309,10 @@ export function reconcileState(db, threadLifecycle) {
518
309
  }
519
310
  continue;
520
311
  }
521
- if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
522
- // Alive: populate spawnedThreads[] if not already present
312
+ if (await isThreadAlive(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt)) {
313
+ // Alive: populate spawnedThreads[] if not already present.
314
+ // THREAD-scoped so a live broker worker (dead launcher pid, live forked
315
+ // child in the job) is not misjudged dead and orphaned across a self-update.
523
316
  if (!spawnedThreads.some((t) => t.threadId === pidEntry.threadId)) {
524
317
  spawnedThreads.push({
525
318
  pid: pidEntry.pid,
@@ -531,6 +324,9 @@ export function reconcileState(db, threadLifecycle) {
531
324
  ...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
532
325
  });
533
326
  log.info(`[reconcileState] Restored thread ${pidEntry.threadId} PID=${pidEntry.pid} from PID file`);
327
+ // Safe cutover (Phase 3): pull pre-broker processes into jobs+registry
328
+ // now rather than waiting for a supervisor restart. Idempotent.
329
+ void brokerAdopt(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt, pidEntry.threadType ?? "agent");
534
330
  }
535
331
  }
536
332
  else {
@@ -591,318 +387,138 @@ export function reconcileState(db, threadLifecycle) {
591
387
  log.warn(`[reconcileState] Failed to sweep orphan MCP configs: ${errorMessage(err)}`);
592
388
  }
593
389
  }
594
- // descendantPid → the tracked thread it was witnessed under (in-memory: an
595
- // orphan spanning a server restart is intentionally not tracked — see header).
596
- const orphanLineage = new Map();
597
- const orphanSnapshotBreaker = makeSnapshotBreaker("win32-process");
598
- /** One `Get-CimInstance Win32_Process` snapshot giving ppid AND creation time for
599
- * every pid — the only single call that yields both, which the sweep needs to
600
- * walk lineage and enforce the pid-reuse guards. Async (never blocks the event
601
- * loop) and behind the shared circuit breaker: on the corporate-agent failure
602
- * observed on this machine it resolves null, and the sweep degrades to a no-op. */
603
- function takeWindowsProcessSnapshot() {
604
- return new Promise((resolve) => {
605
- if (process.platform !== "win32" || orphanSnapshotBreaker.isOpen()) {
606
- resolve(null);
607
- return;
608
- }
609
- execFile("powershell", [
610
- "-NoProfile",
611
- "-Command",
612
- 'Get-CimInstance Win32_Process | ForEach-Object { try { "$($_.ProcessId) $($_.ParentProcessId) $($_.CreationDate.ToUniversalTime().Ticks)" } catch {} }',
613
- ], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
614
- if (err) {
615
- orphanSnapshotBreaker.recordFailure(err);
616
- resolve(null);
617
- return;
618
- }
619
- orphanSnapshotBreaker.recordSuccess();
620
- const snap = new Map();
621
- for (const m of out.matchAll(/^(\d+) (\d+) (\d+)$/gm)) {
622
- snap.set(Number(m[1]), {
623
- ppid: Number(m[2]),
624
- startMs: Math.round(Number(m[3]) / 10000 - 62135596800000), // .NET ticks → Unix ms
625
- });
626
- }
627
- resolve(snap);
628
- });
629
- });
630
- }
631
- /**
632
- * Pure core of the orphan sweep — no IO, no clock, fully deterministic so the
633
- * pid-reuse guard and the snapshot-unavailable path can be tested directly.
634
- * Mutates `lineage` in place (records new descendants, drops dead/recycled ones).
635
- */
636
- export function computeOrphanSweep(params) {
637
- const { snapshot, trackedThreads, lineage, protectedPids, toleranceMs } = params;
638
- const result = {
639
- orphans: [],
640
- reusedPidsSkipped: [],
641
- olderThanParentSkipped: [],
642
- recorded: 0,
643
- };
644
- // Snapshot unavailable → never guess, never kill, never mutate lineage.
645
- if (snapshot === null)
646
- return result;
647
- const aliveOwners = new Set();
648
- for (const t of trackedThreads)
649
- if (t.alive)
650
- aliveOwners.add(t.threadId);
651
- // Children adjacency, built once from the snapshot.
652
- const children = new Map();
653
- for (const [pid, e] of snapshot) {
654
- const list = children.get(e.ppid);
655
- if (list)
656
- list.push(pid);
657
- else
658
- children.set(e.ppid, [pid]);
659
- }
660
- // Step A — witness lineage: record every descendant of each ALIVE tracked
661
- // thread while its subtree chain is still intact. Excludes the root pid itself
662
- // (that thread is managed by the worker-cleanup path, not here).
663
- for (const t of trackedThreads) {
664
- if (!t.alive || !snapshot.has(t.pid))
665
- continue;
666
- const seen = new Set([t.pid]);
667
- const stack = [...(children.get(t.pid) ?? [])];
668
- while (stack.length) {
669
- const d = stack.pop();
670
- if (seen.has(d))
671
- continue; // cycle / pid-reuse loop guard
672
- seen.add(d);
673
- const e = snapshot.get(d);
674
- if (!e)
675
- continue;
676
- lineage.set(d, {
677
- ownerThreadId: t.threadId,
678
- ownerPid: t.pid,
679
- ownerStartedAt: t.startedAt,
680
- descendantStartMs: e.startMs,
681
- });
682
- result.recorded++;
683
- for (const c of children.get(d) ?? [])
684
- if (!seen.has(c))
685
- stack.push(c);
686
- }
687
- }
688
- // Step B — evaluate: a recorded descendant whose owner is no longer alive is an
689
- // orphan candidate, subject to both pid-reuse guards.
690
- for (const [pid, entry] of [...lineage]) {
691
- // Prune a descendant that has left the process table FIRST, before the
692
- // owner-liveness check — otherwise a dead descendant of a still-live owner is
693
- // never evaluated here (its owner short-circuits below) and never re-recorded
694
- // by Step A (it is gone), so it lingers in `lineage` for the owner's entire
695
- // lifetime. A long-lived thread that churns many short-lived children (e.g. a
696
- // claude-agent spawning a bash.exe per hook) would grow the map without bound.
697
- const cur = snapshot.get(pid);
698
- if (!cur) {
699
- lineage.delete(pid); // descendant already gone — nothing to reap
700
- continue;
701
- }
702
- if (aliveOwners.has(entry.ownerThreadId))
703
- continue; // owner still alive → not orphaned
704
- // Never touch a pid managed elsewhere (a live tracked thread, the server, us).
705
- if (protectedPids.has(pid))
706
- continue;
707
- // GUARD 1 — same-process: a recycled pid now hosts a different process whose
708
- // creation time differs from the one we recorded. Drop it; do NOT kill.
709
- if (Math.abs(cur.startMs - entry.descendantStartMs) > toleranceMs) {
710
- lineage.delete(pid);
711
- result.reusedPidsSkipped.push(pid);
712
- continue;
713
- }
714
- // GUARD 2 — a child cannot predate its parent. If this pid started before the
715
- // owning thread did, it is not really our descendant (recorded owner is wrong
716
- // or the pid was reused across the boundary). Keep it for logging, never kill.
717
- if (entry.ownerStartedAt !== undefined && cur.startMs < entry.ownerStartedAt - toleranceMs) {
718
- result.olderThanParentSkipped.push(pid);
719
- continue;
720
- }
721
- result.orphans.push({ pid, ownerThreadId: entry.ownerThreadId, startMs: cur.startMs });
722
- }
723
- return result;
724
- }
725
- /** True when the sweep is allowed to actually kill confirmed orphans. Default is
726
- * log-only (see header): a week of real candidate logs comes first. */
727
- function orphanSweepWillKill() {
728
- const v = process.env.ORPHAN_SWEEP_KILL;
729
- return v === "1" || v === "true";
730
- }
731
390
  /**
732
- * Run one orphan-descendant sweep on the existing worker-cleanup cadence. Takes a
733
- * single process snapshot (no new timer, no second snapshot on the hot liveness
734
- * path), records lineage for live threads, and logs — or, when ORPHAN_SWEEP_KILL
735
- * is set, reaps — confirmed orphans. Best-effort and self-contained; any failure
736
- * degrades to doing nothing.
391
+ * Kill the process tree rooted at `pid` via the supervisor's broker (native
392
+ * TerminateProcess over the whole tree, identity-checked when `spawnedAtMs` is
393
+ * given, outcome recorded in the broker's on-disk registry). Never spawns a
394
+ * child process and never blocks the event loop.
395
+ *
396
+ * Broker unreachable → degraded best-effort: one native `process.kill(pid)`
397
+ * (root only — Node cannot enumerate a tree without shelling out, and that
398
+ * machinery is gone by design) followed by a conservative liveness re-check.
399
+ * A survived kill keeps the PID file so the keeper/cleanup policy loops retry
400
+ * and escalate — same contract as before.
737
401
  */
738
- export async function sweepOrphanDescendants() {
739
- const snapshot = await takeWindowsProcessSnapshot();
740
- if (snapshot === null) {
741
- log.debug("[orphan-sweep] process snapshot unavailable — skipping (never kill on partial data)");
742
- return { orphans: [], reusedPidsSkipped: [], olderThanParentSkipped: [], recorded: 0 };
402
+ export async function killProcessTree(pid, threadId, caller, spawnedAtMs) {
403
+ // Self-protection: NEVER tree-kill our own process or the authoritative
404
+ // server PID (a recycled worker PID could point at us — killing it would be
405
+ // the traceless server death this layer exists to prevent). The broker has
406
+ // the same guard server-side; this is defense in depth.
407
+ if (pid === process.pid) {
408
+ log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
409
+ return;
743
410
  }
744
- const trackedThreads = readPidFiles().map((e) => ({
745
- threadId: e.threadId,
746
- pid: e.pid,
747
- startedAt: e.startedAt,
748
- alive: isProcessAlive(e.pid, e.startedAt) && isSameProcess(e.pid, e.startedAt),
749
- }));
750
- // Protected: every live tracked-thread pid (managed by worker-cleanup), the
751
- // authoritative server pid, and our own pid — these are never orphan targets.
752
- const protectedPids = new Set([process.pid]);
753
411
  const serverPid = readServerPid();
754
- if (serverPid !== undefined)
755
- protectedPids.add(serverPid);
756
- for (const t of trackedThreads)
757
- if (t.alive)
758
- protectedPids.add(t.pid);
759
- const result = computeOrphanSweep({
760
- snapshot,
761
- trackedThreads,
762
- lineage: orphanLineage,
763
- protectedPids,
764
- toleranceMs: PID_IDENTITY_TOLERANCE_MS,
412
+ if (serverPid !== undefined && pid === serverPid) {
413
+ log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
414
+ return;
415
+ }
416
+ log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
417
+ const result = await brokerKillTree(pid, {
418
+ threadId,
419
+ killedBy: caller ?? "mcp",
420
+ reason: caller,
421
+ startedAtMs: spawnedAtMs,
765
422
  });
766
- for (const pid of result.reusedPidsSkipped)
767
- log.debug(`[orphan-sweep] pid ${pid} recycled since we recorded it — dropped, not killed`);
768
- for (const pid of result.olderThanParentSkipped)
769
- log.warn(`[orphan-sweep] pid ${pid} is older than its recorded owner — refusing to kill`);
770
- const willKill = orphanSweepWillKill();
771
- for (const orphan of result.orphans) {
772
- if (!willKill) {
773
- log.warn(`[orphan-sweep] CANDIDATE (log-only): pid ${orphan.pid} orphaned from dead thread ${orphan.ownerThreadId} — set ORPHAN_SWEEP_KILL=1 to reap`);
774
- continue;
775
- }
776
- log.warn(`[orphan-sweep] REAPING orphan pid ${orphan.pid} (owner thread ${orphan.ownerThreadId} is dead)`);
423
+ let stillAlive;
424
+ if (result !== null) {
425
+ if (result.refused)
426
+ log.error(`[process] broker REFUSED kill of pid ${pid}: ${result.error ?? "protected pid"}`);
427
+ else if (result.identityMismatch)
428
+ log.warn(`[process] broker: pid ${pid} was recycled (identity mismatch) — our process is already gone, nothing killed`);
429
+ stillAlive = result.alive;
430
+ }
431
+ else {
432
+ // Degraded path (no supervisor): best-effort native root kill, then re-check.
433
+ log.warn(`[process] broker unreachable for kill of pid ${pid} (thread ${threadId}) — degraded native root-only kill`);
777
434
  try {
778
- await killProcessTree(orphan.pid, orphan.ownerThreadId, "orphan-sweep", orphan.startMs);
435
+ process.kill(pid);
779
436
  }
780
437
  catch (err) {
781
- log.warn(`[orphan-sweep] kill of orphan pid ${orphan.pid} failed: ${errorMessage(err)}`);
782
- }
783
- finally {
784
- orphanLineage.delete(orphan.pid);
438
+ log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
785
439
  }
440
+ await new Promise((resolve) => setTimeout(resolve, 500));
441
+ stillAlive = await isProcessAlive(pid, spawnedAtMs);
786
442
  }
787
- return result;
788
- }
789
- // taskkill on a WEDGED process tree (e.g. a quota-stuck agent whose child
790
- // node/bash/git processes sit in blocking IO) can take well over 10s to force
791
- // the whole /T tree down. A short execFile timeout aborts taskkill mid-kill and
792
- // surfaces as "Command failed" with no stderr, so the process survives and the
793
- // keeper wrongly flags it UNKILLABLE. Give taskkill headroom to finish; the
794
- // caller still verifies liveness afterwards, so a fast kill returns immediately.
795
- const TASKKILL_TIMEOUT_MS = 60_000;
796
- export function killProcessTree(pid, threadId, caller, spawnedAtMs) {
797
- return new Promise((resolve) => {
798
- // Self-protection: NEVER tree-kill our own process or the authoritative
799
- // server PID. The leading hypothesis for the traceless server death is a
800
- // keeper/worker-reaper tree-killing a RECYCLED worker PID that the OS
801
- // reassigned to the node server (or one of its ancestors). `taskkill /T`
802
- // would then take the whole server tree down with no signal, no [fatal] and
803
- // no crash dump — exactly the observed symptom. Refuse and log loudly.
804
- if (pid === process.pid) {
805
- log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
806
- resolve();
807
- return;
443
+ if (!stillAlive) {
444
+ log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
445
+ try {
446
+ unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
808
447
  }
809
- const serverPid = readServerPid();
810
- if (serverPid !== undefined && pid === serverPid) {
811
- log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
812
- resolve();
813
- return;
448
+ catch {
449
+ /* PID file already gone (exit handler may have raced us) */
814
450
  }
815
- log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
816
- const cleanupPidFile = () => {
817
- const pidFile = join(PROCESS_PIDS_DIR, `${threadId}.pid`);
451
+ return;
452
+ }
453
+ // Conservative: a survived kill keeps the PID file so a later sweep retries
454
+ // against fresh evidence (keeper/worker-cleanup own the escalation policy).
455
+ log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt — NOT removing PID file`);
456
+ }
457
+ /**
458
+ * Kill a worker by its THREAD, not by the pid we recorded at spawn.
459
+ *
460
+ * For a launcher-forked worker the recorded pid is the launcher, which has long
461
+ * since exited; `killProcessTree(launcherPid)` therefore routes to the broker's
462
+ * KillByPid, hits its "AlreadyDead" fast-path (the launcher pid is gone) and
463
+ * returns BEFORE the job-kill fallback — so the live forked child is never
464
+ * touched. Routing by threadId (pid omitted) takes the broker's KillByThread
465
+ * path instead: it terminates the whole Job Object (every live member), or, when
466
+ * the job assign failed (Avecto downgrade), the registry-identity survivor the
467
+ * broker adopted. This is the kill that actually reaches the real agent.
468
+ *
469
+ * Broker unreachable → degraded best-effort native kill of `fallbackPid` (self/
470
+ * server-guarded), and we NEVER remove the PID file or claim death: a later sweep
471
+ * retries against fresh evidence. `brokerKillTree` also returns null for an
472
+ * unknown/already-dead thread (HTTP 404); the degraded native kill of a dead pid
473
+ * is a harmless no-op there, and the caller's own `isThreadAlive` gate — not this
474
+ * function — decides teardown.
475
+ */
476
+ export async function killThreadTree(threadId, caller, fallbackPid, spawnedAtMs) {
477
+ log.info(`[process] killThreadTree thread=${threadId} (caller=${caller ?? "?"})`);
478
+ // Route by thread: pid omitted → broker KillByThread (job kill of the real child).
479
+ const result = await brokerKillTree(undefined, {
480
+ threadId,
481
+ killedBy: caller ?? "mcp",
482
+ reason: caller,
483
+ startedAtMs: spawnedAtMs,
484
+ });
485
+ if (result !== null) {
486
+ if (result.refused)
487
+ log.error(`[process] broker REFUSED kill of thread ${threadId}: ${result.error ?? "protected"}`);
488
+ if (!result.alive) {
489
+ // No live member remains — safe to drop the stale PID file.
818
490
  try {
819
- unlinkSync(pidFile);
491
+ unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
820
492
  }
821
493
  catch {
822
494
  /* PID file already gone (exit handler may have raced us) */
823
495
  }
824
- };
825
- const verifyDead = (attempt) => {
826
- // spawnedAtMs matters here for orphan-kill callers reaping FRESH pids: a
827
- // failed kill must not let a pre-spawn snapshot falsely confirm death and
828
- // unlink a live process's pid file. A genuinely dead child is usually
829
- // caught by the kill(0) ESRCH probe or a post-spawn snapshot refresh; if
830
- // neither lands within the 3 attempts (refresh in flight / breaker open),
831
- // we resolve as "still alive" and KEEP the pid file — conservative, and a
832
- // later sweep finishes the cleanup against a fresher snapshot.
833
- if (!isProcessAlive(pid, spawnedAtMs)) {
834
- log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
835
- cleanupPidFile();
836
- resolve();
837
- return;
838
- }
839
- if (attempt >= 3) {
840
- log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after ${attempt} kill attempts — NOT removing PID file`);
841
- resolve();
842
- return;
843
- }
844
- // Retry kill after a short delay
845
- log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt ${attempt} — retrying`);
846
- if (process.platform === "win32") {
847
- // Escalation, retry-path only: the happy-path in-process `taskkill /F /T`
848
- // already failed to verify-dead. If the failure was EPERM because the
849
- // server's own tree is job-object-constrained / I/O-wedged, no in-process
850
- // kill will ever succeed — so first spawn a DETACHED, fresh sibling
851
- // taskkill that does not inherit our wedged handles.
852
- spawnDetachedKillerWin(pid);
853
- // Native TerminateProcess (via libuv) as well as taskkill — they take
854
- // different paths (taskkill is an external process with its own token;
855
- // process.kill issues the syscall directly from this process), so one
856
- // can succeed where the other fails on a stubborn process.
857
- try {
858
- process.kill(pid);
859
- }
860
- catch (err) {
861
- log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
862
- }
863
- // `taskkill /T` walks the tree top-down and can stall when a hung child
864
- // blocks the parent's termination — the exact case where the root sat
865
- // alive for hours. Kill each descendant leaves-first so a wedged child is
866
- // reaped before its parent, freeing the root to die. Logged at info (from
867
- // entry through kill count) so a week of logs can answer whether this
868
- // escalation path ever actually runs, or is dead weight.
869
- log.info(`[process] leaf-first kill ESCALATION entered for pid=${pid} thread=${threadId} (attempt ${attempt}) — taskkill /F /T did not verify-dead`);
870
- getWindowsProcessTreeLeafFirst(pid, threadId, (tree) => {
871
- killPidsSequentiallyWin(tree, (killed) => {
872
- log.info(`[process] leaf-first escalation for pid=${pid} thread=${threadId}: issued taskkill to ${tree.length} process(es), ${killed} reported killed`);
873
- setTimeout(() => verifyDead(attempt + 1), 1000);
874
- });
875
- });
876
- }
877
- else {
878
- try {
879
- process.kill(pid, "SIGKILL");
880
- }
881
- catch {
882
- /* process already gone (ESRCH) — verifyDead confirms */
883
- }
884
- setTimeout(() => verifyDead(attempt + 1), 1000);
885
- }
886
- };
887
- if (process.platform === "win32") {
888
- execFile("taskkill", ["/F", "/T", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
889
- if (err) {
890
- log.warn(`[process] Kill process ${pid} (thread ${threadId}) failed: ${errorMessage(err)}`);
891
- }
892
- // Always verify — taskkill can report success but process lingers,
893
- // or report failure but process is actually dead.
894
- setTimeout(() => verifyDead(1), 500);
895
- });
896
496
  }
897
497
  else {
898
- try {
899
- process.kill(pid, "SIGTERM");
900
- }
901
- catch (err) {
902
- log.debug(`[process] Kill process ${pid} (thread ${threadId}): ${errorMessage(err)}`);
903
- }
904
- setTimeout(() => verifyDead(1), 500);
498
+ log.warn(`[process] thread ${threadId} still has a live member after kill — NOT removing PID file`);
905
499
  }
906
- });
500
+ return;
501
+ }
502
+ // Degraded path (broker unreachable, or unknown/already-dead thread → 404).
503
+ // Best-effort native kill of the known pid; never remove the PID file or claim
504
+ // death — treat-alive on the unknown, exactly the audited-safe contract.
505
+ if (fallbackPid === undefined)
506
+ return;
507
+ if (fallbackPid === process.pid) {
508
+ log.error(`[process] REFUSED degraded kill of pid ${fallbackPid} — matches self (self-protection) thread=${threadId}`);
509
+ return;
510
+ }
511
+ const serverPid = readServerPid();
512
+ if (serverPid !== undefined && fallbackPid === serverPid) {
513
+ log.error(`[process] REFUSED degraded kill of pid ${fallbackPid} — matches server.pid (self-protection) thread=${threadId}`);
514
+ return;
515
+ }
516
+ log.warn(`[process] broker unreachable for kill of thread ${threadId} — degraded native kill of pid ${fallbackPid}`);
517
+ try {
518
+ process.kill(fallbackPid);
519
+ }
520
+ catch (err) {
521
+ log.debug(`[process] native kill PID ${fallbackPid} failed: ${errorMessage(err)}`);
522
+ }
907
523
  }
908
524
  //# sourceMappingURL=process.service.js.map