sensorium-mcp 3.1.0-refactor.16 → 3.1.0-refactor.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/agent-settings.d.ts +6 -0
- package/dist/config/agent-settings.d.ts.map +1 -1
- package/dist/config/agent-settings.js +22 -0
- package/dist/config/agent-settings.js.map +1 -1
- package/dist/config/detail-grant.d.ts +39 -0
- package/dist/config/detail-grant.d.ts.map +1 -0
- package/dist/config/detail-grant.js +61 -0
- package/dist/config/detail-grant.js.map +1 -0
- package/dist/config/index.d.ts +1 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +1 -0
- package/dist/config/index.js.map +1 -1
- package/dist/dashboard/routes/copilot.d.ts.map +1 -1
- package/dist/dashboard/routes/copilot.js +17 -1
- package/dist/dashboard/routes/copilot.js.map +1 -1
- package/dist/dashboard/routes/threads-read.d.ts.map +1 -1
- package/dist/dashboard/routes/threads-read.js +9 -2
- package/dist/dashboard/routes/threads-read.js.map +1 -1
- package/dist/dashboard/routes/threads.js +3 -3
- package/dist/dashboard/routes/threads.js.map +1 -1
- package/dist/dashboard/routes.d.ts.map +1 -1
- package/dist/dashboard/routes.js +11 -4
- package/dist/dashboard/routes.js.map +1 -1
- package/dist/dashboard/spa.html +11 -11
- package/dist/index.js +48 -13
- package/dist/index.js.map +1 -1
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/logger.js +21 -2
- package/dist/lib/logger.js.map +1 -1
- package/dist/server/factory.js +16 -6
- package/dist/server/factory.js.map +1 -1
- package/dist/server/http-server.d.ts.map +1 -1
- package/dist/server/http-server.js +10 -7
- package/dist/server/http-server.js.map +1 -1
- package/dist/services/agent-spawn.service.d.ts +3 -3
- package/dist/services/agent-spawn.service.d.ts.map +1 -1
- package/dist/services/agent-spawn.service.js +14 -8
- package/dist/services/agent-spawn.service.js.map +1 -1
- package/dist/services/background-runner.d.ts.map +1 -1
- package/dist/services/background-runner.js +6 -12
- package/dist/services/background-runner.js.map +1 -1
- package/dist/services/broker-client.d.ts +181 -0
- package/dist/services/broker-client.d.ts.map +1 -0
- package/dist/services/broker-client.js +323 -0
- package/dist/services/broker-client.js.map +1 -0
- package/dist/services/claude-spawn.service.d.ts +1 -1
- package/dist/services/claude-spawn.service.d.ts.map +1 -1
- package/dist/services/claude-spawn.service.js +25 -25
- package/dist/services/claude-spawn.service.js.map +1 -1
- package/dist/services/codex-spawn.service.d.ts +1 -1
- package/dist/services/codex-spawn.service.d.ts.map +1 -1
- package/dist/services/codex-spawn.service.js +19 -24
- package/dist/services/codex-spawn.service.js.map +1 -1
- package/dist/services/copilot-spawn.service.d.ts +1 -1
- package/dist/services/copilot-spawn.service.d.ts.map +1 -1
- package/dist/services/copilot-spawn.service.js +45 -36
- package/dist/services/copilot-spawn.service.js.map +1 -1
- package/dist/services/dispatcher/poller.d.ts.map +1 -1
- package/dist/services/dispatcher/poller.js +24 -5
- package/dist/services/dispatcher/poller.js.map +1 -1
- package/dist/services/flight-recorder.service.d.ts.map +1 -1
- package/dist/services/flight-recorder.service.js +41 -2
- package/dist/services/flight-recorder.service.js.map +1 -1
- package/dist/services/keeper.service.d.ts +9 -0
- package/dist/services/keeper.service.d.ts.map +1 -1
- package/dist/services/keeper.service.js +147 -31
- package/dist/services/keeper.service.js.map +1 -1
- package/dist/services/process.service.d.ts +40 -62
- package/dist/services/process.service.d.ts.map +1 -1
- package/dist/services/process.service.js +118 -613
- package/dist/services/process.service.js.map +1 -1
- package/dist/services/scheduler.d.ts +1 -0
- package/dist/services/scheduler.d.ts.map +1 -1
- package/dist/services/scheduler.js +58 -11
- package/dist/services/scheduler.js.map +1 -1
- package/dist/services/self-update.service.d.ts +14 -3
- package/dist/services/self-update.service.d.ts.map +1 -1
- package/dist/services/self-update.service.js +141 -72
- package/dist/services/self-update.service.js.map +1 -1
- package/dist/services/spawn-common.d.ts +49 -23
- package/dist/services/spawn-common.d.ts.map +1 -1
- package/dist/services/spawn-common.js +262 -47
- package/dist/services/spawn-common.js.map +1 -1
- package/dist/services/spawn-exit-watcher.d.ts +32 -0
- package/dist/services/spawn-exit-watcher.d.ts.map +1 -0
- package/dist/services/spawn-exit-watcher.js +102 -0
- package/dist/services/spawn-exit-watcher.js.map +1 -0
- package/dist/services/thread-health.service.d.ts +1 -1
- package/dist/services/thread-health.service.d.ts.map +1 -1
- package/dist/services/thread-health.service.js +6 -6
- package/dist/services/thread-health.service.js.map +1 -1
- package/dist/services/worker-cleanup.service.d.ts.map +1 -1
- package/dist/services/worker-cleanup.service.js +9 -92
- package/dist/services/worker-cleanup.service.js.map +1 -1
- package/dist/tools/defs/session-defs.js +1 -1
- package/dist/tools/defs/session-defs.js.map +1 -1
- package/dist/tools/defs/utility-defs.d.ts.map +1 -1
- package/dist/tools/defs/utility-defs.js +4 -3
- package/dist/tools/defs/utility-defs.js.map +1 -1
- package/dist/tools/delegate-tool.d.ts +1 -1
- package/dist/tools/delegate-tool.d.ts.map +1 -1
- package/dist/tools/delegate-tool.js +5 -5
- package/dist/tools/delegate-tool.js.map +1 -1
- package/dist/tools/session-tools.d.ts.map +1 -1
- package/dist/tools/session-tools.js +26 -9
- package/dist/tools/session-tools.js.map +1 -1
- package/dist/tools/utility-tools.js +1 -1
- package/dist/tools/utility-tools.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { execFile, spawn } from "node:child_process";
|
|
2
1
|
import { mkdirSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
|
|
3
2
|
import { homedir } from "node:os";
|
|
4
3
|
import { join } from "node:path";
|
|
@@ -6,6 +5,7 @@ import { log } from "../lib/logger.js";
|
|
|
6
5
|
import { errorMessage } from "../lib/utils.js";
|
|
7
6
|
import { getAllThreads } from "../data/memory/thread-registry.js";
|
|
8
7
|
import { getDashboardSessions } from "../sessions/dashboard-registry.js";
|
|
8
|
+
import { brokerAdopt, brokerCheckAlive, brokerKillTree } from "./broker-client.js";
|
|
9
9
|
const BASE_DIR = join(homedir(), ".remote-copilot-mcp");
|
|
10
10
|
const LOGS_DIR = join(BASE_DIR, "logs");
|
|
11
11
|
const PIDS_DIR = join(BASE_DIR, "pids");
|
|
@@ -59,294 +59,39 @@ export function deletePendingTaskMailbox(threadId) {
|
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
|
-
/**
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Liveness + identity in one question: is `pid` alive AND (when `spawnedAtMs`
|
|
64
|
+
* is given) still the process we spawned?
|
|
65
|
+
*
|
|
66
|
+
* Two-step check, neither of which ever spawns a child process:
|
|
67
|
+
* 1. `process.kill(pid, 0)` — a plain syscall. ESRCH (any non-EPERM error) is
|
|
68
|
+
* a definitive "dead" and needs no broker (this also keeps dev mode
|
|
69
|
+
* without a supervisor fully functional for the common case).
|
|
70
|
+
* 2. Otherwise the answer is NOT trustworthy on Windows — kill(0) succeeds
|
|
71
|
+
* for kernel-zombie PIDs held open by a lingering handle, and EPERM is
|
|
72
|
+
* what a parent sees for its own terminated child. Ask the supervisor's
|
|
73
|
+
* process broker, which answers from a native process snapshot and also
|
|
74
|
+
* verifies creation-time identity (PID-reuse guard).
|
|
75
|
+
*
|
|
76
|
+
* Broker unreachable → "cannot confirm → treat alive", exactly the audited-safe
|
|
77
|
+
* degradation the deleted tasklist/Get-Process breakers had.
|
|
78
|
+
*/
|
|
79
|
+
export async function isProcessAlive(pid, spawnedAtMs) {
|
|
68
80
|
try {
|
|
69
81
|
process.kill(pid, 0);
|
|
70
82
|
}
|
|
71
83
|
catch (err) {
|
|
72
|
-
// ESRCH (or any non-EPERM error) = no such process → dead. EPERM = kill(0)
|
|
73
|
-
// was denied; do NOT trust that as alive — fall through to the win32 table
|
|
74
|
-
// check below (see next comment for why). On POSIX, EPERM does mean a live
|
|
75
|
-
// process owned by another user, so the fall-through returns true there.
|
|
76
84
|
if (err.code !== "EPERM")
|
|
77
85
|
return false;
|
|
78
86
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
// - kill(0) succeeds for a process that already TERMINATED but whose PID is
|
|
82
|
-
// still held open by a lingering handle — a kernel "zombie".
|
|
83
|
-
// - OpenProcess from the PARENT of such a dead child returns ACCESS_DENIED
|
|
84
|
-
// (→EPERM) for that same terminated PID (a non-parent instead sees ESRCH).
|
|
85
|
-
// Both cases wedged the keeper into treating an unkillable zombie as alive
|
|
86
|
-
// forever, never restarting the thread. The process table excludes terminated
|
|
87
|
-
// PIDs even while their handle lingers, so make it authoritative for both
|
|
88
|
-
// paths. (tasklist unavailable → isPidInWindowsTable conservatively returns
|
|
89
|
-
// true, so we never declare a genuinely live process dead.)
|
|
90
|
-
if (process.platform === "win32")
|
|
91
|
-
return isPidInWindowsTable(pid, spawnedAtMs);
|
|
92
|
-
return true;
|
|
93
|
-
}
|
|
94
|
-
const WIN_PID_SNAPSHOT_TTL_MS = 2000;
|
|
95
|
-
// A snapshot older than this is discarded rather than trusted: a very stale pid
|
|
96
|
-
// set would mark freshly-spawned processes dead (they're absent from it), and a
|
|
97
|
-
// stale start-time map is merely incomplete. Normal refreshes complete in well
|
|
98
|
-
// under a second, so this bound is only reached when refreshes keep failing —
|
|
99
|
-
// at which point the getters degrade to null (= "can't confirm" → alive).
|
|
100
|
-
const WIN_SNAPSHOT_MAX_STALE_MS = 15_000;
|
|
101
|
-
// Circuit breaker for the snapshot commands. When the OS throttles external
|
|
102
|
-
// process spawns (battery / Energy Saver / EDR), tasklist and powershell hang
|
|
103
|
-
// their full 5s timeout on EVERY call. The former synchronous snapshots then
|
|
104
|
-
// froze the event loop for 5s at a time, so newly-spawned agent children could
|
|
105
|
-
// not complete the MCP initialize/tools-list handshake and started TOOLLESS —
|
|
106
|
-
// then exited cleanly and were respawned forever (the 2026-07-17 respawn storm,
|
|
107
|
-
// see implementation_plans/2026-07-18_toolless-respawn-storm.md). Refreshes are
|
|
108
|
-
// now async (never on the event loop) and, after repeated failures, paused
|
|
109
|
-
// entirely for a cooldown instead of hammering a throttled OS every sweep.
|
|
110
|
-
const SNAPSHOT_BREAKER_THRESHOLD = 3;
|
|
111
|
-
const SNAPSHOT_BREAKER_COOLDOWN_MS = 60_000;
|
|
112
|
-
function makeSnapshotBreaker(label) {
|
|
113
|
-
let consecutiveFailures = 0;
|
|
114
|
-
let openUntil = 0;
|
|
115
|
-
return {
|
|
116
|
-
isOpen: () => Date.now() < openUntil,
|
|
117
|
-
recordSuccess: () => {
|
|
118
|
-
consecutiveFailures = 0;
|
|
119
|
-
},
|
|
120
|
-
recordFailure: (err) => {
|
|
121
|
-
consecutiveFailures++;
|
|
122
|
-
if (consecutiveFailures >= SNAPSHOT_BREAKER_THRESHOLD) {
|
|
123
|
-
openUntil = Date.now() + SNAPSHOT_BREAKER_COOLDOWN_MS;
|
|
124
|
-
// Reset on open so re-opening after the cooldown requires THRESHOLD
|
|
125
|
-
// fresh failures — one lingering failure must not instantly re-trip.
|
|
126
|
-
consecutiveFailures = 0;
|
|
127
|
-
log.warn(`[process] ${label} snapshot failed ${SNAPSHOT_BREAKER_THRESHOLD}x consecutively (${errorMessage(err)}) — pausing refreshes for ${SNAPSHOT_BREAKER_COOLDOWN_MS / 1000}s`);
|
|
128
|
-
}
|
|
129
|
-
else {
|
|
130
|
-
log.debug(`[process] ${label} snapshot failed: ${errorMessage(err)}`);
|
|
131
|
-
}
|
|
132
|
-
},
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
/** Authoritative Windows liveness check via the process table, which — unlike
|
|
136
|
-
* OpenProcess/process.kill(0) — does NOT list a terminated process whose PID is
|
|
137
|
-
* still held by a lingering handle. One `tasklist` snapshot of ALL pids serves
|
|
138
|
-
* every pid in a sweep, and it is refreshed asynchronously in the background:
|
|
139
|
-
* the getter reads the cached snapshot and NEVER blocks the event loop (a 5s
|
|
140
|
-
* synchronous hang here is what starved MCP handshakes and caused the toolless
|
|
141
|
-
* respawn storm). Callers treat null as "can't confirm dead" → alive. */
|
|
142
|
-
let winPidSnapshot = null;
|
|
143
|
-
let winPidRefreshInFlight = false;
|
|
144
|
-
const pidSnapshotBreaker = makeSnapshotBreaker("tasklist");
|
|
145
|
-
function refreshWindowsPidSet() {
|
|
146
|
-
if (winPidRefreshInFlight || pidSnapshotBreaker.isOpen())
|
|
147
|
-
return;
|
|
148
|
-
winPidRefreshInFlight = true;
|
|
149
|
-
// The snapshot's ts is the REFRESH START: the process table is read at (or
|
|
150
|
-
// after) this instant, so any pid spawned later can legitimately be absent.
|
|
151
|
-
const refreshStartedAt = Date.now();
|
|
152
|
-
// /NH = no header, /FO CSV = quoted columns; PID is column 2: "image","PID",...
|
|
153
|
-
execFile("tasklist", ["/NH", "/FO", "CSV"], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
154
|
-
winPidRefreshInFlight = false;
|
|
155
|
-
if (err) {
|
|
156
|
-
pidSnapshotBreaker.recordFailure(err);
|
|
157
|
-
return;
|
|
158
|
-
}
|
|
159
|
-
pidSnapshotBreaker.recordSuccess();
|
|
160
|
-
const pids = new Set();
|
|
161
|
-
for (const m of out.matchAll(/^"[^"]*","(\d+)"/gm))
|
|
162
|
-
pids.add(Number(m[1]));
|
|
163
|
-
winPidSnapshot = { pids, ts: refreshStartedAt };
|
|
164
|
-
});
|
|
165
|
-
}
|
|
166
|
-
function getWindowsPidSet() {
|
|
167
|
-
const now = Date.now();
|
|
168
|
-
if (!winPidSnapshot || now - winPidSnapshot.ts >= WIN_PID_SNAPSHOT_TTL_MS)
|
|
169
|
-
refreshWindowsPidSet();
|
|
170
|
-
if (winPidSnapshot && now - winPidSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
|
|
171
|
-
return winPidSnapshot.pids;
|
|
172
|
-
return null; // no usable snapshot — conservative: never declare a real process dead
|
|
173
|
-
}
|
|
174
|
-
function isPidInWindowsTable(pid, spawnedAtMs) {
|
|
175
|
-
const pids = getWindowsPidSet();
|
|
176
|
-
if (pids === null)
|
|
177
|
-
return true; // tasklist unavailable — never declare a real process dead
|
|
178
|
-
if (pids.has(pid))
|
|
87
|
+
if (process.platform !== "win32") {
|
|
88
|
+
// POSIX: kill(0) success or EPERM both mean a live process.
|
|
179
89
|
return true;
|
|
180
|
-
// Absent from the table — but if the process was spawned AFTER the snapshot
|
|
181
|
-
// was taken, the absence proves nothing (the table simply predates it).
|
|
182
|
-
// Treat as alive; the refresh already triggered above will testify shortly.
|
|
183
|
-
return spawnedAtMs !== undefined && spawnedAtMs >= winPidSnapshot.ts;
|
|
184
|
-
}
|
|
185
|
-
// Tolerance between the pid-file's recorded startedAt (Date.now() at spawn) and
|
|
186
|
-
// the OS process creation time. Wide enough to absorb spawn latency and a wall-
|
|
187
|
-
// clock step (NTP/VM resume) between the two readings, yet far below the minutes-
|
|
188
|
-
// to-hours gap a recycled PID produces — so PID reuse is still caught.
|
|
189
|
-
const PID_IDENTITY_TOLERANCE_MS = 60_000;
|
|
190
|
-
const WIN_START_SNAPSHOT_TTL_MS = 2000;
|
|
191
|
-
/** One powershell snapshot of every pid's creation time (Unix ms), reused for
|
|
192
|
-
* all pids in a keeper sweep and refreshed asynchronously in the background —
|
|
193
|
-
* same non-blocking pattern as the tasklist snapshot above. Staleness is safe
|
|
194
|
-
* here: a pid's start time never changes, so an old map is merely missing new
|
|
195
|
-
* pids, and a missing entry already means "can't tell" → trust PID. StartTime
|
|
196
|
-
* is unreadable for some protected processes; those are simply omitted. */
|
|
197
|
-
let winStartSnapshot = null;
|
|
198
|
-
let winStartRefreshInFlight = false;
|
|
199
|
-
const startSnapshotBreaker = makeSnapshotBreaker("start-time");
|
|
200
|
-
function refreshWindowsStartTimeMap() {
|
|
201
|
-
if (winStartRefreshInFlight || startSnapshotBreaker.isOpen())
|
|
202
|
-
return;
|
|
203
|
-
winStartRefreshInFlight = true;
|
|
204
|
-
const refreshStartedAt = Date.now();
|
|
205
|
-
execFile("powershell", [
|
|
206
|
-
"-NoProfile",
|
|
207
|
-
"-Command",
|
|
208
|
-
'Get-Process | ForEach-Object { try { "$($_.Id) $($_.StartTime.ToUniversalTime().Ticks)" } catch {} }',
|
|
209
|
-
], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
210
|
-
winStartRefreshInFlight = false;
|
|
211
|
-
if (err) {
|
|
212
|
-
startSnapshotBreaker.recordFailure(err);
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
|
-
startSnapshotBreaker.recordSuccess();
|
|
216
|
-
const starts = new Map();
|
|
217
|
-
for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
|
|
218
|
-
starts.set(Number(m[1]), Math.round(Number(m[2]) / 10000 - 62135596800000)); // .NET ticks (100ns from year 1) → Unix ms
|
|
219
|
-
}
|
|
220
|
-
winStartSnapshot = { starts, ts: refreshStartedAt };
|
|
221
|
-
});
|
|
222
|
-
}
|
|
223
|
-
function getWindowsStartTimeMap() {
|
|
224
|
-
const now = Date.now();
|
|
225
|
-
if (!winStartSnapshot || now - winStartSnapshot.ts >= WIN_START_SNAPSHOT_TTL_MS)
|
|
226
|
-
refreshWindowsStartTimeMap();
|
|
227
|
-
if (winStartSnapshot && now - winStartSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
|
|
228
|
-
return winStartSnapshot.starts;
|
|
229
|
-
return null; // no usable snapshot — conservative: caller trusts the PID
|
|
230
|
-
}
|
|
231
|
-
/** Windows process creation time as Unix ms, or undefined if unavailable.
|
|
232
|
-
* Lets us tell OUR process apart from an unrelated one the OS assigned the
|
|
233
|
-
* same (recycled) PID after ours exited. */
|
|
234
|
-
function processStartMs(pid) {
|
|
235
|
-
if (process.platform !== "win32")
|
|
236
|
-
return undefined;
|
|
237
|
-
return getWindowsStartTimeMap()?.get(pid);
|
|
238
|
-
}
|
|
239
|
-
/** True if the live PID is (still) the process we spawned, judged by start time.
|
|
240
|
-
* Guards against PID reuse: a recycled PID now owned by a different process has
|
|
241
|
-
* a start time far from what we recorded, so we stop treating it as ours (which
|
|
242
|
-
* otherwise wedges the keeper into an endless "unkillable" kill loop). */
|
|
243
|
-
function isSameProcess(pid, startedAt) {
|
|
244
|
-
if (startedAt === undefined)
|
|
245
|
-
return true; // nothing recorded to compare — trust PID
|
|
246
|
-
// Our process was spawned AFTER the current start-time map was taken: any
|
|
247
|
-
// entry the map holds for this pid describes a PREVIOUS owner of the (since
|
|
248
|
-
// recycled) pid, not our child — comparing against it would wrongly reject
|
|
249
|
-
// our own fresh process. Trust the PID until a fresher map exists.
|
|
250
|
-
if (process.platform === "win32" &&
|
|
251
|
-
winStartSnapshot !== null &&
|
|
252
|
-
startedAt >= winStartSnapshot.ts) {
|
|
253
|
-
return true;
|
|
254
|
-
}
|
|
255
|
-
const actual = processStartMs(pid);
|
|
256
|
-
if (actual === undefined)
|
|
257
|
-
return true; // can't read start time — keep prior behavior
|
|
258
|
-
return Math.abs(actual - startedAt) < PID_IDENTITY_TOLERANCE_MS;
|
|
259
|
-
}
|
|
260
|
-
/** Descendants of rootPid (including root), ordered leaves-first (post-order),
|
|
261
|
-
* from one process-table snapshot, delivered via callback (async so the kill
|
|
262
|
-
* escalation path never blocks the event loop either). Fallback for when
|
|
263
|
-
* `taskkill /T` fails to tear the whole tree down: a single wedged descendant
|
|
264
|
-
* can make taskkill abort the tree, so killing each PID individually lets the
|
|
265
|
-
* still-killable relatives (often including the root) go down anyway. */
|
|
266
|
-
function getWindowsProcessTreeLeafFirst(rootPid, threadId, done) {
|
|
267
|
-
execFile("powershell", [
|
|
268
|
-
"-NoProfile",
|
|
269
|
-
"-Command",
|
|
270
|
-
'Get-CimInstance Win32_Process | ForEach-Object { "$($_.ProcessId) $($_.ParentProcessId)" }',
|
|
271
|
-
], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
272
|
-
if (err) {
|
|
273
|
-
// "Could not look" — distinct from "found nothing" below: the snapshot
|
|
274
|
-
// itself failed (the corporate-agent CIM failure seen on this machine),
|
|
275
|
-
// so we cannot enumerate the tree at all and fall back to the root pid.
|
|
276
|
-
log.warn(`[process] leaf-first walk pid=${rootPid} thread=${threadId}: process-tree snapshot FAILED (${errorMessage(err)}) — could NOT enumerate the tree; falling back to root pid only`);
|
|
277
|
-
done([rootPid]);
|
|
278
|
-
return;
|
|
279
|
-
}
|
|
280
|
-
const children = new Map();
|
|
281
|
-
for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
|
|
282
|
-
const pid = Number(m[1]);
|
|
283
|
-
const ppid = Number(m[2]);
|
|
284
|
-
const list = children.get(ppid);
|
|
285
|
-
if (list)
|
|
286
|
-
list.push(pid);
|
|
287
|
-
else
|
|
288
|
-
children.set(ppid, [pid]);
|
|
289
|
-
}
|
|
290
|
-
const ordered = [];
|
|
291
|
-
const seen = new Set();
|
|
292
|
-
const visit = (pid) => {
|
|
293
|
-
if (seen.has(pid))
|
|
294
|
-
return; // guard against cycles / PID-reuse loops
|
|
295
|
-
seen.add(pid);
|
|
296
|
-
for (const child of children.get(pid) ?? [])
|
|
297
|
-
visit(child);
|
|
298
|
-
ordered.push(pid); // post-order → leaves first, root last
|
|
299
|
-
};
|
|
300
|
-
visit(rootPid);
|
|
301
|
-
// "Found nothing" reads as "root + 0 descendant(s)" — clearly not the same
|
|
302
|
-
// as the snapshot-failed warning above.
|
|
303
|
-
log.info(`[process] leaf-first walk pid=${rootPid} thread=${threadId} enumerated ${ordered.length} process(es): root + ${ordered.length - 1} descendant(s)`);
|
|
304
|
-
done(ordered);
|
|
305
|
-
});
|
|
306
|
-
}
|
|
307
|
-
/** Force-kill each PID in order, one at a time, tolerating any that refuse (a
|
|
308
|
-
* wedged leaf must not block the ones after it). Async so a sweep over a deep
|
|
309
|
-
* tree never blocks the server event loop. */
|
|
310
|
-
function killPidsSequentiallyWin(pids, done) {
|
|
311
|
-
let i = 0;
|
|
312
|
-
let killed = 0; // taskkill invocations that returned no error (reported killed)
|
|
313
|
-
const next = () => {
|
|
314
|
-
if (i >= pids.length) {
|
|
315
|
-
done(killed);
|
|
316
|
-
return;
|
|
317
|
-
}
|
|
318
|
-
const pid = pids[i++];
|
|
319
|
-
execFile("taskkill", ["/F", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
|
|
320
|
-
if (!err)
|
|
321
|
-
killed++;
|
|
322
|
-
next();
|
|
323
|
-
});
|
|
324
|
-
};
|
|
325
|
-
next();
|
|
326
|
-
}
|
|
327
|
-
/** Last-resort escalation for EPERM-wedged trees: spawn taskkill as a DETACHED,
|
|
328
|
-
* FRESH process so it does NOT inherit the server's wedged tree handles. When
|
|
329
|
-
* the server's own process tree is job-object-constrained / I/O-wedged, an
|
|
330
|
-
* in-process `taskkill /F /T` returns EPERM — but a sibling process spawned
|
|
331
|
-
* with its own handles (detached, own stdio) killed the exact same zombie trees
|
|
332
|
-
* instantly in this session's experiment. Nested securevault contexts (which
|
|
333
|
-
* once made detached killers cross-context Access-Denied, commit 1b7ded6) are
|
|
334
|
-
* gone — there is a single context now — so a detached killer in the server's
|
|
335
|
-
* own context is safe. Fire-and-forget: unref'd so the server never waits. */
|
|
336
|
-
function spawnDetachedKillerWin(pid) {
|
|
337
|
-
try {
|
|
338
|
-
const child = spawn("taskkill", ["/F", "/T", "/PID", String(pid)], {
|
|
339
|
-
detached: true,
|
|
340
|
-
windowsHide: true,
|
|
341
|
-
stdio: "ignore",
|
|
342
|
-
});
|
|
343
|
-
child.on("error", (err) => log.warn(`[process] detached killer for PID ${pid} errored: ${errorMessage(err)}`));
|
|
344
|
-
child.unref();
|
|
345
|
-
log.warn(`[process] ESCALATION: spawned DETACHED fresh taskkill /F /T /PID ${pid} (bypasses server's EPERM-wedged tree)`);
|
|
346
|
-
}
|
|
347
|
-
catch (err) {
|
|
348
|
-
log.warn(`[process] detached killer spawn for PID ${pid} failed: ${errorMessage(err)}`);
|
|
349
90
|
}
|
|
91
|
+
const result = await brokerCheckAlive(pid, spawnedAtMs);
|
|
92
|
+
if (result === null)
|
|
93
|
+
return true; // cannot confirm → never declare a live process dead
|
|
94
|
+
return result.alive;
|
|
350
95
|
}
|
|
351
96
|
export function readPidFiles() {
|
|
352
97
|
const entries = [];
|
|
@@ -381,40 +126,42 @@ export function readPidFiles() {
|
|
|
381
126
|
}
|
|
382
127
|
return entries;
|
|
383
128
|
}
|
|
384
|
-
export function findAliveThread(threadId) {
|
|
129
|
+
export async function findAliveThread(threadId) {
|
|
385
130
|
for (let i = spawnedThreads.length - 1; i >= 0; i--) {
|
|
386
131
|
const thread = spawnedThreads[i];
|
|
387
132
|
if (thread.threadId !== threadId)
|
|
388
133
|
continue;
|
|
389
|
-
if (isProcessAlive(thread.pid, thread.startedAt)
|
|
134
|
+
if (await isProcessAlive(thread.pid, thread.startedAt))
|
|
390
135
|
return thread;
|
|
391
136
|
log.warn(`[findAliveThread] Thread ${threadId} PID ${thread.pid} stale in spawnedThreads - removing`);
|
|
392
137
|
spawnedThreads.splice(i, 1);
|
|
393
138
|
}
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
139
|
+
for (const entry of readPidFiles()) {
|
|
140
|
+
if (entry.threadId !== threadId)
|
|
141
|
+
continue;
|
|
142
|
+
if (!(await isProcessAlive(entry.pid, entry.startedAt)))
|
|
143
|
+
continue;
|
|
144
|
+
const restored = {
|
|
145
|
+
pid: entry.pid,
|
|
146
|
+
threadId,
|
|
147
|
+
name: entry.name ?? `Thread ${threadId}`,
|
|
148
|
+
startedAt: entry.startedAt ?? Date.now(),
|
|
149
|
+
createdAt: entry.startedAt ?? Date.now(),
|
|
150
|
+
logFile: "",
|
|
151
|
+
...(entry.threadType ? { threadType: entry.threadType } : {}),
|
|
152
|
+
};
|
|
153
|
+
spawnedThreads.push(restored);
|
|
154
|
+
log.info(`[findAliveThread] Restored thread ${threadId} PID=${entry.pid} from PID file`);
|
|
155
|
+
return restored;
|
|
156
|
+
}
|
|
157
|
+
return undefined;
|
|
411
158
|
}
|
|
412
|
-
export const isThreadRunning = (threadId) => findAliveThread(threadId) !== undefined;
|
|
413
|
-
export function getActiveThreadIds() {
|
|
159
|
+
export const isThreadRunning = async (threadId) => (await findAliveThread(threadId)) !== undefined;
|
|
160
|
+
export async function getActiveThreadIds() {
|
|
414
161
|
// Workers are disposable and not eligible for reconnect — exclude them.
|
|
415
|
-
const
|
|
416
|
-
|
|
417
|
-
|
|
162
|
+
const candidates = spawnedThreads.filter((t) => t.threadType !== "worker");
|
|
163
|
+
const aliveFlags = await Promise.all(candidates.map((t) => isProcessAlive(t.pid, t.startedAt)));
|
|
164
|
+
const spawned = candidates.filter((_, i) => aliveFlags[i]).map((t) => t.threadId);
|
|
418
165
|
// Also include threads from active MCP sessions (root threads that aren't
|
|
419
166
|
// in spawnedThreads because they connected externally, not spawned by us).
|
|
420
167
|
const sessionThreadIds = getDashboardSessions()
|
|
@@ -464,14 +211,16 @@ export function pruneOldThreadLogs() {
|
|
|
464
211
|
}
|
|
465
212
|
/**
|
|
466
213
|
* Check liveness of a thread directly via its PID file, bypassing the
|
|
467
|
-
* spawnedThreads[] in-memory cache.
|
|
468
|
-
*
|
|
214
|
+
* spawnedThreads[] in-memory cache. Returns the live PID together with its
|
|
215
|
+
* recorded spawn identity (startedAt) so callers can make identity-checked
|
|
216
|
+
* kill/alive calls, or undefined if no PID file exists or the process is dead
|
|
217
|
+
* (stale file is deleted).
|
|
469
218
|
*/
|
|
470
|
-
export function findAliveThreadViaPidFile(threadId) {
|
|
219
|
+
export async function findAliveThreadViaPidFile(threadId) {
|
|
471
220
|
const entry = readPidFiles().find((e) => e.threadId === threadId);
|
|
472
221
|
if (!entry)
|
|
473
222
|
return undefined;
|
|
474
|
-
if (!
|
|
223
|
+
if (!(await isProcessAlive(entry.pid, entry.startedAt))) {
|
|
475
224
|
try {
|
|
476
225
|
unlinkSync(entry.filePath);
|
|
477
226
|
}
|
|
@@ -480,7 +229,7 @@ export function findAliveThreadViaPidFile(threadId) {
|
|
|
480
229
|
}
|
|
481
230
|
return undefined;
|
|
482
231
|
}
|
|
483
|
-
return entry.pid;
|
|
232
|
+
return { pid: entry.pid, startedAt: entry.startedAt };
|
|
484
233
|
}
|
|
485
234
|
/**
|
|
486
235
|
* Reconcile in-memory state (spawnedThreads[]) with the two authoritative
|
|
@@ -490,7 +239,7 @@ export function findAliveThreadViaPidFile(threadId) {
|
|
|
490
239
|
*
|
|
491
240
|
* Replaces the old restoreFromPidFiles() + cleanupStalePidFiles() pair.
|
|
492
241
|
*/
|
|
493
|
-
export function reconcileState(db, threadLifecycle) {
|
|
242
|
+
export async function reconcileState(db, threadLifecycle) {
|
|
494
243
|
const pidEntries = readPidFiles();
|
|
495
244
|
const dbThreadMap = new Map(getAllThreads(db)
|
|
496
245
|
.filter((t) => ["active", "exited", "exiting", "spawning", "stuck"].includes(t.status))
|
|
@@ -503,7 +252,7 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
503
252
|
// If the process is still alive it's an orphan that must be reaped — never
|
|
504
253
|
// just drop the PID file, or the live process becomes untrackable.
|
|
505
254
|
// killProcessTree removes the PID file itself once the process is confirmed dead.
|
|
506
|
-
if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
|
|
255
|
+
if (await isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
|
|
507
256
|
log.warn(`[reconcileState] Live process for thread ${pidEntry.threadId} (PID=${pidEntry.pid}) has no active DB record — killing orphan`);
|
|
508
257
|
void killProcessTree(pidEntry.pid, pidEntry.threadId, "reconcile-orphan", pidEntry.startedAt);
|
|
509
258
|
}
|
|
@@ -518,7 +267,7 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
518
267
|
}
|
|
519
268
|
continue;
|
|
520
269
|
}
|
|
521
|
-
if (isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
|
|
270
|
+
if (await isProcessAlive(pidEntry.pid, pidEntry.startedAt)) {
|
|
522
271
|
// Alive: populate spawnedThreads[] if not already present
|
|
523
272
|
if (!spawnedThreads.some((t) => t.threadId === pidEntry.threadId)) {
|
|
524
273
|
spawnedThreads.push({
|
|
@@ -531,6 +280,9 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
531
280
|
...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
|
|
532
281
|
});
|
|
533
282
|
log.info(`[reconcileState] Restored thread ${pidEntry.threadId} PID=${pidEntry.pid} from PID file`);
|
|
283
|
+
// Safe cutover (Phase 3): pull pre-broker processes into jobs+registry
|
|
284
|
+
// now rather than waiting for a supervisor restart. Idempotent.
|
|
285
|
+
void brokerAdopt(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt, pidEntry.threadType ?? "agent");
|
|
534
286
|
}
|
|
535
287
|
}
|
|
536
288
|
else {
|
|
@@ -591,318 +343,71 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
591
343
|
log.warn(`[reconcileState] Failed to sweep orphan MCP configs: ${errorMessage(err)}`);
|
|
592
344
|
}
|
|
593
345
|
}
|
|
594
|
-
// descendantPid → the tracked thread it was witnessed under (in-memory: an
|
|
595
|
-
// orphan spanning a server restart is intentionally not tracked — see header).
|
|
596
|
-
const orphanLineage = new Map();
|
|
597
|
-
const orphanSnapshotBreaker = makeSnapshotBreaker("win32-process");
|
|
598
|
-
/** One `Get-CimInstance Win32_Process` snapshot giving ppid AND creation time for
|
|
599
|
-
* every pid — the only single call that yields both, which the sweep needs to
|
|
600
|
-
* walk lineage and enforce the pid-reuse guards. Async (never blocks the event
|
|
601
|
-
* loop) and behind the shared circuit breaker: on the corporate-agent failure
|
|
602
|
-
* observed on this machine it resolves null, and the sweep degrades to a no-op. */
|
|
603
|
-
function takeWindowsProcessSnapshot() {
|
|
604
|
-
return new Promise((resolve) => {
|
|
605
|
-
if (process.platform !== "win32" || orphanSnapshotBreaker.isOpen()) {
|
|
606
|
-
resolve(null);
|
|
607
|
-
return;
|
|
608
|
-
}
|
|
609
|
-
execFile("powershell", [
|
|
610
|
-
"-NoProfile",
|
|
611
|
-
"-Command",
|
|
612
|
-
'Get-CimInstance Win32_Process | ForEach-Object { try { "$($_.ProcessId) $($_.ParentProcessId) $($_.CreationDate.ToUniversalTime().Ticks)" } catch {} }',
|
|
613
|
-
], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
614
|
-
if (err) {
|
|
615
|
-
orphanSnapshotBreaker.recordFailure(err);
|
|
616
|
-
resolve(null);
|
|
617
|
-
return;
|
|
618
|
-
}
|
|
619
|
-
orphanSnapshotBreaker.recordSuccess();
|
|
620
|
-
const snap = new Map();
|
|
621
|
-
for (const m of out.matchAll(/^(\d+) (\d+) (\d+)$/gm)) {
|
|
622
|
-
snap.set(Number(m[1]), {
|
|
623
|
-
ppid: Number(m[2]),
|
|
624
|
-
startMs: Math.round(Number(m[3]) / 10000 - 62135596800000), // .NET ticks → Unix ms
|
|
625
|
-
});
|
|
626
|
-
}
|
|
627
|
-
resolve(snap);
|
|
628
|
-
});
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
/**
|
|
632
|
-
* Pure core of the orphan sweep — no IO, no clock, fully deterministic so the
|
|
633
|
-
* pid-reuse guard and the snapshot-unavailable path can be tested directly.
|
|
634
|
-
* Mutates `lineage` in place (records new descendants, drops dead/recycled ones).
|
|
635
|
-
*/
|
|
636
|
-
export function computeOrphanSweep(params) {
|
|
637
|
-
const { snapshot, trackedThreads, lineage, protectedPids, toleranceMs } = params;
|
|
638
|
-
const result = {
|
|
639
|
-
orphans: [],
|
|
640
|
-
reusedPidsSkipped: [],
|
|
641
|
-
olderThanParentSkipped: [],
|
|
642
|
-
recorded: 0,
|
|
643
|
-
};
|
|
644
|
-
// Snapshot unavailable → never guess, never kill, never mutate lineage.
|
|
645
|
-
if (snapshot === null)
|
|
646
|
-
return result;
|
|
647
|
-
const aliveOwners = new Set();
|
|
648
|
-
for (const t of trackedThreads)
|
|
649
|
-
if (t.alive)
|
|
650
|
-
aliveOwners.add(t.threadId);
|
|
651
|
-
// Children adjacency, built once from the snapshot.
|
|
652
|
-
const children = new Map();
|
|
653
|
-
for (const [pid, e] of snapshot) {
|
|
654
|
-
const list = children.get(e.ppid);
|
|
655
|
-
if (list)
|
|
656
|
-
list.push(pid);
|
|
657
|
-
else
|
|
658
|
-
children.set(e.ppid, [pid]);
|
|
659
|
-
}
|
|
660
|
-
// Step A — witness lineage: record every descendant of each ALIVE tracked
|
|
661
|
-
// thread while its subtree chain is still intact. Excludes the root pid itself
|
|
662
|
-
// (that thread is managed by the worker-cleanup path, not here).
|
|
663
|
-
for (const t of trackedThreads) {
|
|
664
|
-
if (!t.alive || !snapshot.has(t.pid))
|
|
665
|
-
continue;
|
|
666
|
-
const seen = new Set([t.pid]);
|
|
667
|
-
const stack = [...(children.get(t.pid) ?? [])];
|
|
668
|
-
while (stack.length) {
|
|
669
|
-
const d = stack.pop();
|
|
670
|
-
if (seen.has(d))
|
|
671
|
-
continue; // cycle / pid-reuse loop guard
|
|
672
|
-
seen.add(d);
|
|
673
|
-
const e = snapshot.get(d);
|
|
674
|
-
if (!e)
|
|
675
|
-
continue;
|
|
676
|
-
lineage.set(d, {
|
|
677
|
-
ownerThreadId: t.threadId,
|
|
678
|
-
ownerPid: t.pid,
|
|
679
|
-
ownerStartedAt: t.startedAt,
|
|
680
|
-
descendantStartMs: e.startMs,
|
|
681
|
-
});
|
|
682
|
-
result.recorded++;
|
|
683
|
-
for (const c of children.get(d) ?? [])
|
|
684
|
-
if (!seen.has(c))
|
|
685
|
-
stack.push(c);
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
// Step B — evaluate: a recorded descendant whose owner is no longer alive is an
|
|
689
|
-
// orphan candidate, subject to both pid-reuse guards.
|
|
690
|
-
for (const [pid, entry] of [...lineage]) {
|
|
691
|
-
// Prune a descendant that has left the process table FIRST, before the
|
|
692
|
-
// owner-liveness check — otherwise a dead descendant of a still-live owner is
|
|
693
|
-
// never evaluated here (its owner short-circuits below) and never re-recorded
|
|
694
|
-
// by Step A (it is gone), so it lingers in `lineage` for the owner's entire
|
|
695
|
-
// lifetime. A long-lived thread that churns many short-lived children (e.g. a
|
|
696
|
-
// claude-agent spawning a bash.exe per hook) would grow the map without bound.
|
|
697
|
-
const cur = snapshot.get(pid);
|
|
698
|
-
if (!cur) {
|
|
699
|
-
lineage.delete(pid); // descendant already gone — nothing to reap
|
|
700
|
-
continue;
|
|
701
|
-
}
|
|
702
|
-
if (aliveOwners.has(entry.ownerThreadId))
|
|
703
|
-
continue; // owner still alive → not orphaned
|
|
704
|
-
// Never touch a pid managed elsewhere (a live tracked thread, the server, us).
|
|
705
|
-
if (protectedPids.has(pid))
|
|
706
|
-
continue;
|
|
707
|
-
// GUARD 1 — same-process: a recycled pid now hosts a different process whose
|
|
708
|
-
// creation time differs from the one we recorded. Drop it; do NOT kill.
|
|
709
|
-
if (Math.abs(cur.startMs - entry.descendantStartMs) > toleranceMs) {
|
|
710
|
-
lineage.delete(pid);
|
|
711
|
-
result.reusedPidsSkipped.push(pid);
|
|
712
|
-
continue;
|
|
713
|
-
}
|
|
714
|
-
// GUARD 2 — a child cannot predate its parent. If this pid started before the
|
|
715
|
-
// owning thread did, it is not really our descendant (recorded owner is wrong
|
|
716
|
-
// or the pid was reused across the boundary). Keep it for logging, never kill.
|
|
717
|
-
if (entry.ownerStartedAt !== undefined && cur.startMs < entry.ownerStartedAt - toleranceMs) {
|
|
718
|
-
result.olderThanParentSkipped.push(pid);
|
|
719
|
-
continue;
|
|
720
|
-
}
|
|
721
|
-
result.orphans.push({ pid, ownerThreadId: entry.ownerThreadId, startMs: cur.startMs });
|
|
722
|
-
}
|
|
723
|
-
return result;
|
|
724
|
-
}
|
|
725
|
-
/** True when the sweep is allowed to actually kill confirmed orphans. Default is
|
|
726
|
-
* log-only (see header): a week of real candidate logs comes first. */
|
|
727
|
-
function orphanSweepWillKill() {
|
|
728
|
-
const v = process.env.ORPHAN_SWEEP_KILL;
|
|
729
|
-
return v === "1" || v === "true";
|
|
730
|
-
}
|
|
731
346
|
/**
|
|
732
|
-
*
|
|
733
|
-
*
|
|
734
|
-
*
|
|
735
|
-
*
|
|
736
|
-
*
|
|
347
|
+
* Kill the process tree rooted at `pid` via the supervisor's broker (native
|
|
348
|
+
* TerminateProcess over the whole tree, identity-checked when `spawnedAtMs` is
|
|
349
|
+
* given, outcome recorded in the broker's on-disk registry). Never spawns a
|
|
350
|
+
* child process and never blocks the event loop.
|
|
351
|
+
*
|
|
352
|
+
* Broker unreachable → degraded best-effort: one native `process.kill(pid)`
|
|
353
|
+
* (root only — Node cannot enumerate a tree without shelling out, and that
|
|
354
|
+
* machinery is gone by design) followed by a conservative liveness re-check.
|
|
355
|
+
* A survived kill keeps the PID file so the keeper/cleanup policy loops retry
|
|
356
|
+
* and escalate — same contract as before.
|
|
737
357
|
*/
|
|
738
|
-
export async function
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
358
|
+
export async function killProcessTree(pid, threadId, caller, spawnedAtMs) {
|
|
359
|
+
// Self-protection: NEVER tree-kill our own process or the authoritative
|
|
360
|
+
// server PID (a recycled worker PID could point at us — killing it would be
|
|
361
|
+
// the traceless server death this layer exists to prevent). The broker has
|
|
362
|
+
// the same guard server-side; this is defense in depth.
|
|
363
|
+
if (pid === process.pid) {
|
|
364
|
+
log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
365
|
+
return;
|
|
743
366
|
}
|
|
744
|
-
const trackedThreads = readPidFiles().map((e) => ({
|
|
745
|
-
threadId: e.threadId,
|
|
746
|
-
pid: e.pid,
|
|
747
|
-
startedAt: e.startedAt,
|
|
748
|
-
alive: isProcessAlive(e.pid, e.startedAt) && isSameProcess(e.pid, e.startedAt),
|
|
749
|
-
}));
|
|
750
|
-
// Protected: every live tracked-thread pid (managed by worker-cleanup), the
|
|
751
|
-
// authoritative server pid, and our own pid — these are never orphan targets.
|
|
752
|
-
const protectedPids = new Set([process.pid]);
|
|
753
367
|
const serverPid = readServerPid();
|
|
754
|
-
if (serverPid !== undefined)
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
const result =
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
toleranceMs: PID_IDENTITY_TOLERANCE_MS,
|
|
368
|
+
if (serverPid !== undefined && pid === serverPid) {
|
|
369
|
+
log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
|
|
373
|
+
const result = await brokerKillTree(pid, {
|
|
374
|
+
threadId,
|
|
375
|
+
killedBy: caller ?? "mcp",
|
|
376
|
+
reason: caller,
|
|
377
|
+
startedAtMs: spawnedAtMs,
|
|
765
378
|
});
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
log.warn(`[
|
|
379
|
+
let stillAlive;
|
|
380
|
+
if (result !== null) {
|
|
381
|
+
if (result.refused)
|
|
382
|
+
log.error(`[process] broker REFUSED kill of pid ${pid}: ${result.error ?? "protected pid"}`);
|
|
383
|
+
else if (result.identityMismatch)
|
|
384
|
+
log.warn(`[process] broker: pid ${pid} was recycled (identity mismatch) — our process is already gone, nothing killed`);
|
|
385
|
+
stillAlive = result.alive;
|
|
386
|
+
}
|
|
387
|
+
else {
|
|
388
|
+
// Degraded path (no supervisor): best-effort native root kill, then re-check.
|
|
389
|
+
log.warn(`[process] broker unreachable for kill of pid ${pid} (thread ${threadId}) — degraded native root-only kill`);
|
|
777
390
|
try {
|
|
778
|
-
|
|
391
|
+
process.kill(pid);
|
|
779
392
|
}
|
|
780
393
|
catch (err) {
|
|
781
|
-
log.
|
|
782
|
-
}
|
|
783
|
-
finally {
|
|
784
|
-
orphanLineage.delete(orphan.pid);
|
|
394
|
+
log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
|
|
785
395
|
}
|
|
396
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
397
|
+
stillAlive = await isProcessAlive(pid, spawnedAtMs);
|
|
786
398
|
}
|
|
787
|
-
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
// the whole /T tree down. A short execFile timeout aborts taskkill mid-kill and
|
|
792
|
-
// surfaces as "Command failed" with no stderr, so the process survives and the
|
|
793
|
-
// keeper wrongly flags it UNKILLABLE. Give taskkill headroom to finish; the
|
|
794
|
-
// caller still verifies liveness afterwards, so a fast kill returns immediately.
|
|
795
|
-
const TASKKILL_TIMEOUT_MS = 60_000;
|
|
796
|
-
export function killProcessTree(pid, threadId, caller, spawnedAtMs) {
|
|
797
|
-
return new Promise((resolve) => {
|
|
798
|
-
// Self-protection: NEVER tree-kill our own process or the authoritative
|
|
799
|
-
// server PID. The leading hypothesis for the traceless server death is a
|
|
800
|
-
// keeper/worker-reaper tree-killing a RECYCLED worker PID that the OS
|
|
801
|
-
// reassigned to the node server (or one of its ancestors). `taskkill /T`
|
|
802
|
-
// would then take the whole server tree down with no signal, no [fatal] and
|
|
803
|
-
// no crash dump — exactly the observed symptom. Refuse and log loudly.
|
|
804
|
-
if (pid === process.pid) {
|
|
805
|
-
log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
806
|
-
resolve();
|
|
807
|
-
return;
|
|
808
|
-
}
|
|
809
|
-
const serverPid = readServerPid();
|
|
810
|
-
if (serverPid !== undefined && pid === serverPid) {
|
|
811
|
-
log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
812
|
-
resolve();
|
|
813
|
-
return;
|
|
814
|
-
}
|
|
815
|
-
log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
|
|
816
|
-
const cleanupPidFile = () => {
|
|
817
|
-
const pidFile = join(PROCESS_PIDS_DIR, `${threadId}.pid`);
|
|
818
|
-
try {
|
|
819
|
-
unlinkSync(pidFile);
|
|
820
|
-
}
|
|
821
|
-
catch {
|
|
822
|
-
/* PID file already gone (exit handler may have raced us) */
|
|
823
|
-
}
|
|
824
|
-
};
|
|
825
|
-
const verifyDead = (attempt) => {
|
|
826
|
-
// spawnedAtMs matters here for orphan-kill callers reaping FRESH pids: a
|
|
827
|
-
// failed kill must not let a pre-spawn snapshot falsely confirm death and
|
|
828
|
-
// unlink a live process's pid file. A genuinely dead child is usually
|
|
829
|
-
// caught by the kill(0) ESRCH probe or a post-spawn snapshot refresh; if
|
|
830
|
-
// neither lands within the 3 attempts (refresh in flight / breaker open),
|
|
831
|
-
// we resolve as "still alive" and KEEP the pid file — conservative, and a
|
|
832
|
-
// later sweep finishes the cleanup against a fresher snapshot.
|
|
833
|
-
if (!isProcessAlive(pid, spawnedAtMs)) {
|
|
834
|
-
log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
|
|
835
|
-
cleanupPidFile();
|
|
836
|
-
resolve();
|
|
837
|
-
return;
|
|
838
|
-
}
|
|
839
|
-
if (attempt >= 3) {
|
|
840
|
-
log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after ${attempt} kill attempts — NOT removing PID file`);
|
|
841
|
-
resolve();
|
|
842
|
-
return;
|
|
843
|
-
}
|
|
844
|
-
// Retry kill after a short delay
|
|
845
|
-
log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt ${attempt} — retrying`);
|
|
846
|
-
if (process.platform === "win32") {
|
|
847
|
-
// Escalation, retry-path only: the happy-path in-process `taskkill /F /T`
|
|
848
|
-
// already failed to verify-dead. If the failure was EPERM because the
|
|
849
|
-
// server's own tree is job-object-constrained / I/O-wedged, no in-process
|
|
850
|
-
// kill will ever succeed — so first spawn a DETACHED, fresh sibling
|
|
851
|
-
// taskkill that does not inherit our wedged handles.
|
|
852
|
-
spawnDetachedKillerWin(pid);
|
|
853
|
-
// Native TerminateProcess (via libuv) as well as taskkill — they take
|
|
854
|
-
// different paths (taskkill is an external process with its own token;
|
|
855
|
-
// process.kill issues the syscall directly from this process), so one
|
|
856
|
-
// can succeed where the other fails on a stubborn process.
|
|
857
|
-
try {
|
|
858
|
-
process.kill(pid);
|
|
859
|
-
}
|
|
860
|
-
catch (err) {
|
|
861
|
-
log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
|
|
862
|
-
}
|
|
863
|
-
// `taskkill /T` walks the tree top-down and can stall when a hung child
|
|
864
|
-
// blocks the parent's termination — the exact case where the root sat
|
|
865
|
-
// alive for hours. Kill each descendant leaves-first so a wedged child is
|
|
866
|
-
// reaped before its parent, freeing the root to die. Logged at info (from
|
|
867
|
-
// entry through kill count) so a week of logs can answer whether this
|
|
868
|
-
// escalation path ever actually runs, or is dead weight.
|
|
869
|
-
log.info(`[process] leaf-first kill ESCALATION entered for pid=${pid} thread=${threadId} (attempt ${attempt}) — taskkill /F /T did not verify-dead`);
|
|
870
|
-
getWindowsProcessTreeLeafFirst(pid, threadId, (tree) => {
|
|
871
|
-
killPidsSequentiallyWin(tree, (killed) => {
|
|
872
|
-
log.info(`[process] leaf-first escalation for pid=${pid} thread=${threadId}: issued taskkill to ${tree.length} process(es), ${killed} reported killed`);
|
|
873
|
-
setTimeout(() => verifyDead(attempt + 1), 1000);
|
|
874
|
-
});
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
else {
|
|
878
|
-
try {
|
|
879
|
-
process.kill(pid, "SIGKILL");
|
|
880
|
-
}
|
|
881
|
-
catch {
|
|
882
|
-
/* process already gone (ESRCH) — verifyDead confirms */
|
|
883
|
-
}
|
|
884
|
-
setTimeout(() => verifyDead(attempt + 1), 1000);
|
|
885
|
-
}
|
|
886
|
-
};
|
|
887
|
-
if (process.platform === "win32") {
|
|
888
|
-
execFile("taskkill", ["/F", "/T", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
|
|
889
|
-
if (err) {
|
|
890
|
-
log.warn(`[process] Kill process ${pid} (thread ${threadId}) failed: ${errorMessage(err)}`);
|
|
891
|
-
}
|
|
892
|
-
// Always verify — taskkill can report success but process lingers,
|
|
893
|
-
// or report failure but process is actually dead.
|
|
894
|
-
setTimeout(() => verifyDead(1), 500);
|
|
895
|
-
});
|
|
399
|
+
if (!stillAlive) {
|
|
400
|
+
log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
|
|
401
|
+
try {
|
|
402
|
+
unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
|
|
896
403
|
}
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
process.kill(pid, "SIGTERM");
|
|
900
|
-
}
|
|
901
|
-
catch (err) {
|
|
902
|
-
log.debug(`[process] Kill process ${pid} (thread ${threadId}): ${errorMessage(err)}`);
|
|
903
|
-
}
|
|
904
|
-
setTimeout(() => verifyDead(1), 500);
|
|
404
|
+
catch {
|
|
405
|
+
/* PID file already gone (exit handler may have raced us) */
|
|
905
406
|
}
|
|
906
|
-
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
// Conservative: a survived kill keeps the PID file so a later sweep retries
|
|
410
|
+
// against fresh evidence (keeper/worker-cleanup own the escalation policy).
|
|
411
|
+
log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt — NOT removing PID file`);
|
|
907
412
|
}
|
|
908
413
|
//# sourceMappingURL=process.service.js.map
|