sensorium-mcp 3.1.0-refactor.9 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/agent-settings.d.ts +6 -0
- package/dist/config/agent-settings.d.ts.map +1 -1
- package/dist/config/agent-settings.js +22 -0
- package/dist/config/agent-settings.js.map +1 -1
- package/dist/config/detail-grant.d.ts +39 -0
- package/dist/config/detail-grant.d.ts.map +1 -0
- package/dist/config/detail-grant.js +61 -0
- package/dist/config/detail-grant.js.map +1 -0
- package/dist/config/index.d.ts +2 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +5 -3
- package/dist/config/index.js.map +1 -1
- package/dist/config/misc-settings.d.ts +2 -0
- package/dist/config/misc-settings.d.ts.map +1 -1
- package/dist/config/misc-settings.js +22 -0
- package/dist/config/misc-settings.js.map +1 -1
- package/dist/config/settings-store.d.ts +41 -2
- package/dist/config/settings-store.d.ts.map +1 -1
- package/dist/config/settings-store.js +151 -14
- package/dist/config/settings-store.js.map +1 -1
- package/dist/dashboard/routes/copilot.d.ts.map +1 -1
- package/dist/dashboard/routes/copilot.js +17 -1
- package/dist/dashboard/routes/copilot.js.map +1 -1
- package/dist/dashboard/routes/settings.d.ts +2 -0
- package/dist/dashboard/routes/settings.d.ts.map +1 -1
- package/dist/dashboard/routes/settings.js +25 -1
- package/dist/dashboard/routes/settings.js.map +1 -1
- package/dist/dashboard/routes/threads-read.d.ts.map +1 -1
- package/dist/dashboard/routes/threads-read.js +15 -3
- package/dist/dashboard/routes/threads-read.js.map +1 -1
- package/dist/dashboard/routes/threads.d.ts.map +1 -1
- package/dist/dashboard/routes/threads.js +13 -4
- package/dist/dashboard/routes/threads.js.map +1 -1
- package/dist/dashboard/routes.d.ts.map +1 -1
- package/dist/dashboard/routes.js +14 -5
- package/dist/dashboard/routes.js.map +1 -1
- package/dist/dashboard/spa.html +11 -11
- package/dist/data/daily-marker.js +3 -3
- package/dist/data/daily-marker.js.map +1 -1
- package/dist/data/file-storage.d.ts.map +1 -1
- package/dist/data/file-storage.js +3 -3
- package/dist/data/file-storage.js.map +1 -1
- package/dist/data/memory/bootstrap.d.ts.map +1 -1
- package/dist/data/memory/bootstrap.js +2 -3
- package/dist/data/memory/bootstrap.js.map +1 -1
- package/dist/data/memory/migration-runner.d.ts.map +1 -1
- package/dist/data/memory/migration-runner.js +4 -7
- package/dist/data/memory/migration-runner.js.map +1 -1
- package/dist/data/memory/schema.js +2 -2
- package/dist/data/memory/schema.js.map +1 -1
- package/dist/index.js +67 -21
- package/dist/index.js.map +1 -1
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/logger.js +21 -2
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/paths.d.ts +29 -0
- package/dist/lib/paths.d.ts.map +1 -0
- package/dist/lib/paths.js +36 -0
- package/dist/lib/paths.js.map +1 -0
- package/dist/lib/types.d.ts +1 -0
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/server/factory.js +16 -6
- package/dist/server/factory.js.map +1 -1
- package/dist/server/http-server.d.ts.map +1 -1
- package/dist/server/http-server.js +52 -20
- package/dist/server/http-server.js.map +1 -1
- package/dist/services/agent-spawn.service.d.ts +3 -3
- package/dist/services/agent-spawn.service.d.ts.map +1 -1
- package/dist/services/agent-spawn.service.js +30 -9
- package/dist/services/agent-spawn.service.js.map +1 -1
- package/dist/services/background-runner.d.ts.map +1 -1
- package/dist/services/background-runner.js +22 -12
- package/dist/services/background-runner.js.map +1 -1
- package/dist/services/broker-client.d.ts +209 -0
- package/dist/services/broker-client.d.ts.map +1 -0
- package/dist/services/broker-client.js +359 -0
- package/dist/services/broker-client.js.map +1 -0
- package/dist/services/claude-spawn.service.d.ts +1 -1
- package/dist/services/claude-spawn.service.d.ts.map +1 -1
- package/dist/services/claude-spawn.service.js +25 -25
- package/dist/services/claude-spawn.service.js.map +1 -1
- package/dist/services/codex-spawn.service.d.ts +1 -1
- package/dist/services/codex-spawn.service.d.ts.map +1 -1
- package/dist/services/codex-spawn.service.js +19 -24
- package/dist/services/codex-spawn.service.js.map +1 -1
- package/dist/services/copilot-spawn.service.d.ts +1 -1
- package/dist/services/copilot-spawn.service.d.ts.map +1 -1
- package/dist/services/copilot-spawn.service.js +86 -37
- package/dist/services/copilot-spawn.service.js.map +1 -1
- package/dist/services/dispatcher/broker.d.ts +4 -3
- package/dist/services/dispatcher/broker.d.ts.map +1 -1
- package/dist/services/dispatcher/broker.js +23 -11
- package/dist/services/dispatcher/broker.js.map +1 -1
- package/dist/services/dispatcher/lock.js +2 -2
- package/dist/services/dispatcher/lock.js.map +1 -1
- package/dist/services/dispatcher/poller.d.ts.map +1 -1
- package/dist/services/dispatcher/poller.js +24 -5
- package/dist/services/dispatcher/poller.js.map +1 -1
- package/dist/services/flight-recorder.service.d.ts +37 -0
- package/dist/services/flight-recorder.service.d.ts.map +1 -0
- package/dist/services/flight-recorder.service.js +210 -0
- package/dist/services/flight-recorder.service.js.map +1 -0
- package/dist/services/keeper.service.d.ts +9 -0
- package/dist/services/keeper.service.d.ts.map +1 -1
- package/dist/services/keeper.service.js +198 -39
- package/dist/services/keeper.service.js.map +1 -1
- package/dist/services/maintenance-signal.d.ts.map +1 -1
- package/dist/services/maintenance-signal.js +2 -3
- package/dist/services/maintenance-signal.js.map +1 -1
- package/dist/services/process.service.d.ts +76 -61
- package/dist/services/process.service.d.ts.map +1 -1
- package/dist/services/process.service.js +224 -608
- package/dist/services/process.service.js.map +1 -1
- package/dist/services/reconnect-snapshot.service.js +2 -2
- package/dist/services/reconnect-snapshot.service.js.map +1 -1
- package/dist/services/scheduler.d.ts +1 -0
- package/dist/services/scheduler.d.ts.map +1 -1
- package/dist/services/scheduler.js +69 -14
- package/dist/services/scheduler.js.map +1 -1
- package/dist/services/self-update.service.d.ts +113 -15
- package/dist/services/self-update.service.d.ts.map +1 -1
- package/dist/services/self-update.service.js +382 -109
- package/dist/services/self-update.service.js.map +1 -1
- package/dist/services/spawn-common.d.ts +49 -23
- package/dist/services/spawn-common.d.ts.map +1 -1
- package/dist/services/spawn-common.js +262 -47
- package/dist/services/spawn-common.js.map +1 -1
- package/dist/services/spawn-exit-watcher.d.ts +43 -0
- package/dist/services/spawn-exit-watcher.d.ts.map +1 -0
- package/dist/services/spawn-exit-watcher.js +127 -0
- package/dist/services/spawn-exit-watcher.js.map +1 -0
- package/dist/services/telegram.d.ts +3 -2
- package/dist/services/telegram.d.ts.map +1 -1
- package/dist/services/telegram.js +8 -5
- package/dist/services/telegram.js.map +1 -1
- package/dist/services/thread-health.service.d.ts +1 -1
- package/dist/services/thread-health.service.d.ts.map +1 -1
- package/dist/services/thread-health.service.js +11 -7
- package/dist/services/thread-health.service.js.map +1 -1
- package/dist/services/thread-lifecycle.service.js +2 -2
- package/dist/services/thread-lifecycle.service.js.map +1 -1
- package/dist/services/topic.service.d.ts +1 -1
- package/dist/services/topic.service.d.ts.map +1 -1
- package/dist/services/topic.service.js +3 -2
- package/dist/services/topic.service.js.map +1 -1
- package/dist/services/worker-cleanup.service.d.ts.map +1 -1
- package/dist/services/worker-cleanup.service.js +25 -99
- package/dist/services/worker-cleanup.service.js.map +1 -1
- package/dist/test-bootstrap.d.ts +39 -0
- package/dist/test-bootstrap.d.ts.map +1 -0
- package/dist/test-bootstrap.js +78 -0
- package/dist/test-bootstrap.js.map +1 -0
- package/dist/tools/defs/session-defs.js +1 -1
- package/dist/tools/defs/session-defs.js.map +1 -1
- package/dist/tools/defs/utility-defs.d.ts.map +1 -1
- package/dist/tools/defs/utility-defs.js +4 -3
- package/dist/tools/defs/utility-defs.js.map +1 -1
- package/dist/tools/delegate-tool.d.ts +1 -1
- package/dist/tools/delegate-tool.d.ts.map +1 -1
- package/dist/tools/delegate-tool.js +37 -6
- package/dist/tools/delegate-tool.js.map +1 -1
- package/dist/tools/session-tools.d.ts.map +1 -1
- package/dist/tools/session-tools.js +34 -20
- package/dist/tools/session-tools.js.map +1 -1
- package/dist/tools/utility-tools.js +1 -1
- package/dist/tools/utility-tools.js.map +1 -1
- package/dist/tools/wait/context.d.ts +13 -0
- package/dist/tools/wait/context.d.ts.map +1 -1
- package/dist/tools/wait/message-delivery.d.ts.map +1 -1
- package/dist/tools/wait/message-delivery.js +2 -3
- package/dist/tools/wait/message-delivery.js.map +1 -1
- package/dist/tools/wait/poll-loop.d.ts.map +1 -1
- package/dist/tools/wait/poll-loop.js +8 -1
- package/dist/tools/wait/poll-loop.js.map +1 -1
- package/package.json +2 -2
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { execFile, spawn } from "node:child_process";
|
|
2
1
|
import { mkdirSync, readdirSync, readFileSync, statSync, unlinkSync } from "node:fs";
|
|
3
|
-
import { homedir } from "node:os";
|
|
4
2
|
import { join } from "node:path";
|
|
5
3
|
import { log } from "../lib/logger.js";
|
|
4
|
+
import { dataDir } from "../lib/paths.js";
|
|
6
5
|
import { errorMessage } from "../lib/utils.js";
|
|
7
6
|
import { getAllThreads } from "../data/memory/thread-registry.js";
|
|
8
7
|
import { getDashboardSessions } from "../sessions/dashboard-registry.js";
|
|
9
|
-
|
|
8
|
+
import { brokerAdopt, brokerCheckAlive, brokerCheckAliveByThread, brokerKillTree, } from "./broker-client.js";
|
|
9
|
+
const BASE_DIR = dataDir();
|
|
10
10
|
const LOGS_DIR = join(BASE_DIR, "logs");
|
|
11
11
|
const PIDS_DIR = join(BASE_DIR, "pids");
|
|
12
12
|
const SERVER_PID_FILE = join(BASE_DIR, "server.pid");
|
|
@@ -59,294 +59,67 @@ export function deletePendingTaskMailbox(threadId) {
|
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
|
-
/**
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
62
|
+
/**
|
|
63
|
+
* Liveness + identity in one question: is `pid` alive AND (when `spawnedAtMs`
|
|
64
|
+
* is given) still the process we spawned?
|
|
65
|
+
*
|
|
66
|
+
* Two-step check, neither of which ever spawns a child process:
|
|
67
|
+
* 1. `process.kill(pid, 0)` — a plain syscall. ESRCH (any non-EPERM error) is
|
|
68
|
+
* a definitive "dead" and needs no broker (this also keeps dev mode
|
|
69
|
+
* without a supervisor fully functional for the common case).
|
|
70
|
+
* 2. Otherwise the answer is NOT trustworthy on Windows — kill(0) succeeds
|
|
71
|
+
* for kernel-zombie PIDs held open by a lingering handle, and EPERM is
|
|
72
|
+
* what a parent sees for its own terminated child. Ask the supervisor's
|
|
73
|
+
* process broker, which answers from a native process snapshot and also
|
|
74
|
+
* verifies creation-time identity (PID-reuse guard).
|
|
75
|
+
*
|
|
76
|
+
* Broker unreachable → "cannot confirm → treat alive", exactly the audited-safe
|
|
77
|
+
* degradation the deleted tasklist/Get-Process breakers had.
|
|
78
|
+
*/
|
|
79
|
+
export async function isProcessAlive(pid, spawnedAtMs) {
|
|
68
80
|
try {
|
|
69
81
|
process.kill(pid, 0);
|
|
70
82
|
}
|
|
71
83
|
catch (err) {
|
|
72
|
-
// ESRCH (or any non-EPERM error) = no such process → dead. EPERM = kill(0)
|
|
73
|
-
// was denied; do NOT trust that as alive — fall through to the win32 table
|
|
74
|
-
// check below (see next comment for why). On POSIX, EPERM does mean a live
|
|
75
|
-
// process owned by another user, so the fall-through returns true there.
|
|
76
84
|
if (err.code !== "EPERM")
|
|
77
85
|
return false;
|
|
78
86
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
// - kill(0) succeeds for a process that already TERMINATED but whose PID is
|
|
82
|
-
// still held open by a lingering handle — a kernel "zombie".
|
|
83
|
-
// - OpenProcess from the PARENT of such a dead child returns ACCESS_DENIED
|
|
84
|
-
// (→EPERM) for that same terminated PID (a non-parent instead sees ESRCH).
|
|
85
|
-
// Both cases wedged the keeper into treating an unkillable zombie as alive
|
|
86
|
-
// forever, never restarting the thread. The process table excludes terminated
|
|
87
|
-
// PIDs even while their handle lingers, so make it authoritative for both
|
|
88
|
-
// paths. (tasklist unavailable → isPidInWindowsTable conservatively returns
|
|
89
|
-
// true, so we never declare a genuinely live process dead.)
|
|
90
|
-
if (process.platform === "win32")
|
|
91
|
-
return isPidInWindowsTable(pid, spawnedAtMs);
|
|
92
|
-
return true;
|
|
93
|
-
}
|
|
94
|
-
const WIN_PID_SNAPSHOT_TTL_MS = 2000;
|
|
95
|
-
// A snapshot older than this is discarded rather than trusted: a very stale pid
|
|
96
|
-
// set would mark freshly-spawned processes dead (they're absent from it), and a
|
|
97
|
-
// stale start-time map is merely incomplete. Normal refreshes complete in well
|
|
98
|
-
// under a second, so this bound is only reached when refreshes keep failing —
|
|
99
|
-
// at which point the getters degrade to null (= "can't confirm" → alive).
|
|
100
|
-
const WIN_SNAPSHOT_MAX_STALE_MS = 15_000;
|
|
101
|
-
// Circuit breaker for the snapshot commands. When the OS throttles external
|
|
102
|
-
// process spawns (battery / Energy Saver / EDR), tasklist and powershell hang
|
|
103
|
-
// their full 5s timeout on EVERY call. The former synchronous snapshots then
|
|
104
|
-
// froze the event loop for 5s at a time, so newly-spawned agent children could
|
|
105
|
-
// not complete the MCP initialize/tools-list handshake and started TOOLLESS —
|
|
106
|
-
// then exited cleanly and were respawned forever (the 2026-07-17 respawn storm,
|
|
107
|
-
// see implementation_plans/2026-07-18_toolless-respawn-storm.md). Refreshes are
|
|
108
|
-
// now async (never on the event loop) and, after repeated failures, paused
|
|
109
|
-
// entirely for a cooldown instead of hammering a throttled OS every sweep.
|
|
110
|
-
const SNAPSHOT_BREAKER_THRESHOLD = 3;
|
|
111
|
-
const SNAPSHOT_BREAKER_COOLDOWN_MS = 60_000;
|
|
112
|
-
function makeSnapshotBreaker(label) {
|
|
113
|
-
let consecutiveFailures = 0;
|
|
114
|
-
let openUntil = 0;
|
|
115
|
-
return {
|
|
116
|
-
isOpen: () => Date.now() < openUntil,
|
|
117
|
-
recordSuccess: () => {
|
|
118
|
-
consecutiveFailures = 0;
|
|
119
|
-
},
|
|
120
|
-
recordFailure: (err) => {
|
|
121
|
-
consecutiveFailures++;
|
|
122
|
-
if (consecutiveFailures >= SNAPSHOT_BREAKER_THRESHOLD) {
|
|
123
|
-
openUntil = Date.now() + SNAPSHOT_BREAKER_COOLDOWN_MS;
|
|
124
|
-
// Reset on open so re-opening after the cooldown requires THRESHOLD
|
|
125
|
-
// fresh failures — one lingering failure must not instantly re-trip.
|
|
126
|
-
consecutiveFailures = 0;
|
|
127
|
-
log.warn(`[process] ${label} snapshot failed ${SNAPSHOT_BREAKER_THRESHOLD}x consecutively (${errorMessage(err)}) — pausing refreshes for ${SNAPSHOT_BREAKER_COOLDOWN_MS / 1000}s`);
|
|
128
|
-
}
|
|
129
|
-
else {
|
|
130
|
-
log.debug(`[process] ${label} snapshot failed: ${errorMessage(err)}`);
|
|
131
|
-
}
|
|
132
|
-
},
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
/** Authoritative Windows liveness check via the process table, which — unlike
|
|
136
|
-
* OpenProcess/process.kill(0) — does NOT list a terminated process whose PID is
|
|
137
|
-
* still held by a lingering handle. One `tasklist` snapshot of ALL pids serves
|
|
138
|
-
* every pid in a sweep, and it is refreshed asynchronously in the background:
|
|
139
|
-
* the getter reads the cached snapshot and NEVER blocks the event loop (a 5s
|
|
140
|
-
* synchronous hang here is what starved MCP handshakes and caused the toolless
|
|
141
|
-
* respawn storm). Callers treat null as "can't confirm dead" → alive. */
|
|
142
|
-
let winPidSnapshot = null;
|
|
143
|
-
let winPidRefreshInFlight = false;
|
|
144
|
-
const pidSnapshotBreaker = makeSnapshotBreaker("tasklist");
|
|
145
|
-
function refreshWindowsPidSet() {
|
|
146
|
-
if (winPidRefreshInFlight || pidSnapshotBreaker.isOpen())
|
|
147
|
-
return;
|
|
148
|
-
winPidRefreshInFlight = true;
|
|
149
|
-
// The snapshot's ts is the REFRESH START: the process table is read at (or
|
|
150
|
-
// after) this instant, so any pid spawned later can legitimately be absent.
|
|
151
|
-
const refreshStartedAt = Date.now();
|
|
152
|
-
// /NH = no header, /FO CSV = quoted columns; PID is column 2: "image","PID",...
|
|
153
|
-
execFile("tasklist", ["/NH", "/FO", "CSV"], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
154
|
-
winPidRefreshInFlight = false;
|
|
155
|
-
if (err) {
|
|
156
|
-
pidSnapshotBreaker.recordFailure(err);
|
|
157
|
-
return;
|
|
158
|
-
}
|
|
159
|
-
pidSnapshotBreaker.recordSuccess();
|
|
160
|
-
const pids = new Set();
|
|
161
|
-
for (const m of out.matchAll(/^"[^"]*","(\d+)"/gm))
|
|
162
|
-
pids.add(Number(m[1]));
|
|
163
|
-
winPidSnapshot = { pids, ts: refreshStartedAt };
|
|
164
|
-
});
|
|
165
|
-
}
|
|
166
|
-
function getWindowsPidSet() {
|
|
167
|
-
const now = Date.now();
|
|
168
|
-
if (!winPidSnapshot || now - winPidSnapshot.ts >= WIN_PID_SNAPSHOT_TTL_MS)
|
|
169
|
-
refreshWindowsPidSet();
|
|
170
|
-
if (winPidSnapshot && now - winPidSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
|
|
171
|
-
return winPidSnapshot.pids;
|
|
172
|
-
return null; // no usable snapshot — conservative: never declare a real process dead
|
|
173
|
-
}
|
|
174
|
-
function isPidInWindowsTable(pid, spawnedAtMs) {
|
|
175
|
-
const pids = getWindowsPidSet();
|
|
176
|
-
if (pids === null)
|
|
177
|
-
return true; // tasklist unavailable — never declare a real process dead
|
|
178
|
-
if (pids.has(pid))
|
|
179
|
-
return true;
|
|
180
|
-
// Absent from the table — but if the process was spawned AFTER the snapshot
|
|
181
|
-
// was taken, the absence proves nothing (the table simply predates it).
|
|
182
|
-
// Treat as alive; the refresh already triggered above will testify shortly.
|
|
183
|
-
return spawnedAtMs !== undefined && spawnedAtMs >= winPidSnapshot.ts;
|
|
184
|
-
}
|
|
185
|
-
// Tolerance between the pid-file's recorded startedAt (Date.now() at spawn) and
|
|
186
|
-
// the OS process creation time. Wide enough to absorb spawn latency and a wall-
|
|
187
|
-
// clock step (NTP/VM resume) between the two readings, yet far below the minutes-
|
|
188
|
-
// to-hours gap a recycled PID produces — so PID reuse is still caught.
|
|
189
|
-
const PID_IDENTITY_TOLERANCE_MS = 60_000;
|
|
190
|
-
const WIN_START_SNAPSHOT_TTL_MS = 2000;
|
|
191
|
-
/** One powershell snapshot of every pid's creation time (Unix ms), reused for
|
|
192
|
-
* all pids in a keeper sweep and refreshed asynchronously in the background —
|
|
193
|
-
* same non-blocking pattern as the tasklist snapshot above. Staleness is safe
|
|
194
|
-
* here: a pid's start time never changes, so an old map is merely missing new
|
|
195
|
-
* pids, and a missing entry already means "can't tell" → trust PID. StartTime
|
|
196
|
-
* is unreadable for some protected processes; those are simply omitted. */
|
|
197
|
-
let winStartSnapshot = null;
|
|
198
|
-
let winStartRefreshInFlight = false;
|
|
199
|
-
const startSnapshotBreaker = makeSnapshotBreaker("start-time");
|
|
200
|
-
function refreshWindowsStartTimeMap() {
|
|
201
|
-
if (winStartRefreshInFlight || startSnapshotBreaker.isOpen())
|
|
202
|
-
return;
|
|
203
|
-
winStartRefreshInFlight = true;
|
|
204
|
-
const refreshStartedAt = Date.now();
|
|
205
|
-
execFile("powershell", [
|
|
206
|
-
"-NoProfile",
|
|
207
|
-
"-Command",
|
|
208
|
-
'Get-Process | ForEach-Object { try { "$($_.Id) $($_.StartTime.ToUniversalTime().Ticks)" } catch {} }',
|
|
209
|
-
], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
210
|
-
winStartRefreshInFlight = false;
|
|
211
|
-
if (err) {
|
|
212
|
-
startSnapshotBreaker.recordFailure(err);
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
|
-
startSnapshotBreaker.recordSuccess();
|
|
216
|
-
const starts = new Map();
|
|
217
|
-
for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
|
|
218
|
-
starts.set(Number(m[1]), Math.round(Number(m[2]) / 10000 - 62135596800000)); // .NET ticks (100ns from year 1) → Unix ms
|
|
219
|
-
}
|
|
220
|
-
winStartSnapshot = { starts, ts: refreshStartedAt };
|
|
221
|
-
});
|
|
222
|
-
}
|
|
223
|
-
function getWindowsStartTimeMap() {
|
|
224
|
-
const now = Date.now();
|
|
225
|
-
if (!winStartSnapshot || now - winStartSnapshot.ts >= WIN_START_SNAPSHOT_TTL_MS)
|
|
226
|
-
refreshWindowsStartTimeMap();
|
|
227
|
-
if (winStartSnapshot && now - winStartSnapshot.ts < WIN_SNAPSHOT_MAX_STALE_MS)
|
|
228
|
-
return winStartSnapshot.starts;
|
|
229
|
-
return null; // no usable snapshot — conservative: caller trusts the PID
|
|
230
|
-
}
|
|
231
|
-
/** Windows process creation time as Unix ms, or undefined if unavailable.
|
|
232
|
-
* Lets us tell OUR process apart from an unrelated one the OS assigned the
|
|
233
|
-
* same (recycled) PID after ours exited. */
|
|
234
|
-
function processStartMs(pid) {
|
|
235
|
-
if (process.platform !== "win32")
|
|
236
|
-
return undefined;
|
|
237
|
-
return getWindowsStartTimeMap()?.get(pid);
|
|
238
|
-
}
|
|
239
|
-
/** True if the live PID is (still) the process we spawned, judged by start time.
|
|
240
|
-
* Guards against PID reuse: a recycled PID now owned by a different process has
|
|
241
|
-
* a start time far from what we recorded, so we stop treating it as ours (which
|
|
242
|
-
* otherwise wedges the keeper into an endless "unkillable" kill loop). */
|
|
243
|
-
function isSameProcess(pid, startedAt) {
|
|
244
|
-
if (startedAt === undefined)
|
|
245
|
-
return true; // nothing recorded to compare — trust PID
|
|
246
|
-
// Our process was spawned AFTER the current start-time map was taken: any
|
|
247
|
-
// entry the map holds for this pid describes a PREVIOUS owner of the (since
|
|
248
|
-
// recycled) pid, not our child — comparing against it would wrongly reject
|
|
249
|
-
// our own fresh process. Trust the PID until a fresher map exists.
|
|
250
|
-
if (process.platform === "win32" &&
|
|
251
|
-
winStartSnapshot !== null &&
|
|
252
|
-
startedAt >= winStartSnapshot.ts) {
|
|
87
|
+
if (process.platform !== "win32") {
|
|
88
|
+
// POSIX: kill(0) success or EPERM both mean a live process.
|
|
253
89
|
return true;
|
|
254
90
|
}
|
|
255
|
-
const
|
|
256
|
-
if (
|
|
257
|
-
return true; //
|
|
258
|
-
return
|
|
259
|
-
}
|
|
260
|
-
/** Descendants of rootPid (including root), ordered leaves-first (post-order),
|
|
261
|
-
* from one process-table snapshot, delivered via callback (async so the kill
|
|
262
|
-
* escalation path never blocks the event loop either). Fallback for when
|
|
263
|
-
* `taskkill /T` fails to tear the whole tree down: a single wedged descendant
|
|
264
|
-
* can make taskkill abort the tree, so killing each PID individually lets the
|
|
265
|
-
* still-killable relatives (often including the root) go down anyway. */
|
|
266
|
-
function getWindowsProcessTreeLeafFirst(rootPid, threadId, done) {
|
|
267
|
-
execFile("powershell", [
|
|
268
|
-
"-NoProfile",
|
|
269
|
-
"-Command",
|
|
270
|
-
'Get-CimInstance Win32_Process | ForEach-Object { "$($_.ProcessId) $($_.ParentProcessId)" }',
|
|
271
|
-
], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
272
|
-
if (err) {
|
|
273
|
-
// "Could not look" — distinct from "found nothing" below: the snapshot
|
|
274
|
-
// itself failed (the corporate-agent CIM failure seen on this machine),
|
|
275
|
-
// so we cannot enumerate the tree at all and fall back to the root pid.
|
|
276
|
-
log.warn(`[process] leaf-first walk pid=${rootPid} thread=${threadId}: process-tree snapshot FAILED (${errorMessage(err)}) — could NOT enumerate the tree; falling back to root pid only`);
|
|
277
|
-
done([rootPid]);
|
|
278
|
-
return;
|
|
279
|
-
}
|
|
280
|
-
const children = new Map();
|
|
281
|
-
for (const m of out.matchAll(/^(\d+) (\d+)$/gm)) {
|
|
282
|
-
const pid = Number(m[1]);
|
|
283
|
-
const ppid = Number(m[2]);
|
|
284
|
-
const list = children.get(ppid);
|
|
285
|
-
if (list)
|
|
286
|
-
list.push(pid);
|
|
287
|
-
else
|
|
288
|
-
children.set(ppid, [pid]);
|
|
289
|
-
}
|
|
290
|
-
const ordered = [];
|
|
291
|
-
const seen = new Set();
|
|
292
|
-
const visit = (pid) => {
|
|
293
|
-
if (seen.has(pid))
|
|
294
|
-
return; // guard against cycles / PID-reuse loops
|
|
295
|
-
seen.add(pid);
|
|
296
|
-
for (const child of children.get(pid) ?? [])
|
|
297
|
-
visit(child);
|
|
298
|
-
ordered.push(pid); // post-order → leaves first, root last
|
|
299
|
-
};
|
|
300
|
-
visit(rootPid);
|
|
301
|
-
// "Found nothing" reads as "root + 0 descendant(s)" — clearly not the same
|
|
302
|
-
// as the snapshot-failed warning above.
|
|
303
|
-
log.info(`[process] leaf-first walk pid=${rootPid} thread=${threadId} enumerated ${ordered.length} process(es): root + ${ordered.length - 1} descendant(s)`);
|
|
304
|
-
done(ordered);
|
|
305
|
-
});
|
|
306
|
-
}
|
|
307
|
-
/** Force-kill each PID in order, one at a time, tolerating any that refuse (a
|
|
308
|
-
* wedged leaf must not block the ones after it). Async so a sweep over a deep
|
|
309
|
-
* tree never blocks the server event loop. */
|
|
310
|
-
function killPidsSequentiallyWin(pids, done) {
|
|
311
|
-
let i = 0;
|
|
312
|
-
let killed = 0; // taskkill invocations that returned no error (reported killed)
|
|
313
|
-
const next = () => {
|
|
314
|
-
if (i >= pids.length) {
|
|
315
|
-
done(killed);
|
|
316
|
-
return;
|
|
317
|
-
}
|
|
318
|
-
const pid = pids[i++];
|
|
319
|
-
execFile("taskkill", ["/F", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
|
|
320
|
-
if (!err)
|
|
321
|
-
killed++;
|
|
322
|
-
next();
|
|
323
|
-
});
|
|
324
|
-
};
|
|
325
|
-
next();
|
|
91
|
+
const result = await brokerCheckAlive(pid, spawnedAtMs);
|
|
92
|
+
if (result === null)
|
|
93
|
+
return true; // cannot confirm → never declare a live process dead
|
|
94
|
+
return result.alive;
|
|
326
95
|
}
|
|
327
|
-
/**
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Thread-scoped liveness — the correct question for a broker-spawned worker.
|
|
98
|
+
*
|
|
99
|
+
* On Windows claude.exe/copilot.exe are LAUNCHERS: they fork the real agent into
|
|
100
|
+
* the thread's Job Object under a NEW pid, then exit. ~20s later the launcher pid
|
|
101
|
+
* we recorded at spawn is gone, so a pid-first `isProcessAlive(launcherPid)`
|
|
102
|
+
* returns false and orphans the still-working child. The thread's JOB is the
|
|
103
|
+
* source of truth; ask the broker by threadId:
|
|
104
|
+
* - null (unreachable) → cannot confirm → treat ALIVE (never kill on unknown),
|
|
105
|
+
* exactly the audited-safe degradation contract.
|
|
106
|
+
* - known:false (404) → the broker has NEVER managed this thread (dev mode /
|
|
107
|
+
* pre-broker process) → fall back to the pid check.
|
|
108
|
+
* - known:true → use the job/registry verdict (kernel truth).
|
|
109
|
+
*
|
|
110
|
+
* Off Windows there is no launcher-fork indirection (the pid IS the real child),
|
|
111
|
+
* so defer straight to the pid check.
|
|
112
|
+
*/
|
|
113
|
+
export async function isThreadAlive(threadId, fallbackPid, spawnedAtMs) {
|
|
114
|
+
if (process.platform === "win32") {
|
|
115
|
+
const verdict = await brokerCheckAliveByThread(threadId);
|
|
116
|
+
if (verdict === null)
|
|
117
|
+
return true; // unreachable → never declare a live worker dead
|
|
118
|
+
if (verdict.known)
|
|
119
|
+
return verdict.alive; // job/registry truth
|
|
120
|
+
// known:false → not a broker-managed thread; fall through to the pid check.
|
|
349
121
|
}
|
|
122
|
+
return fallbackPid !== undefined ? isProcessAlive(fallbackPid, spawnedAtMs) : false;
|
|
350
123
|
}
|
|
351
124
|
export function readPidFiles() {
|
|
352
125
|
const entries = [];
|
|
@@ -381,40 +154,46 @@ export function readPidFiles() {
|
|
|
381
154
|
}
|
|
382
155
|
return entries;
|
|
383
156
|
}
|
|
384
|
-
export function findAliveThread(threadId) {
|
|
157
|
+
export async function findAliveThread(threadId) {
|
|
385
158
|
for (let i = spawnedThreads.length - 1; i >= 0; i--) {
|
|
386
159
|
const thread = spawnedThreads[i];
|
|
387
160
|
if (thread.threadId !== threadId)
|
|
388
161
|
continue;
|
|
389
|
-
if (isProcessAlive(thread.pid, thread.startedAt)
|
|
162
|
+
if (await isProcessAlive(thread.pid, thread.startedAt))
|
|
390
163
|
return thread;
|
|
391
164
|
log.warn(`[findAliveThread] Thread ${threadId} PID ${thread.pid} stale in spawnedThreads - removing`);
|
|
392
165
|
spawnedThreads.splice(i, 1);
|
|
393
166
|
}
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
167
|
+
for (const entry of readPidFiles()) {
|
|
168
|
+
if (entry.threadId !== threadId)
|
|
169
|
+
continue;
|
|
170
|
+
if (!(await isProcessAlive(entry.pid, entry.startedAt)))
|
|
171
|
+
continue;
|
|
172
|
+
const restored = {
|
|
173
|
+
pid: entry.pid,
|
|
174
|
+
threadId,
|
|
175
|
+
name: entry.name ?? `Thread ${threadId}`,
|
|
176
|
+
startedAt: entry.startedAt ?? Date.now(),
|
|
177
|
+
createdAt: entry.startedAt ?? Date.now(),
|
|
178
|
+
logFile: "",
|
|
179
|
+
...(entry.threadType ? { threadType: entry.threadType } : {}),
|
|
180
|
+
};
|
|
181
|
+
spawnedThreads.push(restored);
|
|
182
|
+
log.info(`[findAliveThread] Restored thread ${threadId} PID=${entry.pid} from PID file`);
|
|
183
|
+
return restored;
|
|
184
|
+
}
|
|
185
|
+
return undefined;
|
|
411
186
|
}
|
|
412
|
-
export
|
|
413
|
-
export function getActiveThreadIds() {
|
|
187
|
+
export async function getActiveThreadIds() {
|
|
414
188
|
// Workers are disposable and not eligible for reconnect — exclude them.
|
|
415
|
-
const
|
|
416
|
-
|
|
417
|
-
|
|
189
|
+
const candidates = spawnedThreads.filter((t) => t.threadType !== "worker");
|
|
190
|
+
// THREAD-scoped liveness: a live broker non-worker thread's recorded pid is a
|
|
191
|
+
// launcher that exits ~20s post-spawn while its forked child lives on in the
|
|
192
|
+
// job, so a pid-first isProcessAlive(t.pid) under-reports it as dead and drops
|
|
193
|
+
// it from the reconnect/adopt set. Already async with every caller awaiting, so
|
|
194
|
+
// this is a drop-in correctness fix with no sync→async blast radius.
|
|
195
|
+
const aliveFlags = await Promise.all(candidates.map((t) => isThreadAlive(t.threadId, t.pid, t.startedAt)));
|
|
196
|
+
const spawned = candidates.filter((_, i) => aliveFlags[i]).map((t) => t.threadId);
|
|
418
197
|
// Also include threads from active MCP sessions (root threads that aren't
|
|
419
198
|
// in spawnedThreads because they connected externally, not spawned by us).
|
|
420
199
|
const sessionThreadIds = getDashboardSessions()
|
|
@@ -464,14 +243,23 @@ export function pruneOldThreadLogs() {
|
|
|
464
243
|
}
|
|
465
244
|
/**
|
|
466
245
|
* Check liveness of a thread directly via its PID file, bypassing the
|
|
467
|
-
* spawnedThreads[] in-memory cache.
|
|
468
|
-
*
|
|
246
|
+
* spawnedThreads[] in-memory cache. Returns the live PID together with its
|
|
247
|
+
* recorded spawn identity (startedAt) so callers can make identity-checked
|
|
248
|
+
* kill/alive calls, or undefined if no PID file exists or the process is dead
|
|
249
|
+
* (stale file is deleted).
|
|
469
250
|
*/
|
|
470
|
-
export function findAliveThreadViaPidFile(threadId) {
|
|
251
|
+
export async function findAliveThreadViaPidFile(threadId) {
|
|
471
252
|
const entry = readPidFiles().find((e) => e.threadId === threadId);
|
|
472
253
|
if (!entry)
|
|
473
254
|
return undefined;
|
|
474
|
-
|
|
255
|
+
// THREAD-scoped liveness before unlinking. The recorded pid is a launcher that
|
|
256
|
+
// exits ~20s after spawn while its forked child lives on in the job, so a
|
|
257
|
+
// pid-first isProcessAlive() reads dead for a LIVE agent and would delete its pid
|
|
258
|
+
// file — destroying the reconcile anchor a post-self-update adopt/restore needs.
|
|
259
|
+
// Ask the broker by threadId: unreachable → treat-alive (never unlink on unknown),
|
|
260
|
+
// known:false → fall back to the pid check (dev mode / pre-broker), known:true →
|
|
261
|
+
// job/registry truth. Only a genuinely-gone thread reaches the unlink.
|
|
262
|
+
if (!(await isThreadAlive(threadId, entry.pid, entry.startedAt))) {
|
|
475
263
|
try {
|
|
476
264
|
unlinkSync(entry.filePath);
|
|
477
265
|
}
|
|
@@ -480,7 +268,7 @@ export function findAliveThreadViaPidFile(threadId) {
|
|
|
480
268
|
}
|
|
481
269
|
return undefined;
|
|
482
270
|
}
|
|
483
|
-
return entry.pid;
|
|
271
|
+
return { pid: entry.pid, startedAt: entry.startedAt };
|
|
484
272
|
}
|
|
485
273
|
/**
|
|
486
274
|
* Reconcile in-memory state (spawnedThreads[]) with the two authoritative
|
|
@@ -490,7 +278,7 @@ export function findAliveThreadViaPidFile(threadId) {
|
|
|
490
278
|
*
|
|
491
279
|
* Replaces the old restoreFromPidFiles() + cleanupStalePidFiles() pair.
|
|
492
280
|
*/
|
|
493
|
-
export function reconcileState(db, threadLifecycle) {
|
|
281
|
+
export async function reconcileState(db, threadLifecycle) {
|
|
494
282
|
const pidEntries = readPidFiles();
|
|
495
283
|
const dbThreadMap = new Map(getAllThreads(db)
|
|
496
284
|
.filter((t) => ["active", "exited", "exiting", "spawning", "stuck"].includes(t.status))
|
|
@@ -502,10 +290,13 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
502
290
|
// PID file references a thread with no active DB record (archived/removed).
|
|
503
291
|
// If the process is still alive it's an orphan that must be reaped — never
|
|
504
292
|
// just drop the PID file, or the live process becomes untrackable.
|
|
505
|
-
//
|
|
506
|
-
|
|
293
|
+
// killThreadTree removes the PID file itself once the process is confirmed dead.
|
|
294
|
+
// Liveness/kill are THREAD-scoped: a broker-spawned worker's launcher pid is
|
|
295
|
+
// dead here even while its forked child lives, so a pid-first check/kill would
|
|
296
|
+
// both miss the live orphan and (via KillByPid AlreadyDead) fail to reap it.
|
|
297
|
+
if (await isThreadAlive(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt)) {
|
|
507
298
|
log.warn(`[reconcileState] Live process for thread ${pidEntry.threadId} (PID=${pidEntry.pid}) has no active DB record — killing orphan`);
|
|
508
|
-
void
|
|
299
|
+
void killThreadTree(pidEntry.threadId, "reconcile-orphan", pidEntry.pid, pidEntry.startedAt);
|
|
509
300
|
}
|
|
510
301
|
else {
|
|
511
302
|
log.warn(`[reconcileState] PID file for thread ${pidEntry.threadId} has no DB record — removing`);
|
|
@@ -518,8 +309,10 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
518
309
|
}
|
|
519
310
|
continue;
|
|
520
311
|
}
|
|
521
|
-
if (
|
|
522
|
-
// Alive: populate spawnedThreads[] if not already present
|
|
312
|
+
if (await isThreadAlive(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt)) {
|
|
313
|
+
// Alive: populate spawnedThreads[] if not already present.
|
|
314
|
+
// THREAD-scoped so a live broker worker (dead launcher pid, live forked
|
|
315
|
+
// child in the job) is not misjudged dead and orphaned across a self-update.
|
|
523
316
|
if (!spawnedThreads.some((t) => t.threadId === pidEntry.threadId)) {
|
|
524
317
|
spawnedThreads.push({
|
|
525
318
|
pid: pidEntry.pid,
|
|
@@ -531,6 +324,9 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
531
324
|
...(pidEntry.threadType ? { threadType: pidEntry.threadType } : {}),
|
|
532
325
|
});
|
|
533
326
|
log.info(`[reconcileState] Restored thread ${pidEntry.threadId} PID=${pidEntry.pid} from PID file`);
|
|
327
|
+
// Safe cutover (Phase 3): pull pre-broker processes into jobs+registry
|
|
328
|
+
// now rather than waiting for a supervisor restart. Idempotent.
|
|
329
|
+
void brokerAdopt(pidEntry.threadId, pidEntry.pid, pidEntry.startedAt, pidEntry.threadType ?? "agent");
|
|
534
330
|
}
|
|
535
331
|
}
|
|
536
332
|
else {
|
|
@@ -591,318 +387,138 @@ export function reconcileState(db, threadLifecycle) {
|
|
|
591
387
|
log.warn(`[reconcileState] Failed to sweep orphan MCP configs: ${errorMessage(err)}`);
|
|
592
388
|
}
|
|
593
389
|
}
|
|
594
|
-
// descendantPid → the tracked thread it was witnessed under (in-memory: an
|
|
595
|
-
// orphan spanning a server restart is intentionally not tracked — see header).
|
|
596
|
-
const orphanLineage = new Map();
|
|
597
|
-
const orphanSnapshotBreaker = makeSnapshotBreaker("win32-process");
|
|
598
|
-
/** One `Get-CimInstance Win32_Process` snapshot giving ppid AND creation time for
|
|
599
|
-
* every pid — the only single call that yields both, which the sweep needs to
|
|
600
|
-
* walk lineage and enforce the pid-reuse guards. Async (never blocks the event
|
|
601
|
-
* loop) and behind the shared circuit breaker: on the corporate-agent failure
|
|
602
|
-
* observed on this machine it resolves null, and the sweep degrades to a no-op. */
|
|
603
|
-
function takeWindowsProcessSnapshot() {
|
|
604
|
-
return new Promise((resolve) => {
|
|
605
|
-
if (process.platform !== "win32" || orphanSnapshotBreaker.isOpen()) {
|
|
606
|
-
resolve(null);
|
|
607
|
-
return;
|
|
608
|
-
}
|
|
609
|
-
execFile("powershell", [
|
|
610
|
-
"-NoProfile",
|
|
611
|
-
"-Command",
|
|
612
|
-
'Get-CimInstance Win32_Process | ForEach-Object { try { "$($_.ProcessId) $($_.ParentProcessId) $($_.CreationDate.ToUniversalTime().Ticks)" } catch {} }',
|
|
613
|
-
], { encoding: "utf-8", windowsHide: true, timeout: 5000 }, (err, out) => {
|
|
614
|
-
if (err) {
|
|
615
|
-
orphanSnapshotBreaker.recordFailure(err);
|
|
616
|
-
resolve(null);
|
|
617
|
-
return;
|
|
618
|
-
}
|
|
619
|
-
orphanSnapshotBreaker.recordSuccess();
|
|
620
|
-
const snap = new Map();
|
|
621
|
-
for (const m of out.matchAll(/^(\d+) (\d+) (\d+)$/gm)) {
|
|
622
|
-
snap.set(Number(m[1]), {
|
|
623
|
-
ppid: Number(m[2]),
|
|
624
|
-
startMs: Math.round(Number(m[3]) / 10000 - 62135596800000), // .NET ticks → Unix ms
|
|
625
|
-
});
|
|
626
|
-
}
|
|
627
|
-
resolve(snap);
|
|
628
|
-
});
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
/**
|
|
632
|
-
* Pure core of the orphan sweep — no IO, no clock, fully deterministic so the
|
|
633
|
-
* pid-reuse guard and the snapshot-unavailable path can be tested directly.
|
|
634
|
-
* Mutates `lineage` in place (records new descendants, drops dead/recycled ones).
|
|
635
|
-
*/
|
|
636
|
-
export function computeOrphanSweep(params) {
|
|
637
|
-
const { snapshot, trackedThreads, lineage, protectedPids, toleranceMs } = params;
|
|
638
|
-
const result = {
|
|
639
|
-
orphans: [],
|
|
640
|
-
reusedPidsSkipped: [],
|
|
641
|
-
olderThanParentSkipped: [],
|
|
642
|
-
recorded: 0,
|
|
643
|
-
};
|
|
644
|
-
// Snapshot unavailable → never guess, never kill, never mutate lineage.
|
|
645
|
-
if (snapshot === null)
|
|
646
|
-
return result;
|
|
647
|
-
const aliveOwners = new Set();
|
|
648
|
-
for (const t of trackedThreads)
|
|
649
|
-
if (t.alive)
|
|
650
|
-
aliveOwners.add(t.threadId);
|
|
651
|
-
// Children adjacency, built once from the snapshot.
|
|
652
|
-
const children = new Map();
|
|
653
|
-
for (const [pid, e] of snapshot) {
|
|
654
|
-
const list = children.get(e.ppid);
|
|
655
|
-
if (list)
|
|
656
|
-
list.push(pid);
|
|
657
|
-
else
|
|
658
|
-
children.set(e.ppid, [pid]);
|
|
659
|
-
}
|
|
660
|
-
// Step A — witness lineage: record every descendant of each ALIVE tracked
|
|
661
|
-
// thread while its subtree chain is still intact. Excludes the root pid itself
|
|
662
|
-
// (that thread is managed by the worker-cleanup path, not here).
|
|
663
|
-
for (const t of trackedThreads) {
|
|
664
|
-
if (!t.alive || !snapshot.has(t.pid))
|
|
665
|
-
continue;
|
|
666
|
-
const seen = new Set([t.pid]);
|
|
667
|
-
const stack = [...(children.get(t.pid) ?? [])];
|
|
668
|
-
while (stack.length) {
|
|
669
|
-
const d = stack.pop();
|
|
670
|
-
if (seen.has(d))
|
|
671
|
-
continue; // cycle / pid-reuse loop guard
|
|
672
|
-
seen.add(d);
|
|
673
|
-
const e = snapshot.get(d);
|
|
674
|
-
if (!e)
|
|
675
|
-
continue;
|
|
676
|
-
lineage.set(d, {
|
|
677
|
-
ownerThreadId: t.threadId,
|
|
678
|
-
ownerPid: t.pid,
|
|
679
|
-
ownerStartedAt: t.startedAt,
|
|
680
|
-
descendantStartMs: e.startMs,
|
|
681
|
-
});
|
|
682
|
-
result.recorded++;
|
|
683
|
-
for (const c of children.get(d) ?? [])
|
|
684
|
-
if (!seen.has(c))
|
|
685
|
-
stack.push(c);
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
// Step B — evaluate: a recorded descendant whose owner is no longer alive is an
|
|
689
|
-
// orphan candidate, subject to both pid-reuse guards.
|
|
690
|
-
for (const [pid, entry] of [...lineage]) {
|
|
691
|
-
// Prune a descendant that has left the process table FIRST, before the
|
|
692
|
-
// owner-liveness check — otherwise a dead descendant of a still-live owner is
|
|
693
|
-
// never evaluated here (its owner short-circuits below) and never re-recorded
|
|
694
|
-
// by Step A (it is gone), so it lingers in `lineage` for the owner's entire
|
|
695
|
-
// lifetime. A long-lived thread that churns many short-lived children (e.g. a
|
|
696
|
-
// claude-agent spawning a bash.exe per hook) would grow the map without bound.
|
|
697
|
-
const cur = snapshot.get(pid);
|
|
698
|
-
if (!cur) {
|
|
699
|
-
lineage.delete(pid); // descendant already gone — nothing to reap
|
|
700
|
-
continue;
|
|
701
|
-
}
|
|
702
|
-
if (aliveOwners.has(entry.ownerThreadId))
|
|
703
|
-
continue; // owner still alive → not orphaned
|
|
704
|
-
// Never touch a pid managed elsewhere (a live tracked thread, the server, us).
|
|
705
|
-
if (protectedPids.has(pid))
|
|
706
|
-
continue;
|
|
707
|
-
// GUARD 1 — same-process: a recycled pid now hosts a different process whose
|
|
708
|
-
// creation time differs from the one we recorded. Drop it; do NOT kill.
|
|
709
|
-
if (Math.abs(cur.startMs - entry.descendantStartMs) > toleranceMs) {
|
|
710
|
-
lineage.delete(pid);
|
|
711
|
-
result.reusedPidsSkipped.push(pid);
|
|
712
|
-
continue;
|
|
713
|
-
}
|
|
714
|
-
// GUARD 2 — a child cannot predate its parent. If this pid started before the
|
|
715
|
-
// owning thread did, it is not really our descendant (recorded owner is wrong
|
|
716
|
-
// or the pid was reused across the boundary). Keep it for logging, never kill.
|
|
717
|
-
if (entry.ownerStartedAt !== undefined && cur.startMs < entry.ownerStartedAt - toleranceMs) {
|
|
718
|
-
result.olderThanParentSkipped.push(pid);
|
|
719
|
-
continue;
|
|
720
|
-
}
|
|
721
|
-
result.orphans.push({ pid, ownerThreadId: entry.ownerThreadId, startMs: cur.startMs });
|
|
722
|
-
}
|
|
723
|
-
return result;
|
|
724
|
-
}
|
|
725
|
-
/** True when the sweep is allowed to actually kill confirmed orphans. Default is
|
|
726
|
-
* log-only (see header): a week of real candidate logs comes first. */
|
|
727
|
-
function orphanSweepWillKill() {
|
|
728
|
-
const v = process.env.ORPHAN_SWEEP_KILL;
|
|
729
|
-
return v === "1" || v === "true";
|
|
730
|
-
}
|
|
731
390
|
/**
|
|
732
|
-
*
|
|
733
|
-
*
|
|
734
|
-
*
|
|
735
|
-
*
|
|
736
|
-
*
|
|
391
|
+
* Kill the process tree rooted at `pid` via the supervisor's broker (native
|
|
392
|
+
* TerminateProcess over the whole tree, identity-checked when `spawnedAtMs` is
|
|
393
|
+
* given, outcome recorded in the broker's on-disk registry). Never spawns a
|
|
394
|
+
* child process and never blocks the event loop.
|
|
395
|
+
*
|
|
396
|
+
* Broker unreachable → degraded best-effort: one native `process.kill(pid)`
|
|
397
|
+
* (root only — Node cannot enumerate a tree without shelling out, and that
|
|
398
|
+
* machinery is gone by design) followed by a conservative liveness re-check.
|
|
399
|
+
* A survived kill keeps the PID file so the keeper/cleanup policy loops retry
|
|
400
|
+
* and escalate — same contract as before.
|
|
737
401
|
*/
|
|
738
|
-
export async function
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
402
|
+
export async function killProcessTree(pid, threadId, caller, spawnedAtMs) {
|
|
403
|
+
// Self-protection: NEVER tree-kill our own process or the authoritative
|
|
404
|
+
// server PID (a recycled worker PID could point at us — killing it would be
|
|
405
|
+
// the traceless server death this layer exists to prevent). The broker has
|
|
406
|
+
// the same guard server-side; this is defense in depth.
|
|
407
|
+
if (pid === process.pid) {
|
|
408
|
+
log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
409
|
+
return;
|
|
743
410
|
}
|
|
744
|
-
const trackedThreads = readPidFiles().map((e) => ({
|
|
745
|
-
threadId: e.threadId,
|
|
746
|
-
pid: e.pid,
|
|
747
|
-
startedAt: e.startedAt,
|
|
748
|
-
alive: isProcessAlive(e.pid, e.startedAt) && isSameProcess(e.pid, e.startedAt),
|
|
749
|
-
}));
|
|
750
|
-
// Protected: every live tracked-thread pid (managed by worker-cleanup), the
|
|
751
|
-
// authoritative server pid, and our own pid — these are never orphan targets.
|
|
752
|
-
const protectedPids = new Set([process.pid]);
|
|
753
411
|
const serverPid = readServerPid();
|
|
754
|
-
if (serverPid !== undefined)
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
const result =
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
toleranceMs: PID_IDENTITY_TOLERANCE_MS,
|
|
412
|
+
if (serverPid !== undefined && pid === serverPid) {
|
|
413
|
+
log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
log.info(`[process] killProcessTree pid=${pid} thread=${threadId} (caller=${caller ?? "?"})`);
|
|
417
|
+
const result = await brokerKillTree(pid, {
|
|
418
|
+
threadId,
|
|
419
|
+
killedBy: caller ?? "mcp",
|
|
420
|
+
reason: caller,
|
|
421
|
+
startedAtMs: spawnedAtMs,
|
|
765
422
|
});
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
log.warn(`[
|
|
423
|
+
let stillAlive;
|
|
424
|
+
if (result !== null) {
|
|
425
|
+
if (result.refused)
|
|
426
|
+
log.error(`[process] broker REFUSED kill of pid ${pid}: ${result.error ?? "protected pid"}`);
|
|
427
|
+
else if (result.identityMismatch)
|
|
428
|
+
log.warn(`[process] broker: pid ${pid} was recycled (identity mismatch) — our process is already gone, nothing killed`);
|
|
429
|
+
stillAlive = result.alive;
|
|
430
|
+
}
|
|
431
|
+
else {
|
|
432
|
+
// Degraded path (no supervisor): best-effort native root kill, then re-check.
|
|
433
|
+
log.warn(`[process] broker unreachable for kill of pid ${pid} (thread ${threadId}) — degraded native root-only kill`);
|
|
777
434
|
try {
|
|
778
|
-
|
|
435
|
+
process.kill(pid);
|
|
779
436
|
}
|
|
780
437
|
catch (err) {
|
|
781
|
-
log.
|
|
782
|
-
}
|
|
783
|
-
finally {
|
|
784
|
-
orphanLineage.delete(orphan.pid);
|
|
438
|
+
log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
|
|
785
439
|
}
|
|
440
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
441
|
+
stillAlive = await isProcessAlive(pid, spawnedAtMs);
|
|
786
442
|
}
|
|
787
|
-
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
// the whole /T tree down. A short execFile timeout aborts taskkill mid-kill and
|
|
792
|
-
// surfaces as "Command failed" with no stderr, so the process survives and the
|
|
793
|
-
// keeper wrongly flags it UNKILLABLE. Give taskkill headroom to finish; the
|
|
794
|
-
// caller still verifies liveness afterwards, so a fast kill returns immediately.
|
|
795
|
-
const TASKKILL_TIMEOUT_MS = 60_000;
|
|
796
|
-
export function killProcessTree(pid, threadId, caller, spawnedAtMs) {
|
|
797
|
-
return new Promise((resolve) => {
|
|
798
|
-
// Self-protection: NEVER tree-kill our own process or the authoritative
|
|
799
|
-
// server PID. The leading hypothesis for the traceless server death is a
|
|
800
|
-
// keeper/worker-reaper tree-killing a RECYCLED worker PID that the OS
|
|
801
|
-
// reassigned to the node server (or one of its ancestors). `taskkill /T`
|
|
802
|
-
// would then take the whole server tree down with no signal, no [fatal] and
|
|
803
|
-
// no crash dump — exactly the observed symptom. Refuse and log loudly.
|
|
804
|
-
if (pid === process.pid) {
|
|
805
|
-
log.error(`[process] REFUSED kill of pid ${pid} — matches self (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
806
|
-
resolve();
|
|
807
|
-
return;
|
|
443
|
+
if (!stillAlive) {
|
|
444
|
+
log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
|
|
445
|
+
try {
|
|
446
|
+
unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
|
|
808
447
|
}
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
log.error(`[process] REFUSED kill of pid ${pid} — matches server.pid (self-protection) caller=${caller ?? "?"} thread=${threadId}`);
|
|
812
|
-
resolve();
|
|
813
|
-
return;
|
|
448
|
+
catch {
|
|
449
|
+
/* PID file already gone (exit handler may have raced us) */
|
|
814
450
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
// Conservative: a survived kill keeps the PID file so a later sweep retries
|
|
454
|
+
// against fresh evidence (keeper/worker-cleanup own the escalation policy).
|
|
455
|
+
log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt — NOT removing PID file`);
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Kill a worker by its THREAD, not by the pid we recorded at spawn.
|
|
459
|
+
*
|
|
460
|
+
* For a launcher-forked worker the recorded pid is the launcher, which has long
|
|
461
|
+
* since exited; `killProcessTree(launcherPid)` therefore routes to the broker's
|
|
462
|
+
* KillByPid, hits its "AlreadyDead" fast-path (the launcher pid is gone) and
|
|
463
|
+
* returns BEFORE the job-kill fallback — so the live forked child is never
|
|
464
|
+
* touched. Routing by threadId (pid omitted) takes the broker's KillByThread
|
|
465
|
+
* path instead: it terminates the whole Job Object (every live member), or, when
|
|
466
|
+
* the job assign failed (Avecto downgrade), the registry-identity survivor the
|
|
467
|
+
* broker adopted. This is the kill that actually reaches the real agent.
|
|
468
|
+
*
|
|
469
|
+
* Broker unreachable → degraded best-effort native kill of `fallbackPid` (self/
|
|
470
|
+
* server-guarded), and we NEVER remove the PID file or claim death: a later sweep
|
|
471
|
+
* retries against fresh evidence. `brokerKillTree` also returns null for an
|
|
472
|
+
* unknown/already-dead thread (HTTP 404); the degraded native kill of a dead pid
|
|
473
|
+
* is a harmless no-op there, and the caller's own `isThreadAlive` gate — not this
|
|
474
|
+
* function — decides teardown.
|
|
475
|
+
*/
|
|
476
|
+
export async function killThreadTree(threadId, caller, fallbackPid, spawnedAtMs) {
|
|
477
|
+
log.info(`[process] killThreadTree thread=${threadId} (caller=${caller ?? "?"})`);
|
|
478
|
+
// Route by thread: pid omitted → broker KillByThread (job kill of the real child).
|
|
479
|
+
const result = await brokerKillTree(undefined, {
|
|
480
|
+
threadId,
|
|
481
|
+
killedBy: caller ?? "mcp",
|
|
482
|
+
reason: caller,
|
|
483
|
+
startedAtMs: spawnedAtMs,
|
|
484
|
+
});
|
|
485
|
+
if (result !== null) {
|
|
486
|
+
if (result.refused)
|
|
487
|
+
log.error(`[process] broker REFUSED kill of thread ${threadId}: ${result.error ?? "protected"}`);
|
|
488
|
+
if (!result.alive) {
|
|
489
|
+
// No live member remains — safe to drop the stale PID file.
|
|
818
490
|
try {
|
|
819
|
-
unlinkSync(
|
|
491
|
+
unlinkSync(join(PROCESS_PIDS_DIR, `${threadId}.pid`));
|
|
820
492
|
}
|
|
821
493
|
catch {
|
|
822
494
|
/* PID file already gone (exit handler may have raced us) */
|
|
823
495
|
}
|
|
824
|
-
};
|
|
825
|
-
const verifyDead = (attempt) => {
|
|
826
|
-
// spawnedAtMs matters here for orphan-kill callers reaping FRESH pids: a
|
|
827
|
-
// failed kill must not let a pre-spawn snapshot falsely confirm death and
|
|
828
|
-
// unlink a live process's pid file. A genuinely dead child is usually
|
|
829
|
-
// caught by the kill(0) ESRCH probe or a post-spawn snapshot refresh; if
|
|
830
|
-
// neither lands within the 3 attempts (refresh in flight / breaker open),
|
|
831
|
-
// we resolve as "still alive" and KEEP the pid file — conservative, and a
|
|
832
|
-
// later sweep finishes the cleanup against a fresher snapshot.
|
|
833
|
-
if (!isProcessAlive(pid, spawnedAtMs)) {
|
|
834
|
-
log.info(`[process] Killed process tree for thread ${threadId} PID=${pid}`);
|
|
835
|
-
cleanupPidFile();
|
|
836
|
-
resolve();
|
|
837
|
-
return;
|
|
838
|
-
}
|
|
839
|
-
if (attempt >= 3) {
|
|
840
|
-
log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after ${attempt} kill attempts — NOT removing PID file`);
|
|
841
|
-
resolve();
|
|
842
|
-
return;
|
|
843
|
-
}
|
|
844
|
-
// Retry kill after a short delay
|
|
845
|
-
log.warn(`[process] PID ${pid} (thread ${threadId}) still alive after kill attempt ${attempt} — retrying`);
|
|
846
|
-
if (process.platform === "win32") {
|
|
847
|
-
// Escalation, retry-path only: the happy-path in-process `taskkill /F /T`
|
|
848
|
-
// already failed to verify-dead. If the failure was EPERM because the
|
|
849
|
-
// server's own tree is job-object-constrained / I/O-wedged, no in-process
|
|
850
|
-
// kill will ever succeed — so first spawn a DETACHED, fresh sibling
|
|
851
|
-
// taskkill that does not inherit our wedged handles.
|
|
852
|
-
spawnDetachedKillerWin(pid);
|
|
853
|
-
// Native TerminateProcess (via libuv) as well as taskkill — they take
|
|
854
|
-
// different paths (taskkill is an external process with its own token;
|
|
855
|
-
// process.kill issues the syscall directly from this process), so one
|
|
856
|
-
// can succeed where the other fails on a stubborn process.
|
|
857
|
-
try {
|
|
858
|
-
process.kill(pid);
|
|
859
|
-
}
|
|
860
|
-
catch (err) {
|
|
861
|
-
log.debug(`[process] native kill PID ${pid} failed: ${errorMessage(err)}`);
|
|
862
|
-
}
|
|
863
|
-
// `taskkill /T` walks the tree top-down and can stall when a hung child
|
|
864
|
-
// blocks the parent's termination — the exact case where the root sat
|
|
865
|
-
// alive for hours. Kill each descendant leaves-first so a wedged child is
|
|
866
|
-
// reaped before its parent, freeing the root to die. Logged at info (from
|
|
867
|
-
// entry through kill count) so a week of logs can answer whether this
|
|
868
|
-
// escalation path ever actually runs, or is dead weight.
|
|
869
|
-
log.info(`[process] leaf-first kill ESCALATION entered for pid=${pid} thread=${threadId} (attempt ${attempt}) — taskkill /F /T did not verify-dead`);
|
|
870
|
-
getWindowsProcessTreeLeafFirst(pid, threadId, (tree) => {
|
|
871
|
-
killPidsSequentiallyWin(tree, (killed) => {
|
|
872
|
-
log.info(`[process] leaf-first escalation for pid=${pid} thread=${threadId}: issued taskkill to ${tree.length} process(es), ${killed} reported killed`);
|
|
873
|
-
setTimeout(() => verifyDead(attempt + 1), 1000);
|
|
874
|
-
});
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
else {
|
|
878
|
-
try {
|
|
879
|
-
process.kill(pid, "SIGKILL");
|
|
880
|
-
}
|
|
881
|
-
catch {
|
|
882
|
-
/* process already gone (ESRCH) — verifyDead confirms */
|
|
883
|
-
}
|
|
884
|
-
setTimeout(() => verifyDead(attempt + 1), 1000);
|
|
885
|
-
}
|
|
886
|
-
};
|
|
887
|
-
if (process.platform === "win32") {
|
|
888
|
-
execFile("taskkill", ["/F", "/T", "/PID", String(pid)], { timeout: TASKKILL_TIMEOUT_MS, windowsHide: true }, (err) => {
|
|
889
|
-
if (err) {
|
|
890
|
-
log.warn(`[process] Kill process ${pid} (thread ${threadId}) failed: ${errorMessage(err)}`);
|
|
891
|
-
}
|
|
892
|
-
// Always verify — taskkill can report success but process lingers,
|
|
893
|
-
// or report failure but process is actually dead.
|
|
894
|
-
setTimeout(() => verifyDead(1), 500);
|
|
895
|
-
});
|
|
896
496
|
}
|
|
897
497
|
else {
|
|
898
|
-
|
|
899
|
-
process.kill(pid, "SIGTERM");
|
|
900
|
-
}
|
|
901
|
-
catch (err) {
|
|
902
|
-
log.debug(`[process] Kill process ${pid} (thread ${threadId}): ${errorMessage(err)}`);
|
|
903
|
-
}
|
|
904
|
-
setTimeout(() => verifyDead(1), 500);
|
|
498
|
+
log.warn(`[process] thread ${threadId} still has a live member after kill — NOT removing PID file`);
|
|
905
499
|
}
|
|
906
|
-
|
|
500
|
+
return;
|
|
501
|
+
}
|
|
502
|
+
// Degraded path (broker unreachable, or unknown/already-dead thread → 404).
|
|
503
|
+
// Best-effort native kill of the known pid; never remove the PID file or claim
|
|
504
|
+
// death — treat-alive on the unknown, exactly the audited-safe contract.
|
|
505
|
+
if (fallbackPid === undefined)
|
|
506
|
+
return;
|
|
507
|
+
if (fallbackPid === process.pid) {
|
|
508
|
+
log.error(`[process] REFUSED degraded kill of pid ${fallbackPid} — matches self (self-protection) thread=${threadId}`);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
const serverPid = readServerPid();
|
|
512
|
+
if (serverPid !== undefined && fallbackPid === serverPid) {
|
|
513
|
+
log.error(`[process] REFUSED degraded kill of pid ${fallbackPid} — matches server.pid (self-protection) thread=${threadId}`);
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
log.warn(`[process] broker unreachable for kill of thread ${threadId} — degraded native kill of pid ${fallbackPid}`);
|
|
517
|
+
try {
|
|
518
|
+
process.kill(fallbackPid);
|
|
519
|
+
}
|
|
520
|
+
catch (err) {
|
|
521
|
+
log.debug(`[process] native kill PID ${fallbackPid} failed: ${errorMessage(err)}`);
|
|
522
|
+
}
|
|
907
523
|
}
|
|
908
524
|
//# sourceMappingURL=process.service.js.map
|