pi-crew 0.11.40 → 0.11.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +75 -0
- package/dist/index.mjs +505 -505
- package/package.json +1 -1
- package/src/runtime/foreground-watchdog.ts +213 -26
- package/src/runtime/process/zombie-scanner.ts +97 -42
- package/src/runtime/task-runner/live-executor.ts +146 -45
- package/src/state/event-log/event-log.ts +42 -0
package/package.json
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* assistant has no way to know when a run completes or gets stuck without
|
|
7
7
|
* manual polling. This watchdog monitors active runs and:
|
|
8
8
|
*
|
|
9
|
-
* 1. Detects hung runs (active status,
|
|
9
|
+
* 1. Detects hung runs (active status, manifest not updated for >2 min with
|
|
10
|
+
* no agent activity evidence — wave 13: a live registry claim or a fresh
|
|
11
|
+
* task heartbeat withholds the verdict, see CLAIMED_RUN_HUNG_MS)
|
|
10
12
|
* 2. Injects a followUp message via pi.sendUserMessage() so the assistant
|
|
11
13
|
* is automatically notified — no manual sleep+check needed.
|
|
12
14
|
* 3. Cleans up after itself when the run completes or the session ends.
|
|
@@ -19,6 +21,7 @@
|
|
|
19
21
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
22
|
import { getCrewEnv, getCrewEnvInt } from "../config/env-vars.ts";
|
|
21
23
|
import { TEAM_TERMINAL_TASK_STATUSES } from "../state/contracts.ts";
|
|
24
|
+
import { activeRunEntries } from "../state/stores/active-run-registry.ts";
|
|
22
25
|
import { loadRunManifestById } from "../state/stores/state-store.ts";
|
|
23
26
|
import type { TeamTaskState } from "../state/types.ts";
|
|
24
27
|
import { readCrewAgents } from "./crew-agent-records.ts";
|
|
@@ -128,6 +131,170 @@ function resolveCheckIntervalMs(overrideMs: number | undefined): number {
|
|
|
128
131
|
return DEFAULT_CHECK_INTERVAL_MS;
|
|
129
132
|
}
|
|
130
133
|
|
|
134
|
+
// ── Wave 13 (F1, GAP-0): hung-verdict liveness short-circuit ────────────────
|
|
135
|
+
//
|
|
136
|
+
// Live evidence (2026-10-24/25): five false "appears hung" alerts in one
|
|
137
|
+
// session, all direct single-agent runs executing ONE long tool call (a
|
|
138
|
+
// 20-min CI poll loop, a 20-min test suite). The hung verdict
|
|
139
|
+
// (isLikelyOrphanedActiveRun: manifest stale ≥2 min + no agent activity
|
|
140
|
+
// within 5 min) is structurally blind while a tool executes — every
|
|
141
|
+
// liveness write is event-driven and no pi events fire mid-tool — so a
|
|
142
|
+
// HEALTHY run reads as hung ~5-10 min into a long tool. The two signals
|
|
143
|
+
// other consumers already trust, and this verdict ignores, are consulted
|
|
144
|
+
// here IN THE WATCHDOG (never inside isLikelyOrphanedActiveRun — that
|
|
145
|
+
// predicate is shared with the dashboard's isDisplayActiveRun, GAP-4):
|
|
146
|
+
// (a) a LIVE active-run-registry claim, and
|
|
147
|
+
// (b) a fresh task heartbeat (kept fresh through long tools by F2's
|
|
148
|
+
// live-executor timer).
|
|
149
|
+
// The two branches are INDEPENDENT ORs: the registry drops non-async
|
|
150
|
+
// entries once the manifest is >30 min stale, so past that horizon only the
|
|
151
|
+
// heartbeat carries liveness — never require both.
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Wave 13 (F1, GAP-0): hard ceiling for how long a LIVE active-run-registry
|
|
155
|
+
* claim can shield a run from the watchdog's hung verdict. A claimed run
|
|
156
|
+
* whose NEWEST liveness evidence (claim assertion time, task heartbeats/
|
|
157
|
+
* activity) is older than this still alerts — a genuine in-process deadlock
|
|
158
|
+
* (claim held, nothing progressing) is still caught — while healthy claimed
|
|
159
|
+
* runs mid-long-tool are not false-alarmed by the 2-min ladder. 45 min is
|
|
160
|
+
* 1.5× the registry's own 30-min freshness horizon and 9× the 5-min activity
|
|
161
|
+
* shield, so only a truly stuck claim trips it. Unclaimed runs keep the
|
|
162
|
+
* fast 2-min ladder unchanged; ORPHANED_ACTIVE_RUN_MS (process-status.ts,
|
|
163
|
+
* shared with the dashboard — GAP-4) is NOT touched.
|
|
164
|
+
*/
|
|
165
|
+
export const CLAIMED_RUN_HUNG_MS = 45 * 60 * 1000;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Wave 13 (F1): freshness window for task-heartbeat liveness evidence —
|
|
169
|
+
* mirrors ACTIVE_EVIDENCE_TTL_MS in stale-reconciler.ts (5 min).
|
|
170
|
+
*/
|
|
171
|
+
const TASK_HEARTBEAT_EVIDENCE_MS = 5 * 60 * 1000;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Wave 13 (F1): read-only active-run-registry claim consult result.
|
|
175
|
+
* `claimedAtMs` is the registry entry's `updatedAt` (the moment the claim
|
|
176
|
+
* was asserted — registerActiveRun fires at dispatch); undefined when the
|
|
177
|
+
* entry carries no parseable timestamp.
|
|
178
|
+
*/
|
|
179
|
+
interface LiveRunClaim {
|
|
180
|
+
claimedAtMs?: number;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Wave 13 (F1, GAP-0): consult the active-run registry for a LIVE claim on
|
|
185
|
+
* `runId`. Read-only mirror of findLiveRegistryEntry (stale-reconciler G24):
|
|
186
|
+
* activeRunEntries() applies the full liveness filter itself (terminal
|
|
187
|
+
* status, dead async PID, >30-min-stale non-async manifests, symlink
|
|
188
|
+
* safety), so an entry found here means some live session/runner claims the
|
|
189
|
+
* run RIGHT NOW. Inlined via the registry store instead of importing
|
|
190
|
+
* stale-reconciler (its reaping graph must not enter the watchdog's import
|
|
191
|
+
* closure); the async-PID half of hasLiveRunClaim is deliberately omitted —
|
|
192
|
+
* async runs never reach this verdict path (isLikelyOrphanedActiveRun
|
|
193
|
+
* excludes them). Fail-open: any registry error ⇒ no claim — a broken
|
|
194
|
+
* registry must never shield a genuinely hung run (G24 contract).
|
|
195
|
+
*/
|
|
196
|
+
function consultActiveRunRegistry(runId: string): LiveRunClaim | undefined {
|
|
197
|
+
try {
|
|
198
|
+
const entry = activeRunEntries().find((item) => item.runId === runId);
|
|
199
|
+
if (!entry) return undefined;
|
|
200
|
+
const claimedAtMs = entry.updatedAt ? Date.parse(entry.updatedAt) : Number.NaN;
|
|
201
|
+
return { claimedAtMs: Number.isFinite(claimedAtMs) ? claimedAtMs : undefined };
|
|
202
|
+
} catch {
|
|
203
|
+
return undefined;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
let liveClaimConsult: (runId: string) => LiveRunClaim | undefined = consultActiveRunRegistry;
|
|
208
|
+
|
|
209
|
+
/** @internal Test seam — replace the registry claim consult (null = real). */
|
|
210
|
+
export function setLiveClaimConsultForTests(consult: ((runId: string) => LiveRunClaim | undefined) | null): void {
|
|
211
|
+
liveClaimConsult = consult ?? consultActiveRunRegistry;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Wave 13 (F1): does any running/waiting task carry FRESH live evidence?
|
|
216
|
+
* Behavioral mirror of hasRecentActiveEvidence (stale-reconciler.ts): a
|
|
217
|
+
* running/waiting task whose `heartbeat.lastSeenAt` (with `alive !== false`)
|
|
218
|
+
* or `agentProgress.lastActivityAt` is within TASK_HEARTBEAT_EVIDENCE_MS
|
|
219
|
+
* counts. Absent/unreadable tasks (`[]`) ⇒ false — fail-open to the ladder,
|
|
220
|
+
* same as WD-STALE-RECORD.
|
|
221
|
+
*/
|
|
222
|
+
function hasFreshTaskHeartbeatEvidence(tasks: TeamTaskState[] | undefined, now: number): boolean {
|
|
223
|
+
if (!Array.isArray(tasks)) return false;
|
|
224
|
+
return tasks.some((task) => {
|
|
225
|
+
if (task.status !== "running" && task.status !== "waiting") return false;
|
|
226
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
227
|
+
if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= TASK_HEARTBEAT_EVIDENCE_MS) return true;
|
|
228
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
229
|
+
return Number.isFinite(activityAt) && now - activityAt <= TASK_HEARTBEAT_EVIDENCE_MS;
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Wave 13 (F1): newest liveness-evidence timestamp (epoch ms) among the
|
|
235
|
+
* claim assertion time and the running/waiting tasks' heartbeat/activity
|
|
236
|
+
* marks — everything EXCEPT manifest.updatedAt, which is the frozen
|
|
237
|
+
* artifact under judgment, not evidence (the registry's own filter already
|
|
238
|
+
* bounds it at 30 min for entry presence; counting it here would make the
|
|
239
|
+
* CLAIMED_RUN_HUNG_MS ceiling unreachable). undefined when no signal
|
|
240
|
+
* carries a finite timestamp.
|
|
241
|
+
*/
|
|
242
|
+
function newestClaimEvidenceMs(tasks: TeamTaskState[] | undefined, claim: LiveRunClaim): number | undefined {
|
|
243
|
+
let newest: number | undefined;
|
|
244
|
+
const consider = (ms: number): void => {
|
|
245
|
+
if (Number.isFinite(ms) && (newest === undefined || ms > newest)) newest = ms;
|
|
246
|
+
};
|
|
247
|
+
if (claim.claimedAtMs !== undefined) consider(claim.claimedAtMs);
|
|
248
|
+
for (const task of tasks ?? []) {
|
|
249
|
+
if (task.status !== "running" && task.status !== "waiting") continue;
|
|
250
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
251
|
+
consider(heartbeatAt);
|
|
252
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
253
|
+
consider(activityAt);
|
|
254
|
+
}
|
|
255
|
+
return newest;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Wave 13 (F1): hung-verdict liveness short-circuit result. */
|
|
259
|
+
interface LivenessShortCircuit {
|
|
260
|
+
/** True ⇒ the hung verdict is withheld this cycle (live evidence present). */
|
|
261
|
+
suppress: boolean;
|
|
262
|
+
/** Debug-log reason (PI_CREW_DEBUG-gated). */
|
|
263
|
+
reason: string;
|
|
264
|
+
/** Claim present but its evidence aged past CLAIMED_RUN_HUNG_MS — the alert detail must name it. */
|
|
265
|
+
claimExhausted: boolean;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Wave 13 (F1, GAP-0): should the hung verdict for this cycle be withheld?
|
|
270
|
+
* Two INDEPENDENT OR branches (never require both):
|
|
271
|
+
* (b) a fresh task heartbeat (≤5 min) ⇒ suppress outright;
|
|
272
|
+
* (a) a live registry claim ⇒ suppress while the newest liveness evidence
|
|
273
|
+
* is ≤ CLAIMED_RUN_HUNG_MS old; past the ceiling the ladder proceeds
|
|
274
|
+
* (claimExhausted ⇒ the alert names the claimed-run threshold).
|
|
275
|
+
* No claim and no fresh heartbeat ⇒ the existing 2-min ladder is unchanged
|
|
276
|
+
* (an unclaimed zombie still alerts fast).
|
|
277
|
+
*/
|
|
278
|
+
function assessClaimedRunLiveness(runId: string, tasks: TeamTaskState[] | undefined, now: number): LivenessShortCircuit {
|
|
279
|
+
if (hasFreshTaskHeartbeatEvidence(tasks, now)) {
|
|
280
|
+
return { suppress: true, reason: "fresh task heartbeat", claimExhausted: false };
|
|
281
|
+
}
|
|
282
|
+
const claim = liveClaimConsult(runId);
|
|
283
|
+
if (claim) {
|
|
284
|
+
const evidenceMs = newestClaimEvidenceMs(tasks, claim);
|
|
285
|
+
const evidenceAgeMs = evidenceMs !== undefined ? now - evidenceMs : undefined;
|
|
286
|
+
if (evidenceAgeMs !== undefined && evidenceAgeMs <= CLAIMED_RUN_HUNG_MS) {
|
|
287
|
+
return {
|
|
288
|
+
suppress: true,
|
|
289
|
+
reason: `live registry claim (newest evidence ${Math.max(0, Math.round(evidenceAgeMs / 1000))}s old)`,
|
|
290
|
+
claimExhausted: false,
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
return { suppress: false, reason: "claim evidence older than CLAIMED_RUN_HUNG_MS", claimExhausted: true };
|
|
294
|
+
}
|
|
295
|
+
return { suppress: false, reason: "no live claim", claimExhausted: false };
|
|
296
|
+
}
|
|
297
|
+
|
|
131
298
|
/**
|
|
132
299
|
* Pre-emit state re-verify loader. Injectable for tests to simulate the run
|
|
133
300
|
* going terminal between the cycle's initial read and the emit point.
|
|
@@ -471,39 +638,59 @@ export function startForegroundWatchdog(opts: WatchdogOptions): void {
|
|
|
471
638
|
// Keep monitoring: if the manifest eventually finalizes, the
|
|
472
639
|
// terminal branch delivers the completion notice.
|
|
473
640
|
} else {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
641
|
+
// Wave 13 (F1, GAP-0): liveness short-circuit BEFORE any hung alert.
|
|
642
|
+
// The verdict above is structurally blind while a tool executes; a
|
|
643
|
+
// live registry claim or fresh task heartbeat withholds it. A
|
|
644
|
+
// suppressed cycle means the run left the hung state — reset the
|
|
645
|
+
// no-progress counter so a later genuine alert gets the full drip cap.
|
|
646
|
+
const liveness = assessClaimedRunLiveness(runId, reverified.tasks, now);
|
|
647
|
+
if (liveness.suppress) {
|
|
648
|
+
consecutiveHungNotices = 0;
|
|
649
|
+
debugLog(`[foreground-watchdog] suppress hung verdict for ${runId}: ${liveness.reason}`);
|
|
650
|
+
} else {
|
|
651
|
+
// Wave 13 (F3, GAP-2): the detail carries its EVIDENCE time (the
|
|
652
|
+
// manifest read the verdict was based on) — "as of <ISO>" — not the
|
|
653
|
+
// emit time, so a user re-checking can tell a stale/queued alert
|
|
654
|
+
// apart from one about the last few minutes. An exhausted claim
|
|
655
|
+
// names the claimed-run threshold so the user knows it is not the
|
|
656
|
+
// 2-min ladder.
|
|
657
|
+
const claimNote = liveness.claimExhausted
|
|
658
|
+
? `, claimed-run hung threshold exceeded (no live evidence for >${Math.round(CLAIMED_RUN_HUNG_MS / 60_000)} min)`
|
|
659
|
+
: "";
|
|
660
|
+
const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}, as of ${manifest.updatedAt}${claimNote}`;
|
|
661
|
+
consecutiveHungNotices += 1;
|
|
662
|
+
try {
|
|
663
|
+
if (consecutiveHungNotices <= HUNG_NOTICE_CAP) {
|
|
664
|
+
// WD-DEDUP: alert-once per runId at module scope — even a
|
|
665
|
+
// fresh watchdog instance must not re-alert the same run.
|
|
666
|
+
if (!hungAlertAlreadySent(runId)) {
|
|
667
|
+
// Mark BEFORE queueing so a racing cycle cannot double-queue.
|
|
668
|
+
updateHungAlertRecord(runId, { alerted: true });
|
|
669
|
+
// WD-DELIVER-GUARD: the host queues a followUp while the
|
|
670
|
+
// session streams and delivers it arbitrarily later — so
|
|
671
|
+
// hand it over only at the idle boundary, re-verifying the
|
|
672
|
+
// run state at THAT moment.
|
|
673
|
+
queueHungAlertDelivery(
|
|
674
|
+
pi,
|
|
675
|
+
cwd,
|
|
676
|
+
runId,
|
|
677
|
+
`pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
|
|
678
|
+
opts.isSessionIdle,
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
} else if (consecutiveHungNotices === HUNG_NOTICE_CAP + 1) {
|
|
682
|
+
// One-time handoff, then silence — still monitoring, no more drips.
|
|
487
683
|
queueHungAlertDelivery(
|
|
488
684
|
pi,
|
|
489
685
|
cwd,
|
|
490
686
|
runId,
|
|
491
|
-
`pi-crew watchdog: run ${runId}
|
|
687
|
+
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
492
688
|
opts.isSessionIdle,
|
|
493
689
|
);
|
|
494
690
|
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
queueHungAlertDelivery(
|
|
498
|
-
pi,
|
|
499
|
-
cwd,
|
|
500
|
-
runId,
|
|
501
|
-
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
502
|
-
opts.isSessionIdle,
|
|
503
|
-
);
|
|
691
|
+
} catch {
|
|
692
|
+
/* non-critical */
|
|
504
693
|
}
|
|
505
|
-
} catch {
|
|
506
|
-
/* non-critical */
|
|
507
694
|
}
|
|
508
695
|
}
|
|
509
696
|
// Keep monitoring (ARCH-5: silently past the cap — no more notices).
|
|
@@ -97,11 +97,51 @@ export interface ForeignPiProcess {
|
|
|
97
97
|
cmd: string;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
/** Wave-14 F4 (GAP-3): injectable seam for `scanZombieSubagents`.
|
|
101
|
+
*
|
|
102
|
+
* Every ambient read the scanner performs is replaceable so classification
|
|
103
|
+
* tests can drive a SYNTHETIC /proc on every OS (previously 7 tests skipped
|
|
104
|
+
* forever on macOS/Windows CI). Defaults preserve behavior EXACTLY — each
|
|
105
|
+
* read-site resolves `opts.X ?? <the previous inline expression>` (the same
|
|
106
|
+
* injection pattern as reap-group.ts ReapGroupOptions, wave 12b). Production
|
|
107
|
+
* callers are no-arg (doctor.ts, orphan-reaper.ts) and see zero change. */
|
|
108
|
+
export interface ZombieScanOptions {
|
|
109
|
+
/** Platform gate. Default `process.platform`. */
|
|
110
|
+
platform?: NodeJS.Platform;
|
|
111
|
+
/** Real uid of the scanner process. Default `tryGetUid` (try/catch around
|
|
112
|
+
* `process.getuid?.()` — undefined on win32). */
|
|
113
|
+
getuid?: () => number | undefined;
|
|
114
|
+
/** File reader for /proc paths. Default `fs.readFileSync(p, "utf-8")` with
|
|
115
|
+
* read errors mapped to `undefined` (the previous per-helper catch). */
|
|
116
|
+
readProcFile?: (p: string) => string | undefined;
|
|
117
|
+
/** Candidate pid enumeration. Default: readdir of the real `/proc`. */
|
|
118
|
+
listProcPids?: () => number[];
|
|
119
|
+
/** Pid liveness probe. Default `isPidAlive` (`process.kill(pid, 0)` with the
|
|
120
|
+
* R7-A2 ESRCH/ENOENT semantics). */
|
|
121
|
+
pidAlive?: (pid: number) => boolean;
|
|
122
|
+
/** Scanner's own parent pid — the ancestor-chain starting point. Default
|
|
123
|
+
* `process.ppid`. Injection point for deterministic ancestor tests. */
|
|
124
|
+
getParentPid?: () => number;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Normalized /proc read: `undefined` = unreadable (missing, EPERM, …). */
|
|
128
|
+
type ReadProcFile = (p: string) => string | undefined;
|
|
129
|
+
|
|
130
|
+
/** Default seam reader — real fs with the previous per-helper catch folded in. */
|
|
131
|
+
const defaultReadProcFile: ReadProcFile = (p) => {
|
|
132
|
+
try {
|
|
133
|
+
return fs.readFileSync(p, "utf-8");
|
|
134
|
+
} catch {
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
|
|
100
139
|
/** Read /proc/<pid>/environ as a key=value record. Returns {} if unreadable. */
|
|
101
|
-
function readProcEnviron(pid: number): Record<string, string> {
|
|
140
|
+
function readProcEnviron(pid: number, read: ReadProcFile = defaultReadProcFile): Record<string, string> {
|
|
102
141
|
try {
|
|
103
142
|
// /proc/<pid>/environ is NUL-separated key=value pairs.
|
|
104
|
-
const raw =
|
|
143
|
+
const raw = read(`/proc/${pid}/environ`);
|
|
144
|
+
if (raw === undefined) return {};
|
|
105
145
|
const out: Record<string, string> = {};
|
|
106
146
|
for (const entry of raw.split("\0")) {
|
|
107
147
|
const eq = entry.indexOf("=");
|
|
@@ -114,9 +154,13 @@ function readProcEnviron(pid: number): Record<string, string> {
|
|
|
114
154
|
}
|
|
115
155
|
|
|
116
156
|
/** Read /proc/<pid>/stat to get ppid + starttime + elapsed. Returns undefined if unreadable. */
|
|
117
|
-
function readProcStat(
|
|
157
|
+
function readProcStat(
|
|
158
|
+
pid: number,
|
|
159
|
+
read: ReadProcFile = defaultReadProcFile,
|
|
160
|
+
): { ppid: number; starttime: number | undefined; elapsedSec: number | undefined } | undefined {
|
|
118
161
|
try {
|
|
119
|
-
const stat =
|
|
162
|
+
const stat = read(`/proc/${pid}/stat`);
|
|
163
|
+
if (stat === undefined) return undefined;
|
|
120
164
|
// stat format: pid (comm) state ppid ... starttime ...
|
|
121
165
|
// Shared paren-aware cut (proc-stat.ts) — MUST index identically with
|
|
122
166
|
// surface-spawn's PI_CREW_PARENT_START_TIME capture.
|
|
@@ -125,7 +169,7 @@ function readProcStat(pid: number): { ppid: number; starttime: number | undefine
|
|
|
125
169
|
const ppid = Number.parseInt(rest[PROC_STAT_PPID_INDEX] ?? "", 10);
|
|
126
170
|
const starttimeTicksRaw = Number.parseInt(rest[PROC_STAT_STARTTIME_INDEX] ?? "", 10);
|
|
127
171
|
const starttime = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
|
|
128
|
-
const elapsedSec = computeElapsedSec(starttime);
|
|
172
|
+
const elapsedSec = computeElapsedSec(starttime, read);
|
|
129
173
|
return { ppid: Number.isFinite(ppid) ? ppid : 0, starttime, elapsedSec };
|
|
130
174
|
} catch {
|
|
131
175
|
return undefined;
|
|
@@ -133,9 +177,10 @@ function readProcStat(pid: number): { ppid: number; starttime: number | undefine
|
|
|
133
177
|
}
|
|
134
178
|
|
|
135
179
|
/** /proc/uptime first field (seconds since boot); undefined when unreadable. */
|
|
136
|
-
function readUptimeSec(): number | undefined {
|
|
180
|
+
function readUptimeSec(read: ReadProcFile = defaultReadProcFile): number | undefined {
|
|
137
181
|
try {
|
|
138
|
-
const uptimeRaw =
|
|
182
|
+
const uptimeRaw = read("/proc/uptime");
|
|
183
|
+
if (uptimeRaw === undefined) return undefined;
|
|
139
184
|
const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
|
|
140
185
|
return Number.isFinite(uptimeSec) ? uptimeSec : undefined;
|
|
141
186
|
} catch {
|
|
@@ -148,22 +193,22 @@ function readUptimeSec(): number | undefined {
|
|
|
148
193
|
* ticks — boot time = now - uptime, so startedAt = boot + starttime/CLK_TCK.
|
|
149
194
|
* Best-effort ISO string; undefined when /proc/uptime is unavailable.
|
|
150
195
|
*/
|
|
151
|
-
function computeStartedAtIso(starttime: number | undefined): string | undefined {
|
|
196
|
+
function computeStartedAtIso(starttime: number | undefined, read: ReadProcFile = defaultReadProcFile): string | undefined {
|
|
152
197
|
if (starttime === undefined || !Number.isFinite(starttime)) return undefined;
|
|
153
|
-
const uptimeSec = readUptimeSec();
|
|
198
|
+
const uptimeSec = readUptimeSec(read);
|
|
154
199
|
if (uptimeSec === undefined) return undefined;
|
|
155
200
|
const bootMs = Date.now() - uptimeSec * 1000;
|
|
156
201
|
return new Date(bootMs + (starttime / 100) * 1000).toISOString();
|
|
157
202
|
}
|
|
158
203
|
|
|
159
|
-
function computeElapsedSec(starttimeTicks: number | undefined): number | undefined {
|
|
204
|
+
function computeElapsedSec(starttimeTicks: number | undefined, read: ReadProcFile = defaultReadProcFile): number | undefined {
|
|
160
205
|
if (starttimeTicks === undefined || !Number.isFinite(starttimeTicks)) return undefined;
|
|
161
206
|
try {
|
|
162
207
|
// Linux CLK_TCK is virtually always 100 (sysconf(_SC_CLK_TCK)). Reading it
|
|
163
208
|
// portably from Node requires a native addon; hardcoding 100 matches every
|
|
164
209
|
// mainstream Linux distro and keeps this dependency-free.
|
|
165
210
|
const ticksPerSec = 100;
|
|
166
|
-
const uptimeSec = readUptimeSec();
|
|
211
|
+
const uptimeSec = readUptimeSec(read);
|
|
167
212
|
if (uptimeSec === undefined) return undefined;
|
|
168
213
|
// starttime (ticks since boot) → process age in seconds = uptime - starttime/ticksPerSec.
|
|
169
214
|
const startAgeSec = starttimeTicks / ticksPerSec;
|
|
@@ -188,24 +233,26 @@ function isPidAlive(pid: number): boolean {
|
|
|
188
233
|
}
|
|
189
234
|
}
|
|
190
235
|
|
|
191
|
-
function readProcCmdline(pid: number): string {
|
|
192
|
-
return readProcCmdlineTokens(pid).join(" ").trim() || `pid ${pid}`;
|
|
236
|
+
function readProcCmdline(pid: number, read: ReadProcFile = defaultReadProcFile): string {
|
|
237
|
+
return readProcCmdlineTokens(pid, read).join(" ").trim() || `pid ${pid}`;
|
|
193
238
|
}
|
|
194
239
|
|
|
195
240
|
/** /proc/<pid>/cmdline as argv tokens (NUL-split). Empty array if unreadable. */
|
|
196
|
-
function readProcCmdlineTokens(pid: number): string[] {
|
|
241
|
+
function readProcCmdlineTokens(pid: number, read: ReadProcFile = defaultReadProcFile): string[] {
|
|
197
242
|
try {
|
|
198
243
|
// /proc/<pid>/cmdline is NUL-separated argv.
|
|
199
|
-
const raw =
|
|
244
|
+
const raw = read(`/proc/${pid}/cmdline`);
|
|
245
|
+
if (raw === undefined) return [];
|
|
200
246
|
return raw.split("\0").filter(Boolean);
|
|
201
247
|
} catch {
|
|
202
248
|
return [];
|
|
203
249
|
}
|
|
204
250
|
}
|
|
205
251
|
|
|
206
|
-
function readProcRssKb(pid: number): number {
|
|
252
|
+
function readProcRssKb(pid: number, read: ReadProcFile = defaultReadProcFile): number {
|
|
207
253
|
try {
|
|
208
|
-
const status =
|
|
254
|
+
const status = read(`/proc/${pid}/status`);
|
|
255
|
+
if (status === undefined) return 0;
|
|
209
256
|
const match = status.match(/^VmRSS:\s+(\d+)\s+kB/m);
|
|
210
257
|
return match ? Number.parseInt(match[1] ?? "", 10) : 0;
|
|
211
258
|
} catch {
|
|
@@ -221,8 +268,8 @@ function readProcRssKb(pid: number): number {
|
|
|
221
268
|
* non-Linux platforms the scanner returns an empty result with a note in
|
|
222
269
|
* `errors` (zombie detection is best-effort; the doctor report still renders).
|
|
223
270
|
*/
|
|
224
|
-
function listCandidatePids(): number[] {
|
|
225
|
-
if (
|
|
271
|
+
function listCandidatePids(platform: NodeJS.Platform = process.platform): number[] {
|
|
272
|
+
if (platform !== "linux") return [];
|
|
226
273
|
const pids: number[] = [];
|
|
227
274
|
try {
|
|
228
275
|
for (const entry of fs.readdirSync("/proc")) {
|
|
@@ -241,23 +288,30 @@ function listCandidatePids(): number[] {
|
|
|
241
288
|
* any scan errors. Callers decide what to do with the result; this module
|
|
242
289
|
* has no side effects.
|
|
243
290
|
*/
|
|
244
|
-
export function scanZombieSubagents(): ZombieScanResult {
|
|
291
|
+
export function scanZombieSubagents(opts: ZombieScanOptions = {}): ZombieScanResult {
|
|
292
|
+
// Wave-14 F4: resolve the seam ONCE — every default below is the exact
|
|
293
|
+
// previous inline expression, so no-arg callers are byte-for-byte behavior
|
|
294
|
+
// preserved (doctor.ts:990, orphan-reaper.ts:306).
|
|
295
|
+
const platform = opts.platform ?? process.platform;
|
|
296
|
+
const read = opts.readProcFile ?? defaultReadProcFile;
|
|
297
|
+
const pidAlive = opts.pidAlive ?? isPidAlive;
|
|
245
298
|
const result: ZombieScanResult = { zombies: [], live: [], foreign: [], errors: [] };
|
|
246
|
-
if (
|
|
247
|
-
result.errors.push("zombie scan is Linux-only (/proc required); skipping on " +
|
|
299
|
+
if (platform !== "linux") {
|
|
300
|
+
result.errors.push("zombie scan is Linux-only (/proc required); skipping on " + platform);
|
|
248
301
|
return result;
|
|
249
302
|
}
|
|
250
303
|
|
|
251
|
-
const myUid = tryGetUid();
|
|
304
|
+
const myUid = (opts.getuid ?? tryGetUid)();
|
|
252
305
|
// R3-16: self + ancestors are live hosting sessions (main session / crew
|
|
253
306
|
// host chain) — never reportable as foreign, even when they carry markers.
|
|
254
|
-
const ancestors = collectAncestorPids();
|
|
255
|
-
|
|
307
|
+
const ancestors = collectAncestorPids(read, opts.getParentPid);
|
|
308
|
+
const listPids = opts.listProcPids ?? (() => listCandidatePids(platform));
|
|
309
|
+
for (const pid of listPids()) {
|
|
256
310
|
try {
|
|
257
311
|
// Cheap rejection first: only inspect processes we own (avoid scanning system procs).
|
|
258
|
-
if (myUid !== undefined && getProcUid(pid) !== myUid) continue;
|
|
312
|
+
if (myUid !== undefined && getProcUid(pid, read) !== myUid) continue;
|
|
259
313
|
|
|
260
|
-
const environ = readProcEnviron(pid);
|
|
314
|
+
const environ = readProcEnviron(pid, read);
|
|
261
315
|
// AUTHORITATIVE GATE: a process is a pi-crew sub-agent ONLY if it carries
|
|
262
316
|
// PI_CREW_KIND=subagent. The user's main session never sets this, so it can
|
|
263
317
|
// never be matched — this is the fix for accidentally killing main sessions.
|
|
@@ -271,18 +325,18 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
271
325
|
// them (pi-crew also scrubs these markers from worker env).
|
|
272
326
|
const hasPiMarker = environ.AI_AGENT === "pi" || environ.PI_CODING_AGENT === "true";
|
|
273
327
|
if (hasPiMarker && pid !== process.pid && !ancestors.has(pid)) {
|
|
274
|
-
const tokens = readProcCmdlineTokens(pid);
|
|
328
|
+
const tokens = readProcCmdlineTokens(pid, read);
|
|
275
329
|
if (looksLikePiBinary(tokens)) {
|
|
276
|
-
const foreignStat = readProcStat(pid);
|
|
330
|
+
const foreignStat = readProcStat(pid, read);
|
|
277
331
|
const ppid = foreignStat?.ppid ?? 0;
|
|
278
332
|
result.foreign.push({
|
|
279
333
|
pid,
|
|
280
334
|
ppid,
|
|
281
|
-
orphaned: ppid <= 1 || !
|
|
335
|
+
orphaned: ppid <= 1 || !pidAlive(ppid),
|
|
282
336
|
headless: isHeadlessArgv(tokens),
|
|
283
|
-
rssKb: readProcRssKb(pid),
|
|
337
|
+
rssKb: readProcRssKb(pid, read),
|
|
284
338
|
elapsedSec: foreignStat?.elapsedSec,
|
|
285
|
-
cmd: readProcCmdline(pid),
|
|
339
|
+
cmd: readProcCmdline(pid, read),
|
|
286
340
|
});
|
|
287
341
|
}
|
|
288
342
|
}
|
|
@@ -290,7 +344,7 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
290
344
|
}
|
|
291
345
|
|
|
292
346
|
const crewParentPid = Number.parseInt(environ.PI_CREW_PARENT_PID ?? "", 10);
|
|
293
|
-
const stat = readProcStat(pid);
|
|
347
|
+
const stat = readProcStat(pid, read);
|
|
294
348
|
// Surface identity (T12): strict allowlist — an unrelated PI_CREW_SURFACE
|
|
295
349
|
// value must not half-populate the fields doctor relies on for pane close.
|
|
296
350
|
const surface = environ.PI_CREW_SURFACE === "tmux" || environ.PI_CREW_SURFACE === "herdr" ? environ.PI_CREW_SURFACE : undefined;
|
|
@@ -298,17 +352,17 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
298
352
|
pid,
|
|
299
353
|
ppid: stat?.ppid ?? 0,
|
|
300
354
|
crewParentPid: Number.isFinite(crewParentPid) ? crewParentPid : 0,
|
|
301
|
-
parentAlive: Number.isFinite(crewParentPid) &&
|
|
355
|
+
parentAlive: Number.isFinite(crewParentPid) && pidAlive(crewParentPid),
|
|
302
356
|
role: environ.PI_CREW_ROLE,
|
|
303
357
|
surface,
|
|
304
358
|
surfacePaneId: environ.PI_CREW_SURFACE_PANE || undefined,
|
|
305
|
-
rssKb: readProcRssKb(pid),
|
|
359
|
+
rssKb: readProcRssKb(pid, read),
|
|
306
360
|
elapsedSec: stat?.elapsedSec,
|
|
307
361
|
// M-7: starttime (ticks) + derived wall-clock start — pid identity pin
|
|
308
362
|
// for kill-time PID-reuse checks and prompt-kill triage.
|
|
309
363
|
starttime: stat?.starttime,
|
|
310
|
-
startedAt: computeStartedAtIso(stat?.starttime),
|
|
311
|
-
cmd: readProcCmdline(pid),
|
|
364
|
+
startedAt: computeStartedAtIso(stat?.starttime, read),
|
|
365
|
+
cmd: readProcCmdline(pid, read),
|
|
312
366
|
};
|
|
313
367
|
|
|
314
368
|
if (entry.parentAlive) {
|
|
@@ -338,10 +392,11 @@ function tryGetUid(): number | undefined {
|
|
|
338
392
|
}
|
|
339
393
|
}
|
|
340
394
|
|
|
341
|
-
function getProcUid(pid: number): number | undefined {
|
|
395
|
+
function getProcUid(pid: number, read: ReadProcFile = defaultReadProcFile): number | undefined {
|
|
342
396
|
try {
|
|
343
397
|
// /proc/<pid>/status has Uid: <real> <eff> <sav> <fs>
|
|
344
|
-
const status =
|
|
398
|
+
const status = read(`/proc/${pid}/status`);
|
|
399
|
+
if (status === undefined) return undefined;
|
|
345
400
|
const match = status.match(/^Uid:\s+(\d+)/m);
|
|
346
401
|
return match ? Number.parseInt(match[1] ?? "", 10) : undefined;
|
|
347
402
|
} catch {
|
|
@@ -393,13 +448,13 @@ function isHeadlessArgv(tokens: string[]): boolean {
|
|
|
393
448
|
* it sit on this chain and carry markers when pi-in-pi). Bounded at 64 hops to
|
|
394
449
|
* survive a hypothetical pid cycle.
|
|
395
450
|
*/
|
|
396
|
-
function collectAncestorPids(): Set<number> {
|
|
451
|
+
function collectAncestorPids(read: ReadProcFile = defaultReadProcFile, getParentPid?: () => number): Set<number> {
|
|
397
452
|
const chain = new Set<number>();
|
|
398
|
-
let current = process.ppid;
|
|
453
|
+
let current = getParentPid !== undefined ? getParentPid() : process.ppid;
|
|
399
454
|
let guard = 0;
|
|
400
455
|
while (current > 1 && guard++ < 64) {
|
|
401
456
|
chain.add(current);
|
|
402
|
-
const stat = readProcStat(current);
|
|
457
|
+
const stat = readProcStat(current, read);
|
|
403
458
|
if (!stat || stat.ppid <= 0) break;
|
|
404
459
|
current = stat.ppid;
|
|
405
460
|
}
|