pi-crew 0.11.37 → 0.11.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +89 -0
  2. package/agents/critic.md +13 -0
  3. package/agents/oracle.md +14 -0
  4. package/dist/index.mjs +531 -526
  5. package/package.json +4 -3
  6. package/skills/delegation-patterns/SKILL.md +16 -0
  7. package/skills/real-test-pi-crew/SKILL.md +13 -2
  8. package/src/config/env-allowlist-registry.ts +22 -1
  9. package/src/config/env-vars.ts +23 -0
  10. package/src/extension/registration/foreground-run-controller.ts +21 -1
  11. package/src/extension/registration/lifecycle-handlers.ts +12 -0
  12. package/src/extension/team-tool/doctor.ts +125 -1
  13. package/src/prompt/prompt-runtime.ts +67 -5
  14. package/src/runtime/child-pi/child-pi-kill.ts +68 -2
  15. package/src/runtime/child-pi/child-pi-spawn.ts +20 -0
  16. package/src/runtime/child-pi/child-pi.ts +3 -1
  17. package/src/runtime/foreground-watchdog.ts +394 -17
  18. package/src/runtime/heartbeat/worker-heartbeat.ts +4 -0
  19. package/src/runtime/model/model-fallback.ts +5 -0
  20. package/src/runtime/process/orphan-reaper.ts +197 -22
  21. package/src/runtime/process/proc-stat.ts +8 -0
  22. package/src/runtime/process/reap-group.ts +499 -0
  23. package/src/runtime/process/run-descendant-sweep.ts +744 -0
  24. package/src/runtime/recovery/crash-recovery.ts +202 -0
  25. package/src/runtime/rpc/frame-client.ts +21 -1
  26. package/src/runtime/stale-reconciler.ts +343 -2
  27. package/src/runtime/task-runner/child-executor.ts +15 -1
  28. package/src/runtime/task-runner/post-execution.ts +112 -1
  29. package/src/runtime/task-runner/state-helpers.ts +28 -1
  30. package/src/runtime/verification/verify-gate-memo.ts +127 -0
  31. package/src/runtime/verification/verify-gate.ts +127 -1
  32. package/src/state/coordination/mailbox.ts +68 -0
  33. package/src/state/stores/state-store.ts +8 -0
  34. package/src/state/types.ts +61 -0
@@ -0,0 +1,744 @@
1
+ /**
2
+ * run-descendant-sweep.ts — fix-wave 11b (F7): reap the DESCENDANTS of a
3
+ * TERMINAL pi-crew run by matching the pi-crew env markers they inherited.
4
+ *
5
+ * Why this module exists (the incident, SPEC §1): run
6
+ * `team_20261022110510_cefb9164af126460` reached `completed` at 2026-10-22T13:02Z
7
+ * and still had SIX grandchildren alive 24 h later (`dpi-backend` ×5, `caddy`;
8
+ * 399 MB RSS). They carried `PI_CREW_KIND=subagent` and
9
+ * `PI_CREW_PARENT_PID=<dead runner pid>`. No layer of the 11a teardown stack
10
+ * could reach them, and never will: pi's bash tool spawns EVERY command with
11
+ * `detached: true` (GAP-0), so each bash command gets its OWN process group.
12
+ * A group kill rooted at the worker reaches the worker's group only — the bash
13
+ * grandchildren live in a group of their own.
14
+ *
15
+ * The only identity that survives that topology is the ENVIRONMENT: the child
16
+ * inherits the worker's env, so every descendant still carries the pi-crew
17
+ * markers of the run it belongs to. This module therefore turns env markers
18
+ * into kill candidates, scoped to ONE run, and only after that run is terminal.
19
+ *
20
+ * Markers actually used (VERIFIED in src — never invented):
21
+ * - `PI_CREW_BROKER_RUN_ID` — `child-pi-spawn.ts:382` writes
22
+ * `built.env.PI_CREW_BROKER_RUN_ID = input.runId` for every team worker
23
+ * (the env key is spelled by `PI_CREW_RUN_ID_ENV` in
24
+ * `scratchpad-lifecycle.ts:66`, whose VALUE is the broker name). This is
25
+ * the strongest marker ⇒ `matchedBy: "run-id"`.
26
+ * `PI_CREW_RUN_ID` itself is NOT matched: `env-allowlist-registry.ts:322`
27
+ * classifies it `test-only` ("fixture name; src uses PI_CREW_BROKER_RUN_ID")
28
+ * and grep confirms no src writer — matching it would only widen the kill
29
+ * surface onto test fixtures.
30
+ * - `PI_CREW_KIND=subagent` (`pi-args.ts:564`), `PI_CREW_TASK_ID`
31
+ * (`child-pi-spawn.ts:434,443`) and `PI_CREW_PARENT_PID`
32
+ * (`child-pi-spawn.ts:191` = the SPAWNING process pid) ⇒
33
+ * `matchedBy: "task-id"`.
34
+ * - `PI_CREW_PARENT_PID` alone, when the recorded parent is already dead ⇒
35
+ * `matchedBy: "parent-pid"` (the shape the incident's survivors had).
36
+ *
37
+ * Safety model — the sweep is additive, never a wider kill surface:
38
+ * 1. Discovery is Linux `/proc` only; anything else yields `[]` (fail-open).
39
+ * 2. Our own pid and every ancestor of it are never candidates.
40
+ * 3. Same uid only: we can only reason about our own processes.
41
+ * 4. The candidate must have STARTED no earlier than the run (starttime is
42
+ * converted to wall clock via /proc/uptime; > 60 s of pre-run clock skew
43
+ * is rejected) — a process that predates the run cannot belong to it.
44
+ * 5. The candidate's group must not be our own group nor our own pid.
45
+ * 6. Every signal goes through `reapGroup()` — the identity-proven ladder in
46
+ * reap-group.ts — anchored on the candidate pid, never a blind
47
+ * `kill(-pgid)`. Candidates without a usable pgid are skipped
48
+ * (`no-pgid`), never signalled individually.
49
+ * 7. EXTRA rung for the sweep: the group's LEADER must either be gone, or be
50
+ * another matched candidate of this same run. A live, unmatched leader
51
+ * means the group belongs to a session that is still running ⇒
52
+ * `leader-alive-unmatched`, nothing signalled.
53
+ * 8. `dryRun` reports and signals nothing; `PI_CREW_REAP_RUN_DESCENDANTS=0`
54
+ * disables the whole sweep.
55
+ * 9. (11b.1) A RECORDED ancestor (the detached runner pid, a worker pid) is
56
+ * the tree's ROOT, never its descendant ⇒ it is never a candidate, even
57
+ * when it is still alive. Only its children/grandchildren are, so a live
58
+ * recorded pid cannot veto the sweep the way the old all-or-nothing
59
+ * "every recorded pid is dead" gate did.
60
+ * 10. (11b.3) A recorded SURFACE pane pid (`manifest.surface.workerPids`) is
61
+ * something the operator may still be watching in a mux pane ⇒ while it is
62
+ * ALIVE it is never a candidate. It is not excluded unconditionally like a
63
+ * recorded ancestor: once it is dead there is nothing left to kill, so the
64
+ * pid simply falls out of /proc (and a recycled pid must still satisfy
65
+ * every start-time/uid/marker proof above).
66
+ * 11. (11b.4) The `run-id` MARKER ALONE IS NOT PROOF OF DESCENT. Inheriting
67
+ * `PI_CREW_BROKER_RUN_ID` only says "some ancestor of mine was part of this
68
+ * run": an operator shell opened inside a worker's pane, or a dev server a
69
+ * worker started and left running, carries the same env after the spawn
70
+ * session is long gone. The `run-id` branch therefore also demands a
71
+ * `PI_CREW_PARENT_PID` the run actually RECORDED (runner / worker /
72
+ * surface pane / session pid — the last added by 11c for SYNC runs, which
73
+ * record no runner) that is ALREADY DEAD — the same descent proof the
74
+ * `task-id` and `parent-pid` branches have always required. A same-uid
75
+ * process that inherited the env but has no such parent is not reaped
76
+ * (a documented miss, never a wrong kill).
77
+ * 12. (11b.4) A candidate holding a CONTROLLING TERMINAL (`/proc/<pid>/stat`
78
+ * field 7 `tty_nr !== 0`) is an INTERACTIVE process — typically the shell
79
+ * the operator is typing into. It is skipped with reason `interactive-tty`
80
+ * and never signalled, on EVERY branch (a tty is not a run resource). The
81
+ * incident shape (a detached dev server, `caddy`) has no controlling
82
+ * terminal, so the real leaks are unaffected.
83
+ *
84
+ * Escalation: like every other `reapGroup` caller, the SIGKILL rung comes back
85
+ * as a `pending` promise. The caller registers it with the EXISTING drain
86
+ * owners (`trackReapEscalation`/`drainReapEscalations` in stale-reconciler.ts,
87
+ * `flushReapEscalationsSync` in reap-group.ts) — this module deliberately has
88
+ * no drain of its own.
89
+ *
90
+ * Known limits (documented, not fixed here): a descendant that `setsid()`s or
91
+ * runs inside a container/runtime with its own init (docker, systemd-run) is a
92
+ * group we cannot prove and is left alone; and pi's bash tool remains the
93
+ * primary leak source — the upstream fix (a `detached: false`/tracked-children
94
+ * default) is what would make this sweep unnecessary.
95
+ */
96
+
97
+ import * as fs from "node:fs";
98
+ import { getCrewEnvBool } from "../../config/env-vars.ts";
99
+ import { logInternalError } from "../../utils/internal-error.ts";
100
+ import { fieldsAfterComm, PROC_STAT_PPID_INDEX, PROC_STAT_TTY_INDEX } from "./proc-stat.ts";
101
+ import { type ProcIdentity, readProcIdentity, reapGroup } from "./reap-group.ts";
102
+
103
+ /** Default grace window between SIGTERM and SIGKILL (ms). */
104
+ const DEFAULT_GRACEFUL_MS = 2_000;
105
+ /** Default hard cap after SIGKILL before giving up (ms). */
106
+ const DEFAULT_CAP_MS = 2_000;
107
+ /**
108
+ * Allowed clock skew when comparing a candidate's start time against the run
109
+ * start. A process that started MORE than this before the run cannot belong to
110
+ * it; the window absorbs `createdAt` being stamped before the spawn and the
111
+ * coarseness of `/proc/<pid>/stat` starttime ticks.
112
+ */
113
+ const PRE_RUN_SKEW_MS = 60_000;
114
+ /** Linux `CLK_TCK` — hardcoded like zombie-scanner.ts (100 on every mainstream distro). */
115
+ const CLOCK_TICKS_PER_SEC = 100;
116
+ /** Bound on the "already signalled this group" registry (see idempotency note). */
117
+ const MAX_TRACKED_GROUPS = 256;
118
+
119
+ /**
120
+ * Kill switch (default ON). `PI_CREW_REAP_RUN_DESCENDANTS=0` makes the sweep a
121
+ * complete no-op without a redeploy — the escape hatch for a platform whose
122
+ * /proc semantics turn out to over-match.
123
+ */
124
+ export function resolveRunDescendantSweepEnabled(): boolean {
125
+ return getCrewEnvBool("PI_CREW_REAP_RUN_DESCENDANTS") ?? true;
126
+ }
127
+
128
+ /** One process proven to belong to a terminal run. */
129
+ export interface RunDescendant {
130
+ pid: number;
131
+ pgid: number | undefined;
132
+ starttime: string | undefined;
133
+ uid: number | undefined;
134
+ /** Which marker proved descent — highest available priority first. */
135
+ matchedBy: "run-id" | "task-id" | "parent-pid";
136
+ cmd: string;
137
+ rssKb: number;
138
+ /**
139
+ * 11b.4: `/proc/<pid>/stat` field 7. `0`/undefined ⇒ no controlling terminal;
140
+ * anything else marks an INTERACTIVE process, skipped by safety rung 12.
141
+ */
142
+ ttyNr?: number;
143
+ }
144
+
145
+ /** What identifies a run's process tree. All values come from run state. */
146
+ export interface RunMarkers {
147
+ runId: string;
148
+ /** `manifest.async.pid` — the detached team-runner, when recorded. */
149
+ runnerPid?: number;
150
+ /** Recorded worker pids of the run (heartbeat.pid / checkpoint.childPid). */
151
+ workerPids: number[];
152
+ /** Task ids of the run (PI_CREW_TASK_ID values its workers carried). */
153
+ taskIds: string[];
154
+ /**
155
+ * Wave 11c (battery 2026-10-24 F2): `manifest.sessionPid` — the pid of the
156
+ * process that created the run. A SYNC (foreground) run has no async
157
+ * runner, so its leaked bash grandchildren carry
158
+ * `PI_CREW_PARENT_PID=<session pi pid>` — a pid NO other field records,
159
+ * which made the descent proof unsatisfiable and the leak a permanent
160
+ * documented miss. Accepted as a proof parent for the run-id branch with
161
+ * the SAME dead-only guard as every other parent: while the session pi is
162
+ * ALIVE it is not proof of anything (the operator may still be typing in
163
+ * it) — the leak is only reaped once that session is gone.
164
+ */
165
+ sessionPid?: number;
166
+ /**
167
+ * 11b.3: pids recorded in `manifest.surface.workerPids` (the mux-surface
168
+ * controller's view of each worker). A pane pid the operator can still see
169
+ * must not be reaped while it is alive — see safety rung 10.
170
+ */
171
+ surfacePids?: number[];
172
+ /** Run start (epoch ms) — a candidate that predates it cannot belong to it. */
173
+ startedAtMs: number;
174
+ }
175
+
176
+ /**
177
+ * One `/proc` row captured ONCE per reconcile pass (wave 11b.4, LOW-4).
178
+ *
179
+ * Before 11b.4 every run in the terminal lane built its own `/proc` view, so a
180
+ * pass with the full 25-run budget read `/proc/<pid>/stat` + `status` for every
181
+ * process on the host 25 times. The snapshot makes it ONE read per pid per
182
+ * pass; the readers stay injectable so tests and the doctor path are unchanged.
183
+ */
184
+ export interface ProcSnapshotEntry {
185
+ identity: ProcIdentity;
186
+ /**
187
+ * The process' `PI_CREW_*` env. Read ONLY for same-uid processes: a foreign
188
+ * uid is rejected by rung 3 before its env is ever consulted, and
189
+ * `/proc/<pid>/environ` of another user is unreadable anyway — so skipping it
190
+ * avoids a pile of EACCES throws on a busy host.
191
+ */
192
+ environ: Record<string, string> | undefined;
193
+ /** `/proc/<pid>/stat` field 7; `0` ⇒ no controlling terminal (safety rung 12). */
194
+ ttyNr: number | undefined;
195
+ }
196
+
197
+ /** A reusable `/proc` view for one reconcile pass. */
198
+ export interface ProcSnapshot {
199
+ /** Every pid the snapshot covers (`/proc` enumeration order). */
200
+ pids: number[];
201
+ /** pid → row. A pid MISSING here had no readable identity when captured. */
202
+ entries: Map<number, ProcSnapshotEntry>;
203
+ }
204
+
205
+ /** Injectable dependencies — every side effect is replaceable in tests. */
206
+ export interface SweepOptions {
207
+ /** Report candidates without signalling (doctor). */
208
+ dryRun?: boolean;
209
+ /** /proc enumeration (tests inject a fixed pid list). Default `readdirSync("/proc")`. */
210
+ listPids?: () => number[];
211
+ /** /proc/<pid>/environ reader (tests inject marker maps). */
212
+ readEnviron?: (pid: number) => Record<string, string> | undefined;
213
+ /** /proc identity reader (tests inject fixtures). Default `readProcIdentity`. */
214
+ readIdentity?: (pid: number) => ProcIdentity | undefined;
215
+ /** Group signal sender (the reap-group `signalGroup` seam). */
216
+ killGroup?: (pgid: number, signal: NodeJS.Signals) => void;
217
+ /** Group liveness probe (the reap-group `groupAlive` seam). */
218
+ groupAlive?: (pgid: number) => boolean;
219
+ /** Grace window between SIGTERM and SIGKILL on the group path (ms). */
220
+ gracefulMs?: number;
221
+ /** Hard cap after SIGKILL on the group path (ms). */
222
+ capMs?: number;
223
+ /** Platform override (tests simulate non-Linux). Default `process.platform`. */
224
+ platform?: NodeJS.Platform;
225
+ /** Candidate start (epoch ms) resolver — tests pin it, default reads /proc/uptime. */
226
+ candidateStartMs?: (identity: ProcIdentity) => number | undefined;
227
+ /** `/proc/<pid>/stat` field 7 (tty_nr) reader — tests inject fixtures. */
228
+ readTtyNr?: (pid: number) => number | undefined;
229
+ /**
230
+ * 11b.4 (LOW-4): a `/proc` view shared by every sweep in a reconcile pass.
231
+ * When absent the sweep builds its own (one per call), so the public contract
232
+ * and the doctor path are unchanged.
233
+ */
234
+ snapshot?: ProcSnapshot;
235
+ }
236
+
237
+ /** Sweep verdict: what was signalled, what was refused, what is still in flight. */
238
+ export interface RunSweepResult {
239
+ /** Candidates actually handed to the group reaper (empty on dryRun). */
240
+ reaped: RunDescendant[];
241
+ skipped: { pid: number; reason: string }[];
242
+ /** SIGKILL escalations — register with the existing drain owners; never dropped. */
243
+ pending: Promise<unknown>[];
244
+ /**
245
+ * Every candidate discovery proved to belong to the run (dryRun included).
246
+ * `reaped` is the subset that was signalled; doctor reports THIS list.
247
+ */
248
+ descendants: RunDescendant[];
249
+ }
250
+
251
+ // ── default /proc readers (all best-effort, none throw) ──────────────────────
252
+
253
+ /** `/proc/<pid>/environ` as a key=value record; undefined when unreadable. */
254
+ function readProcEnviron(pid: number): Record<string, string> | undefined {
255
+ try {
256
+ const raw = fs.readFileSync(`/proc/${pid}/environ`, "utf-8");
257
+ const out: Record<string, string> = {};
258
+ for (const entry of raw.split("\0")) {
259
+ const eq = entry.indexOf("=");
260
+ if (eq > 0) out[entry.slice(0, eq)] = entry.slice(eq + 1);
261
+ }
262
+ return out;
263
+ } catch {
264
+ return undefined;
265
+ }
266
+ }
267
+
268
+ /** Numeric pid directories under /proc; `[]` when /proc is unreadable. */
269
+ function listProcPids(): number[] {
270
+ const pids: number[] = [];
271
+ try {
272
+ for (const entry of fs.readdirSync("/proc")) {
273
+ if (/^\d+$/.test(entry)) pids.push(Number.parseInt(entry, 10));
274
+ }
275
+ } catch {
276
+ /* no /proc — the platform gate above already handles the real case */
277
+ }
278
+ return pids;
279
+ }
280
+
281
+ /**
282
+ * Parent pid of a process, from the shared paren-aware stat parser. Used ONLY to
283
+ * walk OUR OWN ancestry (the skip set): that is real kernel state regardless of
284
+ * any injected candidate reader, so it is deliberately not injectable.
285
+ */
286
+ function readParentPid(pid: number): number | undefined {
287
+ try {
288
+ const fields = fieldsAfterComm(fs.readFileSync(`/proc/${pid}/stat`, "utf-8"));
289
+ if (!fields) return undefined;
290
+ const ppid = Number.parseInt(fields[PROC_STAT_PPID_INDEX] ?? "", 10);
291
+ return Number.isFinite(ppid) && ppid > 0 ? ppid : undefined;
292
+ } catch {
293
+ return undefined;
294
+ }
295
+ }
296
+
297
+ /** Our own ancestor chain (never a candidate: they host this session). */
298
+ function selfAncestors(): Set<number> {
299
+ const chain = new Set<number>();
300
+ let current = process.ppid;
301
+ let guard = 0;
302
+ while (current > 1 && guard++ < 64) {
303
+ chain.add(current);
304
+ const next = readParentPid(current);
305
+ if (next === undefined || chain.has(next)) break;
306
+ current = next;
307
+ }
308
+ return chain;
309
+ }
310
+
311
+ /** `/proc/<pid>/cmdline` for the report; `pid <n>` when unreadable. */
312
+ function readProcCmdline(pid: number): string {
313
+ try {
314
+ const raw = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8");
315
+ return raw.split("\0").filter(Boolean).join(" ").trim() || `pid ${pid}`;
316
+ } catch {
317
+ return `pid ${pid}`;
318
+ }
319
+ }
320
+
321
+ /**
322
+ * `/proc/<pid>/stat` field 7 (tty_nr): `0` ⇒ no controlling terminal, any other
323
+ * value ⇒ the process is attached to a terminal. Read from the shared parser so
324
+ * the field index can never drift from its siblings. undefined when unreadable —
325
+ * and an unreadable tty is treated as "no tty": the identity read that got us
326
+ * here already succeeded, so this is a gone-process race and `reapGroup` refuses
327
+ * a gone candidate on its own.
328
+ */
329
+ function readProcTtyNr(pid: number): number | undefined {
330
+ try {
331
+ const fields = fieldsAfterComm(fs.readFileSync(`/proc/${pid}/stat`, "utf-8"));
332
+ if (!fields) return undefined;
333
+ const raw = Number.parseInt(fields[PROC_STAT_TTY_INDEX] ?? "", 10);
334
+ return Number.isFinite(raw) ? raw : undefined;
335
+ } catch {
336
+ return undefined;
337
+ }
338
+ }
339
+
340
+ /** `/proc/<pid>/status` VmRSS in kB; 0 when unreadable. */
341
+ function readProcRssKb(pid: number): number {
342
+ try {
343
+ const match = /^VmRSS:\s+(\d+)\s+kB/m.exec(fs.readFileSync(`/proc/${pid}/status`, "utf-8"));
344
+ return match ? Number.parseInt(match[1] ?? "", 10) : 0;
345
+ } catch {
346
+ return 0;
347
+ }
348
+ }
349
+
350
+ /** /proc/uptime first field (seconds since boot); undefined when unreadable. */
351
+ function readUptimeSec(): number | undefined {
352
+ try {
353
+ const seconds = Number.parseFloat(fs.readFileSync("/proc/uptime", "utf-8").split(" ")[0] ?? "");
354
+ return Number.isFinite(seconds) ? seconds : undefined;
355
+ } catch {
356
+ return undefined;
357
+ }
358
+ }
359
+
360
+ /**
361
+ * Wall-clock start of a candidate, from `starttime` ticks: boot = now - uptime,
362
+ * start = boot + starttime/CLK_TCK. undefined when either side is unreadable —
363
+ * an unprovable start time is a rejection, never a guess.
364
+ */
365
+ function defaultCandidateStartMs(identity: ProcIdentity): number | undefined {
366
+ if (identity.starttime === undefined) return undefined;
367
+ const ticks = Number(identity.starttime);
368
+ if (!Number.isFinite(ticks)) return undefined;
369
+ const uptimeSec = readUptimeSec();
370
+ if (uptimeSec === undefined) return undefined;
371
+ return Date.now() - uptimeSec * 1000 + (ticks / CLOCK_TICKS_PER_SEC) * 1000;
372
+ }
373
+
374
+ /** Liveness by pid: ESRCH/ENOENT ⇒ gone, EPERM ⇒ alive but not ours. */
375
+ function isPidAlive(pid: number): boolean {
376
+ if (!Number.isInteger(pid) || pid <= 1) return false;
377
+ try {
378
+ process.kill(pid, 0);
379
+ return true;
380
+ } catch (err) {
381
+ const code = (err as NodeJS.ErrnoException).code;
382
+ return code !== "ESRCH" && code !== "ENOENT";
383
+ }
384
+ }
385
+
386
+ // ── /proc snapshot (one per reconcile pass) ─────────────────────────────────
387
+
388
+ /**
389
+ * Capture the `/proc` view one reconcile pass reuses for every run (11b.4,
390
+ * LOW-4). Reads `/proc/<pid>/stat` + `status` once per pid instead of once per
391
+ * RUN, which is what made a full 25-run budget 25 full-host scans. Never throws:
392
+ * an unreadable `/proc` yields an empty snapshot (the sweep is best-effort).
393
+ *
394
+ * Only same-uid processes get their `environ` read — a foreign uid is rejected by
395
+ * safety rung 3 before its env is consulted (and its environ is unreadable, so
396
+ * an eager read would be a pile of EACCES throws on a busy host).
397
+ */
398
+ export function buildProcSnapshot(opts: SweepOptions = {}): ProcSnapshot {
399
+ const readIdentity = opts.readIdentity ?? readProcIdentity;
400
+ const readEnviron = opts.readEnviron ?? readProcEnviron;
401
+ const readTtyNr = opts.readTtyNr ?? readProcTtyNr;
402
+ const selfUid = typeof process.getuid === "function" ? process.getuid() : undefined;
403
+ const pids: number[] = [];
404
+ const entries = new Map<number, ProcSnapshotEntry>();
405
+ for (const pid of (opts.listPids ?? listProcPids)()) {
406
+ if (!Number.isInteger(pid) || pid <= 1) continue;
407
+ const identity = readIdentity(pid);
408
+ // Gone between readdir and read: nothing left to prove, nothing to reap.
409
+ if (!identity) continue;
410
+ const sameUid = identity.uid !== undefined && selfUid !== undefined && identity.uid === selfUid;
411
+ pids.push(pid);
412
+ entries.set(pid, { identity, environ: sameUid ? readEnviron(pid) : undefined, ttyNr: readTtyNr(pid) });
413
+ }
414
+ return { pids, entries };
415
+ }
416
+
417
+ // ── marker matching ─────────────────────────────────────────────────────────
418
+
419
+ /**
420
+ * Match a candidate's inherited env against the run markers, highest priority
421
+ * first (run-id → task-id → parent-pid). Returns the winning marker or
422
+ * undefined.
423
+ *
424
+ * `recordedParents` (runner + worker pids) is the set that may serve as a
425
+ * descent PROOF for a candidate's `PI_CREW_PARENT_PID`; `proofParents` is the
426
+ * wider set the `run-id` branch accepts (it also admits recorded surface pane
427
+ * pids — see MEDIUM-3 in the caller).
428
+ */
429
+ function matchMarkers(
430
+ environ: Record<string, string>,
431
+ markers: RunMarkers,
432
+ recordedParents: ReadonlySet<number>,
433
+ proofParents: ReadonlySet<number>,
434
+ taskIds: ReadonlySet<string>,
435
+ ): RunDescendant["matchedBy"] | undefined {
436
+ const parentPid = Number.parseInt(environ.PI_CREW_PARENT_PID ?? "", 10);
437
+ const hasParent = Number.isInteger(parentPid) && parentPid > 1;
438
+ // 11b.4 (HIGH-1, safety rung 11): the run id is NECESSARY but never
439
+ // sufficient. Every descendant of a run inherits it — including the shell an
440
+ // operator opens inside a worker's pane and a dev server a worker leaves
441
+ // running — so matching it alone turns any same-uid process that ever saw the
442
+ // env into a kill candidate. The candidate must additionally name a parent
443
+ // this run RECORDED and that parent must already be DEAD: exactly the descent
444
+ // proof the other two branches require, and exactly the shape of the incident
445
+ // survivors (`PI_CREW_PARENT_PID` = the recorded runner, dead for 24 h).
446
+ if (
447
+ environ.PI_CREW_BROKER_RUN_ID !== undefined &&
448
+ environ.PI_CREW_BROKER_RUN_ID === markers.runId &&
449
+ hasParent &&
450
+ proofParents.has(parentPid) &&
451
+ !isPidAlive(parentPid)
452
+ ) {
453
+ return "run-id";
454
+ }
455
+ // The worker shape: the process must carry the worker kind, belong to a task
456
+ // of THIS run, and name a parent the run actually recorded (the runner pid
457
+ // for a bash grandchild — the worker inherited PI_CREW_PARENT_PID from the
458
+ // runner — or a worker pid for a nested worker).
459
+ if (environ.PI_CREW_KIND === "subagent") {
460
+ const taskId = environ.PI_CREW_TASK_ID;
461
+ // 11b.1 (per-candidate proof): the recorded parent this candidate names
462
+ // must ALSO be gone. While it is alive the candidate may still be doing
463
+ // that worker's work, so a `task-id` match alone is not proof of a leak —
464
+ // the parent-pid branch below has always required the same thing.
465
+ if (taskId !== undefined && taskIds.has(taskId) && hasParent && recordedParents.has(parentPid) && !isPidAlive(parentPid)) {
466
+ return "task-id";
467
+ }
468
+ }
469
+ // The orphan shape of the incident survivors: only the dead recorded parent
470
+ // is available as proof.
471
+ if (hasParent && markers.workerPids.includes(parentPid) && !isPidAlive(parentPid)) return "parent-pid";
472
+ return undefined;
473
+ }
474
+
475
+ /**
476
+ * Enumerate the processes that PROVABLY belong to `markers.runId`. Pure
477
+ * discovery — never signals. Returns `[]` on any platform without /proc and
478
+ * when the kill switch is off (fail-open: the sweep is best-effort cleanup).
479
+ */
480
+ export function collectRunDescendants(markers: RunMarkers, opts: SweepOptions = {}): RunDescendant[] {
481
+ if (!resolveRunDescendantSweepEnabled()) return [];
482
+ if ((opts.platform ?? process.platform) !== "linux") return [];
483
+ const readIdentity = opts.readIdentity ?? readProcIdentity;
484
+ const candidateStartMs = opts.candidateStartMs ?? defaultCandidateStartMs;
485
+ const selfUid = typeof process.getuid === "function" ? process.getuid() : undefined;
486
+ // Our own group: never a candidate's target group (belt to reap-group's rung).
487
+ const selfPgrp = readIdentity(process.pid)?.pgid;
488
+ // 11b.4 (LOW-4): ONE /proc view per call — supplied by the caller (a whole
489
+ // reconcile pass) or built here (doctor, tests, the lone reconciler path).
490
+ const snapshot = opts.snapshot ?? buildProcSnapshot(opts);
491
+ const recordedParents = new Set<number>(markers.workerPids);
492
+ if (markers.runnerPid !== undefined) recordedParents.add(markers.runnerPid);
493
+ // 11b.3: pids the surface controller recorded for this run's panes.
494
+ const surfacePids = new Set<number>((markers.surfacePids ?? []).filter((pid) => Number.isInteger(pid) && pid > 1));
495
+ // 11b.4 (MEDIUM-3): the surface pids are ALSO acceptable descent proofs for
496
+ // the run-id branch. Today this set is redundant with the recorded parents
497
+ // (`manifest.surface.workerPids` holds worker pids, which are already
498
+ // recorded parents) — it is kept as a guard in case surface pids ever
499
+ // diverge from what the tasks recorded. Defence in depth, no added risk: a
500
+ // proof parent must still be DEAD, and every other rung still applies.
501
+ const proofParents = new Set<number>([...recordedParents, ...surfacePids]);
502
+ // Wave 11c (battery F2): the run's SPAWNER pid (`manifest.sessionPid`) is a
503
+ // recorded parent too — for a SYNC run it is the only pid its leaked
504
+ // descendants can name. Proof-parent only (NOT a recorded ancestor for the
505
+ // candidate-exclusion set): the session pi hosts the run, it is not a worker
506
+ // of it. The dead-only guard inside matchMarkers keeps a LIVE session from
507
+ // ever serving as proof — the operator may still be using that very pi.
508
+ if (markers.sessionPid !== undefined && Number.isInteger(markers.sessionPid) && markers.sessionPid > 1) {
509
+ proofParents.add(markers.sessionPid);
510
+ }
511
+ const taskIds = new Set<string>(markers.taskIds);
512
+ const ancestors = selfAncestors();
513
+ const descendants: RunDescendant[] = [];
514
+ for (const pid of snapshot.pids) {
515
+ if (!Number.isInteger(pid) || pid <= 1) continue;
516
+ // Never ourselves, never a process that hosts this session.
517
+ if (pid === process.pid || ancestors.has(pid)) continue;
518
+ // 11b.1 per-candidate rule: a RECORDED ancestor is the root of the tree we
519
+ // are cleaning up, never a descendant of it. Excluding it is what makes the
520
+ // sweep safe WITHOUT the old all-or-nothing "every recorded pid is dead"
521
+ // gate: a live worker keeps running (its own group reap / orphan reaper
522
+ // owns it) while the leaked grandchildren whose recorded parent is already
523
+ // gone are still reaped. Trade-off, stated plainly: a recorded pid that was
524
+ // recycled onto a genuine leaked descendant is skipped here — a miss, never
525
+ // a wrong kill.
526
+ if (recordedParents.has(pid)) continue;
527
+ // 11b.3: a recorded surface pane pid that is STILL ALIVE may be the pane the
528
+ // operator is watching right now (a run that just went terminal keeps its
529
+ // mux panes until they are closed). Never a candidate while alive; once it
530
+ // is dead the pid is gone from /proc anyway, so nothing is left to reap.
531
+ if (surfacePids.has(pid) && isPidAlive(pid)) continue;
532
+ const entry = snapshot.entries.get(pid);
533
+ if (!entry) continue;
534
+ const identity = entry.identity;
535
+ // Same uid only — an unknown uid cannot be proven to be ours either.
536
+ if (identity.uid === undefined || selfUid === undefined || identity.uid !== selfUid) continue;
537
+ // The PID-reuse-proof field, and the run-scope proof: must exist and must
538
+ // not predate the run.
539
+ if (identity.starttime === undefined) continue;
540
+ const startMs = candidateStartMs(identity);
541
+ if (startMs === undefined || startMs < markers.startedAtMs - PRE_RUN_SKEW_MS) continue;
542
+ // Never hand our own group (or our own pid as a pgid) to the reaper.
543
+ if (identity.pgid === process.pid || (selfPgrp !== undefined && identity.pgid === selfPgrp)) continue;
544
+ const environ = entry.environ;
545
+ if (!environ) continue;
546
+ const matchedBy = matchMarkers(environ, markers, recordedParents, proofParents, taskIds);
547
+ if (matchedBy === undefined) continue;
548
+ descendants.push({
549
+ pid,
550
+ pgid: identity.pgid,
551
+ starttime: identity.starttime,
552
+ uid: identity.uid,
553
+ matchedBy,
554
+ cmd: readProcCmdline(pid),
555
+ rssKb: readProcRssKb(pid),
556
+ ...(entry.ttyNr === undefined ? {} : { ttyNr: entry.ttyNr }),
557
+ });
558
+ }
559
+ return descendants;
560
+ }
561
+
562
+ // ── idempotency: one signal per group incarnation ───────────────────────────
563
+ //
564
+ // The stale reconciler runs on a timer and may look at the same terminal run
565
+ // more than once. Re-signalling an already-reaped group is not just noise: it
566
+ // is a second chance to hit a RECYCLED pgid (the group drained in between). The
567
+ // registry therefore remembers `pgid:leader-starttime` for every accepted
568
+ // signal — the leader starttime makes a new incarnation a NEW key — and is
569
+ // bounded so a long-lived host cannot grow it without limit. An escalation that
570
+ // ended `stuck`/`refused` is forgotten, so the next tick may retry it.
571
+ const signalledGroups = new Map<string, number>();
572
+
573
+ function rememberSignalledGroup(key: string): void {
574
+ signalledGroups.set(key, Date.now());
575
+ while (signalledGroups.size > MAX_TRACKED_GROUPS) {
576
+ const oldest = signalledGroups.keys().next();
577
+ if (oldest.done) break;
578
+ signalledGroups.delete(oldest.value);
579
+ }
580
+ }
581
+
582
+ /** Test seam: forget every remembered group (idempotency reset). */
583
+ export function __resetSignalledGroupsForTest(): void {
584
+ signalledGroups.clear();
585
+ }
586
+
587
+ /** Test seam: how many group incarnations are remembered. */
588
+ export function __signalledGroupCountForTest(): number {
589
+ return signalledGroups.size;
590
+ }
591
+
592
+ /** Default group signal — defensively refuses non-positive pgids. */
593
+ function defaultKillGroup(pgid: number, signal: NodeJS.Signals): void {
594
+ if (!Number.isInteger(pgid) || pgid <= 1) return;
595
+ try {
596
+ process.kill(-pgid, signal);
597
+ } catch {
598
+ /* group already gone — nothing to deliver */
599
+ }
600
+ }
601
+
602
+ /** Default group liveness: `kill(-pgid, 0)`; EPERM means alive but not ours. */
603
+ function defaultGroupAlive(pgid: number): boolean {
604
+ if (!Number.isInteger(pgid) || pgid <= 1) return false;
605
+ try {
606
+ process.kill(-pgid, 0);
607
+ return true;
608
+ } catch (err) {
609
+ return (err as NodeJS.ErrnoException).code === "EPERM";
610
+ }
611
+ }
612
+
613
+ /**
614
+ * Reap the proven descendants of a TERMINAL run. Every signal goes through the
615
+ * guarded group ladder (`reapGroup`), one signal per group incarnation; the
616
+ * candidates that share a pgid are reported individually but signalled once.
617
+ * Never throws.
618
+ *
619
+ * MUST only be called for a TERMINAL run. 11b.1: the recorded ancestors need NOT
620
+ * be dead any more — a recorded pid is never a candidate (see
621
+ * `collectRunDescendants`), so a live worker cannot veto reaping the leaked
622
+ * grandchildren whose own recorded parent is gone. `stale-reconciler.ts`
623
+ * enforces the terminal-run precondition before calling, and the doctor path is
624
+ * `dryRun`.
625
+ */
626
+ export function reapRunDescendants(markers: RunMarkers, opts: SweepOptions = {}): RunSweepResult {
627
+ const result: RunSweepResult = { reaped: [], skipped: [], pending: [], descendants: [] };
628
+ if (!resolveRunDescendantSweepEnabled()) return result;
629
+ // Same platform gate `collectRunDescendants` applies — checked here so a
630
+ // non-Linux host never pays for a /proc snapshot it cannot use.
631
+ if ((opts.platform ?? process.platform) !== "linux") return result;
632
+ // 11b.4 (LOW-4): ONE snapshot for discovery AND for the leader/identity
633
+ // lookups below. A caller that runs several sweeps in one reconcile pass
634
+ // (`crash-recovery`'s terminal lane) passes its own; everyone else gets a
635
+ // per-call view, unchanged from before.
636
+ const snapshot = opts.snapshot ?? buildProcSnapshot(opts);
637
+ const descendants = collectRunDescendants(markers, { ...opts, snapshot });
638
+ result.descendants = descendants;
639
+ if (descendants.length === 0) return result;
640
+ const readIdentity = opts.readIdentity ?? readProcIdentity;
641
+ // Snapshot first (cheap, already read); a pid missing from it appeared after
642
+ // the capture ⇒ the live read is the fresher truth.
643
+ const identityOf = (pid: number): ProcIdentity | undefined => snapshot.entries.get(pid)?.identity ?? readIdentity(pid);
644
+ const signalGroup = opts.killGroup ?? defaultKillGroup;
645
+ const groupAlive = opts.groupAlive ?? defaultGroupAlive;
646
+ const selfPgrp = readIdentity(process.pid)?.pgid;
647
+ const matchedPids = new Set<number>(descendants.map((descendant) => descendant.pid));
648
+ // Candidates are grouped by the group we would signal: one reapGroup call per
649
+ // pgid, so a sibling pair of bash grandchildren cannot be signalled twice.
650
+ const byGroup = new Map<number, RunDescendant[]>();
651
+ for (const descendant of descendants) {
652
+ const pgid = descendant.pgid;
653
+ if (
654
+ pgid === undefined ||
655
+ !Number.isInteger(pgid) ||
656
+ pgid <= 1 ||
657
+ pgid === process.pid ||
658
+ (selfPgrp !== undefined && pgid === selfPgrp)
659
+ ) {
660
+ result.skipped.push({ pid: descendant.pid, reason: "no-pgid" });
661
+ continue;
662
+ }
663
+ const members = byGroup.get(pgid);
664
+ if (members) members.push(descendant);
665
+ else byGroup.set(pgid, [descendant]);
666
+ }
667
+ for (const [pgid, members] of byGroup) {
668
+ // Report-only pass (doctor): nothing is signalled by construction, and the
669
+ // reconciler keys its "do not consume the marker" rule on THIS reason — so
670
+ // it is decided first, before any skip that could otherwise win the report
671
+ // (a dry run must never look like a retryable skip).
672
+ if (opts.dryRun) {
673
+ for (const member of members) result.skipped.push({ pid: member.pid, reason: "dry-run" });
674
+ continue;
675
+ }
676
+ // Safety rung 12 (11b.4, HIGH-1 follow-up): an INTERACTIVE candidate — any
677
+ // member holding a controlling terminal — is never signalled. This is the
678
+ // operator's shell (or a tool attached to it), not a run resource, and
679
+ // there is nothing to "clean up" about it. Applied per GROUP because the
680
+ // signal is per group: if any member is interactive, the whole group is
681
+ // left alone (a documented miss, never a wrong kill).
682
+ if (members.some((member) => (member.ttyNr ?? 0) !== 0)) {
683
+ for (const member of members) result.skipped.push({ pid: member.pid, reason: "interactive-tty" });
684
+ continue;
685
+ }
686
+ // Sweep-only rung: a live leader that is NOT itself a matched candidate
687
+ // means this group belongs to a session that is still running (or to a
688
+ // stranger) — never signal it. A dead leader, or a leader that is one of
689
+ // our own candidates, is safe: reapGroup re-validates identity anyway.
690
+ const leader = identityOf(pgid);
691
+ if (leader && !matchedPids.has(pgid)) {
692
+ logInternalError(
693
+ "run-descendant-sweep.leader-alive-unmatched",
694
+ new Error(`pgid ${pgid} has a live leader ${pgid} that carries no marker of run ${markers.runId}`),
695
+ `candidates=${members.map((member) => member.pid).join(",")}`,
696
+ "debug",
697
+ );
698
+ for (const member of members) result.skipped.push({ pid: member.pid, reason: "leader-alive-unmatched" });
699
+ continue;
700
+ }
701
+ const groupKey = `${pgid}:${leader?.starttime ?? "dead"}`;
702
+ if (signalledGroups.has(groupKey)) {
703
+ for (const member of members) result.skipped.push({ pid: member.pid, reason: "already-swept" });
704
+ continue;
705
+ }
706
+ const anchor = members[0];
707
+ if (anchor === undefined) continue;
708
+ const expected: ProcIdentity = { pid: anchor.pid, pgid, starttime: anchor.starttime, uid: anchor.uid };
709
+ const reap = reapGroup(pgid, anchor.pid, {
710
+ expected,
711
+ readIdentity,
712
+ signalGroup,
713
+ groupAlive,
714
+ name: `run-sweep:${markers.runId}`,
715
+ ...(opts.gracefulMs !== undefined ? { gracefulMs: opts.gracefulMs } : {}),
716
+ ...(opts.capMs !== undefined ? { capMs: opts.capMs } : {}),
717
+ ...(opts.platform !== undefined ? { platform: opts.platform } : {}),
718
+ });
719
+ if (reap.refusal !== undefined) {
720
+ // Nothing was signalled. Leave the key unset so a later tick can retry
721
+ // once the group/identity state changes.
722
+ logInternalError(
723
+ "run-descendant-sweep.refused",
724
+ new Error(`group reap refused for pgid ${pgid} (${reap.refusal})`),
725
+ `run=${markers.runId} pids=${members.map((member) => member.pid).join(",")}`,
726
+ "debug",
727
+ );
728
+ for (const member of members) result.skipped.push({ pid: member.pid, reason: `group-refused:${reap.refusal}` });
729
+ continue;
730
+ }
731
+ rememberSignalledGroup(groupKey);
732
+ // A no-op catch keeps a stuck/failed escalation from wedging the registry:
733
+ // the group did not finish dying, so the next sweep may try again.
734
+ void reap.pending.then(
735
+ (outcome) => {
736
+ if (outcome === "stuck" || outcome === "refused") signalledGroups.delete(groupKey);
737
+ },
738
+ () => signalledGroups.delete(groupKey),
739
+ );
740
+ result.pending.push(reap.pending);
741
+ for (const member of members) result.reaped.push(member);
742
+ }
743
+ return result;
744
+ }