pi-crew 0.11.37 → 0.11.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/agents/critic.md +13 -0
- package/agents/oracle.md +14 -0
- package/dist/index.mjs +531 -526
- package/package.json +4 -3
- package/skills/delegation-patterns/SKILL.md +16 -0
- package/skills/real-test-pi-crew/SKILL.md +13 -2
- package/src/config/env-allowlist-registry.ts +22 -1
- package/src/config/env-vars.ts +23 -0
- package/src/extension/registration/foreground-run-controller.ts +21 -1
- package/src/extension/registration/lifecycle-handlers.ts +12 -0
- package/src/extension/team-tool/doctor.ts +125 -1
- package/src/prompt/prompt-runtime.ts +67 -5
- package/src/runtime/child-pi/child-pi-kill.ts +68 -2
- package/src/runtime/child-pi/child-pi-spawn.ts +20 -0
- package/src/runtime/child-pi/child-pi.ts +3 -1
- package/src/runtime/foreground-watchdog.ts +394 -17
- package/src/runtime/heartbeat/worker-heartbeat.ts +4 -0
- package/src/runtime/model/model-fallback.ts +5 -0
- package/src/runtime/process/orphan-reaper.ts +197 -22
- package/src/runtime/process/proc-stat.ts +8 -0
- package/src/runtime/process/reap-group.ts +499 -0
- package/src/runtime/process/run-descendant-sweep.ts +744 -0
- package/src/runtime/recovery/crash-recovery.ts +202 -0
- package/src/runtime/rpc/frame-client.ts +21 -1
- package/src/runtime/stale-reconciler.ts +343 -2
- package/src/runtime/task-runner/child-executor.ts +15 -1
- package/src/runtime/task-runner/post-execution.ts +112 -1
- package/src/runtime/task-runner/state-helpers.ts +28 -1
- package/src/runtime/verification/verify-gate-memo.ts +127 -0
- package/src/runtime/verification/verify-gate.ts +127 -1
- package/src/state/coordination/mailbox.ts +68 -0
- package/src/state/stores/state-store.ts +8 -0
- package/src/state/types.ts +61 -0
|
@@ -0,0 +1,744 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* run-descendant-sweep.ts — fix-wave 11b (F7): reap the DESCENDANTS of a
|
|
3
|
+
* TERMINAL pi-crew run by matching the pi-crew env markers they inherited.
|
|
4
|
+
*
|
|
5
|
+
* Why this module exists (the incident, SPEC §1): run
|
|
6
|
+
* `team_20261022110510_cefb9164af126460` reached `completed` at 2026-10-22T13:02Z
|
|
7
|
+
* and still had SIX grandchildren alive 24 h later (`dpi-backend` ×5, `caddy`;
|
|
8
|
+
* 399 MB RSS). They carried `PI_CREW_KIND=subagent` and
|
|
9
|
+
* `PI_CREW_PARENT_PID=<dead runner pid>`. No layer of the 11a teardown stack
|
|
10
|
+
* could reach them, and never will: pi's bash tool spawns EVERY command with
|
|
11
|
+
* `detached: true` (GAP-0), so each bash command gets its OWN process group.
|
|
12
|
+
* A group kill rooted at the worker reaches the worker's group only — the bash
|
|
13
|
+
* grandchildren live in a group of their own.
|
|
14
|
+
*
|
|
15
|
+
* The only identity that survives that topology is the ENVIRONMENT: the child
|
|
16
|
+
* inherits the worker's env, so every descendant still carries the pi-crew
|
|
17
|
+
* markers of the run it belongs to. This module therefore turns env markers
|
|
18
|
+
* into kill candidates, scoped to ONE run, and only after that run is terminal.
|
|
19
|
+
*
|
|
20
|
+
* Markers actually used (VERIFIED in src — never invented):
|
|
21
|
+
* - `PI_CREW_BROKER_RUN_ID` — `child-pi-spawn.ts:382` writes
|
|
22
|
+
* `built.env.PI_CREW_BROKER_RUN_ID = input.runId` for every team worker
|
|
23
|
+
* (the env key is spelled by `PI_CREW_RUN_ID_ENV` in
|
|
24
|
+
* `scratchpad-lifecycle.ts:66`, whose VALUE is the broker name). This is
|
|
25
|
+
* the strongest marker ⇒ `matchedBy: "run-id"`.
|
|
26
|
+
* `PI_CREW_RUN_ID` itself is NOT matched: `env-allowlist-registry.ts:322`
|
|
27
|
+
* classifies it `test-only` ("fixture name; src uses PI_CREW_BROKER_RUN_ID")
|
|
28
|
+
* and grep confirms no src writer — matching it would only widen the kill
|
|
29
|
+
* surface onto test fixtures.
|
|
30
|
+
* - `PI_CREW_KIND=subagent` (`pi-args.ts:564`), `PI_CREW_TASK_ID`
|
|
31
|
+
* (`child-pi-spawn.ts:434,443`) and `PI_CREW_PARENT_PID`
|
|
32
|
+
* (`child-pi-spawn.ts:191` = the SPAWNING process pid) ⇒
|
|
33
|
+
* `matchedBy: "task-id"`.
|
|
34
|
+
* - `PI_CREW_PARENT_PID` alone, when the recorded parent is already dead ⇒
|
|
35
|
+
* `matchedBy: "parent-pid"` (the shape the incident's survivors had).
|
|
36
|
+
*
|
|
37
|
+
* Safety model — the sweep is additive, never a wider kill surface:
|
|
38
|
+
* 1. Discovery is Linux `/proc` only; anything else yields `[]` (fail-open).
|
|
39
|
+
* 2. Our own pid and every ancestor of it are never candidates.
|
|
40
|
+
* 3. Same uid only: we can only reason about our own processes.
|
|
41
|
+
* 4. The candidate must have STARTED no earlier than the run (starttime is
|
|
42
|
+
* converted to wall clock via /proc/uptime; > 60 s of pre-run clock skew
|
|
43
|
+
* is rejected) — a process that predates the run cannot belong to it.
|
|
44
|
+
* 5. The candidate's group must not be our own group nor our own pid.
|
|
45
|
+
* 6. Every signal goes through `reapGroup()` — the identity-proven ladder in
|
|
46
|
+
* reap-group.ts — anchored on the candidate pid, never a blind
|
|
47
|
+
* `kill(-pgid)`. Candidates without a usable pgid are skipped
|
|
48
|
+
* (`no-pgid`), never signalled individually.
|
|
49
|
+
* 7. EXTRA rung for the sweep: the group's LEADER must either be gone, or be
|
|
50
|
+
* another matched candidate of this same run. A live, unmatched leader
|
|
51
|
+
* means the group belongs to a session that is still running ⇒
|
|
52
|
+
* `leader-alive-unmatched`, nothing signalled.
|
|
53
|
+
* 8. `dryRun` reports and signals nothing; `PI_CREW_REAP_RUN_DESCENDANTS=0`
|
|
54
|
+
* disables the whole sweep.
|
|
55
|
+
* 9. (11b.1) A RECORDED ancestor (the detached runner pid, a worker pid) is
|
|
56
|
+
* the tree's ROOT, never its descendant ⇒ it is never a candidate, even
|
|
57
|
+
* when it is still alive. Only its children/grandchildren are, so a live
|
|
58
|
+
* recorded pid cannot veto the sweep the way the old all-or-nothing
|
|
59
|
+
* "every recorded pid is dead" gate did.
|
|
60
|
+
* 10. (11b.3) A recorded SURFACE pane pid (`manifest.surface.workerPids`) is
|
|
61
|
+
* something the operator may still be watching in a mux pane ⇒ while it is
|
|
62
|
+
* ALIVE it is never a candidate. It is not excluded unconditionally like a
|
|
63
|
+
* recorded ancestor: once it is dead there is nothing left to kill, so the
|
|
64
|
+
* pid simply falls out of /proc (and a recycled pid must still satisfy
|
|
65
|
+
* every start-time/uid/marker proof above).
|
|
66
|
+
* 11. (11b.4) The `run-id` MARKER ALONE IS NOT PROOF OF DESCENT. Inheriting
|
|
67
|
+
* `PI_CREW_BROKER_RUN_ID` only says "some ancestor of mine was part of this
|
|
68
|
+
* run": an operator shell opened inside a worker's pane, or a dev server a
|
|
69
|
+
* worker started and left running, carries the same env after the spawn
|
|
70
|
+
* session is long gone. The `run-id` branch therefore also demands a
|
|
71
|
+
* `PI_CREW_PARENT_PID` the run actually RECORDED (runner / worker /
|
|
72
|
+
* surface pane / session pid — the last added by 11c for SYNC runs, which
|
|
73
|
+
* record no runner) that is ALREADY DEAD — the same descent proof the
|
|
74
|
+
* `task-id` and `parent-pid` branches have always required. A same-uid
|
|
75
|
+
* process that inherited the env but has no such parent is not reaped
|
|
76
|
+
* (a documented miss, never a wrong kill).
|
|
77
|
+
* 12. (11b.4) A candidate holding a CONTROLLING TERMINAL (`/proc/<pid>/stat`
|
|
78
|
+
* field 7 `tty_nr !== 0`) is an INTERACTIVE process — typically the shell
|
|
79
|
+
* the operator is typing into. It is skipped with reason `interactive-tty`
|
|
80
|
+
* and never signalled, on EVERY branch (a tty is not a run resource). The
|
|
81
|
+
* incident shape (a detached dev server, `caddy`) has no controlling
|
|
82
|
+
* terminal, so the real leaks are unaffected.
|
|
83
|
+
*
|
|
84
|
+
* Escalation: like every other `reapGroup` caller, the SIGKILL rung comes back
|
|
85
|
+
* as a `pending` promise. The caller registers it with the EXISTING drain
|
|
86
|
+
* owners (`trackReapEscalation`/`drainReapEscalations` in stale-reconciler.ts,
|
|
87
|
+
* `flushReapEscalationsSync` in reap-group.ts) — this module deliberately has
|
|
88
|
+
* no drain of its own.
|
|
89
|
+
*
|
|
90
|
+
* Known limits (documented, not fixed here): a descendant that `setsid()`s or
|
|
91
|
+
* runs inside a container/runtime with its own init (docker, systemd-run) is a
|
|
92
|
+
* group we cannot prove and is left alone; and pi's bash tool remains the
|
|
93
|
+
* primary leak source — the upstream fix (a `detached: false`/tracked-children
|
|
94
|
+
* default) is what would make this sweep unnecessary.
|
|
95
|
+
*/
|
|
96
|
+
|
|
97
|
+
import * as fs from "node:fs";
|
|
98
|
+
import { getCrewEnvBool } from "../../config/env-vars.ts";
|
|
99
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
100
|
+
import { fieldsAfterComm, PROC_STAT_PPID_INDEX, PROC_STAT_TTY_INDEX } from "./proc-stat.ts";
|
|
101
|
+
import { type ProcIdentity, readProcIdentity, reapGroup } from "./reap-group.ts";
|
|
102
|
+
|
|
103
|
+
/** Default grace window between SIGTERM and SIGKILL (ms). */
|
|
104
|
+
const DEFAULT_GRACEFUL_MS = 2_000;
|
|
105
|
+
/** Default hard cap after SIGKILL before giving up (ms). */
|
|
106
|
+
const DEFAULT_CAP_MS = 2_000;
|
|
107
|
+
/**
|
|
108
|
+
* Allowed clock skew when comparing a candidate's start time against the run
|
|
109
|
+
* start. A process that started MORE than this before the run cannot belong to
|
|
110
|
+
* it; the window absorbs `createdAt` being stamped before the spawn and the
|
|
111
|
+
* coarseness of `/proc/<pid>/stat` starttime ticks.
|
|
112
|
+
*/
|
|
113
|
+
const PRE_RUN_SKEW_MS = 60_000;
|
|
114
|
+
/** Linux `CLK_TCK` — hardcoded like zombie-scanner.ts (100 on every mainstream distro). */
|
|
115
|
+
const CLOCK_TICKS_PER_SEC = 100;
|
|
116
|
+
/** Bound on the "already signalled this group" registry (see idempotency note). */
|
|
117
|
+
const MAX_TRACKED_GROUPS = 256;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Kill switch (default ON). `PI_CREW_REAP_RUN_DESCENDANTS=0` makes the sweep a
|
|
121
|
+
* complete no-op without a redeploy — the escape hatch for a platform whose
|
|
122
|
+
* /proc semantics turn out to over-match.
|
|
123
|
+
*/
|
|
124
|
+
export function resolveRunDescendantSweepEnabled(): boolean {
|
|
125
|
+
return getCrewEnvBool("PI_CREW_REAP_RUN_DESCENDANTS") ?? true;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** One process proven to belong to a terminal run. */
|
|
129
|
+
export interface RunDescendant {
|
|
130
|
+
pid: number;
|
|
131
|
+
pgid: number | undefined;
|
|
132
|
+
starttime: string | undefined;
|
|
133
|
+
uid: number | undefined;
|
|
134
|
+
/** Which marker proved descent — highest available priority first. */
|
|
135
|
+
matchedBy: "run-id" | "task-id" | "parent-pid";
|
|
136
|
+
cmd: string;
|
|
137
|
+
rssKb: number;
|
|
138
|
+
/**
|
|
139
|
+
* 11b.4: `/proc/<pid>/stat` field 7. `0`/undefined ⇒ no controlling terminal;
|
|
140
|
+
* anything else marks an INTERACTIVE process, skipped by safety rung 12.
|
|
141
|
+
*/
|
|
142
|
+
ttyNr?: number;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** What identifies a run's process tree. All values come from run state. */
|
|
146
|
+
export interface RunMarkers {
|
|
147
|
+
runId: string;
|
|
148
|
+
/** `manifest.async.pid` — the detached team-runner, when recorded. */
|
|
149
|
+
runnerPid?: number;
|
|
150
|
+
/** Recorded worker pids of the run (heartbeat.pid / checkpoint.childPid). */
|
|
151
|
+
workerPids: number[];
|
|
152
|
+
/** Task ids of the run (PI_CREW_TASK_ID values its workers carried). */
|
|
153
|
+
taskIds: string[];
|
|
154
|
+
/**
|
|
155
|
+
* Wave 11c (battery 2026-10-24 F2): `manifest.sessionPid` — the pid of the
|
|
156
|
+
* process that created the run. A SYNC (foreground) run has no async
|
|
157
|
+
* runner, so its leaked bash grandchildren carry
|
|
158
|
+
* `PI_CREW_PARENT_PID=<session pi pid>` — a pid NO other field records,
|
|
159
|
+
* which made the descent proof unsatisfiable and the leak a permanent
|
|
160
|
+
* documented miss. Accepted as a proof parent for the run-id branch with
|
|
161
|
+
* the SAME dead-only guard as every other parent: while the session pi is
|
|
162
|
+
* ALIVE it is not proof of anything (the operator may still be typing in
|
|
163
|
+
* it) — the leak is only reaped once that session is gone.
|
|
164
|
+
*/
|
|
165
|
+
sessionPid?: number;
|
|
166
|
+
/**
|
|
167
|
+
* 11b.3: pids recorded in `manifest.surface.workerPids` (the mux-surface
|
|
168
|
+
* controller's view of each worker). A pane pid the operator can still see
|
|
169
|
+
* must not be reaped while it is alive — see safety rung 10.
|
|
170
|
+
*/
|
|
171
|
+
surfacePids?: number[];
|
|
172
|
+
/** Run start (epoch ms) — a candidate that predates it cannot belong to it. */
|
|
173
|
+
startedAtMs: number;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* One `/proc` row captured ONCE per reconcile pass (wave 11b.4, LOW-4).
|
|
178
|
+
*
|
|
179
|
+
* Before 11b.4 every run in the terminal lane built its own `/proc` view, so a
|
|
180
|
+
* pass with the full 25-run budget read `/proc/<pid>/stat` + `status` for every
|
|
181
|
+
* process on the host 25 times. The snapshot makes it ONE read per pid per
|
|
182
|
+
* pass; the readers stay injectable so tests and the doctor path are unchanged.
|
|
183
|
+
*/
|
|
184
|
+
export interface ProcSnapshotEntry {
|
|
185
|
+
identity: ProcIdentity;
|
|
186
|
+
/**
|
|
187
|
+
* The process' `PI_CREW_*` env. Read ONLY for same-uid processes: a foreign
|
|
188
|
+
* uid is rejected by rung 3 before its env is ever consulted, and
|
|
189
|
+
* `/proc/<pid>/environ` of another user is unreadable anyway — so skipping it
|
|
190
|
+
* avoids a pile of EACCES throws on a busy host.
|
|
191
|
+
*/
|
|
192
|
+
environ: Record<string, string> | undefined;
|
|
193
|
+
/** `/proc/<pid>/stat` field 7; `0` ⇒ no controlling terminal (safety rung 12). */
|
|
194
|
+
ttyNr: number | undefined;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** A reusable `/proc` view for one reconcile pass. */
|
|
198
|
+
export interface ProcSnapshot {
|
|
199
|
+
/** Every pid the snapshot covers (`/proc` enumeration order). */
|
|
200
|
+
pids: number[];
|
|
201
|
+
/** pid → row. A pid MISSING here had no readable identity when captured. */
|
|
202
|
+
entries: Map<number, ProcSnapshotEntry>;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Injectable dependencies — every side effect is replaceable in tests. */
|
|
206
|
+
export interface SweepOptions {
|
|
207
|
+
/** Report candidates without signalling (doctor). */
|
|
208
|
+
dryRun?: boolean;
|
|
209
|
+
/** /proc enumeration (tests inject a fixed pid list). Default `readdirSync("/proc")`. */
|
|
210
|
+
listPids?: () => number[];
|
|
211
|
+
/** /proc/<pid>/environ reader (tests inject marker maps). */
|
|
212
|
+
readEnviron?: (pid: number) => Record<string, string> | undefined;
|
|
213
|
+
/** /proc identity reader (tests inject fixtures). Default `readProcIdentity`. */
|
|
214
|
+
readIdentity?: (pid: number) => ProcIdentity | undefined;
|
|
215
|
+
/** Group signal sender (the reap-group `signalGroup` seam). */
|
|
216
|
+
killGroup?: (pgid: number, signal: NodeJS.Signals) => void;
|
|
217
|
+
/** Group liveness probe (the reap-group `groupAlive` seam). */
|
|
218
|
+
groupAlive?: (pgid: number) => boolean;
|
|
219
|
+
/** Grace window between SIGTERM and SIGKILL on the group path (ms). */
|
|
220
|
+
gracefulMs?: number;
|
|
221
|
+
/** Hard cap after SIGKILL on the group path (ms). */
|
|
222
|
+
capMs?: number;
|
|
223
|
+
/** Platform override (tests simulate non-Linux). Default `process.platform`. */
|
|
224
|
+
platform?: NodeJS.Platform;
|
|
225
|
+
/** Candidate start (epoch ms) resolver — tests pin it, default reads /proc/uptime. */
|
|
226
|
+
candidateStartMs?: (identity: ProcIdentity) => number | undefined;
|
|
227
|
+
/** `/proc/<pid>/stat` field 7 (tty_nr) reader — tests inject fixtures. */
|
|
228
|
+
readTtyNr?: (pid: number) => number | undefined;
|
|
229
|
+
/**
|
|
230
|
+
* 11b.4 (LOW-4): a `/proc` view shared by every sweep in a reconcile pass.
|
|
231
|
+
* When absent the sweep builds its own (one per call), so the public contract
|
|
232
|
+
* and the doctor path are unchanged.
|
|
233
|
+
*/
|
|
234
|
+
snapshot?: ProcSnapshot;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Sweep verdict: what was signalled, what was refused, what is still in flight. */
|
|
238
|
+
export interface RunSweepResult {
|
|
239
|
+
/** Candidates actually handed to the group reaper (empty on dryRun). */
|
|
240
|
+
reaped: RunDescendant[];
|
|
241
|
+
skipped: { pid: number; reason: string }[];
|
|
242
|
+
/** SIGKILL escalations — register with the existing drain owners; never dropped. */
|
|
243
|
+
pending: Promise<unknown>[];
|
|
244
|
+
/**
|
|
245
|
+
* Every candidate discovery proved to belong to the run (dryRun included).
|
|
246
|
+
* `reaped` is the subset that was signalled; doctor reports THIS list.
|
|
247
|
+
*/
|
|
248
|
+
descendants: RunDescendant[];
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ── default /proc readers (all best-effort, none throw) ──────────────────────
|
|
252
|
+
|
|
253
|
+
/** `/proc/<pid>/environ` as a key=value record; undefined when unreadable. */
|
|
254
|
+
function readProcEnviron(pid: number): Record<string, string> | undefined {
|
|
255
|
+
try {
|
|
256
|
+
const raw = fs.readFileSync(`/proc/${pid}/environ`, "utf-8");
|
|
257
|
+
const out: Record<string, string> = {};
|
|
258
|
+
for (const entry of raw.split("\0")) {
|
|
259
|
+
const eq = entry.indexOf("=");
|
|
260
|
+
if (eq > 0) out[entry.slice(0, eq)] = entry.slice(eq + 1);
|
|
261
|
+
}
|
|
262
|
+
return out;
|
|
263
|
+
} catch {
|
|
264
|
+
return undefined;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Numeric pid directories under /proc; `[]` when /proc is unreadable. */
|
|
269
|
+
function listProcPids(): number[] {
|
|
270
|
+
const pids: number[] = [];
|
|
271
|
+
try {
|
|
272
|
+
for (const entry of fs.readdirSync("/proc")) {
|
|
273
|
+
if (/^\d+$/.test(entry)) pids.push(Number.parseInt(entry, 10));
|
|
274
|
+
}
|
|
275
|
+
} catch {
|
|
276
|
+
/* no /proc — the platform gate above already handles the real case */
|
|
277
|
+
}
|
|
278
|
+
return pids;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Parent pid of a process, from the shared paren-aware stat parser. Used ONLY to
|
|
283
|
+
* walk OUR OWN ancestry (the skip set): that is real kernel state regardless of
|
|
284
|
+
* any injected candidate reader, so it is deliberately not injectable.
|
|
285
|
+
*/
|
|
286
|
+
function readParentPid(pid: number): number | undefined {
|
|
287
|
+
try {
|
|
288
|
+
const fields = fieldsAfterComm(fs.readFileSync(`/proc/${pid}/stat`, "utf-8"));
|
|
289
|
+
if (!fields) return undefined;
|
|
290
|
+
const ppid = Number.parseInt(fields[PROC_STAT_PPID_INDEX] ?? "", 10);
|
|
291
|
+
return Number.isFinite(ppid) && ppid > 0 ? ppid : undefined;
|
|
292
|
+
} catch {
|
|
293
|
+
return undefined;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Our own ancestor chain (never a candidate: they host this session). */
|
|
298
|
+
function selfAncestors(): Set<number> {
|
|
299
|
+
const chain = new Set<number>();
|
|
300
|
+
let current = process.ppid;
|
|
301
|
+
let guard = 0;
|
|
302
|
+
while (current > 1 && guard++ < 64) {
|
|
303
|
+
chain.add(current);
|
|
304
|
+
const next = readParentPid(current);
|
|
305
|
+
if (next === undefined || chain.has(next)) break;
|
|
306
|
+
current = next;
|
|
307
|
+
}
|
|
308
|
+
return chain;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** `/proc/<pid>/cmdline` for the report; `pid <n>` when unreadable. */
|
|
312
|
+
function readProcCmdline(pid: number): string {
|
|
313
|
+
try {
|
|
314
|
+
const raw = fs.readFileSync(`/proc/${pid}/cmdline`, "utf-8");
|
|
315
|
+
return raw.split("\0").filter(Boolean).join(" ").trim() || `pid ${pid}`;
|
|
316
|
+
} catch {
|
|
317
|
+
return `pid ${pid}`;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* `/proc/<pid>/stat` field 7 (tty_nr): `0` ⇒ no controlling terminal, any other
|
|
323
|
+
* value ⇒ the process is attached to a terminal. Read from the shared parser so
|
|
324
|
+
* the field index can never drift from its siblings. undefined when unreadable —
|
|
325
|
+
* and an unreadable tty is treated as "no tty": the identity read that got us
|
|
326
|
+
* here already succeeded, so this is a gone-process race and `reapGroup` refuses
|
|
327
|
+
* a gone candidate on its own.
|
|
328
|
+
*/
|
|
329
|
+
function readProcTtyNr(pid: number): number | undefined {
|
|
330
|
+
try {
|
|
331
|
+
const fields = fieldsAfterComm(fs.readFileSync(`/proc/${pid}/stat`, "utf-8"));
|
|
332
|
+
if (!fields) return undefined;
|
|
333
|
+
const raw = Number.parseInt(fields[PROC_STAT_TTY_INDEX] ?? "", 10);
|
|
334
|
+
return Number.isFinite(raw) ? raw : undefined;
|
|
335
|
+
} catch {
|
|
336
|
+
return undefined;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** `/proc/<pid>/status` VmRSS in kB; 0 when unreadable. */
|
|
341
|
+
function readProcRssKb(pid: number): number {
|
|
342
|
+
try {
|
|
343
|
+
const match = /^VmRSS:\s+(\d+)\s+kB/m.exec(fs.readFileSync(`/proc/${pid}/status`, "utf-8"));
|
|
344
|
+
return match ? Number.parseInt(match[1] ?? "", 10) : 0;
|
|
345
|
+
} catch {
|
|
346
|
+
return 0;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/** /proc/uptime first field (seconds since boot); undefined when unreadable. */
|
|
351
|
+
function readUptimeSec(): number | undefined {
|
|
352
|
+
try {
|
|
353
|
+
const seconds = Number.parseFloat(fs.readFileSync("/proc/uptime", "utf-8").split(" ")[0] ?? "");
|
|
354
|
+
return Number.isFinite(seconds) ? seconds : undefined;
|
|
355
|
+
} catch {
|
|
356
|
+
return undefined;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Wall-clock start of a candidate, from `starttime` ticks: boot = now - uptime,
|
|
362
|
+
* start = boot + starttime/CLK_TCK. undefined when either side is unreadable —
|
|
363
|
+
* an unprovable start time is a rejection, never a guess.
|
|
364
|
+
*/
|
|
365
|
+
function defaultCandidateStartMs(identity: ProcIdentity): number | undefined {
|
|
366
|
+
if (identity.starttime === undefined) return undefined;
|
|
367
|
+
const ticks = Number(identity.starttime);
|
|
368
|
+
if (!Number.isFinite(ticks)) return undefined;
|
|
369
|
+
const uptimeSec = readUptimeSec();
|
|
370
|
+
if (uptimeSec === undefined) return undefined;
|
|
371
|
+
return Date.now() - uptimeSec * 1000 + (ticks / CLOCK_TICKS_PER_SEC) * 1000;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** Liveness by pid: ESRCH/ENOENT ⇒ gone, EPERM ⇒ alive but not ours. */
|
|
375
|
+
function isPidAlive(pid: number): boolean {
|
|
376
|
+
if (!Number.isInteger(pid) || pid <= 1) return false;
|
|
377
|
+
try {
|
|
378
|
+
process.kill(pid, 0);
|
|
379
|
+
return true;
|
|
380
|
+
} catch (err) {
|
|
381
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
382
|
+
return code !== "ESRCH" && code !== "ENOENT";
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// ── /proc snapshot (one per reconcile pass) ─────────────────────────────────
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Capture the `/proc` view one reconcile pass reuses for every run (11b.4,
|
|
390
|
+
* LOW-4). Reads `/proc/<pid>/stat` + `status` once per pid instead of once per
|
|
391
|
+
* RUN, which is what made a full 25-run budget 25 full-host scans. Never throws:
|
|
392
|
+
* an unreadable `/proc` yields an empty snapshot (the sweep is best-effort).
|
|
393
|
+
*
|
|
394
|
+
* Only same-uid processes get their `environ` read — a foreign uid is rejected by
|
|
395
|
+
* safety rung 3 before its env is consulted (and its environ is unreadable, so
|
|
396
|
+
* an eager read would be a pile of EACCES throws on a busy host).
|
|
397
|
+
*/
|
|
398
|
+
export function buildProcSnapshot(opts: SweepOptions = {}): ProcSnapshot {
|
|
399
|
+
const readIdentity = opts.readIdentity ?? readProcIdentity;
|
|
400
|
+
const readEnviron = opts.readEnviron ?? readProcEnviron;
|
|
401
|
+
const readTtyNr = opts.readTtyNr ?? readProcTtyNr;
|
|
402
|
+
const selfUid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
|
403
|
+
const pids: number[] = [];
|
|
404
|
+
const entries = new Map<number, ProcSnapshotEntry>();
|
|
405
|
+
for (const pid of (opts.listPids ?? listProcPids)()) {
|
|
406
|
+
if (!Number.isInteger(pid) || pid <= 1) continue;
|
|
407
|
+
const identity = readIdentity(pid);
|
|
408
|
+
// Gone between readdir and read: nothing left to prove, nothing to reap.
|
|
409
|
+
if (!identity) continue;
|
|
410
|
+
const sameUid = identity.uid !== undefined && selfUid !== undefined && identity.uid === selfUid;
|
|
411
|
+
pids.push(pid);
|
|
412
|
+
entries.set(pid, { identity, environ: sameUid ? readEnviron(pid) : undefined, ttyNr: readTtyNr(pid) });
|
|
413
|
+
}
|
|
414
|
+
return { pids, entries };
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// ── marker matching ─────────────────────────────────────────────────────────
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Match a candidate's inherited env against the run markers, highest priority
|
|
421
|
+
* first (run-id → task-id → parent-pid). Returns the winning marker or
|
|
422
|
+
* undefined.
|
|
423
|
+
*
|
|
424
|
+
* `recordedParents` (runner + worker pids) is the set that may serve as a
|
|
425
|
+
* descent PROOF for a candidate's `PI_CREW_PARENT_PID`; `proofParents` is the
|
|
426
|
+
* wider set the `run-id` branch accepts (it also admits recorded surface pane
|
|
427
|
+
* pids — see MEDIUM-3 in the caller).
|
|
428
|
+
*/
|
|
429
|
+
function matchMarkers(
|
|
430
|
+
environ: Record<string, string>,
|
|
431
|
+
markers: RunMarkers,
|
|
432
|
+
recordedParents: ReadonlySet<number>,
|
|
433
|
+
proofParents: ReadonlySet<number>,
|
|
434
|
+
taskIds: ReadonlySet<string>,
|
|
435
|
+
): RunDescendant["matchedBy"] | undefined {
|
|
436
|
+
const parentPid = Number.parseInt(environ.PI_CREW_PARENT_PID ?? "", 10);
|
|
437
|
+
const hasParent = Number.isInteger(parentPid) && parentPid > 1;
|
|
438
|
+
// 11b.4 (HIGH-1, safety rung 11): the run id is NECESSARY but never
|
|
439
|
+
// sufficient. Every descendant of a run inherits it — including the shell an
|
|
440
|
+
// operator opens inside a worker's pane and a dev server a worker leaves
|
|
441
|
+
// running — so matching it alone turns any same-uid process that ever saw the
|
|
442
|
+
// env into a kill candidate. The candidate must additionally name a parent
|
|
443
|
+
// this run RECORDED and that parent must already be DEAD: exactly the descent
|
|
444
|
+
// proof the other two branches require, and exactly the shape of the incident
|
|
445
|
+
// survivors (`PI_CREW_PARENT_PID` = the recorded runner, dead for 24 h).
|
|
446
|
+
if (
|
|
447
|
+
environ.PI_CREW_BROKER_RUN_ID !== undefined &&
|
|
448
|
+
environ.PI_CREW_BROKER_RUN_ID === markers.runId &&
|
|
449
|
+
hasParent &&
|
|
450
|
+
proofParents.has(parentPid) &&
|
|
451
|
+
!isPidAlive(parentPid)
|
|
452
|
+
) {
|
|
453
|
+
return "run-id";
|
|
454
|
+
}
|
|
455
|
+
// The worker shape: the process must carry the worker kind, belong to a task
|
|
456
|
+
// of THIS run, and name a parent the run actually recorded (the runner pid
|
|
457
|
+
// for a bash grandchild — the worker inherited PI_CREW_PARENT_PID from the
|
|
458
|
+
// runner — or a worker pid for a nested worker).
|
|
459
|
+
if (environ.PI_CREW_KIND === "subagent") {
|
|
460
|
+
const taskId = environ.PI_CREW_TASK_ID;
|
|
461
|
+
// 11b.1 (per-candidate proof): the recorded parent this candidate names
|
|
462
|
+
// must ALSO be gone. While it is alive the candidate may still be doing
|
|
463
|
+
// that worker's work, so a `task-id` match alone is not proof of a leak —
|
|
464
|
+
// the parent-pid branch below has always required the same thing.
|
|
465
|
+
if (taskId !== undefined && taskIds.has(taskId) && hasParent && recordedParents.has(parentPid) && !isPidAlive(parentPid)) {
|
|
466
|
+
return "task-id";
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
// The orphan shape of the incident survivors: only the dead recorded parent
|
|
470
|
+
// is available as proof.
|
|
471
|
+
if (hasParent && markers.workerPids.includes(parentPid) && !isPidAlive(parentPid)) return "parent-pid";
|
|
472
|
+
return undefined;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Enumerate the processes that PROVABLY belong to `markers.runId`. Pure
|
|
477
|
+
* discovery — never signals. Returns `[]` on any platform without /proc and
|
|
478
|
+
* when the kill switch is off (fail-open: the sweep is best-effort cleanup).
|
|
479
|
+
*/
|
|
480
|
+
export function collectRunDescendants(markers: RunMarkers, opts: SweepOptions = {}): RunDescendant[] {
|
|
481
|
+
if (!resolveRunDescendantSweepEnabled()) return [];
|
|
482
|
+
if ((opts.platform ?? process.platform) !== "linux") return [];
|
|
483
|
+
const readIdentity = opts.readIdentity ?? readProcIdentity;
|
|
484
|
+
const candidateStartMs = opts.candidateStartMs ?? defaultCandidateStartMs;
|
|
485
|
+
const selfUid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
|
486
|
+
// Our own group: never a candidate's target group (belt to reap-group's rung).
|
|
487
|
+
const selfPgrp = readIdentity(process.pid)?.pgid;
|
|
488
|
+
// 11b.4 (LOW-4): ONE /proc view per call — supplied by the caller (a whole
|
|
489
|
+
// reconcile pass) or built here (doctor, tests, the lone reconciler path).
|
|
490
|
+
const snapshot = opts.snapshot ?? buildProcSnapshot(opts);
|
|
491
|
+
const recordedParents = new Set<number>(markers.workerPids);
|
|
492
|
+
if (markers.runnerPid !== undefined) recordedParents.add(markers.runnerPid);
|
|
493
|
+
// 11b.3: pids the surface controller recorded for this run's panes.
|
|
494
|
+
const surfacePids = new Set<number>((markers.surfacePids ?? []).filter((pid) => Number.isInteger(pid) && pid > 1));
|
|
495
|
+
// 11b.4 (MEDIUM-3): the surface pids are ALSO acceptable descent proofs for
|
|
496
|
+
// the run-id branch. Today this set is redundant with the recorded parents
|
|
497
|
+
// (`manifest.surface.workerPids` holds worker pids, which are already
|
|
498
|
+
// recorded parents) — it is kept as a guard in case surface pids ever
|
|
499
|
+
// diverge from what the tasks recorded. Defence in depth, no added risk: a
|
|
500
|
+
// proof parent must still be DEAD, and every other rung still applies.
|
|
501
|
+
const proofParents = new Set<number>([...recordedParents, ...surfacePids]);
|
|
502
|
+
// Wave 11c (battery F2): the run's SPAWNER pid (`manifest.sessionPid`) is a
|
|
503
|
+
// recorded parent too — for a SYNC run it is the only pid its leaked
|
|
504
|
+
// descendants can name. Proof-parent only (NOT a recorded ancestor for the
|
|
505
|
+
// candidate-exclusion set): the session pi hosts the run, it is not a worker
|
|
506
|
+
// of it. The dead-only guard inside matchMarkers keeps a LIVE session from
|
|
507
|
+
// ever serving as proof — the operator may still be using that very pi.
|
|
508
|
+
if (markers.sessionPid !== undefined && Number.isInteger(markers.sessionPid) && markers.sessionPid > 1) {
|
|
509
|
+
proofParents.add(markers.sessionPid);
|
|
510
|
+
}
|
|
511
|
+
const taskIds = new Set<string>(markers.taskIds);
|
|
512
|
+
const ancestors = selfAncestors();
|
|
513
|
+
const descendants: RunDescendant[] = [];
|
|
514
|
+
for (const pid of snapshot.pids) {
|
|
515
|
+
if (!Number.isInteger(pid) || pid <= 1) continue;
|
|
516
|
+
// Never ourselves, never a process that hosts this session.
|
|
517
|
+
if (pid === process.pid || ancestors.has(pid)) continue;
|
|
518
|
+
// 11b.1 per-candidate rule: a RECORDED ancestor is the root of the tree we
|
|
519
|
+
// are cleaning up, never a descendant of it. Excluding it is what makes the
|
|
520
|
+
// sweep safe WITHOUT the old all-or-nothing "every recorded pid is dead"
|
|
521
|
+
// gate: a live worker keeps running (its own group reap / orphan reaper
|
|
522
|
+
// owns it) while the leaked grandchildren whose recorded parent is already
|
|
523
|
+
// gone are still reaped. Trade-off, stated plainly: a recorded pid that was
|
|
524
|
+
// recycled onto a genuine leaked descendant is skipped here — a miss, never
|
|
525
|
+
// a wrong kill.
|
|
526
|
+
if (recordedParents.has(pid)) continue;
|
|
527
|
+
// 11b.3: a recorded surface pane pid that is STILL ALIVE may be the pane the
|
|
528
|
+
// operator is watching right now (a run that just went terminal keeps its
|
|
529
|
+
// mux panes until they are closed). Never a candidate while alive; once it
|
|
530
|
+
// is dead the pid is gone from /proc anyway, so nothing is left to reap.
|
|
531
|
+
if (surfacePids.has(pid) && isPidAlive(pid)) continue;
|
|
532
|
+
const entry = snapshot.entries.get(pid);
|
|
533
|
+
if (!entry) continue;
|
|
534
|
+
const identity = entry.identity;
|
|
535
|
+
// Same uid only — an unknown uid cannot be proven to be ours either.
|
|
536
|
+
if (identity.uid === undefined || selfUid === undefined || identity.uid !== selfUid) continue;
|
|
537
|
+
// The PID-reuse-proof field, and the run-scope proof: must exist and must
|
|
538
|
+
// not predate the run.
|
|
539
|
+
if (identity.starttime === undefined) continue;
|
|
540
|
+
const startMs = candidateStartMs(identity);
|
|
541
|
+
if (startMs === undefined || startMs < markers.startedAtMs - PRE_RUN_SKEW_MS) continue;
|
|
542
|
+
// Never hand our own group (or our own pid as a pgid) to the reaper.
|
|
543
|
+
if (identity.pgid === process.pid || (selfPgrp !== undefined && identity.pgid === selfPgrp)) continue;
|
|
544
|
+
const environ = entry.environ;
|
|
545
|
+
if (!environ) continue;
|
|
546
|
+
const matchedBy = matchMarkers(environ, markers, recordedParents, proofParents, taskIds);
|
|
547
|
+
if (matchedBy === undefined) continue;
|
|
548
|
+
descendants.push({
|
|
549
|
+
pid,
|
|
550
|
+
pgid: identity.pgid,
|
|
551
|
+
starttime: identity.starttime,
|
|
552
|
+
uid: identity.uid,
|
|
553
|
+
matchedBy,
|
|
554
|
+
cmd: readProcCmdline(pid),
|
|
555
|
+
rssKb: readProcRssKb(pid),
|
|
556
|
+
...(entry.ttyNr === undefined ? {} : { ttyNr: entry.ttyNr }),
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
return descendants;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// ── idempotency: one signal per group incarnation ───────────────────────────
|
|
563
|
+
//
|
|
564
|
+
// The stale reconciler runs on a timer and may look at the same terminal run
|
|
565
|
+
// more than once. Re-signalling an already-reaped group is not just noise: it
|
|
566
|
+
// is a second chance to hit a RECYCLED pgid (the group drained in between). The
|
|
567
|
+
// registry therefore remembers `pgid:leader-starttime` for every accepted
|
|
568
|
+
// signal — the leader starttime makes a new incarnation a NEW key — and is
|
|
569
|
+
// bounded so a long-lived host cannot grow it without limit. An escalation that
|
|
570
|
+
// ended `stuck`/`refused` is forgotten, so the next tick may retry it.
|
|
571
|
+
const signalledGroups = new Map<string, number>();
|
|
572
|
+
|
|
573
|
+
function rememberSignalledGroup(key: string): void {
|
|
574
|
+
signalledGroups.set(key, Date.now());
|
|
575
|
+
while (signalledGroups.size > MAX_TRACKED_GROUPS) {
|
|
576
|
+
const oldest = signalledGroups.keys().next();
|
|
577
|
+
if (oldest.done) break;
|
|
578
|
+
signalledGroups.delete(oldest.value);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/** Test seam: forget every remembered group (idempotency reset). */
|
|
583
|
+
export function __resetSignalledGroupsForTest(): void {
|
|
584
|
+
signalledGroups.clear();
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/** Test seam: how many group incarnations are remembered. */
|
|
588
|
+
export function __signalledGroupCountForTest(): number {
|
|
589
|
+
return signalledGroups.size;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/** Default group signal — defensively refuses non-positive pgids. */
|
|
593
|
+
function defaultKillGroup(pgid: number, signal: NodeJS.Signals): void {
|
|
594
|
+
if (!Number.isInteger(pgid) || pgid <= 1) return;
|
|
595
|
+
try {
|
|
596
|
+
process.kill(-pgid, signal);
|
|
597
|
+
} catch {
|
|
598
|
+
/* group already gone — nothing to deliver */
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** Default group liveness: `kill(-pgid, 0)`; EPERM means alive but not ours. */
|
|
603
|
+
function defaultGroupAlive(pgid: number): boolean {
|
|
604
|
+
if (!Number.isInteger(pgid) || pgid <= 1) return false;
|
|
605
|
+
try {
|
|
606
|
+
process.kill(-pgid, 0);
|
|
607
|
+
return true;
|
|
608
|
+
} catch (err) {
|
|
609
|
+
return (err as NodeJS.ErrnoException).code === "EPERM";
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* Reap the proven descendants of a TERMINAL run. Every signal goes through the
|
|
615
|
+
* guarded group ladder (`reapGroup`), one signal per group incarnation; the
|
|
616
|
+
* candidates that share a pgid are reported individually but signalled once.
|
|
617
|
+
* Never throws.
|
|
618
|
+
*
|
|
619
|
+
* MUST only be called for a TERMINAL run. 11b.1: the recorded ancestors need NOT
|
|
620
|
+
* be dead any more — a recorded pid is never a candidate (see
|
|
621
|
+
* `collectRunDescendants`), so a live worker cannot veto reaping the leaked
|
|
622
|
+
* grandchildren whose own recorded parent is gone. `stale-reconciler.ts`
|
|
623
|
+
* enforces the terminal-run precondition before calling, and the doctor path is
|
|
624
|
+
* `dryRun`.
|
|
625
|
+
*/
|
|
626
|
+
export function reapRunDescendants(markers: RunMarkers, opts: SweepOptions = {}): RunSweepResult {
|
|
627
|
+
const result: RunSweepResult = { reaped: [], skipped: [], pending: [], descendants: [] };
|
|
628
|
+
if (!resolveRunDescendantSweepEnabled()) return result;
|
|
629
|
+
// Same platform gate `collectRunDescendants` applies — checked here so a
|
|
630
|
+
// non-Linux host never pays for a /proc snapshot it cannot use.
|
|
631
|
+
if ((opts.platform ?? process.platform) !== "linux") return result;
|
|
632
|
+
// 11b.4 (LOW-4): ONE snapshot for discovery AND for the leader/identity
|
|
633
|
+
// lookups below. A caller that runs several sweeps in one reconcile pass
|
|
634
|
+
// (`crash-recovery`'s terminal lane) passes its own; everyone else gets a
|
|
635
|
+
// per-call view, unchanged from before.
|
|
636
|
+
const snapshot = opts.snapshot ?? buildProcSnapshot(opts);
|
|
637
|
+
const descendants = collectRunDescendants(markers, { ...opts, snapshot });
|
|
638
|
+
result.descendants = descendants;
|
|
639
|
+
if (descendants.length === 0) return result;
|
|
640
|
+
const readIdentity = opts.readIdentity ?? readProcIdentity;
|
|
641
|
+
// Snapshot first (cheap, already read); a pid missing from it appeared after
|
|
642
|
+
// the capture ⇒ the live read is the fresher truth.
|
|
643
|
+
const identityOf = (pid: number): ProcIdentity | undefined => snapshot.entries.get(pid)?.identity ?? readIdentity(pid);
|
|
644
|
+
const signalGroup = opts.killGroup ?? defaultKillGroup;
|
|
645
|
+
const groupAlive = opts.groupAlive ?? defaultGroupAlive;
|
|
646
|
+
const selfPgrp = readIdentity(process.pid)?.pgid;
|
|
647
|
+
const matchedPids = new Set<number>(descendants.map((descendant) => descendant.pid));
|
|
648
|
+
// Candidates are grouped by the group we would signal: one reapGroup call per
|
|
649
|
+
// pgid, so a sibling pair of bash grandchildren cannot be signalled twice.
|
|
650
|
+
const byGroup = new Map<number, RunDescendant[]>();
|
|
651
|
+
for (const descendant of descendants) {
|
|
652
|
+
const pgid = descendant.pgid;
|
|
653
|
+
if (
|
|
654
|
+
pgid === undefined ||
|
|
655
|
+
!Number.isInteger(pgid) ||
|
|
656
|
+
pgid <= 1 ||
|
|
657
|
+
pgid === process.pid ||
|
|
658
|
+
(selfPgrp !== undefined && pgid === selfPgrp)
|
|
659
|
+
) {
|
|
660
|
+
result.skipped.push({ pid: descendant.pid, reason: "no-pgid" });
|
|
661
|
+
continue;
|
|
662
|
+
}
|
|
663
|
+
const members = byGroup.get(pgid);
|
|
664
|
+
if (members) members.push(descendant);
|
|
665
|
+
else byGroup.set(pgid, [descendant]);
|
|
666
|
+
}
|
|
667
|
+
for (const [pgid, members] of byGroup) {
|
|
668
|
+
// Report-only pass (doctor): nothing is signalled by construction, and the
|
|
669
|
+
// reconciler keys its "do not consume the marker" rule on THIS reason — so
|
|
670
|
+
// it is decided first, before any skip that could otherwise win the report
|
|
671
|
+
// (a dry run must never look like a retryable skip).
|
|
672
|
+
if (opts.dryRun) {
|
|
673
|
+
for (const member of members) result.skipped.push({ pid: member.pid, reason: "dry-run" });
|
|
674
|
+
continue;
|
|
675
|
+
}
|
|
676
|
+
// Safety rung 12 (11b.4, HIGH-1 follow-up): an INTERACTIVE candidate — any
|
|
677
|
+
// member holding a controlling terminal — is never signalled. This is the
|
|
678
|
+
// operator's shell (or a tool attached to it), not a run resource, and
|
|
679
|
+
// there is nothing to "clean up" about it. Applied per GROUP because the
|
|
680
|
+
// signal is per group: if any member is interactive, the whole group is
|
|
681
|
+
// left alone (a documented miss, never a wrong kill).
|
|
682
|
+
if (members.some((member) => (member.ttyNr ?? 0) !== 0)) {
|
|
683
|
+
for (const member of members) result.skipped.push({ pid: member.pid, reason: "interactive-tty" });
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
// Sweep-only rung: a live leader that is NOT itself a matched candidate
|
|
687
|
+
// means this group belongs to a session that is still running (or to a
|
|
688
|
+
// stranger) — never signal it. A dead leader, or a leader that is one of
|
|
689
|
+
// our own candidates, is safe: reapGroup re-validates identity anyway.
|
|
690
|
+
const leader = identityOf(pgid);
|
|
691
|
+
if (leader && !matchedPids.has(pgid)) {
|
|
692
|
+
logInternalError(
|
|
693
|
+
"run-descendant-sweep.leader-alive-unmatched",
|
|
694
|
+
new Error(`pgid ${pgid} has a live leader ${pgid} that carries no marker of run ${markers.runId}`),
|
|
695
|
+
`candidates=${members.map((member) => member.pid).join(",")}`,
|
|
696
|
+
"debug",
|
|
697
|
+
);
|
|
698
|
+
for (const member of members) result.skipped.push({ pid: member.pid, reason: "leader-alive-unmatched" });
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
const groupKey = `${pgid}:${leader?.starttime ?? "dead"}`;
|
|
702
|
+
if (signalledGroups.has(groupKey)) {
|
|
703
|
+
for (const member of members) result.skipped.push({ pid: member.pid, reason: "already-swept" });
|
|
704
|
+
continue;
|
|
705
|
+
}
|
|
706
|
+
const anchor = members[0];
|
|
707
|
+
if (anchor === undefined) continue;
|
|
708
|
+
const expected: ProcIdentity = { pid: anchor.pid, pgid, starttime: anchor.starttime, uid: anchor.uid };
|
|
709
|
+
const reap = reapGroup(pgid, anchor.pid, {
|
|
710
|
+
expected,
|
|
711
|
+
readIdentity,
|
|
712
|
+
signalGroup,
|
|
713
|
+
groupAlive,
|
|
714
|
+
name: `run-sweep:${markers.runId}`,
|
|
715
|
+
...(opts.gracefulMs !== undefined ? { gracefulMs: opts.gracefulMs } : {}),
|
|
716
|
+
...(opts.capMs !== undefined ? { capMs: opts.capMs } : {}),
|
|
717
|
+
...(opts.platform !== undefined ? { platform: opts.platform } : {}),
|
|
718
|
+
});
|
|
719
|
+
if (reap.refusal !== undefined) {
|
|
720
|
+
// Nothing was signalled. Leave the key unset so a later tick can retry
|
|
721
|
+
// once the group/identity state changes.
|
|
722
|
+
logInternalError(
|
|
723
|
+
"run-descendant-sweep.refused",
|
|
724
|
+
new Error(`group reap refused for pgid ${pgid} (${reap.refusal})`),
|
|
725
|
+
`run=${markers.runId} pids=${members.map((member) => member.pid).join(",")}`,
|
|
726
|
+
"debug",
|
|
727
|
+
);
|
|
728
|
+
for (const member of members) result.skipped.push({ pid: member.pid, reason: `group-refused:${reap.refusal}` });
|
|
729
|
+
continue;
|
|
730
|
+
}
|
|
731
|
+
rememberSignalledGroup(groupKey);
|
|
732
|
+
// A no-op catch keeps a stuck/failed escalation from wedging the registry:
|
|
733
|
+
// the group did not finish dying, so the next sweep may try again.
|
|
734
|
+
void reap.pending.then(
|
|
735
|
+
(outcome) => {
|
|
736
|
+
if (outcome === "stuck" || outcome === "refused") signalledGroups.delete(groupKey);
|
|
737
|
+
},
|
|
738
|
+
() => signalledGroups.delete(groupKey),
|
|
739
|
+
);
|
|
740
|
+
result.pending.push(reap.pending);
|
|
741
|
+
for (const member of members) result.reaped.push(member);
|
|
742
|
+
}
|
|
743
|
+
return result;
|
|
744
|
+
}
|