pi-crew 0.11.37 → 0.11.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +89 -0
  2. package/agents/critic.md +13 -0
  3. package/agents/oracle.md +14 -0
  4. package/dist/index.mjs +531 -526
  5. package/package.json +4 -3
  6. package/skills/delegation-patterns/SKILL.md +16 -0
  7. package/skills/real-test-pi-crew/SKILL.md +13 -2
  8. package/src/config/env-allowlist-registry.ts +22 -1
  9. package/src/config/env-vars.ts +23 -0
  10. package/src/extension/registration/foreground-run-controller.ts +21 -1
  11. package/src/extension/registration/lifecycle-handlers.ts +12 -0
  12. package/src/extension/team-tool/doctor.ts +125 -1
  13. package/src/prompt/prompt-runtime.ts +67 -5
  14. package/src/runtime/child-pi/child-pi-kill.ts +68 -2
  15. package/src/runtime/child-pi/child-pi-spawn.ts +20 -0
  16. package/src/runtime/child-pi/child-pi.ts +3 -1
  17. package/src/runtime/foreground-watchdog.ts +394 -17
  18. package/src/runtime/heartbeat/worker-heartbeat.ts +4 -0
  19. package/src/runtime/model/model-fallback.ts +5 -0
  20. package/src/runtime/process/orphan-reaper.ts +197 -22
  21. package/src/runtime/process/proc-stat.ts +8 -0
  22. package/src/runtime/process/reap-group.ts +499 -0
  23. package/src/runtime/process/run-descendant-sweep.ts +744 -0
  24. package/src/runtime/recovery/crash-recovery.ts +202 -0
  25. package/src/runtime/rpc/frame-client.ts +21 -1
  26. package/src/runtime/stale-reconciler.ts +343 -2
  27. package/src/runtime/task-runner/child-executor.ts +15 -1
  28. package/src/runtime/task-runner/post-execution.ts +112 -1
  29. package/src/runtime/task-runner/state-helpers.ts +28 -1
  30. package/src/runtime/verification/verify-gate-memo.ts +127 -0
  31. package/src/runtime/verification/verify-gate.ts +127 -1
  32. package/src/state/coordination/mailbox.ts +68 -0
  33. package/src/state/stores/state-store.ts +8 -0
  34. package/src/state/types.ts +61 -0
@@ -0,0 +1,499 @@
1
+ /**
2
+ * reap-group.ts — fix-wave 11a (F5): THE single place that may signal a POSIX
3
+ * process GROUP, with an identity-proven refusal ladder and a bounded
4
+ * escalation ladder.
5
+ *
6
+ * Why this module exists: every pre-wave reap path sent ONE signal to ONE pid
7
+ * (`orphan-reaper.ts`, `child-pi-kill.ts`) or blindly group-killed a stored
8
+ * pgid (`owned-process.ts` distrusts nothing). Both shapes fail the incident
9
+ * that motivated wave 11 (run `team_20261022110510_cefb9164af126460` completed
10
+ * but 6 grandchildren survived 24 h): a pid-only SIGTERM never reaches
11
+ * descendants, while a raw `kill(-pgid)` can hit an UNRELATED group after PID
12
+ * reuse. The guard ladder below is what makes a group kill defensible:
13
+ *
14
+ * 1. POSIX only, and /proc must be readable (the identity source).
15
+ * 2. `pgid` must be an integer > 1 — NEVER `kill(-1)` / `kill(0)`, which
16
+ * would signal every process we are allowed to signal, and never 1 (init's
17
+ * group).
18
+ * 3. never our own group (its REAL pgrp, read once from /proc — see
19
+ * `selfIdentity()`), never our pid, never our parent's pid.
20
+ * 4. the candidate must have a live identity, OR (candidate already gone) a
21
+ * caller-supplied snapshot (`expected`) that PROVES the group leader —
22
+ * `expected.pid === expected.pgid === pgid` + a starttime — because a
23
+ * dead candidate cannot prove anything by itself.
24
+ * 5. same uid as us (we can only reason about our own processes).
25
+ * 6. the candidate's REAL pgid must equal the requested pgid (PID-reuse).
26
+ * 7. the identity must carry a starttime (the reuse-proof field).
27
+ * 8. if a snapshot exists, its starttime must match the live one.
28
+ *
29
+ * Wave 11a REVIEW FIXES (round 1) baked into the ladder:
30
+ * - CRITICAL #1 `selfIdentity()`: comparing `pgid` with {pid, ppid} misses a
31
+ * host started under `setsid`/a wrapper, whose pgrp is neither ⇒ the old
32
+ * rung PASSED and `kill(-ownPgrp)` took out the host session.
33
+ * - MAJOR #2 dead-candidate path: `identity := expected` made rungs 4/6/7/8
34
+ * tautologies over the snapshot itself (blind `kill(-pgid)`). The snapshot
35
+ * is now only usable as LEADER PROOF, and the kernel's "a pgid is not
36
+ * recycled while the group still has members" property is what makes
37
+ * "leader gone but group alive" safe.
38
+ *
39
+ * Escalation: SIGTERM is delivered SYNCHRONOUSLY (postmortem-safe — a hard exit
40
+ * right after the call still leaves the graceful signal sent), then a bounded
41
+ * poll → re-validate → SIGKILL → bounded poll. The poll timers are unref'd so a
42
+ * reaper never pins the event loop — which is exactly why the SIGKILL rung needs
43
+ * explicit owners (see the "deferred SIGKILL deliveries" block below):
44
+ * `drainReapEscalations()` awaits the armed promises under a REF'D keep-alive,
45
+ * and `flushReapEscalationsSync()` delivers the last rung synchronously on
46
+ * session shutdown. Nothing here throws: every failure is a typed refusal or
47
+ * outcome.
48
+ *
49
+ * Scope note (wave 11a): a worker's bash grandchildren live in PI's bash-tool
50
+ * group, not the worker's group (GAP-0) — this module cannot reach them. That
51
+ * is the run-scoped descendant sweep (F7) in wave 11b.
52
+ */
53
+
54
+ import * as fs from "node:fs";
55
+ import { getCrewEnvBool } from "../../config/env-vars.ts";
56
+ import { logInternalError } from "../../utils/internal-error.ts";
57
+ import { fieldsAfterComm, PROC_STAT_PGID_INDEX, PROC_STAT_STARTTIME_INDEX } from "./proc-stat.ts";
58
+
59
+ /** Default grace window between SIGTERM and SIGKILL (ms). */
60
+ const DEFAULT_GRACEFUL_MS = 2_000;
61
+ /** Default hard cap after SIGKILL before declaring the group stuck (ms). */
62
+ const DEFAULT_CAP_MS = 2_000;
63
+ /** Poll interval for group-liveness (ms). */
64
+ const POLL_INTERVAL_MS = 20;
65
+
66
+ /**
67
+ * Kill switch (host-process only, default ON). `PI_CREW_REAP_GROUP=0` degrades
68
+ * every group-kill caller to its pre-wave pid-only behaviour, without a
69
+ * redeploy — the escape hatch if a platform turns out to mis-report pgids.
70
+ */
71
+ export function resolveReapGroupEnabled(): boolean {
72
+ return getCrewEnvBool("PI_CREW_REAP_GROUP") ?? true;
73
+ }
74
+
75
+ /** A process identity read from /proc — the reuse-proof triple plus pid/pgid. */
76
+ export interface ProcIdentity {
77
+ pid: number;
78
+ pgid: number | undefined;
79
+ starttime: string | undefined;
80
+ uid: number | undefined;
81
+ }
82
+
83
+ /** Every way we refuse to signal a group. Each maps to one ladder rung. */
84
+ export type GroupRefusalReason =
85
+ | "platform-not-posix"
86
+ | "proc-unavailable"
87
+ | "pgid-invalid"
88
+ | "pgid-init-or-self"
89
+ | "uid-mismatch"
90
+ | "identity-missing"
91
+ | "starttime-mismatch"
92
+ | "not-group-member";
93
+
94
+ /** Terminal verdict for a group reap attempt. */
95
+ export type GroupReapOutcome = "gone" | "killed-after-escalation" | "stuck" | "refused";
96
+
97
+ /** Injectable dependencies (every side effect is replaceable in tests). */
98
+ export interface ReapGroupOptions {
99
+ /** Grace window between SIGTERM and SIGKILL. Default 2000 ms. */
100
+ gracefulMs?: number;
101
+ /** Hard cap after SIGKILL before giving up. Default 2000 ms. */
102
+ capMs?: number;
103
+ /** Platform override (tests simulate win32). Default `process.platform`. */
104
+ platform?: NodeJS.Platform;
105
+ /** /proc identity reader (tests inject fixtures). Default `readProcIdentity`. */
106
+ readIdentity?: (pid: number) => ProcIdentity | undefined;
107
+ /** Group signal sender. Default `process.kill(-pgid, signal)`. */
108
+ signalGroup?: (pgid: number, signal: NodeJS.Signals) => void;
109
+ /** Group liveness probe. Default `process.kill(-pgid, 0)`. */
110
+ groupAlive?: (pgid: number) => boolean;
111
+ /** Snapshot recorded at spawn time — required when the candidate is gone. */
112
+ expected?: ProcIdentity;
113
+ /** Diagnostic label used in log messages. */
114
+ name?: string;
115
+ }
116
+
117
+ /** Sync verdict + the authoritative async verdict. */
118
+ export interface GroupReapResult {
119
+ /**
120
+ * Provisional sync verdict: `refused` (nothing signalled), `gone` (SIGTERM
121
+ * already cleared the group) or `killed-after-escalation` (escalation
122
+ * armed — `pending` is authoritative).
123
+ */
124
+ outcome: GroupReapOutcome;
125
+ /** Set when the guard ladder refused. */
126
+ refusal?: GroupRefusalReason;
127
+ /** Authoritative verdict; already resolved when no escalation was needed. */
128
+ pending: Promise<GroupReapOutcome>;
129
+ }
130
+
131
+ /** `/proc/<pid>/status` first `Uid:` field (real uid). Never throws. */
132
+ function readUidFromStatus(pid: number): number | undefined {
133
+ try {
134
+ const status = fs.readFileSync(`/proc/${pid}/status`, "utf8");
135
+ const match = /^Uid:\s+(\d+)/m.exec(status);
136
+ if (!match) return undefined;
137
+ const uid = Number(match[1]);
138
+ return Number.isInteger(uid) ? uid : undefined;
139
+ } catch {
140
+ return undefined;
141
+ }
142
+ }
143
+
144
+ /**
145
+ * Read pid/pgid/starttime/uid from /proc. Reuses the paren-aware stat parser in
146
+ * proc-stat.ts (comm may contain spaces AND parens — a naive split corrupts
147
+ * every field after it). Returns undefined for anything unreadable: /proc is
148
+ * absent (non-Linux), the pid is gone, or the line is malformed.
149
+ */
150
+ export function readProcIdentity(pid: number): ProcIdentity | undefined {
151
+ if (!Number.isInteger(pid) || pid <= 0) return undefined;
152
+ let stat: string;
153
+ try {
154
+ stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
155
+ } catch {
156
+ return undefined;
157
+ }
158
+ const fields = fieldsAfterComm(stat);
159
+ if (!fields) return undefined;
160
+ const rawPgid = Number(fields[PROC_STAT_PGID_INDEX]);
161
+ const pgid = Number.isInteger(rawPgid) && rawPgid > 0 ? rawPgid : undefined;
162
+ const rawStarttime = fields[PROC_STAT_STARTTIME_INDEX];
163
+ const starttime = rawStarttime && Number.isFinite(Number(rawStarttime)) ? rawStarttime : undefined;
164
+ return { pid, pgid, starttime, uid: readUidFromStatus(pid) };
165
+ }
166
+
167
+ /**
168
+ * OUR OWN identity as read from /proc — the authoritative answer to "which
169
+ * process group am I in?". Read ONCE per process and cached (a POSIX process
170
+ * cannot change its group after start), so the ladder never re-reads /proc for
171
+ * a value that cannot move. A FAILED read is NOT cached (MINOR #3): /proc can be
172
+ * momentarily unreadable (mount race, sandbox start-up) and caching that would
173
+ * turn a transient failure into a permanent refuse-everything state.
174
+ *
175
+ * Why not {process.pid, process.ppid} (the pre-review rung 3): a host started
176
+ * under `setsid`, a tmux/systemd/supervisor wrapper, or any launcher that puts
177
+ * it in a fresh group has a pgrp that is NEITHER its pid NOR its parent's pid
178
+ * — the old comparison PASSED for that group, and `kill(-pgrp)` then killed the
179
+ * whole host session. Review finding CRITICAL #1 (wave 11a).
180
+ */
181
+ let selfIdentityCache: ProcIdentity | undefined;
182
+ let selfIdentityLoaded = false;
183
+ /** Injected ONLY by tests (the ladder never accepts a per-call reader for self). */
184
+ let selfIdentityReader: (pid: number) => ProcIdentity | undefined = readProcIdentity;
185
+
186
+ /**
187
+ * Lazily read + cache our own /proc identity (undefined ⇒ unreadable RIGHT NOW;
188
+ * the next call retries). Only a successful read is cached (MINOR #3).
189
+ */
190
+ function selfIdentity(): ProcIdentity | undefined {
191
+ if (selfIdentityLoaded) return selfIdentityCache;
192
+ const identity = selfIdentityReader(process.pid);
193
+ if (!identity) {
194
+ logInternalError(
195
+ "reap-group.self-identity",
196
+ new Error("cannot read our own /proc identity — refusing group signals, will retry"),
197
+ undefined,
198
+ "debug",
199
+ );
200
+ return undefined;
201
+ }
202
+ selfIdentityCache = identity;
203
+ selfIdentityLoaded = true;
204
+ return identity;
205
+ }
206
+
207
+ /** Test seam: forget the cached self identity so the next call re-reads /proc. */
208
+ export function __resetSelfIdentityForTest(): void {
209
+ selfIdentityCache = undefined;
210
+ selfIdentityLoaded = false;
211
+ selfIdentityReader = readProcIdentity;
212
+ }
213
+
214
+ /** Test seam: install a synthetic self identity (undefined ⇒ "cannot read"). */
215
+ export function __setSelfIdentityForTest(identity: ProcIdentity | undefined): void {
216
+ selfIdentityCache = identity;
217
+ selfIdentityLoaded = true;
218
+ }
219
+
220
+ /**
221
+ * Test seam: swap the /proc reader used for OUR OWN identity. Needed to pin the
222
+ * MINOR #3 retry rule (a FAILED self read must not be cached) without depending
223
+ * on a real /proc — the per-call `opts.readIdentity` deliberately cannot do it.
224
+ */
225
+ export function __setSelfIdentityReaderForTest(reader: ((pid: number) => ProcIdentity | undefined) | undefined): void {
226
+ selfIdentityReader = reader ?? readProcIdentity;
227
+ }
228
+
229
+ /**
230
+ * Run the refusal ladder. Returns the FIRST failing rung, or undefined when the
231
+ * group is safe to signal. Pure with respect to injected `readIdentity` /
232
+ * `groupAlive`; the self identity is cached module state (seam above).
233
+ */
234
+ export function assertSafeGroup(pgid: number, candidatePid: number, opts: ReapGroupOptions = {}): GroupRefusalReason | undefined {
235
+ const platform = opts.platform ?? process.platform;
236
+ const readIdentity = opts.readIdentity ?? readProcIdentity;
237
+ const groupAlive = opts.groupAlive ?? defaultGroupAlive;
238
+ // Rung 1: our own /proc identity must be readable — it is BOTH the /proc
239
+ // availability probe AND the rung-3 comparison. MINOR #4 (review): this is
240
+ // the cached SELF read, never the injected `readIdentity`: the injection point
241
+ // exists for the CANDIDATE, and letting a test hook vouch for our own identity
242
+ // is precisely the shape of CRITICAL #1.
243
+ if (platform === "win32") return "platform-not-posix";
244
+ const self = selfIdentity();
245
+ if (!self) return "proc-unavailable";
246
+ // Rung 2: a non-positive pgid would signal far too much (kill(-1)/kill(0));
247
+ // pgid 1 is init's group — reaping it is never a worker cleanup.
248
+ if (!Number.isInteger(pgid) || pgid <= 1) return "pgid-invalid";
249
+ // Rung 3: never our own group (its REAL pgrp — CRITICAL #1), never our pid,
250
+ // never our parent's pid. Unknown self pgrp ⇒ refuse: that is exactly the
251
+ // state in which `kill(-pgid)` could take out the host session.
252
+ if (self.pgid === undefined) return "identity-missing";
253
+ if (pgid === process.pid || pgid === process.ppid || pgid === self.pgid) return "pgid-init-or-self";
254
+ const selfUid = typeof process.getuid === "function" ? process.getuid() : undefined;
255
+ // Rung 4: a LIVE candidate identity, or (dead candidate) leader-proof.
256
+ const live = readIdentity(candidatePid);
257
+ if (!live) {
258
+ // The candidate is gone, so nothing about IT can be verified any more. A
259
+ // snapshot is only evidence when it PROVES the group leader: the candidate
260
+ // we were asked to signal must BE the leader (`candidatePid === pgid`, and
261
+ // the snapshot must agree on both pid and pgid — MINOR #1) and must carry
262
+ // the starttime. Without that, every later rung would compare the snapshot
263
+ // with itself ⇒ tautology ⇒ blind kill(-pgid) after PID reuse (MAJOR #2).
264
+ const expected = opts.expected;
265
+ if (!expected || candidatePid !== pgid || expected.pid !== pgid || expected.pgid !== pgid || expected.starttime === undefined) {
266
+ return "identity-missing";
267
+ }
268
+ // Rung 5 (dead path): the uid proof is MANDATORY here (MINOR #2). The live
269
+ // /proc path always carries one, so an absent uid means we cannot separate
270
+ // our own process from a stranger's: signalling then fails with EPERM AFTER
271
+ // the caller already reported `reaped`, which leaves a false `stuck`.
272
+ if (expected.uid === undefined) return "identity-missing";
273
+ if (selfUid !== undefined && expected.uid !== selfUid) return "uid-mismatch";
274
+ // The kernel does NOT recycle a pgid while the group still has a member,
275
+ // so the leader is the only pid whose starttime the snapshot pins. If the
276
+ // leader is still there it must be the SAME incarnation AND still lead that
277
+ // group (MINOR #1: a pid that left the group via setpgid is no evidence).
278
+ const leader = readIdentity(pgid);
279
+ if (leader) {
280
+ if (leader.pgid !== pgid) return "not-group-member";
281
+ if (leader.starttime === undefined) return "identity-missing";
282
+ if (leader.starttime !== expected.starttime) return "starttime-mismatch";
283
+ if (leader.uid !== undefined && selfUid !== undefined && leader.uid !== selfUid) return "uid-mismatch";
284
+ return undefined;
285
+ }
286
+ // Leader gone: a still-live group means surviving members hold the pgid
287
+ // (so it cannot have been recycled) ⇒ our old group. An empty group means
288
+ // there is nothing to justify the kill and nothing to kill.
289
+ if (groupAlive(pgid)) {
290
+ // TOCTOU, documented on purpose (MINOR #7): between this probe and the
291
+ // caller's `signalGroup` the LAST member may exit and the pgid be recycled
292
+ // by an unrelated group. That window cannot be closed without a pidfd-style
293
+ // handle (one signal per pid, no group target). It is accepted because the
294
+ // alternative — re-introducing the snapshot as identity — made the very
295
+ // rungs above tautological (MAJOR #2).
296
+ return undefined;
297
+ }
298
+ return "identity-missing";
299
+ }
300
+ // Rung 5: same uid (a missing uid only happens for injected snapshots; the
301
+ // live /proc path always carries one — the remaining rungs still hold).
302
+ if (live.uid !== undefined && selfUid !== undefined && live.uid !== selfUid) return "uid-mismatch";
303
+ // Rung 6: PID reuse — the candidate must actually belong to that group.
304
+ if (live.pgid !== pgid) return "not-group-member";
305
+ // Rung 7: starttime is the field that survives PID reuse.
306
+ if (live.starttime === undefined) return "identity-missing";
307
+ // Rung 8: the snapshot and the live process must be the SAME incarnation.
308
+ const expectedStart = opts.expected?.starttime;
309
+ if (expectedStart !== undefined && live.starttime !== expectedStart) return "starttime-mismatch";
310
+ return undefined;
311
+ }
312
+
313
+ /** Unref'd sleep so an in-flight reap never pins the event loop. */
314
+ const delay = (ms: number): Promise<void> =>
315
+ new Promise((resolve) => {
316
+ const timer = setTimeout(resolve, Math.max(0, ms));
317
+ timer.unref?.();
318
+ });
319
+
320
+ /** Poll `predicate` until true or the deadline passes; returns the final value. */
321
+ async function pollUntil(predicate: () => boolean, timeoutMs: number): Promise<boolean> {
322
+ const deadline = Date.now() + Math.max(0, timeoutMs);
323
+ while (Date.now() < deadline) {
324
+ await delay(Math.min(POLL_INTERVAL_MS, Math.max(0, deadline - Date.now())));
325
+ if (predicate()) return true;
326
+ }
327
+ return predicate();
328
+ }
329
+
330
+ /** Default group liveness: `kill(-pgid, 0)`; EPERM means "alive, not ours". */
331
+ function defaultGroupAlive(pgid: number): boolean {
332
+ if (!Number.isInteger(pgid) || pgid <= 1) return false;
333
+ try {
334
+ process.kill(-pgid, 0);
335
+ return true;
336
+ } catch (err) {
337
+ return (err as NodeJS.ErrnoException).code === "EPERM";
338
+ }
339
+ }
340
+
341
+ /** Default group signal. Defensively refuses non-positive pgids (kill(-1)/kill(0)). */
342
+ function defaultSignalGroup(pgid: number, signal: NodeJS.Signals): void {
343
+ if (!Number.isInteger(pgid) || pgid <= 1) return;
344
+ try {
345
+ process.kill(-pgid, signal);
346
+ } catch {
347
+ /* group already gone — nothing to deliver */
348
+ }
349
+ }
350
+
351
+ /** Log a refusal/abnormal outcome at debug (never above: refusals are expected). */
352
+ function logDebug(scope: string, message: string, opts: ReapGroupOptions): void {
353
+ logInternalError(scope, new Error(message), `name=${opts.name ?? "-"}`, "debug");
354
+ }
355
+
356
+ // ── deferred SIGKILL deliveries (wave 11a review, MAJOR #3) ───────────────────
357
+ //
358
+ // `reapGroup` hands the SIGKILL rung back as a promise, and every production
359
+ // caller is SYNCHRONOUS (the stale-run reconciler, `killProcessTree`): they can
360
+ // at most register the promise with a drain owner. The escalation's own poll
361
+ // timers are unref'd (they must not keep a finished run alive), which means an
362
+ // un-awaited escalation is abandoned the moment the host's loop drains — the
363
+ // SIGKILL is silently never delivered (measured: an unsettled top-level await
364
+ // over unref'd timers exits 13).
365
+ //
366
+ // Two owners close that hole:
367
+ // 1. `drainReapEscalations()` (stale-reconciler.ts) awaits the registered
368
+ // promises while holding a REF'D keep-alive interval.
369
+ // 2. `flushReapEscalationsSync()` (below) is for session shutdown: it
370
+ // re-validates and delivers the SIGKILL rung synchronously, so the last
371
+ // rung still happens in a process that is about to exit.
372
+ /** One armed escalation: everything needed to re-validate and signal it later. */
373
+ interface ArmedGroupEscalation {
374
+ pgid: number;
375
+ candidatePid: number;
376
+ opts: ReapGroupOptions;
377
+ /** Set once {@link flushReapEscalationsSync} delivered the SIGKILL — the async
378
+ * escalation must then never signal a second time. */
379
+ deliveredSync?: boolean;
380
+ }
381
+
382
+ const armedEscalations = new Set<ArmedGroupEscalation>();
383
+
384
+ /**
385
+ * Deliver the SIGKILL rung of every ARMED escalation SYNCHRONOUSLY — the
386
+ * session-shutdown path. Each entry is re-validated against the ladder at THIS
387
+ * instant (the group may have drained, or the pgid been recycled since the
388
+ * SIGTERM), so a stale entry never becomes a blind kill. Best-effort per entry:
389
+ * one failure cannot stop the rest. The registry is cleared either way. Returns
390
+ * the number of groups signalled. Never throws.
391
+ */
392
+ export function flushReapEscalationsSync(): number {
393
+ const entries = [...armedEscalations];
394
+ armedEscalations.clear();
395
+ let delivered = 0;
396
+ for (const entry of entries) {
397
+ try {
398
+ const refusal = assertSafeGroup(entry.pgid, entry.candidatePid, entry.opts);
399
+ if (refusal) {
400
+ logDebug("reap-group.flush-refused", `no deferred SIGKILL for pgid=${entry.pgid} (${refusal})`, entry.opts);
401
+ continue;
402
+ }
403
+ (entry.opts.signalGroup ?? defaultSignalGroup)(entry.pgid, "SIGKILL");
404
+ entry.deliveredSync = true;
405
+ delivered++;
406
+ } catch (err) {
407
+ logInternalError(
408
+ "reap-group.flush-error",
409
+ err instanceof Error ? err : new Error(String(err)),
410
+ `pgid=${entry.pgid} name=${entry.opts.name ?? "-"}`,
411
+ "warn",
412
+ );
413
+ }
414
+ }
415
+ if (delivered > 0) logDebug("reap-group.flush", `delivered ${delivered} deferred group SIGKILL(s) at shutdown`, {});
416
+ return delivered;
417
+ }
418
+
419
+ /** Test seam: how many escalations are still armed (registry must empty on flush). */
420
+ export function __armedEscalationCountForTest(): number {
421
+ return armedEscalations.size;
422
+ }
423
+
424
+ /** Test seam: drop the armed registry without signalling anything. */
425
+ export function __resetArmedEscalationsForTest(): void {
426
+ armedEscalations.clear();
427
+ }
428
+
429
+ /**
430
+ * Reap a whole process group. See the module header for the safety model.
431
+ * Refused ⇒ nothing is signalled. Accepted ⇒ SIGTERM now, escalation in
432
+ * `pending` (re-validated before SIGKILL).
433
+ */
434
+ export function reapGroup(pgid: number, candidatePid: number, opts: ReapGroupOptions = {}): GroupReapResult {
435
+ const signalGroup = opts.signalGroup ?? defaultSignalGroup;
436
+ const groupAlive = opts.groupAlive ?? defaultGroupAlive;
437
+ const gracefulMs = opts.gracefulMs ?? DEFAULT_GRACEFUL_MS;
438
+ const capMs = opts.capMs ?? DEFAULT_CAP_MS;
439
+ const refusal = assertSafeGroup(pgid, candidatePid, opts);
440
+ if (refusal) {
441
+ logDebug("reap-group.refused", `refusing to signal pgid=${pgid} (${refusal})`, opts);
442
+ return { outcome: "refused", refusal, pending: Promise.resolve("refused") };
443
+ }
444
+ // Postmortem-safe prefix: the graceful signal is out BEFORE any await, so a
445
+ // host that dies right here still leaves SIGTERM delivered.
446
+ signalGroup(pgid, "SIGTERM");
447
+ if (!groupAlive(pgid)) return { outcome: "gone", pending: Promise.resolve("gone") };
448
+ // MINOR #5 (review): the escalation promise is handed to callers who may
449
+ // `void` it. It must therefore NEVER reject — an unhandled rejection is fatal
450
+ // on Node 22 — so every rung is wrapped and the worst case is a typed
451
+ // "stuck" outcome.
452
+ // MAJOR #3 (review): the escalation is also REGISTERED, so the two owners
453
+ // above can await it (drain) or deliver it synchronously (shutdown flush).
454
+ const armed: ArmedGroupEscalation = { pgid, candidatePid, opts };
455
+ armedEscalations.add(armed);
456
+ const pending = (async (): Promise<GroupReapOutcome> => {
457
+ try {
458
+ if (await pollUntil(() => !groupAlive(pgid), gracefulMs)) return "gone";
459
+ // The shutdown flush may already have delivered SIGKILL synchronously —
460
+ // signalling a second time would be a double kill of a possibly recycled
461
+ // group, so stop here.
462
+ if (armed.deliveredSync === true) return "killed-after-escalation";
463
+ // Re-validate immediately before SIGKILL: between the SIGTERM prefix and
464
+ // here the group may have drained and the pgid been recycled by an
465
+ // unrelated group. A failure here is a refusal, not an escalation.
466
+ const recheck = assertSafeGroup(pgid, candidatePid, opts);
467
+ if (recheck) {
468
+ logDebug("reap-group.sigkill-refused", `escalation refused for pgid=${pgid} (${recheck})`, opts);
469
+ return "refused";
470
+ }
471
+ signalGroup(pgid, "SIGKILL");
472
+ if (await pollUntil(() => !groupAlive(pgid), capMs)) return "killed-after-escalation";
473
+ logInternalError(
474
+ "reap-group.stuck",
475
+ new Error(`process group ${pgid} still alive after SIGKILL cap (${capMs}ms); giving up`),
476
+ `name=${opts.name ?? "-"}`,
477
+ "warn",
478
+ );
479
+ return "stuck";
480
+ } catch (err) {
481
+ logInternalError(
482
+ "reap-group.escalation-error",
483
+ err instanceof Error ? err : new Error(String(err)),
484
+ `pgid=${pgid} name=${opts.name ?? "-"}`,
485
+ "warn",
486
+ );
487
+ return "stuck";
488
+ }
489
+ })();
490
+ // Retire the entry the moment the escalation settles: a healthy escalation
491
+ // leaves nothing behind for the shutdown flush (and an empty registry is what
492
+ // the flush contract promises). Both handlers swallow — this derived promise
493
+ // must never reject, nothing awaits it.
494
+ void pending.then(
495
+ () => armedEscalations.delete(armed),
496
+ () => armedEscalations.delete(armed),
497
+ );
498
+ return { outcome: "killed-after-escalation", pending };
499
+ }