pi-crew 0.11.37 → 0.11.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/agents/critic.md +13 -0
- package/agents/oracle.md +14 -0
- package/dist/index.mjs +531 -526
- package/package.json +4 -3
- package/skills/delegation-patterns/SKILL.md +16 -0
- package/skills/real-test-pi-crew/SKILL.md +13 -2
- package/src/config/env-allowlist-registry.ts +22 -1
- package/src/config/env-vars.ts +23 -0
- package/src/extension/registration/foreground-run-controller.ts +21 -1
- package/src/extension/registration/lifecycle-handlers.ts +12 -0
- package/src/extension/team-tool/doctor.ts +125 -1
- package/src/prompt/prompt-runtime.ts +67 -5
- package/src/runtime/child-pi/child-pi-kill.ts +68 -2
- package/src/runtime/child-pi/child-pi-spawn.ts +20 -0
- package/src/runtime/child-pi/child-pi.ts +3 -1
- package/src/runtime/foreground-watchdog.ts +394 -17
- package/src/runtime/heartbeat/worker-heartbeat.ts +4 -0
- package/src/runtime/model/model-fallback.ts +5 -0
- package/src/runtime/process/orphan-reaper.ts +197 -22
- package/src/runtime/process/proc-stat.ts +8 -0
- package/src/runtime/process/reap-group.ts +499 -0
- package/src/runtime/process/run-descendant-sweep.ts +744 -0
- package/src/runtime/recovery/crash-recovery.ts +202 -0
- package/src/runtime/rpc/frame-client.ts +21 -1
- package/src/runtime/stale-reconciler.ts +343 -2
- package/src/runtime/task-runner/child-executor.ts +15 -1
- package/src/runtime/task-runner/post-execution.ts +112 -1
- package/src/runtime/task-runner/state-helpers.ts +28 -1
- package/src/runtime/verification/verify-gate-memo.ts +127 -0
- package/src/runtime/verification/verify-gate.ts +127 -1
- package/src/state/coordination/mailbox.ts +68 -0
- package/src/state/stores/state-store.ts +8 -0
- package/src/state/types.ts +61 -0
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* reap-group.ts — fix-wave 11a (F5): THE single place that may signal a POSIX
|
|
3
|
+
* process GROUP, with an identity-proven refusal ladder and a bounded
|
|
4
|
+
* escalation ladder.
|
|
5
|
+
*
|
|
6
|
+
* Why this module exists: every pre-wave reap path sent ONE signal to ONE pid
|
|
7
|
+
* (`orphan-reaper.ts`, `child-pi-kill.ts`) or blindly group-killed a stored
|
|
8
|
+
* pgid (`owned-process.ts` distrusts nothing). Both shapes fail the incident
|
|
9
|
+
* that motivated wave 11 (run `team_20261022110510_cefb9164af126460` completed
|
|
10
|
+
* but 6 grandchildren survived 24 h): a pid-only SIGTERM never reaches
|
|
11
|
+
* descendants, while a raw `kill(-pgid)` can hit an UNRELATED group after PID
|
|
12
|
+
* reuse. The guard ladder below is what makes a group kill defensible:
|
|
13
|
+
*
|
|
14
|
+
* 1. POSIX only, and /proc must be readable (the identity source).
|
|
15
|
+
* 2. `pgid` must be an integer > 1 — NEVER `kill(-1)` / `kill(0)`, which
|
|
16
|
+
* would signal every process we are allowed to signal, and never 1 (init's
|
|
17
|
+
* group).
|
|
18
|
+
* 3. never our own group (its REAL pgrp, read once from /proc — see
|
|
19
|
+
* `selfIdentity()`), never our pid, never our parent's pid.
|
|
20
|
+
* 4. the candidate must have a live identity, OR (candidate already gone) a
|
|
21
|
+
* caller-supplied snapshot (`expected`) that PROVES the group leader —
|
|
22
|
+
* `expected.pid === expected.pgid === pgid` + a starttime — because a
|
|
23
|
+
* dead candidate cannot prove anything by itself.
|
|
24
|
+
* 5. same uid as us (we can only reason about our own processes).
|
|
25
|
+
* 6. the candidate's REAL pgid must equal the requested pgid (PID-reuse).
|
|
26
|
+
* 7. the identity must carry a starttime (the reuse-proof field).
|
|
27
|
+
* 8. if a snapshot exists, its starttime must match the live one.
|
|
28
|
+
*
|
|
29
|
+
* Wave 11a REVIEW FIXES (round 1) baked into the ladder:
|
|
30
|
+
* - CRITICAL #1 `selfIdentity()`: comparing `pgid` with {pid, ppid} misses a
|
|
31
|
+
* host started under `setsid`/a wrapper, whose pgrp is neither ⇒ the old
|
|
32
|
+
* rung PASSED and `kill(-ownPgrp)` took out the host session.
|
|
33
|
+
* - MAJOR #2 dead-candidate path: `identity := expected` made rungs 4/6/7/8
|
|
34
|
+
* tautologies over the snapshot itself (blind `kill(-pgid)`). The snapshot
|
|
35
|
+
* is now only usable as LEADER PROOF, and the kernel's "a pgid is not
|
|
36
|
+
* recycled while the group still has members" property is what makes
|
|
37
|
+
* "leader gone but group alive" safe.
|
|
38
|
+
*
|
|
39
|
+
* Escalation: SIGTERM is delivered SYNCHRONOUSLY (postmortem-safe — a hard exit
|
|
40
|
+
* right after the call still leaves the graceful signal sent), then a bounded
|
|
41
|
+
* poll → re-validate → SIGKILL → bounded poll. The poll timers are unref'd so a
|
|
42
|
+
* reaper never pins the event loop — which is exactly why the SIGKILL rung needs
|
|
43
|
+
* explicit owners (see the "deferred SIGKILL deliveries" block below):
|
|
44
|
+
* `drainReapEscalations()` awaits the armed promises under a REF'D keep-alive,
|
|
45
|
+
* and `flushReapEscalationsSync()` delivers the last rung synchronously on
|
|
46
|
+
* session shutdown. Nothing here throws: every failure is a typed refusal or
|
|
47
|
+
* outcome.
|
|
48
|
+
*
|
|
49
|
+
* Scope note (wave 11a): a worker's bash grandchildren live in PI's bash-tool
|
|
50
|
+
* group, not the worker's group (GAP-0) — this module cannot reach them. That
|
|
51
|
+
* is the run-scoped descendant sweep (F7) in wave 11b.
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
import * as fs from "node:fs";
|
|
55
|
+
import { getCrewEnvBool } from "../../config/env-vars.ts";
|
|
56
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
57
|
+
import { fieldsAfterComm, PROC_STAT_PGID_INDEX, PROC_STAT_STARTTIME_INDEX } from "./proc-stat.ts";
|
|
58
|
+
|
|
59
|
+
/** Default grace window between SIGTERM and SIGKILL (ms). */
|
|
60
|
+
const DEFAULT_GRACEFUL_MS = 2_000;
|
|
61
|
+
/** Default hard cap after SIGKILL before declaring the group stuck (ms). */
|
|
62
|
+
const DEFAULT_CAP_MS = 2_000;
|
|
63
|
+
/** Poll interval for group-liveness (ms). */
|
|
64
|
+
const POLL_INTERVAL_MS = 20;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Kill switch (host-process only, default ON). `PI_CREW_REAP_GROUP=0` degrades
|
|
68
|
+
* every group-kill caller to its pre-wave pid-only behaviour, without a
|
|
69
|
+
* redeploy — the escape hatch if a platform turns out to mis-report pgids.
|
|
70
|
+
*/
|
|
71
|
+
export function resolveReapGroupEnabled(): boolean {
|
|
72
|
+
return getCrewEnvBool("PI_CREW_REAP_GROUP") ?? true;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** A process identity read from /proc — the reuse-proof triple plus pid/pgid. */
|
|
76
|
+
export interface ProcIdentity {
|
|
77
|
+
pid: number;
|
|
78
|
+
pgid: number | undefined;
|
|
79
|
+
starttime: string | undefined;
|
|
80
|
+
uid: number | undefined;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Every way we refuse to signal a group. Each maps to one ladder rung. */
|
|
84
|
+
export type GroupRefusalReason =
|
|
85
|
+
| "platform-not-posix"
|
|
86
|
+
| "proc-unavailable"
|
|
87
|
+
| "pgid-invalid"
|
|
88
|
+
| "pgid-init-or-self"
|
|
89
|
+
| "uid-mismatch"
|
|
90
|
+
| "identity-missing"
|
|
91
|
+
| "starttime-mismatch"
|
|
92
|
+
| "not-group-member";
|
|
93
|
+
|
|
94
|
+
/** Terminal verdict for a group reap attempt. */
|
|
95
|
+
export type GroupReapOutcome = "gone" | "killed-after-escalation" | "stuck" | "refused";
|
|
96
|
+
|
|
97
|
+
/** Injectable dependencies (every side effect is replaceable in tests). */
|
|
98
|
+
export interface ReapGroupOptions {
|
|
99
|
+
/** Grace window between SIGTERM and SIGKILL. Default 2000 ms. */
|
|
100
|
+
gracefulMs?: number;
|
|
101
|
+
/** Hard cap after SIGKILL before giving up. Default 2000 ms. */
|
|
102
|
+
capMs?: number;
|
|
103
|
+
/** Platform override (tests simulate win32). Default `process.platform`. */
|
|
104
|
+
platform?: NodeJS.Platform;
|
|
105
|
+
/** /proc identity reader (tests inject fixtures). Default `readProcIdentity`. */
|
|
106
|
+
readIdentity?: (pid: number) => ProcIdentity | undefined;
|
|
107
|
+
/** Group signal sender. Default `process.kill(-pgid, signal)`. */
|
|
108
|
+
signalGroup?: (pgid: number, signal: NodeJS.Signals) => void;
|
|
109
|
+
/** Group liveness probe. Default `process.kill(-pgid, 0)`. */
|
|
110
|
+
groupAlive?: (pgid: number) => boolean;
|
|
111
|
+
/** Snapshot recorded at spawn time — required when the candidate is gone. */
|
|
112
|
+
expected?: ProcIdentity;
|
|
113
|
+
/** Diagnostic label used in log messages. */
|
|
114
|
+
name?: string;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Sync verdict + the authoritative async verdict. */
|
|
118
|
+
export interface GroupReapResult {
|
|
119
|
+
/**
|
|
120
|
+
* Provisional sync verdict: `refused` (nothing signalled), `gone` (SIGTERM
|
|
121
|
+
* already cleared the group) or `killed-after-escalation` (escalation
|
|
122
|
+
* armed — `pending` is authoritative).
|
|
123
|
+
*/
|
|
124
|
+
outcome: GroupReapOutcome;
|
|
125
|
+
/** Set when the guard ladder refused. */
|
|
126
|
+
refusal?: GroupRefusalReason;
|
|
127
|
+
/** Authoritative verdict; already resolved when no escalation was needed. */
|
|
128
|
+
pending: Promise<GroupReapOutcome>;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** `/proc/<pid>/status` first `Uid:` field (real uid). Never throws. */
|
|
132
|
+
function readUidFromStatus(pid: number): number | undefined {
|
|
133
|
+
try {
|
|
134
|
+
const status = fs.readFileSync(`/proc/${pid}/status`, "utf8");
|
|
135
|
+
const match = /^Uid:\s+(\d+)/m.exec(status);
|
|
136
|
+
if (!match) return undefined;
|
|
137
|
+
const uid = Number(match[1]);
|
|
138
|
+
return Number.isInteger(uid) ? uid : undefined;
|
|
139
|
+
} catch {
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Read pid/pgid/starttime/uid from /proc. Reuses the paren-aware stat parser in
|
|
146
|
+
* proc-stat.ts (comm may contain spaces AND parens — a naive split corrupts
|
|
147
|
+
* every field after it). Returns undefined for anything unreadable: /proc is
|
|
148
|
+
* absent (non-Linux), the pid is gone, or the line is malformed.
|
|
149
|
+
*/
|
|
150
|
+
export function readProcIdentity(pid: number): ProcIdentity | undefined {
|
|
151
|
+
if (!Number.isInteger(pid) || pid <= 0) return undefined;
|
|
152
|
+
let stat: string;
|
|
153
|
+
try {
|
|
154
|
+
stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
155
|
+
} catch {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
const fields = fieldsAfterComm(stat);
|
|
159
|
+
if (!fields) return undefined;
|
|
160
|
+
const rawPgid = Number(fields[PROC_STAT_PGID_INDEX]);
|
|
161
|
+
const pgid = Number.isInteger(rawPgid) && rawPgid > 0 ? rawPgid : undefined;
|
|
162
|
+
const rawStarttime = fields[PROC_STAT_STARTTIME_INDEX];
|
|
163
|
+
const starttime = rawStarttime && Number.isFinite(Number(rawStarttime)) ? rawStarttime : undefined;
|
|
164
|
+
return { pid, pgid, starttime, uid: readUidFromStatus(pid) };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* OUR OWN identity as read from /proc — the authoritative answer to "which
|
|
169
|
+
* process group am I in?". Read ONCE per process and cached (a POSIX process
|
|
170
|
+
* cannot change its group after start), so the ladder never re-reads /proc for
|
|
171
|
+
* a value that cannot move. A FAILED read is NOT cached (MINOR #3): /proc can be
|
|
172
|
+
* momentarily unreadable (mount race, sandbox start-up) and caching that would
|
|
173
|
+
* turn a transient failure into a permanent refuse-everything state.
|
|
174
|
+
*
|
|
175
|
+
* Why not {process.pid, process.ppid} (the pre-review rung 3): a host started
|
|
176
|
+
* under `setsid`, a tmux/systemd/supervisor wrapper, or any launcher that puts
|
|
177
|
+
* it in a fresh group has a pgrp that is NEITHER its pid NOR its parent's pid
|
|
178
|
+
* — the old comparison PASSED for that group, and `kill(-pgrp)` then killed the
|
|
179
|
+
* whole host session. Review finding CRITICAL #1 (wave 11a).
|
|
180
|
+
*/
|
|
181
|
+
let selfIdentityCache: ProcIdentity | undefined;
|
|
182
|
+
let selfIdentityLoaded = false;
|
|
183
|
+
/** Injected ONLY by tests (the ladder never accepts a per-call reader for self). */
|
|
184
|
+
let selfIdentityReader: (pid: number) => ProcIdentity | undefined = readProcIdentity;
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Lazily read + cache our own /proc identity (undefined ⇒ unreadable RIGHT NOW;
|
|
188
|
+
* the next call retries). Only a successful read is cached (MINOR #3).
|
|
189
|
+
*/
|
|
190
|
+
function selfIdentity(): ProcIdentity | undefined {
|
|
191
|
+
if (selfIdentityLoaded) return selfIdentityCache;
|
|
192
|
+
const identity = selfIdentityReader(process.pid);
|
|
193
|
+
if (!identity) {
|
|
194
|
+
logInternalError(
|
|
195
|
+
"reap-group.self-identity",
|
|
196
|
+
new Error("cannot read our own /proc identity — refusing group signals, will retry"),
|
|
197
|
+
undefined,
|
|
198
|
+
"debug",
|
|
199
|
+
);
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
|
202
|
+
selfIdentityCache = identity;
|
|
203
|
+
selfIdentityLoaded = true;
|
|
204
|
+
return identity;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Test seam: forget the cached self identity so the next call re-reads /proc. */
|
|
208
|
+
export function __resetSelfIdentityForTest(): void {
|
|
209
|
+
selfIdentityCache = undefined;
|
|
210
|
+
selfIdentityLoaded = false;
|
|
211
|
+
selfIdentityReader = readProcIdentity;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Test seam: install a synthetic self identity (undefined ⇒ "cannot read"). */
|
|
215
|
+
export function __setSelfIdentityForTest(identity: ProcIdentity | undefined): void {
|
|
216
|
+
selfIdentityCache = identity;
|
|
217
|
+
selfIdentityLoaded = true;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Test seam: swap the /proc reader used for OUR OWN identity. Needed to pin the
|
|
222
|
+
* MINOR #3 retry rule (a FAILED self read must not be cached) without depending
|
|
223
|
+
* on a real /proc — the per-call `opts.readIdentity` deliberately cannot do it.
|
|
224
|
+
*/
|
|
225
|
+
export function __setSelfIdentityReaderForTest(reader: ((pid: number) => ProcIdentity | undefined) | undefined): void {
|
|
226
|
+
selfIdentityReader = reader ?? readProcIdentity;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Run the refusal ladder. Returns the FIRST failing rung, or undefined when the
|
|
231
|
+
* group is safe to signal. Pure with respect to injected `readIdentity` /
|
|
232
|
+
* `groupAlive`; the self identity is cached module state (seam above).
|
|
233
|
+
*/
|
|
234
|
+
export function assertSafeGroup(pgid: number, candidatePid: number, opts: ReapGroupOptions = {}): GroupRefusalReason | undefined {
|
|
235
|
+
const platform = opts.platform ?? process.platform;
|
|
236
|
+
const readIdentity = opts.readIdentity ?? readProcIdentity;
|
|
237
|
+
const groupAlive = opts.groupAlive ?? defaultGroupAlive;
|
|
238
|
+
// Rung 1: our own /proc identity must be readable — it is BOTH the /proc
|
|
239
|
+
// availability probe AND the rung-3 comparison. MINOR #4 (review): this is
|
|
240
|
+
// the cached SELF read, never the injected `readIdentity`: the injection point
|
|
241
|
+
// exists for the CANDIDATE, and letting a test hook vouch for our own identity
|
|
242
|
+
// is precisely the shape of CRITICAL #1.
|
|
243
|
+
if (platform === "win32") return "platform-not-posix";
|
|
244
|
+
const self = selfIdentity();
|
|
245
|
+
if (!self) return "proc-unavailable";
|
|
246
|
+
// Rung 2: a non-positive pgid would signal far too much (kill(-1)/kill(0));
|
|
247
|
+
// pgid 1 is init's group — reaping it is never a worker cleanup.
|
|
248
|
+
if (!Number.isInteger(pgid) || pgid <= 1) return "pgid-invalid";
|
|
249
|
+
// Rung 3: never our own group (its REAL pgrp — CRITICAL #1), never our pid,
|
|
250
|
+
// never our parent's pid. Unknown self pgrp ⇒ refuse: that is exactly the
|
|
251
|
+
// state in which `kill(-pgid)` could take out the host session.
|
|
252
|
+
if (self.pgid === undefined) return "identity-missing";
|
|
253
|
+
if (pgid === process.pid || pgid === process.ppid || pgid === self.pgid) return "pgid-init-or-self";
|
|
254
|
+
const selfUid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
|
255
|
+
// Rung 4: a LIVE candidate identity, or (dead candidate) leader-proof.
|
|
256
|
+
const live = readIdentity(candidatePid);
|
|
257
|
+
if (!live) {
|
|
258
|
+
// The candidate is gone, so nothing about IT can be verified any more. A
|
|
259
|
+
// snapshot is only evidence when it PROVES the group leader: the candidate
|
|
260
|
+
// we were asked to signal must BE the leader (`candidatePid === pgid`, and
|
|
261
|
+
// the snapshot must agree on both pid and pgid — MINOR #1) and must carry
|
|
262
|
+
// the starttime. Without that, every later rung would compare the snapshot
|
|
263
|
+
// with itself ⇒ tautology ⇒ blind kill(-pgid) after PID reuse (MAJOR #2).
|
|
264
|
+
const expected = opts.expected;
|
|
265
|
+
if (!expected || candidatePid !== pgid || expected.pid !== pgid || expected.pgid !== pgid || expected.starttime === undefined) {
|
|
266
|
+
return "identity-missing";
|
|
267
|
+
}
|
|
268
|
+
// Rung 5 (dead path): the uid proof is MANDATORY here (MINOR #2). The live
|
|
269
|
+
// /proc path always carries one, so an absent uid means we cannot separate
|
|
270
|
+
// our own process from a stranger's: signalling then fails with EPERM AFTER
|
|
271
|
+
// the caller already reported `reaped`, which leaves a false `stuck`.
|
|
272
|
+
if (expected.uid === undefined) return "identity-missing";
|
|
273
|
+
if (selfUid !== undefined && expected.uid !== selfUid) return "uid-mismatch";
|
|
274
|
+
// The kernel does NOT recycle a pgid while the group still has a member,
|
|
275
|
+
// so the leader is the only pid whose starttime the snapshot pins. If the
|
|
276
|
+
// leader is still there it must be the SAME incarnation AND still lead that
|
|
277
|
+
// group (MINOR #1: a pid that left the group via setpgid is no evidence).
|
|
278
|
+
const leader = readIdentity(pgid);
|
|
279
|
+
if (leader) {
|
|
280
|
+
if (leader.pgid !== pgid) return "not-group-member";
|
|
281
|
+
if (leader.starttime === undefined) return "identity-missing";
|
|
282
|
+
if (leader.starttime !== expected.starttime) return "starttime-mismatch";
|
|
283
|
+
if (leader.uid !== undefined && selfUid !== undefined && leader.uid !== selfUid) return "uid-mismatch";
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
// Leader gone: a still-live group means surviving members hold the pgid
|
|
287
|
+
// (so it cannot have been recycled) ⇒ our old group. An empty group means
|
|
288
|
+
// there is nothing to justify the kill and nothing to kill.
|
|
289
|
+
if (groupAlive(pgid)) {
|
|
290
|
+
// TOCTOU, documented on purpose (MINOR #7): between this probe and the
|
|
291
|
+
// caller's `signalGroup` the LAST member may exit and the pgid be recycled
|
|
292
|
+
// by an unrelated group. That window cannot be closed without a pidfd-style
|
|
293
|
+
// handle (one signal per pid, no group target). It is accepted because the
|
|
294
|
+
// alternative — re-introducing the snapshot as identity — made the very
|
|
295
|
+
// rungs above tautological (MAJOR #2).
|
|
296
|
+
return undefined;
|
|
297
|
+
}
|
|
298
|
+
return "identity-missing";
|
|
299
|
+
}
|
|
300
|
+
// Rung 5: same uid (a missing uid only happens for injected snapshots; the
|
|
301
|
+
// live /proc path always carries one — the remaining rungs still hold).
|
|
302
|
+
if (live.uid !== undefined && selfUid !== undefined && live.uid !== selfUid) return "uid-mismatch";
|
|
303
|
+
// Rung 6: PID reuse — the candidate must actually belong to that group.
|
|
304
|
+
if (live.pgid !== pgid) return "not-group-member";
|
|
305
|
+
// Rung 7: starttime is the field that survives PID reuse.
|
|
306
|
+
if (live.starttime === undefined) return "identity-missing";
|
|
307
|
+
// Rung 8: the snapshot and the live process must be the SAME incarnation.
|
|
308
|
+
const expectedStart = opts.expected?.starttime;
|
|
309
|
+
if (expectedStart !== undefined && live.starttime !== expectedStart) return "starttime-mismatch";
|
|
310
|
+
return undefined;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Unref'd sleep so an in-flight reap never pins the event loop. */
|
|
314
|
+
const delay = (ms: number): Promise<void> =>
|
|
315
|
+
new Promise((resolve) => {
|
|
316
|
+
const timer = setTimeout(resolve, Math.max(0, ms));
|
|
317
|
+
timer.unref?.();
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
/** Poll `predicate` until true or the deadline passes; returns the final value. */
|
|
321
|
+
async function pollUntil(predicate: () => boolean, timeoutMs: number): Promise<boolean> {
|
|
322
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
323
|
+
while (Date.now() < deadline) {
|
|
324
|
+
await delay(Math.min(POLL_INTERVAL_MS, Math.max(0, deadline - Date.now())));
|
|
325
|
+
if (predicate()) return true;
|
|
326
|
+
}
|
|
327
|
+
return predicate();
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** Default group liveness: `kill(-pgid, 0)`; EPERM means "alive, not ours". */
|
|
331
|
+
function defaultGroupAlive(pgid: number): boolean {
|
|
332
|
+
if (!Number.isInteger(pgid) || pgid <= 1) return false;
|
|
333
|
+
try {
|
|
334
|
+
process.kill(-pgid, 0);
|
|
335
|
+
return true;
|
|
336
|
+
} catch (err) {
|
|
337
|
+
return (err as NodeJS.ErrnoException).code === "EPERM";
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Default group signal. Defensively refuses non-positive pgids (kill(-1)/kill(0)). */
|
|
342
|
+
function defaultSignalGroup(pgid: number, signal: NodeJS.Signals): void {
|
|
343
|
+
if (!Number.isInteger(pgid) || pgid <= 1) return;
|
|
344
|
+
try {
|
|
345
|
+
process.kill(-pgid, signal);
|
|
346
|
+
} catch {
|
|
347
|
+
/* group already gone — nothing to deliver */
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** Log a refusal/abnormal outcome at debug (never above: refusals are expected). */
|
|
352
|
+
function logDebug(scope: string, message: string, opts: ReapGroupOptions): void {
|
|
353
|
+
logInternalError(scope, new Error(message), `name=${opts.name ?? "-"}`, "debug");
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// ── deferred SIGKILL deliveries (wave 11a review, MAJOR #3) ───────────────────
|
|
357
|
+
//
|
|
358
|
+
// `reapGroup` hands the SIGKILL rung back as a promise, and every production
|
|
359
|
+
// caller is SYNCHRONOUS (the stale-run reconciler, `killProcessTree`): they can
|
|
360
|
+
// at most register the promise with a drain owner. The escalation's own poll
|
|
361
|
+
// timers are unref'd (they must not keep a finished run alive), which means an
|
|
362
|
+
// un-awaited escalation is abandoned the moment the host's loop drains — the
|
|
363
|
+
// SIGKILL is silently never delivered (measured: an unsettled top-level await
|
|
364
|
+
// over unref'd timers exits 13).
|
|
365
|
+
//
|
|
366
|
+
// Two owners close that hole:
|
|
367
|
+
// 1. `drainReapEscalations()` (stale-reconciler.ts) awaits the registered
|
|
368
|
+
// promises while holding a REF'D keep-alive interval.
|
|
369
|
+
// 2. `flushReapEscalationsSync()` (below) is for session shutdown: it
|
|
370
|
+
// re-validates and delivers the SIGKILL rung synchronously, so the last
|
|
371
|
+
// rung still happens in a process that is about to exit.
|
|
372
|
+
/** One armed escalation: everything needed to re-validate and signal it later. */
|
|
373
|
+
interface ArmedGroupEscalation {
|
|
374
|
+
pgid: number;
|
|
375
|
+
candidatePid: number;
|
|
376
|
+
opts: ReapGroupOptions;
|
|
377
|
+
/** Set once {@link flushReapEscalationsSync} delivered the SIGKILL — the async
|
|
378
|
+
* escalation must then never signal a second time. */
|
|
379
|
+
deliveredSync?: boolean;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
const armedEscalations = new Set<ArmedGroupEscalation>();
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Deliver the SIGKILL rung of every ARMED escalation SYNCHRONOUSLY — the
|
|
386
|
+
* session-shutdown path. Each entry is re-validated against the ladder at THIS
|
|
387
|
+
* instant (the group may have drained, or the pgid been recycled since the
|
|
388
|
+
* SIGTERM), so a stale entry never becomes a blind kill. Best-effort per entry:
|
|
389
|
+
* one failure cannot stop the rest. The registry is cleared either way. Returns
|
|
390
|
+
* the number of groups signalled. Never throws.
|
|
391
|
+
*/
|
|
392
|
+
export function flushReapEscalationsSync(): number {
|
|
393
|
+
const entries = [...armedEscalations];
|
|
394
|
+
armedEscalations.clear();
|
|
395
|
+
let delivered = 0;
|
|
396
|
+
for (const entry of entries) {
|
|
397
|
+
try {
|
|
398
|
+
const refusal = assertSafeGroup(entry.pgid, entry.candidatePid, entry.opts);
|
|
399
|
+
if (refusal) {
|
|
400
|
+
logDebug("reap-group.flush-refused", `no deferred SIGKILL for pgid=${entry.pgid} (${refusal})`, entry.opts);
|
|
401
|
+
continue;
|
|
402
|
+
}
|
|
403
|
+
(entry.opts.signalGroup ?? defaultSignalGroup)(entry.pgid, "SIGKILL");
|
|
404
|
+
entry.deliveredSync = true;
|
|
405
|
+
delivered++;
|
|
406
|
+
} catch (err) {
|
|
407
|
+
logInternalError(
|
|
408
|
+
"reap-group.flush-error",
|
|
409
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
410
|
+
`pgid=${entry.pgid} name=${entry.opts.name ?? "-"}`,
|
|
411
|
+
"warn",
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (delivered > 0) logDebug("reap-group.flush", `delivered ${delivered} deferred group SIGKILL(s) at shutdown`, {});
|
|
416
|
+
return delivered;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/** Test seam: how many escalations are still armed (registry must empty on flush). */
|
|
420
|
+
export function __armedEscalationCountForTest(): number {
|
|
421
|
+
return armedEscalations.size;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/** Test seam: drop the armed registry without signalling anything. */
|
|
425
|
+
export function __resetArmedEscalationsForTest(): void {
|
|
426
|
+
armedEscalations.clear();
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Reap a whole process group. See the module header for the safety model.
|
|
431
|
+
* Refused ⇒ nothing is signalled. Accepted ⇒ SIGTERM now, escalation in
|
|
432
|
+
* `pending` (re-validated before SIGKILL).
|
|
433
|
+
*/
|
|
434
|
+
export function reapGroup(pgid: number, candidatePid: number, opts: ReapGroupOptions = {}): GroupReapResult {
|
|
435
|
+
const signalGroup = opts.signalGroup ?? defaultSignalGroup;
|
|
436
|
+
const groupAlive = opts.groupAlive ?? defaultGroupAlive;
|
|
437
|
+
const gracefulMs = opts.gracefulMs ?? DEFAULT_GRACEFUL_MS;
|
|
438
|
+
const capMs = opts.capMs ?? DEFAULT_CAP_MS;
|
|
439
|
+
const refusal = assertSafeGroup(pgid, candidatePid, opts);
|
|
440
|
+
if (refusal) {
|
|
441
|
+
logDebug("reap-group.refused", `refusing to signal pgid=${pgid} (${refusal})`, opts);
|
|
442
|
+
return { outcome: "refused", refusal, pending: Promise.resolve("refused") };
|
|
443
|
+
}
|
|
444
|
+
// Postmortem-safe prefix: the graceful signal is out BEFORE any await, so a
|
|
445
|
+
// host that dies right here still leaves SIGTERM delivered.
|
|
446
|
+
signalGroup(pgid, "SIGTERM");
|
|
447
|
+
if (!groupAlive(pgid)) return { outcome: "gone", pending: Promise.resolve("gone") };
|
|
448
|
+
// MINOR #5 (review): the escalation promise is handed to callers who may
|
|
449
|
+
// `void` it. It must therefore NEVER reject — an unhandled rejection is fatal
|
|
450
|
+
// on Node 22 — so every rung is wrapped and the worst case is a typed
|
|
451
|
+
// "stuck" outcome.
|
|
452
|
+
// MAJOR #3 (review): the escalation is also REGISTERED, so the two owners
|
|
453
|
+
// above can await it (drain) or deliver it synchronously (shutdown flush).
|
|
454
|
+
const armed: ArmedGroupEscalation = { pgid, candidatePid, opts };
|
|
455
|
+
armedEscalations.add(armed);
|
|
456
|
+
const pending = (async (): Promise<GroupReapOutcome> => {
|
|
457
|
+
try {
|
|
458
|
+
if (await pollUntil(() => !groupAlive(pgid), gracefulMs)) return "gone";
|
|
459
|
+
// The shutdown flush may already have delivered SIGKILL synchronously —
|
|
460
|
+
// signalling a second time would be a double kill of a possibly recycled
|
|
461
|
+
// group, so stop here.
|
|
462
|
+
if (armed.deliveredSync === true) return "killed-after-escalation";
|
|
463
|
+
// Re-validate immediately before SIGKILL: between the SIGTERM prefix and
|
|
464
|
+
// here the group may have drained and the pgid been recycled by an
|
|
465
|
+
// unrelated group. A failure here is a refusal, not an escalation.
|
|
466
|
+
const recheck = assertSafeGroup(pgid, candidatePid, opts);
|
|
467
|
+
if (recheck) {
|
|
468
|
+
logDebug("reap-group.sigkill-refused", `escalation refused for pgid=${pgid} (${recheck})`, opts);
|
|
469
|
+
return "refused";
|
|
470
|
+
}
|
|
471
|
+
signalGroup(pgid, "SIGKILL");
|
|
472
|
+
if (await pollUntil(() => !groupAlive(pgid), capMs)) return "killed-after-escalation";
|
|
473
|
+
logInternalError(
|
|
474
|
+
"reap-group.stuck",
|
|
475
|
+
new Error(`process group ${pgid} still alive after SIGKILL cap (${capMs}ms); giving up`),
|
|
476
|
+
`name=${opts.name ?? "-"}`,
|
|
477
|
+
"warn",
|
|
478
|
+
);
|
|
479
|
+
return "stuck";
|
|
480
|
+
} catch (err) {
|
|
481
|
+
logInternalError(
|
|
482
|
+
"reap-group.escalation-error",
|
|
483
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
484
|
+
`pgid=${pgid} name=${opts.name ?? "-"}`,
|
|
485
|
+
"warn",
|
|
486
|
+
);
|
|
487
|
+
return "stuck";
|
|
488
|
+
}
|
|
489
|
+
})();
|
|
490
|
+
// Retire the entry the moment the escalation settles: a healthy escalation
|
|
491
|
+
// leaves nothing behind for the shutdown flush (and an empty registry is what
|
|
492
|
+
// the flush contract promises). Both handlers swallow — this derived promise
|
|
493
|
+
// must never reject, nothing awaits it.
|
|
494
|
+
void pending.then(
|
|
495
|
+
() => armedEscalations.delete(armed),
|
|
496
|
+
() => armedEscalations.delete(armed),
|
|
497
|
+
);
|
|
498
|
+
return { outcome: "killed-after-escalation", pending };
|
|
499
|
+
}
|