pi-crew 0.11.11 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/dist/index.mjs +551 -519
- package/package.json +1 -1
- package/src/config/config-validation.ts +9 -0
- package/src/config/defaults.ts +7 -0
- package/src/config/env-vars.ts +24 -0
- package/src/config/types.ts +11 -0
- package/src/extension/registration/tool-registration.ts +7 -0
- package/src/extension/registration/verify-gate-tool.ts +263 -0
- package/src/extension/team-tool/run-deadline.ts +22 -1
- package/src/extension/team-tool/run.ts +38 -2
- package/src/extension/team-tool-types.ts +4 -0
- package/src/extension/team-tool.ts +31 -3
- package/src/prompt/inbox-poll.ts +141 -3
- package/src/runtime/broker/crew-broker.ts +48 -2
- package/src/runtime/broker/protocol/lock-busy.ts +47 -1
- package/src/runtime/broker/protocol/request-parsers.ts +8 -1
- package/src/runtime/budget-enforcement.ts +36 -20
- package/src/runtime/child-pi/child-pi-constants.ts +24 -0
- package/src/runtime/child-pi/child-pi-kill.ts +59 -2
- package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
- package/src/runtime/child-pi/child-pi-steering.ts +10 -8
- package/src/runtime/child-pi/child-pi-streams.ts +69 -19
- package/src/runtime/child-pi/child-pi.ts +29 -30
- package/src/runtime/child-pi/owned-process.ts +434 -0
- package/src/runtime/child-pi/postmortem-registry.ts +134 -0
- package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
- package/src/runtime/classifier/classifier-service.ts +277 -61
- package/src/runtime/dispatch-batch.ts +32 -0
- package/src/runtime/live-session/live-agent-control.ts +2 -0
- package/src/runtime/live-session/live-agent-manager.ts +72 -2
- package/src/runtime/live-session/live-session-runtime.ts +435 -40
- package/src/runtime/merge-loop.ts +63 -35
- package/src/runtime/recovery/crash-recovery.ts +141 -3
- package/src/runtime/recovery/retry-executor.ts +29 -0
- package/src/runtime/run-tracker.ts +28 -0
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/task-runner.ts +103 -0
- package/src/runtime/verification/classifier-pre-gate.ts +262 -0
- package/src/runtime/verification/verification-gates.ts +5 -1
- package/src/runtime/verification/verify-gate.ts +394 -0
- package/src/schema/config-schema.ts +13 -0
- package/src/state/contracts.ts +18 -0
- package/src/state/coordination/mailbox.ts +203 -55
- package/src/state/stores/sqlite-run-state.ts +378 -0
- package/src/state/stores/state-store.ts +134 -6
- package/src/ui/agents-jobs-browser.ts +154 -8
- package/src/ui/events-state-source.ts +358 -0
- package/src/ui/run-snapshot-cache.ts +87 -3
- package/src/ui/widget/widget-model.ts +13 -2
|
@@ -16,6 +16,15 @@ import * as fs from "node:fs";
|
|
|
16
16
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
17
17
|
import { killProcessTree } from "./child-pi-kill.ts";
|
|
18
18
|
|
|
19
|
+
/** The wrap-up advisory delivered at the soft turn limit. Shared by BOTH
|
|
20
|
+
* transports so the wording stays single-sourced:
|
|
21
|
+
* - json mode: appended to the steering JSONL file the child polls
|
|
22
|
+
* (PI_CREW_STEERING_FILE) inside onTurnEnd below.
|
|
23
|
+
* - rpc mode (U14): sent over the wire via RpcClient.steer() from
|
|
24
|
+
* child-pi-rpc.ts, gated by isStreaming (an idle steer would QUEUE and
|
|
25
|
+
* poison the NEXT run — PoC nuance 1, upgrade-spec 2026-10-09 U14). */
|
|
26
|
+
export const STEER_WRAP_UP_MESSAGE = "You have reached your turn limit. Wrap up immediately — provide your final answer now.";
|
|
27
|
+
|
|
19
28
|
/** Action emitted by the controller when a `turn_end` event is processed. */
|
|
20
29
|
export type SteeringAction =
|
|
21
30
|
| { kind: "steer" }
|
|
@@ -65,14 +74,7 @@ export class ChildPiSteeringController {
|
|
|
65
74
|
// a failed write must NOT kill the worker.
|
|
66
75
|
if (steeringFile) {
|
|
67
76
|
try {
|
|
68
|
-
fs.appendFileSync(
|
|
69
|
-
steeringFile,
|
|
70
|
-
JSON.stringify({
|
|
71
|
-
type: "steer",
|
|
72
|
-
message: "You have reached your turn limit. Wrap up immediately — provide your final answer now.",
|
|
73
|
-
}) + "\n",
|
|
74
|
-
"utf-8",
|
|
75
|
-
);
|
|
77
|
+
fs.appendFileSync(steeringFile, JSON.stringify({ type: "steer", message: STEER_WRAP_UP_MESSAGE }) + "\n", "utf-8");
|
|
76
78
|
} catch (err) {
|
|
77
79
|
logInternalError("child-pi.steer-write-failed", err instanceof Error ? err : new Error(String(err)), `pid=${pid}`);
|
|
78
80
|
}
|
|
@@ -130,6 +130,28 @@ export function compactChildPiEvent(event: unknown): unknown | undefined {
|
|
|
130
130
|
return record.type ? { type: record.type } : undefined;
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
/**
|
|
134
|
+
* U14 (child-pi protocol v2): `true` when the event is the FINAL assistant
|
|
135
|
+
* event of a run — an assistant `message_end` with stopReason "stop" and no
|
|
136
|
+
* pending toolCall part. Shared by BOTH transports (the json stdout funnel in
|
|
137
|
+
* child-pi.ts and the rpc event subscription in child-pi-rpc.ts) so the
|
|
138
|
+
* final-drain arming condition cannot drift between modes.
|
|
139
|
+
*
|
|
140
|
+
* Extracted from child-pi.ts (U14) — pure motion, zero behavior change.
|
|
141
|
+
*/
|
|
142
|
+
export function isFinalAssistantEvent(event: unknown): boolean {
|
|
143
|
+
const obj = asRecord(event);
|
|
144
|
+
if (obj?.type !== "message_end") return false;
|
|
145
|
+
const message = asRecord(obj.message);
|
|
146
|
+
const role = message?.role;
|
|
147
|
+
if (role !== undefined && role !== "assistant") return false;
|
|
148
|
+
const stopReason =
|
|
149
|
+
typeof message?.stopReason === "string" ? message.stopReason : typeof obj.stopReason === "string" ? obj.stopReason : undefined;
|
|
150
|
+
if (stopReason !== undefined && stopReason !== "stop") return false;
|
|
151
|
+
const content = Array.isArray(message?.content) ? message.content : [];
|
|
152
|
+
return !content.some((part) => asRecord(part)?.type === "toolCall");
|
|
153
|
+
}
|
|
154
|
+
|
|
133
155
|
function displayTextFromCompactEvent(event: unknown): string | undefined {
|
|
134
156
|
const record = asRecord(event);
|
|
135
157
|
if (!record) return undefined;
|
|
@@ -297,27 +319,55 @@ export class ChildPiLineObserver {
|
|
|
297
319
|
parsed = undefined;
|
|
298
320
|
}
|
|
299
321
|
if (parsed !== undefined) {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
322
|
+
this.processParsedEvent(parsed);
|
|
323
|
+
} else {
|
|
324
|
+
// OPT-PHASE2: construct the non-JSON fallback directly when parsing failed,
|
|
325
|
+
// so a broken line triggers exactly ONE (failed) parse instead of two.
|
|
326
|
+
this.dispatchCompact(nonJsonLineResult(line));
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* U14 (child-pi protocol v2): feed an ALREADY-PARSED agent event through
|
|
332
|
+
* the exact same pipeline as a parsed stdout JSON line — raw assistant-text
|
|
333
|
+
* extraction, compaction, onJsonEvent dispatch, transcript append, and
|
|
334
|
+
* display-line emission. The rpc transport (child-pi-rpc.ts) receives
|
|
335
|
+
* parsed event objects from RpcClient.onEvent (the RpcClient owns the
|
|
336
|
+
* stdout JSONL wire), so this is its entry into the observer. Non-object
|
|
337
|
+
* values are dropped (parity with the non-JSON text fallback, which never
|
|
338
|
+
* reaches this path).
|
|
339
|
+
*/
|
|
340
|
+
observeEvent(event: unknown): void {
|
|
341
|
+
if (event === null || typeof event !== "object") return;
|
|
342
|
+
this.processParsedEvent(event);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** Shared post-parse pipeline for BOTH transports (text lines + rpc events). */
|
|
346
|
+
private processParsedEvent(parsed: unknown): void {
|
|
347
|
+
const rawTexts = extractText(parsed);
|
|
348
|
+
if (rawTexts.length > 0) {
|
|
349
|
+
// F9: trim from the front if the push would exceed the cap. Slice's
|
|
350
|
+
// second arg excludes the index, so this drops the oldest entries
|
|
351
|
+
// while keeping the freshly pushed tail.
|
|
352
|
+
this.rawTextEvents.push(...rawTexts);
|
|
353
|
+
const rawOverflow = this.rawTextEvents.length - ChildPiLineObserver.MAX_RAW_TEXT_EVENTS;
|
|
354
|
+
if (rawOverflow > 0) this.rawTextEvents.splice(0, rawOverflow);
|
|
355
|
+
// Also capture raw assistant text as intermediate findings — the last raw
|
|
356
|
+
// text may be a partial answer before the worker ran out of budget.
|
|
357
|
+
const last = rawTexts[rawTexts.length - 1];
|
|
358
|
+
if (last.trim().length > 0) {
|
|
359
|
+
this.intermediateFindings.push(last.trim());
|
|
360
|
+
const findingsOverflow = this.intermediateFindings.length - ChildPiLineObserver.MAX_INTERMEDIATE_FINDINGS;
|
|
361
|
+
if (findingsOverflow > 0) this.intermediateFindings.splice(0, findingsOverflow);
|
|
316
362
|
}
|
|
317
363
|
}
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
|
|
364
|
+
// (preParsed is always defined here — the line-based JSON.parse happened
|
|
365
|
+
// in emitLine; the rpc path never had a line at all.)
|
|
366
|
+
this.dispatchCompact(compactChildPiLine("", parsed));
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/** Dispatch a compacted line/event to the consumer callbacks + transcript. */
|
|
370
|
+
private dispatchCompact(compact: { persistedLine: string; event?: unknown; displayLine?: string; json: boolean }): void {
|
|
321
371
|
if (compact.event !== undefined) {
|
|
322
372
|
try {
|
|
323
373
|
this.input.onJsonEvent?.(compact.event);
|
|
@@ -3,7 +3,6 @@ import * as fs from "node:fs";
|
|
|
3
3
|
import * as path from "node:path";
|
|
4
4
|
import type { AgentConfig } from "../../agents/agent-config.ts";
|
|
5
5
|
import { loadConfig } from "../../config/config.ts";
|
|
6
|
-
import { getCrewEnv } from "../../config/env-vars.ts";
|
|
7
6
|
import type { PiTeamsConfig } from "../../config/types.ts";
|
|
8
7
|
import { registerChildProcess, unregisterChildProcess } from "../../extension/crew-cleanup.ts";
|
|
9
8
|
import type { WorkerExitStatus } from "../../state/types.ts";
|
|
@@ -18,12 +17,23 @@ import { classifyOnExit, getSurfaceRuntimeController, makeTerminalEventProbe } f
|
|
|
18
17
|
import type { SurfaceGateEnvSnapshot } from "../surface/resolve-surface.ts";
|
|
19
18
|
import type { SurfaceProvider } from "../surface/surface-provider.ts";
|
|
20
19
|
import { prepareSurfaceSpawn, type SurfaceExitInfo, type SurfaceSpawnOutcome, waitForSurfaceExit } from "../surface/surface-spawn.ts";
|
|
21
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
FINAL_DRAIN_MS,
|
|
22
|
+
HARD_KILL_MS,
|
|
23
|
+
POST_EXIT_STDIO_GUARD_MS,
|
|
24
|
+
RESPONSE_TIMEOUT_MS,
|
|
25
|
+
resolveResponseTimeoutMs,
|
|
26
|
+
} from "./child-pi-constants.ts";
|
|
22
27
|
import { clearHardKillTimer, killProcessTree, registerActiveChild, unregisterActiveChild } from "./child-pi-kill.ts";
|
|
28
|
+
// U14 (child-pi protocol v2): the rpc transport branch — opt-in via
|
|
29
|
+
// PI_CREW_CHILD_PI_MODE=rpc; json stays the default transport (phase 1).
|
|
30
|
+
import { resolveChildPiRpcMode, tryChildPiRpcRun } from "./child-pi-rpc.ts";
|
|
23
31
|
import { buildFinalChildPiSpawnOptions, prepareSpawnContext } from "./child-pi-spawn.ts";
|
|
24
32
|
import { ChildPiSteeringController } from "./child-pi-steering.ts";
|
|
25
33
|
// Internal helpers for active-child bookkeeping (extracted to child-pi-kill.ts).
|
|
26
|
-
|
|
34
|
+
// U14: isFinalAssistantEvent moved to child-pi-streams.ts (shared by BOTH
|
|
35
|
+
// transports) and is re-exported below to keep the module surface stable.
|
|
36
|
+
import { ChildPiLineObserver, isFinalAssistantEvent } from "./child-pi-streams.ts";
|
|
27
37
|
// Phase 2.3: the six timer constructs moved to child-pi-timers.ts (pure motion).
|
|
28
38
|
import { createChildPiTimers } from "./child-pi-timers.ts";
|
|
29
39
|
import { runMockChildPi } from "./mock-fixtures.ts";
|
|
@@ -51,7 +61,7 @@ export {
|
|
|
51
61
|
// owns the canary + filter + spread sequence; lives in child-pi-spawn.ts.
|
|
52
62
|
export { buildChildPiSpawnOptions, buildFinalChildPiSpawnOptions } from "./child-pi-spawn.ts";
|
|
53
63
|
// ── Re-export from child-pi-streams.ts (H-7 decomposition step 4) ──
|
|
54
|
-
export { ChildPiLineObserver } from "./child-pi-streams.ts";
|
|
64
|
+
export { ChildPiLineObserver, isFinalAssistantEvent } from "./child-pi-streams.ts";
|
|
55
65
|
|
|
56
66
|
import { checkCrewDepth, cleanupTempDir, resolveHermeticWorkers } from "../model/pi-args.ts";
|
|
57
67
|
import { attachPostExitStdioGuard, trySignalChild } from "../process/post-exit-stdio-guard.ts";
|
|
@@ -399,19 +409,6 @@ function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
|
399
409
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : undefined;
|
|
400
410
|
}
|
|
401
411
|
|
|
402
|
-
function isFinalAssistantEvent(event: unknown): boolean {
|
|
403
|
-
const obj = asRecord(event);
|
|
404
|
-
if (obj?.type !== "message_end") return false;
|
|
405
|
-
const message = asRecord(obj.message);
|
|
406
|
-
const role = message?.role;
|
|
407
|
-
if (role !== undefined && role !== "assistant") return false;
|
|
408
|
-
const stopReason =
|
|
409
|
-
typeof message?.stopReason === "string" ? message.stopReason : typeof obj.stopReason === "string" ? obj.stopReason : undefined;
|
|
410
|
-
if (stopReason !== undefined && stopReason !== "stop") return false;
|
|
411
|
-
const content = Array.isArray(message?.content) ? message.content : [];
|
|
412
|
-
return !content.some((part) => asRecord(part)?.type === "toolCall");
|
|
413
|
-
}
|
|
414
|
-
|
|
415
412
|
/**
|
|
416
413
|
* Discover the project crew config for the surface gate. The discovered config
|
|
417
414
|
* DEFAULTS surface off (`visibleAgents: []`) — headless unless explicitly opted
|
|
@@ -789,6 +786,17 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
789
786
|
// awaited) or null → fall through to the classic headless spawn below.
|
|
790
787
|
const surfaceResult = await trySurfaceBranch(input, depthEnv, builtArgs, mergedEnv, builtEnv, tempDir);
|
|
791
788
|
if (surfaceResult) return surfaceResult;
|
|
789
|
+
// U14 (child-pi protocol v2 — upgrade-spec 2026-10-09): `--mode rpc`
|
|
790
|
+
// transport, opt-in via PI_CREW_CHILD_PI_MODE=rpc (json remains the
|
|
791
|
+
// phase-1 default — the flag is the BOTH-directions regression lever).
|
|
792
|
+
// Returns null when rpc cannot run here (no absolute cli script) so the
|
|
793
|
+
// json branch below stays the fail-safe path. Surface TUI panes are
|
|
794
|
+
// unaffected — trySurfaceBranch consumed builtArgs above and never
|
|
795
|
+
// reaches this point in surface mode.
|
|
796
|
+
if (resolveChildPiRpcMode()) {
|
|
797
|
+
const rpcResult = await tryChildPiRpcRun({ input, effectiveTask, ctx: spawnPrep.ctx, workerSession, sessionRecoveryEnabled });
|
|
798
|
+
if (rpcResult) return rpcResult;
|
|
799
|
+
}
|
|
792
800
|
try {
|
|
793
801
|
return await new Promise<ChildPiRunResult>((resolve) => {
|
|
794
802
|
// Compose the final SpawnOptions: canary + filter + spread are now
|
|
@@ -841,19 +849,10 @@ export async function runChildPi(input: ChildPiRunInput): Promise<ChildPiRunResu
|
|
|
841
849
|
let finalDrainFiredMonotonicMs: number | undefined;
|
|
842
850
|
const spawnMonotonicMs = performance.now();
|
|
843
851
|
let finalAssistantEventMonotonicMs: number | undefined;
|
|
844
|
-
// FIX (Round 14)
|
|
845
|
-
//
|
|
846
|
-
//
|
|
847
|
-
|
|
848
|
-
// built-in default.
|
|
849
|
-
const RESPONSE_TIMEOUT_MIN_MS = 1_000;
|
|
850
|
-
const RESPONSE_TIMEOUT_MAX_MS = 3_600_000;
|
|
851
|
-
const responseTimeoutEnv = Number.parseInt(getCrewEnv("PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS") ?? "", 10);
|
|
852
|
-
const envInRange =
|
|
853
|
-
Number.isFinite(responseTimeoutEnv) &&
|
|
854
|
-
responseTimeoutEnv >= RESPONSE_TIMEOUT_MIN_MS &&
|
|
855
|
-
responseTimeoutEnv <= RESPONSE_TIMEOUT_MAX_MS;
|
|
856
|
-
const responseTimeoutMs = envInRange ? responseTimeoutEnv : (input.responseTimeoutMs ?? RESPONSE_TIMEOUT_MS);
|
|
852
|
+
// FIX (Round 14) — the env-override bounds live in
|
|
853
|
+
// resolveResponseTimeoutMs (child-pi-constants.ts, U14: shared with the
|
|
854
|
+
// rpc transport so watchdog budgets cannot drift between modes).
|
|
855
|
+
const responseTimeoutMs = resolveResponseTimeoutMs(input);
|
|
857
856
|
let responseTimeoutHit = false;
|
|
858
857
|
let forcedFinalDrain = false;
|
|
859
858
|
let abortRequested = input.signal?.aborted === true;
|
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* owned-process.ts — U15: OwnedProcess pattern, Node 22 port.
|
|
3
|
+
*
|
|
4
|
+
* Ported from gajae-code `packages/coding-agent/src/runtime/process-lifecycle.ts`
|
|
5
|
+
* (fork pi-mono v0.8.2) per spec U15 of pi-crew-upgrade-spec-2026-10-09.md:
|
|
6
|
+
* the Bun `ptree.spawn` core is re-expressed with `node:child_process.spawn`
|
|
7
|
+
* `{detached:true}` — pi-crew's child-pi spawns already use detached+setsid
|
|
8
|
+
* (child-pi-spawn.ts), they were only missing the group-kill discipline:
|
|
9
|
+
*
|
|
10
|
+
* - Process-group ownership: a detached child is its own group leader, so
|
|
11
|
+
* `process.kill(-pgid, sig)` signals the WHOLE descendant tree — including
|
|
12
|
+
* `sh -c "worker &"` backgrounded descendants, pi-crew's real zombie
|
|
13
|
+
* vector (proof-of-pattern round 9: root SIGKILL left 2 children
|
|
14
|
+
* reparented to ppid=1 while the group stayed signalable).
|
|
15
|
+
* - Escalating dispose with hard cap: SIGTERM group → poll gracefulMs (2s)
|
|
16
|
+
* → SIGKILL → poll cap (2s) → warn + give up. A wedged child can never
|
|
17
|
+
* block shutdown forever.
|
|
18
|
+
* - Root-exit drain-reconcile (250ms): when the root exits on its own but
|
|
19
|
+
* the group still has live members, the owner reaps the group — no child
|
|
20
|
+
* outlives its owner.
|
|
21
|
+
* - PID-recycle guard: once terminal, dispose() is a settled no-op and
|
|
22
|
+
* never re-probes a pgid the OS may have recycled into an unrelated group.
|
|
23
|
+
* - Idempotent/concurrent dispose share ONE in-flight promise; bounded
|
|
24
|
+
* `awaitExit` never rejects.
|
|
25
|
+
* - Postmortem registry (postmortem-registry.ts) reaps every still-live
|
|
26
|
+
* owned group on exit/SIGINT/SIGTERM/SIGHUP/uncaught/unhandled; owners
|
|
27
|
+
* deregister only AFTER teardown completed.
|
|
28
|
+
*
|
|
29
|
+
* GOTCHA (round 9, MUST keep): do NOT `unref()` the root child handle. The
|
|
30
|
+
* root-exit drain-reconcile needs the 'exit' event delivered; unref'ing the
|
|
31
|
+
* handle starves exit-event delivery when the rest of the loop is idle and
|
|
32
|
+
* the reconcile silently never runs. Internal poll/delay timers unref
|
|
33
|
+
* THEMSELVES (they must not pin the loop), but the child handle stays ref'd.
|
|
34
|
+
*
|
|
35
|
+
* Known limit (spec, grep-verified in the fork): no parent-death detection —
|
|
36
|
+
* if THIS owner process is SIGKILLed, no postmortem runs. That case is owned
|
|
37
|
+
* by the complementary child-side PI_CREW_PARENT_PID guard (zombie-scanner +
|
|
38
|
+
* background-runner parent-guard), which stays in place untouched.
|
|
39
|
+
*
|
|
40
|
+
* rpc-socket-security.ts is explicitly NOT ported (spec: defer until an UDS
|
|
41
|
+
* transport exists — see U14/U15 lane notes).
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import { type ChildProcess, spawn } from "node:child_process";
|
|
45
|
+
import * as fs from "node:fs";
|
|
46
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
47
|
+
import { registerPostmortemCleanup } from "./postmortem-registry.ts";
|
|
48
|
+
|
|
49
|
+
const DEFAULT_GRACEFUL_MS = 2_000;
|
|
50
|
+
// Hard cap for how long dispose() waits after SIGKILL before giving up so a
|
|
51
|
+
// wedged, unkillable child can never block shutdown forever.
|
|
52
|
+
const SIGKILL_REAP_CAP_MS = 2_000;
|
|
53
|
+
// After the root process exits on its own, how long to wait for the process
|
|
54
|
+
// group to drain before deregistering. Clean children drain immediately; a
|
|
55
|
+
// root that backgrounded descendants keeps the owner registered past this
|
|
56
|
+
// window and the group is then reaped by dispose().
|
|
57
|
+
const ROOT_EXIT_DRAIN_MS = 250;
|
|
58
|
+
const POLL_INTERVAL_MS = 20;
|
|
59
|
+
|
|
60
|
+
const isPosix = process.platform !== "win32";
|
|
61
|
+
|
|
62
|
+
const delay = (ms: number): Promise<void> =>
|
|
63
|
+
new Promise((resolve) => {
|
|
64
|
+
const timer = setTimeout(resolve, Math.max(0, ms));
|
|
65
|
+
timer.unref?.();
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
/** Poll `predicate` until true or `timeoutMs` elapses. Returns the final value. */
|
|
69
|
+
async function pollUntil(predicate: () => boolean, timeoutMs: number, intervalMs = POLL_INTERVAL_MS): Promise<boolean> {
|
|
70
|
+
if (predicate()) return true;
|
|
71
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
72
|
+
while (Date.now() < deadline) {
|
|
73
|
+
await delay(Math.min(intervalMs, Math.max(0, deadline - Date.now())));
|
|
74
|
+
if (predicate()) return true;
|
|
75
|
+
}
|
|
76
|
+
return predicate();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Whether a POSIX process group still has any member (zombies count as alive). */
|
|
80
|
+
function groupAlive(pgid: number): boolean {
|
|
81
|
+
try {
|
|
82
|
+
process.kill(-pgid, 0);
|
|
83
|
+
return true;
|
|
84
|
+
} catch (err) {
|
|
85
|
+
// EPERM => the group exists but we cannot signal it; treat as alive.
|
|
86
|
+
return (err as NodeJS.ErrnoException).code === "EPERM";
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Read the process-group id of `pid` from /proc (Linux). Returns undefined
|
|
92
|
+
* when /proc is unavailable (non-Linux) or the stat line is unparseable.
|
|
93
|
+
* Authoritative group-leadership check on Linux.
|
|
94
|
+
*/
|
|
95
|
+
function readPgidFromProc(pid: number): number | undefined {
|
|
96
|
+
try {
|
|
97
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
98
|
+
// Layout: pid (comm) state ppid pgrp session ... — comm may contain
|
|
99
|
+
// spaces and parens, so fields are counted AFTER the last ')'.
|
|
100
|
+
const close = stat.lastIndexOf(")");
|
|
101
|
+
if (close === -1) return undefined;
|
|
102
|
+
const fields = stat
|
|
103
|
+
.slice(close + 2)
|
|
104
|
+
.trim()
|
|
105
|
+
.split(/\s+/);
|
|
106
|
+
const pgrp = Number(fields[2]);
|
|
107
|
+
return Number.isInteger(pgrp) && pgrp > 0 ? pgrp : undefined;
|
|
108
|
+
} catch {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Options shared by {@link spawnOwnedProcess} and {@link adoptOwnedProcess}. */
|
|
114
|
+
export interface OwnedProcessOptions {
|
|
115
|
+
/** Grace period (ms) between SIGTERM and SIGKILL on dispose. Default 2000. */
|
|
116
|
+
gracefulMs?: number;
|
|
117
|
+
/** Label used in diagnostics. */
|
|
118
|
+
name?: string;
|
|
119
|
+
/**
|
|
120
|
+
* When aborted, the owned process tree is disposed (escalating kill).
|
|
121
|
+
* The listener is removed on settle so long-lived signals never leak.
|
|
122
|
+
*/
|
|
123
|
+
signal?: AbortSignal;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export interface SpawnOwnedOptions extends OwnedProcessOptions {
|
|
127
|
+
cwd?: string;
|
|
128
|
+
env?: NodeJS.ProcessEnv;
|
|
129
|
+
/**
|
|
130
|
+
* Spawn the child as its own process-group leader so the whole descendant
|
|
131
|
+
* tree can be signalled on dispose. Defaults to `true` on POSIX. Has no
|
|
132
|
+
* effect on Windows, where teardown falls back to single-process kill.
|
|
133
|
+
*/
|
|
134
|
+
processGroup?: boolean;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Result of a bounded {@link OwnedProcess.awaitExit}. */
|
|
138
|
+
export interface AwaitExitResult {
|
|
139
|
+
/** `true` when the process has exited; `false` when the timeout fired first. */
|
|
140
|
+
exited: boolean;
|
|
141
|
+
/** Exit code if known, else `null`. */
|
|
142
|
+
code: number | null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** A child process owned by the runtime with guaranteed group teardown. */
|
|
146
|
+
export interface OwnedProcess {
|
|
147
|
+
readonly child: ChildProcess;
|
|
148
|
+
readonly pid: number | undefined;
|
|
149
|
+
/** Process-group id when the child leads its own group, else undefined. */
|
|
150
|
+
readonly pgid: number | undefined;
|
|
151
|
+
/** Resolves with the exit code when the root child exits. Never rejects. */
|
|
152
|
+
readonly exited: Promise<number | null>;
|
|
153
|
+
/** `true` once dispose() has started. */
|
|
154
|
+
readonly disposed: boolean;
|
|
155
|
+
/**
|
|
156
|
+
* `true` once teardown/reconciliation confirmed the tree is gone. A late
|
|
157
|
+
* dispose() is then a settled no-op (PID-recycle guard).
|
|
158
|
+
*/
|
|
159
|
+
readonly terminated: boolean;
|
|
160
|
+
/**
|
|
161
|
+
* Wait for the root child to exit, optionally bounded by `timeoutMs`.
|
|
162
|
+
* Never rejects.
|
|
163
|
+
*/
|
|
164
|
+
awaitExit(opts?: { timeoutMs?: number }): Promise<AwaitExitResult>;
|
|
165
|
+
/**
|
|
166
|
+
* Idempotently terminate the owned process *group*: SIGTERM the group,
|
|
167
|
+
* wait `gracefulMs`, then SIGKILL, polling group liveness throughout.
|
|
168
|
+
* Removes the abort listener and deregisters from the live-owner set only
|
|
169
|
+
* after teardown has completed. Repeated/concurrent calls return the same
|
|
170
|
+
* in-flight promise.
|
|
171
|
+
*/
|
|
172
|
+
dispose(): Promise<void>;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const liveOwners = new Set<OwnedProcess>();
|
|
176
|
+
let ownedPostmortemRegistered = false;
|
|
177
|
+
|
|
178
|
+
function ensureOwnedPostmortem(): void {
|
|
179
|
+
if (ownedPostmortemRegistered) return;
|
|
180
|
+
ownedPostmortemRegistered = true;
|
|
181
|
+
registerPostmortemCleanup("child-pi:owned-processes", () => disposeAllOwnedProcesses());
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Create the OwnedProcess wrapper around a spawned (or adoptable) child. */
|
|
185
|
+
function createOwnedProcess(child: ChildProcess, pgid: number | undefined, opts: OwnedProcessOptions): OwnedProcess {
|
|
186
|
+
const gracefulMs = opts.gracefulMs ?? DEFAULT_GRACEFUL_MS;
|
|
187
|
+
|
|
188
|
+
ensureOwnedPostmortem();
|
|
189
|
+
|
|
190
|
+
// exited: resolves exactly once with the exit code (null on spawn error or
|
|
191
|
+
// signal death without a code). Never rejects — reconcile relies on that.
|
|
192
|
+
let exitSettled = false;
|
|
193
|
+
let resolveExit!: (code: number | null) => void;
|
|
194
|
+
const exited = new Promise<number | null>((resolve) => {
|
|
195
|
+
resolveExit = resolve;
|
|
196
|
+
});
|
|
197
|
+
const settleExit = (code: number | null): void => {
|
|
198
|
+
if (exitSettled) return;
|
|
199
|
+
exitSettled = true;
|
|
200
|
+
resolveExit(code);
|
|
201
|
+
};
|
|
202
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
203
|
+
// Already exited before adoption/creation (signal death leaves exitCode
|
|
204
|
+
// null with signalCode set) — settle immediately, the events are gone.
|
|
205
|
+
settleExit(child.exitCode);
|
|
206
|
+
} else {
|
|
207
|
+
child.once("exit", (code) => settleExit(code));
|
|
208
|
+
// Spawn failures ('error', pid undefined) must not leave a hanging
|
|
209
|
+
// promise — resolve null so drain-reconcile can deregister.
|
|
210
|
+
child.once("error", () => settleExit(null));
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
let disposed = false;
|
|
214
|
+
let disposePromise: Promise<void> | undefined;
|
|
215
|
+
let deregistered = false;
|
|
216
|
+
// Terminal once teardown/reconciliation has confirmed the group is gone. A
|
|
217
|
+
// late dispose() must then be a true no-op and never re-probe a pgid the OS
|
|
218
|
+
// may have recycled into an unrelated group.
|
|
219
|
+
let terminated = false;
|
|
220
|
+
let onAbort: (() => void) | undefined;
|
|
221
|
+
|
|
222
|
+
const removeAbort = (): void => {
|
|
223
|
+
if (onAbort && opts.signal) {
|
|
224
|
+
opts.signal.removeEventListener("abort", onAbort);
|
|
225
|
+
onAbort = undefined;
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
const deregister = (): void => {
|
|
230
|
+
if (deregistered) return;
|
|
231
|
+
deregistered = true;
|
|
232
|
+
terminated = true;
|
|
233
|
+
liveOwners.delete(owner);
|
|
234
|
+
removeAbort();
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
const signalTree = (signal: NodeJS.Signals): void => {
|
|
238
|
+
const pid = child.pid;
|
|
239
|
+
if (pid === undefined) return;
|
|
240
|
+
if (pgid !== undefined) {
|
|
241
|
+
try {
|
|
242
|
+
// Negative pid signals the entire process group (child is leader).
|
|
243
|
+
process.kill(-pgid, signal);
|
|
244
|
+
} catch {
|
|
245
|
+
// Group already gone; nothing to do.
|
|
246
|
+
}
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
// Single-process fallback (Windows / non-group child, e.g. the U14 rpc
|
|
250
|
+
// branch which spawns without detached). child.kill() is best-effort;
|
|
251
|
+
// fall through to process.kill() when it reports no delivery.
|
|
252
|
+
if (signal === "SIGKILL" || !child.kill(signal)) {
|
|
253
|
+
try {
|
|
254
|
+
process.kill(pid, signal);
|
|
255
|
+
} catch {
|
|
256
|
+
/* already gone */
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
const owner: OwnedProcess = {
|
|
262
|
+
child,
|
|
263
|
+
get pid() {
|
|
264
|
+
return child.pid;
|
|
265
|
+
},
|
|
266
|
+
get pgid() {
|
|
267
|
+
return pgid;
|
|
268
|
+
},
|
|
269
|
+
get exited() {
|
|
270
|
+
return exited;
|
|
271
|
+
},
|
|
272
|
+
get disposed() {
|
|
273
|
+
return disposed;
|
|
274
|
+
},
|
|
275
|
+
get terminated() {
|
|
276
|
+
return terminated;
|
|
277
|
+
},
|
|
278
|
+
async awaitExit({ timeoutMs }: { timeoutMs?: number } = {}): Promise<AwaitExitResult> {
|
|
279
|
+
const exitedResult = exited.then((code) => ({ exited: true as const, code }));
|
|
280
|
+
if (timeoutMs === undefined) return exitedResult;
|
|
281
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
282
|
+
const timeout = new Promise<AwaitExitResult>((resolve) => {
|
|
283
|
+
timer = setTimeout(() => resolve({ exited: false, code: child.exitCode }), Math.max(0, timeoutMs));
|
|
284
|
+
timer.unref?.();
|
|
285
|
+
});
|
|
286
|
+
try {
|
|
287
|
+
return await Promise.race([exitedResult, timeout]);
|
|
288
|
+
} finally {
|
|
289
|
+
if (timer) clearTimeout(timer);
|
|
290
|
+
}
|
|
291
|
+
},
|
|
292
|
+
dispose(): Promise<void> {
|
|
293
|
+
// Already terminal (e.g. clean drain reconciled and deregistered):
|
|
294
|
+
// never re-probe the pgid; treat dispose as a settled no-op.
|
|
295
|
+
if (terminated) {
|
|
296
|
+
disposed = true;
|
|
297
|
+
if (!disposePromise) disposePromise = Promise.resolve();
|
|
298
|
+
return disposePromise;
|
|
299
|
+
}
|
|
300
|
+
if (disposePromise) return disposePromise;
|
|
301
|
+
disposed = true;
|
|
302
|
+
removeAbort();
|
|
303
|
+
disposePromise = (async () => {
|
|
304
|
+
try {
|
|
305
|
+
if (pgid !== undefined) {
|
|
306
|
+
// Group ownership: reap until the whole group is gone, even if
|
|
307
|
+
// the root has already exited (it may have backgrounded children).
|
|
308
|
+
if (!groupAlive(pgid)) return;
|
|
309
|
+
signalTree("SIGTERM");
|
|
310
|
+
if (await pollUntil(() => !groupAlive(pgid), gracefulMs)) return;
|
|
311
|
+
signalTree("SIGKILL");
|
|
312
|
+
if (!(await pollUntil(() => !groupAlive(pgid), SIGKILL_REAP_CAP_MS))) {
|
|
313
|
+
logInternalError(
|
|
314
|
+
"child-pi.owned-sigkill-cap",
|
|
315
|
+
new Error(`owned process group still alive after SIGKILL cap (${SIGKILL_REAP_CAP_MS}ms); giving up`),
|
|
316
|
+
`name=${opts.name} pgid=${pgid}`,
|
|
317
|
+
"warn",
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
// Single-process fallback (Windows / processGroup:false / rpc child).
|
|
323
|
+
if (child.exitCode !== null) return;
|
|
324
|
+
signalTree("SIGTERM");
|
|
325
|
+
if ((await owner.awaitExit({ timeoutMs: gracefulMs })).exited) return;
|
|
326
|
+
signalTree("SIGKILL");
|
|
327
|
+
await owner.awaitExit({ timeoutMs: SIGKILL_REAP_CAP_MS });
|
|
328
|
+
} catch (err) {
|
|
329
|
+
logInternalError(
|
|
330
|
+
"child-pi.owned-dispose-failed",
|
|
331
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
332
|
+
`name=${opts.name}`,
|
|
333
|
+
);
|
|
334
|
+
} finally {
|
|
335
|
+
// Deregister only AFTER teardown completed so a postmortem firing
|
|
336
|
+
// mid-grace still sees the owner and awaits this in-flight dispose.
|
|
337
|
+
deregister();
|
|
338
|
+
}
|
|
339
|
+
})();
|
|
340
|
+
return disposePromise;
|
|
341
|
+
},
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
liveOwners.add(owner);
|
|
345
|
+
|
|
346
|
+
// When the root exits on its own (not via dispose), reconcile ownership by
|
|
347
|
+
// the *group*. After a short drain window: if the group is empty, deregister;
|
|
348
|
+
// if descendants are still alive, reap the owned group (no child outlives its
|
|
349
|
+
// owner). Either way the owner never lingers holding a stale pgid that the OS
|
|
350
|
+
// could later recycle and a stray dispose could mis-signal.
|
|
351
|
+
void exited.finally(() => {
|
|
352
|
+
if (disposed) return; // dispose() owns deregistration
|
|
353
|
+
if (pgid === undefined) {
|
|
354
|
+
deregister();
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
void (async () => {
|
|
358
|
+
const drained = await pollUntil(() => !groupAlive(pgid), ROOT_EXIT_DRAIN_MS);
|
|
359
|
+
if (disposed) return;
|
|
360
|
+
if (drained) {
|
|
361
|
+
deregister();
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
// Root exited but the owned group still has descendants: reap them.
|
|
365
|
+
// dispose() escalates SIGTERM→SIGKILL and deregisters in its finally.
|
|
366
|
+
await owner.dispose();
|
|
367
|
+
})();
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
if (opts.signal) {
|
|
371
|
+
if (opts.signal.aborted) {
|
|
372
|
+
void owner.dispose();
|
|
373
|
+
} else {
|
|
374
|
+
onAbort = () => void owner.dispose();
|
|
375
|
+
opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
return owner;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Spawn a child process owned by the runtime. The returned {@link OwnedProcess}
|
|
384
|
+
* is registered for postmortem cleanup and tears down its whole process group
|
|
385
|
+
* on dispose/abort. POSIX: spawned `detached:true` so the child is its own
|
|
386
|
+
* session+group leader (pgid === pid).
|
|
387
|
+
*/
|
|
388
|
+
export function spawnOwnedProcess(command: string, args: string[], opts: SpawnOwnedOptions = {}): OwnedProcess {
|
|
389
|
+
const useGroup = (opts.processGroup ?? true) && isPosix;
|
|
390
|
+
const child = spawn(command, args, {
|
|
391
|
+
cwd: opts.cwd,
|
|
392
|
+
env: opts.env,
|
|
393
|
+
stdio: "ignore",
|
|
394
|
+
detached: useGroup,
|
|
395
|
+
});
|
|
396
|
+
// Round-9 gotcha: do NOT child.unref() — the root handle must stay ref'd
|
|
397
|
+
// or the exit event starves when the rest of the loop is idle, killing the
|
|
398
|
+
// drain-reconcile above. Callers that want fire-and-forget semantics get
|
|
399
|
+
// them from the postmortem registry instead.
|
|
400
|
+
return createOwnedProcess(child, useGroup ? child.pid : undefined, opts);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Adopt an ALREADY-spawned child into the owned-process registry. Group
|
|
405
|
+
* leadership is resolved conservatively:
|
|
406
|
+
* - Linux: read the child's real pgrp from /proc — authoritative (covers
|
|
407
|
+
* the U14 rpc branch, whose RpcClient spawns WITHOUT detached: pgrp ≠ pid
|
|
408
|
+
* ⇒ single-process fallback, exactly the deferred behavior noted there).
|
|
409
|
+
* - Other POSIX: a live group with id === child.pid implies the child leads
|
|
410
|
+
* it (same pid-recycle exposure as the legacy killProcessPid group path).
|
|
411
|
+
* - Windows: no process groups ⇒ single-process fallback.
|
|
412
|
+
*/
|
|
413
|
+
export function adoptOwnedProcess(child: ChildProcess, opts: OwnedProcessOptions = {}): OwnedProcess {
|
|
414
|
+
let pgid: number | undefined;
|
|
415
|
+
if (isPosix && child.pid !== undefined) {
|
|
416
|
+
const pgrp = readPgidFromProc(child.pid);
|
|
417
|
+
if (pgrp !== undefined) {
|
|
418
|
+
pgid = pgrp === child.pid ? child.pid : undefined;
|
|
419
|
+
} else if (groupAlive(child.pid)) {
|
|
420
|
+
pgid = child.pid;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
return createOwnedProcess(child, pgid, opts);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** Number of currently live owned processes. Exposed for leak assertions/tests. */
|
|
427
|
+
export function liveOwnedProcessCount(): number {
|
|
428
|
+
return liveOwners.size;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Dispose every live owned process (owner-scoped teardown, postmortem, tests). */
|
|
432
|
+
export async function disposeAllOwnedProcesses(): Promise<void> {
|
|
433
|
+
await Promise.allSettled([...liveOwners].map((owner) => owner.dispose()));
|
|
434
|
+
}
|