pi-crew 0.11.38 → 0.11.39

package/src/runtime/recovery/crash-recovery.ts

@@ -24,14 +24,17 @@ import { recordFromTask, upsertCrewAgent } from "../crew-agent-records.ts";
 import { isWorkerHeartbeatStale } from "../heartbeat/worker-heartbeat.ts";
 import { terminateLiveAgentsForRun } from "../live-session/live-agent-manager.ts";
 import type { ManifestCache } from "../manifest-cache.ts";
+import { buildProcSnapshot, type ProcSnapshot } from "../process/run-descendant-sweep.ts";
 import { checkProcessLiveness } from "../process-status.ts";
 import { mapConcurrent } from "../scheduling/parallel-utils.ts";
 import {
+	drainReapEscalations,
 	hasLiveRunClaim,
 	isIntentionalWait,
 	isPlanApprovalPendingEffective,
 	type ReconcileResult,
 	reconcileStaleRun,
+	sweepTerminalRunDescendants,
 } from "../stale-reconciler.ts";
 
 export interface RecoveryPlan {
@@ -1023,12 +1026,156 @@ export function applyReconcileAtOpenFastPath(
 	};
 }
 
+/**
+ * Wave 11b.2: run statuses whose descendants are fair game for the startup
+ * sweep. Deliberately NARROWER than `TEAM_TERMINAL_RUN_STATUSES`, which also
+ * counts `blocked`: a blocked run is intentionally parked (plan approval / ask)
+ * and is enumerated by the reconcile lane, so its descendants must not be swept
+ * here. `queued`/`planning`/`running` never appear either — those runs may
+ * legitimately own live processes.
+ */
+const TERMINAL_RUN_STATUSES_FOR_SWEEP: ReadonlySet<TeamRunManifest["status"]> = new Set(["completed", "failed", "cancelled"]);
+
+/**
+ * Wave 11b.2: hard cap on terminal runs swept in ONE pass. Each visit costs a
+ * /proc enumeration (plus a lock + two manifest reads), so an install with
+ * hundreds of historical runs must not turn startup into a /proc storm; the
+ * remainder are picked up on later passes (startup + the periodic reconcile),
+ * newest first, which is also where leaks are most likely to still exist.
+ */
+const MAX_UNSWEPT_TERMINAL_RUNS_PER_PASS = 25;
+
+/**
+ * Wave 11b.3 (grace window): a terminal run is only swept once its `updatedAt`
+ * is at least this old. `status` flips to `completed` the moment the last task
+ * finishes, while the run's mux panes / live sessions can still be attached for
+ * a while afterwards — the operator may be reading that output. Sweeping
+ * immediately would kill processes the operator is still using, so the lane
+ * waits one minute. A run that is still inside the window is NOT stamped: it
+ * simply stays eligible and is picked up by a later pass (startup + the periodic
+ * reconcile), which is exactly the reachability the 11b.2 marker provides.
+ */
+const TERMINAL_SWEEP_MIN_AGE_MS = 60_000;
+
+/**
+ * Wave 11b.3 (old-run starvation): how deep the terminal lane reads the manifest
+ * cache. `list(50)` (the call the reconcile lane uses) is a top-N by `createdAt`
+ * — with 50 newer runs in between, the run that actually leaked (the 10-22
+ * incident: completed a day before its grandchildren died) would never be seen.
+ * 200 is a per-lane window only; `DEFAULT_CACHE.manifestMaxEntries` and the
+ * global `list()` default stay untouched.
+ */
+const TERMINAL_SWEEP_CACHE_WINDOW = 200;
+
+/**
+ * Wave 11b.4 (HIGH-2): how long a run marked `descendantSweepPending` waits
+ * before the lane tries it again. A pending flag means the sweep PROVED
+ * descendants it could not signal (a live unmatched group leader, a candidate
+ * with no usable pgid, a refused group reap, an interactive tty) — the leak is
+ * still out there, but the blocker is usually another live process, so retrying
+ * on the very next pass would burn a /proc scan per pass for as long as that
+ * process lives. Ten minutes is short enough that a real leak is reaped while
+ * the operator is still in the same session, and long enough that a stuck run
+ * cannot dominate the 25-per-pass budget.
+ */
+const TERMINAL_SWEEP_RETRY_MS = 10 * 60_000;
+
+function isTerminalRunStatus(status: TeamRunManifest["status"]): boolean {
+	return TERMINAL_RUN_STATUSES_FOR_SWEEP.has(status);
+}
+
+/**
+ * Wave 11b.2 + 11b.4: is this TERMINAL run still entitled to a sweep attempt?
+ *
+ * Never swept ⇒ yes. Swept and clean ⇒ no (that is what the marker is for).
+ * Swept but `descendantSweepPending` ⇒ yes again, once the bounded retry
+ * interval has elapsed — otherwise a run whose candidates were all skipped
+ * would never be looked at again and 11b.4's whole point (a real leak must stay
+ * reachable) would be lost.
+ */
+function isTerminalSweepEligible(manifest: TeamRunManifest, now: number): boolean {
+	if (!isTerminalRunStatus(manifest.status)) return false;
+	if (manifest.descendantSweepAt === undefined) return true;
+	if (manifest.descendantSweepPending !== true) return false;
+	const sweptAt = Date.parse(manifest.descendantSweepAt);
+	return Number.isFinite(sweptAt) && now - sweptAt > TERMINAL_SWEEP_RETRY_MS;
+}
+
+/**
+ * Epoch ms of the run's last state change (`updatedAt`), falling back to
+ * `createdAt`. `TeamRunManifest` has no `completedAt` (verified in
+ * state/types.ts) and `updatedAt` is the field every status write refreshes, so
+ * it is the correct "when did this run become terminal" proxy. A manifest whose
+ * timestamps are both unparseable is treated as ANCIENT (0) — it must not starve
+ * a leak just because its clock field is broken.
+ */
+function terminalSinceMs(manifest: TeamRunManifest): number {
+	const updated = Date.parse(manifest.updatedAt);
+	if (Number.isFinite(updated)) return updated;
+	const created = Date.parse(manifest.createdAt);
+	return Number.isFinite(created) ? created : 0;
+}
+
+/**
+ * Wave 11b.2 + 11b.3: terminal runs whose descendant sweep marker is absent,
+ * OLDEST first, capped at `MAX_UNSWEPT_TERMINAL_RUNS_PER_PASS`.
+ *
+ * This list is what makes the sweep reachable after the fact: the pre-11b.2
+ * enumeration (`status === running | blocked`) structurally could not see a run
+ * that had already persisted as `completed` — the shape of the incident run,
+ * whose leaked grandchildren outlived it by a day. It is intentionally NOT
+ * filtered by `currentSessionId`: a terminal run has no live claim to respect
+ * (the marker, checked again under the run lock, is the idempotency guard), and
+ * the sweep itself refuses to signal anything it cannot prove descends from
+ * THIS run with a dead recorded parent.
+ *
+ * Ordering (11b.3): OLDEST `updatedAt` first, deliberately the opposite of the
+ * reconcile lane. The cap is 25 /proc scans per pass, so whatever order is
+ * chosen decides which leaks STARVE — and a leak from an old run is the one
+ * that has been burning memory longest (the incident ran for 24 h). Newest-first
+ * would let a steady stream of fresh runs push an old leak out of the cap
+ * forever; oldest-first guarantees every eligible run is visited within
+ * `ceil(eligible / 25)` passes.
+ *
+ * Age gate (11b.3): a run whose `updatedAt` is younger than
+ * `TERMINAL_SWEEP_MIN_AGE_MS` is skipped and left UNSTAMPED (see the constant).
+ *
+ * Retry gate (11b.4, HIGH-2): a stamped run comes back only while it still
+ * carries `descendantSweepPending` AND `TERMINAL_SWEEP_RETRY_MS` has elapsed
+ * since the stamp — see `isTerminalSweepEligible`. Selection is still
+ * oldest-first and still capped, so a pending run competes for the same 25-scan
+ * budget as a never-swept one rather than getting a lane of its own.
+ */
+function selectUnsweptTerminalRunIds(manifestCache: ManifestCache, now = Date.now()): string[] {
+	return manifestCache
+		.list(TERMINAL_SWEEP_CACHE_WINDOW)
+		.filter((manifest) => isTerminalSweepEligible(manifest, now))
+		.map((manifest) => ({ runId: manifest.runId, terminalSinceMs: terminalSinceMs(manifest) }))
+		.filter((entry) => now - entry.terminalSinceMs >= TERMINAL_SWEEP_MIN_AGE_MS)
+		.sort((a, b) => a.terminalSinceMs - b.terminalSinceMs)
+		.slice(0, MAX_UNSWEPT_TERMINAL_RUNS_PER_PASS)
+		.map((entry) => entry.runId);
+}
+
 export async function reconcileAllStaleRuns(
 	cwd: string,
 	manifestCache: ManifestCache,
 	now = Date.now(),
 	currentSessionId?: string,
 ): Promise<ReconcileResult[]> {
+	// Wave 11b.2 (F7 reachability): startup reconciliation used to enumerate
+	// `running | blocked` only, so a run that had ALREADY persisted as terminal —
+	// the common case the next morning — never reached `reconcileStaleRun` and its
+	// leaked descendants (the 390 MB / 24 h bash grandchildren behind
+	// team_20261022110510) were never reaped. A leak outlives its run, so the
+	// sweep must be reachable from a LATER pass: every terminal run that has not
+	// been swept yet (`manifest.descendantSweepAt` absent) is enumerated here and
+	// swept once, under its own run lock, in a lane that never runs the stale
+	// verdict tree (a terminal run is not stale — it is finished).
+	// Wave 11b.3 adds the two bounds: the grace window (`updatedAt` at least a
+	// minute old, so a just-finished run keeps its panes) and the deeper cache
+	// window (oldest first), both documented on the constants above.
+	const terminalRunIds = selectUnsweptTerminalRunIds(manifestCache, now);
 	// Capture runIds to reconcile BEFORE acquiring locks — avoids TOCTOU between cache iteration and lock acquisition.
 	const runIds = manifestCache
 		.list(50)
@@ -1123,6 +1270,11 @@ export async function reconcileAllStaleRuns(
 					out.push(result);
 				}
 			});
+			// Wave 11a review (MAJOR #3): the SYNC reconcile arms a group SIGKILL
+			// escalation it cannot await. This async owner awaits it OUTSIDE the run
+			// lock (so the lock hold time is unchanged) — the sweep must not resolve
+			// while a SIGKILL is still pending delivery.
+			await drainReapEscalations();
 			return out;
 		} catch (error) {
 			logInternalError("crash-recovery.reconcileStaleRuns", error, `runId=${runId}`, "warn");
@@ -1130,6 +1282,56 @@ export async function reconcileAllStaleRuns(
 		}
 	});
 	const results = perRun.flat();
+	// Wave 11b.2: terminal-run descendant sweep lane. Deliberately SEPARATE from
+	// the reconcile lane above: a terminal run must never enter the stale verdict
+	// tree (`updateRunStatus`, task repair, `crew.run.reconciled_stale`), it only
+	// needs its leaked descendants reaped once. Same per-run error isolation and
+	// per-run lock as the reconcile lane; contributes no ReconcileResult.
+	if (terminalRunIds.length > 0) {
+		logInternalError(
+			"crash-recovery.terminal-descendant-sweep",
+			new Error(`sweeping descendants of ${terminalRunIds.length} unswept terminal run(s)`),
+			terminalRunIds.join(","),
+			"debug",
+		);
+	}
+	// Wave 11b.4 (LOW-4): ONE /proc snapshot for the whole lane. Built lazily —
+	// a pass whose every run bails out below (already clean, resumed, inside the
+	// grace window) never pays for it. Before 11b.4 each run built its own, so a
+	// full 25-run budget read /proc for every process on the host 25 times.
+	let passSnapshot: ProcSnapshot | undefined;
+	const sweepSnapshot = (): ProcSnapshot => (passSnapshot ??= buildProcSnapshot());
+	await mapConcurrent(terminalRunIds, 2, async (runId): Promise<void> => {
+		try {
+			const loaded = await loadRunManifestByIdAsync(cwd, runId); // best-effort; re-read under the lock below
+			if (!loaded) return;
+			await withRunLock(loaded.manifest, async () => {
+				const fresh = await loadRunManifestByIdAsync(cwd, runId); // consistent read inside the lock
+				if (!fresh) return;
+				// Idempotency: a concurrent pass (or the run's own completion path)
+				// already stamped this run clean ⇒ nothing to do, and no /proc scan.
+				// 11b.4 (HIGH-2): a run stamped `descendantSweepPending` is NOT done
+				// — it is due again once the retry interval has elapsed, which is
+				// what the cache selection just said; re-checked here because the
+				// fresh read is authoritative and another pass may have just run it.
+				if (!isTerminalSweepEligible(fresh.manifest, now)) return;
+				// The run may have flipped back to running/blocked (resume) between
+				// the cache list and this lock — that run belongs to the reconcile
+				// lane, and its descendants are legitimate again.
+				if (!isTerminalRunStatus(fresh.manifest.status)) return;
+				// 11b.3: re-check the grace window on the FRESH manifest — a run that
+				// resumed and finished again since the cache snapshot is young again,
+				// and its panes may have just been re-attached.
+				if (now - terminalSinceMs(fresh.manifest) < TERMINAL_SWEEP_MIN_AGE_MS) return;
+				sweepTerminalRunDescendants(fresh.manifest, fresh.tasks, { snapshot: sweepSnapshot() });
+			});
+			// Wave 11a review MAJOR #3 (same owner as the reconcile lane): the sync
+			// sweep arms a group SIGKILL escalation it cannot await.
+			await drainReapEscalations();
+		} catch (error) {
+			logInternalError("crash-recovery.terminal-descendant-sweep", error, `runId=${runId}`, "warn");
+		}
+	});
 	// US-002 (2026-09-22): structured stale-lock sweep. Runs after reconciliation
 	// so locks whose holder died (kill -9 / crash) do not linger. The sweep is
 	// strictly safer than acquire-time steal (stale AND holder-dead only), so a

package/src/runtime/rpc/frame-client.ts

@@ -52,10 +52,30 @@ export interface RpcSpawnHandle {
 /** Spawn function: receives the pi argv (after `--mode rpc` etc.), returns a handle. */
 export type RpcSpawnFn = (argv: string[]) => RpcSpawnHandle;
 
+/**
+ * Spawn options for the frame client's child (wave 11a, F4). Exported as a pure
+ * function so the platform-dependent group-leadership flag is unit-testable
+ * without spawning a real pi binary. POSIX: `detached` makes the child its own
+ * process-group leader, so a group-targeted teardown can reach its descendants.
+ * `detached` does NOT detach the stdio pipes — frames still round-trip through
+ * the same fd 0/1/2. Windows has no process groups (single-process kill
+ * fallback), so it never detaches.
+ */
+export function frameClientSpawnOptions(platform: NodeJS.Platform): {
+	stdio: ["pipe", "pipe", "pipe"];
+	detached: boolean;
+	windowsHide: boolean;
+} {
+	return { stdio: ["pipe", "pipe", "pipe"], detached: platform !== "win32", windowsHide: true };
+}
+
 /** Default spawner: host pi binary (PI_TEAMS_PI_BIN aware) with piped stdio. */
 function defaultSpawnFn(argv: string[]): RpcSpawnHandle {
 	const { command, args } = getPiSpawnCommand(argv);
-	const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
+	// Wave 11a (F4): mirror the headless spawn branch (child-pi-spawn.ts) — see
+	// frameClientSpawnOptions above for why `detached` is on and why it does not
+	// break piped stdio.
+	const child = spawn(command, args, frameClientSpawnOptions(process.platform));
 	return {
 		stdin: child.stdin as Writable,
 		stdout: child.stdout as Readable,