pi-crew 0.11.21 → 0.11.22

package/src/runtime/live-session/live-session-runtime.ts

@@ -868,6 +868,49 @@ export async function probeLiveSessionRuntime(): Promise<LiveSessionUnavailableR
 	};
 }
 
+// DR-02 C1: the per-agent control-request dedupe Set previously grew for
+// the whole run lifetime — fed by BOTH the 500ms/1000ms control poll and the
+// realtime subscription, with no eviction. Bounded FIFO eviction: a Set
+// preserves insertion order, so deleting the first-added id evicts the OLDEST
+// request id (mirrors the NotificationRouter.SEEN_MAP_MAX_SIZE bounded-seen
+// pattern). Worst case after an eviction is one redelivery of a very old
+// control request; the consumers (steer / follow-up) are idempotent by design.
+export const MAX_SEEN_CONTROL_REQUEST_IDS = 10_000;
+
+class BoundedSeenControlRequestIdSet extends Set<string> {
+	override add(id: string): this {
+		super.add(id);
+		while (this.size > MAX_SEEN_CONTROL_REQUEST_IDS) {
+			const oldest = this.values().next().value;
+			if (oldest === undefined) break;
+			this.delete(oldest);
+		}
+		return this;
+	}
+}
+
+/** @internal — test seam for the C1 cap (see live-agent-control-dedupe tests). */
+export const __test__BoundedSeenControlRequestIdSet = BoundedSeenControlRequestIdSet;
+
+/**
+ * R6-9 (DR-03 round-6): best-effort usage extraction for the fault/watchdog
+ * escape returns. The success return reports usageFromSessionStats(session),
+ * but the fault/watchdog catch paths previously carried NO usage — whenever
+ * embeddedWorkers routed work to the embedded tier and the task faulted, the
+ * burned turns vanished from the budget (under-count). The session may be
+ * disposed or partially unusable after the fault, so stats collection is
+ * guarded — it can never turn a fault into a crash. Returns undefined when no
+ * usable stats surface exists.
+ */
+function faultPathUsage(session: LiveSessionLike | undefined): UsageState | undefined {
+	try {
+		return usageFromSessionStats(session);
+	} catch (error) {
+		logInternalError("live-session.fault-usage-stats", error, "session stats unavailable on fault path");
+		return undefined;
+	}
+}
+
 export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<LiveSessionRunResult> {
 	// Cold-start race fix: ensure the hot module graph is warm before touching
 	// any module. Under tsx, concurrent first-imports race module-record
@@ -1406,7 +1449,7 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
 		registerLiveAgentModel(agentId, resolvedModelRef ?? "");
 		streamOut = createStreamingOutput(input.manifest, input.task.id);
 		let controlCursor: LiveAgentControlCursor = { offset: 0 };
-		const seenControlRequestIds = new Set<string>();
+		const seenControlRequestIds: Set<string> = new BoundedSeenControlRequestIdSet();
 		let controlBusy = false;
 		const pollControl = async () => {
 			if (!isCurrent() || controlBusy || !session) return;
@@ -1793,12 +1836,16 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
 				},
 			});
 			updateLiveAgentStatus(agentId, "failed");
+			// R6-9: turns burned before the watchdog fired still count — capture the
+			// partial usage (guarded; session was disposed by the watchdog timer).
+			const watchdogUsage = faultPathUsage(session);
 			return {
 				available: true,
 				exitCode: 1,
 				stdout: stdoutTail.value().trim(),
 				stderr: error.message,
 				jsonEvents,
+				usage: watchdogUsage,
 				error: error.message,
 				...(embedded ? { fault: { kind: "turn-watchdog" as const, detail: error.message, countAsFault: true } } : {}),
 			};
@@ -1820,12 +1867,15 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
 			// drop-reference (synchronous — never awaits the stuck abort): drain +
 			// terminate + dispose + UNREGISTER, so the hung session holds no slot.
 			dropLiveAgentReference(agentId);
+			// R6-9: partial usage from the turns completed before the tool hung.
+			const hungUsage = faultPathUsage(session);
 			return {
 				available: true,
 				exitCode: 1,
 				stdout: stdoutTail.value().trim(),
 				stderr: error.message,
 				jsonEvents,
+				usage: hungUsage,
 				error: error.message,
 				...(embedded ? { fault: { kind: "hung-tool" as const, timeoutMs: error.timeoutMs, countAsFault: true } } : {}),
 			};
@@ -1861,6 +1911,9 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
 			stdout: stdoutTail.value().trim(),
 			stderr: message,
 			jsonEvents,
+			// R6-9: generic fault path — partial usage from completed turns so the
+			// budget sees the real spend even when the task failed.
+			usage: faultPathUsage(session),
 			error: message,
 			...(embeddedCatchFault ? { fault: embeddedCatchFault } : {}),
 		};

package/src/runtime/model/model-fallback.ts

@@ -606,7 +606,29 @@ export function resolveModelFallbackPolicy(
 	} else {
 		maxAutoFallbacks = config?.maxAutoFallbacks;
 	}
-	const order = (env.PI_CREW_MODEL_FALLBACK_ORDER as "parentFirst" | "asIs" | undefined) ?? config?.order;
+	// J5 (DR-03 round-6): validate the order env against the two legal values.
+	// The previous unvalidated cast let arbitrary strings flow into
+	// orderAutoFallbacks' `order === "asIs"` checks — any typo (e.g.
+	// "parent-first") silently behaved as parentFirst. Invalid → warn + fall
+	// back to config/default, mirroring the PI_CREW_MAX_AUTO_FALLBACKS
+	// clamp-warn sibling pattern above.
+	const envOrder = env.PI_CREW_MODEL_FALLBACK_ORDER;
+	let order: "parentFirst" | "asIs" | undefined;
+	if (envOrder) {
+		if (envOrder === "parentFirst" || envOrder === "asIs") {
+			order = envOrder;
+		} else {
+			logInternalError(
+				"model-fallback.model-fallback-order-invalid",
+				undefined,
+				`PI_CREW_MODEL_FALLBACK_ORDER="${envOrder}" is invalid (expected "parentFirst" or "asIs"), falling back to config`,
+				"warn",
+			);
+			order = config?.order;
+		}
+	} else {
+		order = config?.order;
+	}
 	const requireCredentials =
 		env.PI_CREW_MODEL_REQUIRE_CREDENTIALS === "1"
 			? true

package/src/runtime/output/pi-json-output.ts

@@ -20,6 +20,14 @@ export interface ParsedPiJsonOutput {
 	 * "429 ... overloaded"). Used to detect runs that exited 0 but produced
 	 * nothing because the model was rate-limited — see task-runner 429 fix. */
 	errorMessages?: string[];
+	/** DR-01 (deep review 2026-10-14): true when the LAST assistant
+	 * `message_end` event in the stream carried no non-empty text — the
+	 * session settled without a final answer. Upstream 429 exhaustion
+	 * manifests as a chain of EMPTY assistant message_end events (content [],
+	 * usage 0) between auto-retry cycles — never as stopReason:"error" — so
+	 * `errorMessages` stays empty and `finalText` keeps an early pre-outage
+	 * sentence. undefined when no assistant message_end was parsed at all. */
+	settledWithoutAnswer?: boolean;
 }
 
 function asRecord(value: unknown): Record<string, unknown> | undefined {
@@ -82,12 +90,23 @@ export function extractText(value: unknown): string[] {
 	return text.filter((entry) => entry.trim().length > 0);
 }
 
+/** True for `message_end` events whose message is assistant-role. Feeds the
+ *  settledWithoutAnswer flag (DR-01): see ParsedPiJsonOutput.settledWithoutAnswer. */
+function isAssistantMessageEnd(event: unknown): boolean {
+	const obj = asRecord(event);
+	if (obj?.type !== "message_end") return false;
+	const message = asRecord(obj.message);
+	return message?.role === "assistant";
+}
+
 export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
 	let jsonEvents = 0;
 	const textEvents: string[] = [];
 	const patches: string[] = [];
 	const errorMessages: string[] = [];
 	let usage: ParsedPiUsage | undefined;
+	let sawAssistantMessageEnd = false;
+	let lastAssistantMessageHadText = false;
 	for (const line of stdout.split("\n")) {
 		const trimmed = line.trim();
 		if (!trimmed) continue;
@@ -98,7 +117,12 @@ export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
 			continue;
 		}
 		jsonEvents++;
-		textEvents.push(...extractText(event));
+		const eventTexts = extractText(event);
+		textEvents.push(...eventTexts);
+		if (isAssistantMessageEnd(event)) {
+			sawAssistantMessageEnd = true;
+			lastAssistantMessageHadText = eventTexts.length > 0;
+		}
 		// Extract unified patches from tool_result events
 		extractPatch(event, patches);
 		// Extract provider/model error messages from message_end events (429 fix).
@@ -114,6 +138,7 @@ export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
 		usage,
 		patches: patches.length > 0 ? patches : undefined,
 		errorMessages: errorMessages.length > 0 ? errorMessages : undefined,
+		settledWithoutAnswer: sawAssistantMessageEnd ? !lastAssistantMessageHadText : undefined,
 	};
 }
 

package/src/runtime/policy-engine.ts

@@ -104,6 +104,24 @@ export function evaluateCrewPolicy(input: PolicyEngineInput): PolicyDecision[] {
 		}
 	}
 
+	// R4-4 (DR-02 wave 2): needs_attention is a TERMINAL-without-result status
+	// (surface-lost / no-yield paths in post-execution) and the scheduler counts
+	// it as done, so a run can otherwise complete green while a task never
+	// submitted its result. Surface it as a NON-blocking notify decision —
+	// detection-first: no status flip, no block, no retry (re-queue/recovery is
+	// owned by the surface-degrade drain machinery, not the policy engine).
+	const needsAttentionTasks = input.tasks.filter((t) => t.status === "needs_attention");
+	if (needsAttentionTasks.length > 0) {
+		const ids = needsAttentionTasks.map((t) => t.id).join(", ");
+		decisions.push(
+			decision(
+				"notify",
+				"task_needs_attention",
+				`${needsAttentionTasks.length} task(s) finished needs_attention without submitting a result: ${ids}.`,
+			),
+		);
+	}
+
 	if (decisions.length === 0 && input.tasks.length > 0 && input.tasks.every((task) => task.status === "completed")) {
 		decisions.push(decision("closeout", "run_complete", "All tasks completed and no policy blockers were found."));
 	}

package/src/runtime/process/zombie-scanner.ts

@@ -60,6 +60,13 @@ export interface ZombieSubagent {
 	surfacePaneId?: string;
 	rssKb: number;
 	elapsedSec: number | undefined;
+	/** M-7 (DR-04 round-7): /proc/<pid>/stat starttime — clock ticks since boot.
+	 *  Stable pid identity: unlike the pid number, starttime changes when the OS
+	 *  recycles a dead pid for a NEW process, so kill-time re-checks can detect
+	 *  PID reuse between scan and kill. */
+	starttime?: number;
+	/** M-7: wall-clock start time (ISO) derived from starttime + /proc/uptime. */
+	startedAt?: string;
 	cmd: string;
 }
 
@@ -106,8 +113,8 @@ function readProcEnviron(pid: number): Record<string, string> {
 	}
 }
 
-/** Read /proc/<pid>/stat to get ppid + elapsed. Returns undefined if unreadable. */
-function readProcStat(pid: number): { ppid: number; elapsedSec: number | undefined } | undefined {
+/** Read /proc/<pid>/stat to get ppid + starttime + elapsed. Returns undefined if unreadable. */
+function readProcStat(pid: number): { ppid: number; starttime: number | undefined; elapsedSec: number | undefined } | undefined {
 	try {
 		const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8");
 		// stat format: pid (comm) state ppid ... starttime ...
@@ -117,14 +124,38 @@ function readProcStat(pid: number): { ppid: number; elapsedSec: number | undefin
 		if (!rest) return undefined;
 		const ppid = Number.parseInt(rest[PROC_STAT_PPID_INDEX] ?? "", 10);
 		const starttimeTicksRaw = Number.parseInt(rest[PROC_STAT_STARTTIME_INDEX] ?? "", 10);
-		const starttimeTicks = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
-		const elapsedSec = computeElapsedSec(starttimeTicks);
-		return { ppid: Number.isFinite(ppid) ? ppid : 0, elapsedSec };
+		const starttime = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
+		const elapsedSec = computeElapsedSec(starttime);
+		return { ppid: Number.isFinite(ppid) ? ppid : 0, starttime, elapsedSec };
 	} catch {
 		return undefined;
 	}
 }
 
+/** /proc/uptime first field (seconds since boot); undefined when unreadable. */
+function readUptimeSec(): number | undefined {
+	try {
+		const uptimeRaw = fs.readFileSync("/proc/uptime", "utf-8");
+		const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
+		return Number.isFinite(uptimeSec) ? uptimeSec : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+/**
+ * M-7 (DR-04 round-7): wall-clock start time of a pid from its starttime
+ * ticks — boot time = now - uptime, so startedAt = boot + starttime/CLK_TCK.
+ * Best-effort ISO string; undefined when /proc/uptime is unavailable.
+ */
+function computeStartedAtIso(starttime: number | undefined): string | undefined {
+	if (starttime === undefined || !Number.isFinite(starttime)) return undefined;
+	const uptimeSec = readUptimeSec();
+	if (uptimeSec === undefined) return undefined;
+	const bootMs = Date.now() - uptimeSec * 1000;
+	return new Date(bootMs + (starttime / 100) * 1000).toISOString();
+}
+
 function computeElapsedSec(starttimeTicks: number | undefined): number | undefined {
 	if (starttimeTicks === undefined || !Number.isFinite(starttimeTicks)) return undefined;
 	try {
@@ -132,10 +163,8 @@ function computeElapsedSec(starttimeTicks: number | undefined): number | undefin
 		// portably from Node requires a native addon; hardcoding 100 matches every
 		// mainstream Linux distro and keeps this dependency-free.
 		const ticksPerSec = 100;
-		// /proc/uptime: first field is seconds since boot.
-		const uptimeRaw = fs.readFileSync("/proc/uptime", "utf-8");
-		const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
-		if (!Number.isFinite(uptimeSec)) return undefined;
+		const uptimeSec = readUptimeSec();
+		if (uptimeSec === undefined) return undefined;
 		// starttime (ticks since boot) → process age in seconds = uptime - starttime/ticksPerSec.
 		const startAgeSec = starttimeTicks / ticksPerSec;
 		return Math.max(0, uptimeSec - startAgeSec);
@@ -271,6 +300,10 @@ export function scanZombieSubagents(): ZombieScanResult {
 				surfacePaneId: environ.PI_CREW_SURFACE_PANE || undefined,
 				rssKb: readProcRssKb(pid),
 				elapsedSec: stat?.elapsedSec,
+				// M-7: starttime (ticks) + derived wall-clock start — pid identity pin
+				// for kill-time PID-reuse checks and prompt-kill triage.
+				starttime: stat?.starttime,
+				startedAt: computeStartedAtIso(stat?.starttime),
 				cmd: readProcCmdline(pid),
 			};
 
@@ -392,9 +425,13 @@ export function formatZombieReport(scan: ZombieScanResult): string {
 
 	if (scan.zombies.length > 0) {
 		lines.push(`### Zombies — parent dead (${scan.zombies.length})`);
-		lines.push("These sub-agents are orphaned. Safe to kill after review:");
+		lines.push("These sub-agents are orphaned. Safe to kill, but do it PROMPTLY after this");
+		lines.push("scan: an unreviewed delay re-opens the PID-reuse window (the OS can recycle");
+		lines.push("a dead zombie's PID for an unrelated process). Before killing, re-check that");
+		lines.push("the PID's STARTED column below still matches — a newer starttime means the");
+		lines.push("process you are about to kill is NOT the zombie that was scanned.");
 		lines.push("");
-		lines.push("  PID       PARENT  RSS       ROLE          SURFACE          CMD");
+		lines.push("  PID       PARENT  AGE      STARTED              RSS       ROLE          SURFACE          CMD");
 		for (const z of scan.zombies) {
 			lines.push(formatZombieRow(z));
 		}
@@ -405,7 +442,7 @@ export function formatZombieReport(scan: ZombieScanResult): string {
 		lines.push(`### Live — parent still running (${scan.live.length})`);
 		lines.push("NOT zombies. Do not kill (parent PID is alive and may still reap them).");
 		lines.push("");
-		lines.push("  PID       PARENT  RSS       ROLE          SURFACE          CMD");
+		lines.push("  PID       PARENT  AGE      STARTED              RSS       ROLE          SURFACE          CMD");
 		for (const l of scan.live) {
 			lines.push(formatZombieRow(l));
 		}
@@ -435,7 +472,9 @@ export function formatZombieReport(scan: ZombieScanResult): string {
 		lines.push("");
 	}
 
-	lines.push("To kill a zombie: `kill <PID>` (the OS will reap it). This tool never kills.");
+	lines.push("To kill a zombie: `kill <PID>` (the OS will reap it). Kill soon after the");
+	lines.push("scan — a PID listed here can be reused by an unrelated process if you wait;");
+	lines.push("verify its starttime/age first. This tool never kills.");
 	return lines.join("\n");
 }
 
@@ -443,7 +482,30 @@ export function formatZombieReport(scan: ZombieScanResult): string {
 function formatZombieRow(z: ZombieSubagent): string {
 	// SURFACE = "kind:paneId" (T12) — the pane doctor closes when orphaned.
 	const surface = z.surface ? `${z.surface}:${z.surfacePaneId ?? "?"}` : "-";
-	return `  ${String(z.pid).padEnd(9)}${String(z.crewParentPid).padEnd(8)}${formatRss(z.rssKb).padEnd(10)}${(z.role ?? "?").padEnd(14)}${surface.padEnd(16)}${z.cmd.slice(0, 44)}`;
+	// M-7: AGE + STARTED — pid identity + triage (old zombies = long-leaked;
+	// STARTED mismatch at kill time = PID was reused, do NOT kill).
+	return (
+		`  ${String(z.pid).padEnd(9)}${String(z.crewParentPid).padEnd(8)}${formatAge(z.elapsedSec).padEnd(9)}` +
+		`${formatStartedAt(z.startedAt).padEnd(21)}${formatRss(z.rssKb).padEnd(10)}${(z.role ?? "?").padEnd(14)}` +
+		`${surface.padEnd(16)}${z.cmd.slice(0, 28)}`
+	);
+}
+
+/** M-7: age as a compact human duration ("37s", "5m", "3h", "2d"). */
+function formatAge(sec: number | undefined): string {
+	if (sec === undefined || !Number.isFinite(sec) || sec < 0) return "-";
+	if (sec < 60) return `${Math.floor(sec)}s`;
+	if (sec < 3600) return `${Math.floor(sec / 60)}m`;
+	if (sec < 86400) return `${Math.floor(sec / 3600)}h`;
+	return `${Math.floor(sec / 86400)}d`;
+}
+
+/** M-7: ISO startedAt → "MM-DD HH:MM:SS" (UTC, no year — scan-time triage). */
+function formatStartedAt(iso: string | undefined): string {
+	if (!iso) return "-";
+	const d = new Date(iso);
+	if (Number.isNaN(d.getTime())) return "-";
+	return d.toISOString().slice(5, 19).replace("T", " ");
 }
 
 /** One table row for the R3-16 foreign section — state carries the triage flags. */
@@ -469,4 +531,6 @@ export const __test = {
 	looksLikePiBinary,
 	isHeadlessArgv,
 	collectAncestorPids,
+	// M-7: starttime helpers (ticks → wall-clock start).
+	computeStartedAtIso,
 };

package/src/runtime/process-status.ts

@@ -140,7 +140,12 @@ export function isDisplayActiveRun(run: TeamRunManifest, agents: CrewAgentRecord
 		if (Number.isFinite(updatedAt) && now - updatedAt > STALE_ACTIVE_RUN_MS) return false;
 	}
 	// Grace period: show completed runs for a few seconds so users see the result.
-	if (run.status === "completed" || run.status === "failed" || run.status === "cancelled") {
+	// R5-4 (DR-02 wave 2): "blocked" is terminal (isFinishedRunStatus) but NOT an
+	// error the user caused — without the grace branch it fell through to
+	// `!isActiveRunStatus` and vanished from the widget instantly while "failed"
+	// lingered 10min. Blocked now gets the same ERROR_VISIBILITY_GRACE_MS so the
+	// block reason stays inspectable.
+	if (run.status === "completed" || run.status === "failed" || run.status === "cancelled" || run.status === "blocked") {
 		const grace = run.status === "completed" ? COMPLETED_VISIBILITY_GRACE_MS : ERROR_VISIBILITY_GRACE_MS;
 		const lastAgentActivity = agents.reduce<number>((max, agent) => {
 			const ts = agent.completedAt ?? agent.startedAt;