pi-subagents 0.47.1 → 0.49.0

package/src/runs/background/chain-append.ts

@@ -4,7 +4,7 @@ import * as path from "node:path";
 import { writeAtomicJson } from "../../shared/atomic-json.ts";
 import { appendJsonl } from "../../shared/artifacts.ts";
 import type { AsyncParallelGroupStatus, AsyncStatus, WorkflowGraphNode, WorkflowGraphSnapshot } from "../../shared/types.ts";
-import { readStatus } from "../../shared/utils.ts";
+import { PROMPT_REDACTED, readStatus } from "../../shared/utils.ts";
 import type { DynamicRunnerGroup, ParallelStepGroup, RunnerStep, RunnerSubagentStep, RunnerCheckpointStep } from "../shared/parallel-utils.ts";
 import { isCheckpointRunnerStep, isDynamicRunnerGroup, isParallelGroup } from "../shared/parallel-utils.ts";
 
@@ -19,6 +19,7 @@ export interface ChainAppendRequest {
 export interface ChainAppendResult {
 	request: ChainAppendRequest;
 	pendingCount: number;
+	bookkeepingError?: string;
 }
 
 type StatusStep = NonNullable<AsyncStatus["steps"]>[number];
@@ -69,6 +70,7 @@ export function enqueueChainAppendRequest(input: {
 	runId: string;
 	steps: RunnerStep[];
 	now?: number;
+	admit?: (persist: () => void) => void;
 }): ChainAppendResult {
 	const status = readStatus(input.asyncDir);
 	if (!status) throw new Error(`No async run status found for '${input.runId}'.`);
@@ -85,20 +87,36 @@ export function enqueueChainAppendRequest(input: {
 		steps: input.steps,
 	};
 	fs.mkdirSync(appendDir(input.asyncDir), { recursive: true });
-	writeAtomicJson(appendRequestPath(input.asyncDir, request), request);
-	const pendingCount = countPendingChainAppendRequests(input.asyncDir);
+	const persist = () => writeAtomicJson(appendRequestPath(input.asyncDir, request), request);
+	if (input.admit) input.admit(persist);
+	else persist();
+	let pendingCount = 1;
+	const bookkeepingErrors: string[] = [];
+	try {
+		pendingCount = countPendingChainAppendRequests(input.asyncDir);
+	} catch (error) {
+		bookkeepingErrors.push(`pending append count failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
 	const statusPath = path.join(input.asyncDir, "status.json");
 	const updatedStatus = { ...status, pendingAppends: pendingCount, lastUpdate: request.createdAt };
-	writeAtomicJson(statusPath, updatedStatus);
-	appendJsonl(path.join(input.asyncDir, "events.jsonl"), JSON.stringify({
-		type: "subagent.chain.append.requested",
-		ts: request.createdAt,
-		runId: input.runId,
-		requestId: request.id,
-		stepCount: input.steps.length,
-		pendingAppends: pendingCount,
-	}));
-	return { request, pendingCount };
+	try {
+		writeAtomicJson(statusPath, updatedStatus);
+	} catch (error) {
+		bookkeepingErrors.push(`status update failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	try {
+		appendJsonl(path.join(input.asyncDir, "events.jsonl"), JSON.stringify({
+			type: "subagent.chain.append.requested",
+			ts: request.createdAt,
+			runId: input.runId,
+			requestId: request.id,
+			stepCount: input.steps.length,
+			pendingAppends: pendingCount,
+		}));
+	} catch (error) {
+		bookkeepingErrors.push(`event append failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	return { request, pendingCount, ...(bookkeepingErrors.length > 0 ? { bookkeepingError: bookkeepingErrors.join("; ") } : {}) };
 }
 
 function readAppendRequest(filePath: string): ChainAppendRequest | undefined {
@@ -135,8 +153,8 @@ const MAX_STATUS_STEP_DESCRIPTION_CHARS = 160;
 
 /** Bounded one-line per-step task description persisted into status.json for fleet display. */
 export function statusStepDescription(task: string | undefined): string | undefined {
-	const description = task?.replace(/\s+/g, " ").trim();
-	if (!description) return undefined;
+	if (!task?.trim()) return undefined;
+	const description = PROMPT_REDACTED;
 	return description.length > MAX_STATUS_STEP_DESCRIPTION_CHARS
 		? `${description.slice(0, MAX_STATUS_STEP_DESCRIPTION_CHARS - 1)}…`
 		: description;

package/src/runs/background/fleet-view.ts

@@ -1,6 +1,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import type { AgentToolResult } from "@earendil-works/pi-agent-core";
+import { safeTerminalText } from "../../shared/display-text.ts";
 import { formatDuration, formatModelThinking, formatTokens, shortenPath } from "../../shared/formatters.ts";
 import { formatActivityLabel } from "../../shared/status-format.ts";
 import {
@@ -196,7 +197,7 @@ function readSessionTranscriptTail(sessionFile: string, maxLines: number, truste
 		try {
 			const parsed = JSON.parse(line) as unknown;
 			const messageLine = sessionMessageLine(parsed);
-			if (messageLine) lines.push(messageLine);
+			if (messageLine) lines.push(...messageLine.split(/\r?\n/));
 		} catch {
 			malformed++;
 		}
@@ -410,6 +411,18 @@ function appendKnownArtifacts(lines: string[], input: { outputPaths: string[]; s
 	for (const artifact of artifacts) lines.push(`  ${artifact}`);
 }
 
+/**
+ * Sanitizes each assembled line independently.
+ *
+ * Sanitizing the joined response would let a single binary fragment in child
+ * output collapse the whole transcript, including run state, artifact paths, and
+ * warnings, into the binary placeholder. Per-line sanitization keeps that damage
+ * to the offending line.
+ */
+function safeTranscriptLines(lines: string[]): string {
+	return lines.map((line) => safeTerminalText(line)).join("\n");
+}
+
 function appendTranscriptBody(lines: string[], sourceLabel: string, sourceLines: string[], truncated: boolean): void {
 	lines.push(`${sourceLabel}${truncated ? " (tail truncated)" : ""}:`);
 	if (sourceLines.length === 0) {
@@ -470,7 +483,7 @@ export function formatAsyncRunTranscript(status: AsyncStatus, asyncDir: string,
 		for (const warning of warnings) lines.push(`  ${warning}`);
 	}
 	appendTranscriptBody(lines, transcriptSource, transcriptLines, truncated);
-	return lines.join("\n");
+	return safeTranscriptLines(lines);
 }
 
 export function formatNestedRunTranscript(run: NestedRunSummary, options: TranscriptOptions = {}): string {
@@ -488,7 +501,7 @@ export function formatNestedRunTranscript(run: NestedRunSummary, options: Transc
 	appendKnownArtifacts(lines, { outputPaths: [], sessionFile: run.sessionFile });
 	if (!run.sessionFile) {
 		appendTranscriptBody(lines, "Transcript tail", [], false);
-		return lines.join("\n");
+		return safeTranscriptLines(lines);
 	}
 	const sessionTail = readSessionTranscriptTail(run.sessionFile, lineLimit, options.sessionRoots ?? []);
 	if (sessionTail.warnings.length) {
@@ -496,7 +509,7 @@ export function formatNestedRunTranscript(run: NestedRunSummary, options: Transc
 		for (const warning of sessionTail.warnings) lines.push(`  ${warning}`);
 	}
 	appendTranscriptBody(lines, `Session transcript tail from ${run.sessionFile}`, sessionTail.lines, false);
-	return lines.join("\n");
+	return safeTranscriptLines(lines);
 }
 
 export function formatAsyncResultTranscript(data: {
@@ -536,5 +549,5 @@ export function formatAsyncResultTranscript(data: {
 	].filter((line): line is string => Boolean(line));
 	appendKnownArtifacts(lines, { outputPaths: [], sessionFile, resultPath });
 	appendTranscriptBody(lines, "Result transcript tail", transcriptLines.filter((line) => line.trim()), output.split(/\r?\n/).length > lineLimit);
-	return lines.join("\n");
+	return safeTranscriptLines(lines);
 }

package/src/runs/background/owned-process-tree.ts

@@ -0,0 +1,104 @@
+import { spawnSync } from "node:child_process";
+import type { ProcessTreeTerminalV1 } from "../../shared/types.ts";
+
+const DEFAULT_TERM_GRACE_MS = 3000;
+const DEFAULT_KILL_VERIFY_MS = 1000;
+const VERIFY_INTERVAL_MS = 25;
+
+type SignalResult = "sent" | "absent" | { diagnostic: string };
+
+function diagnostic(error: unknown): string {
+	return error instanceof Error ? error.message : String(error);
+}
+
+function signalProcess(id: number, signal: NodeJS.Signals): SignalResult {
+	try {
+		process.kill(id, signal);
+		return "sent";
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === "ESRCH") return "absent";
+		return { diagnostic: diagnostic(error) };
+	}
+}
+
+function activeProcessGroupMembers(processGroupId: number): number[] | { diagnostic: string } {
+	const result = spawnSync("ps", ["-axo", "pid=,pgid=,stat="], { encoding: "utf-8" });
+	if (result.error || result.status !== 0) {
+		return { diagnostic: result.error ? diagnostic(result.error) : (result.stderr.trim() || `ps exited with ${result.status}`) };
+	}
+	const members: number[] = [];
+	for (const line of result.stdout.split("\n")) {
+		const match = /^\s*(\d+)\s+(\d+)\s+(\S+)/.exec(line);
+		if (!match || Number(match[2]) !== processGroupId || match[3]!.startsWith("Z")) continue;
+		members.push(Number(match[1]));
+	}
+	return members;
+}
+
+async function waitUntilGroupTerminal(
+	processGroupId: number,
+	timeoutMs: number,
+): Promise<false | { state: "enumeration-failed" | "still-active"; diagnostic: string }> {
+	const deadline = Date.now() + timeoutMs;
+	while (true) {
+		const members = activeProcessGroupMembers(processGroupId);
+		if (Array.isArray(members) && members.length === 0) return false;
+		const remaining = deadline - Date.now();
+		if (remaining <= 0) {
+			if (!Array.isArray(members)) return { state: "enumeration-failed", diagnostic: members.diagnostic };
+			return { state: "still-active", diagnostic: `Process group ${processGroupId} still has active members: ${members.join(", ")}.` };
+		}
+		await new Promise<void>((resolve) => setTimeout(resolve, Math.min(VERIFY_INTERVAL_MS, remaining)));
+	}
+}
+
+function observed(processGroupId: number): ProcessTreeTerminalV1 {
+	return { state: "observed", mechanism: "posix-process-group", processGroupId, verifiedAt: Date.now() };
+}
+
+/** Owns one writer process group and arbitrates its cleanup exactly once. */
+export interface OwnedProcessTreeController {
+	terminate(): Promise<ProcessTreeTerminalV1>;
+	finishAfterWriterClose(): Promise<ProcessTreeTerminalV1>;
+}
+
+export function createOwnedProcessTreeController(
+	pid: number,
+	options: { termGraceMs?: number; killVerifyMs?: number } = {},
+): OwnedProcessTreeController {
+	let termination: Promise<ProcessTreeTerminalV1> | undefined;
+	const posixGroupOwned = process.platform !== "win32";
+	const target = posixGroupOwned ? -pid : pid;
+
+	const terminate = (): Promise<ProcessTreeTerminalV1> => {
+		if (termination) return termination;
+		termination = (async () => {
+			if (!posixGroupOwned) {
+				signalProcess(target, "SIGTERM");
+				return { state: "unknown", reason: "unsupported-platform" };
+			}
+			const term = signalProcess(target, "SIGTERM");
+			if (term !== "sent" && term !== "absent") {
+				return { state: "unknown", reason: "signal-failed", diagnostic: term.diagnostic };
+			}
+			const termExit = await waitUntilGroupTerminal(pid, options.termGraceMs ?? DEFAULT_TERM_GRACE_MS);
+			if (termExit === false) return observed(pid);
+
+			const kill = signalProcess(target, "SIGKILL");
+			if (kill !== "sent" && kill !== "absent") {
+				const members = activeProcessGroupMembers(pid);
+				if (!Array.isArray(members) || members.length > 0) {
+					return { state: "unknown", reason: "signal-failed", diagnostic: kill.diagnostic };
+				}
+			}
+			const killExit = await waitUntilGroupTerminal(pid, options.killVerifyMs ?? DEFAULT_KILL_VERIFY_MS);
+			if (killExit !== false) {
+				return { state: "unknown", reason: "verification-failed", diagnostic: killExit.diagnostic };
+			}
+			return observed(pid);
+		})();
+		return termination;
+	};
+
+	return { terminate, finishAfterWriterClose: terminate };
+}

package/src/runs/background/process-terminal.ts

@@ -10,6 +10,7 @@ import {
 	type ProcessTerminalV1,
 } from "../../shared/types.ts";
 import { canonicalSessionId, inspectSessionLease } from "../shared/session-lease.ts";
+import { releaseActiveRunIndex } from "./active-run-index.ts";
 
 export interface ProcessTerminalCandidate {
 	version: 1;
@@ -40,9 +41,19 @@ function validProcessInstance(value: unknown, kind?: "runner" | "pi-writer"): va
 	if (typeof value.closeObservedAt !== "number" || !Number.isFinite(value.closeObservedAt)) return false;
 	if (typeof value.exitCode !== "number" && value.exitCode !== null) return false;
 	if (typeof value.signal !== "string" && value.signal !== null) return false;
-	return value.kind === "runner"
-		? value.attempt === undefined
-		: typeof value.attempt === "number" && Number.isInteger(value.attempt) && value.attempt >= 0;
+	if (value.kind === "runner") return value.attempt === undefined;
+	if (typeof value.attempt !== "number" || !Number.isInteger(value.attempt) || value.attempt < 0 || !isRecord(value.processTree)) return false;
+	if (value.processTree.state === "observed") {
+		return value.processTree.mechanism === "posix-process-group"
+			&& typeof value.processTree.processGroupId === "number"
+			&& Number.isInteger(value.processTree.processGroupId)
+			&& value.processTree.processGroupId > 0
+			&& typeof value.processTree.verifiedAt === "number"
+			&& Number.isFinite(value.processTree.verifiedAt);
+	}
+	return value.processTree.state === "unknown"
+		&& ["unsupported-platform", "signal-failed", "verification-failed"].includes(String(value.processTree.reason))
+		&& (value.processTree.diagnostic === undefined || typeof value.processTree.diagnostic === "string");
 }
 
 function validInstance(value: unknown): value is ProcessInstanceExitV1 {
@@ -249,6 +260,8 @@ export function finalizeProcessTerminal(
 				proof = unknownProof(runId, runnerClose.processInstanceId, "canonical-session-release-unverified");
 			} else if (inconsistentWriters || (allWriters.length === 0 && expectedEntries.length === 0)) {
 				proof = unknownProof(runId, runnerClose.processInstanceId, "writer-close-unverified");
+			} else if (allWriters.some((writer) => writer.kind === "pi-writer" && writer.processTree.state !== "observed")) {
+				proof = unknownProof(runId, runnerClose.processInstanceId, "process-tree-unverified");
 			} else {
 				const runner: ProcessInstanceExitV1 = { kind: "runner", ...runnerClose };
 				const canonicalSession = session && sessionProjection(candidate, session);
@@ -271,6 +284,7 @@ export function finalizeProcessTerminal(
 	try {
 		writeAtomicJson(processTerminalPath(asyncDir), proof);
 		durable = true;
+		if (proof.state === "observed") releaseActiveRunIndex(asyncDir);
 		overlayStatus(asyncDir, proof, candidateForOverlay);
 		fs.appendFileSync(path.join(asyncDir, "events.jsonl"), `${JSON.stringify({ type: "subagent.run.process_terminal", lifecycleArtifactVersion: SUBAGENT_LIFECYCLE_ARTIFACT_VERSION, ts: Date.now(), runId, processTerminal: proof })}\n`, "utf-8");
 	} catch {

package/src/runs/background/resume-guidance.ts

@@ -1,6 +1,19 @@
 import * as fs from "node:fs";
 import type { AsyncRunSummary } from "./async-status.ts";
 
+const INTERCOM_DETACH_ERROR = "detached for intercom coordination";
+
+function isIntercomDetached(run: AsyncRunSummary): boolean {
+	return run.steps.some((step) => step.execution?.status === "detached"
+		|| step.error?.toLowerCase().includes(INTERCOM_DETACH_ERROR) === true)
+		|| run.error?.toLowerCase().includes(INTERCOM_DETACH_ERROR) === true;
+}
+
+function formatIntercomDetachGuidance(run: AsyncRunSummary): string | undefined {
+	if (!isIntercomDetached(run)) return undefined;
+	return `Run "${run.id}" detached for intercom coordination. Reply to the supervisor request first, then wait with subagent_wait({ id: "${run.id}" }). Use subagent({ action: "status", id: "${run.id}" }) to recover the result; do not resume or launch a replacement while it remains detached.`;
+}
+
 export function formatAsyncReviveCommand(run: AsyncRunSummary): string | undefined {
 	const step = run.steps.find((candidate) => candidate.status === "failed" && candidate.sessionFile && fs.existsSync(candidate.sessionFile));
 	if (!step) {
@@ -15,19 +28,26 @@ export function formatAsyncReviveCommand(run: AsyncRunSummary): string | undefin
 
 export function formatResumeFirstFailedRunDetail(run: AsyncRunSummary): string | undefined {
 	if (run.state !== "failed") return undefined;
+	const detachGuidance = formatIntercomDetachGuidance(run);
+	if (detachGuidance) return detachGuidance;
 	const command = formatAsyncReviveCommand(run);
 	if (!command) return undefined;
 	return `Resume-first: failed run "${run.id}" has a persisted child session. Revive the original run with ${command} before reporting failure or launching a replacement. Launch a replacement only if revive fails or the user explicitly asks for one.`;
 }
 
 export function formatResumeFirstFailedRunsNote(runs: AsyncRunSummary[]): string {
-	const resumable = runs
-		.filter((run) => run.state === "failed")
+	const failedRuns = runs.filter((run) => run.state === "failed");
+	const detachGuidance = failedRuns
+		.map(formatIntercomDetachGuidance)
+		.filter((guidance): guidance is string => Boolean(guidance));
+	const resumable = failedRuns
+		.filter((run) => !isIntercomDetached(run))
 		.map((run) => ({ run, command: formatAsyncReviveCommand(run) }))
 		.filter((entry): entry is { run: AsyncRunSummary; command: string } => Boolean(entry.command));
-	if (resumable.length === 0) return "";
-	const guidance = resumable.length === 1
-		? `failed run "${resumable[0]!.run.id}" has a persisted child session. Revive the original run with ${resumable[0]!.command}`
-		: `${resumable.length} failed runs have persisted child sessions. Inspect status and revive each original run before retrying`;
-	return ` Resume-first: ${guidance} before reporting failure or launching a replacement. Launch a replacement only if revive fails or the user explicitly asks for one.`;
+	const resumeGuidance = resumable.length === 0
+		? ""
+		: resumable.length === 1
+			? ` Resume-first: failed run "${resumable[0]!.run.id}" has a persisted child session. Revive the original run with ${resumable[0]!.command} before reporting failure or launching a replacement. Launch a replacement only if revive fails or the user explicitly asks for one.`
+			: ` Resume-first: ${resumable.length} failed runs have persisted child sessions. Inspect status and revive each original run before reporting failure or launching a replacement. Launch a replacement only if revive fails or the user explicitly asks for one.`;
+	return `${detachGuidance.length > 0 ? ` ${detachGuidance.join(" ")}` : ""}${resumeGuidance}`;
 }

package/src/runs/background/retained-children.ts

@@ -8,6 +8,8 @@ const MAX_TASK_SUMMARY_LENGTH = 120;
 export interface RetainedChild {
 	runId: string;
 	parentRunId?: string;
+	workflowKey?: string;
+	state: "complete" | "failed" | "paused";
 	agent: string;
 	taskSummary: string;
 	completedAt: number;
@@ -33,16 +35,20 @@ function boundedTaskSummary(value: string | undefined): string {
 }
 
 export function listRetainedChildren(asyncDirRoot: string, sessionId: string): RetainedChild[] {
-	return listAsyncRuns(asyncDirRoot, { sessionId, states: ["complete"], reconcile: false })
+	return listAsyncRuns(asyncDirRoot, { sessionId, states: ["complete", "failed", "paused"], reconcile: false })
 		.flatMap((run) => {
 			if (!run.parentWorkflowRunId || run.steps.length !== 1) return [];
+			if (run.state !== "complete" && run.state !== "failed" && run.state !== "paused") return [];
 			const step = run.steps[0]!;
-			if ((step.status !== "complete" && step.status !== "completed") || !retainedSessionFile(step.sessionFile ?? run.sessionFile)) return [];
-			const completedAt = run.endedAt;
+			const terminalStep = step.status === "complete" || step.status === "completed" || step.status === "failed" || step.status === "paused";
+			if (!terminalStep || !retainedSessionFile(step.sessionFile ?? run.sessionFile)) return [];
+			const completedAt = run.endedAt ?? run.lastUpdate;
 			if (completedAt === undefined) return [];
 			return [{
 				runId: run.id,
 				...(run.parentWorkflowRunId ? { parentRunId: run.parentWorkflowRunId } : {}),
+				...(run.workflowKey ? { workflowKey: run.workflowKey } : {}),
+				state: run.state,
 				agent: step.agent,
 				taskSummary: boundedTaskSummary(step.description),
 				completedAt,
@@ -55,13 +61,15 @@ export function listRetainedChildren(asyncDirRoot: string, sessionId: string): R
 }
 
 export function formatRetainedChildren(children: RetainedChild[]): string {
-	if (children.length === 0) return "No completed retained children in the active parent session.";
+	if (children.length === 0) return "No recoverable retained children in the active parent session.";
 	return [
-		`Completed retained children (newest first, last ${MAX_RETAINED_CHILDREN}):`,
+		`Recoverable retained children (newest first, last ${MAX_RETAINED_CHILDREN}):`,
 		...children.flatMap((child) => [
-			`- ${child.runId} | ${child.agent} | ${new Date(child.completedAt).toISOString()}`,
+			`- ${child.runId} | ${child.agent} | ${child.state} | ${new Date(child.completedAt).toISOString()}`,
+			...(child.parentRunId ? [`  workflow: ${child.parentRunId}${child.workflowKey ? ` (${child.workflowKey})` : ""}`] : []),
 			`  task: ${child.taskSummary || "(no task summary)"}`,
 			`  session: ${child.sessionPath}`,
+			`  resume: subagent({ action: "resume", id: "${child.runId}", message: "..." })`,
 			...(child.tokenTotals ? [`  tokens: input ${child.tokenTotals.input}, output ${child.tokenTotals.output}, total ${child.tokenTotals.total}`] : []),
 		]),
 	].join("\n");

package/src/runs/background/run-status.ts

@@ -1,12 +1,14 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import type { AgentToolResult } from "@earendil-works/pi-agent-core";
+import { safeTerminalText } from "../../shared/display-text.ts";
 import { formatAsyncRunList, formatAsyncRunOutputPath, formatAsyncRunProgressLabel, listAsyncRuns } from "./async-status.ts";
 import { formatAsyncResultTranscript, formatAsyncRunTranscript, formatNestedRunTranscript, inspectSubagentFleet } from "./fleet-view.ts";
 import { formatNestedRunStatusLines } from "../shared/nested-render.ts";
 import { formatModelThinking } from "../../shared/formatters.ts";
 import { formatActivityLabel } from "../../shared/status-format.ts";
 import { DIRS, type AsyncStatus, type Details, type ForegroundResumeRun, type NestedRunSummary, type SteeringStatus, type SubagentState } from "../../shared/types.ts";
+import { inspectActiveAsyncCapacityOwner, type ActiveAsyncCapacityInspection } from "./active-async-capacity.ts";
 import { readStatus } from "../../shared/utils.ts";
 import { resolveSubagentIntercomTarget } from "../../intercom/intercom-bridge.ts";
 import { resolveSubagentResultStatus } from "../../intercom/result-intercom.ts";
@@ -19,6 +21,7 @@ import { reconcileAsyncRun, reconcileNestedAsyncDescendants } from "./stale-run-
 import { attachRootChildrenToSteps, findNestedRouteForRootId, projectNestedRegistryForRoot, type NestedRunResolutionScope } from "../shared/nested-events.ts";
 import { readMissionBinding } from "../../missions/lifecycle.ts";
 import { formatWorkflowJsonPreview } from "../../workflows/scripted-workflow.ts";
+import { formatRunFanoutBudget, getRunFanoutBudgetSnapshot, readRunFanoutBudgetDescriptor } from "../shared/run-fanout-budget.ts";
 
 interface RunStatusParams {
 	action?: string;
@@ -30,6 +33,66 @@ interface RunStatusParams {
 	lines?: number;
 }
 
+function formatProcessTerminal(value: AsyncStatus["processTerminal"] | undefined): string {
+	if (!value) return "missing";
+	return `${value.state}${value.reason ? ` (${value.reason})` : ""}${value.runnerProcessInstanceId ? ` · runner ${value.runnerProcessInstanceId}` : ""}`;
+}
+
+function debugProcessTerminal(asyncDir: string, status: AsyncStatus): { sidecar?: AsyncStatus["processTerminal"]; overlay?: AsyncStatus["processTerminal"] } {
+	const expected = { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId };
+	return {
+		sidecar: readProcessTerminal(asyncDir, expected),
+		overlay: sanitizeProcessTerminal(status.processTerminal, expected, path.join(asyncDir, "status.json")),
+	};
+}
+
+function formatCapacityOwner(inspect: ActiveAsyncCapacityInspection): string[] {
+	if (!inspect.owner) return [`Active capacity: ${inspect.release.state} — ${inspect.release.reason}`];
+	const owner = inspect.owner;
+	return [
+		`Active capacity: ${inspect.release.state} — ${inspect.release.reason}`,
+		`Capacity owner: ${inspect.relation} slot ${owner.slot}, ${owner.kind}, generation ${owner.generation}`,
+		`Capacity session: ${owner.ownerSessionId}`,
+		owner.sourceRunId ? `Capacity source run: ${owner.sourceRunId}` : undefined,
+		`Capacity async dir: ${owner.asyncDir}`,
+		owner.runnerProcessInstanceId ? `Capacity runner: ${owner.runnerProcessInstanceId}` : undefined,
+		owner.runnerStartedAt !== undefined ? `Capacity runner started: ${new Date(owner.runnerStartedAt).toISOString()}` : undefined,
+	].filter((line): line is string => line !== undefined);
+}
+
+function formatWorkflowDebug(status: AsyncStatus): string[] {
+	if (status.mode !== "workflow" && !status.parentWorkflowRunId && !status.workflowKey) return [];
+	const lines = [
+		status.parentWorkflowRunId ? `Workflow parent: ${status.parentWorkflowRunId}${status.workflowKey ? ` (${status.workflowKey})` : ""}` : undefined,
+		status.mode === "workflow" ? `Workflow children: ${(status.steps ?? []).length}` : undefined,
+	].filter((line): line is string => line !== undefined);
+	for (const [index, step] of (status.steps ?? []).entries()) {
+		lines.push(`  ${index + 1}. key ${step.workflowKey ?? "n/a"} · ${step.agent} · ${step.status} · async ${step.async === undefined ? "unknown" : step.async ? "yes" : "no"}${step.runId ? ` · run ${step.runId}` : ""}`);
+	}
+	return lines;
+}
+
+function formatRunLifecycleDebug(input: { status: AsyncStatus; asyncDir: string; sidecarProcessTerminal: AsyncStatus["processTerminal"] | undefined; overlayProcessTerminal: AsyncStatus["processTerminal"] | undefined; capacity: ActiveAsyncCapacityInspection }): string {
+	const { status, asyncDir, sidecarProcessTerminal, overlayProcessTerminal, capacity } = input;
+	const lines = [
+		"Run lifecycle debug",
+		`Run: ${status.runId}`,
+		`Dir: ${asyncDir}`,
+		`Status file: ${path.join(asyncDir, "status.json")}`,
+		`Process terminal file: ${path.join(asyncDir, "process-terminal.json")}`,
+		`Session: ${status.sessionId ?? "unknown"}`,
+		`State: ${status.state}`,
+		`Mode: ${status.mode}`,
+		status.parentWorkflowRunId ? `Workflow parent: ${status.parentWorkflowRunId}` : undefined,
+		status.workflowKey ? `Workflow key: ${status.workflowKey}` : undefined,
+		`Status process terminal: ${formatProcessTerminal(overlayProcessTerminal)}`,
+		`Sidecar process terminal: ${formatProcessTerminal(sidecarProcessTerminal)}`,
+		...formatCapacityOwner(capacity),
+		...formatWorkflowDebug(status),
+	].filter((line): line is string => line !== undefined);
+	return lines.join("\n");
+}
+
 interface RunStatusDeps {
 	asyncDirRoot?: string;
 	resultsDir?: string;
@@ -38,6 +101,7 @@ interface RunStatusDeps {
 	state?: SubagentState;
 	nested?: NestedRunResolutionScope;
 	sessionRoots?: string[];
+	activeCapacityRoot?: string;
 }
 
 function hasExistingSessionFile(value: unknown): value is string {
@@ -49,12 +113,16 @@ function formatCheckpointGuidance(runId: string | undefined, checkpoint: AsyncSt
 	return `Checkpoint: ${checkpoint.name}${checkpoint.message ? ` — ${checkpoint.message}` : ""}\nApprove: subagent({ action: "approve-checkpoint", id: "${runId}" })\nReject: subagent({ action: "reject-checkpoint", id: "${runId}" })`;
 }
 
-function formatResumeGuidance(runId: string | undefined, children: Array<{ agent?: unknown; sessionFile?: unknown }>, fallbackSessionFile?: unknown, options: { stopped?: boolean } = {}): string {
+function formatResumeGuidance(runId: string | undefined, children: Array<{ agent?: unknown; sessionFile?: unknown; runId?: unknown; workflowKey?: unknown }>, fallbackSessionFile?: unknown, options: { stopped?: boolean } = {}): string {
 	if (options.stopped) return "Resume: unavailable; stopped runs are not resumable. Start a new run instead.";
 	const knownChildren = children
 		.map((child, index) => ({ child, index }))
 		.filter(({ child }) => typeof child.agent === "string");
 	if (!runId || knownChildren.length === 0) return "Resume: unavailable; no child session file was persisted.";
+	const workflowChildren = knownChildren.filter(({ child }) => typeof child.runId === "string" && child.runId.trim() && hasExistingSessionFile(child.sessionFile));
+	if (workflowChildren.length > 0) {
+		return workflowChildren.map(({ child }) => `Revive workflow child${typeof child.workflowKey === "string" && child.workflowKey.trim() ? ` '${child.workflowKey}'` : ""}: subagent({ action: "resume", id: "${child.runId}", message: "..." })`).join("\n");
+	}
 	const singleSessionFile = knownChildren[0]?.child.sessionFile ?? fallbackSessionFile;
 	if (children.length === 1 && knownChildren.length === 1 && hasExistingSessionFile(singleSessionFile)) {
 		return `Revive: subagent({ action: "resume", id: "${runId}", message: "..." })`;
@@ -177,7 +245,7 @@ function formatRememberedForegroundTranscript(run: ForegroundResumeRun, options:
 	lines.push("Result transcript tail:");
 	if (outputLines.length === 0) lines.push("  (no recovered final output available yet)");
 	else for (const line of outputLines) lines.push(`  ${line}`);
-	return lines.join("\n");
+	return lines.map((line) => safeTerminalText(line)).join("\n");
 }
 
 function formatNestedExactStatus(rootRunId: string, run: NestedRunSummary): string {
@@ -262,7 +330,9 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 	let location;
 	try {
 		const requestedId = params.id ?? params.runId;
-		if (!params.dir && requestedId) {
+		if (params.action === "debug.run") {
+			location = resolveAsyncRunLocation(params, asyncDirRoot, resultsDir);
+		} else if (!params.dir && requestedId) {
 			const resolved = resolveSubagentRunId(requestedId, { asyncDirRoot, resultsDir, state: deps.state, nested: deps.nested });
 			if (resolved?.kind === "foreground") {
 				const run = deps.state?.foregroundRuns?.get(resolved.id);
@@ -330,6 +400,14 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 		}
 		const status = reconciliation.status;
 		if (!status && diskStatus?.displayDismissedAt !== undefined) {
+			if (params.action === "debug.run") {
+				const { sidecar, overlay } = debugProcessTerminal(asyncDir, diskStatus);
+				const capacity = inspectActiveAsyncCapacityOwner({ runId: diskStatus.runId, sessionId: diskStatus.sessionId, asyncDir }, { rootDir: deps.activeCapacityRoot, liveWorkflowRunIds: new Set(deps.state?.workflowControllers?.keys() ?? []) });
+				return {
+					content: [{ type: "text", text: formatRunLifecycleDebug({ status: diskStatus, asyncDir, sidecarProcessTerminal: sidecar, overlayProcessTerminal: overlay, capacity }) }],
+					details: { mode: "single", results: [], ...((sidecar ?? overlay) ? { lifecycleStatus: { processTerminal: sidecar ?? overlay } } : {}) },
+				};
+			}
 			if (params.view === "transcript") {
 				if (currentSessionId && diskStatus.sessionId !== currentSessionId) {
 					return {
@@ -349,6 +427,14 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 		const logPath = path.join(asyncDir, `subagent-log-${effectiveRunId}.md`);
 		const eventsPath = path.join(asyncDir, "events.jsonl");
 		if (status) {
+			if (params.action === "debug.run") {
+				const { sidecar, overlay } = debugProcessTerminal(asyncDir, status);
+				const capacity = inspectActiveAsyncCapacityOwner({ runId: status.runId, sessionId: status.sessionId, asyncDir }, { rootDir: deps.activeCapacityRoot, liveWorkflowRunIds: new Set(deps.state?.workflowControllers?.keys() ?? []) });
+				return {
+					content: [{ type: "text", text: formatRunLifecycleDebug({ status, asyncDir, sidecarProcessTerminal: sidecar, overlayProcessTerminal: overlay, capacity }) }],
+					details: { mode: "single", results: [], ...((sidecar ?? overlay) ? { lifecycleStatus: { processTerminal: sidecar ?? overlay } } : {}) },
+				};
+			}
 			if (params.view === "transcript") {
 				if (currentSessionId && status.sessionId !== currentSessionId) {
 					return {
@@ -389,6 +475,8 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 			const steeringText = formatSteeringSummary(status);
 			const processTerminal = readProcessTerminal(asyncDir, { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId })
 				?? sanitizeProcessTerminal(status.processTerminal, { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId }, path.join(asyncDir, "status.json"));
+			const runFanoutBudgetDescriptor = readRunFanoutBudgetDescriptor(asyncDir);
+			const runFanoutBudget = runFanoutBudgetDescriptor ? getRunFanoutBudgetSnapshot(runFanoutBudgetDescriptor) : status.runFanoutBudget;
 			let missionId: string | undefined;
 			try {
 				missionId = readMissionBinding(asyncDir)?.missionId;
@@ -410,6 +498,7 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 				statusActivityText ? `Activity: ${statusActivityText}` : undefined,
 				steeringText ? `Steering: ${steeringText}` : undefined,
 				`Mode: ${status.mode}`,
+				runFanoutBudget ? formatRunFanoutBudget(runFanoutBudget) : undefined,
 				status.parentWorkflowRunId ? `Workflow parent: ${status.parentWorkflowRunId}${status.workflowKey ? ` (${status.workflowKey})` : ""}` : undefined,
 				status.mode === "workflow" && workflowReturnPreview !== undefined ? `Return: ${workflowReturnPreview}` : undefined,
 				status.mode === "workflow" && workflowEmitPreview !== undefined ? `Latest emit: ${workflowEmitPreview}` : undefined,
@@ -466,14 +555,21 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 			if (fs.existsSync(logPath)) lines.push(`Log: ${logPath}`);
 			if (fs.existsSync(eventsPath)) lines.push(`Events: ${eventsPath}`);
 
-			return { content: [{ type: "text", text: lines.join("\n") }], details: { mode: "single", results: [], ...(processTerminal ? { lifecycleStatus: { processTerminal } } : {}) } };
+			return { content: [{ type: "text", text: lines.join("\n") }], details: { mode: "single", results: [], ...(runFanoutBudget ? { runFanoutBudget } : {}), ...(processTerminal ? { lifecycleStatus: { processTerminal } } : {}) } };
 		}
 	}
 
 	if (resultPath) {
+		if (params.action === "debug.run") {
+			return {
+				content: [{ type: "text", text: "Run lifecycle debug needs an async run directory with status.json." }],
+				isError: true,
+				details: { mode: "single", results: [] },
+			};
+		}
 		try {
 			const raw = fs.readFileSync(resultPath, "utf-8");
-			const data = JSON.parse(raw) as { id?: string; runId?: string; toolCallId?: string; agent?: string; success?: boolean; summary?: string; output?: string; exitCode?: number; state?: string; stopped?: boolean; timedOut?: boolean; turnBudgetExceeded?: boolean; processSignal?: string | null; sessionFile?: string; parallelHandoff?: { path?: string }; results?: Array<{ agent?: string; output?: string; summary?: string; sessionFile?: string; state?: string; success?: boolean; exitCode?: number | null; stopped?: boolean; timedOut?: boolean; turnBudgetExceeded?: boolean; interrupted?: boolean; processSignal?: string | null }> };
+			const data = JSON.parse(raw) as { id?: string; runId?: string; toolCallId?: string; agent?: string; success?: boolean; summary?: string; output?: string; exitCode?: number; state?: string; stopped?: boolean; timedOut?: boolean; turnBudgetExceeded?: boolean; processSignal?: string | null; sessionFile?: string; parallelHandoff?: { path?: string }; results?: Array<{ agent?: string; runId?: string; workflowKey?: string; output?: string; summary?: string; sessionFile?: string; state?: string; success?: boolean; exitCode?: number | null; stopped?: boolean; timedOut?: boolean; turnBudgetExceeded?: boolean; interrupted?: boolean; processSignal?: string | null }> };
 			if (params.view === "transcript") {
 				try {
 					return { content: [{ type: "text", text: formatAsyncResultTranscript(data, resultPath, { index: params.index, lines: params.lines }) }], details: { mode: "single", results: [] } };

package/src/runs/background/stale-run-reconciler.ts

@@ -1,7 +1,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { writeAtomicJson } from "../../shared/atomic-json.ts";
-import { updateActiveRunIndex } from "./active-run-index.ts";
+import { releaseActiveRunIndex, updateActiveRunIndex } from "./active-run-index.ts";
 import { readStatus } from "../../shared/utils.ts";
 import { DIRS, type AsyncParallelGroupStatus, type AsyncStatus, type NestedRunSummary, type SubagentRunMode } from "../../shared/types.ts";
 import { resolveEffectiveThinking } from "../../shared/model-info.ts";
@@ -261,7 +261,7 @@ function writeFailedRepair(asyncDir: string, status: AsyncStatus, resultPath: st
 	const repair = buildFailedRepair(status, asyncDir, now, reason);
 	writeAtomicJson(resultPath, repair.result);
 	writeAtomicJson(path.join(asyncDir, "status.json"), repair.status);
-	updateActiveRunIndex(asyncDir, repair.status.state);
+	if (repair.status.processTerminal?.state === "observed") releaseActiveRunIndex(asyncDir);
 	appendJsonlBestEffort(path.join(asyncDir, "events.jsonl"), {
 		type: "subagent.run.repaired_stale",
 		ts: now,
@@ -351,7 +351,7 @@ export function reconcileAsyncRun(asyncDir: string, options: ReconcileAsyncRunOp
 			: undefined;
 		if (terminalStatus) {
 			writeAtomicJson(path.join(asyncDir, "status.json"), terminalStatus);
-			updateActiveRunIndex(asyncDir, terminalStatus.state);
+			if (terminalStatus.processTerminal?.state === "observed") releaseActiveRunIndex(asyncDir);
 			return { status: terminalStatus, repaired: true, resultPath, message: "Existing async result file was used to repair stale running status." };
 		}
 		if (effectiveStatus.displayDismissedAt === undefined) return { status: effectiveStatus, repaired: false, resultPath };