pi-subagents 0.47.0 → 0.48.0

package/src/runs/background/async-job-tracker.ts

@@ -15,14 +15,15 @@ import {
 	SUBAGENT_CONTROL_INTERCOM_EVENT,
 	SUBAGENT_STEERING_NOTICE_EVENT,
 } from "../../shared/types.ts";
-import { readStatus } from "../../shared/utils.ts";
+import { readStatus, resolveWatchPath } from "../../shared/utils.ts";
 import { normalizeParallelGroups } from "./parallel-groups.ts";
 import { reconcileAsyncRun, reconcileNestedAsyncDescendants } from "./stale-run-reconciler.ts";
-import { hasLiveNestedDescendants, updateAsyncJobNestedProjection } from "../shared/nested-events.ts";
+import { findNestedRouteForRootId, hasLiveNestedDescendants, updateAsyncJobNestedProjection } from "../shared/nested-events.ts";
 import { listAsyncRuns, type AsyncRunSummary } from "./async-status.ts";
 
 interface AsyncJobTrackerOptions {
 	completionRetentionMs?: number;
+	/** Slow safety sweep for liveness repair when filesystem events are missed. */
 	pollIntervalMs?: number;
 	resultsDir?: string;
 	widgetEnabled?: boolean;
@@ -34,6 +35,9 @@ const CONTROL_EVENT_READ_CHUNK_BYTES = 64 * 1024;
 const MAX_CONTROL_EVENT_LINE_BYTES = 1024 * 1024;
 const CONTROL_EVENT_SCAN_WINDOW_BYTES = 2 * 1024 * 1024;
 const MAX_RECENT_FLEET_JOBS = 20;
+const DEFAULT_LIVENESS_INTERVAL_MS = 5000;
+const EVENT_REFRESH_DEBOUNCE_MS = 25;
+const WATCH_ATTACHMENT_RETRY_MS = 100;
 
 function rememberFleetJob(state: SubagentState, job: AsyncJobState): void {
 	state.fleetJobs ??= new Map();
@@ -51,11 +55,17 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 	handleComplete: (data: unknown) => void;
 	resetJobs: (ctx?: ExtensionContext) => void;
 	restoreActiveJobs: (ctx?: ExtensionContext) => void;
+	dispose: () => void;
 } {
 	const completionRetentionMs = options.completionRetentionMs ?? 10000;
-	const pollIntervalMs = options.pollIntervalMs ?? POLL_INTERVAL_MS;
+	const livenessIntervalMs = options.pollIntervalMs ?? DEFAULT_LIVENESS_INTERVAL_MS;
 	const resultsDir = options.resultsDir ?? DIRS.results;
 	const steeringNoticeSeen = new Map<string, number>();
+	const jobWatchers = new Map<string, { watchers: Map<string, fs.FSWatcher>; retryTimer?: ReturnType<typeof setTimeout> }>();
+	const refreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
+	const runningJobIds = new Set<string>();
+	let rootWatcher: fs.FSWatcher | undefined;
+	let nextLivenessAt = Date.now() + livenessIntervalMs;
 	const rerenderWidget = (ctx: ExtensionContext, jobs = Array.from(state.asyncJobs.values())) => {
 		if (state.widgetsSuspended) return;
 		renderWidget(ctx, options.widgetEnabled === false ? [] : jobs);
@@ -96,6 +106,14 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 			throw error;
 		}
 	};
+	const findNestedRouteForJob = (runId: string) => {
+		try {
+			return findNestedRouteForRootId(runId);
+		} catch (error) {
+			console.error(`Failed to resolve nested event route for '${runId}':`, error);
+			return undefined;
+		}
+	};
 	const summaryToJob = (run: AsyncRunSummary): AsyncJobState => {
 		const groups = normalizeParallelGroups(run.parallelGroups, run.steps.length, run.chainStepCount ?? run.steps.length);
 		const activeGroup = run.currentStep !== undefined
@@ -144,6 +162,7 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 			totalTokens: run.totalTokens,
 			sessionFile: run.sessionFile,
 			controlEventCursor: restoredControlEventCursor(run.asyncDir),
+			nestedRoute: findNestedRouteForJob(run.id),
 			nestedChildren: run.nestedChildren,
 			parentWorkflowRunId: run.parentWorkflowRunId,
 			workflowKey: run.workflowKey,
@@ -160,6 +179,7 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 		cancelCleanup(asyncId);
 		const timer = setTimeout(() => {
 			state.cleanupTimers.delete(asyncId);
+			closeJobWatcher(asyncId);
 			state.asyncJobs.delete(asyncId);
 			rerenderLastWidget();
 		}, completionRetentionMs);
@@ -282,140 +302,245 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 		}
 	};
 
+	const closeJobWatcher = (asyncId: string) => {
+		const watched = jobWatchers.get(asyncId);
+		if (watched?.retryTimer) clearTimeout(watched.retryTimer);
+		for (const watcher of watched?.watchers.values() ?? []) watcher.close();
+		jobWatchers.delete(asyncId);
+		const timer = refreshTimers.get(asyncId);
+		if (timer) clearTimeout(timer);
+		refreshTimers.delete(asyncId);
+		runningJobIds.delete(asyncId);
+	};
+
+	const refreshJob = (job: AsyncJobState): boolean => {
+		const widgetStateBefore = widgetRenderKey(job);
+		let nestedRefreshFailed = false;
+		const refreshNestedProjection = () => {
+			try {
+				updateAsyncJobNestedProjection(job);
+			} catch (error) {
+				nestedRefreshFailed = true;
+				console.error(`Failed to refresh nested async descendants for '${job.asyncDir}':`, error);
+			}
+		};
+		try {
+			emitNewControlEvents(job);
+			try {
+				if (job.nestedRoute) reconcileNestedAsyncDescendants(job.nestedRoute, { resultsDir, kill: options.kill, now: options.now });
+			} catch (error) {
+				nestedRefreshFailed = true;
+				console.error(`Failed to refresh nested async descendants for '${job.asyncDir}':`, error);
+			}
+			refreshNestedProjection();
+			const reconciliation = reconcileAsyncRun(job.asyncDir, {
+				resultsDir,
+				kill: options.kill,
+				now: options.now,
+				startedRun: {
+					runId: job.asyncId,
+					pid: job.pid,
+					sessionId: job.sessionId,
+					mode: job.mode,
+					agents: job.agents,
+					chainStepCount: job.chainStepCount,
+					parallelGroups: job.parallelGroups,
+					startedAt: job.startedAt,
+					sessionFile: job.sessionFile,
+				},
+			});
+			const status = reconciliation.status ?? readStatus(job.asyncDir);
+			if (status) {
+				const previousStatus = job.status;
+				job.status = status.state;
+				if (job.status === "running") runningJobIds.add(job.asyncId);
+				else runningJobIds.delete(job.asyncId);
+				if (job.status !== "complete" && job.status !== "failed" && job.status !== "paused" && job.status !== "stopped") cancelCleanup(job.asyncId);
+				job.sessionId = status.sessionId ?? job.sessionId;
+				job.activityState = status.activityState;
+				job.lastActivityAt = status.lastActivityAt ?? job.lastActivityAt;
+				job.currentTool = status.currentTool;
+				job.currentToolStartedAt = status.currentToolStartedAt;
+				job.currentPath = status.currentPath;
+				job.turnCount = status.turnCount ?? job.turnCount;
+				job.toolCount = status.toolCount ?? job.toolCount;
+				job.steering = status.steering ?? job.steering;
+				job.mode = status.mode;
+				job.parentWorkflowRunId = status.parentWorkflowRunId ?? job.parentWorkflowRunId;
+				job.workflowKey = status.workflowKey ?? job.workflowKey;
+				job.workflow = status.workflow ?? job.workflow;
+				job.currentStep = status.currentStep ?? job.currentStep;
+				job.chainStepCount = status.chainStepCount ?? job.chainStepCount;
+				job.startedAt = status.startedAt ?? job.startedAt;
+				if (status.lastUpdate !== undefined) job.updatedAt = status.lastUpdate;
+				if (status.steps?.length) {
+					const groups = normalizeParallelGroups(status.parallelGroups, status.steps.length, status.chainStepCount ?? status.steps.length);
+					job.parallelGroups = groups.length ? groups : job.parallelGroups;
+					job.hasParallelGroups = groups.length > 0 || job.hasParallelGroups;
+					const activeGroup = status.currentStep !== undefined
+						? groups.find((group) => status.currentStep! >= group.start && status.currentStep! < group.start + group.count)
+						: undefined;
+					const visibleSteps = activeGroup
+						? status.steps.slice(activeGroup.start, activeGroup.start + activeGroup.count).map((step, index) => ({ ...step, index: activeGroup.start + index }))
+						: status.steps.map((step, index) => ({ ...step, index }));
+					job.activeParallelGroup = Boolean(activeGroup);
+					job.agents = visibleSteps.map((step) => step.agent);
+					job.steps = visibleSteps;
+					refreshNestedProjection();
+					job.stepsTotal = visibleSteps.length;
+					job.runningSteps = visibleSteps.filter((step) => step.status === "running").length;
+					job.completedSteps = visibleSteps.filter((step) => step.status === "complete" || step.status === "completed").length;
+					if (status.state === "complete") job.completedSteps = visibleSteps.length;
+				}
+				job.sessionDir = status.sessionDir ?? job.sessionDir;
+				job.outputFile = status.outputFile ?? job.outputFile;
+				job.totalTokens = status.totalTokens ?? job.totalTokens;
+				job.timeoutMs = status.timeoutMs ?? job.timeoutMs;
+				job.deadlineAt = status.deadlineAt ?? job.deadlineAt;
+				job.timedOut = status.timedOut ?? job.timedOut;
+				job.stopped = status.stopped ?? job.stopped;
+				job.turnBudget = status.turnBudget ?? job.turnBudget;
+				job.turnBudgetExceeded = status.turnBudgetExceeded ?? job.turnBudgetExceeded;
+				job.wrapUpRequested = status.wrapUpRequested ?? job.wrapUpRequested;
+				job.sessionFile = status.sessionFile ?? job.sessionFile;
+				if (job.status === "complete" || job.status === "failed" || job.status === "paused" || job.status === "stopped") {
+					rememberFleetJob(state, job);
+					if (!nestedRefreshFailed && !hasLiveNestedDescendants(job.nestedChildren) && (previousStatus !== job.status || !state.cleanupTimers.has(job.asyncId))) {
+						scheduleCleanup(job.asyncId);
+					}
+				}
+				return widgetRenderKey(job) !== widgetStateBefore;
+			}
+			if (job.status === "queued") {
+				job.status = "running";
+				job.updatedAt = Date.now();
+				runningJobIds.add(job.asyncId);
+			}
+		} catch (error) {
+			if (job.status !== "failed") {
+				console.error(`Failed to read async status for '${job.asyncDir}':`, error);
+				job.status = "failed";
+				job.updatedAt = Date.now();
+			}
+			runningJobIds.delete(job.asyncId);
+			rememberFleetJob(state, job);
+			if (!hasLiveNestedDescendants(job.nestedChildren) && !state.cleanupTimers.has(job.asyncId)) scheduleCleanup(job.asyncId);
+		}
+		return widgetRenderKey(job) !== widgetStateBefore;
+	};
+
+	const scheduleJobRefresh = (asyncId: string, delayMs = EVENT_REFRESH_DEBOUNCE_MS) => {
+		if (refreshTimers.has(asyncId)) return;
+		const timer = setTimeout(() => {
+			refreshTimers.delete(asyncId);
+			const job = state.asyncJobs.get(asyncId);
+			if (job && refreshJob(job)) rerenderLastWidget();
+		}, delayMs);
+		timer.unref?.();
+		refreshTimers.set(asyncId, timer);
+	};
+
+	const watchJob = (job: AsyncJobState) => {
+		const watched = jobWatchers.get(job.asyncId) ?? { watchers: new Map<string, fs.FSWatcher>() };
+		const active = job.status === "queued" || job.status === "running";
+		const watchPath = (watchPath: string): boolean => {
+			if (watched.watchers.has(watchPath)) return true;
+			const nestedEventSink = job.nestedRoute?.eventSink === watchPath;
+			let watcher: fs.FSWatcher;
+			try {
+				watcher = fs.watch(resolveWatchPath(watchPath), (_event, file) => {
+					const rawFileName = file?.toString();
+					const fileName = rawFileName ?? path.basename(watchPath);
+					const nestedEventFile = nestedEventSink && (!rawFileName || rawFileName.endsWith(".json") || rawFileName.endsWith(".jsonl"));
+					if (fileName === "status.json" || fileName === "events.jsonl" || nestedEventFile) {
+						scheduleJobRefresh(job.asyncId);
+					}
+					if (watchPath !== job.asyncDir && !nestedEventSink) {
+						watcher.close();
+						watched.watchers.delete(watchPath);
+						watchJob(job);
+					}
+				});
+			} catch {
+				return false;
+			}
+			watcher.on("error", () => {
+				watcher.close();
+				watched.watchers.delete(watchPath);
+				if (watchPath.endsWith("status.json") || nestedEventSink) watchJob(job);
+			});
+			watcher.unref?.();
+			watched.watchers.set(watchPath, watcher);
+			return true;
+		};
+		watchPath(job.asyncDir);
+		const statusPath = path.join(job.asyncDir, "status.json");
+		const hadStatusWatcher = watched.watchers.has(statusPath);
+		const statusWatched = watchPath(statusPath);
+		if (statusWatched && !hadStatusWatcher) scheduleJobRefresh(job.asyncId);
+		watchPath(path.join(job.asyncDir, "events.jsonl"));
+		if (job.nestedRoute) watchPath(job.nestedRoute.eventSink);
+		if (!statusWatched && active && !watched.retryTimer) {
+			watched.retryTimer = setTimeout(() => {
+				watched.retryTimer = undefined;
+				const current = state.asyncJobs.get(job.asyncId);
+				if (current) watchJob(current);
+			}, WATCH_ATTACHMENT_RETRY_MS);
+			watched.retryTimer.unref?.();
+		} else if (statusWatched && watched.retryTimer) {
+			clearTimeout(watched.retryTimer);
+			watched.retryTimer = undefined;
+		}
+		if (watched.watchers.size > 0 || watched.retryTimer) jobWatchers.set(job.asyncId, watched);
+		else jobWatchers.delete(job.asyncId);
+	};
+
+	const watchAsyncRoot = () => {
+		if (rootWatcher) return;
+		try {
+			rootWatcher = fs.watch(resolveWatchPath(asyncDirRoot), (_event, file) => {
+				const runDirName = file?.toString();
+				for (const job of state.asyncJobs.values()) {
+					if (jobWatchers.has(job.asyncId) || (runDirName && path.basename(job.asyncDir) !== runDirName)) continue;
+					watchJob(job);
+					if (jobWatchers.has(job.asyncId)) scheduleJobRefresh(job.asyncId);
+				}
+			});
+			rootWatcher.on("error", () => {
+				rootWatcher?.close();
+				rootWatcher = undefined;
+			});
+			rootWatcher.unref?.();
+		} catch {
+			// The liveness sweep retries if the async root is not available yet.
+		}
+	};
+
 	const ensurePoller = () => {
+		watchAsyncRoot();
 		if (state.poller) return;
+		nextLivenessAt = Date.now() + livenessIntervalMs;
 		state.poller = setInterval(() => {
 			if (state.asyncJobs.size === 0) {
 				rerenderLastWidget([]);
-				if (state.poller) {
-					clearInterval(state.poller);
-					state.poller = null;
-				}
+				if (state.poller) clearInterval(state.poller);
+				state.poller = null;
+				rootWatcher?.close();
+				rootWatcher = undefined;
 				return;
 			}
-
-			let widgetChanged = false;
-			for (const job of state.asyncJobs.values()) {
-				const widgetStateBefore = widgetRenderKey(job);
-				let nestedRefreshFailed = false;
-				const refreshNestedProjection = () => {
-					try {
-						updateAsyncJobNestedProjection(job);
-					} catch (error) {
-						nestedRefreshFailed = true;
-						console.error(`Failed to refresh nested async descendants for '${job.asyncDir}':`, error);
-					}
-				};
-				const reconcileNestedDescendants = () => {
-					try {
-						if (job.nestedRoute) reconcileNestedAsyncDescendants(job.nestedRoute, { resultsDir, kill: options.kill, now: options.now });
-					} catch (error) {
-						nestedRefreshFailed = true;
-						console.error(`Failed to refresh nested async descendants for '${job.asyncDir}':`, error);
-					}
-					refreshNestedProjection();
-				};
-				try {
-					emitNewControlEvents(job);
-					reconcileNestedDescendants();
-					const reconciliation = reconcileAsyncRun(job.asyncDir, {
-						resultsDir,
-						kill: options.kill,
-						now: options.now,
-						startedRun: {
-							runId: job.asyncId,
-							pid: job.pid,
-							sessionId: job.sessionId,
-							mode: job.mode,
-							agents: job.agents,
-							chainStepCount: job.chainStepCount,
-							parallelGroups: job.parallelGroups,
-							startedAt: job.startedAt,
-							sessionFile: job.sessionFile,
-						},
-					});
-					const status = reconciliation.status ?? readStatus(job.asyncDir);
-					if (status) {
-						const previousStatus = job.status;
-						job.status = status.state;
-						if (job.status !== "complete" && job.status !== "failed" && job.status !== "paused" && job.status !== "stopped") cancelCleanup(job.asyncId);
-						job.sessionId = status.sessionId ?? job.sessionId;
-						job.activityState = status.activityState;
-						job.lastActivityAt = status.lastActivityAt ?? job.lastActivityAt;
-						job.currentTool = status.currentTool;
-						job.currentToolStartedAt = status.currentToolStartedAt;
-						job.currentPath = status.currentPath;
-						job.turnCount = status.turnCount ?? job.turnCount;
-						job.toolCount = status.toolCount ?? job.toolCount;
-						job.steering = status.steering ?? job.steering;
-						job.mode = status.mode;
-						job.parentWorkflowRunId = status.parentWorkflowRunId ?? job.parentWorkflowRunId;
-						job.workflowKey = status.workflowKey ?? job.workflowKey;
-						job.workflow = status.workflow ?? job.workflow;
-						job.currentStep = status.currentStep ?? job.currentStep;
-						job.chainStepCount = status.chainStepCount ?? job.chainStepCount;
-						job.startedAt = status.startedAt ?? job.startedAt;
-						if (status.lastUpdate !== undefined) job.updatedAt = status.lastUpdate;
-						if (status.steps?.length) {
-							const groups = normalizeParallelGroups(status.parallelGroups, status.steps.length, status.chainStepCount ?? status.steps.length);
-							job.parallelGroups = groups.length ? groups : job.parallelGroups;
-							job.hasParallelGroups = groups.length > 0 || job.hasParallelGroups;
-							const activeGroup = status.currentStep !== undefined
-								? groups.find((group) => status.currentStep! >= group.start && status.currentStep! < group.start + group.count)
-								: undefined;
-							const visibleSteps = activeGroup
-								? status.steps.slice(activeGroup.start, activeGroup.start + activeGroup.count).map((step, index) => ({ ...step, index: activeGroup.start + index }))
-								: status.steps.map((step, index) => ({ ...step, index }));
-							job.activeParallelGroup = Boolean(activeGroup);
-							job.agents = visibleSteps.map((step) => step.agent);
-							job.steps = visibleSteps;
-							refreshNestedProjection();
-							job.stepsTotal = visibleSteps.length;
-							job.runningSteps = visibleSteps.filter((step) => step.status === "running").length;
-							job.completedSteps = visibleSteps.filter((step) => step.status === "complete" || step.status === "completed").length;
-							if (status.state === "complete") job.completedSteps = visibleSteps.length;
-						}
-						job.sessionDir = status.sessionDir ?? job.sessionDir;
-						job.outputFile = status.outputFile ?? job.outputFile;
-						job.totalTokens = status.totalTokens ?? job.totalTokens;
-						job.timeoutMs = status.timeoutMs ?? job.timeoutMs;
-						job.deadlineAt = status.deadlineAt ?? job.deadlineAt;
-						job.timedOut = status.timedOut ?? job.timedOut;
-						job.stopped = status.stopped ?? job.stopped;
-						job.turnBudget = status.turnBudget ?? job.turnBudget;
-						job.turnBudgetExceeded = status.turnBudgetExceeded ?? job.turnBudgetExceeded;
-						job.wrapUpRequested = status.wrapUpRequested ?? job.wrapUpRequested;
-						job.sessionFile = status.sessionFile ?? job.sessionFile;
-						if (job.status === "complete" || job.status === "failed" || job.status === "paused" || job.status === "stopped") {
-							rememberFleetJob(state, job);
-							if (!nestedRefreshFailed && !hasLiveNestedDescendants(job.nestedChildren) && (previousStatus !== job.status || !state.cleanupTimers.has(job.asyncId))) {
-								scheduleCleanup(job.asyncId);
-							}
-						}
-						if (widgetRenderKey(job) !== widgetStateBefore) widgetChanged = true;
-						continue;
-					}
-					if (job.status === "queued") {
-						job.status = "running";
-						job.updatedAt = Date.now();
-					}
-				} catch (error) {
-					if (job.status !== "failed") {
-						console.error(`Failed to read async status for '${job.asyncDir}':`, error);
-						job.status = "failed";
-						job.updatedAt = Date.now();
-					}
-					rememberFleetJob(state, job);
-					if (!hasLiveNestedDescendants(job.nestedChildren) && !state.cleanupTimers.has(job.asyncId)) {
-						scheduleCleanup(job.asyncId);
-					}
+			const now = Date.now();
+			if (now >= nextLivenessAt) {
+				nextLivenessAt = now + livenessIntervalMs;
+				let widgetChanged = false;
+				for (const job of state.asyncJobs.values()) {
+					watchJob(job);
+					if (refreshJob(job)) widgetChanged = true;
 				}
-				if (widgetRenderKey(job) !== widgetStateBefore) widgetChanged = true;
+				if (widgetChanged) rerenderLastWidget();
 			}
-
-			if (widgetChanged) rerenderLastWidget();
-			else if (Array.from(state.asyncJobs.values()).some((job) => job.status === "running")) requestLastWidgetRender();
-		}, pollIntervalMs);
+			if (runningJobIds.size > 0) requestLastWidgetRender();
+		}, Math.min(POLL_INTERVAL_MS, livenessIntervalMs));
 		state.poller.unref?.();
 	};
 
@@ -457,7 +582,10 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 			workflowKey: info.workflowKey,
 			controlEventCursor: 0,
 		});
-		rememberFleetJob(state, state.asyncJobs.get(info.id)!);
+		const job = state.asyncJobs.get(info.id)!;
+		rememberFleetJob(state, job);
+		watchJob(job);
+		scheduleJobRefresh(info.id, 0);
 		ensurePoller();
 		rerenderLastWidget();
 	};
@@ -471,9 +599,14 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 		let nestedRefreshFailed = false;
 		if (job) {
 			job.status = result.state ?? (result.success ? "complete" : "failed");
+			runningJobIds.delete(asyncId);
 			job.stopped = result.stopped ?? job.stopped;
 			job.updatedAt = Date.now();
-			if (result.asyncDir) job.asyncDir = result.asyncDir;
+			if (result.asyncDir && result.asyncDir !== job.asyncDir) {
+				closeJobWatcher(asyncId);
+				job.asyncDir = result.asyncDir;
+				watchJob(job);
+			}
 			try {
 				updateAsyncJobNestedProjection(job);
 			} catch (error) {
@@ -486,10 +619,20 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 		if (!nestedRefreshFailed && !hasLiveNestedDescendants(job?.nestedChildren)) scheduleCleanup(asyncId);
 	};
 
+	const dispose = () => {
+		if (state.poller) clearInterval(state.poller);
+		state.poller = null;
+		rootWatcher?.close();
+		rootWatcher = undefined;
+		for (const asyncId of jobWatchers.keys()) closeJobWatcher(asyncId);
+		for (const timer of refreshTimers.values()) clearTimeout(timer);
+		refreshTimers.clear();
+		runningJobIds.clear();
+	};
+
 	const resetJobs = (ctx?: ExtensionContext) => {
-		for (const timer of state.cleanupTimers.values()) {
-			clearTimeout(timer);
-		}
+		dispose();
+		for (const timer of state.cleanupTimers.values()) clearTimeout(timer);
 		state.cleanupTimers.clear();
 		state.asyncJobs.clear();
 		state.fleetJobs?.clear();
@@ -515,12 +658,14 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
 		for (const run of runs) {
 			const job = summaryToJob(run);
 			state.asyncJobs.set(run.id, job);
+			if (job.status === "running") runningJobIds.add(job.asyncId);
 			rememberFleetJob(state, job);
+			watchJob(job);
 		}
 		if (runs.length === 0) return;
 		ensurePoller();
 		rerenderLastWidget();
 	};
 
-	return { ensurePoller, refreshWidget, handleStarted, handleComplete, resetJobs, restoreActiveJobs };
+	return { ensurePoller, refreshWidget, handleStarted, handleComplete, resetJobs, restoreActiveJobs, dispose };
 }

package/src/runs/background/async-resume.ts

@@ -5,6 +5,7 @@ import type { AgentConfig } from "../../agents/agents.ts";
 import { validateAcceptanceInput } from "../shared/acceptance.ts";
 import { validateToolBudgetConfig } from "../shared/tool-budget.ts";
 import { intersectSubagentCapabilityCeilings, parseSubagentCapabilityCeiling, type ResolvedSubagentCapabilityCeiling } from "../shared/capability-ceiling.ts";
+import { validateRunFanoutBudgetDescriptor } from "../shared/run-fanout-budget.ts";
 import { resolveTurnBudgetConfig } from "../shared/turn-budget.ts";
 import { reconcileAsyncRun } from "./stale-run-reconciler.ts";
 
@@ -307,7 +308,7 @@ export function readAsyncRecoveryDescriptor(asyncDir: string | undefined): Steer
 		"subagentOnlyExtensions", "mcpDirectTools", "systemPrompt", "systemPromptMode", "inheritProjectContext", "inheritSkills", "skills",
 		"skillPath", "agentFilePath", "completionGuard", "memory", "outputPath", "outputMode", "structuredOutputSchema", "acceptance", "sessionDir", "artifactConfig",
 		"artifactsDir", "maxOutput", "controlConfig", "absoluteDeadlineAt", "initialTurnBudget", "initialToolBudget", "maxSubagentDepth", "share", "capabilityCeiling",
-		"launchResolvedExtensions",
+		"launchResolvedExtensions", "runFanoutBudget",
 	]);
 	for (const field of Object.keys(parsed)) {
 		if (!allowedFields.has(field)) throw new Error(`Invalid async recovery descriptor '${descriptorPath}': unknown field '${field}'.`);
@@ -317,6 +318,11 @@ export function readAsyncRecoveryDescriptor(asyncDir: string | undefined): Steer
 		if (typeof parsed[field] !== "string" || !(parsed[field] as string).trim()) throw new Error(`Invalid async recovery descriptor '${descriptorPath}': ${field} must be a non-empty string.`);
 	}
 	if (parsed.version !== 1) throw new Error(`Invalid async recovery descriptor '${descriptorPath}': version must be 1.`);
+	try {
+		parsed.runFanoutBudget = validateRunFanoutBudgetDescriptor(parsed.runFanoutBudget);
+	} catch (error) {
+		throw new Error(`Invalid async recovery descriptor '${descriptorPath}': ${error instanceof Error ? error.message : String(error)}`);
+	}
 	if (parsed.capabilityCeiling !== undefined) parsed.capabilityCeiling = parseSubagentCapabilityCeiling(parsed.capabilityCeiling, `async recovery descriptor '${descriptorPath}' capabilityCeiling`);
 	if (parsed.agentContract !== undefined) {
 		if (!parsed.agentContract || typeof parsed.agentContract !== "object" || Array.isArray(parsed.agentContract)) throw new Error(`Invalid async recovery descriptor '${descriptorPath}': agentContract must be an object.`);

package/src/runs/background/async-status.ts

@@ -5,12 +5,14 @@ import { formatActivityLabel, formatParallelOutcome } from "../../shared/status-
 import { type ActivityState, type AsyncJobStep, type AsyncParallelGroupStatus, type AsyncStatus, type CostSummary, type Details, type LaunchResolvedChildExtensionsV1, type RuntimeAcknowledgedChildExtensionsV1, type NestedRunSummary, type SteeringStatus, type SubagentRunMode, type TokenUsage, type TurnBudgetState, type UsageBudgetState, type ChainCheckpointState } from "../../shared/types.ts";
 import type { ResolvedSubagentCapabilityCeiling, SubagentCapabilityAudit } from "../shared/capability-ceiling.ts";
 import { pruneStatusCacheForAsyncRoot, readStatus } from "../../shared/utils.ts";
-import { attachRootChildrenToSteps, buildNestedRouteIndex, type NestedRoute, projectNestedEvents } from "../shared/nested-events.ts";
+import { attachRootChildrenToSteps, buildNestedRouteIndex, findNestedRouteForRootId, type NestedRoute, projectNestedEvents } from "../shared/nested-events.ts";
 import { formatNestedRunStatusLines } from "../shared/nested-render.ts";
+import { formatRunFanoutBudget, getRunFanoutBudgetSnapshot, readRunFanoutBudgetDescriptor } from "../shared/run-fanout-budget.ts";
 import { flatToLogicalStepIndex, normalizeParallelGroups } from "./parallel-groups.ts";
 import { contextModeLabel, summarizeContextModes, type ContextMode, type ContextSummary } from "../shared/context-mode.ts";
 import { reconcileAsyncRun, reconcileNestedAsyncDescendants } from "./stale-run-reconciler.ts";
 import { readProcessTerminal, sanitizeProcessTerminal } from "./process-terminal.ts";
+import { ACTIVE_RUN_INDEX_DIR, isActiveAsyncState, readActiveRunIndex, releaseActiveRunIndex, updateActiveRunIndex } from "./active-run-index.ts";
 
 interface AsyncRunStepSummary {
 	index: number;
@@ -105,6 +107,7 @@ export interface AsyncRunSummary {
 	nestedChildren?: NestedRunSummary[];
 	nestedWarnings?: string[];
 	processTerminal?: AsyncStatus["processTerminal"];
+	runFanoutBudget?: AsyncStatus["runFanoutBudget"];
 	launchResolvedExtensions?: LaunchResolvedChildExtensionsV1;
 	runtimeAcknowledgedExtensions?: RuntimeAcknowledgedChildExtensionsV1;
 	capabilityCeiling?: ResolvedSubagentCapabilityCeiling;
@@ -160,7 +163,7 @@ type TargetedAsyncRunResolution =
  * accepting a path whose canonical location escaped the async root.
  */
 export function resolveTargetedAsyncRun(asyncDirRoot: string, id: string, sessionId?: string): TargetedAsyncRunResolution {
-	if (!id || id === "." || id === ".." || path.basename(id) !== id) return { kind: "reject" };
+	if (!id || id === "." || id === ".." || id === ACTIVE_RUN_INDEX_DIR || path.basename(id) !== id) return { kind: "reject" };
 	const asyncDir = path.join(asyncDirRoot, id);
 	let entryStat: fs.Stats;
 	try {
@@ -221,6 +224,8 @@ function statusToSummary(asyncDir: string, status: AsyncStatus & { cwd?: string
 	const { activityState, lastActivityAt } = deriveAsyncActivityState(asyncDir, status);
 	const processTerminal = readProcessTerminal(asyncDir, { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId })
 		?? sanitizeProcessTerminal(status.processTerminal, { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId }, path.join(asyncDir, "status.json"));
+	const runFanoutBudgetDescriptor = readRunFanoutBudgetDescriptor(asyncDir);
+	const runFanoutBudget = runFanoutBudgetDescriptor ? getRunFanoutBudgetSnapshot(runFanoutBudgetDescriptor) : status.runFanoutBudget;
 	const steps = status.steps ?? [];
 	const chainStepCount = status.chainStepCount ?? steps.length;
 	const parallelGroups = normalizeParallelGroups(status.parallelGroups, steps.length, chainStepCount);
@@ -326,6 +331,7 @@ function statusToSummary(asyncDir: string, status: AsyncStatus & { cwd?: string
 		...(nestedChildren.length ? { nestedChildren } : {}),
 		...(nestedWarnings.length ? { nestedWarnings } : {}),
 		...(processTerminal ? { processTerminal } : {}),
+		...(runFanoutBudget ? { runFanoutBudget } : {}),
 		...(status.launchContractDigest ? { launchContractDigest: status.launchContractDigest } : {}),
 		...(status.launchResolvedExtensions ? { launchResolvedExtensions: status.launchResolvedExtensions } : {}),
 		...(status.runtimeAcknowledgedExtensions ? { runtimeAcknowledgedExtensions: status.runtimeAcknowledgedExtensions } : {}),
@@ -371,6 +377,12 @@ function sortRuns(runs: AsyncRunSummary[]): AsyncRunSummary[] {
 export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions = {}): AsyncRunSummary[] {
 	let entries: string[];
 	let scannedCompleteRoot = false;
+	let usedActiveIndex = false;
+	const activeOnly = options.runId === undefined
+		&& options.entryLimit === undefined
+		&& options.states !== undefined
+		&& options.states.length > 0
+		&& options.states.every(isActiveAsyncState);
 	try {
 		if (options.runId !== undefined) {
 			const resolution = resolveTargetedAsyncRun(asyncDirRoot, options.runId, options.sessionId);
@@ -382,8 +394,15 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
 						&& resolveTargetedAsyncRun(asyncDirRoot, entry, options.sessionId).kind === "exact"
 					)
 					: [];
+		} else if (activeOnly) {
+			entries = (readActiveRunIndex(asyncDirRoot) ?? []).filter((entry) => {
+				if (resolveTargetedAsyncRun(asyncDirRoot, entry).kind === "exact") return true;
+				updateActiveRunIndex(path.join(asyncDirRoot, entry), "failed");
+				return false;
+			});
+			usedActiveIndex = true;
 		} else {
-			entries = fs.readdirSync(asyncDirRoot).filter((entry) => isAsyncRunDir(asyncDirRoot, entry));
+			entries = fs.readdirSync(asyncDirRoot).filter((entry) => entry !== ACTIVE_RUN_INDEX_DIR && isAsyncRunDir(asyncDirRoot, entry));
 			scannedCompleteRoot = true;
 		}
 	} catch (error) {
@@ -424,6 +443,7 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
 	// entirely when no active runs match.
 	let nestedRouteIndex: Map<string, NestedRoute> | undefined;
 	const resolveNestedRoute = (rootRunId: string): NestedRoute | undefined => {
+		if (usedActiveIndex) return findNestedRouteForRootId(rootRunId);
 		if (!nestedRouteIndex) nestedRouteIndex = buildNestedRouteIndex();
 		return nestedRouteIndex.get(rootRunId);
 	};
@@ -433,7 +453,15 @@ export function listAsyncRuns(asyncDirRoot: string, options: AsyncRunListOptions
 			? undefined
 			: reconcileAsyncRun(asyncDir, { resultsDir: options.resultsDir, kill: options.kill, now: options.now });
 		const status = (reconciliation?.status ?? readStatus(asyncDir)) as (AsyncStatus & { cwd?: string }) | null;
-		if (!status) continue;
+		if (!status) {
+			if (usedActiveIndex) updateActiveRunIndex(asyncDir, "failed");
+			continue;
+		}
+		if (usedActiveIndex && !isActiveAsyncState(status.state)) {
+			const processTerminal = readProcessTerminal(asyncDir, { runId: status.runId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId });
+			if (processTerminal?.state === "observed") releaseActiveRunIndex(asyncDir);
+		}
+		if (status.displayDismissedAt !== undefined) continue;
 		// Filter before the nested-route lookup: the lookup builds an index over
 		// the nested-events directory, so deferring it for filtered-out runs keeps
 		// restoration at load from scanning that directory when no active runs
@@ -536,6 +564,7 @@ export function formatAsyncRunList(runs: AsyncRunSummary[], heading = "Active as
 		const attached = new Set(run.steps.flatMap((step) => step.children?.map((child) => child.id) ?? []));
 		const unattached = run.nestedChildren?.filter((child) => !attached.has(child.id)) ?? [];
 		lines.push(...formatNestedRunStatusLines(unattached, { indent: "  ", maxLines: 12 }));
+		if (run.runFanoutBudget) lines.push(`  ${formatRunFanoutBudget(run.runFanoutBudget)}`);
 		if (run.error) lines.push(`  Error: ${run.error}`);
 		for (const warning of run.nestedWarnings ?? []) lines.push(`  Warning: ${warning}`);
 		const outputPath = formatAsyncRunOutputPath(run);