pi-subagents 0.29.0 → 0.31.0

package/src/runs/background/result-watcher.ts

@@ -21,7 +21,7 @@ import { projectNestedRegistryForRoot, sanitizeSummary } from "../shared/nested-
 const WATCHER_RESTART_DELAY_MS = 3000;
 const POLL_INTERVAL_MS = 3000;
 
-type ResultWatcherFs = Pick<typeof fs, "existsSync" | "readFileSync" | "unlinkSync" | "readdirSync" | "mkdirSync" | "watch">;
+type ResultWatcherFs = Pick<typeof fs, "existsSync" | "readFileSync" | "unlinkSync" | "readdirSync" | "mkdirSync" | "realpathSync" | "watch">;
 
 type ResultWatcherTimers = {
 	setTimeout: typeof setTimeout;
@@ -91,6 +91,14 @@ function shouldFallBackToPolling(error: unknown): boolean {
 	return code === "EMFILE" || code === "ENOSPC";
 }
 
+function resolveNativeWatchDir(fsApi: ResultWatcherFs, resultsDir: string): string {
+	try {
+		return fsApi.realpathSync.native(resultsDir);
+	} catch {
+		return resultsDir;
+	}
+}
+
 export function createResultWatcher(
 	pi: { events: IntercomEventBus },
 	state: SubagentState,
@@ -264,7 +272,8 @@ export function createResultWatcher(
 			state.watcherRestartTimer = null;
 		}
 		try {
-			state.watcher = fsApi.watch(resultsDir, (ev, file) => {
+			const watchDir = resolveNativeWatchDir(fsApi, resultsDir);
+			state.watcher = fsApi.watch(watchDir, (ev, file) => {
 				if (ev !== "rename" || !file) return;
 				const fileName = file.toString();
 				if (!fileName.endsWith(".json")) return;

package/src/runs/background/run-status.ts

@@ -205,6 +205,7 @@ export function inspectSubagentStatus(params: RunStatusParams, deps: RunStatusDe
 				statusActivityText ? `Activity: ${statusActivityText}` : undefined,
 				`Mode: ${status.mode}`,
 				`Progress: ${progressLabel}`,
+				status.pendingAppends ? `Pending appends: ${status.pendingAppends}` : undefined,
 				`Started: ${started}`,
 				`Updated: ${updated}`,
 				`Dir: ${asyncDir}`,

package/src/runs/background/stale-run-reconciler.ts

@@ -48,9 +48,14 @@ function isNotFoundError(error: unknown): boolean {
 		&& (error as NodeJS.ErrnoException).code === "ENOENT";
 }
 
-function appendJsonl(filePath: string, payload: object): void {
-	fs.mkdirSync(path.dirname(filePath), { recursive: true });
-	fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, "utf-8");
+function appendJsonlBestEffort(filePath: string, payload: object): void {
+	try {
+		fs.mkdirSync(path.dirname(filePath), { recursive: true });
+		fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, "utf-8");
+	} catch {
+		// Repair status/result writes are the important path. A broken or full
+		// diagnostic event log must not make stale-run reconciliation fail.
+	}
 }
 
 function readStatusFile(asyncDir: string): AsyncStatus | null {
@@ -223,7 +228,7 @@ function writeFailedRepair(asyncDir: string, status: AsyncStatus, resultPath: st
 	const repair = buildFailedRepair(status, asyncDir, now, reason);
 	writeAtomicJson(resultPath, repair.result);
 	writeAtomicJson(path.join(asyncDir, "status.json"), repair.status);
-	appendJsonl(path.join(asyncDir, "events.jsonl"), {
+	appendJsonlBestEffort(path.join(asyncDir, "events.jsonl"), {
 		type: "subagent.run.repaired_stale",
 		ts: now,
 		runId: repair.status.runId,

package/src/runs/background/subagent-runner.ts

@@ -4,7 +4,7 @@ import * as path from "node:path";
 import { pathToFileURL } from "node:url";
 import type { Message } from "@earendil-works/pi-ai";
 import { writeAtomicJson } from "../../shared/atomic-json.ts";
-import { appendJsonl, getArtifactPaths } from "../../shared/artifacts.ts";
+import { appendJsonl as appendRawJsonl, getArtifactPaths } from "../../shared/artifacts.ts";
 import { PI_CODING_AGENT_PACKAGE, getPiSpawnCommand, resolveInstalledPiPackageRoot } from "../shared/pi-spawn.ts";
 import { captureSingleOutputSnapshot, finalizeSingleOutput, formatSavedOutputReference, resolveSingleOutput, type SingleOutputSnapshot } from "../shared/single-output.ts";
 import {
@@ -78,6 +78,8 @@ import { resolveEffectiveThinking } from "../../shared/model-info.ts";
 import { writeInitialProgressFile } from "../../shared/settings.ts";
 import { resolveSubagentIntercomTarget } from "../../intercom/intercom-bridge.ts";
 import { acceptanceFailureMessage, aggregateAcceptanceReport, evaluateAcceptance, formatAcceptancePrompt, stripAcceptanceReport } from "../shared/acceptance.ts";
+import { waitForImportedAsyncRoot } from "./chain-root-attachment.ts";
+import { appendRunnerStepsToStatus, consumeChainAppendRequests, countPendingChainAppendRequests } from "./chain-append.ts";
 
 interface SubagentRunConfig {
 	id: string;
@@ -129,6 +131,79 @@ interface StepResult {
 }
 
 const ASYNC_INTERRUPT_SIGNAL: NodeJS.Signals = process.platform === "win32" ? "SIGBREAK" : "SIGUSR2";
+const DEFAULT_MAX_ASYNC_EVENTS_BYTES = 50 * 1024 * 1024;
+const ASYNC_EVENTS_MAX_BYTES_ENV = "PI_SUBAGENT_ASYNC_EVENTS_MAX_BYTES";
+const TRUNCATED_EVENT_TYPE = "subagent.events.truncated";
+const TRUNCATION_MARKER_RESERVE_BYTES = 512;
+
+interface AsyncEventLogState {
+	bytes: number;
+	diagnosticsTruncated: boolean;
+}
+
+const asyncEventLogStates = new Map<string, AsyncEventLogState>();
+
+function maxAsyncEventsBytes(): number {
+	const raw = process.env[ASYNC_EVENTS_MAX_BYTES_ENV];
+	if (!raw) return DEFAULT_MAX_ASYNC_EVENTS_BYTES;
+	const parsed = Number(raw);
+	if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_MAX_ASYNC_EVENTS_BYTES;
+	return Math.floor(parsed);
+}
+
+function eventLogState(filePath: string): AsyncEventLogState {
+	let state = asyncEventLogStates.get(filePath);
+	if (state) return state;
+	let bytes = 0;
+	try {
+		bytes = fs.statSync(filePath).size;
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+			// Diagnostic event accounting is best-effort; writes below are also safe.
+		}
+	}
+	state = { bytes, diagnosticsTruncated: false };
+	asyncEventLogStates.set(filePath, state);
+	return state;
+}
+
+function appendJsonl(filePath: string, line: string): void {
+	try {
+		appendRawJsonl(filePath, line);
+		const state = asyncEventLogStates.get(filePath);
+		if (state) state.bytes += Buffer.byteLength(`${line}\n`, "utf-8");
+	} catch {
+		// Async event logging is diagnostic and must not fail the run.
+	}
+}
+
+function appendDiagnosticJsonl(filePath: string, line: string, droppedEventType?: string): void {
+	if (!line.trim()) return;
+	const state = eventLogState(filePath);
+	if (state.diagnosticsTruncated) return;
+	const maxBytes = maxAsyncEventsBytes();
+	const chunkBytes = Buffer.byteLength(`${line}\n`, "utf-8");
+	const diagnosticBudget = Math.max(0, maxBytes - TRUNCATION_MARKER_RESERVE_BYTES);
+	if (state.bytes + chunkBytes <= diagnosticBudget) {
+		appendJsonl(filePath, line);
+		return;
+	}
+
+	const marker = JSON.stringify({
+		type: TRUNCATED_EVENT_TYPE,
+		ts: Date.now(),
+		maxBytes,
+		droppedEventType,
+	});
+	if (state.bytes + Buffer.byteLength(`${marker}\n`, "utf-8") <= maxBytes) {
+		appendJsonl(filePath, marker);
+	}
+	state.diagnosticsTruncated = true;
+}
+
+function shouldPersistChildEvent(event: Record<string, unknown>): boolean {
+	return event.type !== "message_update";
+}
 
 function findLatestSessionFile(sessionDir: string): string | null {
 	try {
@@ -272,14 +347,15 @@ function runPiStreaming(
 
 		const appendChildEvent = (event: Record<string, unknown>) => {
 			if (!childEventContext) return;
-			appendJsonl(childEventContext.eventsPath, JSON.stringify({
+			if (!shouldPersistChildEvent(event)) return;
+			appendDiagnosticJsonl(childEventContext.eventsPath, JSON.stringify({
 				...event,
 				subagentSource: "child",
 				subagentRunId: childEventContext.runId,
 				subagentStepIndex: childEventContext.stepIndex,
 				subagentAgent: childEventContext.agent,
 				observedAt: Date.now(),
-			}));
+			}), typeof event.type === "string" ? event.type : undefined);
 		};
 
 		const appendChildLine = (type: "subagent.child.stdout" | "subagent.child.stderr", line: string) => {
@@ -601,6 +677,30 @@ async function runSingleStep(
 	structuredOutputSchemaPath?: string;
 	acceptance?: import("../../shared/types.ts").AcceptanceLedger;
 }> {
+	if (step.importAsyncRoot) {
+		const imported = await waitForImportedAsyncRoot(step.importAsyncRoot);
+		try {
+			fs.writeFileSync(ctx.outputFile, imported.output, "utf-8");
+		} catch {
+			// Output files are observability only for imported roots.
+		}
+		return {
+			agent: imported.agent,
+			output: imported.output,
+			exitCode: imported.exitCode,
+			error: imported.error,
+			sessionFile: imported.sessionFile,
+			intercomTarget: imported.intercomTarget,
+			model: imported.model,
+			attemptedModels: imported.attemptedModels,
+			modelAttempts: imported.modelAttempts,
+			structuredOutput: imported.structuredOutput,
+			structuredOutputPath: imported.structuredOutputPath,
+			structuredOutputSchemaPath: imported.structuredOutputSchemaPath,
+			acceptance: imported.acceptance,
+		};
+	}
+
 	const effectiveStructuredOutput = step.structuredOutput ?? (step.structuredOutputSchema
 		? createStructuredOutputRuntime(step.structuredOutputSchema, path.join(path.dirname(ctx.outputFile), "structured-output"))
 		: undefined);
@@ -650,6 +750,7 @@ async function runSingleStep(
 			}
 		}
 		const { args, env, tempDir } = buildPiArgs({
+			parentSessionId: step.parentSessionId,
 			baseArgs: ["--mode", "json", "-p"],
 			task,
 			sessionEnabled,
@@ -658,8 +759,10 @@ async function runSingleStep(
 			model: candidate,
 			inheritProjectContext: step.inheritProjectContext,
 			inheritSkills: step.inheritSkills,
+			requireReadTool: Boolean(step.skills?.length),
 			tools: step.tools,
 			extensions: step.extensions,
+			subagentOnlyExtensions: step.subagentOnlyExtensions,
 			systemPrompt: step.systemPrompt,
 			systemPromptMode: step.systemPromptMode,
 			mcpDirectTools: step.mcpDirectTools,
@@ -1112,6 +1215,45 @@ async function runSubagent(config: SubagentRunConfig): Promise<void> {
 		writeAtomicJson(statusPath, statusPayload);
 		emitNestedSelfEvent(statusPayload.state === "running" || statusPayload.state === "queued" ? "subagent.nested.updated" : "subagent.nested.completed");
 	};
+	const consumePendingAppendRequests = (): void => {
+		if (statusPayload.mode !== "chain" || statusPayload.state !== "running") return;
+		const requests = consumeChainAppendRequests(asyncDir);
+		if (requests.length === 0) {
+			const pendingAppends = countPendingChainAppendRequests(asyncDir);
+			if ((statusPayload.pendingAppends ?? 0) !== pendingAppends) {
+				statusPayload.pendingAppends = pendingAppends;
+				statusPayload.lastUpdate = Date.now();
+				writeStatusPayload();
+			}
+			return;
+		}
+		const appendedSteps = requests.flatMap((request) => request.steps);
+		steps.push(...appendedSteps);
+		const now = Date.now();
+		const pendingAppends = countPendingChainAppendRequests(asyncDir);
+		const added = appendRunnerStepsToStatus({
+			status: statusPayload,
+			steps: appendedSteps,
+			now,
+			pendingAppends,
+		});
+		mutatingFailureStates.push(...Array.from({ length: added.addedFlatSteps }, () => createMutatingFailureState()));
+		pendingToolResults.push(...Array.from({ length: added.addedFlatSteps }, () => undefined));
+		if (config.childIntercomTargets) {
+			config.childIntercomTargets = statusPayload.steps.map((statusStep, index) => resolveSubagentIntercomTarget(id, statusStep.agent, index));
+		}
+		writeStatusPayload();
+		for (const request of requests) {
+			appendJsonl(eventsPath, JSON.stringify({
+				type: "subagent.chain.append.accepted",
+				ts: now,
+				runId: id,
+				requestId: request.id,
+				stepCount: request.steps.length,
+				pendingAppends,
+			}));
+		}
+	};
 	const markDynamicGraphGroup = (stepIndex: number, status: "completed" | "failed" | "running", error?: string, acceptance?: import("../../shared/types.ts").AcceptanceLedger): void => {
 		const groupNode = statusPayload.workflowGraph?.nodes.find((node) => node.id === `step-${stepIndex}`);
 		if (!groupNode) return;
@@ -1403,10 +1545,14 @@ async function runSubagent(config: SubagentRunConfig): Promise<void> {
 	);
 
 	let flatIndex = 0;
+	let stepCursor = 0;
 
-	for (let stepIndex = 0; stepIndex < steps.length; stepIndex++) {
+	while (true) {
 		if (interrupted) break;
-		const step = steps[stepIndex];
+		consumePendingAppendRequests();
+		if (stepCursor >= steps.length) break;
+		const stepIndex = stepCursor++;
+		const step = steps[stepIndex]!;
 
 		if (isDynamicRunnerGroup(step)) {
 			const groupStartFlatIndex = flatIndex;
@@ -1835,7 +1981,7 @@ async function runSubagent(config: SubagentRunConfig): Promise<void> {
 							outputs,
 							sessionDir: taskSessionDir,
 							artifactsDir, artifactConfig, id,
-							flatIndex: fi, flatStepCount: flatSteps.length,
+							flatIndex: fi, flatStepCount: Math.max(statusPayload.steps.length, 1),
 							outputFile: path.join(asyncDir, `output-${fi}.log`),
 							piPackageRoot: config.piPackageRoot,
 							piArgv1: config.piArgv1,
@@ -2000,7 +2146,7 @@ async function runSubagent(config: SubagentRunConfig): Promise<void> {
 				outputs,
 				sessionDir: config.sessionDir,
 				artifactsDir, artifactConfig, id,
-				flatIndex, flatStepCount: flatSteps.length,
+				flatIndex, flatStepCount: Math.max(statusPayload.steps.length, 1),
 				outputFile: path.join(asyncDir, `output-${flatIndex}.log`),
 				piPackageRoot: config.piPackageRoot,
 				piArgv1: config.piArgv1,
@@ -2131,11 +2277,12 @@ async function runSubagent(config: SubagentRunConfig): Promise<void> {
 	}
 
 	const resultMode = config.resultMode ?? statusPayload.mode;
-	const agentName = flatSteps.length === 1
-		? flatSteps[0].agent
+	const finalFlatAgents = statusPayload.steps.map((step) => step.agent);
+	const agentName = finalFlatAgents.length === 1
+		? finalFlatAgents[0]!
 		: resultMode === "parallel"
-			? `parallel:${flatSteps.map((s) => s.agent).join("+")}`
-			: `chain:${flatSteps.map((s) => s.agent).join("->")}`;
+			? `parallel:${finalFlatAgents.join("+")}`
+			: `chain:${finalFlatAgents.join("->")}`;
 	let sessionFile: string | undefined;
 	let shareUrl: string | undefined;
 	let gistUrl: string | undefined;

package/src/runs/foreground/chain-execution.ts

@@ -102,6 +102,7 @@ interface ParallelChainRunInput {
 	globalTaskIndex: number;
 	sessionDirForIndex: (idx?: number) => string | undefined;
 	sessionFileForIndex?: (idx?: number) => string | undefined;
+	sessionFileForTask?: (agentName: string, idx?: number) => string | undefined;
 	shareEnabled: boolean;
 	artifactConfig: ArtifactConfig;
 	artifactsDir: string;
@@ -136,6 +137,8 @@ interface ParallelChainRunInput {
 	worktreeSetup?: WorktreeSetup;
 	maxSubagentDepth: number;
 	nestedRoute?: NestedRouteInfo;
+	timeoutMs?: number;
+	deadlineAt?: number;
 }
 
 function buildChainExecutionDetails(input: ChainExecutionDetailsInput): Details {
@@ -266,6 +269,7 @@ async function runParallelChainTasks(input: ParallelChainRunInput): Promise<Sing
 				? createStructuredOutputRuntime(task.outputSchema, path.join(input.chainDir, "structured-output"))
 				: undefined;
 			const result = await runSync(input.ctx.cwd, input.agents, task.agent, taskStr, {
+				parentSessionId: input.ctx.sessionManager.getSessionId() ?? undefined,
 				cwd: taskCwd,
 				signal: input.signal,
 				interruptSignal: interruptController.signal,
@@ -274,7 +278,8 @@ async function runParallelChainTasks(input: ParallelChainRunInput): Promise<Sing
 				runId: input.runId,
 				index: input.globalTaskIndex + taskIndex,
 				sessionDir: input.sessionDirForIndex(input.globalTaskIndex + taskIndex),
-				sessionFile: input.sessionFileForIndex?.(input.globalTaskIndex + taskIndex),
+				sessionFile: input.sessionFileForTask?.(task.agent, input.globalTaskIndex + taskIndex)
+					?? input.sessionFileForIndex?.(input.globalTaskIndex + taskIndex),
 				share: input.shareEnabled,
 				artifactsDir: input.artifactConfig.enabled ? input.artifactsDir : undefined,
 				artifactConfig: input.artifactConfig,
@@ -293,6 +298,8 @@ async function runParallelChainTasks(input: ParallelChainRunInput): Promise<Sing
 				structuredOutput: structuredRuntime,
 				acceptance: task.acceptance,
 				acceptanceContext: { mode: "chain" },
+				timeoutMs: input.timeoutMs,
+				deadlineAt: input.deadlineAt,
 				onUpdate: input.onUpdate
 					? (progressUpdate) => {
 						const stepResults = progressUpdate.details?.results || [];
@@ -365,6 +372,7 @@ interface ChainExecutionParams {
 	shareEnabled: boolean;
 	sessionDirForIndex: (idx?: number) => string | undefined;
 	sessionFileForIndex?: (idx?: number) => string | undefined;
+	sessionFileForTask?: (agentName: string, idx?: number) => string | undefined;
 	artifactsDir: string;
 	artifactConfig: ArtifactConfig;
 	includeProgress?: boolean;
@@ -395,6 +403,8 @@ interface ChainExecutionParams {
 	nestedRoute?: NestedRouteInfo;
 	worktreeSetupHook?: string;
 	worktreeSetupHookTimeoutMs?: number;
+	timeoutMs?: number;
+	deadlineAt?: number;
 }
 
 interface ChainExecutionResult {
@@ -422,6 +432,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 		shareEnabled,
 		sessionDirForIndex,
 		sessionFileForIndex,
+		sessionFileForTask,
 		artifactsDir,
 		artifactConfig,
 		includeProgress,
@@ -584,6 +595,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 		tuiBehaviorOverrides = result.behaviorOverrides;
 	}
 
+	const deadlineAt = params.deadlineAt ?? (params.timeoutMs !== undefined ? Date.now() + params.timeoutMs : undefined);
 	let prev = "";
 	let globalTaskIndex = 0;
 	let progressCreated = false;
@@ -649,6 +661,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 					globalTaskIndex,
 					sessionDirForIndex,
 					sessionFileForIndex,
+					sessionFileForTask,
 					shareEnabled,
 					artifactConfig,
 					artifactsDir,
@@ -670,6 +683,8 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 					nestedRoute: params.nestedRoute,
 					worktreeSetup,
 					maxSubagentDepth: params.maxSubagentDepth,
+					timeoutMs: params.timeoutMs,
+					deadlineAt,
 				});
 				globalTaskIndex += step.parallel.length;
 
@@ -739,6 +754,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 						output: getSingleResultOutput(result),
 						exitCode: result.exitCode,
 						error: result.error,
+						timedOut: result.timedOut,
 						outputTargetPath,
 						outputTargetExists: outputTargetPath ? fs.existsSync(outputTargetPath) : undefined,
 					};
@@ -855,6 +871,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 				globalTaskIndex,
 				sessionDirForIndex,
 				sessionFileForIndex,
+				sessionFileForTask,
 				shareEnabled,
 				artifactConfig,
 				artifactsDir,
@@ -875,6 +892,8 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 				foregroundControl,
 				nestedRoute: params.nestedRoute,
 				maxSubagentDepth: params.maxSubagentDepth,
+				timeoutMs: params.timeoutMs,
+				deadlineAt,
 			});
 			globalTaskIndex += dynamicParallelStep.parallel.length;
 
@@ -970,6 +989,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 				output: getSingleResultOutput(result),
 				exitCode: result.exitCode,
 				error: result.error,
+				timedOut: result.timedOut,
 			}));
 			prev = aggregateParallelOutputs(taskResults, (i, agent) => `=== Dynamic Item ${i + 1} (${agent}, key ${materialized.items[i]?.key ?? i}) ===`);
 		} else {
@@ -1054,6 +1074,7 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 				? createStructuredOutputRuntime(seqStep.outputSchema, path.join(chainDir, "structured-output"))
 				: undefined;
 			const r = await runSync(ctx.cwd, agents, seqStep.agent, stepTask, {
+				parentSessionId: ctx.sessionManager.getSessionId() ?? undefined,
 				cwd: resolveChildCwd(cwd ?? ctx.cwd, seqStep.cwd),
 				signal,
 				interruptSignal: interruptController.signal,
@@ -1062,7 +1083,8 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 				runId,
 				index: globalTaskIndex,
 				sessionDir: sessionDirForIndex(globalTaskIndex),
-				sessionFile: sessionFileForIndex?.(globalTaskIndex),
+				sessionFile: sessionFileForTask?.(seqStep.agent, globalTaskIndex)
+					?? sessionFileForIndex?.(globalTaskIndex),
 				share: shareEnabled,
 				artifactsDir: artifactConfig.enabled ? artifactsDir : undefined,
 				artifactConfig,
@@ -1081,6 +1103,8 @@ export async function executeChain(params: ChainExecutionParams): Promise<ChainE
 				structuredOutput: structuredRuntime,
 				acceptance: seqStep.acceptance,
 				acceptanceContext: { mode: "chain" },
+				timeoutMs: params.timeoutMs,
+				deadlineAt,
 				onUpdate: onUpdate
 					? (p) => {
 						const stepResults = p.details?.results || [];