pi-crew 0.1.41 → 0.1.44

package/src/runtime/crash-recovery.ts

@@ -1,11 +1,13 @@
 import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
 import type { MetricRegistry } from "../observability/metric-registry.ts";
 import { appendEvent, scanSequence } from "../state/event-log.ts";
+import { withRunLockSync } from "../state/locks.ts";
 import { loadRunManifestById, saveRunTasks, updateRunStatus } from "../state/state-store.ts";
 import type { TeamTaskState } from "../state/types.ts";
 import { isWorkerHeartbeatStale } from "./worker-heartbeat.ts";
 import type { ManifestCache } from "./manifest-cache.ts";
 import { checkProcessLiveness } from "./process-status.ts";
+import { reconcileStaleRun, type ReconcileResult } from "./stale-reconciler.ts";
 
 export interface RecoveryPlan {
 	runId: string;
@@ -51,6 +53,35 @@ export async function applyRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionC
 export function declineRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionContext, "cwd">): void {
 	const loaded = loadRunManifestById(ctx.cwd, plan.runId);
 	if (!loaded) throw new Error(`Run '${plan.runId}' not found.`);
-	updateRunStatus(loaded.manifest, "cancelled", "interrupted-not-resumed");
+	// Log the event first — if appendEvent fails, state remains consistent.
 	appendEvent(loaded.manifest.eventsPath, { type: "crew.run.recovery_declined", runId: plan.runId, message: "Interrupted run was not resumed.", data: { recoveredFromSeq: plan.lastEventSeq } });
+	updateRunStatus(loaded.manifest, "cancelled", "interrupted-not-resumed");
+}
+
+/**
+ * Run 3-phase stale reconciliation on all active runs.
+ * Returns results for each reconciled run.
+ */
+export function reconcileAllStaleRuns(cwd: string, manifestCache: ManifestCache, now = Date.now()): ReconcileResult[] {
+	const results: ReconcileResult[] = [];
+	for (const manifest of manifestCache.list(50)) {
+		if (manifest.status !== "running") continue;
+		const loaded = loadRunManifestById(cwd, manifest.runId);
+		if (!loaded) continue;
+		// Use lock to prevent race with cancel/status handlers modifying the same run
+		withRunLockSync(loaded.manifest, () => {
+			// Re-read inside lock to get freshest data
+			const fresh = loadRunManifestById(cwd, manifest.runId);
+			if (!fresh || fresh.manifest.status !== "running") return;
+			const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
+			if (result.repaired) {
+				updateRunStatus(fresh.manifest, "failed", `Stale run reconciled: ${result.detail}`);
+				appendEvent(fresh.manifest.eventsPath, { type: "crew.run.reconciled_stale", runId: manifest.runId, message: result.detail, data: { verdict: result.verdict } });
+			}
+			if (result.verdict !== "healthy") {
+				results.push(result);
+			}
+		});
+	}
+	return results;
 }

package/src/runtime/crew-agent-runtime.ts

@@ -1,58 +1,59 @@
-import type { TeamTaskStatus } from "../state/contracts.ts";
-import type { CrewActivityState, ModelRoutingState, UsageState } from "../state/types.ts";
-
-export type CrewRuntimeKind = "scaffold" | "child-process" | "live-session";
-export type CrewAgentStatus = "queued" | "running" | "completed" | "failed" | "cancelled" | "stopped";
-
-export interface CrewAgentRecentTool {
-	tool: string;
-	args?: string;
-	endedAt: string;
-}
-
-export interface CrewAgentProgress {
-	currentTool?: string;
-	currentToolArgs?: string;
-	currentToolStartedAt?: string;
-	recentTools: CrewAgentRecentTool[];
-	recentOutput: string[];
-	toolCount: number;
-	tokens?: number;
-	turns?: number;
-	durationMs?: number;
-	lastActivityAt?: string;
-	activityState?: CrewActivityState;
-	failedTool?: string;
-}
-
-export interface CrewAgentRecord {
-	id: string;
-	runId: string;
-	taskId: string;
-	agent: string;
-	role: string;
-	runtime: CrewRuntimeKind;
-	status: CrewAgentStatus;
-	startedAt: string;
-	completedAt?: string;
-	resultArtifactPath?: string;
-	transcriptPath?: string;
-	statusPath?: string;
-	eventsPath?: string;
-	outputPath?: string;
-	toolUses?: number;
-	jsonEvents?: number;
-	model?: string;
-	routing?: ModelRoutingState;
-	usage?: UsageState;
-	progress?: CrewAgentProgress;
-	error?: string;
-}
-
-export function taskStatusToAgentStatus(status: TeamTaskStatus): CrewAgentStatus {
-	if (status === "completed") return "completed";
-	if (status === "failed") return "failed";
-	if (status === "cancelled" || status === "skipped") return "cancelled";
-	if (status === "running") return "running";
-	return "queued";
-}
+import type { TeamTaskStatus } from "../state/contracts.ts";
+import type { CrewActivityState, ModelRoutingState, UsageState } from "../state/types.ts";
+
+export type CrewRuntimeKind = "scaffold" | "child-process" | "live-session";
+export type CrewAgentStatus = "queued" | "running" | "waiting" | "completed" | "failed" | "cancelled" | "stopped";
+
+export interface CrewAgentRecentTool {
+	tool: string;
+	args?: string;
+	endedAt: string;
+}
+
+export interface CrewAgentProgress {
+	currentTool?: string;
+	currentToolArgs?: string;
+	currentToolStartedAt?: string;
+	recentTools: CrewAgentRecentTool[];
+	recentOutput: string[];
+	toolCount: number;
+	tokens?: number;
+	turns?: number;
+	durationMs?: number;
+	lastActivityAt?: string;
+	activityState?: CrewActivityState;
+	failedTool?: string;
+}
+
+export interface CrewAgentRecord {
+	id: string;
+	runId: string;
+	taskId: string;
+	agent: string;
+	role: string;
+	runtime: CrewRuntimeKind;
+	status: CrewAgentStatus;
+	startedAt: string;
+	completedAt?: string;
+	resultArtifactPath?: string;
+	transcriptPath?: string;
+	statusPath?: string;
+	eventsPath?: string;
+	outputPath?: string;
+	toolUses?: number;
+	jsonEvents?: number;
+	model?: string;
+	routing?: ModelRoutingState;
+	usage?: UsageState;
+	progress?: CrewAgentProgress;
+	error?: string;
+}
+
+export function taskStatusToAgentStatus(status: TeamTaskStatus): CrewAgentStatus {
+	if (status === "completed") return "completed";
+	if (status === "failed") return "failed";
+	if (status === "cancelled" || status === "skipped") return "cancelled";
+	if (status === "running") return "running";
+	if (status === "waiting") return "waiting";
+	return "queued";
+}

package/src/runtime/deadletter.ts

@@ -2,6 +2,8 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 import type { TeamRunManifest } from "../state/types.ts";
 
+import { logInternalError } from "../utils/internal-error.ts";
+
 export type DeadletterReason = "max-retries" | "heartbeat-dead" | "manual";
 
 export interface DeadletterEntry {
@@ -18,14 +20,22 @@ export function deadletterPath(manifest: TeamRunManifest): string {
 }
 
 export function appendDeadletter(manifest: TeamRunManifest, entry: DeadletterEntry): void {
-	fs.mkdirSync(manifest.stateRoot, { recursive: true });
-	fs.appendFileSync(deadletterPath(manifest), `${JSON.stringify(entry)}\n`, "utf-8");
+	try {
+		fs.mkdirSync(manifest.stateRoot, { recursive: true });
+		fs.appendFileSync(deadletterPath(manifest), `${JSON.stringify(entry)}\n`, "utf-8");
+	} catch (error) {
+		logInternalError("deadletter.append", error, `taskId=${entry.taskId}`);
+	}
 }
 
-export function readDeadletter(manifest: TeamRunManifest): DeadletterEntry[] {
+export function readDeadletter(manifest: TeamRunManifest, maxEntries = 1000): DeadletterEntry[] {
 	const filePath = deadletterPath(manifest);
 	if (!fs.existsSync(filePath)) return [];
-	return fs.readFileSync(filePath, "utf-8").split(/\r?\n/).filter(Boolean).flatMap((line) => {
+	// Read last maxEntries lines only to limit memory.
+	const raw = fs.readFileSync(filePath, "utf-8");
+	const lines = raw.split(/\r?\n/).filter(Boolean);
+	const tail = lines.slice(-maxEntries);
+	return tail.flatMap((line) => {
 		try {
 			const parsed = JSON.parse(line) as DeadletterEntry;
 			return parsed && typeof parsed.taskId === "string" && typeof parsed.runId === "string" ? [parsed] : [];

package/src/runtime/delivery-coordinator.ts

@@ -0,0 +1,143 @@
+import type { NotificationDescriptor } from "../extension/notification-router.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+
+export interface PendingDelivery {
+	runId: string;
+	payload: unknown;
+	timestamp: number;
+	type: "result" | "notification" | "steer";
+}
+
+export interface DeliveryCoordinatorDeps {
+	/** Emit an event to the active Pi event bus. */
+	emit?: (event: string, data: unknown) => void;
+	/** Send a follow-up message to the active session (for notifications). */
+	sendFollowUp?: (title: string, body: string) => void;
+	/** Send a wake-up message to the active session (for async results). */
+	sendWakeUp?: (message: string) => void;
+}
+
+const PENDING_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+export class DeliveryCoordinator {
+	private ownerSessionId: string | undefined;
+	private active = false;
+	private pending: PendingDelivery[] = [];
+	private readonly deps: DeliveryCoordinatorDeps;
+	private ttlTimer: ReturnType<typeof setInterval> | undefined;
+
+	constructor(deps: DeliveryCoordinatorDeps) {
+		this.deps = deps;
+		this.ttlTimer = setInterval(() => this.evictExpired(), 60_000);
+		this.ttlTimer.unref();
+	}
+
+	activate(sessionId: string): void {
+		this.ownerSessionId = sessionId;
+		this.active = true;
+		this.flushQueuedResults();
+	}
+
+	deactivate(): void {
+		this.active = false;
+		this.ownerSessionId = undefined;
+	}
+
+	isActive(): boolean {
+		return this.active;
+	}
+
+	getPendingCount(): number {
+		return this.pending.length;
+	}
+
+	deliverResult(runId: string, result: unknown): void {
+		if (this.active && this.deps.emit) {
+			try {
+				this.deps.emit("pi-crew:run-result", result);
+			} catch (error) {
+				logInternalError("delivery-coordinator.deliverResult", error, `runId=${runId}`);
+			}
+			return;
+		}
+		this.enqueue({ runId, payload: result, timestamp: Date.now(), type: "result" });
+	}
+
+	deliverNotification(notification: NotificationDescriptor): void {
+		if (this.active && this.deps.sendFollowUp) {
+			try {
+				this.deps.sendFollowUp(notification.title, notification.body ?? "");
+			} catch (error) {
+				logInternalError("delivery-coordinator.deliverNotification", error, `id=${notification.id}`);
+			}
+			if (this.deps.emit) {
+				try {
+					this.deps.emit("pi-crew:notification", notification);
+				} catch { /* secondary delivery, ignore errors */ }
+			}
+			return;
+		}
+		this.enqueue({ runId: notification.runId ?? "", payload: notification, timestamp: Date.now(), type: "notification" });
+	}
+
+	deliverSteer(runId: string, message: string): void {
+		if (this.active && this.deps.sendWakeUp) {
+			try {
+				this.deps.sendWakeUp(message);
+			} catch (error) {
+				logInternalError("delivery-coordinator.deliverSteer", error, `runId=${runId}`);
+			}
+			return;
+		}
+		this.enqueue({ runId, payload: message, timestamp: Date.now(), type: "steer" });
+	}
+
+	flushQueuedResults(): void {
+		if (!this.active || this.pending.length === 0) return;
+		const batch = this.pending.splice(0);
+		for (const delivery of batch) {
+			try {
+				switch (delivery.type) {
+					case "result":
+						this.deliverResult(delivery.runId, delivery.payload);
+						break;
+					case "notification": {
+						const notification = delivery.payload as NotificationDescriptor;
+						this.deliverNotification(notification);
+						break;
+					}
+					case "steer": {
+						const message = typeof delivery.payload === "string" ? delivery.payload : String(delivery.payload);
+						this.deliverSteer(delivery.runId, message);
+						break;
+					}
+				}
+			} catch (error) {
+				logInternalError("delivery-coordinator.flush", error, `runId=${delivery.runId} type=${delivery.type}`);
+			}
+		}
+	}
+
+	dispose(): void {
+		this.deactivate();
+		this.pending.length = 0;
+		if (this.ttlTimer) {
+			clearInterval(this.ttlTimer);
+			this.ttlTimer = undefined;
+		}
+	}
+
+	private enqueue(delivery: PendingDelivery): void {
+		this.pending.push(delivery);
+	}
+
+	private evictExpired(): void {
+		const cutoff = Date.now() - PENDING_TTL_MS;
+		const before = this.pending.length;
+		this.pending = this.pending.filter((d) => d.timestamp > cutoff);
+		const evicted = before - this.pending.length;
+		if (evicted > 0) {
+			logInternalError("delivery-coordinator.evict", undefined, `evicted=${evicted} remaining=${this.pending.length}`);
+		}
+	}
+}

package/src/runtime/direct-run.ts

@@ -1,35 +1,35 @@
-import type { AgentConfig } from "../agents/agent-config.ts";
-import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import type { TeamConfig } from "../teams/team-config.ts";
-import type { WorkflowConfig } from "../workflows/workflow-config.ts";
-
-export function isDirectRun(manifest: Pick<TeamRunManifest, "team" | "workflow">): boolean {
-	return manifest.workflow === "direct-agent";
-}
-
-export function directTeamAndWorkflowFromRun(manifest: TeamRunManifest, tasks: TeamTaskState[], agents: AgentConfig[]): { team: TeamConfig; workflow: WorkflowConfig } | undefined {
-	if (!isDirectRun(manifest)) return undefined;
-	const firstTask = tasks[0];
-	const agentName = firstTask?.agent ?? (manifest.team.replace(/^direct-/, "") || "executor");
-	const agent = agents.find((candidate) => candidate.name === agentName);
-	const role = firstTask?.role ?? "agent";
-	const stepId = firstTask?.stepId ?? "01_agent";
-	return {
-		team: {
-			name: manifest.team,
-			description: `Direct subagent run for ${agentName}`,
-			source: "builtin",
-			filePath: "<generated>",
-			roles: [{ name: role, agent: agentName, description: agent?.description }],
-			defaultWorkflow: "direct-agent",
-			workspaceMode: manifest.workspaceMode,
-		},
-		workflow: {
-			name: manifest.workflow ?? "direct-agent",
-			description: `Direct task for ${agentName}`,
-			source: "builtin",
-			filePath: "<generated>",
-			steps: [{ id: stepId, role, task: "{goal}", model: firstTask?.model }],
-		},
-	};
-}
+import type { AgentConfig } from "../agents/agent-config.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import type { TeamConfig } from "../teams/team-config.ts";
+import type { WorkflowConfig } from "../workflows/workflow-config.ts";
+
+export function isDirectRun(manifest: Pick<TeamRunManifest, "team" | "workflow">): boolean {
+	return manifest.workflow === "direct-agent";
+}
+
+export function directTeamAndWorkflowFromRun(manifest: TeamRunManifest, tasks: TeamTaskState[], agents: AgentConfig[]): { team: TeamConfig; workflow: WorkflowConfig } | undefined {
+	if (!isDirectRun(manifest)) return undefined;
+	const firstTask = tasks[0];
+	const agentName = firstTask?.agent ?? (manifest.team.replace(/^direct-/, "") || "executor");
+	const agent = agents.find((candidate) => candidate.name === agentName);
+	const role = firstTask?.role ?? "agent";
+	const stepId = firstTask?.stepId ?? "01_agent";
+	return {
+		team: {
+			name: manifest.team,
+			description: `Direct subagent run for ${agentName}`,
+			source: "builtin",
+			filePath: "<generated>",
+			roles: [{ name: role, agent: agentName, description: agent?.description }],
+			defaultWorkflow: "direct-agent",
+			workspaceMode: manifest.workspaceMode,
+		},
+		workflow: {
+			name: manifest.workflow ?? "direct-agent",
+			description: `Direct task for ${agentName}`,
+			source: "builtin",
+			filePath: "<generated>",
+			steps: [{ id: stepId, role, task: "{goal}", model: firstTask?.model }],
+		},
+	};
+}

package/src/runtime/foreground-control.ts

@@ -1,82 +1,82 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { appendEvent } from "../state/event-log.ts";
-import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import { checkProcessLiveness, isActiveRunStatus } from "./process-status.ts";
-import { readCrewAgents } from "./crew-agent-records.ts";
-
-export type ForegroundControlRequestType = "interrupt" | "status";
-
-export interface ForegroundControlStatus {
-	runId: string;
-	status: TeamRunManifest["status"];
-	active: boolean;
-	asyncPid?: number;
-	asyncAlive?: boolean;
-	runningTasks: string[];
-	runningAgents: string[];
-	controlPath: string;
-	lastRequest?: ForegroundControlRequest;
-}
-
-export interface ForegroundControlRequest {
-	id: string;
-	type: ForegroundControlRequestType;
-	createdAt: string;
-	reason: string;
-	acknowledged: boolean;
-}
-
-export function foregroundControlPath(manifest: TeamRunManifest): string {
-	return path.join(manifest.stateRoot, "foreground-control.json");
-}
-
-function readLastRequest(controlPath: string): ForegroundControlRequest | undefined {
-	if (!fs.existsSync(controlPath)) return undefined;
-	try {
-		const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
-		return parsed.requests?.at(-1);
-	} catch {
-		return undefined;
-	}
-}
-
-export function readForegroundControlStatus(manifest: TeamRunManifest, tasks: TeamTaskState[]): ForegroundControlStatus {
-	const controlPath = foregroundControlPath(manifest);
-	const asyncAlive = manifest.async?.pid !== undefined ? checkProcessLiveness(manifest.async.pid).alive : undefined;
-	return {
-		runId: manifest.runId,
-		status: manifest.status,
-		active: isActiveRunStatus(manifest.status),
-		asyncPid: manifest.async?.pid,
-		asyncAlive,
-		runningTasks: tasks.filter((task) => task.status === "running").map((task) => task.id),
-		runningAgents: readCrewAgents(manifest).filter((agent) => agent.status === "running").map((agent) => agent.id),
-		controlPath,
-		lastRequest: readLastRequest(controlPath),
-	};
-}
-
-export function writeForegroundInterruptRequest(manifest: TeamRunManifest, reason = "User requested foreground interrupt."): ForegroundControlRequest {
-	const controlPath = foregroundControlPath(manifest);
-	let requests: ForegroundControlRequest[] = [];
-	if (fs.existsSync(controlPath)) {
-		try {
-			const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
-			requests = Array.isArray(parsed.requests) ? parsed.requests : [];
-		} catch {
-			requests = [];
-		}
-	}
-	const request: ForegroundControlRequest = {
-		id: `fg_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`,
-		type: "interrupt",
-		createdAt: new Date().toISOString(),
-		reason,
-		acknowledged: false,
-	};
-	fs.mkdirSync(path.dirname(controlPath), { recursive: true });
-	fs.writeFileSync(controlPath, `${JSON.stringify({ requests: [...requests, request] }, null, 2)}\n`, "utf-8");
-	appendEvent(manifest.eventsPath, { type: "foreground.interrupt_requested", runId: manifest.runId, message: reason, data: { requestId: request.id, controlPath } });
-	return request;
-}
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { appendEvent } from "../state/event-log.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { checkProcessLiveness, isActiveRunStatus } from "./process-status.ts";
+import { readCrewAgents } from "./crew-agent-records.ts";
+
+export type ForegroundControlRequestType = "interrupt" | "status";
+
+export interface ForegroundControlStatus {
+	runId: string;
+	status: TeamRunManifest["status"];
+	active: boolean;
+	asyncPid?: number;
+	asyncAlive?: boolean;
+	runningTasks: string[];
+	runningAgents: string[];
+	controlPath: string;
+	lastRequest?: ForegroundControlRequest;
+}
+
+export interface ForegroundControlRequest {
+	id: string;
+	type: ForegroundControlRequestType;
+	createdAt: string;
+	reason: string;
+	acknowledged: boolean;
+}
+
+export function foregroundControlPath(manifest: TeamRunManifest): string {
+	return path.join(manifest.stateRoot, "foreground-control.json");
+}
+
+function readLastRequest(controlPath: string): ForegroundControlRequest | undefined {
+	if (!fs.existsSync(controlPath)) return undefined;
+	try {
+		const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
+		return parsed.requests?.at(-1);
+	} catch {
+		return undefined;
+	}
+}
+
+export function readForegroundControlStatus(manifest: TeamRunManifest, tasks: TeamTaskState[]): ForegroundControlStatus {
+	const controlPath = foregroundControlPath(manifest);
+	const asyncAlive = manifest.async?.pid !== undefined ? checkProcessLiveness(manifest.async.pid).alive : undefined;
+	return {
+		runId: manifest.runId,
+		status: manifest.status,
+		active: isActiveRunStatus(manifest.status),
+		asyncPid: manifest.async?.pid,
+		asyncAlive,
+		runningTasks: tasks.filter((task) => task.status === "running").map((task) => task.id),
+		runningAgents: readCrewAgents(manifest).filter((agent) => agent.status === "running").map((agent) => agent.id),
+		controlPath,
+		lastRequest: readLastRequest(controlPath),
+	};
+}
+
+export function writeForegroundInterruptRequest(manifest: TeamRunManifest, reason = "User requested foreground interrupt."): ForegroundControlRequest {
+	const controlPath = foregroundControlPath(manifest);
+	let requests: ForegroundControlRequest[] = [];
+	if (fs.existsSync(controlPath)) {
+		try {
+			const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
+			requests = Array.isArray(parsed.requests) ? parsed.requests : [];
+		} catch {
+			requests = [];
+		}
+	}
+	const request: ForegroundControlRequest = {
+		id: `fg_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`,
+		type: "interrupt",
+		createdAt: new Date().toISOString(),
+		reason,
+		acknowledged: false,
+	};
+	fs.mkdirSync(path.dirname(controlPath), { recursive: true });
+	fs.writeFileSync(controlPath, `${JSON.stringify({ requests: [...requests, request] }, null, 2)}\n`, "utf-8");
+	appendEvent(manifest.eventsPath, { type: "foreground.interrupt_requested", runId: manifest.runId, message: reason, data: { requestId: request.id, controlPath } });
+	return request;
+}