pi-crew 0.1.44 → 0.1.45

package/src/runtime/overflow-recovery.ts

@@ -1,157 +1,176 @@
-import { logInternalError } from "../utils/internal-error.ts";
-
-export type OverflowPhase = "none" | "compaction" | "retrying" | "recovered" | "failed";
-
-export interface OverflowRecoveryState {
-	taskId: string;
-	runId: string;
-	phase: OverflowPhase;
-	startedAt: number;
-	lastEventAt: number;
-	compactionCount: number;
-	retryCount: number;
-}
-
-export interface OverflowRecoveryCallbacks {
-	onPhaseChange?: (state: OverflowRecoveryState, previousPhase: OverflowPhase) => void;
-	onTimeout?: (state: OverflowRecoveryState) => void;
-}
-
-const PHASE_TIMEOUT_MS = 120_000; // 120 seconds per phase
-
-export class OverflowRecoveryTracker {
-	private states = new Map<string, OverflowRecoveryState>();
-	private timers = new Map<string, ReturnType<typeof setTimeout>>();
-	private callbacks: OverflowRecoveryCallbacks;
-
-	constructor(callbacks: OverflowRecoveryCallbacks = {}) {
-		this.callbacks = callbacks;
-	}
-
-	feedEvent(taskId: string, runId: string, eventType: string): OverflowPhase {
-		const existing = this.states.get(taskId);
-		const now = Date.now();
-
-		if (existing && existing.phase === "recovered") {
-			existing.lastEventAt = now;
-			return "recovered";
-		}
-		if (existing && existing.phase === "failed") {
-			existing.lastEventAt = now;
-			return "failed";
-		}
-
-		let phase: OverflowPhase = existing?.phase ?? "none";
-		let compactionCount = existing?.compactionCount ?? 0;
-		let retryCount = existing?.retryCount ?? 0;
-		const previousPhase = phase;
-
-		switch (eventType) {
-			case "compaction_start":
-				phase = "compaction";
-				compactionCount++;
-				break;
-			case "compaction_end":
-				// After compaction, we expect a retry; stay in compaction until retry starts
-				break;
-			case "auto_retry_start":
-				phase = "retrying";
-				retryCount++;
-				break;
-			case "auto_retry_end":
-				// After retry completes, the agent should produce a response
-				// We consider this recovered but don't finalize until agent_end
-				phase = "recovered";
-				break;
-			case "agent_end":
-				// If we were recovering and agent ends, we're recovered or failed
-				if (phase === "compaction" || phase === "retrying") {
-					phase = "failed";
-				}
-				break;
-			default:
-				// Unknown event type — no phase change
-				break;
-		}
-
-		const state: OverflowRecoveryState = {
-			taskId,
-			runId,
-			phase,
-			startedAt: existing?.startedAt ?? now,
-			lastEventAt: now,
-			compactionCount,
-			retryCount,
-		};
-
-		this.states.set(taskId, state);
-		this.resetTimeout(taskId);
-
-		if (previousPhase !== phase && this.callbacks.onPhaseChange) {
-			try {
-				this.callbacks.onPhaseChange(state, previousPhase);
-			} catch (error) {
-				logInternalError("overflow-recovery.onPhaseChange", error, `taskId=${taskId}`);
-			}
-		}
-
-		return phase;
-	}
-
-	getState(taskId: string): OverflowRecoveryState | undefined {
-		return this.states.get(taskId);
-	}
-
-	getPhase(taskId: string): OverflowPhase {
-		return this.states.get(taskId)?.phase ?? "none";
-	}
-
-	removeTask(taskId: string): void {
-		this.states.delete(taskId);
-		const timer = this.timers.get(taskId);
-		if (timer) {
-			clearTimeout(timer);
-			this.timers.delete(taskId);
-		}
-	}
-
-	dispose(): void {
-		for (const timer of this.timers.values()) clearTimeout(timer);
-		this.timers.clear();
-		this.states.clear();
-	}
-
-	private resetTimeout(taskId: string): void {
-		const existing = this.timers.get(taskId);
-		if (existing) clearTimeout(existing);
-
-		const timer = setTimeout(() => {
-			this.timers.delete(taskId);
-			const state = this.states.get(taskId);
-			if (!state) return;
-			if (state.phase === "recovered" || state.phase === "failed" || state.phase === "none") return;
-
-			const previousPhase = state.phase;
-			state.phase = "failed";
-			state.lastEventAt = Date.now();
-
-			if (this.callbacks.onTimeout) {
-				try {
-					this.callbacks.onTimeout(state);
-				} catch (error) {
-					logInternalError("overflow-recovery.onTimeout", error, `taskId=${taskId}`);
-				}
-			}
-			if (this.callbacks.onPhaseChange) {
-				try {
-					this.callbacks.onPhaseChange(state, previousPhase);
-				} catch (error) {
-					logInternalError("overflow-recovery.onPhaseChange-timeout", error, `taskId=${taskId}`);
-				}
-			}
-		}, PHASE_TIMEOUT_MS);
-
-		timer.unref();
-		this.timers.set(taskId, timer);
-	}
+import { logInternalError } from "../utils/internal-error.ts";
+
+export type OverflowPhase = "none" | "compaction" | "retrying" | "recovered" | "failed";
+
+export interface OverflowRecoveryState {
+	taskId: string;
+	runId: string;
+	phase: OverflowPhase;
+	startedAt: number;
+	lastEventAt: number;
+	compactionCount: number;
+	retryCount: number;
+}
+
+export interface OverflowRecoveryCallbacks {
+	onPhaseChange?: (state: OverflowRecoveryState, previousPhase: OverflowPhase) => void;
+	onTimeout?: (state: OverflowRecoveryState) => void;
+}
+
+const PHASE_TIMEOUT_MS = 120_000; // 120 seconds per phase
+const TERMINAL_STATE_TTL_MS = 5 * 60_000;
+
+export class OverflowRecoveryTracker {
+	private states = new Map<string, OverflowRecoveryState>();
+	private timers = new Map<string, ReturnType<typeof setTimeout>>();
+	private callbacks: OverflowRecoveryCallbacks;
+
+	constructor(callbacks: OverflowRecoveryCallbacks = {}) {
+		this.callbacks = callbacks;
+	}
+
+	feedEvent(taskId: string, runId: string, eventType: string): OverflowPhase {
+		const key = this.keyFor(taskId, runId);
+		const existing = this.states.get(key);
+		const now = Date.now();
+
+		if (existing && existing.phase === "recovered") {
+			existing.lastEventAt = now;
+			return "recovered";
+		}
+		if (existing && existing.phase === "failed") {
+			existing.lastEventAt = now;
+			return "failed";
+		}
+
+		let phase: OverflowPhase = existing?.phase ?? "none";
+		let compactionCount = existing?.compactionCount ?? 0;
+		let retryCount = existing?.retryCount ?? 0;
+		const previousPhase = phase;
+
+		switch (eventType) {
+			case "compaction_start":
+				phase = "compaction";
+				compactionCount++;
+				break;
+			case "compaction_end":
+				// After compaction, we expect a retry; stay in compaction until retry starts
+				break;
+			case "auto_retry_start":
+				phase = "retrying";
+				retryCount++;
+				break;
+			case "auto_retry_end":
+				// After retry completes, the agent should produce a response
+				// We consider this recovered but don't finalize until agent_end
+				phase = "recovered";
+				break;
+			case "agent_end":
+				// If we were recovering and agent ends, we're recovered or failed
+				if (phase === "compaction" || phase === "retrying") {
+					phase = "failed";
+				}
+				break;
+			default:
+				// Unknown event type — no phase change
+				break;
+		}
+
+		const state: OverflowRecoveryState = {
+			taskId,
+			runId,
+			phase,
+			startedAt: existing?.startedAt ?? now,
+			lastEventAt: now,
+			compactionCount,
+			retryCount,
+		};
+
+		this.states.set(key, state);
+		this.resetTimeout(key);
+
+		if (previousPhase !== phase && this.callbacks.onPhaseChange) {
+			try {
+				this.callbacks.onPhaseChange(state, previousPhase);
+			} catch (error) {
+				logInternalError("overflow-recovery.onPhaseChange", error, `taskId=${taskId}`);
+			}
+		}
+
+		return phase;
+	}
+
+	getState(taskId: string, runId?: string): OverflowRecoveryState | undefined {
+		if (runId) return this.states.get(this.keyFor(taskId, runId));
+		return [...this.states.values()].find((state) => state.taskId === taskId);
+	}
+
+	getPhase(taskId: string, runId?: string): OverflowPhase {
+		return this.getState(taskId, runId)?.phase ?? "none";
+	}
+
+	removeTask(taskId: string, runId?: string): void {
+		const keys = runId
+			? [this.keyFor(taskId, runId)]
+			: [...this.states.entries()].filter(([, state]) => state.taskId === taskId).map(([key]) => key);
+		for (const key of keys) this.removeKey(key);
+	}
+
+	dispose(): void {
+		for (const timer of this.timers.values()) clearTimeout(timer);
+		this.timers.clear();
+		this.states.clear();
+	}
+
+	private keyFor(taskId: string, runId: string): string {
+		return `${runId}\u0000${taskId}`;
+	}
+
+	private removeKey(key: string): void {
+		this.states.delete(key);
+		const timer = this.timers.get(key);
+		if (timer) clearTimeout(timer);
+		this.timers.delete(key);
+	}
+
+	private resetTimeout(key: string): void {
+		const existing = this.timers.get(key);
+		if (existing) clearTimeout(existing);
+		const current = this.states.get(key);
+		const timeoutMs = current?.phase === "recovered" || current?.phase === "failed" || current?.phase === "none"
+			? TERMINAL_STATE_TTL_MS
+			: PHASE_TIMEOUT_MS;
+
+		const timer = setTimeout(() => {
+			this.timers.delete(key);
+			const state = this.states.get(key);
+			if (!state) return;
+			if (state.phase === "recovered" || state.phase === "failed" || state.phase === "none") {
+				this.states.delete(key);
+				return;
+			}
+
+			const previousPhase = state.phase;
+			state.phase = "failed";
+			state.lastEventAt = Date.now();
+
+			if (this.callbacks.onTimeout) {
+				try {
+					this.callbacks.onTimeout(state);
+				} catch (error) {
+					logInternalError("overflow-recovery.onTimeout", error, `taskId=${state.taskId}`);
+				}
+			}
+			if (this.callbacks.onPhaseChange) {
+				try {
+					this.callbacks.onPhaseChange(state, previousPhase);
+				} catch (error) {
+					logInternalError("overflow-recovery.onPhaseChange-timeout", error, `taskId=${state.taskId}`);
+				}
+			}
+		}, timeoutMs);
+
+		timer.unref();
+		this.timers.set(key, timer);
+	}
 }

package/src/runtime/parallel-research.ts

@@ -1,44 +1,44 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
-
-export function sourcePiProjects(cwd: string): string[] {
-	const sourceDir = path.join(cwd, "Source");
-	try {
-		return fs.readdirSync(sourceDir, { withFileTypes: true })
-			.filter((entry) => entry.isDirectory() && entry.name.startsWith("pi-"))
-			.map((entry) => `Source/${entry.name}`)
-			.sort();
-	} catch {
-		return [];
-	}
-}
-
-export function chunkProjects(projects: string[], target = 6): string[][] {
-	const chunks = Array.from({ length: Math.min(Math.max(1, target), Math.max(1, projects.length)) }, () => [] as string[]);
-	projects.forEach((project, index) => chunks[index % chunks.length]!.push(project));
-	return chunks.filter((chunk) => chunk.length > 0);
-}
-
-export function expandParallelResearchWorkflow(workflow: WorkflowConfig, cwd: string): WorkflowConfig {
-	if (workflow.name !== "parallel-research") return workflow;
-	const projects = sourcePiProjects(cwd);
-	if (projects.length === 0) return workflow;
-	const chunks = chunkProjects(projects, Math.min(8, Math.max(4, Math.ceil(projects.length / 3))));
-	const exploreSteps: WorkflowStep[] = chunks.map((paths, index) => ({
-		id: `explore-shard-${index + 1}`,
-		role: "explorer",
-		parallelGroup: "explore",
-		reads: paths,
-		task: [`Explore this dynamic shard for: {goal}`, "", "Paths:", ...paths.map((item) => `- ${item}`), "", "Focus on purpose, architecture, runtime/UI patterns, package config, docs, and lessons for pi-crew."].join("\n"),
-	}));
-	return {
-		...workflow,
-		steps: [
-			{ id: "discover", role: "explorer", parallelGroup: "inventory", task: `Quickly inventory and validate ${projects.length} pi-* projects for: {goal}\n\nProjects:\n${projects.map((item) => `- ${item}`).join("\n")}\n\nDo not block shard work; summarize routing notes only.` },
-			...exploreSteps,
-			{ id: "synthesize", role: "analyst", dependsOn: exploreSteps.map((step) => step.id), task: "Synthesize all dynamic shard findings. Identify common patterns, gaps, and concrete recommendations. Use discover output if available, but prioritize completed shard outputs." },
-			{ id: "write", role: "writer", dependsOn: ["synthesize"], output: "research-summary.md", task: "Write a concise final summary with evidence, risks, and actionable next steps." },
-		],
-	};
-}
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
+
+export function sourcePiProjects(cwd: string): string[] {
+	const sourceDir = path.join(cwd, "Source");
+	try {
+		return fs.readdirSync(sourceDir, { withFileTypes: true })
+			.filter((entry) => entry.isDirectory() && entry.name.startsWith("pi-"))
+			.map((entry) => `Source/${entry.name}`)
+			.sort();
+	} catch {
+		return [];
+	}
+}
+
+export function chunkProjects(projects: string[], target = 6): string[][] {
+	const chunks = Array.from({ length: Math.min(Math.max(1, target), Math.max(1, projects.length)) }, () => [] as string[]);
+	projects.forEach((project, index) => chunks[index % chunks.length]!.push(project));
+	return chunks.filter((chunk) => chunk.length > 0);
+}
+
+export function expandParallelResearchWorkflow(workflow: WorkflowConfig, cwd: string): WorkflowConfig {
+	if (workflow.name !== "parallel-research") return workflow;
+	const projects = sourcePiProjects(cwd);
+	if (projects.length === 0) return workflow;
+	const chunks = chunkProjects(projects, Math.min(8, Math.max(4, Math.ceil(projects.length / 3))));
+	const exploreSteps: WorkflowStep[] = chunks.map((paths, index) => ({
+		id: `explore-shard-${index + 1}`,
+		role: "explorer",
+		parallelGroup: "explore",
+		reads: paths,
+		task: [`Explore this dynamic shard for: {goal}`, "", "Paths:", ...paths.map((item) => `- ${item}`), "", "Focus on purpose, architecture, runtime/UI patterns, package config, docs, and lessons for pi-crew."].join("\n"),
+	}));
+	return {
+		...workflow,
+		steps: [
+			{ id: "discover", role: "explorer", parallelGroup: "inventory", task: `Quickly inventory and validate ${projects.length} pi-* projects for: {goal}\n\nProjects:\n${projects.map((item) => `- ${item}`).join("\n")}\n\nDo not block shard work; summarize routing notes only.` },
+			...exploreSteps,
+			{ id: "synthesize", role: "analyst", dependsOn: exploreSteps.map((step) => step.id), task: "Synthesize all dynamic shard findings. Identify common patterns, gaps, and concrete recommendations. Use discover output if available, but prioritize completed shard outputs." },
+			{ id: "write", role: "writer", dependsOn: ["synthesize"], output: "research-summary.md", task: "Write a concise final summary with evidence, risks, and actionable next steps." },
+		],
+	};
+}

package/src/runtime/pi-json-output.ts

@@ -1,111 +1,111 @@
-export interface ParsedPiUsage {
-	input?: number;
-	output?: number;
-	cacheRead?: number;
-	cacheWrite?: number;
-	cost?: number;
-	turns?: number;
-}
-
-export interface ParsedPiJsonOutput {
-	jsonEvents: number;
-	textEvents: string[];
-	finalText?: string;
-	usage?: ParsedPiUsage;
-}
-
-function asRecord(value: unknown): Record<string, unknown> | undefined {
-	return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
-}
-
-function numberField(obj: Record<string, unknown>, keys: string[]): number | undefined {
-	for (const key of keys) {
-		const value = obj[key];
-		if (typeof value === "number" && Number.isFinite(value)) return value;
-	}
-	return undefined;
-}
-
-function mergeUsage(target: ParsedPiUsage, source: ParsedPiUsage): ParsedPiUsage {
-	return {
-		input: source.input ?? target.input,
-		output: source.output ?? target.output,
-		cacheRead: source.cacheRead ?? target.cacheRead,
-		cacheWrite: source.cacheWrite ?? target.cacheWrite,
-		cost: source.cost ?? target.cost,
-		turns: source.turns ?? target.turns,
-	};
-}
-
-function extractUsage(value: unknown): ParsedPiUsage | undefined {
-	const obj = asRecord(value);
-	if (!obj) return undefined;
-	const direct: ParsedPiUsage = {
-		input: numberField(obj, ["input", "inputTokens", "input_tokens"]),
-		output: numberField(obj, ["output", "outputTokens", "output_tokens"]),
-		cacheRead: numberField(obj, ["cacheRead", "cache_read", "cacheReadTokens", "cache_read_tokens"]),
-		cacheWrite: numberField(obj, ["cacheWrite", "cache_write", "cacheWriteTokens", "cache_write_tokens"]),
-		cost: numberField(obj, ["cost", "costUsd", "cost_usd"]),
-		turns: numberField(obj, ["turns", "turnCount", "turn_count"]),
-	};
-	if (Object.values(direct).some((entry) => entry !== undefined)) return direct;
-	for (const key of ["usage", "tokenUsage", "tokens", "stats"]) {
-		const nested = extractUsage(obj[key]);
-		if (nested) return nested;
-	}
-	return undefined;
-}
-
-function textFromContent(content: unknown): string[] {
-	if (typeof content === "string") return [content];
-	if (!Array.isArray(content)) return [];
-	const text: string[] = [];
-	for (const part of content) {
-		const obj = asRecord(part);
-		if (!obj) continue;
-		if (obj.type === "text" && typeof obj.text === "string") text.push(obj.text);
-		else if (typeof obj.content === "string") text.push(obj.content);
-	}
-	return text;
-}
-
-function extractText(value: unknown): string[] {
-	const obj = asRecord(value);
-	if (!obj) return [];
-	const message = asRecord(obj.message);
-	if (message?.role !== undefined && message.role !== "assistant") return [];
-	const text: string[] = [];
-	if (typeof obj.text === "string") text.push(obj.text);
-	if (typeof obj.output === "string") text.push(obj.output);
-	if (typeof obj.finalOutput === "string") text.push(obj.finalOutput);
-	if (typeof obj.final_output === "string") text.push(obj.final_output);
-	if (!message) text.push(...textFromContent(obj.content));
-	if (message) text.push(...textFromContent(message.content));
-	return text.filter((entry) => entry.trim().length > 0);
-}
-
-export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
-	let jsonEvents = 0;
-	const textEvents: string[] = [];
-	let usage: ParsedPiUsage | undefined;
-	for (const line of stdout.split("\n")) {
-		const trimmed = line.trim();
-		if (!trimmed) continue;
-		let event: unknown;
-		try {
-			event = JSON.parse(trimmed) as unknown;
-		} catch {
-			continue;
-		}
-		jsonEvents++;
-		textEvents.push(...extractText(event));
-		const eventUsage = extractUsage(event);
-		if (eventUsage) usage = mergeUsage(usage ?? {}, eventUsage);
-	}
-	return {
-		jsonEvents,
-		textEvents,
-		finalText: textEvents.length > 0 ? textEvents[textEvents.length - 1] : undefined,
-		usage,
-	};
-}
+export interface ParsedPiUsage {
+	input?: number;
+	output?: number;
+	cacheRead?: number;
+	cacheWrite?: number;
+	cost?: number;
+	turns?: number;
+}
+
+export interface ParsedPiJsonOutput {
+	jsonEvents: number;
+	textEvents: string[];
+	finalText?: string;
+	usage?: ParsedPiUsage;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+	return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+}
+
+function numberField(obj: Record<string, unknown>, keys: string[]): number | undefined {
+	for (const key of keys) {
+		const value = obj[key];
+		if (typeof value === "number" && Number.isFinite(value)) return value;
+	}
+	return undefined;
+}
+
+function mergeUsage(target: ParsedPiUsage, source: ParsedPiUsage): ParsedPiUsage {
+	return {
+		input: source.input ?? target.input,
+		output: source.output ?? target.output,
+		cacheRead: source.cacheRead ?? target.cacheRead,
+		cacheWrite: source.cacheWrite ?? target.cacheWrite,
+		cost: source.cost ?? target.cost,
+		turns: source.turns ?? target.turns,
+	};
+}
+
+function extractUsage(value: unknown): ParsedPiUsage | undefined {
+	const obj = asRecord(value);
+	if (!obj) return undefined;
+	const direct: ParsedPiUsage = {
+		input: numberField(obj, ["input", "inputTokens", "input_tokens"]),
+		output: numberField(obj, ["output", "outputTokens", "output_tokens"]),
+		cacheRead: numberField(obj, ["cacheRead", "cache_read", "cacheReadTokens", "cache_read_tokens"]),
+		cacheWrite: numberField(obj, ["cacheWrite", "cache_write", "cacheWriteTokens", "cache_write_tokens"]),
+		cost: numberField(obj, ["cost", "costUsd", "cost_usd"]),
+		turns: numberField(obj, ["turns", "turnCount", "turn_count"]),
+	};
+	if (Object.values(direct).some((entry) => entry !== undefined)) return direct;
+	for (const key of ["usage", "tokenUsage", "tokens", "stats"]) {
+		const nested = extractUsage(obj[key]);
+		if (nested) return nested;
+	}
+	return undefined;
+}
+
+function textFromContent(content: unknown): string[] {
+	if (typeof content === "string") return [content];
+	if (!Array.isArray(content)) return [];
+	const text: string[] = [];
+	for (const part of content) {
+		const obj = asRecord(part);
+		if (!obj) continue;
+		if (obj.type === "text" && typeof obj.text === "string") text.push(obj.text);
+		else if (typeof obj.content === "string") text.push(obj.content);
+	}
+	return text;
+}
+
+function extractText(value: unknown): string[] {
+	const obj = asRecord(value);
+	if (!obj) return [];
+	const message = asRecord(obj.message);
+	if (message?.role !== undefined && message.role !== "assistant") return [];
+	const text: string[] = [];
+	if (typeof obj.text === "string") text.push(obj.text);
+	if (typeof obj.output === "string") text.push(obj.output);
+	if (typeof obj.finalOutput === "string") text.push(obj.finalOutput);
+	if (typeof obj.final_output === "string") text.push(obj.final_output);
+	if (!message) text.push(...textFromContent(obj.content));
+	if (message) text.push(...textFromContent(message.content));
+	return text.filter((entry) => entry.trim().length > 0);
+}
+
+export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
+	let jsonEvents = 0;
+	const textEvents: string[] = [];
+	let usage: ParsedPiUsage | undefined;
+	for (const line of stdout.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		let event: unknown;
+		try {
+			event = JSON.parse(trimmed) as unknown;
+		} catch {
+			continue;
+		}
+		jsonEvents++;
+		textEvents.push(...extractText(event));
+		const eventUsage = extractUsage(event);
+		if (eventUsage) usage = mergeUsage(usage ?? {}, eventUsage);
+	}
+	return {
+		jsonEvents,
+		textEvents,
+		finalText: textEvents.length > 0 ? textEvents[textEvents.length - 1] : undefined,
+		usage,
+	};
+}