pi-crew 0.9.50 → 0.9.52

package/src/runtime/overflow-recovery.ts

@@ -1,206 +0,0 @@
-import { logInternalError } from "../utils/internal-error.ts";
-
-export type OverflowPhase = "none" | "compaction" | "retrying" | "recovered" | "failed";
-
-export interface OverflowRecoveryState {
-	taskId: string;
-	runId: string;
-	phase: OverflowPhase;
-	startedAt: number;
-	lastEventAt: number;
-	compactionCount: number;
-	retryCount: number;
-}
-
-export interface OverflowRecoveryCallbacks {
-	onPhaseChange?: (state: OverflowRecoveryState, previousPhase: OverflowPhase) => void;
-	onTimeout?: (state: OverflowRecoveryState) => void;
-}
-
-const PHASE_TIMEOUT_MS = 120_000; // 120 seconds per phase
-const TERMINAL_STATE_TTL_MS = 5 * 60_000;
-const MAX_TRACKED_STATES = 5000; // Defensive cap to prevent unbounded growth
-
-export class OverflowRecoveryTracker {
-	private states = new Map<string, OverflowRecoveryState>();
-	private timers = new Map<string, ReturnType<typeof setTimeout>>();
-	private callbacks: OverflowRecoveryCallbacks;
-
-	constructor(callbacks: OverflowRecoveryCallbacks = {}) {
-		this.callbacks = callbacks;
-	}
-
-	feedEvent(taskId: string, runId: string, eventType: string): OverflowPhase {
-		const key = this.keyFor(taskId, runId);
-		const existing = this.states.get(key);
-		const now = Date.now();
-
-		if (existing && existing.phase === "recovered") {
-			existing.lastEventAt = now;
-			return "recovered";
-		}
-		if (existing && existing.phase === "failed") {
-			existing.lastEventAt = now;
-			return "failed";
-		}
-
-		let phase: OverflowPhase = existing?.phase ?? "none";
-		let compactionCount = existing?.compactionCount ?? 0;
-		let retryCount = existing?.retryCount ?? 0;
-		const previousPhase = phase;
-
-		switch (eventType) {
-			case "compaction_start":
-				phase = "compaction";
-				compactionCount++;
-				break;
-			case "compaction_end":
-				// After compaction, we expect a retry; stay in compaction until retry starts
-				break;
-			case "auto_retry_start":
-				phase = "retrying";
-				retryCount++;
-				break;
-			case "auto_retry_end":
-				// After retry completes, the agent should produce a response
-				// We consider this recovered but don't finalize until agent_end
-				phase = "recovered";
-				break;
-			case "agent_end":
-				// If we were recovering and agent ends, we're recovered or failed
-				if (phase === "compaction" || phase === "retrying") {
-					phase = "failed";
-				}
-				break;
-			default:
-				// Unknown event type — no phase change
-				break;
-		}
-
-		const state: OverflowRecoveryState = {
-			taskId,
-			runId,
-			phase,
-			startedAt: existing?.startedAt ?? now,
-			lastEventAt: now,
-			compactionCount,
-			retryCount,
-		};
-
-		this.states.set(key, state);
-		this.resetTimeout(key);
-
-		// Defensive cap: if states Map exceeds MAX_TRACKED_STATES, evict the
-		// oldest terminal-state entry. Live states are protected because they
-		// have not yet reached a terminal phase.
-		if (this.states.size > MAX_TRACKED_STATES) {
-			this.evictOldestTerminalState();
-		}
-
-		if (previousPhase !== phase && this.callbacks.onPhaseChange) {
-			try {
-				this.callbacks.onPhaseChange(state, previousPhase);
-			} catch (error) {
-				logInternalError("overflow-recovery.onPhaseChange", error, `taskId=${taskId}`);
-			}
-		}
-
-		return phase;
-	}
-
-	getState(taskId: string, runId?: string): OverflowRecoveryState | undefined {
-		if (runId) return this.states.get(this.keyFor(taskId, runId));
-		return [...this.states.values()].find((state) => state.taskId === taskId);
-	}
-
-	getPhase(taskId: string, runId?: string): OverflowPhase {
-		return this.getState(taskId, runId)?.phase ?? "none";
-	}
-
-	removeTask(taskId: string, runId?: string): void {
-		const keys = runId
-			? [this.keyFor(taskId, runId)]
-			: [...this.states.entries()].filter(([, state]) => state.taskId === taskId).map(([key]) => key);
-		for (const key of keys) this.removeKey(key);
-	}
-
-	/**
-	 * Evict the oldest terminal-state entry (phase is "recovered", "failed",
-	 * or "none"). Used as a defensive cap when states.size exceeds
-	 * MAX_TRACKED_STATES. Live states in "compaction"/"retrying" phases are
-	 * never evicted by this method — they have their own TTL-driven cleanup.
-	 */
-	private evictOldestTerminalState(): void {
-		let oldestKey: string | undefined;
-		let oldestTimestamp = Infinity;
-		for (const [key, state] of this.states) {
-			const isTerminal = state.phase === "recovered" || state.phase === "failed" || state.phase === "none";
-			if (isTerminal && state.lastEventAt < oldestTimestamp) {
-				oldestTimestamp = state.lastEventAt;
-				oldestKey = key;
-			}
-		}
-		if (oldestKey !== undefined) {
-			this.removeKey(oldestKey);
-		}
-	}
-
-	dispose(): void {
-		for (const timer of this.timers.values()) clearTimeout(timer);
-		this.timers.clear();
-		this.states.clear();
-	}
-
-	private keyFor(taskId: string, runId: string): string {
-		return `${runId}\u0000${taskId}`;
-	}
-
-	private removeKey(key: string): void {
-		this.states.delete(key);
-		const timer = this.timers.get(key);
-		if (timer) clearTimeout(timer);
-		this.timers.delete(key);
-	}
-
-	private resetTimeout(key: string): void {
-		const existing = this.timers.get(key);
-		if (existing) clearTimeout(existing);
-		const current = this.states.get(key);
-		const timeoutMs =
-			current?.phase === "recovered" || current?.phase === "failed" || current?.phase === "none"
-				? TERMINAL_STATE_TTL_MS
-				: PHASE_TIMEOUT_MS;
-
-		const timer = setTimeout(() => {
-			this.timers.delete(key);
-			const state = this.states.get(key);
-			if (!state) return;
-			if (state.phase === "recovered" || state.phase === "failed" || state.phase === "none") {
-				this.states.delete(key);
-				return;
-			}
-
-			const previousPhase = state.phase;
-			state.phase = "failed";
-			state.lastEventAt = Date.now();
-
-			if (this.callbacks.onTimeout) {
-				try {
-					this.callbacks.onTimeout(state);
-				} catch (error) {
-					logInternalError("overflow-recovery.onTimeout", error, `taskId=${state.taskId}`);
-				}
-			}
-			if (this.callbacks.onPhaseChange) {
-				try {
-					this.callbacks.onPhaseChange(state, previousPhase);
-				} catch (error) {
-					logInternalError("overflow-recovery.onPhaseChange-timeout", error, `taskId=${state.taskId}`);
-				}
-			}
-		}, timeoutMs);
-
-		timer.unref();
-		this.timers.set(key, timer);
-	}
-}

package/src/runtime/parallel-research.ts

@@ -1,68 +0,0 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
-
-export function sourcePiProjects(cwd: string): string[] {
-	const sourceDir = path.join(cwd, "Source");
-	try {
-		return fs
-			.readdirSync(sourceDir, { withFileTypes: true })
-			.filter((entry) => entry.isDirectory() && entry.name.startsWith("pi-"))
-			.map((entry) => `Source/${entry.name}`)
-			.sort();
-	} catch {
-		return [];
-	}
-}
-
-export function chunkProjects(projects: string[], target = 6): string[][] {
-	const chunks = Array.from({ length: Math.min(Math.max(1, target), Math.max(1, projects.length)) }, () => [] as string[]);
-	projects.forEach((project, index) => chunks[index % chunks.length]!.push(project));
-	return chunks.filter((chunk) => chunk.length > 0);
-}
-
-export function expandParallelResearchWorkflow(workflow: WorkflowConfig, cwd: string): WorkflowConfig {
-	if (workflow.name !== "parallel-research") return workflow;
-	const projects = sourcePiProjects(cwd);
-	if (projects.length === 0) return workflow;
-	const chunks = chunkProjects(projects, Math.min(8, Math.max(4, Math.ceil(projects.length / 3))));
-	const exploreSteps: WorkflowStep[] = chunks.map((paths, index) => ({
-		id: `explore-shard-${index + 1}`,
-		role: "explorer",
-		parallelGroup: "explore",
-		reads: paths,
-		task: [
-			`Explore this dynamic shard for: {goal}`,
-			"",
-			"Paths:",
-			...paths.map((item) => `- ${item}`),
-			"",
-			"Focus on purpose, architecture, runtime/UI patterns, package config, docs, and lessons for pi-crew.",
-		].join("\n"),
-	}));
-	return {
-		...workflow,
-		steps: [
-			{
-				id: "discover",
-				role: "explorer",
-				parallelGroup: "inventory",
-				task: `Quickly inventory and validate ${projects.length} pi-* projects for: {goal}\n\nProjects:\n${projects.map((item) => `- ${item}`).join("\n")}\n\nDo not block shard work; summarize routing notes only.`,
-			},
-			...exploreSteps,
-			{
-				id: "synthesize",
-				role: "analyst",
-				dependsOn: exploreSteps.map((step) => step.id),
-				task: "Synthesize all dynamic shard findings. Identify common patterns, gaps, and concrete recommendations. Use discover output if available, but prioritize completed shard outputs.",
-			},
-			{
-				id: "write",
-				role: "writer",
-				dependsOn: ["synthesize"],
-				output: "research-summary.md",
-				task: "Write a concise final summary with evidence, risks, and actionable next steps.",
-			},
-		],
-	};
-}

package/src/runtime/parallel-utils.ts

@@ -1,160 +0,0 @@
-export interface RunnerSubagentStep {
-	agent: string;
-	task: string;
-	cwd?: string;
-	model?: string;
-	modelCandidates?: string[];
-	tools?: string[];
-	extensions?: string[];
-	mcpDirectTools?: string[];
-	systemPrompt?: string | null;
-	systemPromptMode?: "append" | "replace";
-	inheritProjectContext: boolean;
-	inheritSkills: boolean;
-	skills?: string[];
-	outputPath?: string;
-	sessionFile?: string;
-	maxSubagentDepth?: number;
-}
-
-export interface ParallelStepGroup {
-	parallel: RunnerSubagentStep[];
-	concurrency?: number;
-	failFast?: boolean;
-	worktree?: boolean;
-}
-
-export type RunnerStep = RunnerSubagentStep | ParallelStepGroup;
-
-export function isParallelGroup(step: RunnerStep): step is ParallelStepGroup {
-	return "parallel" in step && Array.isArray(step.parallel);
-}
-
-export function flattenSteps(steps: RunnerStep[]): RunnerSubagentStep[] {
-	const flat: RunnerSubagentStep[] = [];
-	for (const step of steps) {
-		if (isParallelGroup(step)) {
-			for (const task of step.parallel) flat.push(task);
-		} else {
-			flat.push(step);
-		}
-	}
-	return flat;
-}
-
-export async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, i: number) => Promise<R>): Promise<R[]> {
-	// P12 (perf): single-item fast path. Skip the Promise.all + worker-pool
-	// setup when there's exactly one item — just call fn directly. Same return
-	// shape, no concurrency overhead, no micro-task allocation.
-	if (items.length === 1) return [await fn(items[0], 0)];
-	if (items.length === 0) return [];
-	const safeLimit = Math.max(1, Math.floor(limit) || 1);
-	const results: R[] = new Array(items.length);
-	let next = 0;
-
-	const worker = async (_workerIndex: number): Promise<void> => {
-		while (next < items.length) {
-			const i = next++;
-			results[i] = await fn(items[i], i);
-		}
-	};
-
-	await Promise.all(Array.from({ length: Math.min(safeLimit, items.length) }, (_, workerIndex) => worker(workerIndex)));
-	return results;
-}
-
-/**
- * Phase 6: mapConcurrent with AbortSignal and fail-fast support.
- * On abort: returns partial results (may contain undefined entries).
- * On error: throws immediately (fail-fast) and cancels remaining work.
- */
-/** @internal */
-async function mapConcurrentWithSignal<T, R>(
-	items: T[],
-	limit: number,
-	fn: (item: T, i: number, signal: AbortSignal) => Promise<R>,
-	signal?: AbortSignal,
-): Promise<{ results: (R | undefined)[]; aborted: boolean }> {
-	const safeLimit = Math.max(1, Math.floor(limit) || 1);
-	const results: (R | undefined)[] = new Array(items.length);
-	let next = 0;
-	let aborted = false;
-
-	const abortController = new AbortController();
-	const workerSignal = signal ? AbortSignal.any([signal, abortController.signal]) : abortController.signal;
-
-	let rejectFirst: (error: unknown) => void;
-	const firstErrorPromise = new Promise<never>((_, reject) => {
-		rejectFirst = reject;
-	});
-
-	const worker = async (): Promise<void> => {
-		while (!workerSignal.aborted) {
-			const i = next++;
-			if (i >= items.length) return;
-			try {
-				results[i] = await fn(items[i], i, workerSignal);
-			} catch (error) {
-				if (!workerSignal.aborted) {
-					abortController.abort();
-					rejectFirst(error);
-					throw error;
-				}
-			}
-		}
-	};
-
-	const workers = Array.from({ length: Math.min(safeLimit, items.length) }, () => worker());
-
-	try {
-		await Promise.race([Promise.all(workers), firstErrorPromise]);
-	} catch (error) {
-		if (signal?.aborted) {
-			aborted = true;
-			return { results, aborted };
-		}
-		throw error;
-	}
-
-	return { results, aborted: signal?.aborted ?? false };
-}
-
-export interface ParallelTaskResult {
-	agent: string;
-	taskIndex?: number;
-	output: string;
-	exitCode: number | null;
-	error?: string;
-	model?: string;
-	attemptedModels?: string[];
-	outputTargetPath?: string;
-	outputTargetExists?: boolean;
-}
-
-export function aggregateParallelOutputs(
-	results: ParallelTaskResult[],
-	headerFormat: (index: number, agent: string) => string = (i, agent) => `=== Parallel Task ${i + 1} (${agent}) ===`,
-): string {
-	return results
-		.map((r, i) => {
-			const header = headerFormat(r.taskIndex ?? i, r.agent);
-			const hasOutput = Boolean(r.output?.trim());
-			const status =
-				r.exitCode === -1
-					? "SKIPPED"
-					: r.exitCode == null || r.exitCode !== 0
-						? `FAILED (exit code ${r.exitCode})${r.error ? `: ${r.error}` : ""}`
-						: r.error
-							? `WARNING: ${r.error}`
-							: !hasOutput && r.outputTargetPath && r.outputTargetExists === false
-								? `EMPTY OUTPUT (expected output file missing: ${r.outputTargetPath})`
-								: !hasOutput && !r.outputTargetPath
-									? "EMPTY OUTPUT (no textual response returned)"
-									: "";
-			const body = status ? (hasOutput ? `${status}\n${r.output}` : status) : r.output;
-			return `${header}\n${body}`;
-		})
-		.join("\n\n");
-}
-
-export const MAX_PARALLEL_CONCURRENCY = 4;

package/src/runtime/parent-guard.ts

@@ -1,134 +0,0 @@
-/**
- * Parent liveness guard for pi-crew worker processes.
- *
- * Workers call `startParentGuard(parentPid)` at startup. A lightweight
- * interval checks if the parent PID is still alive. When the parent dies
- * (SIGKILL, crash, power loss, terminal close), the worker self-terminates
- * immediately — no sentinel process needed.
- *
- * Note: `process.kill(pid, 0)` works on both Unix and Windows in Node.js
- * for checking process existence. On Windows, it may throw for processes
- * owned by other users (permission error), but correctly detects dead PIDs.
- *
- * Usage in worker entry points:
- * ```ts
- * const parentPid = Number(process.env.PI_CREW_PARENT_PID);
- * if (parentPid > 0) startParentGuard(parentPid);
- * ```
- *
- * ## Trust model
- *
- * `PI_CREW_PARENT_PID` is propagated from the spawning team-runner to each
- * child via the explicit env allow-list in `child-pi.ts`. The PID itself is
- * NOT a secret — process identifiers are public knowledge on Unix-like
- * systems (visible in `ps`, `/proc`, etc.) and carry no privilege.
- *
- * Residual risk: a malicious or compromised child can spoof
- * `process.env.PI_CREW_PARENT_PID` before invoking `startParentGuard()`.
- * This is acceptable because the parent-guard is a self-termination
- * signal, NOT a security boundary:
- *   - It only causes the (already-compromised) child to exit earlier.
- *   - A truly malicious child can simply not call `startParentGuard()`.
- *   - Real protection against hostile children comes from the env-filter
- *     allowlist and redaction — all enforced before spawn.
- *
- * The guard exists for the benign case: a parent dies (user closes the
- * terminal, pi crashes, machine loses power) and we want all detached
- * workers to stop wasting CPU. It is NOT an anti-tampering mechanism.
- */
-
-/**
- * Poll interval for parent liveness checks (in milliseconds).
- * Default: 500ms. A parent killed by SIGKILL is detected within one poll
- * interval (max 500ms latency).
- *
- * For latency-sensitive workloads (e.g., preventing orphaned workers from
- * running with a dead supervisor), you can tune this lower by setting the
- * PI_CREW_PARENT_GUARD_INTERVAL_MS environment variable.
- *
- * WARNING: Values below 100ms significantly increase overhead for large
- * numbers of parallel workers, since each poll issues a process.kill(pid, 0)
- * syscall per worker. Only tune this if immediate detection is critical.
- *
- * FUTURE: An event-based SIGCHLD handler could supplement or replace this
- * polling approach for near-instantaneous parent-death detection on Unix
- * systems, avoiding the polling overhead entirely.
- */
-const POLL_INTERVAL_MS = Number(process.env.PI_CREW_PARENT_GUARD_INTERVAL_MS) || 500;
-
-const guardIntervals = new Map<number, ReturnType<typeof setInterval>>();
-
-function isPidAlive(pid: number): boolean {
-	try {
-		process.kill(pid, 0);
-		return true;
-	} catch {
-		return false;
-	}
-}
-
-function selfTerminate(parentPid: number): never {
-	// Best-effort: try to log why we're dying
-	try {
-		if (typeof process.stderr?.write === "function") {
-			process.stderr.write(`[pi-crew] Parent process ${parentPid} is dead — self-terminating worker ${process.pid}\n`);
-		}
-	} catch {
-		// Ignore
-	}
-	process.exit(124); // 124 = "parent died" exit code
-}
-
-/**
- * Start a lightweight poll that checks if the parent process is still alive.
- * If the parent dies, this worker exits immediately with code 124.
- *
- * FIX: Removed unref() — the guard interval MUST keep the event loop alive
- * to prevent premature worker exit when the parent is still alive but the
- * worker has no other pending work (LLM calls, timers, I/O). Without this,
- * a worker in pure CPU wait could exit even though its parent is alive.
- */
-export function startParentGuard(parentPid: number): void {
-	if (!parentPid || !Number.isFinite(parentPid) || parentPid <= 0) return;
-
-	// Clear any existing guard for this specific parentPid before setting a new one
-	const existing = guardIntervals.get(parentPid);
-	if (existing) clearInterval(existing);
-
-	// Add ±20% jitter to prevent synchronized polling across workers that
-	// start simultaneously (e.g., after pi restart). Without jitter, all
-	// workers would poll at exactly the same interval, creating load spikes.
-	const jitter = POLL_INTERVAL_MS * 0.2 * (Math.random() - 0.5) * 2;
-	const actualInterval = POLL_INTERVAL_MS + jitter;
-
-	const interval = setInterval(() => {
-		// Immediate check on every tick — detects parent death within one poll
-		// interval (max POLL_INTERVAL_MS latency, default 500ms).
-		if (!isPidAlive(parentPid)) {
-			const guard = guardIntervals.get(parentPid);
-			if (guard) clearInterval(guard);
-			guardIntervals.delete(parentPid);
-			selfTerminate(parentPid);
-		}
-	}, actualInterval);
-
-	guardIntervals.set(parentPid, interval);
-
-	// NOTE: Intentionally NOT calling guardInterval.unref() here.
-	// The watchdog timer must keep the event loop alive to ensure the worker
-	// doesn't exit while the parent is alive. If other work (child processes,
-	// timers, I/O) keeps the loop alive, that's fine — the guard runs as a
-	// side effect. If no other work exists, the guard is the only thing
-	// keeping the process alive, and that's by design.
-}
-
-/**
- * Stop the parent guard. Called when the worker finishes normally
- * and doesn't need to watch the parent anymore.
- */
-export function stopParentGuard(): void {
-	for (const interval of guardIntervals.values()) {
-		clearInterval(interval);
-	}
-	guardIntervals.clear();
-}