pi-crew 0.9.53 → 0.9.55

package/src/runtime/run-drift.ts

@@ -0,0 +1,219 @@
+/**
+ * Runtime drift detectors — detect state anomalies in pi-crew runs.
+ *
+ * Pattern origin: GSD-2 ADR-017 drift detection & state reconciliation.
+ * Each detector checks for a specific anomaly and returns a report.
+ * Repair handlers are idempotent — safe to run multiple times.
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import path from "node:path";
+import { logInternalError } from "../utils/internal-error.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export type DriftKind =
+	| "stale-process" // Heartbeat timeout (existing)
+	| "orphaned-claim" // Task claim without task
+	| "orphaned-worktree" // Worktree dir without active run
+	| "missing-timestamps" // State files without timestamps
+	| "status-divergence" // Manifest status ≠ status file
+	| "unregistered-run"; // State dir but no manifest
+
+export interface DriftReport {
+	kind: DriftKind;
+	runId: string;
+	details: string;
+	repaired: boolean;
+	repairResult?: string;
+}
+
+export interface DriftContext {
+	/** Root directory for crew state (.crew/) */
+	crewRoot: string;
+	/** Active run IDs (from registry) */
+	activeRunIds: Set<string>;
+	/** Manifest content if available */
+	manifest?: {
+		runId: string;
+		status: string;
+		cwd: string;
+		[k: string]: unknown;
+	};
+}
+
+// ── Detectors ────────────────────────────────────────────────────────────
+
+/**
+ * Detect task claims that reference tasks not in the manifest.
+ */
+export function detectOrphanedClaim(ctx: DriftContext): DriftReport | null {
+	if (!ctx.manifest) return null;
+	const claimsDir = path.join(ctx.crewRoot, "state", "task-claims");
+	if (!existsSync(claimsDir)) return null;
+
+	const claimFiles = readdirSync(claimsDir).filter((f) => f.endsWith(".json"));
+	for (const file of claimFiles) {
+		try {
+			const claim = JSON.parse(readFileSync(path.join(claimsDir, file), "utf-8"));
+			if (claim.runId === ctx.manifest.runId && claim.taskId) {
+				// Check if task exists in manifest tasks array
+				const tasks = (ctx.manifest as Record<string, unknown>).tasks;
+				if (Array.isArray(tasks) && !tasks.some((t: Record<string, unknown>) => t.id === claim.taskId)) {
+					return {
+						kind: "orphaned-claim",
+						runId: ctx.manifest.runId,
+						details: `Task claim '${claim.taskId}' references non-existent task`,
+						repaired: false,
+					};
+				}
+			}
+		} catch {
+			// Malformed claim file — skip
+		}
+	}
+	return null;
+}
+
+/**
+ * Detect worktree directories that don't belong to any active run.
+ */
+export function detectOrphanedWorktree(ctx: DriftContext): DriftReport | null {
+	const worktreesDir = path.join(ctx.crewRoot, "worktrees");
+	if (!existsSync(worktreesDir)) return null;
+
+	const dirs = readdirSync(worktreesDir, { withFileTypes: true })
+		.filter((d) => d.isDirectory())
+		.map((d) => d.name);
+
+	for (const dir of dirs) {
+		// Extract run ID from worktree dir name (format: <runId>-<taskId> or <runId>)
+		const runId = dir.split("-").slice(0, 5).join("-"); // heuristic: run IDs are timestamp-based
+		if (!ctx.activeRunIds.has(runId) && !ctx.activeRunIds.has(dir)) {
+			return {
+				kind: "orphaned-worktree",
+				runId: dir,
+				details: `Worktree '${dir}' has no active run`,
+				repaired: false,
+			};
+		}
+	}
+	return null;
+}
+
+/**
+ * Detect state files missing required timestamps.
+ */
+export function detectMissingTimestamps(ctx: DriftContext): DriftReport | null {
+	if (!ctx.manifest) return null;
+	const stateDir = path.join(ctx.crewRoot, "state");
+	if (!existsSync(stateDir)) return null;
+
+	// Check manifest has createdAt/updatedAt
+	const m = ctx.manifest as Record<string, unknown>;
+	if (!m.createdAt && !m.updatedAt) {
+		return {
+			kind: "missing-timestamps",
+			runId: ctx.manifest.runId,
+			details: "Manifest missing createdAt/updatedAt timestamps",
+			repaired: false,
+		};
+	}
+	return null;
+}
+
+/**
+ * Detect divergence between manifest status and individual task status files.
+ */
+export function detectStatusDivergence(ctx: DriftContext): DriftReport | null {
+	if (!ctx.manifest) return null;
+	const statusPath = path.join(ctx.crewRoot, "state", `${ctx.manifest.runId}.status`);
+	if (!existsSync(statusPath)) return null;
+
+	try {
+		const status = readFileSync(statusPath, "utf-8").trim();
+		if (status !== ctx.manifest.status) {
+			return {
+				kind: "status-divergence",
+				runId: ctx.manifest.runId,
+				details: `Manifest says '${ctx.manifest.status}' but status file says '${status}'`,
+				repaired: false,
+			};
+		}
+	} catch {
+		// Can't read status file — not drift, might be permissions
+	}
+	return null;
+}
+
+/**
+ * Detect state directories that have no corresponding manifest.
+ */
+export function detectUnregisteredRun(ctx: DriftContext): DriftReport | null {
+	const runsDir = path.join(ctx.crewRoot, "runs");
+	if (!existsSync(runsDir)) return null;
+
+	const runDirs = readdirSync(runsDir, { withFileTypes: true })
+		.filter((d) => d.isDirectory())
+		.map((d) => d.name);
+
+	for (const runId of runDirs) {
+		if (!ctx.activeRunIds.has(runId)) {
+			// Check if it has state files (manifest exists)
+			const manifestPath = path.join(runsDir, runId, "manifest.json");
+			if (existsSync(manifestPath)) {
+				try {
+					const stat = statSync(manifestPath);
+					const ageMs = Date.now() - stat.mtimeMs;
+					// Only flag if older than 1 hour (might be in-progress)
+					if (ageMs > 60 * 60 * 1000) {
+						return {
+							kind: "unregistered-run",
+							runId,
+							details: `Run '${runId}' has manifest but is not in active registry (age: ${Math.round(ageMs / 60000)}m)`,
+							repaired: false,
+						};
+					}
+				} catch {
+					// Can't stat — skip
+				}
+			}
+		}
+	}
+	return null;
+}
+
+// ── Reconciliation Loop ─────────────────────────────────────────────────
+
+const ALL_DETECTORS = [detectOrphanedClaim, detectOrphanedWorktree, detectMissingTimestamps, detectStatusDivergence, detectUnregisteredRun];
+
+/**
+ * Run all drift detectors and collect reports.
+ * Capped at maxPasses repair attempts.
+ *
+ * Pattern origin: GSD-2 ADR-017 — capped at 2 retry passes.
+ */
+export function runDriftDetection(ctx: DriftContext, maxPasses = 2): DriftReport[] {
+	const reports: DriftReport[] = [];
+
+	for (let pass = 0; pass < maxPasses; pass++) {
+		let newFindings = 0;
+
+		for (const detector of ALL_DETECTORS) {
+			try {
+				const report = detector(ctx);
+				if (report) {
+					reports.push(report);
+					newFindings++;
+				}
+			} catch (error) {
+				logInternalError("run-drift", error, `detector=${detector.name} runId=${ctx.manifest?.runId}`);
+			}
+		}
+
+		// If no new findings, stop early
+		if (newFindings === 0) break;
+	}
+
+	return reports;
+}

package/src/runtime/run-tracker.ts

@@ -0,0 +1,116 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { loadRunManifestById } from "../state/state-store.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { projectCrewRoot } from "../utils/paths.ts";
+import { isFinishedRunStatus } from "./process-status.ts";
+
+export interface ActiveRunPromise {
+	promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }>;
+	resolve: (value: { manifest: TeamRunManifest; tasks: TeamTaskState[] }) => void;
+	reject: (reason: unknown) => void;
+}
+
+const activeRunPromises = new Map<string, ActiveRunPromise>();
+
+export function registerRunPromise(runId: string): ActiveRunPromise {
+	let resolve!: (value: { manifest: TeamRunManifest; tasks: TeamTaskState[] }) => void;
+	let reject!: (reason: unknown) => void;
+	const promise = new Promise<{
+		manifest: TeamRunManifest;
+		tasks: TeamTaskState[];
+	}>((res, rej) => {
+		resolve = res;
+		reject = rej;
+	});
+	const entry: ActiveRunPromise = { promise, resolve, reject };
+	activeRunPromises.set(runId, entry);
+	return entry;
+}
+
+export function resolveRunPromise(runId: string, result: { manifest: TeamRunManifest; tasks: TeamTaskState[] }): void {
+	const entry = activeRunPromises.get(runId);
+	if (entry) {
+		entry.resolve(result);
+		activeRunPromises.delete(runId);
+	}
+}
+
+export function rejectRunPromise(runId: string, reason: unknown): void {
+	const entry = activeRunPromises.get(runId);
+	if (entry) {
+		entry.reject(reason);
+		activeRunPromises.delete(runId);
+	}
+}
+
+/**
+ * Wait for a team run to reach a terminal status.
+ * - If the run is already finished on disk, returns immediately.
+ * - If a foreground promise is registered for this runId, awaits it.
+ * - Otherwise falls back to lightweight fs.watchFile-based waiting.
+ */
+export async function waitForRun(
+	runId: string,
+	cwd: string,
+	options: { timeoutMs?: number; pollIntervalMs?: number } = {},
+): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
+	const { timeoutMs = 300_000, pollIntervalMs = 500 } = options;
+	const deadline = Date.now() + timeoutMs;
+
+	// Fast path: already terminal on disk
+	const loaded = loadRunManifestById(cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
+	if (loaded && isFinishedRunStatus(loaded.manifest.status)) {
+		return loaded;
+	}
+
+	// Medium path: foreground promise registered in this process
+	const entry = activeRunPromises.get(runId);
+	if (entry) {
+		let timer: ReturnType<typeof setTimeout> | undefined;
+		const timeoutPromise = new Promise<never>((_, reject) => {
+			timer = setTimeout(() => reject(new Error(`waitForRun timed out after ${timeoutMs}ms`)), timeoutMs);
+		});
+		try {
+			return await Promise.race([entry.promise, timeoutPromise]);
+		} finally {
+			if (timer) clearTimeout(timer);
+		}
+	}
+
+	// Slow path: background run — poll with exponential backoff capped at pollIntervalMs
+	let attempt = 0;
+	while (Date.now() < deadline) {
+		if (attempt === 0) {
+			// Early exit: if the run directory doesn't exist, don't waste time polling.
+			// Use projectCrewRoot() to honour the .pi/teams/ fallback for .pi-based
+			// projects (see issue #29). Without this, the hardcoded `.crew/state/runs/`
+			// path never resolves in projects that use the `.pi/` layout, the throw
+			// escapes via subagent-manager.ts:281, and pi crashes with uncaughtException.
+			const runDir = path.join(projectCrewRoot(cwd), "state", "runs", runId);
+			if (!fs.existsSync(runDir)) {
+				throw new Error(`Run ${runId} not found. No run directory at ${runDir}`);
+			}
+		}
+		const fresh = loadRunManifestById(cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
+		if (fresh && isFinishedRunStatus(fresh.manifest.status)) {
+			return fresh;
+		}
+		const delay = Math.min(pollIntervalMs, 50 * 2 ** Math.min(attempt, 6)); // max ~3.2s
+		await new Promise((r) => setTimeout(r, delay));
+		attempt++;
+	}
+
+	throw new Error(`waitForRun timed out after ${timeoutMs}ms`);
+}
+
+export function hasActiveRunPromise(runId: string): boolean {
+	return activeRunPromises.has(runId);
+}
+
+export function clearRunPromisesForTest(): void {
+	for (const entry of activeRunPromises.values()) {
+		entry.reject(new Error("Cleared by test"));
+	}
+	activeRunPromises.clear();
+}

package/src/runtime/run-worker.ts

@@ -0,0 +1,78 @@
+/**
+ * run-worker.ts — Unified worker spawn facade (CORE-13).
+ *
+ * Consolidates the 4 `runChildPi` call sites (task-runner, run-coalesced-task-
+ * group, dynamic-workflow-context, goal-evaluator) behind a single entry point
+ * that owns the global worker-cap (`withWorkerSlot`) wrap + the `runChildPi`
+ * call + standardized option assembly.
+ *
+ * ─── WHY ───
+ * Before CORE-13, 4 sites each inlined `withWorkerSlot(() => runChildPi(...))`
+ * (or skipped the cap for the judge). The wrapping was inconsistent and easy to
+ * drift (Sprint 2 CORE-2 had to patch 2 of the 4 sites individually). This
+ * facade centralizes the wrap so future changes (e.g. CORE-3 spawn budget) have
+ * exactly ONE place to touch.
+ *
+ * ─── CAP FLAG ───
+ * The goal-judge (goal-evaluator.ts) is intentionally exempt from the global
+ * worker-cap per RFC MAJ#3 (see global-worker-cap.ts module header). Callers
+ * pass `cap: false` to bypass `withWorkerSlot`; all worker spawns default to
+ * `cap: true` (the cap applies).
+ *
+ * ─── WHAT runWorker DOES NOT OWN ───
+ * Retry (`executeWithRetry`) stays at the caller level — the 4 sites have
+ * different retry policies and retry wraps runWorker, not the other way around.
+ * Heartbeat timers, progress persistence, and transcript parsing are also
+ * caller-owned (they need task/manifest state that runWorker does not see).
+ */
+
+import type { ChildPiRunInput, ChildPiRunResult } from "./child-pi.ts";
+import { runChildPi } from "./child-pi.ts";
+import { withWorkerSlot } from "./global-worker-cap.ts";
+
+/**
+ * Input for {@link runWorker}. Extends {@link ChildPiRunInput} with the `cap`
+ * flag that controls whether the global worker-cap semaphore is applied.
+ *
+ * All fields from `ChildPiRunInput` (cwd, task, agent, model, signal, maxTurns,
+ * graceTurns, transcriptPath, steeringFile, skillPaths, onSpawn,
+ * onLifecycleEvent, onStdoutLine, onJsonEvent, etc.) are passed through
+ * unchanged to `runChildPi`.
+ */
+export interface WorkerSpawnInput extends ChildPiRunInput {
+	/**
+	 * Whether to wrap the spawn in the global worker-cap (`withWorkerSlot`).
+	 * Default: `true` (all worker spawns).
+	 *
+	 * Set to `false` for the goal-judge (goal-evaluator.ts), which is exempt
+	 * from the cap per RFC MAJ#3 — see `global-worker-cap.ts` module header.
+	 */
+	cap?: boolean;
+}
+
+/**
+ * Spawn a child-Pi worker, applying the global worker-cap when `cap` is not
+ * `false`. This is the single entry point for all worker spawns; callers should
+ * never call `runChildPi` directly.
+ *
+ * The cap is applied AROUND `runChildPi` so the slot is released on completion
+ * OR throw (deadlock-safe via `withWorkerSlot`'s try/finally).
+ *
+ * Retry is NOT handled here — callers wrap `runWorker` in `executeWithRetry` as
+ * needed (task-runner and run-coalesced use retry; dynamic-workflow-context and
+ * goal-evaluator do not).
+ *
+ * @example
+ * // Worker spawn (cap applied):
+ * const result = await runWorker({ cwd, task, agent, model, signal, ... });
+ *
+ * // Judge spawn (cap bypassed):
+ * const result = await runWorker({ cwd, task, agent, model, signal, cap: false });
+ */
+export async function runWorker(input: WorkerSpawnInput): Promise<ChildPiRunResult> {
+	const { cap = true, ...childPiInput } = input;
+	if (cap) {
+		return withWorkerSlot(() => runChildPi(childPiInput));
+	}
+	return runChildPi(childPiInput);
+}

package/src/runtime/runtime-policy.ts

@@ -0,0 +1,29 @@
+import type { CrewRuntimeConfig } from "../config/config.ts";
+import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
+import { currentCrewDepth } from "./pi-args.ts";
+
+/**
+ * Resolve the effective runtime kind for a given task role using isolation policy.
+ * - scaffold is never overridden — scaffold stays scaffold.
+ * - If already nested (PI_CREW_DEPTH > 0), force child-process to avoid live-session nesting issues.
+ * - If the role appears in `isolationPolicy.isolatedRoles`, use child-process (crash isolation).
+ * - Otherwise, use `isolationPolicy.defaultRuntime` when configured, then fall back to globalKind.
+ */
+export function resolveTaskRuntimeKind(
+	globalKind: CrewRuntimeKind,
+	role: string,
+	isolationPolicy: CrewRuntimeConfig["isolationPolicy"],
+	env: NodeJS.ProcessEnv = process.env,
+): CrewRuntimeKind {
+	if (globalKind === "scaffold") return "scaffold";
+	// Safety: when already inside a pi-crew worker (depth > 0), never nest live-session.
+	// Live-session creates in-process Pi agent sessions, which would recursively
+	// try to use pi-crew, leading to "Cannot read properties of undefined" errors.
+	// Exception: when PI_CREW_MOCK_LIVE_SESSION is set, we're in a test harness
+	// that mocks the live-session path — forcing child-process would spawn a real
+	// pi process and hang the test.
+	if (globalKind === "live-session" && currentCrewDepth(env) > 0 && env.PI_CREW_MOCK_LIVE_SESSION !== "success") return "child-process";
+	const isolatedRoles = isolationPolicy?.isolatedRoles ?? [];
+	if (isolatedRoles.includes(role)) return "child-process";
+	return isolationPolicy?.defaultRuntime ?? globalKind;
+}

package/src/runtime/runtime-resolver.ts

@@ -0,0 +1,172 @@
+import type { PiTeamsConfig } from "../config/config.ts";
+import type { RuntimeResolutionState } from "../state/types.ts";
+import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
+
+export type CrewRuntimeMode = "auto" | "scaffold" | "child-process" | "live-session";
+
+export type CrewRuntimeSafety = "trusted" | "explicit_dry_run" | "blocked";
+
+export interface CrewRuntimeCapabilities {
+	kind: CrewRuntimeKind;
+	requestedMode: CrewRuntimeMode;
+	available: boolean;
+	fallback?: CrewRuntimeKind;
+	steer: boolean;
+	resume: boolean;
+	liveToolActivity: boolean;
+	transcript: boolean;
+	reason?: string;
+	safety: CrewRuntimeSafety;
+}
+
+export function runtimeResolutionState(runtime: CrewRuntimeCapabilities, resolvedAt = new Date().toISOString()): RuntimeResolutionState {
+	return {
+		kind: runtime.kind,
+		requestedMode: runtime.requestedMode,
+		safety: runtime.safety,
+		available: runtime.available,
+		...(runtime.fallback ? { fallback: runtime.fallback } : {}),
+		...(runtime.reason ? { reason: runtime.reason } : {}),
+		resolvedAt,
+	};
+}
+
+export async function isLiveSessionRuntimeAvailable(
+	timeoutMs = 1500,
+	env: NodeJS.ProcessEnv = process.env,
+): Promise<{ available: boolean; reason?: string }> {
+	if (env.PI_CREW_MOCK_LIVE_SESSION === "success") {
+		return {
+			available: true,
+			reason: "Mock live-session runtime is enabled.",
+		};
+	}
+	const probe = async (): Promise<{
+		available: boolean;
+		reason?: string;
+	}> => {
+		try {
+			// LAZY: optional peer dependency — probe at runtime to avoid hard dependency.
+			const mod = await import("@earendil-works/pi-coding-agent");
+			const api = mod as Record<string, unknown>;
+			const required = ["createAgentSession", "DefaultResourceLoader", "SessionManager", "SettingsManager"];
+			const missing = required.filter((name) => typeof api[name] === "undefined");
+			if (missing.length)
+				return {
+					available: false,
+					reason: `Pi SDK live-session exports missing: ${missing.join(", ")}.`,
+				};
+			return { available: true };
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			return {
+				available: false,
+				reason: `Could not load optional Pi SDK live-session runtime: ${message}`,
+			};
+		}
+	};
+	let timer: NodeJS.Timeout | undefined;
+	try {
+		return await Promise.race([
+			probe(),
+			new Promise<{ available: boolean; reason: string }>((resolve) => {
+				timer = setTimeout(
+					() =>
+						resolve({
+							available: false,
+							reason: `Timed out probing optional Pi SDK live-session runtime after ${timeoutMs}ms.`,
+						}),
+					timeoutMs,
+				);
+				timer.unref();
+			}),
+		]);
+	} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
+export async function resolveCrewRuntime(config: PiTeamsConfig, env: NodeJS.ProcessEnv = process.env): Promise<CrewRuntimeCapabilities> {
+	const requestedMode = config.runtime?.mode ?? "auto";
+	const workersDisabled = config.executeWorkers === false || env.PI_CREW_EXECUTE_WORKERS === "0" || env.PI_TEAMS_EXECUTE_WORKERS === "0";
+	if (requestedMode === "scaffold") return scaffoldCaps(requestedMode, undefined, "explicit_dry_run");
+	if (workersDisabled)
+		return scaffoldCaps(
+			requestedMode,
+			"Child worker execution disabled by config/env. Set runtime.mode=scaffold or executeWorkers=false only for dry runs.",
+			"blocked",
+		);
+	if (requestedMode === "child-process") return childCaps(requestedMode);
+	// When a child-process mock is active (tests), force auto-mode to child-process where the mock is active.
+	if (requestedMode === "auto" && env.PI_TEAMS_MOCK_CHILD_PI)
+		return childCaps(requestedMode, "PI_TEAMS_MOCK_CHILD_PI mock forces child-process runtime in auto mode.");
+	if (requestedMode === "live-session") {
+		const live = await isLiveSessionRuntimeAvailable(1500, env);
+		if (live.available) return liveCaps(requestedMode);
+		if (config.runtime?.allowChildProcessFallback === false) return scaffoldCaps(requestedMode, live.reason, "blocked");
+		return {
+			...childCaps(requestedMode),
+			fallback: "child-process",
+			reason: live.reason,
+		};
+	}
+	// auto mode: use child-process unless preferLiveSession is explicitly enabled
+	if (requestedMode === "auto") {
+		// Check for mock env var first (for testing)
+		if (env.PI_CREW_MOCK_LIVE_SESSION === "success") {
+			const live = await isLiveSessionRuntimeAvailable(1500, env);
+			if (live.available) return liveCaps(requestedMode);
+		}
+		// Then check explicit config preference
+		if (config.runtime?.preferLiveSession === true) {
+			const live = await isLiveSessionRuntimeAvailable(1500, env);
+			if (live.available) return liveCaps(requestedMode);
+		}
+	}
+	return childCaps(requestedMode);
+}
+
+function scaffoldCaps(
+	requestedMode: CrewRuntimeMode,
+	reason?: string,
+	safety: CrewRuntimeSafety = "explicit_dry_run",
+): CrewRuntimeCapabilities {
+	return {
+		kind: "scaffold",
+		requestedMode,
+		available: safety !== "blocked",
+		steer: false,
+		resume: false,
+		liveToolActivity: false,
+		transcript: false,
+		safety,
+		...(reason ? { reason } : {}),
+	};
+}
+
+function childCaps(requestedMode: CrewRuntimeMode, reason?: string): CrewRuntimeCapabilities {
+	return {
+		kind: "child-process",
+		requestedMode,
+		available: true,
+		steer: true,
+		resume: false,
+		liveToolActivity: false,
+		transcript: true,
+		safety: "trusted",
+		...(reason ? { reason } : {}),
+	};
+}
+
+function liveCaps(requestedMode: CrewRuntimeMode): CrewRuntimeCapabilities {
+	return {
+		kind: "live-session",
+		requestedMode,
+		available: true,
+		steer: true,
+		resume: true,
+		liveToolActivity: true,
+		transcript: true,
+		safety: "trusted",
+	};
+}