pi-crew 0.9.53 → 0.9.55

package/src/runtime/foreground-control.ts

@@ -0,0 +1,193 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { atomicWriteFile } from "../state/atomic-write.ts";
+import { appendEvent } from "../state/event-log.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import { sleepSync } from "../utils/sleep.ts";
+import { readCrewAgents } from "./crew-agent-records.ts";
+import { checkProcessLiveness, isActiveRunStatus } from "./process-status.ts";
+
+export type ForegroundControlRequestType = "interrupt" | "status";
+
+export interface ForegroundControlStatus {
+	runId: string;
+	status: TeamRunManifest["status"];
+	active: boolean;
+	asyncPid?: number;
+	asyncAlive?: boolean;
+	runningTasks: string[];
+	runningAgents: string[];
+	controlPath: string;
+	lastRequest?: ForegroundControlRequest;
+}
+
+export interface ForegroundControlRequest {
+	id: string;
+	type: ForegroundControlRequestType;
+	createdAt: string;
+	reason: string;
+	acknowledged: boolean;
+}
+
+export function foregroundControlPath(manifest: TeamRunManifest): string {
+	return path.join(manifest.stateRoot, "foreground-control.json");
+}
+
+function readLastRequest(controlPath: string): ForegroundControlRequest | undefined {
+	if (!fs.existsSync(controlPath)) return undefined;
+	try {
+		const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as {
+			requests?: ForegroundControlRequest[];
+		};
+		return parsed.requests?.at(-1);
+	} catch {
+		return undefined;
+	}
+}
+
+export function readForegroundControlStatus(manifest: TeamRunManifest, tasks: TeamTaskState[]): ForegroundControlStatus {
+	const controlPath = foregroundControlPath(manifest);
+	const asyncAlive = manifest.async?.pid !== undefined ? checkProcessLiveness(manifest.async.pid).alive : undefined;
+	return {
+		runId: manifest.runId,
+		status: manifest.status,
+		active: isActiveRunStatus(manifest.status),
+		asyncPid: manifest.async?.pid,
+		asyncAlive,
+		runningTasks: tasks.filter((task) => task.status === "running").map((task) => task.id),
+		runningAgents: readCrewAgents(manifest)
+			.filter((agent) => agent.status === "running")
+			.map((agent) => agent.id),
+		controlPath,
+		lastRequest: readLastRequest(controlPath),
+	};
+}
+
+export function writeForegroundInterruptRequest(
+	manifest: TeamRunManifest,
+	reason = "User requested foreground interrupt.",
+): ForegroundControlRequest {
+	const controlPath = foregroundControlPath(manifest);
+	const lockDir = `${controlPath}.lock`;
+	const pidFile = path.join(lockDir, "pid");
+	let requests: ForegroundControlRequest[] = [];
+
+	// FIX: Use file locking to prevent race condition in read-modify-write
+	// Added stale lock detection similar to event-log.ts
+	const acquireLock = (): void => {
+		const timeout = 5000;
+		const staleMs = 10000;
+		const start = Date.now();
+		while (true) {
+			try {
+				fs.mkdirSync(lockDir, { recursive: true });
+				try {
+					atomicWriteFile(pidFile, String(process.pid));
+				} catch {
+					/* best-effort */
+				}
+				break;
+			} catch {
+				if (Date.now() - start > timeout) {
+					// Check if lock is stale before giving up
+					try {
+						const raw = fs.readFileSync(pidFile, "utf-8").trim();
+						const ownerPid = Number.parseInt(raw, 10);
+						if (!Number.isNaN(ownerPid) && ownerPid !== process.pid) {
+							let alive = false;
+							try {
+								process.kill(ownerPid, 0);
+								alive = true;
+							} catch {
+								/* dead */
+							}
+							if (!alive) {
+								// Lock is stale — clear it and retry
+								fs.rmSync(lockDir, {
+									recursive: true,
+									force: true,
+								});
+								continue;
+							}
+							// Lock held by live process — throw
+							const err = new Error(`Foreground control lock timeout for ${controlPath}`);
+							logInternalError("foreground-control.lock-timeout", err, `controlPath=${controlPath}`);
+							throw err;
+						}
+					} catch {
+						/* no pid file — continue to throw */
+					}
+					const err = new Error(`Foreground control lock timeout for ${controlPath}`);
+					logInternalError("foreground-control.lock-timeout", err, `controlPath=${controlPath}`);
+					throw err;
+				}
+				// Stale detection: if the owning process is dead, remove the stale lock
+				try {
+					const raw = fs.readFileSync(pidFile, "utf-8").trim();
+					const ownerPid = Number.parseInt(raw, 10);
+					if (!Number.isNaN(ownerPid) && ownerPid !== process.pid) {
+						let alive = false;
+						try {
+							process.kill(ownerPid, 0);
+							alive = true;
+						} catch {
+							/* dead */
+						}
+						if (!alive) {
+							const stat = fs.statSync(lockDir);
+							if (Date.now() - stat.mtimeMs > staleMs) {
+								fs.rmSync(lockDir, {
+									recursive: true,
+									force: true,
+								});
+								continue;
+							}
+						}
+					}
+				} catch {
+					/* no pid file — fall through to sleep */
+				}
+				// Brief sleep to avoid CPU spinning
+				sleepSync(10);
+			}
+		}
+	};
+	const releaseLock = (): void => {
+		try {
+			fs.rmSync(lockDir, { recursive: true, force: true });
+		} catch {
+			/* best-effort */
+		}
+	};
+
+	acquireLock();
+	try {
+		if (fs.existsSync(controlPath)) {
+			try {
+				const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as { requests?: ForegroundControlRequest[] };
+				requests = Array.isArray(parsed.requests) ? parsed.requests : [];
+			} catch {
+				requests = [];
+			}
+		}
+		const request: ForegroundControlRequest = {
+			id: `fg_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}`,
+			type: "interrupt",
+			createdAt: new Date().toISOString(),
+			reason,
+			acknowledged: false,
+		};
+		fs.mkdirSync(path.dirname(controlPath), { recursive: true });
+		atomicWriteFile(controlPath, `${JSON.stringify({ requests: [...requests, request] }, null, 2)}\n`);
+		appendEvent(manifest.eventsPath, {
+			type: "foreground.interrupt_requested",
+			runId: manifest.runId,
+			message: reason,
+			data: { requestId: request.id, controlPath },
+		});
+		return request;
+	} finally {
+		releaseLock();
+	}
+}

package/src/runtime/foreground-watchdog.ts

@@ -0,0 +1,132 @@
+/**
+ * Foreground run watchdog — periodically checks that active foreground runs
+ * are making progress and auto-notifies the assistant if a run appears hung.
+ *
+ * Problem: foreground runs run in background via startForegroundRun(). The Pi
+ * assistant has no way to know when a run completes or gets stuck without
+ * manual polling. This watchdog monitors active runs and:
+ *
+ * 1. Detects hung runs (active status, no heartbeat update for >10 min)
+ * 2. Injects a followUp message via pi.sendUserMessage() so the assistant
+ *    is automatically notified — no manual sleep+check needed.
+ * 3. Cleans up after itself when the run completes or the session ends.
+ */
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { loadRunManifestById } from "../state/state-store.ts";
+import { readCrewAgents } from "./crew-agent-records.ts";
+import { isActiveRunStatus, isLikelyOrphanedActiveRun } from "./process-status.ts";
+
+export interface WatchdogOptions {
+	pi: ExtensionAPI;
+	cwd: string;
+	runId: string;
+	/** Check interval in ms. Default: 5 minutes. */
+	checkIntervalMs?: number;
+	/** Maximum time to monitor in ms. Default: 2 hours. */
+	maxMonitorMs?: number;
+}
+
+const DEFAULT_CHECK_INTERVAL_MS = 300_000; // 5 minutes
+const DEFAULT_MAX_MONITOR_MS = 7_200_000; // 2 hours
+
+/** Active watchdog timers — keyed by runId for cleanup. */
+const activeWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
+
+/** Stop a specific watchdog by runId. */
+export function stopWatchdog(runId: string): void {
+	const timer = activeWatchdogs.get(runId);
+	if (timer) {
+		clearTimeout(timer);
+		activeWatchdogs.delete(runId);
+	}
+}
+
+/** @internal Stop all active watchdogs. Called on session shutdown. */
+function stopAllWatchdogs(): void {
+	for (const [runId, timer] of activeWatchdogs) {
+		clearTimeout(timer);
+	}
+	activeWatchdogs.clear();
+}
+
+/**
+ * Start a periodic watchdog for a foreground run.
+ * Checks at regular intervals whether the run is still progressing.
+ * If the run appears hung (no update for >10 min with no active agents),
+ * injects a followUp message into the Pi conversation.
+ *
+ * Automatically stops when:
+ * - The run reaches a terminal status (completed/failed/cancelled)
+ * - The max monitor time is exceeded
+ * - Explicitly stopped via stopWatchdog()
+ */
+export function startForegroundWatchdog(opts: WatchdogOptions): void {
+	const { pi, cwd, runId } = opts;
+	const checkIntervalMs = opts.checkIntervalMs ?? DEFAULT_CHECK_INTERVAL_MS;
+	const maxMonitorMs = opts.maxMonitorMs ?? DEFAULT_MAX_MONITOR_MS;
+	const startTime = Date.now();
+
+	// Don't stack watchdogs for the same run
+	if (activeWatchdogs.has(runId)) return;
+
+	const check = (): void => {
+		// Check if max monitor time exceeded
+		if (Date.now() - startTime > maxMonitorMs) {
+			activeWatchdogs.delete(runId);
+			return;
+		}
+
+		try {
+			const loaded = loadRunManifestById(cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
+			if (!loaded) {
+				// Run not found — stop watchdog
+				activeWatchdogs.delete(runId);
+				return;
+			}
+
+			const { manifest } = loaded;
+
+			// Terminal status — send completion notification and stop
+			if (!isActiveRunStatus(manifest.status)) {
+				const teamName = manifest.team ?? "unknown";
+				try {
+					pi.sendUserMessage(`pi-crew run ${manifest.status}: ${runId} (${teamName}/${manifest.workflow ?? "default"})`, {
+						deliverAs: "followUp",
+					});
+				} catch {
+					/* non-critical */
+				}
+				activeWatchdogs.delete(runId);
+				return;
+			}
+
+			// Check if run appears hung
+			const agents = readCrewAgents(manifest);
+			const now = Date.now();
+			if (isLikelyOrphanedActiveRun(manifest, agents, now)) {
+				const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}`;
+				try {
+					pi.sendUserMessage(
+						`pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
+						{ deliverAs: "followUp" },
+					);
+				} catch {
+					/* non-critical */
+				}
+				// Don't stop — keep monitoring. The assistant or user may intervene.
+			}
+		} catch {
+			// Non-critical — skip this check
+		}
+
+		// Schedule next check
+		const timer = setTimeout(check, checkIntervalMs);
+		timer.unref(); // Don't prevent process exit
+		activeWatchdogs.set(runId, timer);
+	};
+
+	// First check after initial interval
+	const timer = setTimeout(check, checkIntervalMs);
+	timer.unref();
+	activeWatchdogs.set(runId, timer);
+}

package/src/runtime/global-worker-cap.ts

@@ -0,0 +1,96 @@
+/**
+ * global-worker-cap.ts — Global WORKER-process concurrency cap (P1g).
+ *
+ * RFC: research-findings/goal-workflow/13-VISION-RFC.md v0.5 §P1g + MAJ#3.
+ *
+ * Bounds concurrent WORKER spawns (worker turns, executeTeamRun, dynamic-
+ * workflow ctx.agent()/fanOut) to prevent fork-storm DoS. The cap is a fair
+ * async semaphore (FIFO queue) defaulting to max(2, os.cpus().length - 2),
+ * overridable by env PI_CREW_MAX_WORKERS (parse int; invalid/missing → the
+ * computed default).
+ *
+ * ─── WHY THE GOAL-JUDGE IS EXEMPT (RFC MAJ#3) ───
+ * Do NOT route goal-judge spawns through this cap. The judge is naturally
+ * bounded — exactly 1 judge per turn, maxTurns:3, no tools (it emits a short
+ * JSON verdict, not long agentic loops). A goal cannot spawn many judges, so
+ * the judge is NOT a fork-storm vector. Routing the judge through the cap
+ * would risk DEADLOCK under contention: a judge could wait on a worker slot
+ * that never frees (e.g. all slots held by workers waiting on the judge's
+ * verdict). Bounding WORKERS alone bounds the real DoS surface; the exempt
+ * judge cannot starve them. Workers must therefore call acquireWorkerSlot()
+ * around their runChildPi spawns; judge spawns (goal-evaluator) must not.
+ */
+
+import * as os from "node:os";
+import { Semaphore } from "./semaphore.ts";
+
+/**
+ * Resolve the worker-cap capacity from PI_CREW_MAX_WORKERS or the computed
+ * default. Invalid env values (non-numeric, ≤0, NaN) fall back to the default
+ * rather than silently disabling the cap.
+ */
+function resolveCapacity(): number {
+	const env = process.env.PI_CREW_MAX_WORKERS;
+	if (env !== undefined && env !== "") {
+		const parsed = Number.parseInt(env, 10);
+		if (Number.isFinite(parsed) && parsed > 0) {
+			return parsed;
+		}
+		// Invalid env value → fall back to default (don't silently accept 0/negative).
+	}
+	const cpus = os.cpus().length;
+	return Math.max(2, cpus - 2);
+}
+
+let capacity: number = resolveCapacity();
+let semaphore: Semaphore = new Semaphore(capacity);
+
+/** Resolved capacity of the global worker cap (env override or default). */
+export function getWorkerCapCapacity(): number {
+	return capacity;
+}
+
+/**
+ * @internal Test-only: reinitialize the cap with a specific capacity and a
+ * fresh empty queue. Production code must not call this.
+ */
+export function __test_resetCap(testCapacity: number): void {
+	capacity = Math.max(1, testCapacity);
+	semaphore = new Semaphore(capacity);
+}
+
+/**
+ * Acquire a global worker slot. Resolves immediately if under cap, else queues
+ * (FIFO) until a slot frees. MUST be paired with releaseWorkerSlot().
+ *
+ * Used to bound WORKER spawns only. Do NOT route the goal-judge through this
+ * (see the RFC MAJ#3 rationale in the module header).
+ */
+export async function acquireWorkerSlot(): Promise<void> {
+	await semaphore.acquire();
+}
+
+/**
+ * Release a previously acquired worker slot. Over-release (calling release
+ * without a matching acquire) is a no-op (the underlying Semaphore guards it).
+ */
+export function releaseWorkerSlot(): void {
+	semaphore.release();
+}
+
+/**
+ * Convenience: acquire a worker slot, run `fn`, and release on completion OR
+ * throw. The slot is ALWAYS released — including when `fn` rejects — so a
+ * throwing worker never leaks a slot (deadlock prevention).
+ *
+ * Example:
+ *   const result = await withWorkerSlot(() => runChildPi(...));
+ */
+export async function withWorkerSlot<T>(fn: () => Promise<T>): Promise<T> {
+	await acquireWorkerSlot();
+	try {
+		return await fn();
+	} finally {
+		releaseWorkerSlot();
+	}
+}

package/src/runtime/goal-achievement.ts

@@ -0,0 +1,148 @@
+/**
+ * #2 (assessment): goal-achievement detection — kills the "false-green" lie.
+ *
+ * The v0.9.13 assessment found run team_20260626170635 reported terminal-success
+ * ("completed") while its verifier wrote: "did NOT apply ANY of the three
+ * security fixes. git diff --stat for all 6 target files is empty… Tests are
+ * green only because nothing was changed." The run status LIED.
+ *
+ * This module assesses whether a completed run actually achieved its goal, so
+ * the false-green is no longer SILENT. It is deliberately conservative:
+ *   - read-only / doc-only workflows are never accused (they legitimately make
+ *     no project-code edits; their outputs land in gitignored .crew/),
+ *   - a false-green is only flagged for CODE-MUTATING runs (executor /
+ *     test-engineer steps) in a git repo whose working tree is empty,
+ *   - status is downgraded to "failed" ONLY when a corroborating signal (a
+ *     task that actually failed) confirms it; otherwise the suspicion is
+ *     exposed via `goalAchieved=false` + a prominent event + manifest note,
+ *     without breaking a legitimately-no-op run.
+ */
+import { spawnSync } from "node:child_process";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { findRepoRoot } from "../utils/paths.ts";
+import type { WorkflowConfig } from "../workflows/workflow-config.ts";
+
+/**
+ * Roles whose job is to EDIT project source/test files (changes that appear in
+ * `git status`). Deliberately NARROW: writer/verifier/security-reviewer write
+ * reports (often to gitignored .crew/), so an empty diff is NOT a failure
+ * signal for them. Only executor/test-engineer MUST touch project code.
+ */
+export const MUTATING_ROLES = new Set<string>(["executor", "test-engineer"]);
+
+export type GoalAchieved = boolean | "unknown";
+
+export interface GoalAchievementAssessment {
+	achieved: GoalAchieved;
+	reason: string;
+	/** human-readable corroborating signals (for events/manifest) */
+	signals: string[];
+}
+
+/** Does this workflow contain any code-mutating role step? */
+export function workflowIsMutating(workflow: WorkflowConfig | undefined): boolean {
+	const steps = workflow?.steps ?? [];
+	return steps.some((step) => typeof step?.role === "string" && MUTATING_ROLES.has(step.role));
+}
+
+/** Is the working tree of the repo containing `cwd` clean (no changes)? Throws-safe. */
+export function isGitWorkingTreeClean(cwd: string): { clean: boolean; repoRoot: string } | { clean: "unknown"; repoRoot: undefined } {
+	try {
+		const repoRoot = findRepoRoot(cwd);
+		if (!repoRoot) return { clean: "unknown", repoRoot: undefined };
+		// `git status --porcelain` → empty stdout means a clean working tree.
+		const res = spawnSync("git", ["-C", repoRoot, "status", "--porcelain"], { encoding: "utf8", timeout: 5_000 });
+		if (res.error || res.status !== 0) return { clean: "unknown", repoRoot: undefined };
+		return { clean: res.stdout.trim().length === 0, repoRoot };
+	} catch {
+		return { clean: "unknown", repoRoot: undefined };
+	}
+}
+
+/**
+ * Assess whether a completed run achieved its goal. Pure function over manifest
+ * + tasks + workflow — no side effects, easy to unit-test.
+ */
+export function assessGoalAchievement(
+	manifest: TeamRunManifest,
+	tasks: TeamTaskState[],
+	workflow: WorkflowConfig | undefined,
+): GoalAchievementAssessment {
+	// (1) Only code-mutating workflows can be false-green. Read-only/doc runs
+	//     legitimately make no project edits — never accuse them.
+	if (!workflowIsMutating(workflow)) {
+		return {
+			achieved: "unknown",
+			reason: "read-only / doc-only workflow (no executor/test-engineer steps)",
+			signals: [],
+		};
+	}
+
+	// (2) Need a git repo to inspect the working tree.
+	const tree = isGitWorkingTreeClean(manifest.cwd);
+	if (tree.clean === "unknown" || !tree.repoRoot) {
+		return {
+			achieved: "unknown",
+			reason: "not a git repo or git unavailable",
+			signals: [],
+		};
+	}
+
+	// (3) Non-empty working tree → the mutating run made edits. Achieved.
+	if (!tree.clean) {
+		return {
+			achieved: true,
+			reason: "git working tree has changes (mutating run edited project files)",
+			signals: [`repoRoot=${tree.repoRoot}`],
+		};
+	}
+
+	// (4) Mutating run + CLEAN working tree → suspicious. The executor's job was
+	//     to edit code and it left zero diff. This is the false-green signature.
+	const failedTask = tasks.find((t) => t.status === "failed");
+	const signals = ["git working tree clean despite mutating workflow"];
+	if (failedTask) signals.push(`corroborating failed task: ${failedTask.id} (${failedTask.role})`);
+
+	return {
+		// `false` = false-green detected. Whether to downgrade status is decided
+		// by the caller based on the corroborating failed-task signal.
+		achieved: false,
+		reason: "code-mutating run completed but made no project edits (false-green)",
+		signals,
+	};
+}
+
+/**
+ * Apply an assessment to a run result: set manifest.goalAchieved + note, and
+ * decide whether to downgrade status. Returns the (possibly mutated) manifest.
+ *
+ * Downgrade rule (conservative): only flip "completed" → "failed" when a
+ * corroborating failed-task signal is present. Otherwise expose goalAchieved
+ * = false + a manifest note + let the caller emit an event, but leave status
+ * intact so a legitimately-no-op mutating run is not broken.
+ */
+export function applyGoalAchievement(
+	manifest: TeamRunManifest,
+	assessment: GoalAchievementAssessment,
+): { manifest: TeamRunManifest; downgraded: boolean } {
+	const note =
+		assessment.achieved === false
+			? `goal-achievement: FALSE-GREEN — ${assessment.reason}. ${assessment.signals.join("; ")}`
+			: assessment.achieved === true
+				? `goal-achievement: OK — ${assessment.reason}`
+				: `goal-achievement: unknown — ${assessment.reason}`;
+
+	const updated: TeamRunManifest = {
+		...manifest,
+		goalAchieved: assessment.achieved,
+		goalAchievementNote: note,
+	};
+
+	// Downgrade only on the highest-confidence false-green: a failed task
+	// corroborates that the run genuinely did not succeed.
+	const hasCorroboratingFailure = assessment.signals.some((s) => s.startsWith("corroborating failed task"));
+	if (assessment.achieved === false && hasCorroboratingFailure && updated.status === "completed") {
+		return { manifest: { ...updated, status: "failed" }, downgraded: true };
+	}
+	return { manifest: updated, downgraded: false };
+}