pi-crew 0.9.53 → 0.9.55

package/src/runtime/stale-reconciler.ts

@@ -0,0 +1,680 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { errors } from "../errors.ts";
+import { atomicWriteFile, atomicWriteJson } from "../state/atomic-write.ts";
+import { saveRunManifest } from "../state/state-store.ts";
+import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
+import { recordFromTask, upsertCrewAgent } from "./crew-agent-records.ts";
+import { checkProcessLiveness } from "./process-status.ts";
+
+/** Age threshold for orphaned temp directory cleanup: 1 hour. */
+const ORPHAN_TEMP_DIR_AGE_THRESHOLD_MS = 60 * 60 * 1000;
+/** Defense-in-depth: cap the number of /tmp/pi-crew-* entries processed per
+ * reconcile tick. With a few thousand accumulated dirs, processing them all
+ * synchronously can block the main thread for many seconds, causing the
+ * terminal to appear hung. We process in batches; the rest are handled on
+ * subsequent ticks. */
+const ORPHAN_TEMP_SCAN_BATCH_SIZE = 50;
+
+/**
+ * Result of reconciling a single stale run.
+ */
+export interface ReconcileResult {
+	runId: string;
+	/** What was found and what action was taken */
+	verdict: "healthy" | "blocked_awaiting_approval" | "result_exists" | "pid_dead" | "pid_alive_stale" | "no_status";
+	/** Whether repair was applied */
+	repaired: boolean;
+	/** Human-readable detail */
+	detail: string;
+	/** Repaired task state, returned to a locked caller for persistence. */
+	repairedTasks?: TeamTaskState[];
+}
+
+/**
+ * Is this run intentionally waiting for human plan approval?
+ *
+ * Such runs are NOT stale even if their owning session died or their async PID
+ * is no longer live — they are blocked on a human decision, not a crash. Crash
+ * recovery and stale reconciliation must preserve them rather than mark them
+ * failed or orphan-cancel them. See PR #32 (gustavo-pelissaro) for the
+ * original analysis of this failure mode.
+ */
+export function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
+	return manifest.status === "blocked" && manifest.planApproval?.required === true && manifest.planApproval.status === "pending";
+}
+
+const STALE_ALIVE_PID_MS = 24 * 60 * 60 * 1000; // 24 hours
+const ACTIVE_EVIDENCE_TTL_MS = 5 * 60 * 1000;
+/** For no-PID runs, repair when ALL running tasks have heartbeat stale beyond this threshold. */
+const NO_PID_HEARTBEAT_STALE_MS = 5 * 60 * 1000; // 5 minutes — same as heartbeat-gradient deadMs
+
+/**
+ * Phase 1: Check if a result file already exists for the run.
+ * If so, the run completed but status wasn't updated — repair it.
+ */
+function checkResultFile(manifest: TeamRunManifest, tasks: TeamTaskState[]): { found: boolean; repaired: boolean } {
+	// Check if all tasks already have terminal status (result was written but manifest wasn't updated)
+	const allTerminal =
+		tasks.length > 0 &&
+		tasks.every(
+			(t) =>
+				t.status === "completed" ||
+				t.status === "failed" ||
+				t.status === "cancelled" ||
+				t.status === "skipped" ||
+				t.status === "needs_attention",
+		);
+	if (allTerminal) {
+		// All tasks are terminal but manifest status was not updated — repair it.
+		// Derive the run status from task outcomes instead of blindly marking
+		// "completed": a run where every task failed/cancelled is NOT a success.
+		const hasFailed = tasks.some((t) => t.status === "failed" || t.status === "needs_attention");
+		const onlyCancelledOrSkipped = tasks.every((t) => t.status === "cancelled" || t.status === "skipped");
+		manifest.status = hasFailed ? "failed" : onlyCancelledOrSkipped ? "cancelled" : "completed";
+		// Persist manifest status change immediately to make checkResultFile self-contained.
+		// NOTE (OPT-02): Remains sync — reconcileStaleRun → checkResultFile is called from
+		// withRunLockSync in crash-recovery.ts and from sync reconcileOrphanedTempWorkspaces.
+		// Converting to async would require cascading through all callers including sync UI hooks.
+		saveRunManifest(manifest);
+		// Sync agent records even when tasks are already terminal
+		// (e.g., a previous reconcile fixed tasks but crashed before updating agents)
+		for (const task of tasks) {
+			try {
+				upsertCrewAgent(manifest, recordFromTask(manifest, task, "scaffold"));
+			} catch {
+				/* non-critical */
+			}
+		}
+		return { found: true, repaired: false };
+	}
+	return { found: false, repaired: false };
+}
+
+/**
+ * Get process start time in milliseconds since boot from /proc/<pid>/stat.
+ * Returns undefined if the process is gone or /proc is unavailable.
+ *
+ * Platform limitation: Relies on Linux's /proc filesystem. On macOS/Windows,
+ * startTime will be undefined and PID reuse detection is weaker.
+ *
+ * The start time is in the 22nd field (index 21 after comm) of /proc/<pid>/stat.
+ */
+function getProcessStartTime(pid: number): number | undefined {
+	try {
+		const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8");
+		const lastParen = stat.lastIndexOf(")");
+		if (lastParen === -1) return undefined;
+		const fieldsAfterComm = stat
+			.slice(lastParen + 1)
+			.trim()
+			.split(/\s+/);
+		// starttime is at index 19 (the 20th field after comm)
+		const startTimeClockTicks = Number(fieldsAfterComm[19]);
+		if (!Number.isFinite(startTimeClockTicks)) return undefined;
+		// Convert clock ticks to ms using ~100 ticks/sec (CLK_TCK).
+		// The absolute value matters less than uniqueness per PID lifecycle.
+		return Math.floor(startTimeClockTicks * 10);
+	} catch {
+		return undefined;
+	}
+}
+
+/**
+ * Phase 2: Check PID liveness.
+ * Uses process.kill(pid, 0) for the authoritative check, but also verifies
+ * startTime before and after to detect PID recycling. If the OS recycles
+ * the PID between the kill(0) call and the repair action, a newly spawned
+ * unrelated process could be incorrectly identified as the async worker.
+ * The heartbeat file is used as corroborating evidence when the process
+ * appears dead per kill(0).
+ */
+function checkPidLiveness(
+	pid: number | undefined,
+	stateRoot?: string,
+): {
+	alive: boolean;
+	detail: string;
+} {
+	if (pid === undefined || !Number.isInteger(pid) || pid <= 0) {
+		return { alive: false, detail: "no pid recorded" };
+	}
+	// Capture startTime before kill(0) to detect PID recycling.
+	const startTimeBefore = getProcessStartTime(pid);
+	const liveness = checkProcessLiveness(pid);
+	// Re-verify startTime after kill(0) to close the TOCTOU window.
+	// If startTime changed, the PID was recycled to a different process.
+	const startTimeAfter = getProcessStartTime(pid);
+	if (startTimeBefore !== undefined && startTimeAfter !== undefined && startTimeBefore !== startTimeAfter) {
+		// PID was recycled — treat as dead to avoid acting on wrong process.
+		return { alive: false, detail: "pid_recycled" };
+	}
+	// If process is alive per kill(0), we're done.
+	if (liveness.alive) return { alive: true, detail: liveness.detail };
+	// Process is dead per kill(0). ESRCH is AUTHORITATIVE — the PID does not exist.
+	// A frozen/zombie process still has a PID (kill(0) returns 0, or EPERM if we lack
+	// permission to signal it), so reaching here with alive=false means the process
+	// is genuinely gone. PID recycling is already handled above via the startTime
+	// TOCTOU check, so the heartbeat file must NOT override the dead verdict.
+	//
+	// HISTORY: a <5min-old heartbeat.json previously OVERRRODE this verdict (returned
+	// alive:true "process dead but heartbeat Xs old"), delaying repair by ~5min.
+	// Combined with the 60s reconcile cadence this produced a ~300-360s detection lag
+	// for dead workers (observed 353s in production). The heartbeat is now used only
+	// to enrich the detail string, never to override the authoritative kill(0) verdict.
+	let heartbeatNote = "";
+	if (stateRoot) {
+		try {
+			const heartbeatPath = path.join(stateRoot, "heartbeat.json");
+			if (fs.existsSync(heartbeatPath)) {
+				const hb = JSON.parse(fs.readFileSync(heartbeatPath, "utf-8")) as { pid?: number; at?: number };
+				if (hb?.pid === pid && hb?.at) {
+					heartbeatNote = ` (heartbeat was ${Math.round((Date.now() - hb.at) / 1000)}s old)`;
+				}
+			}
+		} catch {
+			/* ignore — best-effort */
+		}
+	}
+	return { alive: false, detail: `${liveness.detail}${heartbeatNote}` };
+}
+
+/**
+ * Phase 3: For dead PIDs, repair immediately.
+ * For alive PIDs, only mark stale if status hasn't updated in STALE_ALIVE_PID_MS.
+ */
+function evaluateStaleness(manifest: TeamRunManifest, pidAlive: boolean, now: number): { stale: boolean; reason: string } {
+	if (!pidAlive) {
+		return { stale: true, reason: "pid_dead" };
+	}
+	const updatedAt = new Date(manifest.updatedAt).getTime();
+	if (!Number.isFinite(updatedAt)) {
+		return { stale: false, reason: "updated_at_invalid" };
+	}
+	if (now - updatedAt > STALE_ALIVE_PID_MS) {
+		return {
+			stale: true,
+			reason: `alive_but_stale_${Math.round((now - updatedAt) / 3600_000)}h`,
+		};
+	}
+	return { stale: false, reason: "alive_and_recent" };
+}
+
+function hasRecentActiveEvidence(tasks: TeamTaskState[], now: number): boolean {
+	return tasks.some((task) => {
+		if (task.status !== "running" && task.status !== "waiting") return false;
+		const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
+		if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= ACTIVE_EVIDENCE_TTL_MS) return true;
+		const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
+		return Number.isFinite(activityAt) && now - activityAt <= ACTIVE_EVIDENCE_TTL_MS;
+	});
+}
+
+/**
+ * Check if a single task is stale (heartbeat AND activity both stale beyond
+ * NO_PID_HEARTBEAT_STALE_MS). Tasks with no heartbeat AND no agent progress
+ * are considered NOT stale (they may be newly spawned and haven't reported yet).
+ */
+function isTaskHeartbeatStale(task: TeamTaskState, now: number): boolean {
+	const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
+	const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
+	// If no heartbeat AND no activity, we can't determine staleness — not stale
+	if (!Number.isFinite(heartbeatAt) && !Number.isFinite(activityAt)) return false;
+	// Compute elapsed from both sources and use the fresher (minimum) one,
+	// mirroring heartbeat-watcher logic: if either source has recent activity,
+	// the task is not stale even if the other is stale.
+	const heartbeatAge = Number.isFinite(heartbeatAt) ? now - heartbeatAt : Infinity;
+	const activityAge = Number.isFinite(activityAt) ? now - activityAt : Infinity;
+	const elapsed = Math.min(heartbeatAge, activityAge);
+	return elapsed > NO_PID_HEARTBEAT_STALE_MS;
+}
+
+/**
+ * For no-PID runs: check if ALL running tasks have heartbeats stale beyond
+ * the no-PID heartbeat threshold, and return the list of individually stale
+ * task IDs. This detects zombie tasks where the worker process died but no
+ * PID was recorded (e.g. live-session /tmp/ workspaces).
+ * Merged from the former allRunningTasksHeartbeatStale and
+ * findIndividuallyStaleTaskIds to avoid redundant duplicate checks.
+ */
+function getRunningTaskStaleness(tasks: TeamTaskState[], now: number): { allStale: boolean; staleTaskIds: string[] } {
+	const runningTasks = tasks.filter((t) => t.status === "running" || t.status === "waiting");
+	if (runningTasks.length === 0) return { allStale: false, staleTaskIds: [] };
+	const staleTaskIds: string[] = [];
+	for (const task of runningTasks) {
+		if (isTaskHeartbeatStale(task, now)) {
+			staleTaskIds.push(task.id);
+		}
+	}
+	return {
+		allStale: staleTaskIds.length === runningTasks.length,
+		staleTaskIds,
+	};
+}
+
+/**
+ * Repair a stale run by marking it as failed and cancelling running tasks.
+ */
+/**
+ * E3/E1 (Round 15): Build a human-actionable error string for a stale-reconciled
+ * task. Explains WHY the run was marked stale (the detected reason) and gives
+ * concrete remediation, instead of the bare 'Stale run reconciled: <reason>'.
+ * Now returns a structured CrewError (E012) so callers also get a machine-
+ * readable code + help hint; `.message` carries the same rich text as before.
+ */
+function buildStaleReconcileError(task: TeamTaskState, reason: string): Error {
+	const heartbeatAgeSeconds = task.heartbeat?.lastSeenAt
+		? Math.round((Date.now() - new Date(task.heartbeat.lastSeenAt).getTime()) / 1000)
+		: undefined;
+	return errors.runStale(reason, heartbeatAgeSeconds);
+}
+
+function repairStaleRun(manifest: TeamRunManifest, tasks: TeamTaskState[], reason: string): TeamTaskState[] {
+	const now = new Date().toISOString();
+	const repairedTasks = tasks.map((task) => {
+		if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
+			return {
+				...task,
+				status: "cancelled" as const,
+				finishedAt: now,
+				// E3/E1 (Round 15): structured CrewError (E012) with code + help hint.
+				error: buildStaleReconcileError(task, reason).message,
+			};
+		}
+		return task;
+	});
+	// Update agent records so widget sees cancelled status immediately
+	for (const task of repairedTasks) {
+		try {
+			upsertCrewAgent(manifest, recordFromTask(manifest, task, "scaffold"));
+		} catch {
+			/* non-critical */
+		}
+	}
+	return repairedTasks;
+}
+
+/**
+ * Three-phase stale run reconciliation.
+ *
+ * 1. Check if result already exists → use it
+ * 2. Check PID liveness
+ * 3. Dead PID → repair immediately; alive PID → only fail if stale > 24h
+ *
+ * NOTE: Callers must provide locking via withRunLock/withRunLockSync when
+ * calling from contexts where concurrent reconciliation of the same runId
+ * could occur (e.g., the auto-repair timer). The crash-recovery.ts caller
+ * already provides this. The reconcileOrphanedTempWorkspaces caller handles
+ * /tmp workspaces where concurrent access is a known benign race (separate
+ * dirs, low consequence of redundant repair).
+ */
+export function reconcileStaleRun(manifest: TeamRunManifest, tasks: TeamTaskState[], now = Date.now()): ReconcileResult {
+	const runId = manifest.runId;
+
+	// Preserve runs intentionally blocked on human plan approval. These are not
+	// crashes even if the owning PID is gone — they are waiting for a decision.
+	// Must short-circuit before Phase 1 (result check) and Phase 2 (PID liveness).
+	if (isPlanApprovalPending(manifest)) {
+		return {
+			runId,
+			verdict: "blocked_awaiting_approval",
+			repaired: false,
+			detail: "Plan approval is pending; blocked run is intentionally waiting and must not be stale-repaired",
+		};
+	}
+
+	// Phase 1: Check if results already exist
+	const phase1 = checkResultFile(manifest, tasks);
+	if (phase1.found) {
+		// checkResultFile sets manifest.status='completed' and saves it,
+		// but we re-save to ensure the completed status is persisted
+		// before returning (avoids TOCTOU where caller might re-read stale data)
+		// NOTE (OPT-02): Remains sync — reconcileStaleRun is called from withRunLockSync in
+		// crash-recovery.ts and from sync reconcileOrphanedTempWorkspaces; cannot await here.
+		saveRunManifest(manifest);
+		return {
+			runId,
+			verdict: "result_exists",
+			repaired: false,
+			detail: "All tasks already terminal — no repair needed",
+		};
+	}
+
+	// Phase 2: Check PID liveness
+	const pid = manifest.async?.pid;
+	const pidStatus = checkPidLiveness(pid, manifest.stateRoot);
+
+	if (pidStatus.detail === "no pid recorded") {
+		// No async PID may be a foreground/live run. Preserve it if task heartbeat
+		// or agent progress proves active work even when manifest.updatedAt is old.
+		if (hasRecentActiveEvidence(tasks, now)) {
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: false,
+				detail: "No PID recorded, but recent task heartbeat/progress exists; not repairing",
+			};
+		}
+		// No PID and no recent activity. If ALL running tasks have stale heartbeats
+		// (beyond NO_PID_HEARTBEAT_STALE_MS = 5min), repair immediately — the worker
+		// process is dead but we have no PID to check. This handles /tmp/ live-session
+		// workspaces where agents exit without calling submit_result.
+		// Merged with individual stale task check to avoid redundant duplicate checks.
+		const { allStale, staleTaskIds } = getRunningTaskStaleness(tasks, now);
+		if (allStale) {
+			const repaired = repairStaleRun(manifest, tasks, "no_pid_heartbeat_stale");
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: true,
+				detail: `No PID; all running task heartbeats stale >${Math.round(NO_PID_HEARTBEAT_STALE_MS / 60_000)}min; repaired ${repaired.filter((t) => t.status === "cancelled").length} tasks`,
+				repairedTasks: repaired,
+			};
+		}
+		// Check for individually stale tasks even when not all are stale.
+		// This handles the case where task A is healthy but task B is a zombie.
+		// We repair only the zombie tasks, not the whole run.
+		// NOTE: Individual stale task repair (this branch) is intentionally
+		// separate from run-level repair (the allStale branch above). Both
+		// paths can coexist — the allStale path cancels the entire run when
+		// ALL tasks are zombies, while this path repairs only the zombie
+		// subset when some tasks are still healthy.
+		if (staleTaskIds.length > 0) {
+			const repaired = repairStaleRun(manifest, tasks, "no_pid_individual_stale_task");
+			// Only return the individually repaired tasks in detail
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: true,
+				detail: `No PID; ${staleTaskIds.length} individually stale task(s) repaired: ${staleTaskIds.join(", ")}`,
+				repairedTasks: repaired.filter((t) => staleTaskIds.includes(t.id)),
+			};
+		}
+		// Fall through: no recent activity but not all tasks stale enough yet.
+		// Check the longer STALE_ALIVE_PID_MS threshold for very old runs.
+		const updatedAt = new Date(manifest.updatedAt).getTime();
+		if (Number.isFinite(updatedAt) && now - updatedAt > STALE_ALIVE_PID_MS) {
+			const repaired = repairStaleRun(manifest, tasks, "no_pid_stale");
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: true,
+				detail: `No PID; stale ${Math.round((now - updatedAt) / 3600_000)}h; repaired ${repaired.filter((t) => t.status === "cancelled").length} tasks`,
+				repairedTasks: repaired,
+			};
+		}
+		return {
+			runId,
+			verdict: "no_status",
+			repaired: false,
+			detail: "No PID recorded; not stale enough to repair",
+		};
+	}
+
+	// Phase 3: Evaluate staleness
+	const staleness = evaluateStaleness(manifest, pidStatus.alive, now);
+	if (!staleness.stale) {
+		return {
+			runId,
+			verdict: "healthy",
+			repaired: false,
+			detail: `PID ${pid}: ${pidStatus.detail}, ${staleness.reason}`,
+		};
+	}
+
+	// Repair
+	const repaired = repairStaleRun(manifest, tasks, staleness.reason);
+	return {
+		runId,
+		verdict: pidStatus.alive ? "pid_alive_stale" : "pid_dead",
+		repaired: true,
+		detail: `PID ${pid}: ${pidStatus.detail}; ${staleness.reason}; repaired ${repaired.filter((t) => t.status === "cancelled").length} tasks`,
+		repairedTasks: repaired,
+	};
+}
+
+/**
+ * Result of orphaned temp workspace reconciliation.
+ */
+export interface OrphanReconcileResult {
+	/** Number of runs repaired (manifests cancelled). */
+	repaired: number;
+	/** Number of /tmp/pi-crew-* directories removed. */
+	cleanedDirs: number;
+}
+
+/**
+ * Scan /tmp (os.tmpdir()) for orphaned pi-crew-* workspaces and reconcile
+ * any stale runs found. This catches runs created by tests or crashed sessions
+ * that the per-CWD auto-repair timer would miss.
+ *
+ * When `cleanupOrphanedTempDirs` is not explicitly set to `false`, directories
+ * older than 1 hour with no remaining running manifests are deleted after
+ * their runs are reconciled.
+ *
+ * @returns Number of runs repaired and directories cleaned.
+ */
+export function reconcileOrphanedTempWorkspaces(
+	now = Date.now(),
+	options?: {
+		cleanupOrphanedTempDirs?: boolean;
+		tmpDir?: string;
+		scanBatchSize?: number;
+	},
+): OrphanReconcileResult {
+	// Injectable tmpDir + scanBatchSize for deterministic unit testing
+	// (Round 19: tests must not depend on global /tmp cleanliness; the
+	// production ORPHAN_TEMP_SCAN_BATCH_SIZE cap could exclude a test's dir
+	// when leftover dirs accumulate). Defaults remain os.tmpdir() + the cap.
+	const tmpDir = options?.tmpDir ?? getSafeTempDir();
+	if (!tmpDir) return { repaired: 0, cleanedDirs: 0 };
+	let repaired = 0;
+	let cleanedDirs = 0;
+	try {
+		const entries = fs.readdirSync(tmpDir, { withFileTypes: true });
+		// Sort for deterministic order; cap to ORPHAN_TEMP_SCAN_BATCH_SIZE per
+		// tick to avoid main-thread stalls when /tmp has thousands of
+		// pi-crew-* dirs from past interrupted test runs.
+		const scanBatch = options?.scanBatchSize ?? ORPHAN_TEMP_SCAN_BATCH_SIZE;
+		const candidates = entries
+			.filter((e) => e.isDirectory() && e.name.startsWith("pi-crew-"))
+			.sort((a, b) => a.name.localeCompare(b.name))
+			.slice(0, scanBatch);
+		for (const entry of candidates) {
+			if (!entry.isDirectory() || !entry.name.startsWith("pi-crew-")) continue;
+			const workspaceDir = path.join(tmpDir, entry.name);
+			const crewDir = path.join(workspaceDir, ".crew");
+			if (!fs.existsSync(crewDir)) continue;
+			const stateRunsDir = path.join(crewDir, "state", "runs");
+			if (!fs.existsSync(stateRunsDir)) continue;
+			let hasRunning = false;
+			try {
+				for (const runDir of fs.readdirSync(stateRunsDir)) {
+					const manifestPath = path.join(stateRunsDir, runDir, "manifest.json");
+					const tasksPath = path.join(stateRunsDir, runDir, "tasks.json");
+					if (!fs.existsSync(manifestPath) || !fs.existsSync(tasksPath)) continue;
+					try {
+						const manifest: TeamRunManifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+						if (manifest.status !== "running") continue;
+						const tasks: TeamTaskState[] = JSON.parse(fs.readFileSync(tasksPath, "utf-8"));
+						const result = reconcileStaleRun(manifest, tasks, now);
+						if (result.repaired && result.repairedTasks) {
+							// Persist repaired tasks (atomic — temp+rename to survive mid-write crash)
+							atomicWriteJson(tasksPath, result.repairedTasks);
+							// Update manifest status
+							const updated = {
+								...manifest,
+								status: "cancelled" as const,
+								updatedAt: new Date(now).toISOString(),
+								summary: `Stale run reconciled: ${result.detail}`,
+							};
+							atomicWriteJson(manifestPath, updated);
+							// Update agent records
+							for (const task of result.repairedTasks) {
+								try {
+									upsertCrewAgent(updated, recordFromTask(updated, task, "scaffold"));
+								} catch {
+									/* non-critical */
+								}
+							}
+							repaired++;
+						}
+						// Persist result_exists manifest status update
+						if (result.verdict === "result_exists") {
+							// checkResultFile (called by reconcileStaleRun at line 518)
+							// already saved manifest.status='completed' at line 66.
+							// No additional manifest write needed here — avoids TOCTOU
+							// window between checkResultFile save and re-write.
+							// Sync agent records (checkResultFile also does this but
+							// we sync here for consistency)
+							for (const task of tasks) {
+								try {
+									upsertCrewAgent(manifest, recordFromTask(manifest, task, "scaffold"));
+								} catch {
+									/* non-critical */
+								}
+							}
+						}
+						// If still running after reconciliation attempt, mark for dir-preserving
+						if (result.verdict === "healthy" || (result.verdict === "no_status" && !result.repaired)) {
+							hasRunning = true;
+						}
+					} catch (err) {
+						// Log warning when skipping a directory due to error.
+						// Note: manifestPath here refers to the variable defined in the
+						// for loop at line 442 (outer scope of this catch block).
+						const scanManifestPath = manifestPath;
+						console.warn(`[stale-reconciler] Skipping manifest due to parse error: ${scanManifestPath}: ${err}`);
+					}
+				}
+			} catch (err) {
+				// Cannot determine running state — treat as if running to prevent
+				// premature cleanup of a potentially active workspace.
+				hasRunning = true;
+				console.warn(`[stale-reconciler] Skipping unreadable runs dir: ${stateRunsDir}: ${err}`);
+			}
+
+			// Post-loop: check if this workspace dir can be cleaned up.
+			// Eligible when cleanup is enabled, no running manifests remain, and
+			// the directory is older than the age threshold.
+			// Re-scan manifests to confirm no running runs remain BEFORE the
+			// cleanup decision (fixes TOCTOU race where a manifest may have
+			// transitioned from 'running' to 'completed' between the main loop
+			// and this re-scan).
+			//
+			// TOCTOU fix: Create a sentinel file before the re-scan. New run
+			// creation should check for this sentinel and refuse/wait if present.
+			// Additionally, do a final check right before cleanup to catch any
+			// new runs created between the re-scan and the cleanup decision.
+			const sentinelPath = path.join(workspaceDir, ".cleanup-in-progress");
+			let canCleanup = !hasRunning;
+
+			// FIX: Capture dirAge BEFORE creating the sentinel file, because writing
+			// .cleanup-in-progress inside workspaceDir updates its mtime, making the
+			// directory appear fresh and defeating the age check.
+			const cleanupEnabled = options?.cleanupOrphanedTempDirs !== false;
+			let dirAge = 0;
+			if (cleanupEnabled && !hasRunning) {
+				try {
+					const stat = fs.statSync(workspaceDir);
+					dirAge = now - stat.mtimeMs;
+				} catch {
+					// Directory disappeared
+				}
+			}
+			if (canCleanup) {
+				// Create sentinel file before re-scan to signal cleanup in progress.
+				// O_EXCL requires workspaceDir to exist — ensure it before writing.
+				try {
+					if (!fs.existsSync(workspaceDir)) {
+						fs.mkdirSync(workspaceDir, { recursive: true });
+					}
+					// Atomic exclusive-create: reserve the sentinel (fails with EEXIST if
+					// another cleanup is in progress), then fill it atomically. The brief
+					// empty-file window is safe — the sentinel "exists" (correctly
+					// signalling in-progress cleanup).
+					const sentinelFd = fs.openSync(sentinelPath, "wx");
+					fs.closeSync(sentinelFd);
+					atomicWriteFile(sentinelPath, JSON.stringify({ startedAt: now }));
+				} catch {
+					// Sentinel already exists (another cleanup in progress) — skip
+					canCleanup = false;
+				}
+			}
+			if (canCleanup) {
+				if (fs.existsSync(stateRunsDir)) {
+					try {
+						for (const runDir of fs.readdirSync(stateRunsDir)) {
+							const manifestPath = path.join(stateRunsDir, runDir, "manifest.json");
+							if (!fs.existsSync(manifestPath)) continue;
+							let manifest: TeamRunManifest | undefined;
+							try {
+								manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+							} catch (err) {
+								// Log warning when skipping a directory due to error
+								console.warn(`[stale-reconciler] Skipping manifest due to parse error: ${manifestPath}: ${err}`);
+								continue;
+							}
+							if (manifest?.status === "running") {
+								canCleanup = false;
+								break;
+							}
+						}
+					} catch (err) {
+						console.warn(`[stale-reconciler] Skipping unreadable runs dir: ${stateRunsDir}: ${err}`);
+					}
+				}
+			}
+			if (canCleanup) {
+				if (dirAge > ORPHAN_TEMP_DIR_AGE_THRESHOLD_MS) {
+					// Final check: re-verify no running manifests appeared since re-scan.
+					let stillClean = true;
+					if (fs.existsSync(stateRunsDir)) {
+						try {
+							for (const runDir of fs.readdirSync(stateRunsDir)) {
+								const manifestPath = path.join(stateRunsDir, runDir, "manifest.json");
+								if (!fs.existsSync(manifestPath)) continue;
+								try {
+									const manifest: TeamRunManifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+									if (manifest.status === "running") {
+										stillClean = false;
+										break;
+									}
+								} catch {
+									/* skip on parse error */
+								}
+							}
+						} catch {
+							stillClean = false;
+						}
+					}
+					if (stillClean) {
+						fs.rmSync(workspaceDir, {
+							recursive: true,
+							force: true,
+						});
+						cleanedDirs++;
+					}
+				}
+				// Clean up sentinel file regardless of outcome
+				try {
+					fs.unlinkSync(sentinelPath);
+				} catch {
+					/* ignore if already gone */
+				}
+			}
+		}
+	} catch {
+		/* skip if tmpdir unreadable */
+	}
+	return { repaired, cleanedDirs };
+}
+
+function getSafeTempDir(): string | undefined {
+	try {
+		return fs.existsSync(os.tmpdir()) ? os.tmpdir() : undefined;
+	} catch {
+		return undefined;
+	}
+}