pi-crew 0.3.6 → 0.3.8

package/src/runtime/stale-reconciler.ts

@@ -1,8 +1,12 @@
 import * as fs from "node:fs";
+import * as os from "node:os";
 import * as path from "node:path";
 import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
-import { checkProcessLiveness } from "./process-status.ts";
 import { recordFromTask, upsertCrewAgent } from "./crew-agent-records.ts";
+import { checkProcessLiveness } from "./process-status.ts";
+
+/** Age threshold for orphaned temp directory cleanup: 1 hour. */
+const ORPHAN_TEMP_DIR_AGE_THRESHOLD_MS = 60 * 60 * 1000;
 
 /**
  * Result of reconciling a single stale run.
@@ -10,7 +14,12 @@ import { recordFromTask, upsertCrewAgent } from "./crew-agent-records.ts";
 export interface ReconcileResult {
 	runId: string;
 	/** What was found and what action was taken */
-	verdict: "healthy" | "result_exists" | "pid_dead" | "pid_alive_stale" | "no_status";
+	verdict:
+		| "healthy"
+		| "result_exists"
+		| "pid_dead"
+		| "pid_alive_stale"
+		| "no_status";
 	/** Whether repair was applied */
 	repaired: boolean;
 	/** Human-readable detail */
@@ -21,6 +30,8 @@ export interface ReconcileResult {
 
 const STALE_ALIVE_PID_MS = 24 * 60 * 60 * 1000; // 24 hours
 const ACTIVE_EVIDENCE_TTL_MS = 5 * 60 * 1000;
+/** For no-PID runs, repair when ALL running tasks have heartbeat stale beyond this threshold. */
+const NO_PID_HEARTBEAT_STALE_MS = 5 * 60 * 1000; // 5 minutes — same as heartbeat-gradient deadMs
 
 /**
  * Phase 1: Check if a result file already exists for the run.
@@ -31,14 +42,28 @@ function checkResultFile(
 	tasks: TeamTaskState[],
 ): { found: boolean; repaired: boolean } {
 	// Check if all tasks already have terminal status (result was written but manifest wasn't updated)
-	const allTerminal = tasks.length > 0 && tasks.every(
-		(t) => t.status === "completed" || t.status === "failed" || t.status === "cancelled" || t.status === "skipped" || t.status === "needs_attention",
-	);
+	const allTerminal =
+		tasks.length > 0 &&
+		tasks.every(
+			(t) =>
+				t.status === "completed" ||
+				t.status === "failed" ||
+				t.status === "cancelled" ||
+				t.status === "skipped" ||
+				t.status === "needs_attention",
+		);
 	if (allTerminal) {
 		// Sync agent records even when tasks are already terminal
 		// (e.g., a previous reconcile fixed tasks but crashed before updating agents)
 		for (const task of tasks) {
-			try { upsertCrewAgent(manifest, recordFromTask(manifest, task, "scaffold")); } catch { /* non-critical */ }
+			try {
+				upsertCrewAgent(
+					manifest,
+					recordFromTask(manifest, task, "scaffold"),
+				);
+			} catch {
+				/* non-critical */
+			}
 		}
 		return { found: true, repaired: false };
 	}
@@ -52,7 +77,10 @@ function checkResultFile(
  * written, treat the PID as alive even if process.kill returns false
  * (handles SIGKILL race where PID hasn't been recycled yet).
  */
-function checkPidLiveness(pid: number | undefined, stateRoot?: string): {
+function checkPidLiveness(
+	pid: number | undefined,
+	stateRoot?: string,
+): {
 	alive: boolean;
 	detail: string;
 } {
@@ -67,13 +95,18 @@ function checkPidLiveness(pid: number | undefined, stateRoot?: string): {
 		const heartbeatPath = path.join(stateRoot, "heartbeat.json");
 		try {
 			if (fs.existsSync(heartbeatPath)) {
-				const hb = JSON.parse(fs.readFileSync(heartbeatPath, "utf-8")) as { pid?: number; at?: number };
+				const hb = JSON.parse(
+					fs.readFileSync(heartbeatPath, "utf-8"),
+				) as { pid?: number; at?: number };
 				if (hb?.pid === pid && hb?.at) {
 					const ageMs = Date.now() - hb.at;
 					// Heartbeat written < 5 min ago → process was alive recently.
 					// Don't repair yet; let the next reconciliation cycle catch it.
 					if (ageMs < 5 * 60_000) {
-						return { alive: true, detail: `process dead but heartbeat ${Math.round(ageMs / 1000)}s old` };
+						return {
+							alive: true,
+							detail: `process dead but heartbeat ${Math.round(ageMs / 1000)}s old`,
+						};
 					}
 				}
 			}
@@ -101,18 +134,76 @@ function evaluateStaleness(
 		return { stale: false, reason: "updated_at_invalid" };
 	}
 	if (now - updatedAt > STALE_ALIVE_PID_MS) {
-		return { stale: true, reason: `alive_but_stale_${Math.round((now - updatedAt) / 3600_000)}h` };
+		return {
+			stale: true,
+			reason: `alive_but_stale_${Math.round((now - updatedAt) / 3600_000)}h`,
+		};
 	}
 	return { stale: false, reason: "alive_and_recent" };
 }
 
 function hasRecentActiveEvidence(tasks: TeamTaskState[], now: number): boolean {
 	return tasks.some((task) => {
-		if (task.status !== "running" && task.status !== "waiting") return false;
-		const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
-		if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= ACTIVE_EVIDENCE_TTL_MS) return true;
-		const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
-		return Number.isFinite(activityAt) && now - activityAt <= ACTIVE_EVIDENCE_TTL_MS;
+		if (task.status !== "running" && task.status !== "waiting")
+			return false;
+		const heartbeatAt = task.heartbeat?.lastSeenAt
+			? new Date(task.heartbeat.lastSeenAt).getTime()
+			: Number.NaN;
+		if (
+			task.heartbeat?.alive !== false &&
+			Number.isFinite(heartbeatAt) &&
+			now - heartbeatAt <= ACTIVE_EVIDENCE_TTL_MS
+		)
+			return true;
+		const activityAt = task.agentProgress?.lastActivityAt
+			? new Date(task.agentProgress.lastActivityAt).getTime()
+			: Number.NaN;
+		return (
+			Number.isFinite(activityAt) &&
+			now - activityAt <= ACTIVE_EVIDENCE_TTL_MS
+		);
+	});
+}
+
+/**
+ * For no-PID runs: check if ALL running tasks have heartbeats stale beyond
+ * the no-PID heartbeat threshold. This detects zombie tasks where the worker
+ * process died but no PID was recorded (e.g. live-session /tmp/ workspaces).
+ * Tasks with no heartbeat AND no agent progress are considered NOT stale
+ * (they may be newly spawned and haven't reported yet).
+ */
+function allRunningTasksHeartbeatStale(
+	tasks: TeamTaskState[],
+	now: number,
+): boolean {
+	const runningTasks = tasks.filter(
+		(t) => t.status === "running" || t.status === "waiting",
+	);
+	if (runningTasks.length === 0) return false;
+	return runningTasks.every((task) => {
+		const heartbeatAt = task.heartbeat?.lastSeenAt
+			? new Date(task.heartbeat.lastSeenAt).getTime()
+			: Number.NaN;
+		const activityAt = task.agentProgress?.lastActivityAt
+			? new Date(task.agentProgress.lastActivityAt).getTime()
+			: Number.NaN;
+		// If no heartbeat AND no activity, we can't determine staleness — assume not stale
+		if (!Number.isFinite(heartbeatAt) && !Number.isFinite(activityAt))
+			return false;
+		// If heartbeat is recent enough, not stale
+		if (
+			Number.isFinite(heartbeatAt) &&
+			now - heartbeatAt <= NO_PID_HEARTBEAT_STALE_MS
+		)
+			return false;
+		// If agent progress is recent enough, not stale
+		if (
+			Number.isFinite(activityAt) &&
+			now - activityAt <= NO_PID_HEARTBEAT_STALE_MS
+		)
+			return false;
+		// Both present and both stale → this task is stale
+		return true;
 	});
 }
 
@@ -126,7 +217,11 @@ function repairStaleRun(
 ): TeamTaskState[] {
 	const now = new Date().toISOString();
 	const repairedTasks = tasks.map((task) => {
-		if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
+		if (
+			task.status === "running" ||
+			task.status === "queued" ||
+			task.status === "waiting"
+		) {
 			return {
 				...task,
 				status: "cancelled" as const,
@@ -138,7 +233,14 @@ function repairStaleRun(
 	});
 	// Update agent records so widget sees cancelled status immediately
 	for (const task of repairedTasks) {
-		try { upsertCrewAgent(manifest, recordFromTask(manifest, task, "scaffold")); } catch { /* non-critical */ }
+		try {
+			upsertCrewAgent(
+				manifest,
+				recordFromTask(manifest, task, "scaffold"),
+			);
+		} catch {
+			/* non-critical */
+		}
 	}
 	return repairedTasks;
 }
@@ -183,8 +285,31 @@ export function reconcileStaleRun(
 				detail: "No PID recorded, but recent task heartbeat/progress exists; not repairing",
 			};
 		}
+		// No PID and no recent activity. If ALL running tasks have stale heartbeats
+		// (beyond NO_PID_HEARTBEAT_STALE_MS = 5min), repair immediately — the worker
+		// process is dead but we have no PID to check. This handles /tmp/ live-session
+		// workspaces where agents exit without calling submit_result.
+		if (allRunningTasksHeartbeatStale(tasks, now)) {
+			const repaired = repairStaleRun(
+				manifest,
+				tasks,
+				"no_pid_heartbeat_stale",
+			);
+			return {
+				runId,
+				verdict: "no_status",
+				repaired: true,
+				detail: `No PID; all running task heartbeats stale >${Math.round(NO_PID_HEARTBEAT_STALE_MS / 60_000)}min; repaired ${repaired.filter((t) => t.status === "cancelled").length} tasks`,
+				repairedTasks: repaired,
+			};
+		}
+		// Fall through: no recent activity but not all tasks stale enough yet.
+		// Check the longer STALE_ALIVE_PID_MS threshold for very old runs.
 		const updatedAt = new Date(manifest.updatedAt).getTime();
-		if (Number.isFinite(updatedAt) && now - updatedAt > STALE_ALIVE_PID_MS) {
+		if (
+			Number.isFinite(updatedAt) &&
+			now - updatedAt > STALE_ALIVE_PID_MS
+		) {
 			const repaired = repairStaleRun(manifest, tasks, "no_pid_stale");
 			return {
 				runId,
@@ -223,3 +348,182 @@ export function reconcileStaleRun(
 		repairedTasks: repaired,
 	};
 }
+
+/**
+ * Result of orphaned temp workspace reconciliation.
+ */
+export interface OrphanReconcileResult {
+	/** Number of runs repaired (manifests cancelled). */
+	repaired: number;
+	/** Number of /tmp/pi-crew-* directories removed. */
+	cleanedDirs: number;
+}
+
+/**
+ * Scan /tmp (os.tmpdir()) for orphaned pi-crew-* workspaces and reconcile
+ * any stale runs found. This catches runs created by tests or crashed sessions
+ * that the per-CWD auto-repair timer would miss.
+ *
+ * When `cleanupOrphanedTempDirs` is not explicitly set to `false`, directories
+ * older than 1 hour with no remaining running manifests are deleted after
+ * their runs are reconciled.
+ *
+ * @returns Number of runs repaired and directories cleaned.
+ */
+export function reconcileOrphanedTempWorkspaces(
+	now = Date.now(),
+	options?: { cleanupOrphanedTempDirs?: boolean },
+): OrphanReconcileResult {
+	const tmpDir = getSafeTempDir();
+	if (!tmpDir) return { repaired: 0, cleanedDirs: 0 };
+	let repaired = 0;
+	let cleanedDirs = 0;
+	try {
+		const entries = fs.readdirSync(tmpDir, { withFileTypes: true });
+		for (const entry of entries) {
+			if (!entry.isDirectory() || !entry.name.startsWith("pi-crew-"))
+				continue;
+			const workspaceDir = path.join(tmpDir, entry.name);
+			const crewDir = path.join(workspaceDir, ".crew");
+			if (!fs.existsSync(crewDir)) continue;
+			const stateRunsDir = path.join(crewDir, "state", "runs");
+			if (!fs.existsSync(stateRunsDir)) continue;
+			let hasRunning = false;
+			try {
+				for (const runDir of fs.readdirSync(stateRunsDir)) {
+					const manifestPath = path.join(
+						stateRunsDir,
+						runDir,
+						"manifest.json",
+					);
+					const tasksPath = path.join(
+						stateRunsDir,
+						runDir,
+						"tasks.json",
+					);
+					if (
+						!fs.existsSync(manifestPath) ||
+						!fs.existsSync(tasksPath)
+					)
+						continue;
+					try {
+						const manifest: TeamRunManifest = JSON.parse(
+							fs.readFileSync(manifestPath, "utf-8"),
+						);
+						if (manifest.status !== "running") continue;
+						const tasks: TeamTaskState[] = JSON.parse(
+							fs.readFileSync(tasksPath, "utf-8"),
+						);
+						const result = reconcileStaleRun(manifest, tasks, now);
+						if (result.repaired && result.repairedTasks) {
+							// Persist repaired tasks
+							fs.writeFileSync(
+								tasksPath,
+								JSON.stringify(result.repairedTasks, null, 2),
+							);
+							// Update manifest status
+							const updated = {
+								...manifest,
+								status: "cancelled" as const,
+								updatedAt: new Date(now).toISOString(),
+								summary: `Stale run reconciled: ${result.detail}`,
+							};
+							fs.writeFileSync(
+								manifestPath,
+								JSON.stringify(updated, null, 2),
+							);
+							// Update agent records
+							for (const task of result.repairedTasks) {
+								try {
+									upsertCrewAgent(
+										updated,
+										recordFromTask(
+											updated,
+											task,
+											"scaffold",
+										),
+									);
+								} catch {
+									/* non-critical */
+								}
+							}
+							repaired++;
+						}
+						// If still running after reconciliation attempt, mark for dir-preserving
+						if (
+							result.verdict === "healthy" ||
+							(result.verdict === "no_status" && !result.repaired)
+						) {
+							hasRunning = true;
+						}
+					} catch {
+						/* skip corrupt manifests */
+					}
+				}
+			} catch {
+				/* skip unreadable dirs */
+			}
+
+			// Post-loop: check if this workspace dir can be cleaned up.
+			// Eligible when cleanup is enabled, no running manifests remain, and
+			// the directory is older than the age threshold.
+			if (!hasRunning) {
+				// Re-scan manifests to confirm no running runs remain
+				// (some may have been cancelled on a previous pass)
+				if (fs.existsSync(stateRunsDir)) {
+					try {
+						for (const runDir of fs.readdirSync(stateRunsDir)) {
+							const manifestPath = path.join(
+								stateRunsDir,
+								runDir,
+								"manifest.json",
+							);
+							if (!fs.existsSync(manifestPath)) continue;
+							try {
+								const manifest: TeamRunManifest = JSON.parse(
+									fs.readFileSync(manifestPath, "utf-8"),
+								);
+								if (manifest.status === "running") {
+									hasRunning = true;
+									break;
+								}
+							} catch {
+								/* skip corrupt */
+							}
+						}
+					} catch {
+						/* skip unreadable */
+					}
+				}
+			}
+
+			const cleanupEnabled = options?.cleanupOrphanedTempDirs !== false;
+			if (cleanupEnabled && !hasRunning) {
+				try {
+					const stat = fs.statSync(workspaceDir);
+					const dirAge = now - stat.mtimeMs;
+					if (dirAge > ORPHAN_TEMP_DIR_AGE_THRESHOLD_MS) {
+						fs.rmSync(workspaceDir, {
+							recursive: true,
+							force: true,
+						});
+						cleanedDirs++;
+					}
+				} catch {
+					/* skip if stat or rm fails */
+				}
+			}
+		}
+	} catch {
+		/* skip if tmpdir unreadable */
+	}
+	return { repaired, cleanedDirs };
+}
+
+function getSafeTempDir(): string | undefined {
+	try {
+		return fs.existsSync(os.tmpdir()) ? os.tmpdir() : undefined;
+	} catch {
+		return undefined;
+	}
+}

package/src/runtime/task-runner.ts

@@ -416,6 +416,8 @@ export async function runTeamTask(
 					skillPaths,
 					maxTurns: input.runtimeConfig?.maxTurns,
 					graceTurns: input.runtimeConfig?.graceTurns,
+					inheritContext: input.runtimeConfig?.inheritContext,
+					parentContext: input.parentContext,
 					onSpawn: (pid) => {
 						try {
 							({ task, tasks } = checkpointTask(
@@ -827,8 +829,13 @@ export async function runTeamTask(
 		// _yieldResult: preserved for future use — yield completion contract not yet wired to task.result
 		let _yieldResult: YieldResult | undefined;
 		let noYield = false;
+		// Child-process workers do not have a submit_result tool — the yield contract
+		// only applies to live-session workers where submit_result is injected by the
+		// runtime. Skipping yield detection for child-process prevents every child
+		// worker from incorrectly being marked needs_attention.
 		const yieldEnabled =
-			input.runtimeConfig?.yield?.enabled ?? DEFAULT_YIELD_CONFIG.enabled;
+			runtimeKind !== "child-process" &&
+			(input.runtimeConfig?.yield?.enabled ?? DEFAULT_YIELD_CONFIG.enabled);
 		if (yieldEnabled && collectedJsonEvents.length > 0) {
 			if (hasYieldInOutput(collectedJsonEvents)) {
 				const yieldEvent = collectedJsonEvents.find((e) =>

package/src/schema/config-schema.ts

@@ -113,6 +113,7 @@ export const PiTeamsReliabilityConfigSchema = Type.Object({
 	}, { additionalProperties: false })),
 	autoRecover: Type.Optional(Type.Boolean()),
 	deadletterThreshold: Type.Optional(Type.Integer({ minimum: 1 })),
+	cleanupOrphanedTempDirs: Type.Optional(Type.Boolean()),
 }, { additionalProperties: false });
 
 export const PiTeamsOtlpConfigSchema = Type.Object({