pi-crew 0.6.1 → 0.6.3

package/src/runtime/task-runner/state-helpers.ts

@@ -1,22 +1,126 @@
+import * as fs from "node:fs";
 import type { TaskCheckpointState, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
 import { loadRunManifestById, saveRunTasks } from "../../state/state-store.ts";
 import { recordFromTask, upsertCrewAgent } from "../crew-agent-records.ts";
+import { logInternalError } from "../../utils/internal-error.ts";
+import { withRunLockSync } from "../../state/locks.ts";
 
 export function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): TeamTaskState[] {
 	return tasks.map((task) => task.id === updated.id ? updated : task);
 }
 
-export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks: TeamTaskState[], updated: TeamTaskState): TeamTaskState[] {
-	const latest = loadRunManifestById(manifest.cwd, manifest.runId)?.tasks ?? fallbackTasks;
-	const merged = updateTask(latest, updated);
-	saveRunTasks(manifest, merged);
-	return merged;
+/**
+ * Persist a single task update using compare-and-swap under the run lock.
+ *
+ * Problem: The naive read-merge-write pattern is vulnerable to a read-modify-write
+ * race. When two parallel task completions race:
+ *   1. Task A loads tasks [A(running), B(running)], writes [A(completed), B(running)]
+ *   2. Task B loads [A(running), B(running)] (stale, before A's write), writes [A(running), B(completed)]
+ *   Result: Task A's completed status is clobbered.
+ *
+ * Solution: Use mtime-based CAS under the run lock. Before writing, stat the tasks file
+ * to record its mtime. After merging, re-stat — if mtime changed, another writer
+ * committed first; retry with the fresh state. This is O(retry) under contention but
+ * converges in the normal single-writer case.
+ *
+ * @param checkpointPhase - Optional checkpoint phase to include in the task state alongside the update.
+ */
+export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks: TeamTaskState[], updated: TeamTaskState, checkpointPhase?: TaskCheckpointState["phase"]): TeamTaskState[] {
+	let baseMtime = 0;
+	try {
+		baseMtime = fs.statSync(manifest.tasksPath).mtimeMs;
+	} catch {
+		// File doesn't exist yet — baseMtime=0 means "anything is fine"
+		baseMtime = 0;
+	}
+
+	let merged: TeamTaskState[] | undefined;
+
+	// Build the task with optional checkpoint phase
+	const taskWithCheckpoint = checkpointPhase
+		? { ...updated, checkpoint: { phase: checkpointPhase, updatedAt: new Date().toISOString() } }
+		: updated;
+
+	try {
+		return withRunLockSync(manifest, () => {
+			retryLoop: for (let attempt = 0; attempt < 100; attempt++) {
+				const latest = loadRunManifestById(manifest.cwd, manifest.runId)?.tasks ?? fallbackTasks;
+				merged = updateTask(latest, taskWithCheckpoint);
+
+				// Re-stat to detect concurrent writes
+				let currentMtime: number;
+				try {
+					currentMtime = fs.statSync(manifest.tasksPath).mtimeMs;
+				} catch {
+					currentMtime = 0;
+				}
+
+				if (currentMtime !== baseMtime) {
+					// Another writer committed — their update is in latest, re-merge on top
+					baseMtime = currentMtime;
+					continue retryLoop;
+				}
+
+				// No concurrent writer — check that our merged result is based on the
+				// same base we observed (no intermediate writer between our load and check)
+				let recheckMtime: number;
+				try {
+					recheckMtime = fs.statSync(manifest.tasksPath).mtimeMs;
+				} catch {
+					// Run state deleted (prune/forget) — nothing to persist.
+					return fallbackTasks;
+				}
+				if (recheckMtime !== baseMtime) {
+					baseMtime = recheckMtime;
+					continue retryLoop;
+				}
+
+				// Final pre-write mtime check to catch any concurrent writer that completed
+				// between the recheck and saveRunTasks
+				let preWriteMtime: number;
+				try {
+					preWriteMtime = fs.statSync(manifest.tasksPath).mtimeMs;
+				} catch {
+					preWriteMtime = 0;
+				}
+				if (preWriteMtime !== baseMtime) {
+					// Another writer committed — retry
+					baseMtime = preWriteMtime;
+					continue retryLoop;
+				}
+
+				break retryLoop;
+			}
+
+			if (merged === undefined) {
+				logInternalError("persistSingleTaskUpdate", new Error("failed to converge after 50 attempts"));
+				throw new Error("persistSingleTaskUpdate: failed to converge after 50 attempts");
+			}
+
+			try {
+				saveRunTasks(manifest, merged);
+			} catch (err) {
+				logInternalError("persistSingleTaskUpdate", err);
+				throw err;
+			}
+			return merged;
+		});
+	} catch (err) {
+		if (merged === undefined) {
+			logInternalError("persistSingleTaskUpdate", err);
+		}
+		throw err;
+	}
 }
 
 export function checkpointTask(manifest: TeamRunManifest, tasks: TeamTaskState[], task: TeamTaskState, phase: TaskCheckpointState["phase"], childPid?: number): { task: TeamTaskState; tasks: TeamTaskState[] } {
 	const checkpoint: TaskCheckpointState = { phase, updatedAt: new Date().toISOString(), ...(childPid ? { childPid } : task.checkpoint?.childPid ? { childPid: task.checkpoint.childPid } : {}) };
 	const nextTask = { ...task, checkpoint };
 	const nextTasks = persistSingleTaskUpdate(manifest, updateTask(tasks, nextTask), nextTask);
-	upsertCrewAgent(manifest, recordFromTask(manifest, nextTask, "child-process"));
+	try {
+		upsertCrewAgent(manifest, recordFromTask(manifest, nextTask, "child-process"));
+	} catch (err) {
+		logInternalError("checkpointTask", err);
+	}
 	return { task: nextTask, tasks: nextTasks };
 }

package/src/runtime/task-runner.ts

@@ -220,7 +220,7 @@ export async function runTeamTask(
 				tasks: updateTask(tasks, cancelledTask),
 			};
 		}
-		tasks = persistSingleTaskUpdate(manifest, tasks, task);
+		tasks = persistSingleTaskUpdate(manifest, tasks, task, "started");
 		if (runtimeKind === "child-process")
 			({ task, tasks } = checkpointTask(
 				manifest,
@@ -261,7 +261,7 @@ export async function runTeamTask(
 						teamRole: { skills: input.teamRoleSkills },
 						step: input.step,
 						override: input.skillOverride,
-						runId: manifest.runId,  // ECC INSTINCT: Enable skill confidence tracking
+						runId: manifest.runId,  
 					})
 				: undefined;
 		const skillBlock = input.skillBlock ?? renderedSkills?.block;
@@ -386,15 +386,29 @@ export async function runTeamTask(
 			let lastRunProgressSummary: ProgressEventSummary | undefined;
 			const persistHeartbeat = (force = false): void => {
 				const now = Date.now();
+				// Skip disk write if throttled (unless forced).
 				if (!force && now - lastHeartbeatPersistedAt < 1000) return;
-				lastHeartbeatPersistedAt = now;
+				try {
+					// Write to disk first, then update in-memory.
+					// Disk state is always <= in-memory state, so a crash never produces
+					// a fresher in-memory heartbeat than what's on disk. This prevents the
+					// stale reconciler from seeing a live heartbeat paired with stale task state
+					// (which could cause false zombie detection).
+					tasks = persistSingleTaskUpdate(manifest, tasks, task);
+				} catch (err) {
+					// Run state may have been deleted by prune/forget/cleanup.
+					// This is not fatal — the run is gone, no point persisting.
+					if ((err as NodeJS.ErrnoException).code === "ENOENT") return;
+					throw err;
+				}
+				// Now update in-memory heartbeat so it is always >= persisted state.
 				task = {
 					...task,
 					heartbeat: touchWorkerHeartbeat(
 						task.heartbeat ?? createWorkerHeartbeat(task.id),
 					),
 				};
-				tasks = persistSingleTaskUpdate(manifest, tasks, task);
+				lastHeartbeatPersistedAt = now;
 			};
 			const persistChildProgress = (
 				event: unknown,
@@ -438,6 +452,10 @@ export async function runTeamTask(
 			for (let i = 0; i < attemptModels.length; i++) {
 				// M1 fix: set transcript path per attempt to avoid mixing across fallback attempts.
 				transcriptPath = `${manifest.artifactsRoot}/transcripts/${task.id}.attempt-${i}.jsonl`;
+				// Ensure transcripts/ subdirectory exists before child-pi appends
+				// to it. appendTranscript uses O_APPEND (no mkdir) for security,
+				// so the caller must create the directory.
+				fs.mkdirSync(path.join(manifest.artifactsRoot, "transcripts"), { recursive: true });
 				const model = attemptModels[i];
 				const attemptStartedAt = new Date();
 				const pendingAttempt: ModelAttemptSummary = {
@@ -472,6 +490,7 @@ export async function runTeamTask(
 					role: task.role,
 					runId: manifest.runId,
 					agentId: task.id,
+					artifactsRoot: manifest.artifactsRoot,
 					onSpawn: (pid) => {
 						try {
 							({ task, tasks } = checkpointTask(
@@ -521,89 +540,90 @@ export async function runTeamTask(
 						// Errors are logged but processing continues so subsequent events still update state.
 						try {
 							appendCrewAgentEvent(manifest, task.id, event);
-						} catch (err) {
-							logInternalError("task-runner.append-crew-agent-event", err, `taskId=${task.id}`);
-						}
-						if (
-							event &&
-							typeof event === "object" &&
-							!Array.isArray(event)
-						)
-							collectedJsonEvents.push(
-								event as Record<string, unknown>,
-							);
-							if (collectedJsonEvents.length > 1000) {
-								collectedJsonEvents.splice(0, collectedJsonEvents.length - 1000);
-							}
-						// Accumulate lifetime usage via message_end events (survives compaction)
-						if (event && typeof event === "object" && (event as Record<string, unknown>).type === "message_end") {
-							const msg = (event as Record<string, unknown>).message as Record<string, unknown> | undefined;
-							if (msg?.role === "assistant") {
-								const usage = msg.usage as Record<string, number> | undefined;
-								if (usage) {
-									task.lifetimeUsage = {
-										input: (task.lifetimeUsage?.input ?? 0) + (usage.input ?? 0),
-										output: (task.lifetimeUsage?.output ?? 0) + (usage.output ?? 0),
-										cacheWrite: (task.lifetimeUsage?.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
-									};
+							if (
+								event &&
+								typeof event === "object" &&
+								!Array.isArray(event)
+							)
+								collectedJsonEvents.push(
+									event as Record<string, unknown>,
+								);
+								if (collectedJsonEvents.length > 1000) {
+									collectedJsonEvents.splice(0, collectedJsonEvents.length - 1000);
+								}
+							// Accumulate lifetime usage via message_end events (survives compaction)
+							if (event && typeof event === "object" && (event as Record<string, unknown>).type === "message_end") {
+								const msg = (event as Record<string, unknown>).message as Record<string, unknown> | undefined;
+								if (msg?.role === "assistant") {
+									const usage = msg.usage as Record<string, number> | undefined;
+									if (usage) {
+										task.lifetimeUsage = {
+											input: (task.lifetimeUsage?.input ?? 0) + (usage.input ?? 0),
+											output: (task.lifetimeUsage?.output ?? 0) + (usage.output ?? 0),
+											cacheWrite: (task.lifetimeUsage?.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
+										};
+									}
 								}
 							}
-						}
-						persistHeartbeat();
-						// Bug #3 fix: Write worker JSON events to background.log for debugging when running in background mode.
-						// This supplements the event log so developers can see what the child Pi worker produced.
-						if (process.env.PI_CREW_BACKGROUND_MODE === "1" && event) {
-							try {
+							persistHeartbeat();
+							// Bug #3 fix: Write worker JSON events to background.log for debugging when running in background mode.
+							// This supplements the event log so developers can see what the child Pi worker produced.
+							if (process.env.PI_CREW_BACKGROUND_MODE === "1" && event) {
 								const bgLogPath = `${manifest.stateRoot}/background.log`;
 								const eventLine = typeof event === "object" && !Array.isArray(event) ? JSON.stringify(event) : String(event);
 								fs.appendFileSync(bgLogPath, `${eventLine}\n`);
-							} catch { /* background log write failures should not affect task */ }
-						}
-						task = {
-							...task,
-							agentProgress: applyAgentProgressEvent(
-								task.agentProgress ?? emptyCrewAgentProgress(),
-								event,
-								task.startedAt,
-							),
-						};
-						tasks = updateTask(tasks, task);
-						// Bridge event to UI event bus for near-instant updates
-						try {
+							}
+							// Apply agentProgress update first, then persist, then update in-memory array.
+							// This ensures disk state is always >= in-memory state, preventing
+							// fresher in-memory state from being lost on crash.
+							tasks = persistSingleTaskUpdate(manifest, tasks, {
+								...task,
+								agentProgress: applyAgentProgressEvent(
+									task.agentProgress ?? emptyCrewAgentProgress(),
+									event,
+									task.startedAt,
+								),
+							});
+							task = {
+								...task,
+								agentProgress: applyAgentProgressEvent(
+									task.agentProgress ?? emptyCrewAgentProgress(),
+									event,
+									task.startedAt,
+								),
+							};
+							tasks = updateTask(tasks, task);
+							// Bridge event to UI event bus for near-instant updates
 							const bridgeEvent = bridgeEventFromJsonEvent(
 								manifest.runId,
 								task.id,
 								event,
 							);
 							if (bridgeEvent) streamBridge?.handler(bridgeEvent);
-						} catch {
-							/* bridge errors should not affect task */
-						}
-						// Feed overflow recovery tracker
-						if (input.onJsonEvent) {
-							try {
+							// Feed overflow recovery tracker
+							if (input.onJsonEvent) {
 								input.onJsonEvent(
 									task.id,
 									manifest.runId,
 									event,
 								);
-							} catch {
-								/* overflow tracking errors should not affect task */
 							}
+							if (
+								!finalCheckpointWritten &&
+								isFinalChildEvent(event)
+							) {
+								finalCheckpointWritten = true;
+								({ task, tasks } = checkpointTask(
+									manifest,
+									tasks,
+									task,
+									"child-stdout-final",
+								));
+							}
+							persistChildProgress(event);
+						} catch (err) {
+							logInternalError("task-runner.on-json-event", err as Error, `taskId=${task.id}`);
 						}
-						if (
-							!finalCheckpointWritten &&
-							isFinalChildEvent(event)
-						) {
-							finalCheckpointWritten = true;
-							({ task, tasks } = checkpointTask(
-								manifest,
-								tasks,
-								task,
-								"child-stdout-final",
-							));
-						}
-						persistChildProgress(event);
 					},
 				});
 				const evidenceStatus = childResult.exitStatus?.cancelled
@@ -881,6 +901,8 @@ export async function runTeamTask(
 			} else {
 				resultArtifact = live.resultArtifact;
 			}
+			// Sync task.resultArtifact with the re-written artifact (if liveText was truthy)
+			task = { ...task, resultArtifact };
 			logArtifact = live.logArtifact;
 			transcriptArtifact = live.transcriptArtifact;
 		} else {
@@ -1065,7 +1087,7 @@ export async function runTeamTask(
 		// 2. Verification contract has commands
 		// 3. Not in scaffold mode (scaffold mode intentionally skips execution)
 		let verificationEvidence: VerificationEvidence = baseEvidence;
-		if (!error && runtimeKind !== "scaffold" && taskPacket.verification?.commands?.length) {
+		if (runtimeKind !== "scaffold" && taskPacket.verification?.commands?.length) {
 			try {
 				const commandResults = await executeVerificationCommands(
 					taskPacket.verification,