pi-crew 0.6.0 → 0.6.3

package/src/state/run-metrics.ts

@@ -1,8 +1,9 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { loadRunManifestById } from "./state-store.ts";
 import { projectCrewRoot } from "../utils/paths.ts";
+import { assertSafePathId } from "../utils/safe-paths.ts";
 import { atomicWriteJson, readJsonFile } from "./atomic-write.ts";
+import { loadRunManifestById } from "./state-store.ts";
 
 /**
  * Run metrics snapshot captured after a run completes (or on demand).
@@ -28,6 +29,7 @@ function metricsDir(cwd: string): string {
 }
 
 function metricsFilePath(cwd: string, runId: string): string {
+	assertSafePathId("runId", runId);
 	return path.join(metricsDir(cwd), `${runId}.json`);
 }
 
@@ -35,8 +37,12 @@ function metricsFilePath(cwd: string, runId: string): string {
  * Collect metrics for a run by reading its manifest, tasks, and event log.
  * Returns undefined if the run cannot be loaded.
  */
-export function collectRunMetrics(cwd: string, runId: string): RunMetrics | undefined {
-	const result = loadRunManifestById(cwd, runId);
+export function collectRunMetrics(
+	cwd: string,
+	runId: string,
+): RunMetrics | undefined {
+	assertSafePathId("runId", runId);
+	const result = loadRunManifestById(cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
 	if (!result) return undefined;
 
 	const { manifest, tasks } = result;
@@ -63,11 +69,17 @@ export function collectRunMetrics(cwd: string, runId: string): RunMetrics | unde
 	// Duration: from run createdAt to updatedAt (manifest timestamps), or 0 if unavailable.
 	const createdAt = new Date(manifest.createdAt).getTime();
 	const updatedAt = new Date(manifest.updatedAt).getTime();
-	const durationMs = isNaN(createdAt) || isNaN(updatedAt) ? 0 : Math.max(0, updatedAt - createdAt);
+	const durationMs =
+		isNaN(createdAt) || isNaN(updatedAt)
+			? 0
+			: Math.max(0, updatedAt - createdAt);
 
 	// Consistency score: proportion of tasks that completed successfully among all non-skipped tasks.
 	const nonSkippedTasks = tasks.filter((t) => t.status !== "skipped");
-	const consistencyScore = nonSkippedTasks.length > 0 ? completedCount / nonSkippedTasks.length : 1.0;
+	const consistencyScore =
+		nonSkippedTasks.length > 0
+			? completedCount / nonSkippedTasks.length
+			: 1.0;
 
 	return {
 		runId,
@@ -96,7 +108,10 @@ export function saveRunMetrics(cwd: string, metrics: RunMetrics): void {
  * Load a previously saved metrics snapshot.
  * Returns undefined if the file does not exist or cannot be parsed.
  */
-export function loadRunMetrics(cwd: string, runId: string): RunMetrics | undefined {
+export function loadRunMetrics(
+	cwd: string,
+	runId: string,
+): RunMetrics | undefined {
 	return readJsonFile<RunMetrics>(metricsFilePath(cwd, runId));
 }
 
@@ -125,10 +140,11 @@ export function getRunMetricsSummary(cwd: string, limit = 25): RunMetrics[] {
 
 	// Sort newest first (by timestamp, then runId as tiebreaker).
 	metrics.sort((a, b) => {
-		const diff = new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime();
+		const diff =
+			new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime();
 		if (diff !== 0) return diff;
 		return b.runId.localeCompare(a.runId);
 	});
 
 	return metrics.slice(0, limit);
-}
+}

package/src/state/state-store.ts

@@ -9,10 +9,13 @@ import { DEFAULT_CACHE, DEFAULT_PATHS } from "../config/defaults.ts";
 import { createRunId, createTaskId } from "../utils/ids.ts";
 import { findRepoRoot, projectCrewRoot, userCrewRoot } from "../utils/paths.ts";
 import { assertSafePathId, resolveContainedRelativePath, resolveRealContainedPath } from "../utils/safe-paths.ts";
-import { withRunLock } from "./locks.ts";
+import { withRunLock, withRunLockSync } from "./locks.ts";
+import { logInternalError } from "../utils/internal-error.ts";
+import { errors } from "../errors.ts";
 import type { TeamConfig } from "../teams/team-config.ts";
 import type { WorkflowConfig } from "../workflows/workflow-config.ts";
 import { toPiSessionId } from "../utils/session-utils.ts";
+import { HealthStore } from "./health-store.ts";
 
 export interface RunPaths {
 	runId: string;
@@ -31,19 +34,43 @@ interface ManifestCacheEntry {
 	tasksMtimeMs: number;
 	tasksSize: number;
 	cachedAt?: number;
+	generation?: number;
 }
 
-const MANIFEST_CACHE_TTL_MS = 30 * 1000; // 30 seconds (FIX: reduced from 5 minutes for faster state updates)
+// Global generation counter incremented on each cache invalidation.
+// Detects staleness even when mtime/size are spoofed by an attacker.
+let manifestCacheGeneration = 0;
+
+const MANIFEST_CACHE_TTL_MS = 15 * 1000; // 15 seconds (FIX: increased from 5s for read-heavy workloads; 5s was too short causing unnecessary cache invalidation)
+const LOAD_MANIFEST_RETRY_LIMIT = 5; // Configurable retry limit for mtime/size stability checks under contention
 const manifestCache = new Map<string, ManifestCacheEntry>();
 
 function setManifestCache(stateRoot: string, entry: ManifestCacheEntry): void {
 	if (manifestCache.has(stateRoot)) manifestCache.delete(stateRoot);
 	entry.cachedAt = Date.now();
+	entry.generation = manifestCacheGeneration;
+	// FIX: Evict all stale entries by TTL before adding new entry.
+	// This ensures entries that are never accessed still get evicted
+	// based on TTL, not just entries that are hit.
+	const now = Date.now();
+	for (const [key, val] of manifestCache.entries()) {
+		if (val.cachedAt && now - val.cachedAt > MANIFEST_CACHE_TTL_MS) {
+			manifestCache.delete(key);
+		}
+	}
 	manifestCache.set(stateRoot, entry);
 	while (manifestCache.size > DEFAULT_CACHE.manifestMaxEntries) {
-		const oldest = manifestCache.keys().next().value;
-		if (!oldest) break;
-		manifestCache.delete(oldest);
+		// FIX: Evict oldest entry by cachedAt (LRU), not insertion order.
+		// cachedAt is set on both initial insertion and cache hits, so
+		// frequently accessed entries bubble to the end and survive longer.
+		let oldestKey: string | undefined;
+		let oldestTime = Infinity;
+		for (const [key, val] of manifestCache.entries()) {
+			const t = val.cachedAt ?? 0;
+			if (t < oldestTime) { oldestTime = t; oldestKey = key; }
+		}
+		if (!oldestKey) break;
+		manifestCache.delete(oldestKey);
 	}
 }
 
@@ -53,6 +80,7 @@ function useProjectState(cwd: string): boolean {
 
 function invalidateRunCache(stateRoot: string): void {
 	manifestCache.delete(stateRoot);
+	manifestCacheGeneration++;
 }
 
 function scopeBaseRoot(cwd: string): string {
@@ -76,10 +104,19 @@ function resolveRunStateRoot(cwd: string, runId: string): string | undefined {
 }
 
 function validateRunManifestPaths(cwd: string, runId: string, manifest: TeamRunManifest, stateRoot: string, tasksPath: string): boolean {
+	// Issue 2 fix: Reject manifests missing status field to prevent undefined
+	// behavior in callers like canTransitionRunStatus(manifest.status, newStatus).
+	if (!manifest.status || typeof manifest.status !== "string") return false;
 	if (manifest.runId !== runId || manifest.stateRoot !== stateRoot || manifest.tasksPath !== tasksPath || manifest.eventsPath !== path.join(stateRoot, "events.jsonl")) return false;
 	const artifactsParent = path.join(scopeBaseRoot(cwd), DEFAULT_PATHS.state.artifactsSubdir);
 	const expectedArtifactsRoot = resolveContainedRelativePath(artifactsParent, runId, "runId");
 	if (manifest.artifactsRoot !== expectedArtifactsRoot) return false;
+	// Always validate artifactsRoot is not a symlink, even when manifest has
+	// no artifacts entries. A symlinked artifactsRoot pointing outside the
+	// artifacts parent is a security violation (could write to attacker-
+	// controlled locations). This check catches the case where the run was
+	// created legitimately but the artifactsRoot was later replaced with a
+	// symlink between manifest creation and load.
 	if (fs.existsSync(expectedArtifactsRoot)) {
 		try {
 			if (fs.lstatSync(expectedArtifactsRoot).isSymbolicLink()) return false;
@@ -87,6 +124,10 @@ function validateRunManifestPaths(cwd: string, runId: string, manifest: TeamRunM
 		} catch {
 			return false;
 		}
+	} else if (manifest.artifacts && manifest.artifacts.length > 0) {
+		// Has artifacts entries but directory doesn't exist - benign state for
+		// runs still in progress.
+		return true;
 	}
 	return true;
 }
@@ -167,8 +208,14 @@ export function createRunManifest(params: {
 	};
 	fs.mkdirSync(paths.stateRoot, { recursive: true });
 	fs.mkdirSync(paths.artifactsRoot, { recursive: true });
-	atomicWriteJson(paths.manifestPath, manifest);
-	atomicWriteJson(paths.tasksPath, tasks);
+	// FIX: Use saveManifestAndTasksAtomicSync and check result to detect
+	// partial failure. If tasksWritten=false when manifestWritten=true,
+	// throw to ensure manifest and tasks are always consistent.
+	const result = saveManifestAndTasksAtomicSync(manifest, tasks);
+	if (!result.manifestWritten || !result.tasksWritten) {
+		throw errors.fileWrite(paths.stateRoot, result.error as unknown as NodeJS.ErrnoException)
+			.withContext(`saveManifestAndTasksAtomicSync: manifestWritten=${result.manifestWritten}, tasksWritten=${result.tasksWritten}`);
+	}
 	appendEvent(paths.eventsPath, {
 		type: "run.created",
 		runId: paths.runId,
@@ -186,18 +233,105 @@ export function createRunManifest(params: {
 }
 
 export function saveRunManifest(manifest: TeamRunManifest): void {
-	atomicWriteJson(path.join(manifest.stateRoot, "manifest.json"), manifest);
+	// FIX: Invalidate cache BEFORE atomic write. The order matters for crash
+	// safety: if we invalidated after the write and crashed before invalidation,
+	// the stale cache entry (up to MANIFEST_CACHE_TTL_MS old) could be served.
+	// By invalidating first, the worst case is a cache miss forcing a disk read,
+	// which is always safe.
 	invalidateRunCache(manifest.stateRoot);
+	const manifestPath = path.join(manifest.stateRoot, "manifest.json");
+	atomicWriteJson(manifestPath, manifest);
+	// FIX: Re-populate cache with actual mtime/size so loadRunManifestById
+	// doesn't miss the cache on next read. Without this, every load until
+	// TTL expires would hit disk because cached 0 !== any real mtime.
+	// NOTE: tasks is set to [] here because saveRunManifest only writes the
+	// manifest file, not tasks.json. Callers that need tasks should call
+	// saveRunTasks or loadRunTasks separately. If loadRunManifestById is called
+	// immediately after saveRunManifest, it may return empty tasks even if
+	// tasks.json exists on disk — the mtime/size cache check should invalidate
+	// on next read, but the returned empty tasks array could confuse callers
+	// that don't re-read.
+	const manifestStat = fs.statSync(manifestPath);
+	setManifestCache(manifest.stateRoot, {
+		manifest,
+		tasks: [],
+		manifestMtimeMs: manifestStat.mtimeMs,
+		manifestSize: manifestStat.size,
+		tasksMtimeMs: 0,
+		tasksSize: 0,
+	});
 }
 
 export async function saveRunManifestAsync(manifest: TeamRunManifest): Promise<void> {
-	await atomicWriteJsonAsync(path.join(manifest.stateRoot, "manifest.json"), manifest);
+	// FIX: Invalidate cache BEFORE atomic write to prevent stale cache serving
+	// after a crash. See saveRunManifest for full explanation.
 	invalidateRunCache(manifest.stateRoot);
+	const manifestPath = path.join(manifest.stateRoot, "manifest.json");
+	await atomicWriteJsonAsync(manifestPath, manifest);
+	// FIX: Re-populate cache with actual mtime/size. See saveRunManifest.
+	const manifestStat = await fs.promises.stat(manifestPath);
+	setManifestCache(manifest.stateRoot, {
+		manifest,
+		tasks: [],
+		manifestMtimeMs: manifestStat.mtimeMs,
+		manifestSize: manifestStat.size,
+		tasksMtimeMs: 0,
+		tasksSize: 0,
+	});
 }
 
 export function saveRunTasks(manifest: TeamRunManifest, tasks: TeamTaskState[]): void {
-	atomicWriteJson(manifest.tasksPath, tasks);
+	// FIX: Invalidate cache BEFORE atomic write to prevent stale cache serving.
 	invalidateRunCache(manifest.stateRoot);
+
+	// Guard: if the run state directory has been removed (prune/forget/cleanup),
+	// silently return — there is nothing to persist and the run is gone.
+	try {
+		fs.statSync(manifest.stateRoot);
+	} catch {
+		return;
+	}
+
+	atomicWriteJson(manifest.tasksPath, tasks);
+	// FIX: Re-populate cache with actual mtime/size for manifest and tasks.
+	// Note: We re-read manifest from disk to get its current mtime/size
+	// since we only wrote tasks here.
+	const manifestPath = path.join(manifest.stateRoot, "manifest.json");
+	let manifestStat: fs.Stats;
+	let tasksStat: fs.Stats;
+	try {
+		manifestStat = fs.statSync(manifestPath);
+		tasksStat = fs.statSync(manifest.tasksPath);
+	} catch {
+		// Run state disappeared between the guard above and now — give up gracefully.
+		return;
+	}
+	// FIX: If cache was evicted, re-read manifest from disk rather than using
+	// a minimal fallback. A stale minimal manifest with only runId populated
+	// would cause manifest.status to be undefined, breaking status checks.
+	// If the manifest cannot be read, throw an error — callers using withRunLock
+	// should never hit this case, and the error indicates a serious problem.
+	// FIX: Also check that the re-read manifest has a status field. If not,
+	// fall back to the manifest parameter's status rather than serving a
+	// degraded manifest that would break status transition checks.
+	const cached = manifestCache.get(manifest.stateRoot);
+	const manifestEntry = cached?.manifest ?? readJsonFile<TeamRunManifest>(manifestPath);
+	if (!manifestEntry) {
+		return; // Run deleted between guard and read
+	}
+	// Preserve current status from the manifest parameter if the on-disk
+	// manifest is missing it (could be a partial write).
+	if (!manifestEntry.status) {
+		manifestEntry.status = manifest.status;
+	}
+	setManifestCache(manifest.stateRoot, {
+		manifest: manifestEntry,
+		tasks,
+		manifestMtimeMs: manifestStat.mtimeMs,
+		manifestSize: manifestStat.size,
+		tasksMtimeMs: tasksStat.mtimeMs,
+		tasksSize: tasksStat.size,
+	});
 }
 
 /**
@@ -208,32 +342,92 @@ export function saveRunTasks(manifest: TeamRunManifest, tasks: TeamTaskState[]):
  * intended use case. Single-update + read-update loops (e.g.
  * persistSingleTaskUpdate) should keep using saveRunTasks.
  */
-export function saveRunTasksCoalesced(manifest: TeamRunManifest, tasks: TeamTaskState[]): void {
-	atomicWriteJsonCoalesced(manifest.tasksPath, tasks);
+/** @internal */
+function saveRunTasksCoalesced(manifest: TeamRunManifest, tasks: TeamTaskState[]): void {
+	// FIX: Invalidate cache BEFORE atomic write to prevent stale cache serving.
 	invalidateRunCache(manifest.stateRoot);
+	try { fs.statSync(manifest.stateRoot); } catch { return; }
+	atomicWriteJsonCoalesced(manifest.tasksPath, tasks);
 }
 
 export async function saveRunTasksAsync(manifest: TeamRunManifest, tasks: TeamTaskState[]): Promise<void> {
-	await atomicWriteJsonAsync(manifest.tasksPath, tasks);
+	// FIX: Invalidate cache BEFORE atomic write to prevent stale cache serving.
 	invalidateRunCache(manifest.stateRoot);
+	try { fs.statSync(manifest.stateRoot); } catch { return; }
+	await atomicWriteJsonAsync(manifest.tasksPath, tasks);
 }
 
 /**
  * Save manifest and tasks files with individual atomic writes.
- *
- * Note: The two writes are individually atomic (via rename) but not
- * jointly atomic — a crash between writes can leave them inconsistent.
- * This is acceptable because crash recovery detects and repairs
- * inconsistent state on next session start.
+ * FIX: Changed from Promise.all (parallel, non-jointly-atomic) to sequential
+ * writes to ensure manifest is written before tasks. A crash between writes
+ * leaves them in a known state (manifest is older, tasks is newer) which
+ * loadRunManifestById's retry loop can detect via mtime comparison.
+ * NOTE: There is no stale-reconciler component — the retry loop provides
+ * best-effort detection of mid-write crashes by re-reading until mtime/size
+ * are stable. For strict atomicity, callers should use withRunLock().
+ * FIX: Returns a result object so callers know which write step failed.
+ * If manifest write succeeds but tasks write fails, the caller can recover.
  */
-export async function saveManifestAndTasksAtomic(manifest: TeamRunManifest, tasks: TeamTaskState[]): Promise<void> {
-	await withRunLock(manifest, async () => {
-		await Promise.all([
-			atomicWriteJsonAsync(path.join(manifest.stateRoot, "manifest.json"), manifest),
-			atomicWriteJsonAsync(manifest.tasksPath, tasks),
-		]);
-		invalidateRunCache(manifest.stateRoot);
-	});
+/** @internal */
+interface SaveManifestAndTasksResult {
+	manifestWritten: boolean;
+	tasksWritten: boolean;
+	error?: string;
+}
+
+async function saveManifestAndTasksAtomic(manifest: TeamRunManifest, tasks: TeamTaskState[]): Promise<SaveManifestAndTasksResult> {
+	let manifestWritten = false;
+	let tasksWritten = false;
+	try {
+		await withRunLock(manifest, async () => {
+			// FIX: Invalidate cache BEFORE writes to prevent stale cache serving.
+			// Sequential writes instead of Promise.all to ensure manifest is
+			// written before tasks. If a crash occurs between writes, manifest is
+			// the older timestamp which stale-reconciler uses to detect inconsistency.
+			invalidateRunCache(manifest.stateRoot);
+			await atomicWriteJsonAsync(path.join(manifest.stateRoot, "manifest.json"), manifest);
+			manifestWritten = true;
+			await atomicWriteJsonAsync(manifest.tasksPath, tasks);
+			tasksWritten = true;
+		});
+	} catch (err) {
+		return {
+			manifestWritten,
+			tasksWritten,
+			// FIX: Use String(err) to safely convert any thrown value (Error,
+			// string, number, null, undefined) to a string. err.message would be
+			// undefined for non-Error throwables, losing useful context.
+			error: String(err),
+		};
+	}
+	return { manifestWritten: true, tasksWritten: true };
+}
+
+/** @internal */
+function saveManifestAndTasksAtomicSync(manifest: TeamRunManifest, tasks: TeamTaskState[]): SaveManifestAndTasksResult {
+	let manifestWritten = false;
+	let tasksWritten = false;
+	try {
+		withRunLockSync(manifest, () => {
+			// FIX: Invalidate cache BEFORE writes to prevent stale cache serving.
+			invalidateRunCache(manifest.stateRoot);
+			atomicWriteJson(path.join(manifest.stateRoot, "manifest.json"), manifest);
+			manifestWritten = true;
+			atomicWriteJson(manifest.tasksPath, tasks);
+			tasksWritten = true;
+		});
+	} catch (err) {
+		return {
+			manifestWritten,
+			tasksWritten,
+			// FIX: Use String(err) to safely convert any thrown value (Error,
+			// string, number, null, undefined) to a string. err.message would be
+			// undefined for non-Error throwables, losing useful context.
+			error: String(err),
+		};
+	}
+	return { manifestWritten: true, tasksWritten: true };
 }
 
 export interface UpdateRunStatusOptions {
@@ -243,7 +437,7 @@ export interface UpdateRunStatusOptions {
 
 export function updateRunStatus(manifest: TeamRunManifest, status: TeamRunManifest["status"], summary?: string, options: UpdateRunStatusOptions = {}): TeamRunManifest {
 	if (!canTransitionRunStatus(manifest.status, status)) {
-		throw new Error(`Invalid run status transition: ${manifest.status} -> ${status}`);
+		throw errors.invalidStatusTransition(manifest.status, status);
 	}
 	const updated: TeamRunManifest = { ...manifest, status, updatedAt: new Date().toISOString(), summary: summary ?? manifest.summary };
 	saveRunManifest(updated);
@@ -281,11 +475,23 @@ export function __test__clearManifestCache(): void {
 async function readJsonFileAsync<T>(filePath: string): Promise<T | undefined> {
 	try {
 		return JSON.parse(await fs.promises.readFile(filePath, "utf-8")) as T;
-	} catch {
+	} catch (err) {
+		const code = (err as NodeJS.ErrnoException).code;
+		if (code !== "ENOENT" && code !== "ENOTDIR") {
+			logInternalError("readJsonFileAsync", err, `filePath=${filePath}`);
+		}
 		return undefined;
 	}
 }
 
+/**
+ * Load a run manifest and its tasks by runId.
+ * WARNING: This function provides best-effort consistency only. The sentinel-based
+ * retry loop does NOT guarantee manifest/tasks consistency under contention —
+ * a concurrent writer can complete a full write cycle between the final stat
+ * and the read. For strict consistency, callers MUST wrap load+modify+save in
+ * withRunLock(). Callers that need guaranteed consistency should use the lock.
+ */
 export function loadRunManifestById(cwd: string, runId: string): { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined {
 	const stateRoot = resolveRunStateRoot(cwd, runId);
 	if (!stateRoot) return undefined;
@@ -299,6 +505,12 @@ export function loadRunManifestById(cwd: string, runId: string): { manifest: Tea
 		return undefined;
 	}
 	const cached = manifestCache.get(stateRoot);
+	// Issue 1 fix: Note that the cache read-check-use sequence below is NOT atomic
+	// under concurrent modification. While individual JS Map operations are atomic,
+	// the sequence spanning multiple filesystem operations is not atomic. Callers MUST
+	// NOT invoke loadRunManifestById concurrently with cache-modifying operations
+	// (setManifestCache, invalidateRunCache) for the same stateRoot. The generation
+	// counter provides some protection, but does not make the read-check-use atomic.
 	let tasksStat: fs.Stats | undefined;
 	try {
 		tasksStat = fs.statSync(tasksPath);
@@ -312,28 +524,43 @@ export function loadRunManifestById(cwd: string, runId: string): { manifest: Tea
 		&& cached.manifestSize === manifestStat.size
 		&& cached.tasksMtimeMs === tasksMtimeMs
 		&& cached.tasksSize === (tasksStat?.size ?? 0)
+		&& cached.generation === manifestCacheGeneration
 	) {
 		// TTL eviction: expire stale entries even if mtime matches
-		if (cached.cachedAt && Date.now() - cached.cachedAt > MANIFEST_CACHE_TTL_MS) {
+		// FIX: Also evict entries where cachedAt is undefined — such entries are
+		// effectively immortal otherwise (the `cached.cachedAt &&` check would skip
+		// them every time). This can happen if a cache entry was created by
+		// setManifestCache that didn't set cachedAt (shouldn't happen in current
+		// code, but defensive against future regressions).
+		if (!cached.cachedAt || Date.now() - cached.cachedAt > MANIFEST_CACHE_TTL_MS) {
 			manifestCache.delete(stateRoot);
 		} else if (!validateRunManifestPaths(cwd, runId, cached.manifest, stateRoot, tasksPath)) {
 			manifestCache.delete(stateRoot);
 			return undefined;
+			} else if (!fs.existsSync(tasksPath)) {
+			// Tasks file was deleted after cache was populated — this is a cache miss,
+			// not a manifest inconsistency. The cache check passes because
+			// tasksMtimeMs=0 and tasksSize=0 match a cache entry written when tasks.json
+			// didn't exist. We fall through to the retry loop which will correctly
+			// detect the missing file and return undefined. The alternative (treating
+			// this as an inconsistency) would require failing the cache hit path, which
+			// is unnecessary since the retry loop handles it correctly anyway.
+			manifestCache.delete(stateRoot);
+			return undefined;
 		} else {
-			return { manifest: cached.manifest, tasks: cached.tasks };
+			return { manifest: cached.manifest, tasks: cached.tasks ?? [] };
 		}
 	}
 
-	// FIX: Re-stat and re-read inside a single synchronous block to close the
-	// TOCTOU window. We use a sentinel-based re-read: if mtime/size changed
-	// between the initial stat and the read, re-read until stable. With file
-	// sizes typically small (<5MB), the extra cost is negligible. Note: this
-	// doesn't fully prevent torn writes — callers needing strict consistency
-	// should use withRunLock() around the whole load+modify+save sequence.
+	// FIX: Sentinel-based retry loop for best-effort consistency. Re-stat and
+	// re-read until mtime/size are stable. The retry limit is now configurable
+	// via LOAD_MANIFEST_RETRY_LIMIT (was hardcoded 3). High contention can still
+	// cause non-convergence — for strict consistency, callers MUST use withRunLock().
+	// Issue 3 fix: Made retry limit configurable instead of hardcoded 3.
 	let attempts = 0;
 	let manifest: TeamRunManifest | undefined;
 	let tasks: TeamTaskState[] | undefined;
-	while (attempts < 3) {
+	while (attempts < LOAD_MANIFEST_RETRY_LIMIT) {
 		const freshStat = fs.statSync(manifestPath);
 		manifest = readJsonFile<TeamRunManifest>(manifestPath);
 		const freshTasksStat = fs.existsSync(tasksPath) ? fs.statSync(tasksPath) : undefined;
@@ -347,6 +574,21 @@ export function loadRunManifestById(cwd: string, runId: string): { manifest: Tea
 		manifestStat = freshStat;
 		tasksStat = freshTasksStat;
 	}
+	// WARNING: Best-effort consistency only — retry loop detected mtime/size
+	// instability. A concurrent writer can still complete a full write cycle
+	// between the final stat and the read. Callers needing strict consistency
+	// MUST use withRunLock() around load+modify+save.
+	if (attempts > 0) {
+		console.warn(`[state-store] loadRunManifestById: retry loop detected instability for run ${runId} after ${attempts} attempt(s) — best-effort only, use withRunLock() for strict consistency`);
+	}
+	// NOTE: manifest mtime may legitimately be >= tasks mtime because
+	// saveManifestAndTasksAtomicSync writes manifest before tasks. However,
+	// if a crash occurs AFTER tasks is written but BEFORE manifest is written,
+	// tasks mtime would be > manifest mtime (the opposite). The retry loop
+	// above detects this crash state by re-reading until mtime/size are stable
+	// — the final stable state is what gets used. Because the retry loop
+	// handles the crash case, we do NOT fail based on this comparison alone.
+	// It does not indicate corruption on its own.
 	if (!manifest || !validateRunManifestPaths(cwd, runId, manifest, stateRoot, tasksPath)) return undefined;
 	setManifestCache(stateRoot, {
 		manifest,
@@ -364,6 +606,7 @@ export async function loadRunManifestByIdAsync(cwd: string, runId: string): Prom
 	if (!stateRoot) return undefined;
 	const manifestPath = path.join(stateRoot, "manifest.json");
 	const tasksPath = path.join(stateRoot, "tasks.json");
+
 	let manifestStat: fs.Stats;
 	try {
 		manifestStat = await fs.promises.stat(manifestPath);
@@ -371,6 +614,8 @@ export async function loadRunManifestByIdAsync(cwd: string, runId: string): Prom
 		return undefined;
 	}
 	const cached = manifestCache.get(stateRoot);
+	// Issue 1 fix: Note that the cache read-check-use sequence below is NOT atomic
+	// under concurrent modification. Same as sync version — see loadRunManifestById.
 	let tasksStat: fs.Stats | undefined;
 	try {
 		tasksStat = await fs.promises.stat(tasksPath);
@@ -378,20 +623,64 @@ export async function loadRunManifestByIdAsync(cwd: string, runId: string): Prom
 		tasksStat = undefined;
 	}
 	const tasksMtimeMs = tasksStat?.mtimeMs ?? 0;
-	if (cached && cached.manifestMtimeMs === manifestStat.mtimeMs && cached.manifestSize === manifestStat.size && cached.tasksMtimeMs === tasksMtimeMs && cached.tasksSize === (tasksStat?.size ?? 0)) {
+	if (cached && cached.manifestMtimeMs === manifestStat.mtimeMs && cached.manifestSize === manifestStat.size && cached.tasksMtimeMs === tasksMtimeMs && cached.tasksSize === (tasksStat?.size ?? 0) && cached.generation === manifestCacheGeneration) {
 		// TTL eviction: expire stale entries even if mtime matches
-		if (cached.cachedAt && Date.now() - cached.cachedAt > MANIFEST_CACHE_TTL_MS) {
+		// FIX: Also evict entries where cachedAt is undefined — such entries are
+		// effectively immortal otherwise (the `cached.cachedAt &&` check would skip
+		// them every time). This can happen if a cache entry was created by
+		// setManifestCache that didn't set cachedAt (shouldn't happen in current
+		// code, but defensive against future regressions).
+		if (!cached.cachedAt || Date.now() - cached.cachedAt > MANIFEST_CACHE_TTL_MS) {
 			manifestCache.delete(stateRoot);
 		} else if (!validateRunManifestPaths(cwd, runId, cached.manifest, stateRoot, tasksPath)) {
 			manifestCache.delete(stateRoot);
 			return undefined;
+			} else if (!fs.existsSync(tasksPath)) {
+				// Tasks file was deleted after cache was populated — do not serve stale cache.
+				manifestCache.delete(stateRoot);
+				return undefined;
 		} else {
-			return { manifest: cached.manifest, tasks: cached.tasks };
+			return { manifest: cached.manifest, tasks: cached.tasks ?? [] };
 		}
 	}
-	const manifest = await readJsonFileAsync<TeamRunManifest>(manifestPath);
+
+	// FIX: Sentinel-based retry loop to close TOCTOU window between stat and read.
+	// Matches the pattern used in the sync loadRunManifestById.
+	// Issue 3 fix: Made retry limit configurable instead of hardcoded 3.
+	let manifest: TeamRunManifest | undefined;
+	let tasks: TeamTaskState[] | undefined;
+	let attempts = 0;
+	while (attempts < LOAD_MANIFEST_RETRY_LIMIT) {
+		const freshStat = await fs.promises.stat(manifestPath);
+		manifest = await readJsonFileAsync<TeamRunManifest>(manifestPath);
+		const freshTasksStat = await fs.promises.stat(tasksPath).catch(() => undefined);
+		tasks = (await readJsonFileAsync<TeamTaskState[]>(tasksPath)) ?? [];
+		// If size/mtime didn't change between stat and read, we're consistent.
+		if (freshStat.mtimeMs === manifestStat.mtimeMs && freshStat.size === manifestStat.size
+			&& (!freshTasksStat || (freshTasksStat.mtimeMs === tasksStat?.mtimeMs && freshTasksStat.size === tasksStat?.size))) {
+			break;
+		}
+		attempts += 1;
+		manifestStat = freshStat;
+		tasksStat = freshTasksStat;
+	}
+	// WARNING: Best-effort consistency only — retry loop detected mtime/size
+	// instability. A concurrent writer can still complete a full write cycle
+	// between the final stat and the read. Callers needing strict consistency
+	// MUST use withRunLock() around load+modify+save.
+	if (attempts > 0) {
+		console.warn(`[state-store] loadRunManifestByIdAsync: retry loop detected instability for run ${runId} after ${attempts} attempt(s) — best-effort only, use withRunLock() for strict consistency`);
+	}
+	// NOTE: manifest mtime may legitimately be >= tasks mtime because
+	// saveManifestAndTasksAtomicSync writes manifest before tasks. However,
+	// if a crash occurs AFTER tasks is written but BEFORE manifest is written,
+	// tasks mtime would be > manifest mtime (the opposite). The retry loop
+	// above detects this crash state by re-reading until mtime/size are stable
+	// — the final stable state is what gets used. Because the retry loop
+	// handles the crash case, we do NOT fail based on this comparison alone.
+	// It does not indicate corruption on its own.
+
 	if (!manifest || !validateRunManifestPaths(cwd, runId, manifest, stateRoot, tasksPath)) return undefined;
-	const tasks = await readJsonFileAsync<TeamTaskState[]>(tasksPath) ?? [];
-	setManifestCache(stateRoot, { manifest, tasks, manifestMtimeMs: manifestStat.mtimeMs, manifestSize: manifestStat.size, tasksMtimeMs, tasksSize: tasksStat?.size ?? 0 });
-	return { manifest, tasks };
+	setManifestCache(stateRoot, { manifest, tasks: tasks ?? [], manifestMtimeMs: manifestStat.mtimeMs, manifestSize: manifestStat.size, tasksMtimeMs, tasksSize: tasksStat?.size ?? 0 });
+	return { manifest, tasks: tasks ?? [] };
 }

package/src/state/task-claims.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { randomUUID, timingSafeEqual } from "node:crypto";
 import type { TeamTaskState } from "./types.ts";
 
 export interface TaskClaimState {
@@ -18,8 +18,15 @@ export function isTaskClaimExpired(claim: TaskClaimState | undefined, now = new
 	return Number.isFinite(parsed) ? parsed <= now.getTime() : true;
 }
 
+export function timingSafeTokenMatch(a: string, b: string): boolean {
+	const bufA = Buffer.from(String(a));
+	const bufB = Buffer.from(String(b));
+	if (bufA.length !== bufB.length) return false;
+	return timingSafeEqual(bufA, bufB);
+}
+
 export function canUseTaskClaim(task: Pick<TeamTaskState, "claim">, owner: string, token: string, now = new Date()): boolean {
-	return task.claim?.owner === owner && task.claim.token === token && !isTaskClaimExpired(task.claim, now);
+	return task.claim?.owner === owner && timingSafeTokenMatch(task.claim.token, token) && !isTaskClaimExpired(task.claim, now);
 }
 
 export function claimTask<T extends TeamTaskState>(task: T, owner: string, leaseMs?: number, now = new Date()): T {