pi-crew 0.9.42 → 0.9.44

package/src/state/locks.ts

@@ -1,7 +1,9 @@
+import { AsyncLocalStorage } from "node:async_hooks";
 import { randomUUID, timingSafeEqual } from "node:crypto";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { DEFAULT_LOCKS } from "../config/defaults.ts";
+import { logInternalError } from "../utils/internal-error.ts";
 import { sleepSync } from "../utils/sleep.ts";
 import { isSymlinkSafePath } from "./atomic-write.ts";
 import type { TeamRunManifest } from "./types.ts";
@@ -103,41 +105,56 @@ function isLockHolderAlive(filePath: string): boolean {
  *
  * See also: SECURITY-ISSUES.md SEC-008 for documented acceptance.
  */
-function readLockSnapshot(filePath: string, staleMs: number): { canSteal: boolean } {
+function readLockSnapshot(filePath: string, staleMs: number, options?: { treatOwnPidAsStealable?: boolean }): { canSteal: boolean } {
+	const treatOwnPidAsStealable = options?.treatOwnPidAsStealable === true;
 	let stat: fs.Stats | undefined;
 	let raw: string | undefined;
 	try {
 		stat = fs.statSync(filePath);
 		raw = fs.readFileSync(filePath, "utf-8");
-	} catch {
-		// File vanished between writeLockFile's EEXIST and now (holder released).
-		// Loop will retry the create; safe to signal "nothing to steal".
+	} catch (error) {
+		// FIX (CI flake): when the lock file vanishes between writeLockFile's
+		// EEXIST and our read here, the holder has released — return canSteal:true
+		// so the acquire loop retries the create instead of throwing a false
+		// "locked" error.
+		const code = (error as NodeJS.ErrnoException).code;
+		if (code === "ENOENT") {
+			return { canSteal: true };
+		}
+		// Transient I/O error — be conservative (don't steal, retry the read on
+		// next attempt).
 		return { canSteal: false };
 	}
 	// Staleness from a single snapshot.
 	let createdAt = parseCreatedAtFromLock(raw);
 	if (createdAt === undefined) createdAt = stat.mtimeMs;
 	const isStale = Date.now() - createdAt > staleMs;
-	// Holder liveness from the SAME snapshot.
-	let isAlive = true; // Unknown holder — assume alive to be safe (matches isLockHolderAlive).
+	let holderPid: number | undefined;
+	let isAlive = true;
 	try {
 		const parsed = JSON.parse(raw) as { pid?: unknown };
-		const pid = typeof parsed.pid === "number" ? parsed.pid : undefined;
-		if (pid !== undefined) {
-			try {
-				process.kill(pid, 0);
-				isAlive = true;
-			} catch (error) {
-				const code = (error as NodeJS.ErrnoException).code;
-				// EPERM/ESRCH → treat as not-alive (stealable), see isLockHolderAlive.
-				isAlive = false;
-			}
-		}
+		holderPid = typeof parsed.pid === "number" ? parsed.pid : undefined;
 	} catch {
 		/* malformed payload — keep isAlive=true */
 	}
-	// Steal if stale OR holder dead — matches the original intent.
-	return { canSteal: isStale || !isAlive };
+	if (holderPid !== undefined) {
+		try {
+			process.kill(holderPid, 0);
+			isAlive = true;
+		} catch (error) {
+			const code = (error as NodeJS.ErrnoException).code;
+			// EPERM/ESRCH → treat as not-alive (stealable), see isLockHolderAlive.
+			isAlive = false;
+		}
+	}
+	// Steal if stale OR holder dead. Optionally steal if holder is our own pid
+	// (used by withRunLock* which is single-process — a fresh lock with our pid
+	// between acquisitions is just a leftover from a previous call that
+	// releaseOwnLock didn't get to delete yet; safe to steal).
+	const isOurOwnHolder = holderPid === process.pid;
+	return {
+		canSteal: isStale || !isAlive || (treatOwnPidAsStealable && isOurOwnHolder),
+	};
 }
 
 /**
@@ -221,6 +238,34 @@ function timingSafeTokenMatch(a: string, b: string): boolean {
 	return timingSafeEqual(bufA, bufB);
 }
 
+/**
+ * Release the lock we (this process) just acquired. Unlike releaseLock, this
+ * unconditionally removes the file because we know we created it in the same
+ * withRunLock* call — token matching is only needed for multi-process scenarios
+ * (where a different holder's token must not be deleted). Within the same process
+ * the new acquire uses a fresh randomUUID token, so token matching would falsely
+ * fail and leak the file across releases.
+ *
+ * Symlink guard is preserved: if a symlink appeared since our writeLockFile, we
+ * don't rm it (defense against attacker-planted symlinks).
+ */
+export function releaseOwnLock(filePath: string, _token: string): void {
+	try {
+		const stat = fs.lstatSync(filePath);
+		if (stat.isSymbolicLink()) return;
+	} catch {
+		/* ENOENT is fine */
+	}
+	try {
+		fs.rmSync(filePath, { force: true });
+	} catch (error) {
+		const code = (error as NodeJS.ErrnoException).code;
+		if (code !== "ENOENT") {
+			logInternalError("lock-release-own", error, filePath, "warn");
+		}
+	}
+}
+
 function releaseLock(filePath: string, token: string): void {
 	// FIX: Do not delete a symlink — it may have been planted by an attacker
 	// after our lock was released. A legitimate lock file should never be a
@@ -265,7 +310,9 @@ function acquireLockWithRetry(filePath: string, staleMs: number, kind: LockKind
 			}
 			// Round 26 (BUG 1): single-snapshot read closes the TOCTOU window between
 			// separate stale + alive reads (which could race stale→fresh).
-			const { canSteal } = readLockSnapshot(filePath, staleMs);
+			// treatOwnPidAsStealable=false (this is the sync file-lock path used
+			// by withFileLockSync — never steal a fresh lock even if it has our pid).
+			const { canSteal } = readLockSnapshot(filePath, staleMs, { treatOwnPidAsStealable: false });
 			if (!canSteal) {
 				throw new Error(`Run '${path.basename(filePath)}' is locked by another operation.`);
 			}
@@ -300,7 +347,13 @@ async function acquireLockWithRetryAsync(filePath: string, staleMs: number, kind
 				throw new Error(`Run '${path.basename(filePath)}' is locked by another operation.`);
 			}
 			// Round 26 (BUG 1): single-snapshot read (see sync variant).
-			const { canSteal } = readLockSnapshot(filePath, staleMs);
+			// FIX (CI flake): treatOwnPidAsStealable=true for the run-lock async path
+			// (withRunLock). Between acquisitions in parallel-research scaffold mode,
+			// the previous releaseOwnLock leaves a microsecond window where the
+			// file still exists with our own pid. Stealing it avoids spurious 'locked'
+			// errors. The sync file-lock path (withFileLockSync) above uses false to
+			// preserve the multi-process safety guarantee.
+			const { canSteal } = readLockSnapshot(filePath, staleMs, { treatOwnPidAsStealable: true });
 			if (!canSteal) {
 				throw new Error(`Run '${path.basename(filePath)}' is locked by another operation.`);
 			}
@@ -379,48 +432,78 @@ export function withFileLockSync<T>(filePath: string, fn: () => T, options: RunL
 	}
 }
 
+// H-1: track re-entrant run-lock acquisitions PER ASYNC CONTEXT (not process-global).
+// Previously a module-global Map caused a bug: when withRunLock (async) yielded at
+// `await fn()`, a concurrent withRunLockSync for the same run — fired from a
+// different async context (e.g. a child stdout event handler) — saw the holder's
+// token and bypassed the file lock entirely, allowing two writers in the critical
+// section simultaneously. AsyncLocalStorage scopes the "held" set to the current
+// async context, so a call from a different async context no longer bypasses and
+// properly serializes against the on-disk lock. True nested calls in the SAME
+// async context (e.g. handleResume -> executeTeamRun -> executeTeamRunCore) still
+// bypass via lockCtx.getStore(), avoiding the deadlock the bypass was designed for.
+const lockCtx = new AsyncLocalStorage<Set<string>>();
+// Round 29: parallel map for withFileLockSync re-entrance. See the comment
+// at the top of withFileLockSync for the full deadlock mechanism.
+const fileLockHeldByUs = new Map<string, string>(); // lockFile -> token
+
 export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, options: RunLockOptions = {}): T {
 	const filePath = lockPath(manifest);
 	const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
-	const existingToken = runLockHeldByUs.get(filePath);
-	if (existingToken) {
-		// Re-entrant: already hold this lock, just run the callback.
+	// H-1: re-entrance check is scoped to the current async context, not process-global.
+	// A call from a DIFFERENT async context (e.g. a child-stdout event handler that
+	// runs while an async holder is awaiting) sees no held set and properly serializes.
+	if (lockCtx.getStore()?.has(filePath)) {
+		// Re-entrant within the same async context — already hold this lock.
 		return fn();
 	}
 	fs.mkdirSync(path.dirname(filePath), { recursive: true });
 	const token = acquireLockWithRetry(filePath, staleMs, "run");
-	runLockHeldByUs.set(filePath, token);
-	try {
-		return fn();
-	} finally {
-		runLockHeldByUs.delete(filePath);
-		releaseLock(filePath, token);
-	}
+	const prevHeld = lockCtx.getStore() ?? new Set<string>();
+	const newHeld = new Set(prevHeld);
+	newHeld.add(filePath);
+	// Enter a new async context (or extend the current one) carrying the held set.
+	// On exit (sync return OR async resume after an awaited child), the context is
+	// restored — so the held set only contains locks acquired within fn()'s subtree.
+	return lockCtx.run(newHeld, () => {
+		try {
+			return fn();
+		} finally {
+			// FIX (CI flake): releaseLock uses token matching to prevent the
+			// "losing contender wipes winner's lock" race in multi-process scenarios.
+			// But within withRunLockSync/withRunLock (same process), the new acquire
+			// uses a FRESH random token — the previous stored token doesn't match the
+			// current token — and the lock file is never removed. Repeat acquisitions
+			// then keep failing with EEXIST because the file lingers.
+			// Since withRunLock* is always single-process, unconditionally delete
+			// (symlink guard is still applied for safety).
+			releaseOwnLock(filePath, token);
+		}
+	});
 }
 
-// Track re-entrant lock acquisitions per lock file path. When a lock is
-// already held by this call stack (handleResume -> executeTeamRun ->
-// executeTeamRunCore), we skip re-acquisition to avoid deadlock.
-const runLockHeldByUs = new Map<string, string>(); // filePath -> token
-// Round 29: parallel map for withFileLockSync re-entrance. See the comment
-// at the top of withFileLockSync for the full deadlock mechanism.
-const fileLockHeldByUs = new Map<string, string>(); // lockFile -> token
-
 export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promise<T>, options: RunLockOptions = {}): Promise<T> {
 	const filePath = lockPath(manifest);
 	const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
-	const existingToken = runLockHeldByUs.get(filePath);
-	if (existingToken) {
-		// Re-entrant: already hold this lock, just run the callback.
+	if (lockCtx.getStore()?.has(filePath)) {
+		// Re-entrant within the same async context.
 		return await fn();
 	}
 	fs.mkdirSync(path.dirname(filePath), { recursive: true });
 	const token = await acquireLockWithRetryAsync(filePath, staleMs, "run");
-	runLockHeldByUs.set(filePath, token);
-	try {
-		return await fn();
-	} finally {
-		runLockHeldByUs.delete(filePath);
-		releaseLock(filePath, token);
-	}
+	const prevHeld = lockCtx.getStore() ?? new Set<string>();
+	const newHeld = new Set(prevHeld);
+	newHeld.add(filePath);
+	// AsyncLocalStorage propagates across await boundaries, so any code that
+	// resumes after an await inside fn() (including microtasks, timers, and
+	// awaited child Pi results) still sees newHeld.
+	return await lockCtx.run(newHeld, async () => {
+		try {
+			return await fn();
+		} finally {
+			// FIX (CI flake): see withRunLockSync above — use releaseOwnLock for
+			// unconditional deletion of the lock file we created (same process).
+			releaseOwnLock(filePath, token);
+		}
+	});
 }

package/src/state/worker-atomic-writer.ts

@@ -137,6 +137,7 @@ function getWorker(): Worker {
 		// Reject ALL pending requests — worker died.
 		for (const [, entry] of pending) entry.reject(error);
 		pending.clear();
+		worker = undefined;
 	});
 	worker.unref(); // don't keep event loop alive (unless tests request it)
 	if (keepRefForTests) worker.ref();

package/src/utils/paths.ts

@@ -44,7 +44,7 @@ export function userPiRoot(): string {
 	// "undefined/.pi/agent" relative to cwd and silently create a junk "undefined/"
 	// tree. Treat the literal "undefined" (and empty) as unset and fall back to
 	// os.homedir().
-	const rawHome = process.env.PI_TEAMS_HOME?.trim();
+	const rawHome = (process.env.PI_TEAMS_HOME ?? process.env.PI_CREW_HOME)?.trim();
 	const home = rawHome && rawHome !== "undefined" ? rawHome : os.homedir();
 	const resolved = path.join(home, ".pi", "agent");
 

package/src/worktree/worktree-manager.ts

@@ -134,7 +134,11 @@ export function findGitRoot(cwd: string): string {
 }
 
 export function assertCleanLeader(repoRoot: string): void {
-	const status = git(repoRoot, ["status", "--porcelain"]);
+	// H-7 follow-up: --untracked-files=no so pi-crew's own auto-created .gitignore
+	// (and any other untracked files the user hasn't staged) doesn't block worktree mode.
+	// The worktree contract is "no tracked changes" — untracked files are safe since
+	// they're either in .gitignore or the user can decide later.
+	const status = git(repoRoot, ["status", "--porcelain", "--untracked-files=no"]);
 	if (status.trim()) {
 		throw new Error("Worktree mode requires a clean leader repository. Commit/stash changes or use workspaceMode: 'single'.");
 	}
@@ -168,7 +172,9 @@ export function clearCleanLeaderCache(): void {
 
 export async function assertCleanLeaderAsync(repoRoot: string): Promise<void> {
 	if (_cleanLeaderCache.has(repoRoot)) return;
-	const status = await gitAsync(repoRoot, ["status", "--porcelain"]);
+	// H-7 follow-up: --untracked-files=no so pi-crew's own auto-created .gitignore
+	// (and any other untracked files the user hasn't staged) doesn't block worktree mode.
+	const status = await gitAsync(repoRoot, ["status", "--porcelain", "--untracked-files=no"]);
 	if (status.trim()) {
 		throw new Error("Worktree mode requires a clean leader repository. Commit/stash changes or use workspaceMode: 'single'.");
 	}
@@ -558,7 +564,7 @@ export function overlaySeedPaths(repoRoot: string, worktreePath: string, seedPat
  * recovered via `git apply` / manual restore. Best-effort: a snapshot failure
  * only logs (it must not block the clean-slate reuse flow).
  */
-function snapshotDirtyWorktree(manifest: TeamRunManifest, task: TeamTaskState, worktreePath: string, dirtyStatus: string): void {
+function snapshotDirtyWorktree(manifest: TeamRunManifest, task: TeamTaskState, worktreePath: string, dirtyStatus: string): boolean {
 	try {
 		const parts: string[] = [
 			`# Worktree recovery snapshot`,
@@ -596,12 +602,14 @@ function snapshotDirtyWorktree(manifest: TeamRunManifest, task: TeamTaskState, w
 			producer: "worktree-manager.snapshotDirtyWorktree",
 			retention: "run",
 		});
+		return true;
 	} catch (err) {
 		logInternalError(
 			"worktree.recovery.snapshotFailed",
 			err instanceof Error ? err : new Error(String(err)),
 			`runId=${manifest.runId}, taskId=${task.id}`,
 		);
+		return false;
 	}
 }
 
@@ -722,14 +730,24 @@ export function prepareTaskWorkspace(manifest: TeamRunManifest, task: TeamTaskSt
 		if (dirtyStatus.trim()) {
 			// Snapshot uncommitted work to a recovery artifact BEFORE discarding, so the
 			// previous run's changes are never silently destroyed on reuse.
-			snapshotDirtyWorktree(manifest, task, worktreePath, dirtyStatus);
-			logInternalError(
-				"worktree.reused.dirty",
-				new Error(`Discarding uncommitted changes in reused worktree at ${worktreePath} (snapshot saved to artifacts)`),
-				`runId=${manifest.runId}, taskId=${task.id}, dirtyStatus=${dirtyStatus.trim()}`,
-			);
-			git(worktreePath, ["checkout", "--", "."]);
-			git(worktreePath, ["clean", "-fd"]);
+			const snapshotOk = snapshotDirtyWorktree(manifest, task, worktreePath, dirtyStatus);
+			if (snapshotOk) {
+				logInternalError(
+					"worktree.reused.dirty",
+					new Error(`Discarding uncommitted changes in reused worktree at ${worktreePath} (snapshot saved to artifacts)`),
+					`runId=${manifest.runId}, taskId=${task.id}, dirtyStatus=${dirtyStatus.trim()}`,
+				);
+				git(worktreePath, ["checkout", "--", "."]);
+				git(worktreePath, ["clean", "-fd"]);
+			} else {
+				logInternalError(
+					"worktree.reused.dirtyPreserved",
+					new Error(
+						`Snapshot failed — preserving dirty worktree at ${worktreePath} (skipping git checkout/clean to prevent data loss)`,
+					),
+					`runId=${manifest.runId}, taskId=${task.id}, dirtyStatus=${dirtyStatus.trim()}`,
+				);
+			}
 		}
 		// Overlay seed paths from config + step-level seedPaths (reused worktree)
 		const globalSeedPaths = loadedConfig.config.worktree?.seedPaths ?? [];
@@ -869,14 +887,24 @@ export async function prepareTaskWorkspaceAsync(
 		}
 		const dirtyStatus = await gitAsync(worktreePath, ["status", "--porcelain"]);
 		if (dirtyStatus.trim()) {
-			snapshotDirtyWorktree(manifest, task, worktreePath, dirtyStatus);
-			logInternalError(
-				"worktree.reused.dirty",
-				new Error(`Discarding uncommitted changes in reused worktree at ${worktreePath} (snapshot saved to artifacts)`),
-				`runId=${manifest.runId}, taskId=${task.id}, dirtyStatus=${dirtyStatus.trim()}`,
-			);
-			await gitAsync(worktreePath, ["checkout", "--", "."]);
-			await gitAsync(worktreePath, ["clean", "-fd"]);
+			const snapshotOk = snapshotDirtyWorktree(manifest, task, worktreePath, dirtyStatus);
+			if (snapshotOk) {
+				logInternalError(
+					"worktree.reused.dirty",
+					new Error(`Discarding uncommitted changes in reused worktree at ${worktreePath} (snapshot saved to artifacts)`),
+					`runId=${manifest.runId}, taskId=${task.id}, dirtyStatus=${dirtyStatus.trim()}`,
+				);
+				await gitAsync(worktreePath, ["checkout", "--", "."]);
+				await gitAsync(worktreePath, ["clean", "-fd"]);
+			} else {
+				logInternalError(
+					"worktree.reused.dirtyPreserved",
+					new Error(
+						`Snapshot failed — preserving dirty worktree at ${worktreePath} (skipping git checkout/clean to prevent data loss)`,
+					),
+					`runId=${manifest.runId}, taskId=${task.id}, dirtyStatus=${dirtyStatus.trim()}`,
+				);
+			}
 		}
 		const globalSeedPaths = loadedConfig.config.worktree?.seedPaths ?? [];
 		const mergedReused = normalizeSeedPaths([...globalSeedPaths, ...(stepSeedPaths ?? [])], repoRoot);