pi-crew 0.6.1 → 0.6.3

package/src/state/blob-store.ts

@@ -1,18 +1,57 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { resolveRealContainedPath } from "../utils/safe-paths.ts";
+import { atomicWriteFile, renameWithRetry } from "./atomic-write.ts";
+import { withFileLockSync } from "./locks.ts";
+import { sleepSync } from "../utils/sleep.ts";
 
 const SHA256_HEX = /^[a-f0-9]{64}$/i;
 
-function validateBlobHash(hash: string): void {
+function validateBlobHash(hash: string, algorithm?: string): void {
 	if (!SHA256_HEX.test(hash)) throw new Error(`Invalid blob hash: ${hash}`);
+	if (algorithm !== undefined && algorithm !== SHA256_PREFIX) {
+		throw new Error(`Invalid blob algorithm: ${algorithm} (expected ${SHA256_PREFIX})`);
+	}
+}
+
+/**
+ * Atomically write a Buffer to a file using temp-file + rename pattern.
+ * Prevents partial writes on crash and handles concurrent writes safely.
+ */
+function atomicWriteBuffer(filePath: string, content: Buffer): void {
+	fs.mkdirSync(path.dirname(filePath), { recursive: true });
+	const tempPath = `${filePath}.${randomUUID()}.tmp`;
+	const O_NOFOLLOW = typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0;
+	const fd = fs.openSync(tempPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | O_NOFOLLOW, 0o600);
+	try {
+		const openedStat = fs.fstatSync(fd);
+		if (!openedStat.isFile()) {
+			throw new Error(`Refusing to write: opened path is not a regular file: ${tempPath}`);
+		}
+		fs.writeSync(fd, content, 0, content.length);
+		renameWithRetry(tempPath, filePath);
+	} catch (error) {
+		try { fs.rmSync(tempPath, { force: true }); } catch { /* best-effort */ }
+		throw error;
+	} finally {
+		// Always close fd; closeSync is safe to call even if fd was already closed
+		try { fs.closeSync(fd); } catch { /* best-effort */ }
+	}
 }
 
 const BLOBS_DIR = "blobs";
 const BLOB_META_DIR = "blob-metadata";
 const SHA256_PREFIX = "sha256";
 
+/**
+ * File-based lock for cross-process metadata write protection.
+ * Uses withFileLockSync which provides O_EXCL atomic create + token-guarded release.
+ */
+function withMetadataLock<T>(metadataPath: string, fn: () => T): T {
+	return withFileLockSync(metadataPath, fn);
+}
+
 export interface BlobMetadata {
 	blobHash: string;
 	blobAlgorithm: string;
@@ -42,6 +81,9 @@ function sha256Of(content: string | Buffer): string {
 /**
  * Write content-addressed blob to the blobs directory under artifactsRoot.
  * Content is deduplicated by hash; metadata sidecar is always written.
+ * FIX: Both content and metadata writes now use atomicWriteFile to prevent
+ * partial writes on crash. The deduplication check is now advisory (the atomic
+ * write handles concurrent writes correctly via O_EXCL temp file pattern).
  */
 export function writeBlob(artifactsRoot: string, input: {
 	content: string | Buffer;
@@ -58,14 +100,19 @@ export function writeBlob(artifactsRoot: string, input: {
 	const algorithm = SHA256_PREFIX;
 	const blobDir = path.join(artifactsRoot, BLOBS_DIR, algorithm);
 	const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
+	// Issue 2 fix: Validate parent directories are not symlinks before mkdirSync.
+	// An attacker could plant a symlink before we create the directory, causing
+	// blob content to be written to an attacker-controlled location.
+	resolveRealContainedPath(artifactsRoot, blobDir);
+	resolveRealContainedPath(artifactsRoot, metaDir);
 	fs.mkdirSync(blobDir, { recursive: true });
 	fs.mkdirSync(metaDir, { recursive: true });
 
 	const blobPath = path.join(blobDir, hash);
-	if (!fs.existsSync(blobPath)) {
-		fs.writeFileSync(blobPath, content, typeof input.content === "string" ? "utf-8" : undefined);
-	}
-
+	// Issue 6 fix: Validate blobPath BEFORE writing to ensure the path is safe.
+	// If resolveRealContainedPath throws after the write, the blob may already
+	// exist at an unexpected location. Validate first to prevent this.
+	resolveRealContainedPath(artifactsRoot, blobPath);
 	const metadata: BlobMetadata = {
 		blobHash: hash,
 		blobAlgorithm: algorithm,
@@ -79,11 +126,90 @@ export function writeBlob(artifactsRoot: string, input: {
 		retention: input.retention ?? "run",
 		createdAt: new Date().toISOString(),
 	};
-
 	const metadataPath = path.join(metaDir, `${hash}.json`);
-	fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2), "utf-8");
 
-	return { hash, algorithm, blobPath: resolveRealContainedPath(artifactsRoot, blobPath), metadataPath: resolveRealContainedPath(artifactsRoot, metadataPath), sizeBytes: metadata.sizeBytes };
+	// Both content and metadata use atomic writes to prevent partial writes on crash.
+	// Content is immutable and content-addressed (same hash = same content), so
+	// concurrent writes to the same hash are safe. Metadata (mime, retention, etc.)
+	// is protected by a per-hash lock to make the check-then-write atomic, preventing
+	// concurrent writers from racing to write different metadata for the same hash.
+	// Issue 3 fix: wrap metadata check-and-write in a lock to make it atomic.
+	// Without the lock, two processes could read the same metadata concurrently,
+	// both pass the check, and the second write would silently overwrite the first.
+	// FIX: Write blob content FIRST, then metadata. If blob write fails, no orphan metadata.
+	// Previous order was metadata first, then blob - causing orphan metadata on blob failure.
+	let blobContentWritten = false;
+	try {
+		// Write blob content first (immutable, content-addressed)
+		atomicWriteBuffer(blobPath, Buffer.isBuffer(content) ? content : Buffer.from(content, "utf-8"));
+		blobContentWritten = true;
+	} catch (error) {
+		// Blob write failed - no metadata to clean up since we haven't written it yet
+		throw error;
+	}
+
+	// Metadata only after blob content is successfully written
+	// Issue 3 fix: Use two-phase commit for metadata - write to temp file first,
+	// then atomically rename to final location. This prevents orphan blobs if
+	// the process crashes after blob write but before metadata rename.
+	let metadataWritten = false;
+	try {
+		withMetadataLock(metadataPath, () => {
+			try {
+				const existingMeta = JSON.parse(fs.readFileSync(metadataPath, "utf-8")) as BlobMetadata;
+				// Compare fields that indicate concurrent write with different metadata.
+				// Note: taskId is intentionally NOT part of the conflict check because
+				// it represents which task produced the blob and may legitimately differ
+				// across processes (last-write-wins for taskId). If concurrent writes
+				// have identical mime/retention/producer/originalPath but different
+				// taskId, the conflict check passes and both writes succeed — the
+				// final taskId reflects whichever write completed last.
+				// Callers should not rely on taskId for correctness-critical decisions.
+				if (existingMeta.mime !== metadata.mime ||
+					existingMeta.retention !== metadata.retention ||
+					existingMeta.producer !== metadata.producer ||
+					existingMeta.originalPath !== metadata.originalPath) {
+					throw new Error(`Concurrent metadata write conflict for blob ${hash}: different metadata values detected. Existing: ${JSON.stringify(existingMeta)}, New: ${JSON.stringify(metadata)}`);
+				}
+			} catch (err) {
+				if (err instanceof Error && err.message.includes("ENOENT")) {
+					// OK - metadata doesn't exist yet, proceed to write
+				} else {
+					// Re-throw non-ENOENT errors (e.g., permission denied, corrupt file)
+					throw err;
+				}
+			}
+			// Issue 3 fix: Two-phase commit for metadata
+			// Write to temp file first, then atomically rename to final location.
+			// Issue 1 fix: Use atomicWriteFile instead of plain writeSync for the temp file
+			// to prevent partial writes on crash. atomicWriteFile uses O_EXCL + write + rename
+			// pattern which is consistent with how blob content is written.
+			const tempMetaPath = `${metadataPath}.${randomUUID()}.tmp`;
+			atomicWriteFile(tempMetaPath, JSON.stringify(metadata, null, 2));
+			renameWithRetry(tempMetaPath, metadataPath);
+			metadataWritten = true;
+		});
+	} catch (error) {
+		// Issue 4 fix: Clean up orphaned blob if metadata write fails.
+		// If metadata write fails (e.g., concurrent conflict), the blob content
+		// is orphaned since no metadata references it. Clean it up to reclaim space.
+		// Issue 8 fix: Do NOT delete blob content on metadata failure.
+		// If metadata write fails due to concurrent conflict (different values),
+		// the blob content is still valid. Another process has written metadata
+		// referencing this blob - deleting the blob would orphan their metadata.
+		// The caller can retry the metadata write if needed.
+		// However, if metadata was never written (metadataWritten === false),
+		// the blob is orphaned and should be cleaned up.
+		if (!blobContentWritten) {
+			try { fs.rmSync(blobPath, { force: true }); } catch { /* best-effort */ }
+		}
+		throw error;
+	}
+
+	// Issue 2 fix: resolve paths before writes and return cached values
+	const resolvedBlobPath = resolveRealContainedPath(artifactsRoot, blobPath);
+	const resolvedMetadataPath = resolveRealContainedPath(artifactsRoot, metadataPath);
+	return { hash, algorithm, blobPath: resolvedBlobPath, metadataPath: resolvedMetadataPath, sizeBytes: metadata.sizeBytes };
 }
 
 /**
@@ -114,4 +240,46 @@ export function readBlobMetadata(artifactsRoot: string, hash: string): BlobMetad
 	} catch {
 		return undefined;
 	}
-}
+}
+/**
+ * Cleanup orphaned blobs - blobs without corresponding metadata entries.
+ * Called periodically to reclaim storage from blobs that were written but whose
+ * metadata write failed (e.g., crash between content and metadata write).
+ * Returns the number of orphaned blobs cleaned up.
+ */
+export function cleanupOrphanedBlobs(artifactsRoot: string): number {
+	const blobDir = path.join(artifactsRoot, BLOBS_DIR, SHA256_PREFIX);
+	const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
+
+	let cleaned = 0;
+	try {
+		const blobFiles = fs.readdirSync(blobDir);
+		for (const blobFile of blobFiles) {
+			// Skip non-hash files (e.g., temp files from atomicWriteBuffer)
+			if (!SHA256_HEX.test(blobFile)) continue;
+
+			const metaPath = path.join(metaDir, `${blobFile}.json`);
+			// Issue 4 fix: Use lock to prevent race between stat and delete.
+			// A concurrent process could write metadata between stat and delete,
+			// causing valid blob to be deleted while metadata remains.
+			withMetadataLock(metaPath, () => {
+				try {
+					fs.statSync(metaPath);
+					// Metadata exists - blob is not orphaned
+				} catch {
+					// Metadata does not exist - blob is orphaned, delete it
+					const blobPath = path.join(blobDir, blobFile);
+					try {
+						fs.rmSync(blobPath, { force: true });
+						cleaned++;
+					} catch {
+						// Best-effort cleanup - continue to next blob
+					}
+				}
+			});
+		}
+	} catch {
+		// Blobs directory doesn't exist or inaccessible - nothing to clean
+	}
+	return cleaned;
+}

package/src/state/crew-init.ts

@@ -3,10 +3,17 @@
  * Called on first team run in a workspace to ensure all required
  * directories and files exist.
  *
- * IMPORTANT: This module must be COMPLETELY self-contained with NO dependencies
- * on other pi-crew modules (especially paths.ts). It is called via dynamic
- * import from child-process contexts (background runners, subagents) where
- * module binding can fail. Keep this file minimal and self-contained.
+ * IMPORTANT: This module is dynamically `import()`'d from concurrent child
+ * Pi subprocesses (3+ parallel subagents). Under load, the `path` namespace
+ * binding can intermittently arrive as `undefined` in jiti's ESM/CJS interop
+ * layer. We therefore use the inline helpers `parseRoot`, `safeJoin`,
+ * `safeDirname`, and `safeResolve` so that critical path operations do not
+ * depend on the `path` namespace binding.
+ *
+ * The `node:path` import is retained as a *fallback* (only used when the
+ * binding is healthy). Don't add new dependencies on other pi-crew modules.
+ *
+ * See: https://github.com/baphuongna/pi-crew/issues/28
  */
 import * as fs from "node:fs";
 import * as path from "node:path";
@@ -49,25 +56,139 @@ team action='cache' action='clear'
  * Inline implementation to avoid module dependency on paths.ts.
  * Matches the logic in src/utils/paths.ts:computeRepoRoot().
  */
-function findProjectRoot(start: string): string | undefined {
+/**
+ * Detect filesystem root for `start` without relying on `path.parse()`.
+ *
+ * **Why this exists**: This module is dynamically `import()`'d from concurrent
+ * child Pi subprocesses (3+ parallel subagents). Under load, the `path` namespace
+ * binding can intermittently arrive as `undefined` in jiti's ESM/CJS interop layer,
+ * crashing `findProjectRoot` with `TypeError: Cannot read properties of undefined
+ * (reading 'parse')` — see https://github.com/baphuongna/pi-crew/issues/28.
+ *
+ * Inlining `parse` for the termination root eliminates the dependency on the
+ * `path` binding for that critical call path.
+ */
+function parseRoot(start: string): string {
+	if (!start) return "/";
+	if (start[0] === "/") return "/";
+	// Windows: "C:\\" or "C:/" -> "C:\\"
+	if (/^[A-Za-z]:[\\/]/.test(start)) return start.slice(0, 3);
+	// UNC: "\\\\server\\share" — find the second path separator.
+	if (start.startsWith("\\\\") || start.startsWith("//")) {
+		const rest = start.slice(2);
+		const firstSep = Math.max(rest.indexOf("\\"), rest.indexOf("/"));
+		if (firstSep === -1) return start;
+		const secondSep = Math.max(
+			rest.indexOf("\\", firstSep + 1),
+			rest.indexOf("/", firstSep + 1),
+		);
+		if (secondSep === -1) return start;
+		// secondSep is an index into `rest`; add 2 to map back to `start`.
+		return start.slice(0, 2 + secondSep);
+	}
+	// Relative path — no fixed root, use start itself as terminator.
+	return start;
+}
+
+/**
+ * Defensive wrappers around `path` for use in dynamic-import contexts.
+ *
+ * **Why these exist**: This module is dynamically `import()`'d from concurrent
+ * child Pi subprocesses (3+ parallel subagents). Under load, the `path` namespace
+ * binding can intermittently arrive as `undefined` in jiti's ESM/CJS interop layer,
+ * crashing `findProjectRoot` with `TypeError: Cannot read properties of undefined
+ * (reading 'parse')` — see https://github.com/baphuongna/pi-crew/issues/28.
+ *
+ * Each helper checks that the corresponding `path` function exists before
+ * calling it, falling back to an inline implementation. This keeps the file
+ * self-contained even if the namespace binding is missing.
+ */
+function safeJoin(...parts: string[]): string {
+	// Cross-platform join — picks the separator based on the parts.
+	// Don't delegate to `path.join` because POSIX/Windows disagree on which
+	// separator is the path separator, and the dynamic-import context (issue
+	// #28) may have a partially-initialized `path` namespace.
+	const filtered = parts.filter(Boolean);
+	if (filtered.length === 0) return "";
+	const sep = filtered.some((p) => p.includes("\\")) ? "\\" : "/";
+	// Detect if the first part begins with a leading separator (or UNC "\\\\")
+	// so we can preserve it. F-8: collapses runs of the separator everywhere
+	// (including the body), but re-prepends the leading separator that the
+	// collapse regex would otherwise eat.
+	const firstPart = filtered[0];
+	let leading = "";
+	if (sep === "\\") {
+		if (firstPart.startsWith("\\\\")) leading = "\\\\";
+		else if (firstPart.startsWith("\\")) leading = "\\";
+	} else if (firstPart.startsWith("/")) {
+		leading = "/";
+	}
+	// Strip the leading separator(s) from the first part before joining, so
+	// the collapse regex doesn't re-collapse them.
+	const firstPartStripped =
+		sep === "\\"
+			? firstPart.replace(/^\\{1,2}/, "")
+			: firstPart.replace(/^\/+/, "");
+	const rest = filtered.slice(1);
+	const joined = [firstPartStripped, ...rest].filter(Boolean).join(sep);
+	// Collapse internal runs of the separator.
+	const collapsed = joined.replace(
+		new RegExp(`${sep === "\\" ? "\\\\" : "/"}{2,}`, "g"),
+		sep,
+	);
+	return leading + collapsed;
+}
+
+function safeDirname(p: string): string {
+	// Cross-platform dirname — handles BOTH `/` and `\` separators.
+	// Note: we don't delegate to `path.dirname` here because on POSIX it treats
+	// backslashes as part of a filename, and on Windows it treats forward
+	// slashes the same way. The dynamic-import context (issue #28) may also
+	// have a partially-initialized `path` namespace. Using a unified inline
+	// implementation ensures consistent behavior across all platforms.
+	const idx = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\"));
+	if (idx === -1) return p; // No separator at all
+	if (idx === 0) return p[0] === "/" || p[0] === "\\" ? p[0] : p; // Root: "/" or "\"
+	// Preserve drive letter roots like "C:\"
+	if (idx === 2 && /[A-Za-z]:/.test(p.slice(0, 2))) return p.slice(0, 3);
+	return p.slice(0, idx);
+}
+
+function safeResolve(p: string, pathDep?: typeof path): string {
+	const dep = pathDep ?? path;
+	if (dep && typeof dep.resolve === "function") return dep.resolve(p);
+	return p;
+}
+
+function findProjectRoot(
+	start: string,
+	pathDep?: typeof path,
+): string | undefined {
 	const dirMarkers = [".git", ".hg", ".svn"];
-	const fileMarkers = ["package.json", "pyproject.toml", "Cargo.toml", "go.mod"];
-	const root = path.parse(start).root;
-	let current = path.resolve(start);
+	const fileMarkers = [
+		"package.json",
+		"pyproject.toml",
+		"Cargo.toml",
+		"go.mod",
+	];
+	// Use `parseRoot` (inlined above) to avoid `path.parse` for the critical
+	// termination root — fixes the jiti namespace race in issue #28.
+	const root = parseRoot(start);
+	let current = safeResolve(start, pathDep);
 	// Walk up to find project root
 	while (current !== root) {
 		for (const marker of dirMarkers) {
-			if (fs.existsSync(path.join(current, marker))) return current;
+			if (fs.existsSync(safeJoin(current, marker))) return current;
 		}
 		for (const marker of fileMarkers) {
-			if (fs.existsSync(path.join(current, marker))) return current;
+			if (fs.existsSync(safeJoin(current, marker))) return current;
 		}
-		const parent = path.dirname(current);
+		const parent = safeDirname(current);
 		if (parent === current) break;
 		current = parent;
 	}
 	// Check root as fallback
-	if (dirMarkers.some((m) => fs.existsSync(path.join(root, m)))) return root;
+	if (dirMarkers.some((m) => fs.existsSync(safeJoin(root, m)))) return root;
 	return undefined;
 }
 
@@ -77,12 +198,12 @@ function findProjectRoot(start: string): string | undefined {
  */
 function computeCrewRoot(cwd: string): string {
 	const repoRoot = findProjectRoot(cwd) ?? cwd;
-	const crewDir = path.join(repoRoot, ".crew");
+	const crewDir = safeJoin(repoRoot, ".crew");
 	// Keep existing .crew/ stable even when .pi/ exists for project config.
 	if (fs.existsSync(crewDir)) return crewDir;
 	// Legacy reuse: if .pi/ already exists, namespace under .pi/teams/
-	const piDir = path.join(repoRoot, ".pi");
-	return fs.existsSync(piDir) ? path.join(piDir, "teams") : crewDir;
+	const piDir = safeJoin(repoRoot, ".pi");
+	return fs.existsSync(piDir) ? safeJoin(piDir, "teams") : crewDir;
 }
 
 /**
@@ -99,12 +220,12 @@ export async function ensureCrewDirectory(cwd: string): Promise<void> {
 	// 1. Create directory structure
 	const dirs = [
 		crewRoot,
-		path.join(crewRoot, "state", "runs"),
-		path.join(crewRoot, "state", "subagents"),
-		path.join(crewRoot, "artifacts"),
-		path.join(crewRoot, "cache"),
-		path.join(crewRoot, "graphs"),
-		path.join(crewRoot, "audit"),
+		safeJoin(crewRoot, "state", "runs"),
+		safeJoin(crewRoot, "state", "subagents"),
+		safeJoin(crewRoot, "artifacts"),
+		safeJoin(crewRoot, "cache"),
+		safeJoin(crewRoot, "graphs"),
+		safeJoin(crewRoot, "audit"),
 	];
 
 	for (const dir of dirs) {
@@ -115,10 +236,10 @@ export async function ensureCrewDirectory(cwd: string): Promise<void> {
 
 	// 2. Create .gitkeep placeholders in directories that should be tracked
 	const placeholders = [
-		path.join(crewRoot, "artifacts", ".gitkeep"),
-		path.join(crewRoot, "cache", ".gitkeep"),
-		path.join(crewRoot, "graphs", ".gitkeep"),
-		path.join(crewRoot, "audit", ".gitkeep"),
+		safeJoin(crewRoot, "artifacts", ".gitkeep"),
+		safeJoin(crewRoot, "cache", ".gitkeep"),
+		safeJoin(crewRoot, "graphs", ".gitkeep"),
+		safeJoin(crewRoot, "audit", ".gitkeep"),
 	];
 
 	for (const placeholder of placeholders) {
@@ -128,12 +249,24 @@ export async function ensureCrewDirectory(cwd: string): Promise<void> {
 	}
 
 	// 3. Write README.md (always overwrite to keep it current)
-	fs.writeFileSync(path.join(crewRoot, "README.md"), CREW_README, "utf-8");
+	fs.writeFileSync(safeJoin(crewRoot, "README.md"), CREW_README, "utf-8");
 
 	// 4. Update .gitignore at project root
 	const repoRoot = findProjectRoot(cwd);
 	if (repoRoot) {
-		const gitignorePath = path.join(repoRoot, ".gitignore");
+		const gitignorePath = safeJoin(repoRoot, ".gitignore");
 		await updateGitignore(gitignorePath);
 	}
-}
+}
+
+// Exported only for regression tests of issue #28.
+// NOT part of the public API — the `__test__` prefix follows the project
+// convention used in atomic-write.ts, state-store.ts, team-runner.ts, etc.
+// See F-4 in the post-fix review for the convention rationale.
+export const __test__internals = {
+	parseRoot,
+	safeJoin,
+	safeDirname,
+	safeResolve,
+	findProjectRoot,
+};