pi-crew 0.9.53 → 0.9.55

package/src/state/blob-store.ts

@@ -0,0 +1,308 @@
+import { createHash, randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { resolveRealContainedPath } from "../utils/safe-paths.ts";
+import { atomicWriteFile, renameWithRetry } from "./atomic-write.ts";
+import { withFileLockSync } from "./locks.ts";
+
+const SHA256_HEX = /^[a-f0-9]{64}$/i;
+
+function validateBlobHash(hash: string, algorithm?: string): void {
+	if (!SHA256_HEX.test(hash)) throw new Error(`Invalid blob hash: ${hash}`);
+	if (algorithm !== undefined && algorithm !== SHA256_PREFIX) {
+		throw new Error(`Invalid blob algorithm: ${algorithm} (expected ${SHA256_PREFIX})`);
+	}
+}
+
+/**
+ * Atomically write a Buffer to a file using temp-file + rename pattern.
+ * Prevents partial writes on crash and handles concurrent writes safely.
+ */
+function atomicWriteBuffer(filePath: string, content: Buffer): void {
+	fs.mkdirSync(path.dirname(filePath), { recursive: true });
+	const tempPath = `${filePath}.${randomUUID()}.tmp`;
+	const O_NOFOLLOW = typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0;
+	const fd = fs.openSync(tempPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | O_NOFOLLOW, 0o600);
+	try {
+		const openedStat = fs.fstatSync(fd);
+		if (!openedStat.isFile()) {
+			throw new Error(`Refusing to write: opened path is not a regular file: ${tempPath}`);
+		}
+		fs.writeSync(fd, content, 0, content.length);
+		renameWithRetry(tempPath, filePath);
+	} catch (error) {
+		try {
+			fs.rmSync(tempPath, { force: true });
+		} catch {
+			/* best-effort */
+		}
+		throw error;
+	} finally {
+		// Always close fd; closeSync is safe to call even if fd was already closed
+		try {
+			fs.closeSync(fd);
+		} catch {
+			/* best-effort */
+		}
+	}
+}
+
+const BLOBS_DIR = "blobs";
+const BLOB_META_DIR = "blob-metadata";
+const SHA256_PREFIX = "sha256";
+
+/**
+ * File-based lock for cross-process metadata write protection.
+ * Uses withFileLockSync which provides O_EXCL atomic create + token-guarded release.
+ */
+function withMetadataLock<T>(metadataPath: string, fn: () => T): T {
+	return withFileLockSync(metadataPath, fn);
+}
+
+export interface BlobMetadata {
+	blobHash: string;
+	blobAlgorithm: string;
+	runId: string;
+	taskId?: string;
+	mime: string;
+	producer: string;
+	originalPath: string;
+	sizeBytes: number;
+	redacted: boolean;
+	retention: "run" | "project" | "temporary";
+	createdAt: string;
+}
+
+export interface BlobWriteResult {
+	hash: string;
+	algorithm: string;
+	blobPath: string;
+	metadataPath: string;
+	sizeBytes: number;
+}
+
+function sha256Of(content: string | Buffer): string {
+	return createHash("sha256")
+		.update(typeof content === "string" ? content : content)
+		.digest("hex");
+}
+
+/**
+ * Write content-addressed blob to the blobs directory under artifactsRoot.
+ * Content is deduplicated by hash; metadata sidecar is always written.
+ * FIX: Both content and metadata writes now use atomicWriteFile to prevent
+ * partial writes on crash. The deduplication check is now advisory (the atomic
+ * write handles concurrent writes correctly via O_EXCL temp file pattern).
+ */
+export function writeBlob(
+	artifactsRoot: string,
+	input: {
+		content: string | Buffer;
+		runId: string;
+		taskId?: string;
+		mime?: string;
+		producer: string;
+		originalPath: string;
+		redacted?: boolean;
+		retention?: BlobMetadata["retention"];
+	},
+): BlobWriteResult {
+	const content = input.content;
+	const hash = sha256Of(content);
+	const algorithm = SHA256_PREFIX;
+	const blobDir = path.join(artifactsRoot, BLOBS_DIR, algorithm);
+	const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
+	// Issue 2 fix: Validate parent directories are not symlinks before mkdirSync.
+	// An attacker could plant a symlink before we create the directory, causing
+	// blob content to be written to an attacker-controlled location.
+	resolveRealContainedPath(artifactsRoot, blobDir);
+	resolveRealContainedPath(artifactsRoot, metaDir);
+	fs.mkdirSync(blobDir, { recursive: true });
+	fs.mkdirSync(metaDir, { recursive: true });
+
+	const blobPath = path.join(blobDir, hash);
+	// Issue 6 fix: Validate blobPath BEFORE writing to ensure the path is safe.
+	// If resolveRealContainedPath throws after the write, the blob may already
+	// exist at an unexpected location. Validate first to prevent this.
+	resolveRealContainedPath(artifactsRoot, blobPath);
+	const metadata: BlobMetadata = {
+		blobHash: hash,
+		blobAlgorithm: algorithm,
+		runId: input.runId,
+		taskId: input.taskId,
+		mime: input.mime ?? "text/plain",
+		producer: input.producer,
+		originalPath: input.originalPath,
+		sizeBytes: Buffer.isBuffer(content) ? content.length : Buffer.byteLength(content, "utf-8"),
+		redacted: input.redacted ?? false,
+		retention: input.retention ?? "run",
+		createdAt: new Date().toISOString(),
+	};
+	const metadataPath = path.join(metaDir, `${hash}.json`);
+
+	// Both content and metadata use atomic writes to prevent partial writes on crash.
+	// Content is immutable and content-addressed (same hash = same content), so
+	// concurrent writes to the same hash are safe. Metadata (mime, retention, etc.)
+	// is protected by a per-hash lock to make the check-then-write atomic, preventing
+	// concurrent writers from racing to write different metadata for the same hash.
+	// Issue 3 fix: wrap metadata check-and-write in a lock to make it atomic.
+	// Without the lock, two processes could read the same metadata concurrently,
+	// both pass the check, and the second write would silently overwrite the first.
+	// FIX: Write blob content FIRST, then metadata. If blob write fails, no orphan metadata.
+	// Previous order was metadata first, then blob - causing orphan metadata on blob failure.
+	let blobContentWritten = false;
+	// Write blob content first (immutable, content-addressed)
+	atomicWriteBuffer(blobPath, Buffer.isBuffer(content) ? content : Buffer.from(content, "utf-8"));
+	blobContentWritten = true;
+
+	// Metadata only after blob content is successfully written
+	// Issue 3 fix: Use two-phase commit for metadata - write to temp file first,
+	// then atomically rename to final location. This prevents orphan blobs if
+	// the process crashes after blob write but before metadata rename.
+	let metadataWritten = false;
+	try {
+		withMetadataLock(metadataPath, () => {
+			try {
+				const existingMeta = JSON.parse(fs.readFileSync(metadataPath, "utf-8")) as BlobMetadata;
+				// Compare fields that indicate concurrent write with different metadata.
+				// Note: taskId is intentionally NOT part of the conflict check because
+				// it represents which task produced the blob and may legitimately differ
+				// across processes (last-write-wins for taskId). If concurrent writes
+				// have identical mime/retention/producer/originalPath but different
+				// taskId, the conflict check passes and both writes succeed — the
+				// final taskId reflects whichever write completed last.
+				// Callers should not rely on taskId for correctness-critical decisions.
+				if (
+					existingMeta.mime !== metadata.mime ||
+					existingMeta.retention !== metadata.retention ||
+					existingMeta.producer !== metadata.producer ||
+					existingMeta.originalPath !== metadata.originalPath
+				) {
+					throw new Error(
+						`Concurrent metadata write conflict for blob ${hash}: different metadata values detected. Existing: ${JSON.stringify(existingMeta)}, New: ${JSON.stringify(metadata)}`,
+					);
+				}
+			} catch (err) {
+				if (err instanceof Error && err.message.includes("ENOENT")) {
+					// OK - metadata doesn't exist yet, proceed to write
+				} else {
+					// Re-throw non-ENOENT errors (e.g., permission denied, corrupt file)
+					throw err;
+				}
+			}
+			// Issue 3 fix: Two-phase commit for metadata
+			// Write to temp file first, then atomically rename to final location.
+			// Issue 1 fix: Use atomicWriteFile instead of plain writeSync for the temp file
+			// to prevent partial writes on crash. atomicWriteFile uses O_EXCL + write + rename
+			// pattern which is consistent with how blob content is written.
+			const tempMetaPath = `${metadataPath}.${randomUUID()}.tmp`;
+			atomicWriteFile(tempMetaPath, JSON.stringify(metadata, null, 2));
+			renameWithRetry(tempMetaPath, metadataPath);
+			metadataWritten = true;
+		});
+	} catch (error) {
+		// Round 24 (BUG 4 note): the catch block previously checked
+		// `if (!blobContentWritten)` — the WRONG variable (the local comment said
+		// `metadataWritten === false`). For a CONTENT-ADDRESSED store the blob path
+		// is the content hash, so the blob may be referenced by another process's
+		// metadata even when OUR metadata write failed (e.g. a concurrent conflict
+		// where the peer already wrote metadata for the same hash). Deleting it
+		// would orphan their metadata. The safe behavior is therefore to NEVER
+		// delete on a metadata write failure and let the periodic
+		// cleanupOrphanedBlobs() reclaim genuinely-orphaned blobs. The guard below
+		// only removes a blob when its CONTENT was never written (a stray/partial
+		// file from a failed content write) — which is the only unambiguously-safe
+		// case to clean up here.
+		if (!blobContentWritten) {
+			try {
+				fs.rmSync(blobPath, { force: true });
+			} catch {
+				/* best-effort */
+			}
+		}
+		throw error;
+	}
+
+	// Issue 2 fix: resolve paths before writes and return cached values
+	const resolvedBlobPath = resolveRealContainedPath(artifactsRoot, blobPath);
+	const resolvedMetadataPath = resolveRealContainedPath(artifactsRoot, metadataPath);
+	return {
+		hash,
+		algorithm,
+		blobPath: resolvedBlobPath,
+		metadataPath: resolvedMetadataPath,
+		sizeBytes: metadata.sizeBytes,
+	};
+}
+
+/**
+ * Read a content-addressed blob by hash.
+ * Validates hash format and enforces path containment.
+ */
+export function readBlob(artifactsRoot: string, hash: string): Buffer | undefined {
+	validateBlobHash(hash);
+	try {
+		const blobDir = path.join(artifactsRoot, BLOBS_DIR, SHA256_PREFIX);
+		const blobPath = resolveRealContainedPath(blobDir, hash);
+		return fs.readFileSync(blobPath);
+	} catch {
+		return undefined;
+	}
+}
+
+/**
+ * Read blob metadata by hash.
+ * Validates hash format and enforces path containment.
+ */
+export function readBlobMetadata(artifactsRoot: string, hash: string): BlobMetadata | undefined {
+	validateBlobHash(hash);
+	try {
+		const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
+		const metaPath = resolveRealContainedPath(metaDir, `${hash}.json`);
+		return JSON.parse(fs.readFileSync(metaPath, "utf-8")) as BlobMetadata;
+	} catch {
+		return undefined;
+	}
+}
+/**
+ * Cleanup orphaned blobs - blobs without corresponding metadata entries.
+ * Called periodically to reclaim storage from blobs that were written but whose
+ * metadata write failed (e.g., crash between content and metadata write).
+ * Returns the number of orphaned blobs cleaned up.
+ */
+export function cleanupOrphanedBlobs(artifactsRoot: string): number {
+	const blobDir = path.join(artifactsRoot, BLOBS_DIR, SHA256_PREFIX);
+	const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
+
+	let cleaned = 0;
+	try {
+		const blobFiles = fs.readdirSync(blobDir);
+		for (const blobFile of blobFiles) {
+			// Skip non-hash files (e.g., temp files from atomicWriteBuffer)
+			if (!SHA256_HEX.test(blobFile)) continue;
+
+			const metaPath = path.join(metaDir, `${blobFile}.json`);
+			// Issue 4 fix: Use lock to prevent race between stat and delete.
+			// A concurrent process could write metadata between stat and delete,
+			// causing valid blob to be deleted while metadata remains.
+			withMetadataLock(metaPath, () => {
+				try {
+					fs.statSync(metaPath);
+					// Metadata exists - blob is not orphaned
+				} catch {
+					// Metadata does not exist - blob is orphaned, delete it
+					const blobPath = path.join(blobDir, blobFile);
+					try {
+						fs.rmSync(blobPath, { force: true });
+						cleaned++;
+					} catch {
+						// Best-effort cleanup - continue to next blob
+					}
+				}
+			});
+		}
+	} catch {
+		// Blobs directory doesn't exist or inaccessible - nothing to clean
+	}
+	return cleaned;
+}

package/src/state/contracts.ts

@@ -0,0 +1,160 @@
+export const TEAM_RUN_STATUSES = ["queued", "planning", "running", "blocked", "completed", "failed", "cancelled"] as const;
+export type TeamRunStatus = (typeof TEAM_RUN_STATUSES)[number];
+
+export const TEAM_TASK_STATUSES = [
+	"queued",
+	"running",
+	"waiting",
+	"completed",
+	"failed",
+	"cancelled",
+	"skipped",
+	"needs_attention",
+] as const;
+export type TeamTaskStatus = (typeof TEAM_TASK_STATUSES)[number];
+
+export const TEAM_TERMINAL_RUN_STATUSES: ReadonlySet<TeamRunStatus> = new Set(["blocked", "completed", "failed", "cancelled"]);
+export const TEAM_TERMINAL_TASK_STATUSES: ReadonlySet<TeamTaskStatus> = new Set([
+	"completed",
+	"failed",
+	"cancelled",
+	"skipped",
+	"needs_attention",
+]);
+
+export const TEAM_RUN_STATUS_TRANSITIONS: Readonly<Record<TeamRunStatus, readonly TeamRunStatus[]>> = {
+	queued: ["planning", "running", "cancelled", "failed"],
+	planning: ["running", "blocked", "cancelled", "failed"],
+	running: ["blocked", "completed", "failed", "cancelled"],
+	blocked: ["running", "cancelled", "failed"],
+	completed: ["running", "cancelled"],
+	failed: ["running", "cancelled"],
+	cancelled: ["running"],
+};
+
+export const TEAM_TASK_STATUS_TRANSITIONS: Readonly<Record<TeamTaskStatus, readonly TeamTaskStatus[]>> = {
+	queued: ["running", "cancelled", "skipped", "failed"],
+	running: ["completed", "failed", "cancelled", "queued", "waiting"],
+	waiting: ["running", "queued", "completed", "failed", "cancelled"],
+	completed: ["queued"],
+	failed: ["queued", "cancelled"],
+	cancelled: ["queued"],
+	skipped: ["queued", "cancelled"],
+	needs_attention: ["queued", "running"],
+};
+
+/** @internal */
+export const TEAM_EVENT_TYPES = [
+	"run.created",
+	"run.queued",
+	"run.planning",
+	"run.running",
+	"run.blocked",
+	"run.completed",
+	"run.failed",
+	"run.cancelled",
+	"task.created",
+	"task.started",
+	"task.progress",
+	"hook.pre_step_started",
+	"hook.pre_step_completed",
+	"hook.pre_step_failed",
+	"hook.pre_step_skipped",
+	"hook.pre_step_optional_failed",
+	"task.blocked",
+	"task.green",
+	"task.red",
+	"task.completed",
+	"task.failed",
+	"task.cancelled",
+	"task.skipped",
+	"task.needs_attention",
+	"review.approved",
+	"review.rejected",
+	"policy.action",
+	"policy.escalated",
+	"recovery.attempted",
+	"recovery.escalated",
+	"branch.stale",
+	"mailbox.timeout",
+	"worktree.cleanup",
+	"worktree.dirty",
+	"async.spawned",
+	"async.started",
+	"async.completed",
+	"async.failed",
+	"async.stale",
+	"task.waiting",
+	"task.resumed",
+	"task.retried",
+	"supervisor.contact",
+	// Budget tracking events
+	"budget.initialized",
+	"budget.warning",
+	"budget.exhausted",
+	// Phase tracking events
+	"phase.started",
+	"phase.completed",
+	"phase.skipped",
+	"phase.failed",
+	// Goal loop events (P0/P1) — autonomous goal-loop coordinator.
+	"goal.loop_start",
+	"goal.turn_start",
+	"goal.turn_evaluated",
+	"goal.budget_warning",
+	"goal.loop_end",
+	"goal.feedback_steered",
+	"goal.state_changed",
+	// Dynamic workflow events (P2) — script-driven orchestration.
+	"dwf.started",
+	"dwf.phase_started",
+	"dwf.phase_completed",
+	"dwf.completed",
+	"dwf.failed",
+	"dwf.trust_denied", // F-01: project .dwf.ts denied without PI_CREW_TRUST_PROJECT_DWF
+	"dwf.log",
+] as const;
+export type TeamEventType = (typeof TEAM_EVENT_TYPES)[number];
+
+export const TEAM_WAKEABLE_EVENT_TYPES: ReadonlySet<TeamEventType> = new Set([
+	"run.blocked",
+	"run.completed",
+	"run.failed",
+	"run.cancelled",
+	"task.completed",
+	"task.failed",
+	"task.cancelled",
+	"task.skipped",
+	"task.needs_attention",
+	"async.completed",
+	"async.failed",
+	"async.stale",
+]);
+
+export function isTeamRunStatus(value: unknown): value is TeamRunStatus {
+	return typeof value === "string" && TEAM_RUN_STATUSES.includes(value as TeamRunStatus);
+}
+
+export function isTeamTaskStatus(value: unknown): value is TeamTaskStatus {
+	return typeof value === "string" && TEAM_TASK_STATUSES.includes(value as TeamTaskStatus);
+}
+
+export function isTerminalRunStatus(status: TeamRunStatus): boolean {
+	return TEAM_TERMINAL_RUN_STATUSES.has(status);
+}
+
+export function isTerminalTaskStatus(status: TeamTaskStatus): boolean {
+	return TEAM_TERMINAL_TASK_STATUSES.has(status);
+}
+
+export function canTransitionRunStatus(from: TeamRunStatus, to: TeamRunStatus): boolean {
+	return from === to || (TEAM_RUN_STATUS_TRANSITIONS[from]?.includes(to) ?? false);
+}
+
+export function canTransitionTaskStatus(from: TeamTaskStatus, to: TeamTaskStatus): boolean {
+	return from === to || (TEAM_TASK_STATUS_TRANSITIONS[from]?.includes(to) ?? false);
+}
+
+export function isWakeableTeamEventType(type: TeamEventType): boolean {
+	return TEAM_WAKEABLE_EVENT_TYPES.has(type);
+}