pi-crew 0.9.67 → 0.10.1

package/src/state/stores/blob-store.ts

@@ -1,308 +0,0 @@
-import { createHash, randomUUID } from "node:crypto";
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { resolveRealContainedPath } from "../../utils/safe-paths.ts";
-import { atomicWriteFile, renameWithRetry } from "../atomic-write.ts";
-import { withFileLockSync } from "../coordination/locks.ts";
-
-const SHA256_HEX = /^[a-f0-9]{64}$/i;
-
-function validateBlobHash(hash: string, algorithm?: string): void {
-	if (!SHA256_HEX.test(hash)) throw new Error(`Invalid blob hash: ${hash}`);
-	if (algorithm !== undefined && algorithm !== SHA256_PREFIX) {
-		throw new Error(`Invalid blob algorithm: ${algorithm} (expected ${SHA256_PREFIX})`);
-	}
-}
-
-/**
- * Atomically write a Buffer to a file using temp-file + rename pattern.
- * Prevents partial writes on crash and handles concurrent writes safely.
- */
-function atomicWriteBuffer(filePath: string, content: Buffer): void {
-	fs.mkdirSync(path.dirname(filePath), { recursive: true });
-	const tempPath = `${filePath}.${randomUUID()}.tmp`;
-	const O_NOFOLLOW = typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0;
-	const fd = fs.openSync(tempPath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | O_NOFOLLOW, 0o600);
-	try {
-		const openedStat = fs.fstatSync(fd);
-		if (!openedStat.isFile()) {
-			throw new Error(`Refusing to write: opened path is not a regular file: ${tempPath}`);
-		}
-		fs.writeSync(fd, content, 0, content.length);
-		renameWithRetry(tempPath, filePath);
-	} catch (error) {
-		try {
-			fs.rmSync(tempPath, { force: true });
-		} catch {
-			/* best-effort */
-		}
-		throw error;
-	} finally {
-		// Always close fd; closeSync is safe to call even if fd was already closed
-		try {
-			fs.closeSync(fd);
-		} catch {
-			/* best-effort */
-		}
-	}
-}
-
-const BLOBS_DIR = "blobs";
-const BLOB_META_DIR = "blob-metadata";
-const SHA256_PREFIX = "sha256";
-
-/**
- * File-based lock for cross-process metadata write protection.
- * Uses withFileLockSync which provides O_EXCL atomic create + token-guarded release.
- */
-function withMetadataLock<T>(metadataPath: string, fn: () => T): T {
-	return withFileLockSync(metadataPath, fn);
-}
-
-export interface BlobMetadata {
-	blobHash: string;
-	blobAlgorithm: string;
-	runId: string;
-	taskId?: string;
-	mime: string;
-	producer: string;
-	originalPath: string;
-	sizeBytes: number;
-	redacted: boolean;
-	retention: "run" | "project" | "temporary";
-	createdAt: string;
-}
-
-export interface BlobWriteResult {
-	hash: string;
-	algorithm: string;
-	blobPath: string;
-	metadataPath: string;
-	sizeBytes: number;
-}
-
-function sha256Of(content: string | Buffer): string {
-	return createHash("sha256")
-		.update(typeof content === "string" ? content : content)
-		.digest("hex");
-}
-
-/**
- * Write content-addressed blob to the blobs directory under artifactsRoot.
- * Content is deduplicated by hash; metadata sidecar is always written.
- * FIX: Both content and metadata writes now use atomicWriteFile to prevent
- * partial writes on crash. The deduplication check is now advisory (the atomic
- * write handles concurrent writes correctly via O_EXCL temp file pattern).
- */
-export function writeBlob(
-	artifactsRoot: string,
-	input: {
-		content: string | Buffer;
-		runId: string;
-		taskId?: string;
-		mime?: string;
-		producer: string;
-		originalPath: string;
-		redacted?: boolean;
-		retention?: BlobMetadata["retention"];
-	},
-): BlobWriteResult {
-	const content = input.content;
-	const hash = sha256Of(content);
-	const algorithm = SHA256_PREFIX;
-	const blobDir = path.join(artifactsRoot, BLOBS_DIR, algorithm);
-	const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
-	// Issue 2 fix: Validate parent directories are not symlinks before mkdirSync.
-	// An attacker could plant a symlink before we create the directory, causing
-	// blob content to be written to an attacker-controlled location.
-	resolveRealContainedPath(artifactsRoot, blobDir);
-	resolveRealContainedPath(artifactsRoot, metaDir);
-	fs.mkdirSync(blobDir, { recursive: true });
-	fs.mkdirSync(metaDir, { recursive: true });
-
-	const blobPath = path.join(blobDir, hash);
-	// Issue 6 fix: Validate blobPath BEFORE writing to ensure the path is safe.
-	// If resolveRealContainedPath throws after the write, the blob may already
-	// exist at an unexpected location. Validate first to prevent this.
-	resolveRealContainedPath(artifactsRoot, blobPath);
-	const metadata: BlobMetadata = {
-		blobHash: hash,
-		blobAlgorithm: algorithm,
-		runId: input.runId,
-		taskId: input.taskId,
-		mime: input.mime ?? "text/plain",
-		producer: input.producer,
-		originalPath: input.originalPath,
-		sizeBytes: Buffer.isBuffer(content) ? content.length : Buffer.byteLength(content, "utf-8"),
-		redacted: input.redacted ?? false,
-		retention: input.retention ?? "run",
-		createdAt: new Date().toISOString(),
-	};
-	const metadataPath = path.join(metaDir, `${hash}.json`);
-
-	// Both content and metadata use atomic writes to prevent partial writes on crash.
-	// Content is immutable and content-addressed (same hash = same content), so
-	// concurrent writes to the same hash are safe. Metadata (mime, retention, etc.)
-	// is protected by a per-hash lock to make the check-then-write atomic, preventing
-	// concurrent writers from racing to write different metadata for the same hash.
-	// Issue 3 fix: wrap metadata check-and-write in a lock to make it atomic.
-	// Without the lock, two processes could read the same metadata concurrently,
-	// both pass the check, and the second write would silently overwrite the first.
-	// FIX: Write blob content FIRST, then metadata. If blob write fails, no orphan metadata.
-	// Previous order was metadata first, then blob - causing orphan metadata on blob failure.
-	let blobContentWritten = false;
-	// Write blob content first (immutable, content-addressed)
-	atomicWriteBuffer(blobPath, Buffer.isBuffer(content) ? content : Buffer.from(content, "utf-8"));
-	blobContentWritten = true;
-
-	// Metadata only after blob content is successfully written
-	// Issue 3 fix: Use two-phase commit for metadata - write to temp file first,
-	// then atomically rename to final location. This prevents orphan blobs if
-	// the process crashes after blob write but before metadata rename.
-	let metadataWritten = false;
-	try {
-		withMetadataLock(metadataPath, () => {
-			try {
-				const existingMeta = JSON.parse(fs.readFileSync(metadataPath, "utf-8")) as BlobMetadata;
-				// Compare fields that indicate concurrent write with different metadata.
-				// Note: taskId is intentionally NOT part of the conflict check because
-				// it represents which task produced the blob and may legitimately differ
-				// across processes (last-write-wins for taskId). If concurrent writes
-				// have identical mime/retention/producer/originalPath but different
-				// taskId, the conflict check passes and both writes succeed — the
-				// final taskId reflects whichever write completed last.
-				// Callers should not rely on taskId for correctness-critical decisions.
-				if (
-					existingMeta.mime !== metadata.mime ||
-					existingMeta.retention !== metadata.retention ||
-					existingMeta.producer !== metadata.producer ||
-					existingMeta.originalPath !== metadata.originalPath
-				) {
-					throw new Error(
-						`Concurrent metadata write conflict for blob ${hash}: different metadata values detected. Existing: ${JSON.stringify(existingMeta)}, New: ${JSON.stringify(metadata)}`,
-					);
-				}
-			} catch (err) {
-				if (err instanceof Error && err.message.includes("ENOENT")) {
-					// OK - metadata doesn't exist yet, proceed to write
-				} else {
-					// Re-throw non-ENOENT errors (e.g., permission denied, corrupt file)
-					throw err;
-				}
-			}
-			// Issue 3 fix: Two-phase commit for metadata
-			// Write to temp file first, then atomically rename to final location.
-			// Issue 1 fix: Use atomicWriteFile instead of plain writeSync for the temp file
-			// to prevent partial writes on crash. atomicWriteFile uses O_EXCL + write + rename
-			// pattern which is consistent with how blob content is written.
-			const tempMetaPath = `${metadataPath}.${randomUUID()}.tmp`;
-			atomicWriteFile(tempMetaPath, JSON.stringify(metadata, null, 2));
-			renameWithRetry(tempMetaPath, metadataPath);
-			metadataWritten = true;
-		});
-	} catch (error) {
-		// Round 24 (BUG 4 note): the catch block previously checked
-		// `if (!blobContentWritten)` — the WRONG variable (the local comment said
-		// `metadataWritten === false`). For a CONTENT-ADDRESSED store the blob path
-		// is the content hash, so the blob may be referenced by another process's
-		// metadata even when OUR metadata write failed (e.g. a concurrent conflict
-		// where the peer already wrote metadata for the same hash). Deleting it
-		// would orphan their metadata. The safe behavior is therefore to NEVER
-		// delete on a metadata write failure and let the periodic
-		// cleanupOrphanedBlobs() reclaim genuinely-orphaned blobs. The guard below
-		// only removes a blob when its CONTENT was never written (a stray/partial
-		// file from a failed content write) — which is the only unambiguously-safe
-		// case to clean up here.
-		if (!blobContentWritten) {
-			try {
-				fs.rmSync(blobPath, { force: true });
-			} catch {
-				/* best-effort */
-			}
-		}
-		throw error;
-	}
-
-	// Issue 2 fix: resolve paths before writes and return cached values
-	const resolvedBlobPath = resolveRealContainedPath(artifactsRoot, blobPath);
-	const resolvedMetadataPath = resolveRealContainedPath(artifactsRoot, metadataPath);
-	return {
-		hash,
-		algorithm,
-		blobPath: resolvedBlobPath,
-		metadataPath: resolvedMetadataPath,
-		sizeBytes: metadata.sizeBytes,
-	};
-}
-
-/**
- * Read a content-addressed blob by hash.
- * Validates hash format and enforces path containment.
- */
-export function readBlob(artifactsRoot: string, hash: string): Buffer | undefined {
-	validateBlobHash(hash);
-	try {
-		const blobDir = path.join(artifactsRoot, BLOBS_DIR, SHA256_PREFIX);
-		const blobPath = resolveRealContainedPath(blobDir, hash);
-		return fs.readFileSync(blobPath);
-	} catch {
-		return undefined;
-	}
-}
-
-/**
- * Read blob metadata by hash.
- * Validates hash format and enforces path containment.
- */
-export function readBlobMetadata(artifactsRoot: string, hash: string): BlobMetadata | undefined {
-	validateBlobHash(hash);
-	try {
-		const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
-		const metaPath = resolveRealContainedPath(metaDir, `${hash}.json`);
-		return JSON.parse(fs.readFileSync(metaPath, "utf-8")) as BlobMetadata;
-	} catch {
-		return undefined;
-	}
-}
-/**
- * Cleanup orphaned blobs - blobs without corresponding metadata entries.
- * Called periodically to reclaim storage from blobs that were written but whose
- * metadata write failed (e.g., crash between content and metadata write).
- * Returns the number of orphaned blobs cleaned up.
- */
-export function cleanupOrphanedBlobs(artifactsRoot: string): number {
-	const blobDir = path.join(artifactsRoot, BLOBS_DIR, SHA256_PREFIX);
-	const metaDir = path.join(artifactsRoot, BLOB_META_DIR);
-
-	let cleaned = 0;
-	try {
-		const blobFiles = fs.readdirSync(blobDir);
-		for (const blobFile of blobFiles) {
-			// Skip non-hash files (e.g., temp files from atomicWriteBuffer)
-			if (!SHA256_HEX.test(blobFile)) continue;
-
-			const metaPath = path.join(metaDir, `${blobFile}.json`);
-			// Issue 4 fix: Use lock to prevent race between stat and delete.
-			// A concurrent process could write metadata between stat and delete,
-			// causing valid blob to be deleted while metadata remains.
-			withMetadataLock(metaPath, () => {
-				try {
-					fs.statSync(metaPath);
-					// Metadata exists - blob is not orphaned
-				} catch {
-					// Metadata does not exist - blob is orphaned, delete it
-					const blobPath = path.join(blobDir, blobFile);
-					try {
-						fs.rmSync(blobPath, { force: true });
-						cleaned++;
-					} catch {
-						// Best-effort cleanup - continue to next blob
-					}
-				}
-			});
-		}
-	} catch {
-		// Blobs directory doesn't exist or inaccessible - nothing to clean
-	}
-	return cleaned;
-}

package/src/state/stores/instinct-store.ts

@@ -1,275 +0,0 @@
-import { randomUUID } from "node:crypto";
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { assertSafePathId } from "../../utils/safe-paths.ts";
-import { atomicWriteFile } from "../atomic-write.ts";
-
-/**
- * Represents a learned instinct that guides agent behavior.
- * Instincts can be project-scoped or global.
- */
-export interface Instinct {
-	/** Unique identifier for this instinct */
-	id: string;
-	/** What triggers this instinct */
-	trigger: string;
-	/** What action to take when triggered */
-	action: string;
-	/** Confidence level: 0.3 (low), 0.6 (medium), 0.9 (high) */
-	confidence: 0.3 | 0.6 | 0.9;
-	/** Whether this instinct applies to a project or globally */
-	scope: "project" | "global";
-	/** Project identifier (undefined for global instincts) */
-	projectId?: string;
-	/** ISO timestamp of when this instinct was created */
-	createdAt: string;
-	/** Examples/evidence supporting this instinct */
-	evidence: string[];
-}
-
-/** Input type for creating a new instinct (excludes auto-generated fields) */
-export type NewInstinct = Omit<Instinct, "id" | "createdAt">;
-
-const INSTINCT_FILE = "instincts.jsonl";
-
-/**
- * InstinctStore manages persistence of learned instincts using JSONL files.
- * - Project instincts: `.crew/instincts/projects/{projectId}/instincts.jsonl`
- * - Global instincts: `.crew/instincts/global/instincts.jsonl`
- */
-export class InstinctStore {
-	private readonly crewRoot: string;
-
-	constructor(crewRoot: string) {
-		this.crewRoot = crewRoot;
-	}
-
-	/**
-	 * Get the file path for project instincts
-	 */
-	private getProjectInstinctPath(projectId: string): string {
-		return path.join(this.crewRoot, "instincts", "projects", projectId, INSTINCT_FILE);
-	}
-
-	/**
-	 * Get the file path for global instincts
-	 */
-	private getGlobalInstinctPath(): string {
-		return path.join(this.crewRoot, "instincts", "global", INSTINCT_FILE);
-	}
-
-	/**
-	 * Ensure a directory exists, creating it recursively if needed
-	 */
-	private ensureDir(dirPath: string): void {
-		fs.mkdirSync(dirPath, { recursive: true });
-	}
-
-	/**
-	 * Parse a JSONL file and return all instincts
-	 */
-	private readInstinctsFromFile(filePath: string): Instinct[] {
-		if (!fs.existsSync(filePath)) {
-			return [];
-		}
-		try {
-			const content = fs.readFileSync(filePath, "utf-8");
-			const lines = content.split("\n").filter((line) => line.trim() !== "");
-			return lines.map((line) => JSON.parse(line) as Instinct);
-		} catch {
-			// If file is corrupted, return empty array
-			return [];
-		}
-	}
-
-	/**
-	 * Append a single instinct to a JSONL file
-	 */
-	private appendInstinctToFile(filePath: string, instinct: Instinct): void {
-		const dir = path.dirname(filePath);
-		this.ensureDir(dir);
-		// ST-3: read existing + append + atomic rewrite. appendFileSync has no fsync and
-		// can interleave bytes on concurrent writes. Callers should wrap this in
-		// withFileLockSync(filePath) to serialize concurrent RMW on the same file.
-		const existing = fs.existsSync(filePath) ? this.readInstinctsFromFile(filePath) : [];
-		existing.push(instinct);
-		const content = existing.map((i) => JSON.stringify(i)).join("\n") + "\n";
-		atomicWriteFile(filePath, content);
-	}
-
-	/**
-	 * Rewrite a JSONL file with the given instincts
-	 */
-	private rewriteFile(filePath: string, instincts: Instinct[]): void {
-		const dir = path.dirname(filePath);
-		this.ensureDir(dir);
-		const content = instincts.map((i) => JSON.stringify(i)).join("\n") + "\n";
-		// ST-3: atomic write (temp + rename + fsync) — crash mid-writeFileSync truncates
-		// the JSONL and readInstinctsFromFile silently returns [] (all instincts lost).
-		atomicWriteFile(filePath, content);
-	}
-
-	/**
-	 * Save a new instinct with auto-generated id and timestamp.
-	 * Direct scope is determined by the instinct's scope field.
-	 *
-	 * @param instinct - The instinct to save (without id and createdAt)
-	 * @returns The saved instinct with generated id and createdAt
-	 */
-	saveInstinct(instinct: NewInstinct): Instinct {
-		const savedInstinct: Instinct = {
-			...instinct,
-			id: randomUUID(),
-			createdAt: new Date().toISOString(),
-		};
-
-		if (savedInstinct.scope === "global") {
-			savedInstinct.projectId = undefined;
-			this.appendInstinctToFile(this.getGlobalInstinctPath(), savedInstinct);
-		} else {
-			if (!savedInstinct.projectId) {
-				throw new Error("Project-scoped instinct requires a projectId");
-			}
-			assertSafePathId("projectId", savedInstinct.projectId);
-			this.appendInstinctToFile(this.getProjectInstinctPath(savedInstinct.projectId), savedInstinct);
-		}
-
-		return savedInstinct;
-	}
-
-	/**
-	 * Get all instincts, optionally filtered by scope.
-	 *
-	 * @param scope - Optional filter: 'project' or 'global'
-	 * @returns Array of instincts matching the filter
-	 */
-	getInstincts(scope?: "project" | "global"): Instinct[] {
-		const results: Instinct[] = [];
-
-		if (!scope || scope === "project") {
-			const projectsDir = path.join(this.crewRoot, "instincts", "projects");
-			if (fs.existsSync(projectsDir)) {
-				for (const projectId of fs
-					.readdirSync(projectsDir, { withFileTypes: true })
-					.filter((e) => e.isDirectory())
-					.map((e) => e.name)) {
-					const filePath = path.join(projectsDir, projectId, INSTINCT_FILE);
-					results.push(...this.readInstinctsFromFile(filePath));
-				}
-			}
-		}
-
-		if (!scope || scope === "global") {
-			results.push(...this.readInstinctsFromFile(this.getGlobalInstinctPath()));
-		}
-
-		return results;
-	}
-
-	/**
-	 * Get all instincts for a specific project.
-	 * Includes both project-scoped instincts and global instincts.
-	 *
-	 * @param projectId - The project identifier
-	 * @returns Array of instincts for the project
-	 */
-	getProjectInstincts(projectId: string): Instinct[] {
-		assertSafePathId("projectId", projectId);
-		const projectInstincts = this.readInstinctsFromFile(this.getProjectInstinctPath(projectId));
-		const globalInstincts = this.readInstinctsFromFile(this.getGlobalInstinctPath());
-		return [...projectInstincts, ...globalInstincts];
-	}
-
-	/**
-	 * Promote a project-scoped instinct to global scope.
-	 * Creates a copy in global instincts and removes from project.
-	 *
-	 * @param instinctId - The instinct id to promote
-	 * @returns The promoted instinct, or null if not found
-	 */
-	promoteInstinct(instinctId: string): Instinct | null {
-		// Search in all project instinct files
-		const projectsDir = path.join(this.crewRoot, "instincts", "projects");
-		if (fs.existsSync(projectsDir)) {
-			for (const projectId of fs
-				.readdirSync(projectsDir, { withFileTypes: true })
-				.filter((e) => e.isDirectory())
-				.map((e) => e.name)) {
-				assertSafePathId("projectId", projectId);
-				const filePath = path.join(projectsDir, projectId, INSTINCT_FILE);
-				const instincts = this.readInstinctsFromFile(filePath);
-				const index = instincts.findIndex((i) => i.id === instinctId);
-
-				if (index !== -1) {
-					const instinct = instincts[index];
-
-					// Create promoted version (global)
-					const promotedInstinct: Instinct = {
-						...instinct,
-						id: randomUUID(), // New id for promoted instinct
-						scope: "global",
-						projectId: undefined,
-						createdAt: new Date().toISOString(),
-					};
-
-					// Add to global instincts
-					this.appendInstinctToFile(this.getGlobalInstinctPath(), promotedInstinct);
-
-					// Remove from project instincts
-					const updatedInstincts = instincts.filter((i) => i.id !== instinctId);
-					this.rewriteFile(filePath, updatedInstincts);
-
-					return promotedInstinct;
-				}
-			}
-		}
-
-		return null;
-	}
-
-	/**
-	 * Delete an instinct by id.
-	 *
-	 * @param instinctId - The instinct id to delete
-	 * @returns true if deleted, false if not found
-	 */
-	deleteInstinct(instinctId: string): boolean {
-		// Search in global instincts first
-		const globalPath = this.getGlobalInstinctPath();
-		let instincts = this.readInstinctsFromFile(globalPath);
-		let index = instincts.findIndex((i) => i.id === instinctId);
-
-		if (index !== -1) {
-			const updatedInstincts = instincts.filter((i) => i.id !== instinctId);
-			this.rewriteFile(globalPath, updatedInstincts);
-			return true;
-		}
-
-		// Search in project instincts
-		const projectsDir = path.join(this.crewRoot, "instincts", "projects");
-		if (fs.existsSync(projectsDir)) {
-			for (const projectId of fs
-				.readdirSync(projectsDir, { withFileTypes: true })
-				.filter((e) => e.isDirectory())
-				.map((e) => e.name)) {
-				assertSafePathId("projectId", projectId);
-				const filePath = path.join(projectsDir, projectId, INSTINCT_FILE);
-				instincts = this.readInstinctsFromFile(filePath);
-				index = instincts.findIndex((i) => i.id === instinctId);
-
-				if (index !== -1) {
-					const updatedInstincts = instincts.filter((i) => i.id !== instinctId);
-					this.rewriteFile(filePath, updatedInstincts);
-					return true;
-				}
-			}
-		}
-
-		return false;
-	}
-}
-
-export {
-	getGlobalStorageDir,
-	getProjectStorageDir,
-} from "../../utils/project-detector.ts";