pi-crew 0.9.53 → 0.9.55

package/src/state/instinct-store.ts

@@ -0,0 +1,275 @@
+import { randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { assertSafePathId } from "../utils/safe-paths.ts";
+import { atomicWriteFile } from "./atomic-write.ts";
+
+/**
+ * Represents a learned instinct that guides agent behavior.
+ * Instincts can be project-scoped or global.
+ */
+export interface Instinct {
+	/** Unique identifier for this instinct */
+	id: string;
+	/** What triggers this instinct */
+	trigger: string;
+	/** What action to take when triggered */
+	action: string;
+	/** Confidence level: 0.3 (low), 0.6 (medium), 0.9 (high) */
+	confidence: 0.3 | 0.6 | 0.9;
+	/** Whether this instinct applies to a project or globally */
+	scope: "project" | "global";
+	/** Project identifier (undefined for global instincts) */
+	projectId?: string;
+	/** ISO timestamp of when this instinct was created */
+	createdAt: string;
+	/** Examples/evidence supporting this instinct */
+	evidence: string[];
+}
+
+/** Input type for creating a new instinct (excludes auto-generated fields) */
+export type NewInstinct = Omit<Instinct, "id" | "createdAt">;
+
+const INSTINCT_FILE = "instincts.jsonl";
+
+/**
+ * InstinctStore manages persistence of learned instincts using JSONL files.
+ * - Project instincts: `.crew/instincts/projects/{projectId}/instincts.jsonl`
+ * - Global instincts: `.crew/instincts/global/instincts.jsonl`
+ */
+export class InstinctStore {
+	private readonly crewRoot: string;
+
+	constructor(crewRoot: string) {
+		this.crewRoot = crewRoot;
+	}
+
+	/**
+	 * Get the file path for project instincts
+	 */
+	private getProjectInstinctPath(projectId: string): string {
+		return path.join(this.crewRoot, "instincts", "projects", projectId, INSTINCT_FILE);
+	}
+
+	/**
+	 * Get the file path for global instincts
+	 */
+	private getGlobalInstinctPath(): string {
+		return path.join(this.crewRoot, "instincts", "global", INSTINCT_FILE);
+	}
+
+	/**
+	 * Ensure a directory exists, creating it recursively if needed
+	 */
+	private ensureDir(dirPath: string): void {
+		fs.mkdirSync(dirPath, { recursive: true });
+	}
+
+	/**
+	 * Parse a JSONL file and return all instincts
+	 */
+	private readInstinctsFromFile(filePath: string): Instinct[] {
+		if (!fs.existsSync(filePath)) {
+			return [];
+		}
+		try {
+			const content = fs.readFileSync(filePath, "utf-8");
+			const lines = content.split("\n").filter((line) => line.trim() !== "");
+			return lines.map((line) => JSON.parse(line) as Instinct);
+		} catch {
+			// If file is corrupted, return empty array
+			return [];
+		}
+	}
+
+	/**
+	 * Append a single instinct to a JSONL file
+	 */
+	private appendInstinctToFile(filePath: string, instinct: Instinct): void {
+		const dir = path.dirname(filePath);
+		this.ensureDir(dir);
+		// ST-3: read existing + append + atomic rewrite. appendFileSync has no fsync and
+		// can interleave bytes on concurrent writes. Callers should wrap this in
+		// withFileLockSync(filePath) to serialize concurrent RMW on the same file.
+		const existing = fs.existsSync(filePath) ? this.readInstinctsFromFile(filePath) : [];
+		existing.push(instinct);
+		const content = existing.map((i) => JSON.stringify(i)).join("\n") + "\n";
+		atomicWriteFile(filePath, content);
+	}
+
+	/**
+	 * Rewrite a JSONL file with the given instincts
+	 */
+	private rewriteFile(filePath: string, instincts: Instinct[]): void {
+		const dir = path.dirname(filePath);
+		this.ensureDir(dir);
+		const content = instincts.map((i) => JSON.stringify(i)).join("\n") + "\n";
+		// ST-3: atomic write (temp + rename + fsync) — crash mid-writeFileSync truncates
+		// the JSONL and readInstinctsFromFile silently returns [] (all instincts lost).
+		atomicWriteFile(filePath, content);
+	}
+
+	/**
+	 * Save a new instinct with auto-generated id and timestamp.
+	 * Direct scope is determined by the instinct's scope field.
+	 *
+	 * @param instinct - The instinct to save (without id and createdAt)
+	 * @returns The saved instinct with generated id and createdAt
+	 */
+	saveInstinct(instinct: NewInstinct): Instinct {
+		const savedInstinct: Instinct = {
+			...instinct,
+			id: randomUUID(),
+			createdAt: new Date().toISOString(),
+		};
+
+		if (savedInstinct.scope === "global") {
+			savedInstinct.projectId = undefined;
+			this.appendInstinctToFile(this.getGlobalInstinctPath(), savedInstinct);
+		} else {
+			if (!savedInstinct.projectId) {
+				throw new Error("Project-scoped instinct requires a projectId");
+			}
+			assertSafePathId("projectId", savedInstinct.projectId);
+			this.appendInstinctToFile(this.getProjectInstinctPath(savedInstinct.projectId), savedInstinct);
+		}
+
+		return savedInstinct;
+	}
+
+	/**
+	 * Get all instincts, optionally filtered by scope.
+	 *
+	 * @param scope - Optional filter: 'project' or 'global'
+	 * @returns Array of instincts matching the filter
+	 */
+	getInstincts(scope?: "project" | "global"): Instinct[] {
+		const results: Instinct[] = [];
+
+		if (!scope || scope === "project") {
+			const projectsDir = path.join(this.crewRoot, "instincts", "projects");
+			if (fs.existsSync(projectsDir)) {
+				for (const projectId of fs
+					.readdirSync(projectsDir, { withFileTypes: true })
+					.filter((e) => e.isDirectory())
+					.map((e) => e.name)) {
+					const filePath = path.join(projectsDir, projectId, INSTINCT_FILE);
+					results.push(...this.readInstinctsFromFile(filePath));
+				}
+			}
+		}
+
+		if (!scope || scope === "global") {
+			results.push(...this.readInstinctsFromFile(this.getGlobalInstinctPath()));
+		}
+
+		return results;
+	}
+
+	/**
+	 * Get all instincts for a specific project.
+	 * Includes both project-scoped instincts and global instincts.
+	 *
+	 * @param projectId - The project identifier
+	 * @returns Array of instincts for the project
+	 */
+	getProjectInstincts(projectId: string): Instinct[] {
+		assertSafePathId("projectId", projectId);
+		const projectInstincts = this.readInstinctsFromFile(this.getProjectInstinctPath(projectId));
+		const globalInstincts = this.readInstinctsFromFile(this.getGlobalInstinctPath());
+		return [...projectInstincts, ...globalInstincts];
+	}
+
+	/**
+	 * Promote a project-scoped instinct to global scope.
+	 * Creates a copy in global instincts and removes from project.
+	 *
+	 * @param instinctId - The instinct id to promote
+	 * @returns The promoted instinct, or null if not found
+	 */
+	promoteInstinct(instinctId: string): Instinct | null {
+		// Search in all project instinct files
+		const projectsDir = path.join(this.crewRoot, "instincts", "projects");
+		if (fs.existsSync(projectsDir)) {
+			for (const projectId of fs
+				.readdirSync(projectsDir, { withFileTypes: true })
+				.filter((e) => e.isDirectory())
+				.map((e) => e.name)) {
+				assertSafePathId("projectId", projectId);
+				const filePath = path.join(projectsDir, projectId, INSTINCT_FILE);
+				const instincts = this.readInstinctsFromFile(filePath);
+				const index = instincts.findIndex((i) => i.id === instinctId);
+
+				if (index !== -1) {
+					const instinct = instincts[index];
+
+					// Create promoted version (global)
+					const promotedInstinct: Instinct = {
+						...instinct,
+						id: randomUUID(), // New id for promoted instinct
+						scope: "global",
+						projectId: undefined,
+						createdAt: new Date().toISOString(),
+					};
+
+					// Add to global instincts
+					this.appendInstinctToFile(this.getGlobalInstinctPath(), promotedInstinct);
+
+					// Remove from project instincts
+					const updatedInstincts = instincts.filter((i) => i.id !== instinctId);
+					this.rewriteFile(filePath, updatedInstincts);
+
+					return promotedInstinct;
+				}
+			}
+		}
+
+		return null;
+	}
+
+	/**
+	 * Delete an instinct by id.
+	 *
+	 * @param instinctId - The instinct id to delete
+	 * @returns true if deleted, false if not found
+	 */
+	deleteInstinct(instinctId: string): boolean {
+		// Search in global instincts first
+		const globalPath = this.getGlobalInstinctPath();
+		let instincts = this.readInstinctsFromFile(globalPath);
+		let index = instincts.findIndex((i) => i.id === instinctId);
+
+		if (index !== -1) {
+			const updatedInstincts = instincts.filter((i) => i.id !== instinctId);
+			this.rewriteFile(globalPath, updatedInstincts);
+			return true;
+		}
+
+		// Search in project instincts
+		const projectsDir = path.join(this.crewRoot, "instincts", "projects");
+		if (fs.existsSync(projectsDir)) {
+			for (const projectId of fs
+				.readdirSync(projectsDir, { withFileTypes: true })
+				.filter((e) => e.isDirectory())
+				.map((e) => e.name)) {
+				assertSafePathId("projectId", projectId);
+				const filePath = path.join(projectsDir, projectId, INSTINCT_FILE);
+				instincts = this.readInstinctsFromFile(filePath);
+				index = instincts.findIndex((i) => i.id === instinctId);
+
+				if (index !== -1) {
+					const updatedInstincts = instincts.filter((i) => i.id !== instinctId);
+					this.rewriteFile(filePath, updatedInstincts);
+					return true;
+				}
+			}
+		}
+
+		return false;
+	}
+}
+
+export {
+	getGlobalStorageDir,
+	getProjectStorageDir,
+} from "../utils/project-detector.ts";

package/src/state/jsonl-writer.ts

@@ -0,0 +1,107 @@
+import * as fs from "node:fs";
+import { logInternalError } from "../utils/internal-error.ts";
+import { redactJsonLine } from "../utils/redaction.ts";
+
+export interface DrainableSource {
+	pause(): void;
+	resume(): void;
+}
+
+export interface JsonlWriteStream {
+	write(chunk: string): boolean;
+	once(event: "drain", listener: () => void): JsonlWriteStream;
+	end(callback?: () => void): void;
+}
+
+const DEFAULT_MAX_JSONL_BYTES = 50 * 1024 * 1024;
+// FIX (Round 21, per-line cap): A single huge line could exhaust memory during
+// redactJsonLine if an upstream caller constructs an enormous string. Cap each
+// line at 1MB by default — large enough for any legitimate event payload, small
+// enough to prevent memory blow-up. Mirrors the upstream oh-my-pi pattern of
+// bounding chunk boundaries in Bun.file().writer().
+const DEFAULT_MAX_LINE_BYTES = 1 * 1024 * 1024;
+
+export interface JsonlWriterDeps {
+	createWriteStream?: (filePath: string) => JsonlWriteStream;
+	maxBytes?: number;
+	maxLineBytes?: number;
+}
+
+export interface JsonlWriter {
+	writeLine(line: string): void;
+	close(): Promise<void>;
+}
+
+export function createJsonlWriter(filePath: string | undefined, source: DrainableSource, deps: JsonlWriterDeps = {}): JsonlWriter {
+	if (!filePath) {
+		return {
+			writeLine() {},
+			async close() {},
+		};
+	}
+
+	const createWriteStream = deps.createWriteStream ?? ((targetPath: string) => fs.createWriteStream(targetPath, { flags: "a" }));
+	let stream: JsonlWriteStream | undefined;
+	try {
+		stream = createWriteStream(filePath);
+	} catch {
+		return {
+			writeLine() {},
+			async close() {},
+		};
+	}
+
+	let backpressured = false;
+	let closed = false;
+	let bytesWritten = 0;
+	let linesDroppedForSize = 0;
+	const maxBytes = deps.maxBytes ?? DEFAULT_MAX_JSONL_BYTES;
+	const maxLineBytes = deps.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES;
+
+	return {
+		writeLine(line: string) {
+			if (!stream || closed || !line.trim()) return;
+			const safeLine = redactJsonLine(line);
+			const chunk = `${safeLine}\n`;
+			const chunkBytes = Buffer.byteLength(chunk, "utf-8");
+			// FIX (Round 21, per-line cap): Drop oversize lines. Without this, a
+			// single huge payload (e.g. a 100MB base64-encoded transcript) would
+			// be buffered in memory by redactJsonLine AND queued in the write
+			// stream. We log the drop so silent loss is visible.
+			if (chunkBytes > maxLineBytes) {
+				linesDroppedForSize++;
+				if (linesDroppedForSize === 1 || linesDroppedForSize % 100 === 0) {
+					logInternalError(
+						"jsonl-writer.lineTooLarge",
+						new Error(`line size ${chunkBytes} exceeds maxLineBytes ${maxLineBytes}`),
+						`file=${filePath} dropped=${linesDroppedForSize}`,
+					);
+				}
+				return;
+			}
+			if (bytesWritten + chunkBytes > maxBytes) return;
+			try {
+				const ok = stream.write(chunk);
+				bytesWritten += chunkBytes;
+				if (!ok && !backpressured) {
+					backpressured = true;
+					source.pause();
+					stream.once("drain", () => {
+						backpressured = false;
+						if (!closed) source.resume();
+					});
+				}
+			} catch (writeError) {
+				// Log the error — silently dropping events is dangerous.
+				logInternalError("jsonl-writer.write", writeError, `file=${filePath}`);
+			}
+		},
+		async close() {
+			if (!stream || closed) return;
+			closed = true;
+			const current = stream;
+			stream = undefined;
+			await new Promise<void>((resolve) => current.end(() => resolve()));
+		},
+	};
+}