pi-crew 0.5.24 → 0.6.0

package/src/state/observation-store.ts

@@ -0,0 +1,177 @@
+/**
+ * Observation capture and compression system.
+ *
+ * Pattern origin: claude-mem — captures tool usage across sessions,
+ * compresses via AI, injects into future sessions.
+ *
+ * This module provides the observation store and compression logic.
+ * Actual capture hooks into the lifecycle events (Pattern 12).
+ */
+
+import { mkdirSync, readFileSync, writeFileSync, existsSync, appendFileSync } from "node:fs";
+import { logInternalError } from "../utils/internal-error.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export interface Observation {
+	tool: string;
+	input: string;
+	output: string;
+	filesRead: string[];
+	filesModified: string[];
+	timestamp: number;
+	sessionId: string;
+	taskId?: string;
+}
+
+export interface CompressedObservation {
+	summary: string;
+	patterns: string[];
+	decisions: string[];
+	filesAffected: string[];
+	relevanceScore: number;
+	timestamp: number;
+	sessionId: string;
+}
+
+export interface ObservationStoreConfig {
+	maxObservations: number;
+	maxCompressed: number;
+	privacyTags: string[];  // tags to strip before storage
+}
+
+const DEFAULT_CONFIG: ObservationStoreConfig = {
+	maxObservations: 1000,
+	maxCompressed: 200,
+	privacyTags: ["<private>", "<secret>", "<credentials>"],
+};
+
+// ── Privacy ──────────────────────────────────────────────────────────────
+
+/**
+ * Strip privacy-tagged content from a string.
+ */
+export function stripPrivacyTags(content: string, config = DEFAULT_CONFIG): string {
+	let result = content;
+	for (const tag of config.privacyTags) {
+		const openTag = tag;
+		const closeTag = tag.replace("<", "</");
+		// Remove everything between open and close tags
+		const regex = new RegExp(`${escapeRegex(openTag)}[\\s\\S]*?${escapeRegex(closeTag)}`, "gi");
+		result = result.replace(regex, "[REDACTED]");
+	}
+	return result;
+}
+
+function escapeRegex(str: string): string {
+	return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// ── Observation Store ────────────────────────────────────────────────────
+
+export class ObservationStore {
+	private observations: Observation[] = [];
+	private compressed: CompressedObservation[] = [];
+	private config: ObservationStoreConfig;
+	private storePath: string;
+
+	constructor(storePath: string, config: Partial<ObservationStoreConfig> = {}) {
+		this.config = { ...DEFAULT_CONFIG, ...config };
+		this.storePath = storePath;
+		if (existsSync(storePath)) {
+			this.load();
+		}
+	}
+
+	/**
+	 * Record a new observation.
+	 */
+	record(observation: Observation): void {
+		// Strip privacy
+		const sanitized: Observation = {
+			...observation,
+			input: stripPrivacyTags(observation.input, this.config),
+			output: stripPrivacyTags(observation.output, this.config),
+		};
+
+		this.observations.push(sanitized);
+
+		// Enforce capacity
+		if (this.observations.length > this.config.maxObservations) {
+			this.observations = this.observations.slice(-this.config.maxObservations);
+		}
+	}
+
+	/**
+	 * Get recent observations.
+	 */
+	getRecent(count = 10): Observation[] {
+		return this.observations.slice(-count);
+	}
+
+	/**
+	 * Store a compressed observation.
+	 */
+	addCompressed(compressed: CompressedObservation): void {
+		this.compressed.push(compressed);
+
+		if (this.compressed.length > this.config.maxCompressed) {
+			this.compressed = this.compressed.slice(-this.config.maxCompressed);
+		}
+	}
+
+	/**
+	 * Get compressed observations for injection.
+	 */
+	getCompressed(limit = 5): CompressedObservation[] {
+		return this.compressed
+			.sort((a, b) => b.relevanceScore - a.relevanceScore)
+			.slice(0, limit);
+	}
+
+	/**
+	 * Format compressed observations for prompt injection.
+	 */
+	injectCompressed(limit = 5): string {
+		const items = this.getCompressed(limit);
+		if (items.length === 0) return "";
+
+		return "## Observations from Previous Sessions\n\n" +
+			items.map((o) =>
+				`### ${o.summary}\n` +
+				`Patterns: ${o.patterns.join(", ")}\n` +
+				`Decisions: ${o.decisions.join(", ")}\n` +
+				`Files: ${o.filesAffected.join(", ")}`,
+			).join("\n\n") +
+			"\n";
+	}
+
+	/**
+	 * Persist to disk.
+	 */
+	save(): void {
+		try {
+			mkdirSync(this.storePath.substring(0, this.storePath.lastIndexOf("/")), { recursive: true });
+			writeFileSync(this.storePath, JSON.stringify({
+				observations: this.observations,
+				compressed: this.compressed,
+			}, null, 2), "utf-8");
+		} catch (error) {
+			logInternalError("observation-store.save", error, `path=${this.storePath}`);
+		}
+	}
+
+	get stats(): { observations: number; compressed: number } {
+		return { observations: this.observations.length, compressed: this.compressed.length };
+	}
+
+	private load(): void {
+		try {
+			const data = JSON.parse(readFileSync(this.storePath, "utf-8"));
+			if (Array.isArray(data.observations)) this.observations = data.observations;
+			if (Array.isArray(data.compressed)) this.compressed = data.compressed;
+		} catch (error) {
+			logInternalError("observation-store.load", error, `path=${this.storePath}`);
+		}
+	}
+}

package/src/utils/fingerprint.ts

@@ -0,0 +1,183 @@
+/**
+ * Incremental fingerprinting — detect file changes to skip unchanged work.
+ *
+ * Pattern origin: Understand-Anything/packages/core/src/fingerprint.ts
+ * Content hash + structural signature per file. Change classifier:
+ * NONE (unchanged), COSMETIC (whitespace/comments only), STRUCTURAL (code changed).
+ * Only STRUCTURAL changes trigger re-analysis.
+ */
+
+import { createHash, type Hash } from "node:crypto";
+import { readFileSync, writeFileSync, existsSync, statSync } from "node:fs";
+import { logInternalError } from "../utils/internal-error.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export type ChangeClass = "NONE" | "COSMETIC" | "STRUCTURAL";
+
+export interface FileFingerprint {
+	path: string;
+	contentHash: string;
+	structuralSignature: string;
+	lastModified: number; // mtime ms
+	changeClass: ChangeClass;
+}
+
+export interface FingerprintDelta {
+	added: string[];
+	removed: string[];
+	modified: FileFingerprint[]; // only STRUCTURAL changes
+	unchanged: number;
+}
+
+// ── Fingerprinting ───────────────────────────────────────────────────────
+
+/**
+ * Compute SHA-256 content hash of a file.
+ */
+export function computeContentHash(filePath: string): string {
+	try {
+		const content = readFileSync(filePath);
+		return createHash("sha256").update(content).digest("hex");
+	} catch {
+		return "";
+	}
+}
+
+/**
+ * Extract a structural signature from source code.
+ *
+ * Captures function signatures, class methods, and import specifiers.
+ * Ignores whitespace, comments, and string content.
+ * Returns a hash of the structural elements.
+ */
+export function computeStructuralSignature(content: string, filePath: string): string {
+	const lines = content.split("\n");
+	const structural: string[] = [];
+
+	for (const line of lines) {
+		const trimmed = line.trim();
+
+		// Skip empty lines and comments
+		if (trimmed.length === 0) continue;
+		if (trimmed.startsWith("//") || trimmed.startsWith("/*") || trimmed.startsWith("*")) continue;
+
+		// Capture structural lines
+		if (
+			// Function/method declarations
+			/^(export\s+)?(async\s+)?function\s/.test(trimmed) ||
+			/^\w+\s*\(/.test(trimmed) && !/^(if|for|while|switch|return|throw|await)/.test(trimmed) ||
+			// Class declarations
+			/^(export\s+)?(abstract\s+)?class\s/.test(trimmed) ||
+			/^(public|private|protected|static|readonly|abstract)\s/.test(trimmed) ||
+			// Interface/type declarations
+			/^(export\s+)?(interface|type)\s/.test(trimmed) ||
+			// Import declarations
+			/^import\s/.test(trimmed) ||
+			// Export declarations
+			/^export\s+(default\s+)?(const|let|var|function|class|interface|type|enum)\s/.test(trimmed)
+		) {
+			structural.push(trimmed);
+		}
+	}
+
+	return createHash("sha256").update(structural.join("\n")).digest("hex");
+}
+
+/**
+ * Classify the change between two fingerprints.
+ */
+export function classifyChange(previous: FileFingerprint | undefined, current: FileFingerprint): ChangeClass {
+	if (!previous) return "STRUCTURAL"; // New file
+
+	if (previous.contentHash === current.contentHash) return "NONE";
+	if (previous.structuralSignature === current.structuralSignature) return "COSMETIC";
+	return "STRUCTURAL";
+}
+
+/**
+ * Compute fingerprint for a single file.
+ */
+export function fingerprintFile(filePath: string): FileFingerprint {
+	const content = readFileSync(filePath, "utf-8");
+	const stat = statSync(filePath);
+
+	return {
+		path: filePath,
+		contentHash: computeContentHash(filePath),
+		structuralSignature: computeStructuralSignature(content, filePath),
+		lastModified: stat.mtimeMs,
+		changeClass: "NONE", // will be classified during delta computation
+	};
+}
+
+// ── Fingerprint Store ────────────────────────────────────────────────────
+
+const MAX_FINGERPRINTS = 10000;
+
+/**
+ * Load fingerprint baseline from a JSON file.
+ */
+export function loadFingerprintBaseline(storePath: string): Map<string, FileFingerprint> {
+	const map = new Map<string, FileFingerprint>();
+	if (!existsSync(storePath)) return map;
+
+	try {
+		const data = JSON.parse(readFileSync(storePath, "utf-8")) as FileFingerprint[];
+		for (const fp of data) {
+			if (map.size >= MAX_FINGERPRINTS) break;
+			map.set(fp.path, fp);
+		}
+	} catch (error) {
+		logInternalError("fingerprint.load", error, `storePath=${storePath}`);
+	}
+	return map;
+}
+
+/**
+ * Save fingerprint baseline to a JSON file.
+ */
+export function saveFingerprintBaseline(storePath: string, fingerprints: Map<string, FileFingerprint>): void {
+	const entries = [...fingerprints.entries()].slice(0, MAX_FINGERPRINTS).map(([, fp]) => fp);
+	writeFileSync(storePath, JSON.stringify(entries, null, 2), "utf-8");
+}
+
+/**
+ * Compute delta between baseline and current fingerprints.
+ *
+ * Only returns STRUCTURAL changes in the `modified` field.
+ */
+export function computeFingerprintDelta(
+	baseline: Map<string, FileFingerprint>,
+	current: Map<string, FileFingerprint>,
+): FingerprintDelta {
+	const added: string[] = [];
+	const removed: string[] = [];
+	const modified: FileFingerprint[] = [];
+	let unchanged = 0;
+
+	// Find added and modified
+	for (const [path, currentFp] of current) {
+		const prev = baseline.get(path);
+		const changeClass = classifyChange(prev, currentFp);
+
+		if (!prev) {
+			added.push(path);
+		} else if (changeClass === "STRUCTURAL") {
+			modified.push({ ...currentFp, changeClass: "STRUCTURAL" });
+		} else if (changeClass === "COSMETIC") {
+			unchanged++; // cosmetic = treated as unchanged for re-analysis
+		} else {
+			unchanged++;
+		}
+	}
+
+	// Find removed
+	for (const path of baseline.keys()) {
+		if (!current.has(path)) {
+			removed.push(path);
+		}
+	}
+
+	return { added, removed, modified, unchanged };
+}

package/src/workflows/discover-workflows.ts

@@ -11,7 +11,7 @@ export interface WorkflowDiscoveryResult {
 	project: WorkflowConfig[];
 }
 
-const STEP_CONFIG_KEYS = new Set(["role", "dependsOn", "parallelGroup", "output", "reads", "model", "skills", "progress", "worktree", "verify", "task"]);
+const STEP_CONFIG_KEYS = new Set(["role", "dependsOn", "parallelGroup", "output", "reads", "model", "skills", "progress", "worktree", "verify", "task", "seedPaths", "preStepScript", "preStepArgs", "preStepTimeout"]);
 
 function parseStepSection(id: string, body: string): WorkflowStep | undefined {
 	const lines = body.trim().split("\n");
@@ -50,6 +50,10 @@ function parseStepSection(id: string, body: string): WorkflowStep | undefined {
 		progress: config.progress === "true" ? true : config.progress === "false" ? false : undefined,
 		worktree: config.worktree === "true" ? true : config.worktree === "false" ? false : undefined,
 		verify: config.verify === "true" ? true : config.verify === "false" ? false : undefined,
+		seedPaths: parseCsv(config.seedPaths) || undefined,
+		preStepScript: config.preStepScript || undefined,
+		preStepArgs: parseCsv(config.preStepArgs) || undefined,
+		preStepTimeout: parseOptionalInteger(config.preStepTimeout) ?? undefined,
 	};
 }
 

package/src/workflows/intermediate-store.ts

@@ -0,0 +1,173 @@
+/**
+ * Phase-gated intermediate store — persist workflow step outputs to disk.
+ *
+ * Pattern origin: Understand-Anything 7-phase pipeline where each phase
+ * writes structured JSON to intermediate/ directory. Phase N reads Phase N-1
+ * output from disk (not context). Enables:
+ * - Context isolation between steps
+ * - Incremental re-runs (skip completed phases)
+ * - Debugging (inspect intermediate outputs)
+ */
+
+import { mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync, unlinkSync } from "node:fs";
+import path from "node:path";
+import { logInternalError } from "../utils/internal-error.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export interface IntermediateOutput {
+	phase: string;
+	stepId: string;
+	timestamp: string;
+	data: unknown;
+}
+
+export interface IntermediateStoreConfig {
+	/** Root directory for intermediates (e.g., ".crew/intermediate/") */
+	intermediateDir: string;
+	/** File patterns to preserve across runs (e.g., ["scan-result.json"]) */
+	preservePatterns: string[];
+}
+
+// ── Store Operations ─────────────────────────────────────────────────────
+
+const DEFAULT_CONFIG: IntermediateStoreConfig = {
+	intermediateDir: ".crew/intermediate",
+	preservePatterns: [],
+};
+
+/**
+ * Ensure the intermediate directory exists.
+ */
+export function ensureIntermediateDir(config: Partial<IntermediateStoreConfig> = {}): string {
+	const dir = config.intermediateDir ?? DEFAULT_CONFIG.intermediateDir;
+	mkdirSync(dir, { recursive: true });
+	return dir;
+}
+
+/**
+ * Write an intermediate output for a phase.
+ *
+ * @param config - Store configuration
+ * @param phase - Phase name (e.g., "explore", "analyze")
+ * @param stepId - Step ID for correlation
+ * @param data - Phase output data
+ * @returns Path to the written file
+ */
+export function writeIntermediate(
+	config: Partial<IntermediateStoreConfig>,
+	phase: string,
+	stepId: string,
+	data: unknown,
+): string {
+	const dir = ensureIntermediateDir(config);
+	const filename = `${phase}-${stepId}.json`;
+	const filePath = path.join(dir, filename);
+
+	const output: IntermediateOutput = {
+		phase,
+		stepId,
+		timestamp: new Date().toISOString(),
+		data,
+	};
+
+	writeFileSync(filePath, JSON.stringify(output, null, 2), "utf-8");
+	return filePath;
+}
+
+/**
+ * Read an intermediate output for a phase.
+ *
+ * @param config - Store configuration
+ * @param phase - Phase name
+ * @param stepId - Step ID
+ * @returns Parsed intermediate output, or undefined if not found
+ */
+export function readIntermediate(
+	config: Partial<IntermediateStoreConfig>,
+	phase: string,
+	stepId: string,
+): IntermediateOutput | undefined {
+	const dir = config.intermediateDir ?? DEFAULT_CONFIG.intermediateDir;
+	const filename = `${phase}-${stepId}.json`;
+	const filePath = path.join(dir, filename);
+
+	if (!existsSync(filePath)) return undefined;
+
+	try {
+		const content = readFileSync(filePath, "utf-8");
+		return JSON.parse(content) as IntermediateOutput;
+	} catch (error) {
+		logInternalError("intermediate-store.read", error, `filePath=${filePath}`);
+		return undefined;
+	}
+}
+
+/**
+ * Read the most recent intermediate output for any phase.
+ *
+ * Useful when you don't know the exact stepId but want the latest
+ * output from a phase (e.g., for incremental re-runs).
+ */
+export function readLatestIntermediate(
+	config: Partial<IntermediateStoreConfig>,
+	phase: string,
+): IntermediateOutput | undefined {
+	const dir = config.intermediateDir ?? DEFAULT_CONFIG.intermediateDir;
+	if (!existsSync(dir)) return undefined;
+
+	const files = readdirSync(dir)
+		.filter((f) => f.startsWith(`${phase}-`) && f.endsWith(".json"))
+		.sort()
+		.reverse(); // most recent first
+
+	if (files.length === 0) return undefined;
+
+	try {
+		const content = readFileSync(path.join(dir, files[0]!), "utf-8");
+		return JSON.parse(content) as IntermediateOutput;
+	} catch {
+		return undefined;
+	}
+}
+
+/**
+ * Clean up intermediate files, preserving specified patterns.
+ *
+ * @param config - Store configuration
+ * @returns Number of files removed
+ */
+export function cleanupIntermediates(config: Partial<IntermediateStoreConfig> = {}): number {
+	const dir = config.intermediateDir ?? DEFAULT_CONFIG.intermediateDir;
+	const preserve = config.preservePatterns ?? DEFAULT_CONFIG.preservePatterns;
+
+	if (!existsSync(dir)) return 0;
+
+	const files = readdirSync(dir);
+	let removed = 0;
+
+	for (const file of files) {
+		const shouldPreserve = preserve.some((pattern) => file.includes(pattern));
+		if (!shouldPreserve) {
+			try {
+				unlinkSync(path.join(dir, file));
+				removed++;
+			} catch (error) {
+				logInternalError("intermediate-store.cleanup", error, `file=${file}`);
+			}
+		}
+	}
+
+	return removed;
+}
+
+/**
+ * Check if a phase has completed (intermediate exists).
+ */
+export function hasPhaseCompleted(
+	config: Partial<IntermediateStoreConfig>,
+	phase: string,
+	stepId: string,
+): boolean {
+	return readIntermediate(config, phase, stepId) !== undefined;
+}

package/src/workflows/workflow-config.ts

@@ -17,6 +17,14 @@ export interface WorkflowStep {
 	/** Per-step files to overlay into the worktree (in addition to global worktree.seedPaths).
 	 * Useful when only certain steps need access to local drafts or scripts. */
 	seedPaths?: string[];
+	/** Path to a deterministic script to run before dispatching the LLM worker.
+	 * Script stdout is injected into the worker's prompt as context.
+	 * Pattern origin: Understand-Anything deterministic pre-step pattern. */
+	preStepScript?: string;
+	/** Arguments for preStepScript. Passed as positional args. */
+	preStepArgs?: string[];
+	/** Timeout in ms for preStepScript. Default: 30000. */
+	preStepTimeout?: number;
 }
 
 export interface WorkflowConfig {