pi-crew 0.9.53 → 0.9.55

package/src/state/observation-store.ts

@@ -0,0 +1,176 @@
+import { existsSync, readFileSync } from "node:fs";
+import { logInternalError } from "../utils/internal-error.ts";
+import { atomicWriteJson } from "./atomic-write.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export interface Observation {
+	tool: string;
+	input: string;
+	output: string;
+	filesRead: string[];
+	filesModified: string[];
+	timestamp: number;
+	sessionId: string;
+	taskId?: string;
+}
+
+export interface CompressedObservation {
+	summary: string;
+	patterns: string[];
+	decisions: string[];
+	filesAffected: string[];
+	relevanceScore: number;
+	timestamp: number;
+	sessionId: string;
+}
+
+export interface ObservationStoreConfig {
+	maxObservations: number;
+	maxCompressed: number;
+	privacyTags: string[]; // tags to strip before storage
+}
+
+const DEFAULT_CONFIG: ObservationStoreConfig = {
+	maxObservations: 1000,
+	maxCompressed: 200,
+	privacyTags: ["<private>", "<secret>", "<credentials>"],
+};
+
+// ── Privacy ──────────────────────────────────────────────────────────────
+
+/**
+ * Strip privacy-tagged content from a string.
+ */
+export function stripPrivacyTags(content: string, config = DEFAULT_CONFIG): string {
+	let result = content;
+	for (const tag of config.privacyTags) {
+		const openTag = tag;
+		const closeTag = tag.replace("<", "</");
+		// Remove everything between open and close tags
+		const regex = new RegExp(`${escapeRegex(openTag)}[\\s\\S]*?${escapeRegex(closeTag)}`, "gi");
+		result = result.replace(regex, "[REDACTED]");
+	}
+	return result;
+}
+
+function escapeRegex(str: string): string {
+	return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// ── Observation Store ────────────────────────────────────────────────────
+
+export class ObservationStore {
+	private observations: Observation[] = [];
+	private compressed: CompressedObservation[] = [];
+	private config: ObservationStoreConfig;
+	private storePath: string;
+
+	constructor(storePath: string, config: Partial<ObservationStoreConfig> = {}) {
+		this.config = { ...DEFAULT_CONFIG, ...config };
+		this.storePath = storePath;
+		if (existsSync(storePath)) {
+			this.load();
+		}
+	}
+
+	/**
+	 * Record a new observation.
+	 */
+	record(observation: Observation): void {
+		// Strip privacy
+		const sanitized: Observation = {
+			...observation,
+			input: stripPrivacyTags(observation.input, this.config),
+			output: stripPrivacyTags(observation.output, this.config),
+		};
+
+		this.observations.push(sanitized);
+
+		// Enforce capacity
+		if (this.observations.length > this.config.maxObservations) {
+			this.observations = this.observations.slice(-this.config.maxObservations);
+		}
+	}
+
+	/**
+	 * Get recent observations.
+	 */
+	getRecent(count = 10): Observation[] {
+		return this.observations.slice(-count);
+	}
+
+	/**
+	 * Store a compressed observation.
+	 */
+	addCompressed(compressed: CompressedObservation): void {
+		this.compressed.push(compressed);
+
+		if (this.compressed.length > this.config.maxCompressed) {
+			this.compressed = this.compressed.slice(-this.config.maxCompressed);
+		}
+	}
+
+	/**
+	 * Get compressed observations for injection.
+	 */
+	getCompressed(limit = 5): CompressedObservation[] {
+		return this.compressed.sort((a, b) => b.relevanceScore - a.relevanceScore).slice(0, limit);
+	}
+
+	/**
+	 * Format compressed observations for prompt injection.
+	 */
+	injectCompressed(limit = 5): string {
+		const items = this.getCompressed(limit);
+		if (items.length === 0) return "";
+
+		return (
+			"## Observations from Previous Sessions\n\n" +
+			items
+				.map(
+					(o) =>
+						`### ${o.summary}\n` +
+						`Patterns: ${o.patterns.join(", ")}\n` +
+						`Decisions: ${o.decisions.join(", ")}\n` +
+						`Files: ${o.filesAffected.join(", ")}`,
+				)
+				.join("\n\n") +
+			"\n"
+		);
+	}
+
+	/**
+	 * Persist to disk.
+	 */
+	save(): void {
+		try {
+			// BUGFIX (Round 12 I4): use atomicWriteJson (temp-file + rename) instead
+			// of raw writeFileSync, so a crash mid-write cannot leave a truncated /
+			// empty file that breaks load() on restart.
+			atomicWriteJson(this.storePath, {
+				observations: this.observations,
+				compressed: this.compressed,
+			});
+		} catch (error) {
+			logInternalError("observation-store.save", error, `path=${this.storePath}`);
+		}
+	}
+
+	get stats(): { observations: number; compressed: number } {
+		return {
+			observations: this.observations.length,
+			compressed: this.compressed.length,
+		};
+	}
+
+	private load(): void {
+		try {
+			const data = JSON.parse(readFileSync(this.storePath, "utf-8"));
+			if (Array.isArray(data.observations)) this.observations = data.observations;
+			if (Array.isArray(data.compressed)) this.compressed = data.compressed;
+		} catch (error) {
+			logInternalError("observation-store.load", error, `path=${this.storePath}`);
+		}
+	}
+}

package/src/state/run-cache.ts

@@ -0,0 +1,193 @@
+import * as crypto from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { projectCrewRoot } from "../utils/paths.ts";
+import { atomicWriteJson } from "./atomic-write.ts";
+import { withFileLockSync } from "./locks.ts";
+import type { TeamTaskState } from "./types.ts";
+
+const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+
+export interface CacheEntry {
+	key: string;
+	runId: string;
+	status: string;
+	tasks: TeamTaskState[];
+	cachedAt: number;
+	expiresAt: number;
+	goal: string;
+	team: string;
+}
+
+interface CacheIndex {
+	[cacheKey: string]: string;
+}
+
+/**
+ * Compute a cache key from run parameters.
+ * Uses SHA-256 hash of normalized goal + team + workflow.
+ */
+export function computeRunCacheKey(goal: string, team: string, workflow: string, _cwd: string): string {
+	const normalized = goal.trim().toLowerCase().replace(/\s+/g, " ");
+	return crypto.createHash("sha256").update(normalized).update(team).update(workflow).update(_cwd).digest("hex").slice(0, 16);
+}
+
+/**
+ * Get the cache directory path.
+ */
+function cacheDir(cwd: string): string {
+	return path.join(projectCrewRoot(cwd), "cache");
+}
+
+/**
+ * Get cached run result if exists and valid.
+ * Returns null if cache miss or expired.
+ */
+export function getCachedRun(cwd: string, cacheKey: string): CacheEntry | null {
+	const dir = cacheDir(cwd);
+	const indexPath = path.join(dir, "index.json");
+
+	if (!fs.existsSync(indexPath)) return null;
+
+	try {
+		const index = JSON.parse(fs.readFileSync(indexPath, "utf-8")) as CacheIndex;
+		const entryPath = index[cacheKey];
+
+		if (!entryPath || !fs.existsSync(entryPath)) return null;
+
+		const entry = JSON.parse(fs.readFileSync(entryPath, "utf-8")) as CacheEntry;
+
+		if (Date.now() > entry.expiresAt) {
+			// Remove expired entry — use lock + atomic write to prevent index corruption
+			withFileLockSync(indexPath, () => {
+				try {
+					fs.unlinkSync(entryPath);
+				} catch {
+					/* ignore */
+				}
+				const updatedIndex = JSON.parse(fs.readFileSync(indexPath, "utf-8")) as CacheIndex;
+				delete updatedIndex[cacheKey];
+				atomicWriteJson(indexPath, updatedIndex);
+			});
+			return null;
+		}
+
+		return entry;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Save run result to cache.
+ */
+export function saveRunToCache(
+	cwd: string,
+	cacheKey: string,
+	runId: string,
+	status: string,
+	tasks: TeamTaskState[],
+	goal: string,
+	team: string,
+	ttlMs: number = DEFAULT_CACHE_TTL_MS,
+): void {
+	const dir = cacheDir(cwd);
+
+	if (!fs.existsSync(dir)) {
+		fs.mkdirSync(dir, { recursive: true });
+	}
+
+	const entry: CacheEntry = {
+		key: cacheKey,
+		runId,
+		status,
+		tasks,
+		cachedAt: Date.now(),
+		expiresAt: Date.now() + ttlMs,
+		goal,
+		team,
+	};
+
+	const entryPath = path.join(dir, `${cacheKey}.json`);
+	// ST-4: atomic entry write (was raw writeFileSync — inverted priority vs the index).
+	atomicWriteJson(entryPath, entry);
+
+	// Update index under a file lock (prevents concurrent saveCache from racing on the
+	// RMW) and write atomically (unique temp + rename — fixes the fixed "index.json.tmp"
+	// name collision the old manual temp+rename had).
+	const indexPath = path.join(dir, "index.json");
+	withFileLockSync(indexPath, () => {
+		const index: CacheIndex = fs.existsSync(indexPath) ? JSON.parse(fs.readFileSync(indexPath, "utf-8")) : {};
+		index[cacheKey] = entryPath;
+		atomicWriteJson(indexPath, index);
+	});
+}
+
+/**
+ * Clear all cache entries.
+ */
+export function clearCache(cwd: string): void {
+	const dir = cacheDir(cwd);
+	if (!fs.existsSync(dir)) return;
+
+	const indexPath = path.join(dir, "index.json");
+	if (fs.existsSync(indexPath)) {
+		try {
+			const index = JSON.parse(fs.readFileSync(indexPath, "utf-8")) as CacheIndex;
+			for (const entryPath of Object.values(index)) {
+				try {
+					fs.unlinkSync(entryPath);
+				} catch {
+					/* ignore */
+				}
+			}
+			fs.unlinkSync(indexPath);
+		} catch {
+			/* ignore */
+		}
+	}
+
+	// Remove entry files not in index
+	const entries = fs.readdirSync(dir).filter((f) => f.endsWith(".json"));
+	for (const entry of entries) {
+		try {
+			fs.unlinkSync(path.join(dir, entry));
+		} catch {
+			/* ignore */
+		}
+	}
+}
+
+/**
+ * Get cache stats.
+ */
+export function getCacheStats(cwd: string): {
+	entries: number;
+	sizeBytes: number;
+} {
+	const dir = cacheDir(cwd);
+	if (!fs.existsSync(dir)) return { entries: 0, sizeBytes: 0 };
+
+	let sizeBytes = 0;
+	let entries = 0;
+	const indexPath = path.join(dir, "index.json");
+
+	if (fs.existsSync(indexPath)) {
+		try {
+			const index = JSON.parse(fs.readFileSync(indexPath, "utf-8")) as CacheIndex;
+			entries = Object.keys(index).length;
+			for (const entryPath of Object.values(index)) {
+				try {
+					const stat = fs.statSync(entryPath);
+					sizeBytes += stat.size;
+				} catch {
+					/* ignore */
+				}
+			}
+		} catch {
+			/* ignore */
+		}
+	}
+
+	return { entries, sizeBytes };
+}

package/src/state/run-graph.ts

@@ -0,0 +1,183 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { projectCrewRoot } from "../utils/paths.ts";
+import { assertSafePathId } from "../utils/safe-paths.ts";
+import { atomicWriteFile } from "./atomic-write.ts";
+import type { TeamRunManifest, TeamTaskState } from "./types.ts";
+
+export interface RunGraphNode {
+	id: string;
+	type: "run" | "task" | "agent" | "artifact" | "file";
+	name: string;
+	metadata?: Record<string, unknown>;
+}
+
+export interface RunGraphEdge {
+	source: string;
+	target: string;
+	type: "dependsOn" | "produces" | "runs" | "contains";
+	weight?: number;
+}
+
+export interface RunGraphLayer {
+	name: string;
+	nodeIds: string[];
+}
+
+export interface RunGraph {
+	version: "1.0.0";
+	runId: string;
+	team: string;
+	workflow: string;
+	createdAt: string;
+	completedAt?: string;
+	status: string;
+	nodes: RunGraphNode[];
+	edges: RunGraphEdge[];
+	layers: RunGraphLayer[];
+}
+
+/**
+ * Build a unified run graph from manifest + tasks.
+ * Consolidates state into a single graph JSON for dashboard/API use.
+ */
+export function buildRunGraph(manifest: TeamRunManifest, tasks: TeamTaskState[]): RunGraph {
+	const nodes: RunGraphNode[] = [];
+	const edges: RunGraphEdge[] = [];
+	const nodeIds = new Set<string>();
+
+	// Add run node
+	const runId = manifest.runId;
+	nodes.push({
+		id: `run:${runId}`,
+		type: "run",
+		name: manifest.goal ?? runId,
+		metadata: {
+			team: manifest.team,
+			workflow: manifest.workflow,
+			status: manifest.status,
+			createdAt: manifest.createdAt,
+		},
+	});
+	nodeIds.add(`run:${runId}`);
+
+	// Add task nodes
+	for (const task of tasks) {
+		const taskId = `task:${task.id}`;
+		if (nodeIds.has(taskId)) continue;
+		nodeIds.add(taskId);
+
+		nodes.push({
+			id: taskId,
+			type: "task",
+			name: task.role,
+			metadata: {
+				status: task.status,
+				startedAt: task.startedAt,
+				finishedAt: task.finishedAt,
+			},
+		});
+
+		// Edge from run to task
+		edges.push({
+			source: `run:${runId}`,
+			target: taskId,
+			type: "contains",
+		});
+
+		// Edges from dependencies
+		for (const dep of task.dependsOn ?? []) {
+			edges.push({
+				source: `task:${dep}`,
+				target: taskId,
+				type: "dependsOn",
+				weight: 1.0,
+			});
+		}
+	}
+
+	// Group by layer (based on phase or role)
+	const layerMap = new Map<string, string[]>();
+	for (const task of tasks) {
+		const layerName = task.adaptive?.phase ?? task.role;
+		if (!layerMap.has(layerName)) layerMap.set(layerName, []);
+		layerMap.get(layerName)!.push(`task:${task.id}`);
+	}
+
+	const layers: RunGraphLayer[] = [...layerMap.entries()].map(([name, nodeIdList]) => ({
+		name,
+		nodeIds: nodeIdList,
+	}));
+
+	return {
+		version: "1.0.0",
+		runId,
+		team: manifest.team ?? "unknown",
+		workflow: manifest.workflow ?? "unknown",
+		createdAt: manifest.createdAt,
+		completedAt: manifest.updatedAt,
+		status: manifest.status,
+		nodes,
+		edges,
+		layers,
+	};
+}
+
+/**
+ * Save run graph to disk.
+ * Uses projectCrewRoot() to honour the .pi/teams/ fallback for .pi-based
+ * projects (issue #29 follow-up). Graph files now live alongside the
+ * manifest at `<crewRoot>/graphs/<runId>.json`.
+ */
+export function saveRunGraph(graph: RunGraph, cwd: string): string {
+	assertSafePathId("runId", graph.runId);
+	const crewRoot = projectCrewRoot(cwd);
+	const graphsDir = path.join(crewRoot, "graphs");
+
+	if (!fs.existsSync(graphsDir)) {
+		fs.mkdirSync(graphsDir, { recursive: true });
+	}
+
+	const graphPath = path.join(graphsDir, `${graph.runId}.json`);
+	atomicWriteFile(graphPath, JSON.stringify(graph, null, 2));
+
+	return graphPath;
+}
+
+/**
+ * Load run graph from disk.
+ */
+export function loadRunGraph(cwd: string, runId: string): RunGraph | null {
+	assertSafePathId("runId", runId);
+	const crewRoot = projectCrewRoot(cwd);
+	const graphPath = path.join(crewRoot, "graphs", `${runId}.json`);
+	if (!fs.existsSync(graphPath)) return null;
+
+	try {
+		return JSON.parse(fs.readFileSync(graphPath, "utf-8")) as RunGraph;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * List all archived run graphs.
+ */
+export function listRunGraphs(cwd: string): string[] {
+	const crewRoot = projectCrewRoot(cwd);
+	const graphsDir = path.join(crewRoot, "graphs");
+	if (!fs.existsSync(graphsDir)) return [];
+
+	return fs
+		.readdirSync(graphsDir)
+		.filter((f) => f.endsWith(".json"))
+		.map((f) => f.replace(/\.json$/, ""));
+}
+
+/**
+ * Build and save run graph from manifest + tasks.
+ */
+export function buildAndSaveRunGraph(manifest: TeamRunManifest, tasks: TeamTaskState[], cwd: string): string {
+	const graph = buildRunGraph(manifest, tasks);
+	return saveRunGraph(graph, cwd);
+}

package/src/state/run-metrics.ts

@@ -0,0 +1,165 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { projectCrewRoot } from "../utils/paths.ts";
+import { assertSafePathId } from "../utils/safe-paths.ts";
+import { atomicWriteJson, readJsonFile } from "./atomic-write.ts";
+import { loadRunManifestById } from "./state-store.ts";
+
+/**
+ * Run metrics snapshot captured after a run completes (or on demand).
+ */
+export interface RunMetrics {
+	runId: string;
+	timestamp: string;
+	taskCount: number;
+	completedCount: number;
+	failedCount: number;
+	totalTokens: number;
+	totalCost: number;
+	durationMs: number;
+	consistencyScore: number;
+}
+
+/** Number of recent metric files to scan when building a summary. */
+const MAX_METRIC_FILES_TO_SCAN = 500;
+
+function metricsDir(cwd: string): string {
+	const repoRoot = projectCrewRoot(cwd);
+	return path.join(repoRoot, "state", "metrics");
+}
+
+function metricsFilePath(cwd: string, runId: string): string {
+	assertSafePathId("runId", runId);
+	return path.join(metricsDir(cwd), `${runId}.json`);
+}
+
+/**
+ * Collect metrics for a run by reading its manifest, tasks, and event log.
+ * Returns undefined if the run cannot be loaded.
+ */
+export function collectRunMetrics(cwd: string, runId: string): RunMetrics | undefined {
+	assertSafePathId("runId", runId);
+	const result = loadRunManifestById(cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency;
+	if (!result) return undefined;
+
+	const { manifest, tasks } = result;
+	const now = new Date().toISOString();
+
+	// Aggregate token/cost from all tasks that have usage data.
+	let totalTokens = 0;
+	let totalCost = 0;
+	for (const task of tasks) {
+		if (task.usage) {
+			totalTokens += (task.usage.input ?? 0) + (task.usage.output ?? 0);
+			totalCost += task.usage.cost ?? 0;
+		}
+	}
+
+	// Count completed vs failed tasks.
+	let completedCount = 0;
+	let failedCount = 0;
+	for (const task of tasks) {
+		if (task.status === "completed") completedCount++;
+		else if (task.status === "failed") failedCount++;
+	}
+
+	// Duration: from run createdAt to updatedAt (manifest timestamps), or 0 if unavailable.
+	const createdAt = new Date(manifest.createdAt).getTime();
+	const updatedAt = new Date(manifest.updatedAt).getTime();
+	const durationMs = Number.isNaN(createdAt) || Number.isNaN(updatedAt) ? 0 : Math.max(0, updatedAt - createdAt);
+
+	// Consistency score: proportion of tasks that completed successfully among all non-skipped tasks.
+	const nonSkippedTasks = tasks.filter((t) => t.status !== "skipped");
+	const consistencyScore = nonSkippedTasks.length > 0 ? completedCount / nonSkippedTasks.length : 1.0;
+
+	return {
+		runId,
+		timestamp: now,
+		taskCount: tasks.length,
+		completedCount,
+		failedCount,
+		totalTokens,
+		totalCost,
+		durationMs,
+		consistencyScore: Math.round(consistencyScore * 1000) / 1000, // 3 decimal places
+	};
+}
+
+/**
+ * Persist a metrics snapshot to .crew/state/metrics/<runId>.json.
+ * Uses atomicWriteJson to ensure safe writes.
+ */
+export function saveRunMetrics(cwd: string, metrics: RunMetrics): void {
+	const dir = metricsDir(cwd);
+	fs.mkdirSync(dir, { recursive: true });
+	atomicWriteJson(metricsFilePath(cwd, metrics.runId), metrics);
+}
+
+/**
+ * Load a previously saved metrics snapshot.
+ * Returns undefined if the file does not exist or cannot be parsed.
+ */
+export function loadRunMetrics(cwd: string, runId: string): RunMetrics | undefined {
+	return readJsonFile<RunMetrics>(metricsFilePath(cwd, runId));
+}
+
+/**
+ * List recent metrics files up to `limit` entries (newest first).
+ * Returns an array of { runId, timestamp, taskCount, completedCount, failedCount, totalTokens, totalCost, durationMs, consistencyScore }.
+ * Gracefully skips files that cannot be read or parsed.
+ *
+ * FIND-04 perf: sort dirents by mtime descending BEFORE reading any of
+ * them, then read at most `limit` files. Malformed files within that
+ * selected window reduce the returned count; the scan does not continue
+ * past the window to backfill them. Previously the function read up
+ * to MAX_METRIC_FILES_TO_SCAN (500) files via loadRunMetrics() and only
+ * THEN sorted+sliced to `limit` (default 25) — wasting 475 readFileSync +
+ * JSON.parse calls on the hot dashboard path. The sort is O(N log N) and
+ * uses the run-id timestamp prefix as a cheap secondary signal when
+ * mtimes tie (e.g. multiple files saved in the same millisecond).
+ */
+export function getRunMetricsSummary(cwd: string, limit = 25): RunMetrics[] {
+	const dir = metricsDir(cwd);
+	let entries: fs.Dirent[] = [];
+	try {
+		entries = fs.readdirSync(dir, { withFileTypes: true });
+	} catch {
+		return [];
+	}
+
+	// Cap the directory-order candidate set before collecting mtimes. This
+	// is a safety valve against unbounded scan cost; the mtime sort below
+	// only reorders entries within this bounded candidate window.
+	const cap = Math.max(0, limit);
+	const sorted = entries
+		.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+		.slice(0, MAX_METRIC_FILES_TO_SCAN)
+		.map((entry) => {
+			const fullPath = path.join(dir, entry.name);
+			let mtimeMs = 0;
+			try {
+				mtimeMs = fs.statSync(fullPath).mtimeMs;
+			} catch {
+				// stat failure: fall through with mtimeMs=0; the sort still
+				// produces a deterministic order (filenames tiebreak).
+			}
+			return { name: entry.name, mtimeMs };
+		})
+		.sort((a, b) => {
+			const diff = b.mtimeMs - a.mtimeMs;
+			if (diff !== 0) return diff;
+			// Filename tiebreaker: the run-id prefix is a YYYYMMDDhhmmss
+			// timestamp (e.g. team_20260720050617_abc123) so a descending
+			// string compare yields newest-first within a tie group.
+			return b.name.localeCompare(a.name);
+		});
+
+	const metrics: RunMetrics[] = [];
+	for (const { name } of sorted.slice(0, cap)) {
+		const runId = name.replace(/\.json$/, "");
+		const m = loadRunMetrics(cwd, runId);
+		if (m) metrics.push(m);
+	}
+
+	return metrics;
+}