pi-crew 0.9.53 → 0.9.55

package/src/utils/env-filter.ts

@@ -0,0 +1,155 @@
+import { isSecretKey } from "./redaction.ts";
+
+// Well-known LLM provider API keys that are intentionally allowlisted in
+// child-pi.ts and async-runner.ts for pass-through to worker processes.
+// These are "secret" by pattern (contain KEY/API) but are safe to allowlist
+// because they are standard provider credentials, not arbitrary secrets.
+const KNOWN_PROVIDER_KEYS = new Set([
+	"MINIMAX_API_KEY",
+	"MINIMAX_GROUP_ID",
+	"OPENAI_API_KEY",
+	"OPENAI_ORG_ID",
+	"ANTHROPIC_API_KEY",
+	"GOOGLE_API_KEY",
+	"GOOGLE_GENERATIVE_LANGUAGE_API_KEY",
+	"AZURE_OPENAI_API_KEY",
+	"AZURE_OPENAI_ENDPOINT",
+	"AWS_ACCESS_KEY_ID",
+	"AWS_SECRET_ACCESS_KEY",
+	"AWS_REGION",
+	"ZEU_API_KEY",
+	"ZERODEV_API_KEY",
+]);
+
+// Map from provider prefix (lowercase) to env var names.
+// Used by providerEnvKeys() to scope API keys per task.
+const PROVIDER_ENV_KEY_MAP: Record<string, string[]> = {
+	minimax: ["MINIMAX_API_KEY", "MINIMAX_GROUP_ID"],
+	openai: ["OPENAI_API_KEY", "OPENAI_ORG_ID"],
+	anthropic: ["ANTHROPIC_API_KEY"],
+	google: ["GOOGLE_API_KEY", "GOOGLE_GENERATIVE_LANGUAGE_API_KEY"],
+	gemini: ["GOOGLE_API_KEY", "GOOGLE_GENERATIVE_LANGUAGE_API_KEY"],
+	azure: ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT"],
+	"azure-openai": ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT"],
+	aws: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION"],
+	bedrock: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION"],
+	zai: ["ZEU_API_KEY"],
+	zerodev: ["ZERODEV_API_KEY"],
+};
+
+function isKnownProviderKey(key: string): boolean {
+	return KNOWN_PROVIDER_KEYS.has(key);
+}
+
+/**
+ * Extract provider prefix from a model ID string and return the corresponding
+ * env var names. Returns empty array for unknown/custom providers or invalid input.
+ *
+ * @param modelId - Model in "provider/modelId" format (e.g. "openai/gpt-4o")
+ * @returns Array of env var names needed for this provider
+ */
+export function providerEnvKeys(modelId: string | undefined): string[] {
+	if (!modelId) return [];
+	const separatorIndex = modelId.indexOf("/");
+	if (separatorIndex <= 0) return [];
+	const provider = modelId.substring(0, separatorIndex).toLowerCase();
+	return PROVIDER_ENV_KEY_MAP[provider] ?? [];
+}
+
+/**
+ * Build a scoped allowlist that includes system vars + only the provider keys
+ * relevant to the assigned model (and its fallback chain).
+ *
+ * @param baseAllowList - Non-provider system vars to always include
+ * @param models - Model IDs to include provider keys for (primary + fallbacks)
+ * @returns Combined allowlist with scoped provider keys
+ */
+export function buildScopedAllowList(baseAllowList: string[], models: (string | undefined)[]): string[] {
+	const providerKeys = new Set<string>();
+	for (const model of models) {
+		for (const key of providerEnvKeys(model)) {
+			providerKeys.add(key);
+		}
+	}
+	return [...baseAllowList, ...providerKeys];
+}
+
+export interface SanitizeEnvOptions {
+	/** Allow-list of env var names to preserve. Supports trailing glob, e.g. `"PI_*"`. */
+	allowList?: string[];
+}
+
+// Keywords that indicate a secret env var
+const SECRET_SUFFIXES = ["token", "api", "key", "password", "passwd", "secret", "credential", "authorization", "private"];
+
+/**
+ * Check if a glob pattern could match secret env vars.
+ * A pattern like "PI_*" is dangerous because it could match PI_TOKEN, PI_API_KEY, etc.
+ *
+ * Exception: `PI_CREW_*` is a controlled namespace — the pi-crew codebase owns
+ * every PI_CREW_* env var (PI_CREW_PARENT_PID, PI_CREW_ADAPTIVE_REPAIR, etc.)
+ * and none of them are secrets. Allowing the glob here lets child Pi processes
+ * inherit our config without needing a per-var allowlist.
+ */
+function isDangerousGlob(pattern: string): boolean {
+	if (!pattern.endsWith("*")) return false;
+	const prefix = pattern.slice(0, -1); // Remove trailing *
+	if (prefix === "") return true; // Single "*" matches everything
+	// PI_CREW_* is the pi-crew controlled namespace — no secrets live here.
+	// This covers PI_CREW_*, PI_CREW_TEAMS_*, PI_CREW_AGENT_*, etc.
+	if (prefix.startsWith("PI_CREW_") || prefix === "PI_CREW") return false;
+	// Check if combining the prefix with any secret suffix would create a secret key
+	for (const suffix of SECRET_SUFFIXES) {
+		if (isSecretKey(prefix + suffix)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+/**
+ * Strip env vars whose keys look like secrets before passing to child processes.
+ *
+ * Default mode (no allowList): deny-list using isSecretKey.
+ * When allowList is provided, only keys matching the allow-list are preserved.
+ */
+export function sanitizeEnvSecrets(env: NodeJS.ProcessEnv, options?: SanitizeEnvOptions): Record<string, string> {
+	const filtered: Record<string, string> = {};
+	if (options?.allowList && options.allowList.length > 0) {
+		// Validate allowlist patterns don't match secrets
+		for (const pattern of options.allowList) {
+			if (isDangerousGlob(pattern)) {
+				throw new Error(`Allowlist pattern "${pattern}" could match secret env vars. Use a more specific pattern.`);
+			}
+			// Validate non-glob entries don't look like secret keys.
+			// Exception 1: if the key exists in env, it was intentionally set and
+			// should be allowed through (user knows what they're doing).
+			// Exception 2: known provider keys (MINIMAX_API_KEY, etc.) are always
+			// allowed because they are standard provider credentials explicitly
+			// listed in async-runner.ts / child-pi.ts allowlists.
+			if (!pattern.endsWith("*") && isSecretKey(pattern) && !(pattern in env) && !isKnownProviderKey(pattern)) {
+				throw new Error(`Allowlist entry "${pattern}" looks like a secret key. Use a more specific pattern.`);
+			}
+		}
+		const matchers = options.allowList.map((p) => {
+			if (p.endsWith("*")) {
+				// Glob pattern: matches keys that start with the prefix AND have
+				// at least one additional character (distinguishes "PI_CREW_*" from "PI_CREW_").
+				// For example, "PI_CREW_*" matches "PI_CREW_DEPTH" but not "PI_CREW_".
+				// This ensures trailing glob patterns require extra chars, not exact-prefix-only matches.
+				const prefix = p.slice(0, -1);
+				return (k: string) => k.startsWith(prefix) && k.length > prefix.length;
+			}
+			// Exact match is case-sensitive; Unix env vars are uppercase by convention.
+			return (k: string) => k === p;
+		});
+		for (const [key, value] of Object.entries(env)) {
+			if (value !== undefined && matchers.some((fn) => fn(key))) filtered[key] = value;
+		}
+		return filtered;
+	}
+	for (const [key, value] of Object.entries(env)) {
+		if (value !== undefined && !isSecretKey(key)) filtered[key] = value;
+	}
+	return filtered;
+}

package/src/utils/file-coalescer.ts

@@ -0,0 +1,98 @@
+import * as fs from "node:fs";
+
+interface TimerApi {
+	setTimeout(handler: () => void, delayMs: number): unknown;
+	clearTimeout(handle: unknown): void;
+}
+
+const defaultTimerApi: TimerApi = {
+	setTimeout: (handler, delayMs) => {
+		const t = setTimeout(handler, delayMs);
+		// Defense in depth: never let a coalescer timer block process exit.
+		// The timer may be cleared before it fires; .unref() is idempotent.
+		if (typeof t === "object" && t && "unref" in t && typeof t.unref === "function") {
+			t.unref();
+		}
+		return t;
+	},
+	clearTimeout: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
+};
+
+export interface FileCoalescer {
+	schedule(file: string, delayMs?: number): boolean;
+	clear(): void;
+}
+
+export function createFileCoalescer(
+	handler: (file: string) => void,
+	defaultDelayMs: number,
+	timerApi: TimerApi = defaultTimerApi,
+): FileCoalescer {
+	const pending = new Map<string, unknown>();
+	return {
+		schedule(file, delayMs = defaultDelayMs) {
+			if (pending.has(file)) return false;
+			const timer = timerApi.setTimeout(() => {
+				pending.delete(file);
+				handler(file);
+			}, delayMs);
+			pending.set(file, timer);
+			return true;
+		},
+		clear() {
+			for (const timer of pending.values()) timerApi.clearTimeout(timer);
+			pending.clear();
+		},
+	};
+}
+
+interface ReadCacheEntry<T> {
+	value: T;
+	mtimeMs: number;
+	size: number;
+	expiresAt: number;
+}
+
+const readCache = new Map<string, ReadCacheEntry<unknown>>();
+const readCacheSizeLimit = 128;
+
+function evictOldestCacheEntry(): void {
+	if (readCache.size < readCacheSizeLimit) return;
+	// Map iteration order is insertion order; first key is LRU.
+	const oldestKey = readCache.keys().next().value;
+	if (oldestKey !== undefined) readCache.delete(oldestKey);
+}
+
+export function clearReadCache(): void {
+	readCache.clear();
+}
+
+export function readJsonFileCoalesced<T>(filePath: string, ttlMs: number, read: () => T): T {
+	const now = Date.now();
+	const stat = (() => {
+		try {
+			const fileStat = fs.statSync(filePath);
+			return { mtimeMs: fileStat.mtimeMs, size: fileStat.size };
+		} catch {
+			return undefined;
+		}
+	})();
+	const cached = readCache.get(filePath);
+	if (cached && stat && cached.expiresAt > now && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) {
+		// Re-insert to implement LRU: move to end of Map.
+		readCache.delete(filePath);
+		readCache.set(filePath, cached);
+		return cached.value as T;
+	}
+	const value = read();
+	if (stat !== undefined) {
+		readCache.set(filePath, {
+			value,
+			mtimeMs: stat.mtimeMs,
+			size: stat.size,
+			expiresAt: now + ttlMs,
+		});
+		evictOldestCacheEntry();
+	}
+	return value;
+}

package/src/utils/fingerprint.ts

@@ -0,0 +1,180 @@
+/**
+ * Incremental fingerprinting — detect file changes to skip unchanged work.
+ *
+ * Pattern origin: Understand-Anything/packages/core/src/fingerprint.ts
+ * Content hash + structural signature per file. Change classifier:
+ * NONE (unchanged), COSMETIC (whitespace/comments only), STRUCTURAL (code changed).
+ * Only STRUCTURAL changes trigger re-analysis.
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { logInternalError } from "../utils/internal-error.ts";
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export type ChangeClass = "NONE" | "COSMETIC" | "STRUCTURAL";
+
+export interface FileFingerprint {
+	path: string;
+	contentHash: string;
+	structuralSignature: string;
+	lastModified: number; // mtime ms
+	changeClass: ChangeClass;
+}
+
+export interface FingerprintDelta {
+	added: string[];
+	removed: string[];
+	modified: FileFingerprint[]; // only STRUCTURAL changes
+	unchanged: number;
+}
+
+// ── Fingerprinting ───────────────────────────────────────────────────────
+
+/**
+ * Compute SHA-256 content hash of a file.
+ */
+export function computeContentHash(filePath: string): string {
+	try {
+		const content = readFileSync(filePath);
+		return createHash("sha256").update(content).digest("hex");
+	} catch {
+		return "";
+	}
+}
+
+/**
+ * Extract a structural signature from source code.
+ *
+ * Captures function signatures, class methods, and import specifiers.
+ * Ignores whitespace, comments, and string content.
+ * Returns a hash of the structural elements.
+ */
+export function computeStructuralSignature(content: string, filePath: string): string {
+	const lines = content.split("\n");
+	const structural: string[] = [];
+
+	for (const line of lines) {
+		const trimmed = line.trim();
+
+		// Skip empty lines and comments
+		if (trimmed.length === 0) continue;
+		if (trimmed.startsWith("//") || trimmed.startsWith("/*") || trimmed.startsWith("*")) continue;
+
+		// Capture structural lines
+		if (
+			// Function/method declarations
+			/^(export\s+)?(async\s+)?function\s/.test(trimmed) ||
+			(/^\w+\s*\(/.test(trimmed) && !/^(if|for|while|switch|return|throw|await)/.test(trimmed)) ||
+			// Class declarations
+			/^(export\s+)?(abstract\s+)?class\s/.test(trimmed) ||
+			/^(public|private|protected|static|readonly|abstract)\s/.test(trimmed) ||
+			// Interface/type declarations
+			/^(export\s+)?(interface|type)\s/.test(trimmed) ||
+			// Import declarations
+			/^import\s/.test(trimmed) ||
+			// Export declarations
+			/^export\s+(default\s+)?(const|let|var|function|class|interface|type|enum)\s/.test(trimmed)
+		) {
+			structural.push(trimmed);
+		}
+	}
+
+	return createHash("sha256").update(structural.join("\n")).digest("hex");
+}
+
+/**
+ * Classify the change between two fingerprints.
+ */
+export function classifyChange(previous: FileFingerprint | undefined, current: FileFingerprint): ChangeClass {
+	if (!previous) return "STRUCTURAL"; // New file
+
+	if (previous.contentHash === current.contentHash) return "NONE";
+	if (previous.structuralSignature === current.structuralSignature) return "COSMETIC";
+	return "STRUCTURAL";
+}
+
+/**
+ * Compute fingerprint for a single file.
+ */
+export function fingerprintFile(filePath: string): FileFingerprint {
+	const content = readFileSync(filePath, "utf-8");
+	const stat = statSync(filePath);
+
+	return {
+		path: filePath,
+		contentHash: computeContentHash(filePath),
+		structuralSignature: computeStructuralSignature(content, filePath),
+		lastModified: stat.mtimeMs,
+		changeClass: "NONE", // will be classified during delta computation
+	};
+}
+
+// ── Fingerprint Store ────────────────────────────────────────────────────
+
+const MAX_FINGERPRINTS = 10000;
+
+/**
+ * Load fingerprint baseline from a JSON file.
+ */
+export function loadFingerprintBaseline(storePath: string): Map<string, FileFingerprint> {
+	const map = new Map<string, FileFingerprint>();
+	if (!existsSync(storePath)) return map;
+
+	try {
+		const data = JSON.parse(readFileSync(storePath, "utf-8")) as FileFingerprint[];
+		for (const fp of data) {
+			if (map.size >= MAX_FINGERPRINTS) break;
+			map.set(fp.path, fp);
+		}
+	} catch (error) {
+		logInternalError("fingerprint.load", error, `storePath=${storePath}`);
+	}
+	return map;
+}
+
+/**
+ * Save fingerprint baseline to a JSON file.
+ */
+export function saveFingerprintBaseline(storePath: string, fingerprints: Map<string, FileFingerprint>): void {
+	const entries = [...fingerprints.entries()].slice(0, MAX_FINGERPRINTS).map(([, fp]) => fp);
+	writeFileSync(storePath, JSON.stringify(entries, null, 2), "utf-8");
+}
+
+/**
+ * Compute delta between baseline and current fingerprints.
+ *
+ * Only returns STRUCTURAL changes in the `modified` field.
+ */
+export function computeFingerprintDelta(baseline: Map<string, FileFingerprint>, current: Map<string, FileFingerprint>): FingerprintDelta {
+	const added: string[] = [];
+	const removed: string[] = [];
+	const modified: FileFingerprint[] = [];
+	let unchanged = 0;
+
+	// Find added and modified
+	for (const [path, currentFp] of current) {
+		const prev = baseline.get(path);
+		const changeClass = classifyChange(prev, currentFp);
+
+		if (!prev) {
+			added.push(path);
+		} else if (changeClass === "STRUCTURAL") {
+			modified.push({ ...currentFp, changeClass: "STRUCTURAL" });
+		} else if (changeClass === "COSMETIC") {
+			unchanged++; // cosmetic = treated as unchanged for re-analysis
+		} else {
+			unchanged++;
+		}
+	}
+
+	// Find removed
+	for (const path of baseline.keys()) {
+		if (!current.has(path)) {
+			removed.push(path);
+		}
+	}
+
+	return { added, removed, modified, unchanged };
+}

package/src/utils/frontmatter.ts

@@ -0,0 +1,70 @@
+export interface ParsedFrontmatter {
+	frontmatter: Record<string, string>;
+	body: string;
+}
+
+export function parseFrontmatter(content: string): ParsedFrontmatter {
+	if (!content.startsWith("---\n") && !content.startsWith("---\r\n")) {
+		return { frontmatter: {}, body: content };
+	}
+
+	const normalized = content.replaceAll("\r\n", "\n");
+	const end = normalized.indexOf("\n---\n", 4);
+	if (end === -1) {
+		// Support frontmatter that ends at EOF without trailing newline after ---.
+		const altEnd = normalized.indexOf("\n---", 4);
+		if (altEnd !== -1 && altEnd + 4 === normalized.length) {
+			const raw = normalized.slice(4, altEnd);
+			const frontmatter = parseLines(raw);
+			return { frontmatter, body: "" };
+		}
+		return { frontmatter: {}, body: content };
+	}
+
+	const raw = normalized.slice(4, end);
+	const body = normalized.slice(end + "\n---\n".length);
+	const frontmatter = parseLines(raw);
+	return { frontmatter, body };
+}
+
+function parseLines(raw: string): Record<string, string> {
+	const frontmatter: Record<string, string> = {};
+	for (const line of raw.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed || trimmed.startsWith("#")) continue;
+		const separator = trimmed.indexOf(":");
+		if (separator === -1) continue;
+		const key = trimmed.slice(0, separator).trim();
+		const value = trimmed.slice(separator + 1).trim();
+		if (key) frontmatter[key] = value;
+	}
+	return frontmatter;
+}
+
+export function parseCsv(value: string | undefined): string[] | undefined {
+	if (value === undefined) return undefined;
+	// Handle quoted values with commas inside.
+	const values = splitCsv(value)
+		.map((item) => item.trim())
+		.filter(Boolean);
+	return values.length > 0 ? [...new Set(values)] : undefined;
+}
+
+function splitCsv(input: string): string[] {
+	const result: string[] = [];
+	let current = "";
+	let inQuotes = false;
+	for (let i = 0; i < input.length; i++) {
+		const char = input[i];
+		if (char === '"') {
+			inQuotes = !inQuotes;
+		} else if (char === "," && !inQuotes) {
+			result.push(current);
+			current = "";
+		} else {
+			current += char;
+		}
+	}
+	result.push(current);
+	return result;
+}

package/src/utils/fs-watch.ts

@@ -0,0 +1,37 @@
+import type { FSWatcher, WatchListener } from "node:fs";
+import * as fs from "node:fs";
+
+/**
+ * Filesystem watcher helpers (slimmed down — pts/2 hang fix 2026-06-16).
+ *
+ * The recursive-watcher helpers (createRecursiveWatcher / watchCrewState /
+ * runIdFromStateRelativePath) were REMOVED: a recursive fs.watch on the run
+ * state tree exploded to O(total run history) inotify watches on Linux and
+ * caused a permanent interactive-session busy-loop. The bounded
+ * {@link RunWatcherRegistry} (one non-recursive watcher per ACTIVE run) now
+ * replaces them. Only the two primitives below survive — they are still used by
+ * manifest-cache, result-watcher, and run-watcher-registry.
+ */
+
+export function closeWatcher(watcher: FSWatcher | null | undefined): void {
+	if (!watcher) {
+		return;
+	}
+
+	try {
+		watcher.close();
+	} catch {
+		// Ignore watcher close errors
+	}
+}
+
+export function watchWithErrorHandler(path: string, listener: WatchListener<string>, onError: (error?: unknown) => void): FSWatcher | null {
+	try {
+		const watcher = fs.watch(path, listener);
+		watcher.on("error", onError);
+		return watcher;
+	} catch (error) {
+		onError(error);
+		return null;
+	}
+}