pi-crew 0.7.5 → 0.7.6

package/src/utils/run-watcher-registry.ts

@@ -0,0 +1,164 @@
+/**
+ * RunWatcherRegistry — bounded per-run filesystem watcher registry.
+ *
+ * PROBLEM (pts/2 interactive-session hang, /home/bom/pts2-hang-investigation-2026-06-16.md):
+ * `watchCrewState` used `fs.watch(<crewRoot>/state, { recursive: true })`. On
+ * Linux, Node implements "recursive" by creating ONE inotify watch PER
+ * SUBDIRECTORY. With many historical runs under `.crew/state/runs/`, this
+ * ballooned to hundreds of watches (109→339 observed) — one per run dir ever —
+ * and the resulting event volume + render amplification produced a permanent
+ * busy-loop (71% CPU, 400KB/s read) even with no active work.
+ *
+ * FIX: instead of recursively watching the whole history, watch a SINGLE
+ * non-recursive watcher on the `runs/` root (to detect new run dirs appearing)
+ * PLUS one non-recursive watcher PER ACTIVE RUN. Total inotify cost is now
+ * O(active runs) — typically 1–5 — not O(total history). Completed runs stop
+ * being watched as soon as they leave the active set (reconciled by buildFrame,
+ * which reads manifest statuses each preload tick).
+ *
+ * The registry is intentionally small and directly unit-testable (a Map of
+ * watchers with add/remove/reconcile/close semantics).
+ */
+import type { FSWatcher } from "node:fs";
+import { closeWatcher, watchWithErrorHandler } from "./fs-watch.ts";
+
+export interface ReconcileResult {
+	added: string[];
+	removed: string[];
+}
+
+export interface ActiveRun {
+	runId: string;
+	runDir: string;
+}
+
+export type RunChangeCallback = (runId: string) => void;
+export type ErrorCallback = (error: unknown) => void;
+
+export class RunWatcherRegistry {
+	private readonly runWatchers = new Map<string, FSWatcher>();
+	private rootWatcher: FSWatcher | undefined;
+	private closed = false;
+
+	/**
+	 * Watch the `runs/` root directory (non-recursive) and invoke `onNewRun`
+	 * whenever a new run subdirectory appears. This is the only way to detect a
+	 * brand-new run, because `crew.run.created` is never emitted by the runtime
+	 * (confirmed: only `crew.run.completed/failed/cancelled` are emitted).
+	 */
+	setRootWatcher(
+		runsDir: string,
+		onNewRun: RunChangeCallback,
+		onError?: ErrorCallback,
+	): void {
+		if (this.closed) return;
+		// Replace any prior root watcher.
+		closeWatcher(this.rootWatcher);
+		this.rootWatcher = watchWithErrorHandler(
+			runsDir,
+			(_eventType, filename) => {
+				if (typeof filename !== "string" || filename.length === 0) return;
+				// fs.watch reports directory entries as bare names (no slash on Linux).
+				// A new run dir appears as `runs/<runId>` → filename = "<runId>".
+				// Filter obviously-not-run-id noise (files, temp, etc.) defensively.
+				const candidate = filename.replace(/\\/g, "/").split("/")[0];
+				if (candidate.length === 0) return;
+				onNewRun(candidate);
+			},
+			(error) => {
+				if (onError) onError(error);
+			},
+		) ?? undefined;
+	}
+
+	/**
+	 * Add a NON-RECURSIVE watcher on a single run directory. Costs exactly ONE
+	 * inotify watch. If a watcher for this runId already exists, close + replace.
+	 * Returns true if a watcher is now active for this runId.
+	 */
+	addRunWatcher(
+		runId: string,
+		runDir: string,
+		onChange: RunChangeCallback,
+		onError?: ErrorCallback,
+	): boolean {
+		if (this.closed) return false;
+		const existing = this.runWatchers.get(runId);
+		if (existing) closeWatcher(existing);
+		const watcher = watchWithErrorHandler(
+			runDir,
+			() => onChange(runId),
+			(error) => {
+				if (onError) onError(error);
+			},
+		);
+		if (watcher) {
+			this.runWatchers.set(runId, watcher);
+			return true;
+		}
+		// watchWithErrorHandler returned null (fs.watch unsupported / dir missing).
+		// Remove any stale entry so hasWatcher() stays honest.
+		this.runWatchers.delete(runId);
+		return false;
+	}
+
+	/** Remove and close a specific run's watcher. No-op if not watched. */
+	removeRunWatcher(runId: string): void {
+		const watcher = this.runWatchers.get(runId);
+		if (watcher) {
+			closeWatcher(watcher);
+			this.runWatchers.delete(runId);
+		}
+	}
+
+	/** Is a run currently being watched? */
+	hasWatcher(runId: string): boolean {
+		return this.runWatchers.has(runId);
+	}
+
+	/**
+	 * Reconcile against the current active-run set: add watchers for active runs
+	 * not yet watched, remove watchers for runs that left the active set. Returns
+	 * which runIds were added / removed (useful for logging + tests).
+	 */
+	reconcile(
+		activeRuns: ActiveRun[],
+		onChange: RunChangeCallback,
+		onError?: ErrorCallback,
+	): ReconcileResult {
+		if (this.closed) return { added: [], removed: [] };
+		const activeIds = new Set(activeRuns.map((r) => r.runId));
+		const added: string[] = [];
+		const removed: string[] = [];
+		// Remove watchers for runs no longer active.
+		for (const runId of [...this.runWatchers.keys()]) {
+			if (!activeIds.has(runId)) {
+				this.removeRunWatcher(runId);
+				removed.push(runId);
+			}
+		}
+		// Add watchers for newly-active runs.
+		for (const { runId, runDir } of activeRuns) {
+			if (!this.runWatchers.has(runId)) {
+				if (this.addRunWatcher(runId, runDir, onChange, onError)) {
+					added.push(runId);
+				}
+			}
+		}
+		return { added, removed };
+	}
+
+	/** Close ALL watchers (per-run + root). Safe to call multiple times. */
+	closeAll(): void {
+		this.closed = true;
+		for (const watcher of this.runWatchers.values()) closeWatcher(watcher);
+		this.runWatchers.clear();
+		closeWatcher(this.rootWatcher);
+		this.rootWatcher = undefined;
+	}
+
+	/** Number of active PER-RUN watchers (excludes the root watcher). */
+	get size(): number {
+		return this.runWatchers.size;
+	}
+}

package/src/workflows/discover-workflows.ts

@@ -11,7 +11,7 @@ export interface WorkflowDiscoveryResult {
 	project: WorkflowConfig[];
 }
 
-const STEP_CONFIG_KEYS = new Set(["role", "dependsOn", "parallelGroup", "output", "reads", "model", "skills", "progress", "worktree", "verify", "task", "seedPaths", "preStepScript", "preStepArgs", "preStepTimeout"]);
+const STEP_CONFIG_KEYS = new Set(["role", "dependsOn", "parallelGroup", "output", "reads", "model", "skills", "progress", "worktree", "verify", "task", "seedPaths", "preStepScript", "preStepArgs", "preStepTimeout", "preStepOptional"]);
 
 function parseStepSection(id: string, body: string): WorkflowStep | undefined {
 	const lines = body.trim().split("\n");
@@ -54,6 +54,7 @@ function parseStepSection(id: string, body: string): WorkflowStep | undefined {
 		preStepScript: config.preStepScript || undefined,
 		preStepArgs: parseCsv(config.preStepArgs) || undefined,
 		preStepTimeout: parseOptionalInteger(config.preStepTimeout) ?? undefined,
+		preStepOptional: config.preStepOptional === "true" || config.preStepOptional === "1",
 	};
 }
 

package/src/workflows/workflow-config.ts

@@ -25,6 +25,11 @@ export interface WorkflowStep {
 	preStepArgs?: string[];
 	/** Timeout in ms for preStepScript. Default: 30000. */
 	preStepTimeout?: number;
+	/** Round 21 (E4): if true, a failing preStepScript does NOT abort the task.
+	 * The failure is logged as a warning and the task proceeds without the
+	 * pre-step output. Use for advisory hooks (e.g. optional test runs) whose
+	 * failure shouldn't block the workflow. Default: false (fail-fast). */
+	preStepOptional?: boolean;
 }
 
 export interface WorkflowConfig {

package/src/runtime/dynamic-script-runner.ts

@@ -1,497 +0,0 @@
-import * as vm from "node:vm";
-import * as acorn from "acorn";
-import { WorkflowSandbox, type SandboxOptions } from "./sandbox.ts";
-
-/**
- * Forbidden globals that could compromise sandbox security or cause side effects.
- * These are checked during AST validation before execution.
- */
-export const FORBIDDEN_GLOBALS = [
-	"Math.random",
-	"require",
-	"import",
-	"module",
-	"exports",
-	"__dirname",
-	"__filename",
-	"process.exit",
-	"process.kill",
-	"process.hrtime",
-	"process.memoryUsage",
-	"process.cpuUsage",
-	"process.binding",
-	"process.dlopen",
-	"process._tickCallback",
-	"eval",
-	"Function",
-	"AsyncFunction",
-	"GeneratorFunction",
-	"Proxy",
-	"Reflect",
-	"WebAssembly",
-	"global",
-	"globalThis",
-	"window",
-	"document",
-	"XMLHttpRequest",
-	"fetch",
-	"WebSocket",
-	"Worker",
-	"SharedArrayBuffer",
-	"Atomics",
-] as const;
-
-// Freeze the array at runtime to ensure it's truly immutable
-Object.freeze(FORBIDDEN_GLOBALS);
-
-export type ForbiddenGlobal = (typeof FORBIDDEN_GLOBALS)[number];
-
-export interface ScriptValidationResult {
-	valid: boolean;
-	errors: ScriptValidationError[];
-	warnings: ScriptValidationWarning[];
-}
-
-export interface ScriptValidationError {
-	type: "forbidden_global" | "forbidden_syntax" | "parse_error";
-	message: string;
-	location?: { line: number; column: number };
-}
-
-export interface ScriptValidationWarning {
-	type: "deprecated_api" | "potentially_unsafe";
-	message: string;
-	location?: { line: number; column: number };
-}
-
-export interface DynamicScriptOptions {
-	timeout?: number;
-	maxTokens?: number;
-	allowAwait?: boolean;
-	allowAsync?: boolean;
-	strictMode?: boolean;
-	/** Enable strict AST whitelist mode (C2) - reject dynamic property access, call expressions */
-	strictAstWhitelist?: boolean;
-}
-
-export interface ScriptExecutionResult {
-	success: boolean;
-	value?: unknown;
-	error?: string;
-	executionTime: number;
-	validation: ScriptValidationResult;
-}
-
-/**
- * DynamicScriptRunner executes JavaScript in a VM sandbox with AST validation
- * and forbidden pattern detection.
- * 
- * Note: AST parsing is simplified without acorn. For full AST validation,
- * add acorn as a dependency.
- */
-export class DynamicScriptRunner {
-	private sandbox: WorkflowSandbox;
-	private defaultTimeout: number;
-	private options: DynamicScriptOptions;
-
-	constructor(options: DynamicScriptOptions = {}) {
-		this.defaultTimeout = options.timeout ?? 30000;
-		this.options = options;
-		this.sandbox = new WorkflowSandbox({
-			timeout: this.defaultTimeout,
-		});
-	}
-
-	/**
-	 * Validate script before execution using full AST parsing (C1).
-	 * Uses acorn for comprehensive syntax tree analysis.
-	 */
-	validate(code: string): ScriptValidationResult {
-		const errors: ScriptValidationError[] = [];
-		const warnings: ScriptValidationWarning[] = [];
-
-		// C1: Full AST parsing with acorn for complete validation
-		let ast: acorn.Node | null = null;
-		try {
-			ast = acorn.parse(code, {
-				ecmaVersion: "latest",
-				sourceType: "script",
-				allowReturnOutsideFunction: true,
-				allowAwaitOutsideFunction: this.options.allowAwait ?? false,
-			});
-		} catch (parseError) {
-			errors.push({
-				type: "parse_error",
-				message: `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
-			});
-			return { valid: false, errors, warnings };
-		}
-
-		// C2: Strict AST whitelist mode - reject dynamic property access and call expressions
-		if (this.options.strictAstWhitelist) {
-			this.validateAstWhitelist(ast, errors);
-		}
-
-		// C4: Bytecode compilation verification - verify script compiles correctly
-		this.verifyCompilation(code, errors);
-
-		// Check for forbidden globals using regex patterns (fallback)
-		this.checkForForbiddenGlobals(code, errors);
-
-		// Check for forbidden syntax patterns
-		this.checkForForbiddenSyntax(code, errors, warnings);
-
-		// Check for potentially unsafe patterns
-		this.checkForPotentiallyUnsafePatterns(code, warnings);
-
-		return {
-			valid: errors.length === 0,
-			errors,
-			warnings,
-		};
-	}
-
-	/**
-	 * C2: Validate AST against strict whitelist - reject dynamic property access,
-	 * call expressions, and other potentially dangerous patterns.
-	 */
-	private validateAstWhitelist(
-		ast: acorn.Node,
-		errors: ScriptValidationError[],
-	): void {
-		const walkNode = (node: acorn.Node): void => {
-			const n = node as acorn.Node & { type: string };
-
-			// Reject MemberExpression (e.g., obj.prop, obj[key])
-			if (n.type === "MemberExpression") {
-				errors.push({
-					type: "forbidden_syntax",
-					message: "Dynamic property access is not allowed in strict whitelist mode",
-				});
-			}
-
-			// Reject CallExpression (e.g., fn(), obj.method())
-			if (n.type === "CallExpression") {
-				errors.push({
-					type: "forbidden_syntax",
-					message: "Call expressions are not allowed in strict whitelist mode",
-				});
-			}
-
-			// Reject NewExpression (e.g., new Foo())
-			if (n.type === "NewExpression") {
-				errors.push({
-					type: "forbidden_syntax",
-					message: "Constructor calls are not allowed in strict whitelist mode",
-				});
-			}
-
-			// Reject UpdateExpression (++a, a--, etc.)
-			if (n.type === "UpdateExpression") {
-				errors.push({
-					type: "forbidden_syntax",
-					message: "Update expressions are not allowed in strict whitelist mode",
-				});
-			}
-
-			// Reject AssignmentExpression (a = b, a += b, etc.)
-			if (n.type === "AssignmentExpression") {
-				errors.push({
-					type: "forbidden_syntax",
-					message: "Assignment expressions are not allowed in strict whitelist mode",
-				});
-			}
-
-			// Reject ForInStatement and ForOfStatement
-			if (n.type === "ForInStatement" || n.type === "ForOfStatement") {
-				errors.push({
-					type: "forbidden_syntax",
-					message: "For-in/for-of loops are not allowed in strict whitelist mode",
-				});
-			}
-
-			// Recurse into children
-			for (const key of Object.keys(n as object)) {
-				const value = (n as unknown as Record<string, unknown>)[key];
-				if (value && typeof value === "object") {
-					if (Array.isArray(value)) {
-						for (const child of value) {
-							if (child && typeof child === "object" && "type" in child) {
-								walkNode(child as acorn.Node);
-							}
-						}
-					} else if ("type" in value) {
-						walkNode(value as acorn.Node);
-					}
-				}
-			}
-		};
-
-		// Cast to access body property (Program node)
-		const programNode = ast as unknown as { body: acorn.Node[] };
-		for (const node of programNode.body) {
-			walkNode(node);
-		}
-	}
-
-	/**
-	 * C4: Verify script compiles to bytecode correctly. If compilation fails,
-	 * the script cannot be executed safely.
-	 */
-	private verifyCompilation(code: string, errors: ScriptValidationError[]): void {
-		try {
-			const wrappedCode = `(function(){ ${code} })()`;
-			new vm.Script(wrappedCode, {
-				filename: "compile-check.js",
-			});
-		} catch (compileError) {
-			errors.push({
-				type: "parse_error",
-				message: `Compilation verification failed: ${compileError instanceof Error ? compileError.message : String(compileError)}`,
-			});
-		}
-	}
-
-	private checkForForbiddenGlobals(code: string, errors: ScriptValidationError[]): void {
-		// Check each forbidden global pattern
-		for (const forbidden of FORBIDDEN_GLOBALS) {
-			if (forbidden.includes(".")) {
-				// Check for member expressions like Math.random, process.exit
-				const [obj, prop] = forbidden.split(".");
-				const pattern = new RegExp(`\\b${obj}\\s*\\.\\s*${prop}\\b`);
-				if (pattern.test(code)) {
-					errors.push({
-						type: "forbidden_global",
-						message: `Forbidden global access: '${forbidden}'`,
-					});
-				}
-			} else {
-				// Check for simple identifiers
-				// But avoid false positives like "myDate" matching "Date"
-				const pattern = new RegExp(`\\b${forbidden}\\b`);
-				if (pattern.test(code)) {
-					errors.push({
-						type: "forbidden_global",
-						message: `Forbidden global: '${forbidden}'`,
-					});
-				}
-			}
-		}
-	}
-
-	private checkForForbiddenSyntax(
-		code: string,
-		errors: ScriptValidationError[],
-		warnings: ScriptValidationWarning[],
-	): void {
-		// Check for eval()
-		if (/\beval\s*\(/.test(code)) {
-			errors.push({
-				type: "forbidden_syntax",
-				message: "eval() is not allowed",
-			});
-		}
-
-		// Check for Function constructor
-		if (/\bnew\s+Function\s*\(/.test(code) || /\bFunction\s*\(\s*['"`]/.test(code)) {
-			errors.push({
-				type: "forbidden_syntax",
-				message: "Function constructor is not allowed",
-			});
-		}
-
-		// Check for AsyncFunction constructor
-		if (/\bnew\s+AsyncFunction\s*\(/.test(code) || /\bAsyncFunction\s*\(\s*['"`]/.test(code)) {
-			errors.push({
-				type: "forbidden_syntax",
-				message: "AsyncFunction constructor is not allowed",
-			});
-		}
-
-		// Check for GeneratorFunction constructor
-		if (/\bnew\s+GeneratorFunction\s*\(/.test(code)) {
-			errors.push({
-				type: "forbidden_syntax",
-				message: "GeneratorFunction constructor is not allowed",
-			});
-		}
-
-		// Check for Promise constructor - warn but don't block
-		if (/\bnew\s+Promise\s*\(/.test(code)) {
-			warnings.push({
-				type: "potentially_unsafe",
-				message: "Direct Promise constructor usage - consider using async/await instead",
-			});
-		}
-	}
-
-	private checkForPotentiallyUnsafePatterns(code: string, warnings: ScriptValidationWarning[]): void {
-		// Check for try-catch with broad catch - warn
-		if (/\bcatch\s*\(\s*\)\s*\{/.test(code)) {
-			warnings.push({
-				type: "potentially_unsafe",
-				message: "Broad catch clause - consider catching specific error types",
-			});
-		}
-
-		// Check for nested function declarations - warn about potential complexity
-		if (/function\s+\w+\s*\([^)]*\)\s*\{[^}]*function\s+/.test(code)) {
-			warnings.push({
-				type: "potentially_unsafe",
-				message: "Nested function declaration - consider extracting to module level",
-			});
-		}
-
-		// Check for with statement - deprecated and potentially unsafe
-		if (/\bwith\s*\(/.test(code)) {
-			warnings.push({
-				type: "potentially_unsafe",
-				message: "with statement is deprecated and potentially unsafe",
-			});
-		}
-	}
-
-	/**
-	 * Execute a script with validation.
-	 * @param code - The JavaScript code to execute
-	 * @param options - Execution options
-	 * @returns The execution result
-	 */
-	execute(code: string, options?: DynamicScriptOptions): ScriptExecutionResult {
-		const startTime = Date.now();
-		const timeout = options?.timeout ?? this.defaultTimeout;
-
-		// Validate first
-		const validation = this.validate(code);
-		if (!validation.valid) {
-			return {
-				success: false,
-				error: validation.errors.map((e) => e.message).join("; "),
-				executionTime: Date.now() - startTime,
-				validation,
-			};
-		}
-
-		try {
-			const value = this.sandbox.execute(code, timeout);
-			return {
-				success: true,
-				value,
-				executionTime: Date.now() - startTime,
-				validation,
-			};
-		} catch (error) {
-			return {
-				success: false,
-				error: error instanceof Error ? error.message : String(error),
-				executionTime: Date.now() - startTime,
-				validation,
-			};
-		}
-	}
-
-	/**
-	 * Execute an async script with validation.
-	 * @param code - The JavaScript code to execute (must be async or return Promise)
-	 * @param options - Execution options
-	 * @returns Promise resolving to the execution result
-	 */
-	async executeAsync(code: string, options?: DynamicScriptOptions): Promise<ScriptExecutionResult> {
-		const startTime = Date.now();
-		const timeout = options?.timeout ?? this.defaultTimeout;
-
-		// Wrap in async IIFE for async/await support
-		const asyncCode = `(async () => { ${code} })()`;
-
-		// Validate the wrapped code (not the original code)
-		const validation = this.validate(asyncCode);
-		if (!validation.valid) {
-			return {
-				success: false,
-				error: validation.errors.map((e) => e.message).join("; "),
-				executionTime: Date.now() - startTime,
-				validation,
-			};
-		}
-
-		try {
-			// Execute using vm directly for async support
-			const script = new vm.Script(asyncCode, {
-				filename: "workflow.js",
-			});
-
-			const result = await script.runInContext(this.sandbox.getContext(), {
-				timeout,
-				displayErrors: true,
-			});
-			return {
-				success: true,
-				value: result,
-				executionTime: Date.now() - startTime,
-				validation,
-			};
-		} catch (error) {
-			return {
-				success: false,
-				error: error instanceof Error ? error.message : String(error),
-				executionTime: Date.now() - startTime,
-				validation,
-			};
-		}
-	}
-
-	/**
-	 * Execute a script without validation (assumes pre-validated).
-	 * Use with caution - prefer execute() for untrusted scripts.
-	 * @internal TEST ONLY — do not use in production code
-	 */
-	private executeUnchecked(code: string, timeout?: number): ScriptExecutionResult {
-		const startTime = Date.now();
-
-		try {
-			const value = this.sandbox.execute(code, timeout ?? this.defaultTimeout);
-			return {
-				success: true,
-				value,
-				executionTime: Date.now() - startTime,
-				validation: { valid: true, errors: [], warnings: [] },
-			};
-		} catch (error) {
-			return {
-				success: false,
-				error: error instanceof Error ? error.message : String(error),
-				executionTime: Date.now() - startTime,
-				validation: { valid: true, errors: [], warnings: [] },
-			};
-		}
-	}
-
-	/**
-	 * Get the list of forbidden globals for documentation.
-	 */
-	getForbiddenGlobals(): readonly string[] {
-		return FORBIDDEN_GLOBALS;
-	}
-}
-
-/**
- * Create a pre-configured script runner for workflow execution.
- */
-export function createScriptRunner(options?: DynamicScriptOptions): DynamicScriptRunner {
-	return new DynamicScriptRunner(options);
-}
-
-/**
- * @internal TEST ONLY — do not use in production code.
- * Exposes DynamicScriptRunner.executeUnchecked for unit testing.
- * NOTE: This function is exported unconditionally. Previous versions gated
- * on NODE_ENV=test || PI_CREW_TEST=1, but Node's test runner doesn't set
- * NODE_ENV=test and PI_CREW_TEST has no consumer to set it. The gate was
- * effectively dead code that blocked legitimate tests. The `__test_` prefix
- * and JSDoc warning are the only protection. Real production code should
- * never import this; a more robust solution (tree-shaking, separate builds)
- * is out of scope here.
- */
-export const __test_executeUnchecked = (runner: DynamicScriptRunner, code: string, timeout?: number): ScriptExecutionResult => {
-	return (runner as unknown as { executeUnchecked: (code: string, timeout?: number) => ScriptExecutionResult }).executeUnchecked(code, timeout);
-};