pi-crew 0.7.4 → 0.7.6

package/src/utils/fs-watch.ts

@@ -1,9 +1,17 @@
 import * as fs from "node:fs";
-import * as path from "node:path";
 import type { FSWatcher, WatchListener } from "node:fs";
 
-/** @internal */
-const FS_WATCH_RETRY_DELAY_MS = 5000;
+/**
+ * Filesystem watcher helpers (slimmed down — pts/2 hang fix 2026-06-16).
+ *
+ * The recursive-watcher helpers (createRecursiveWatcher / watchCrewState /
+ * runIdFromStateRelativePath) were REMOVED: a recursive fs.watch on the run
+ * state tree exploded to O(total run history) inotify watches on Linux and
+ * caused a permanent interactive-session busy-loop. The bounded
+ * {@link RunWatcherRegistry} (one non-recursive watcher per ACTIVE run) now
+ * replaces them. Only the two primitives below survive — they are still used by
+ * manifest-cache, result-watcher, and run-watcher-registry.
+ */
 
 export function closeWatcher(watcher: FSWatcher | null | undefined): void {
 	if (!watcher) {
@@ -31,60 +39,3 @@ export function watchWithErrorHandler(
 		return null;
 	}
 }
-
-/**
- * 1.3 — Watch a directory recursively and invoke `onChange` when any file
- * inside changes. Falls back to `null` on systems where `fs.watch` rejects
- * recursive mode (e.g., Linux when running on older kernels via FUSE/network FS).
- *
- * Callers MUST handle null and fall back to polling. The watcher emits the
- * filename relative to `rootDir` (forward-slash normalised on Windows).
- */
-export function createRecursiveWatcher(
-	rootDir: string,
-	onChange: (relativePath: string) => void,
-	onError: (error: unknown) => void,
-): FSWatcher | null {
-	try {
-		if (!fs.existsSync(rootDir)) fs.mkdirSync(rootDir, { recursive: true });
-		const watcher = fs.watch(rootDir, { recursive: true }, (_eventType, filename) => {
-			if (typeof filename !== "string" || filename.length === 0) return;
-			onChange(filename.replace(/\\/g, "/"));
-		});
-		watcher.on("error", (error) => {
-			try { watcher.close(); } catch { /* ignore */ }
-			onError(error);
-		});
-		return watcher;
-	} catch (error) {
-		onError(error);
-		return null;
-	}
-}
-
-/**
- * Given a path relative to `<crewRoot>/state`, return the runId that owns
- * the change, or undefined if the path doesn't match any tracked run layout.
- */
-export function runIdFromStateRelativePath(relativePath: string): string | undefined {
-	const parts = relativePath.split("/");
-	// Layout is `runs/{runId}/...` — see docs/architecture.md state layer.
-	if (parts.length >= 2 && parts[0] === "runs" && parts[1]) return parts[1];
-	return undefined;
-}
-
-/** Convenience: combine the two helpers for `<crewRoot>/state` watching. */
-export function watchCrewState(
-	stateDir: string,
-	onRunChange: (runId: string) => void,
-	onError: (error: unknown) => void,
-): FSWatcher | null {
-	return createRecursiveWatcher(stateDir, (relativePath) => {
-		const runId = runIdFromStateRelativePath(relativePath);
-		if (runId) onRunChange(runId);
-	}, onError);
-}
-
-// Re-export path helper so callers don't pull node:path just for join.
-/** @internal */
-const joinPath = path.join;

package/src/utils/run-watcher-registry.ts

@@ -0,0 +1,164 @@
+/**
+ * RunWatcherRegistry — bounded per-run filesystem watcher registry.
+ *
+ * PROBLEM (pts/2 interactive-session hang, /home/bom/pts2-hang-investigation-2026-06-16.md):
+ * `watchCrewState` used `fs.watch(<crewRoot>/state, { recursive: true })`. On
+ * Linux, Node implements "recursive" by creating ONE inotify watch PER
+ * SUBDIRECTORY. With many historical runs under `.crew/state/runs/`, this
+ * ballooned to hundreds of watches (109→339 observed) — one per run dir ever —
+ * and the resulting event volume + render amplification produced a permanent
+ * busy-loop (71% CPU, 400KB/s read) even with no active work.
+ *
+ * FIX: instead of recursively watching the whole history, watch a SINGLE
+ * non-recursive watcher on the `runs/` root (to detect new run dirs appearing)
+ * PLUS one non-recursive watcher PER ACTIVE RUN. Total inotify cost is now
+ * O(active runs) — typically 1–5 — not O(total history). Completed runs stop
+ * being watched as soon as they leave the active set (reconciled by buildFrame,
+ * which reads manifest statuses each preload tick).
+ *
+ * The registry is intentionally small and directly unit-testable (a Map of
+ * watchers with add/remove/reconcile/close semantics).
+ */
+import type { FSWatcher } from "node:fs";
+import { closeWatcher, watchWithErrorHandler } from "./fs-watch.ts";
+
+export interface ReconcileResult {
+	added: string[];
+	removed: string[];
+}
+
+export interface ActiveRun {
+	runId: string;
+	runDir: string;
+}
+
+export type RunChangeCallback = (runId: string) => void;
+export type ErrorCallback = (error: unknown) => void;
+
+export class RunWatcherRegistry {
+	private readonly runWatchers = new Map<string, FSWatcher>();
+	private rootWatcher: FSWatcher | undefined;
+	private closed = false;
+
+	/**
+	 * Watch the `runs/` root directory (non-recursive) and invoke `onNewRun`
+	 * whenever a new run subdirectory appears. This is the only way to detect a
+	 * brand-new run, because `crew.run.created` is never emitted by the runtime
+	 * (confirmed: only `crew.run.completed/failed/cancelled` are emitted).
+	 */
+	setRootWatcher(
+		runsDir: string,
+		onNewRun: RunChangeCallback,
+		onError?: ErrorCallback,
+	): void {
+		if (this.closed) return;
+		// Replace any prior root watcher.
+		closeWatcher(this.rootWatcher);
+		this.rootWatcher = watchWithErrorHandler(
+			runsDir,
+			(_eventType, filename) => {
+				if (typeof filename !== "string" || filename.length === 0) return;
+				// fs.watch reports directory entries as bare names (no slash on Linux).
+				// A new run dir appears as `runs/<runId>` → filename = "<runId>".
+				// Filter obviously-not-run-id noise (files, temp, etc.) defensively.
+				const candidate = filename.replace(/\\/g, "/").split("/")[0];
+				if (candidate.length === 0) return;
+				onNewRun(candidate);
+			},
+			(error) => {
+				if (onError) onError(error);
+			},
+		) ?? undefined;
+	}
+
+	/**
+	 * Add a NON-RECURSIVE watcher on a single run directory. Costs exactly ONE
+	 * inotify watch. If a watcher for this runId already exists, close + replace.
+	 * Returns true if a watcher is now active for this runId.
+	 */
+	addRunWatcher(
+		runId: string,
+		runDir: string,
+		onChange: RunChangeCallback,
+		onError?: ErrorCallback,
+	): boolean {
+		if (this.closed) return false;
+		const existing = this.runWatchers.get(runId);
+		if (existing) closeWatcher(existing);
+		const watcher = watchWithErrorHandler(
+			runDir,
+			() => onChange(runId),
+			(error) => {
+				if (onError) onError(error);
+			},
+		);
+		if (watcher) {
+			this.runWatchers.set(runId, watcher);
+			return true;
+		}
+		// watchWithErrorHandler returned null (fs.watch unsupported / dir missing).
+		// Remove any stale entry so hasWatcher() stays honest.
+		this.runWatchers.delete(runId);
+		return false;
+	}
+
+	/** Remove and close a specific run's watcher. No-op if not watched. */
+	removeRunWatcher(runId: string): void {
+		const watcher = this.runWatchers.get(runId);
+		if (watcher) {
+			closeWatcher(watcher);
+			this.runWatchers.delete(runId);
+		}
+	}
+
+	/** Is a run currently being watched? */
+	hasWatcher(runId: string): boolean {
+		return this.runWatchers.has(runId);
+	}
+
+	/**
+	 * Reconcile against the current active-run set: add watchers for active runs
+	 * not yet watched, remove watchers for runs that left the active set. Returns
+	 * which runIds were added / removed (useful for logging + tests).
+	 */
+	reconcile(
+		activeRuns: ActiveRun[],
+		onChange: RunChangeCallback,
+		onError?: ErrorCallback,
+	): ReconcileResult {
+		if (this.closed) return { added: [], removed: [] };
+		const activeIds = new Set(activeRuns.map((r) => r.runId));
+		const added: string[] = [];
+		const removed: string[] = [];
+		// Remove watchers for runs no longer active.
+		for (const runId of [...this.runWatchers.keys()]) {
+			if (!activeIds.has(runId)) {
+				this.removeRunWatcher(runId);
+				removed.push(runId);
+			}
+		}
+		// Add watchers for newly-active runs.
+		for (const { runId, runDir } of activeRuns) {
+			if (!this.runWatchers.has(runId)) {
+				if (this.addRunWatcher(runId, runDir, onChange, onError)) {
+					added.push(runId);
+				}
+			}
+		}
+		return { added, removed };
+	}
+
+	/** Close ALL watchers (per-run + root). Safe to call multiple times. */
+	closeAll(): void {
+		this.closed = true;
+		for (const watcher of this.runWatchers.values()) closeWatcher(watcher);
+		this.runWatchers.clear();
+		closeWatcher(this.rootWatcher);
+		this.rootWatcher = undefined;
+	}
+
+	/** Number of active PER-RUN watchers (excludes the root watcher). */
+	get size(): number {
+		return this.runWatchers.size;
+	}
+}

package/src/workflows/discover-workflows.ts

@@ -11,7 +11,7 @@ export interface WorkflowDiscoveryResult {
 	project: WorkflowConfig[];
 }
 
-const STEP_CONFIG_KEYS = new Set(["role", "dependsOn", "parallelGroup", "output", "reads", "model", "skills", "progress", "worktree", "verify", "task", "seedPaths", "preStepScript", "preStepArgs", "preStepTimeout"]);
+const STEP_CONFIG_KEYS = new Set(["role", "dependsOn", "parallelGroup", "output", "reads", "model", "skills", "progress", "worktree", "verify", "task", "seedPaths", "preStepScript", "preStepArgs", "preStepTimeout", "preStepOptional"]);
 
 function parseStepSection(id: string, body: string): WorkflowStep | undefined {
 	const lines = body.trim().split("\n");
@@ -54,6 +54,7 @@ function parseStepSection(id: string, body: string): WorkflowStep | undefined {
 		preStepScript: config.preStepScript || undefined,
 		preStepArgs: parseCsv(config.preStepArgs) || undefined,
 		preStepTimeout: parseOptionalInteger(config.preStepTimeout) ?? undefined,
+		preStepOptional: config.preStepOptional === "true" || config.preStepOptional === "1",
 	};
 }
 

package/src/workflows/workflow-config.ts

@@ -25,6 +25,11 @@ export interface WorkflowStep {
 	preStepArgs?: string[];
 	/** Timeout in ms for preStepScript. Default: 30000. */
 	preStepTimeout?: number;
+	/** Round 21 (E4): if true, a failing preStepScript does NOT abort the task.
+	 * The failure is logged as a warning and the task proceeds without the
+	 * pre-step output. Use for advisory hooks (e.g. optional test runs) whose
+	 * failure shouldn't block the workflow. Default: false (fail-fast). */
+	preStepOptional?: boolean;
 }
 
 export interface WorkflowConfig {