pi-crew 0.9.16 → 0.9.18

package/src/runtime/task-runner/retrieval-orchestrator.ts

@@ -0,0 +1,310 @@
+/**
+ * M3: Iterative file-retrieval orchestrator.
+ *
+ * Pattern: workers progressively discover relevant context files
+ * (e.g. "which source file handles X?") using ripgrep-driven keyword
+ * search + the existing context-retrieval.ts scoring/convergence
+ * helpers. Max 3 cycles, fall back to in-memory heuristic when
+ * ripgrep is not available (e.g. minimal Windows CI runners).
+ *
+ * Signal flow:
+ *   renderTaskPrompt (in prompt-builder.ts)
+ *     → runRetrievalCycle(task, goal, cwd)
+ *       → cycle 1: rg --files, then score each file
+ *       → cycle 2: refine query, rg --json for keyword filter
+ *       → cycle 3: same; if !shouldContinue, stop early
+ *     → returns top-N files (5..10)
+ *   renderTaskPrompt injects "Suggested files to read (top-N by
+ *   retrieval score):" section before final prompt assembly.
+ *
+ * Production code path — not @experimental. Fallback is mandatory so
+ * the prompt is never blocked by a missing ripgrep binary.
+ */
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { RelevanceEvaluation } from "./context-retrieval.ts";
+import { hasConverged, scoreRelevance, shouldContinue } from "./context-retrieval.ts";
+
+/** Max retrieval cycles per prompt render. Matches context-retrieval.MAX_CYCLES. */
+export const MAX_CYCLES = 3;
+
+/** Hard cap on suggested files injected into the worker prompt. */
+export const MAX_SUGGESTED_FILES = 10;
+
+/** Minimum files suggested when retrieval returns anything. */
+export const MIN_SUGGESTED_FILES = 5;
+
+/** Stopwords dropped during keyword tokenization (lowercase comparison). */
+const STOPWORDS: ReadonlySet<string> = new Set(["the", "a", "an", "and", "or", "to", "of", "in", "for", "on", "is", "are", "be", "with"]);
+
+/** File extensions considered relevant for retrieval. */
+const RELEVANT_EXTS: ReadonlySet<string> = new Set([
+	".ts",
+	".tsx",
+	".js",
+	".jsx",
+	".mjs",
+	".cjs",
+	".md",
+	".markdown",
+	".json",
+	".yaml",
+	".yml",
+]);
+
+/** Result of a single runRetrievalCycle call. */
+export interface RetrievalResult {
+	files: Array<{ path: string; score: number; reason: string }>;
+	cycles: number;
+	converged: boolean;
+	usedFallback: boolean;
+}
+
+interface RipgrepAvailable {
+	available: boolean;
+	version?: string;
+}
+
+let cachedRgCheck: RipgrepAvailable | undefined;
+
+/**
+ * Detect ripgrep availability once per process. Uses `rg --version` and
+ * catches ENOENT or non-zero exit. Cached so the cost (one spawn) is
+ * paid only on the first retrieval cycle.
+ */
+export async function detectRipgrep(): Promise<RipgrepAvailable> {
+	if (cachedRgCheck !== undefined) return cachedRgCheck;
+	return await new Promise<RipgrepAvailable>((resolve) => {
+		let settled = false;
+		try {
+			const child = spawn("rg", ["--version"], { stdio: ["ignore", "pipe", "pipe"] });
+			let stdout = "";
+			child.stdout?.on("data", (chunk) => {
+				stdout += chunk.toString("utf-8");
+			});
+			child.on("error", () => {
+				if (settled) return;
+				settled = true;
+				cachedRgCheck = { available: false };
+				resolve(cachedRgCheck);
+			});
+			child.on("close", (code) => {
+				if (settled) return;
+				settled = true;
+				if (code === 0) {
+					cachedRgCheck = { available: true, version: stdout.split("\n")[0] ?? undefined };
+				} else {
+					cachedRgCheck = { available: false };
+				}
+				resolve(cachedRgCheck);
+			});
+		} catch {
+			if (settled) return;
+			settled = true;
+			cachedRgCheck = { available: false };
+			resolve(cachedRgCheck);
+		}
+	});
+}
+
+/** @internal Test-only: reset the ripgrep detection cache. */
+export function __test_resetRipgrepCache(): void {
+	cachedRgCheck = undefined;
+}
+
+/**
+ * Tokenize a task + goal string into lowercase keywords, dropping
+ * stopwords. Single-letter tokens and pure-punctuation tokens are
+ * dropped. Output is deduped, original-order preserved.
+ */
+export function tokenizeQuery(task: string, goal: string): string[] {
+	const combined = `${task}\n${goal}`.toLowerCase();
+	const tokens = combined
+		.split(/[^a-z0-9_-]+/)
+		.map((t) => t.trim())
+		.filter((t) => t.length >= 2 && !STOPWORDS.has(t));
+	return [...new Set(tokens)];
+}
+
+/** Reason template for a discovery hit. Exported for tests. */
+export function reasonFor(file: string, keywords: string[]): string {
+	const lower = file.toLowerCase();
+	const hits = keywords.filter((k) => lower.includes(k));
+	if (hits.length === 0) return `matched by relevance score (no direct keyword hit in path)`;
+	return `keyword match: ${hits.join(", ")}`;
+}
+
+/**
+ * Run ripgrep with the given args, returning stdout as a string.
+ * Throws on ENOENT / non-zero exit. Caller handles fallback.
+ */
+function runRipgrep(args: string[], cwd: string): Promise<string> {
+	return new Promise<string>((resolve, reject) => {
+		let settled = false;
+		let stdout = "";
+		let stderr = "";
+		try {
+			const child = spawn("rg", args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
+			child.stdout?.on("data", (chunk) => {
+				stdout += chunk.toString("utf-8");
+			});
+			child.stderr?.on("data", (chunk) => {
+				stderr += chunk.toString("utf-8");
+			});
+			child.on("error", (err) => {
+				if (settled) return;
+				settled = true;
+				reject(err);
+			});
+			child.on("close", (code) => {
+				if (settled) return;
+				settled = true;
+				// rg exit code 1 = "no matches" (NOT an error). Any other
+				// non-zero exit IS an error.
+				if (code === 0 || code === 1) {
+					resolve(stdout);
+				} else {
+					reject(new Error(`rg exited ${code}: ${stderr.slice(0, 200)}`));
+				}
+			});
+		} catch (e) {
+			if (settled) return;
+			settled = true;
+			reject(e);
+		}
+	});
+}
+
+/**
+ * In-memory fallback: walk cwd with readdir({recursive:true}), filter
+ * to relevant extensions, score by filename keyword match. Used when
+ * ripgrep is not installed. Mirrors the rg --files path closely so
+ * downstream scoring behaves the same.
+ */
+async function walkFilesFallback(cwd: string, keywords: string[]): Promise<Array<{ path: string; score: number; reason: string }>> {
+	const out: Array<{ path: string; score: number; reason: string }> = [];
+	const lowerCwd = cwd.toLowerCase();
+	async function walk(dir: string): Promise<void> {
+		let entries: fs.Dirent[];
+		try {
+			entries = await fs.promises.readdir(dir, { withFileTypes: true });
+		} catch {
+			return;
+		}
+		for (const entry of entries) {
+			if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
+			const full = path.join(dir, entry.name);
+			if (entry.isDirectory()) {
+				await walk(full);
+				continue;
+			}
+			if (!entry.isFile()) continue;
+			const ext = path.extname(entry.name).toLowerCase();
+			if (!RELEVANT_EXTS.has(ext)) continue;
+			// scoreRelevance expects file content, but for the fallback we
+			// only have the filename. Use the filename as a proxy (the
+			// existing scoreRelevance handles short content gracefully —
+			// path match contributes 0.3, content match 0.05 × log2(N)).
+			const content = "";
+			const score = scoreRelevance(full, content, keywords);
+			if (score > 0) {
+				out.push({ path: path.relative(lowerCwd, full), score, reason: reasonFor(full, keywords) });
+			}
+		}
+	}
+	await walk(cwd);
+	return out;
+}
+
+/**
+ * Iterate up to MAX_CYCLES times. Each cycle: discover files, score
+ * them, check convergence. Stop early on convergence or when
+ * shouldContinue returns false.
+ */
+export async function runRetrievalCycle(task: string, goal: string, cwd: string): Promise<RetrievalResult> {
+	const keywords = tokenizeQuery(task, goal);
+	if (keywords.length === 0) {
+		return { files: [], cycles: 0, converged: true, usedFallback: false };
+	}
+	const rg = await detectRipgrep();
+	const useRg = rg.available;
+	let usedFallback = !useRg;
+	const evaluations: RelevanceEvaluation[] = [];
+	let cycle = 0;
+	let converged = false;
+	for (; cycle < MAX_CYCLES; cycle++) {
+		let discovered: string[] = [];
+		try {
+			if (useRg) {
+				// Cycle 0: enumerate all relevant files via `rg --files`.
+				// Later cycles: filter by keywords via `rg --files | rg pattern`.
+				// We use the simpler `rg --files` + filter strategy because
+				// `rg --json` parsing adds complexity for marginal gain.
+				// `rg --files` respects .gitignore by default; we add an
+				// explicit -g '!node_modules' and -g '!.git' to be safe on
+				// repos that don't ignore them.
+				const stdout = await runRipgrep(["--files", "-g", "!node_modules", "-g", "!.git", cwd], cwd);
+				discovered = stdout
+					.split("\n")
+					.map((p) => p.trim())
+					.filter((p) => p && RELEVANT_EXTS.has(path.extname(p).toLowerCase()))
+					.map((p) => path.relative(cwd, p));
+			} else {
+				discovered = (await walkFilesFallback(cwd, keywords)).map((f) => f.path);
+			}
+		} catch {
+			// rg errored mid-run — switch to fallback for this cycle.
+			usedFallback = true;
+			discovered = (await walkFilesFallback(cwd, keywords)).map((f) => f.path);
+		}
+		// Score each discovered file. Path-only scoring (no file read) so
+		// we don't slow down prompt building for hundreds of files.
+		const seenInThisCycle = new Set<string>();
+		for (const relPath of discovered) {
+			if (seenInThisCycle.has(relPath)) continue;
+			seenInThisCycle.add(relPath);
+			const absPath = path.isAbsolute(relPath) ? relPath : path.join(cwd, relPath);
+			const score = scoreRelevance(absPath, "", keywords);
+			if (score > 0) {
+				evaluations.push({
+					path: absPath,
+					relevance: score,
+					reason: reasonFor(absPath, keywords),
+					missingContext: [],
+				});
+			}
+		}
+		converged = hasConverged(evaluations);
+		if (converged) break;
+		if (!shouldContinue(evaluations, cycle)) break;
+	}
+	// Sort by score desc, take top N (5..10).
+	evaluations.sort((a, b) => b.relevance - a.relevance);
+	const cap = Math.min(MAX_SUGGESTED_FILES, Math.max(MIN_SUGGESTED_FILES, evaluations.length));
+	const top = evaluations.slice(0, cap).map((e) => ({
+		path: path.isAbsolute(e.path) ? path.relative(cwd, e.path) : e.path,
+		score: e.relevance,
+		reason: e.reason,
+	}));
+	return { files: top, cycles: cycle, converged, usedFallback };
+}
+
+/**
+ * Render the "Suggested files to read" section for injection into the
+ * worker prompt. Returns an empty string when retrieval returned no
+ * files. Format is a markdown bullet list (one line per file) prefixed
+ * with a heading so the worker can `grep` for it.
+ */
+export function renderSuggestedFilesSection(result: RetrievalResult): string {
+	if (result.files.length === 0) return "";
+	const lines: string[] = [
+		`# Suggested files to read (top-${result.files.length} by retrieval score)`,
+		`Retrieval ran for ${result.cycles} cycle(s)${result.usedFallback ? " (in-memory fallback, rg unavailable)" : ""}${result.converged ? " and converged" : ""}.`,
+		"",
+	];
+	for (const file of result.files) {
+		lines.push(`- ${file.path} — ${file.reason} (score ${file.score.toFixed(2)})`);
+	}
+	return lines.join("\n");
+}

package/src/runtime/task-runner/run-projection.ts

@@ -1,6 +1,5 @@
-import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
 import type { MailboxMessage } from "../../state/mailbox.ts";
-import type { ArtifactDescriptor } from "../../state/types.ts";
+import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
 
 export interface RunProjectionSource {
 	kind: "events" | "mailbox" | "artifacts" | "ui_metadata" | "runtime_metadata";
@@ -44,7 +43,11 @@ export function transformRunContextBeforeWorkerStart(input: {
 	const completedTasks = input.tasks.filter((t) => t.status === "completed" || t.status === "failed");
 	if (completedTasks.length > 0) {
 		const tasks = completedTasks.slice(0, maxEvents);
-		sources.push({ kind: "events", bounded: true, reference: `tasks:${tasks.length}/${completedTasks.length}` });
+		sources.push({
+			kind: "events",
+			bounded: true,
+			reference: `tasks:${tasks.length}/${completedTasks.length}`,
+		});
 		lines.push(`Previous tasks (${tasks.length}/${completedTasks.length}):`);
 		for (const task of tasks) {
 			lines.push(`- ${task.id}: ${task.status}${task.error ? ` (${task.error})` : ""}`);
@@ -54,7 +57,11 @@ export function transformRunContextBeforeWorkerStart(input: {
 	// Project pending mailbox that is relevant to this worker
 	if (input.pendingMailbox.length > 0) {
 		const messages = input.pendingMailbox.slice(0, maxMailbox);
-		sources.push({ kind: "mailbox", bounded: true, reference: `mailbox:${messages.length}/${input.pendingMailbox.length}` });
+		sources.push({
+			kind: "mailbox",
+			bounded: true,
+			reference: `mailbox:${messages.length}/${input.pendingMailbox.length}`,
+		});
 		lines.push(`Pending messages (${messages.length}/${input.pendingMailbox.length}):`);
 		for (const msg of messages) {
 			lines.push(`- ${msg.kind ?? "message"}: ${msg.body.slice(0, 100)}`);
@@ -64,7 +71,11 @@ export function transformRunContextBeforeWorkerStart(input: {
 	// Project artifact references (not content)
 	if (input.artifacts.length > 0) {
 		const artifacts = input.artifacts.slice(0, maxArtifacts);
-		sources.push({ kind: "artifacts", bounded: true, reference: `artifacts:${artifacts.length}/${input.artifacts.length}` });
+		sources.push({
+			kind: "artifacts",
+			bounded: true,
+			reference: `artifacts:${artifacts.length}/${input.artifacts.length}`,
+		});
 		lines.push(`Available artifacts (${artifacts.length}/${input.artifacts.length}):`);
 		for (const art of artifacts) {
 			lines.push(`- ${art.kind} (${art.producer})`);
@@ -72,8 +83,16 @@ export function transformRunContextBeforeWorkerStart(input: {
 	}
 
 	// Metadata markers — not injected as prompt instructions
-	sources.push({ kind: "ui_metadata", bounded: false, reference: "excluded_from_prompt" });
-	sources.push({ kind: "runtime_metadata", bounded: false, reference: "excluded_from_prompt" });
+	sources.push({
+		kind: "ui_metadata",
+		bounded: false,
+		reference: "excluded_from_prompt",
+	});
+	sources.push({
+		kind: "runtime_metadata",
+		bounded: false,
+		reference: "excluded_from_prompt",
+	});
 
 	return {
 		sources,
@@ -101,4 +120,4 @@ export function convertRunHistoryToWorkerPrompt(input: {
 		"",
 		`Projection sources: ${projection.sources.map((s) => s.kind).join(", ")}`,
 	].join("\n");
-}
+}

package/src/runtime/task-runner/state-helpers.ts

@@ -1,12 +1,13 @@
 import * as fs from "node:fs";
+import { flushPendingAtomicWrites } from "../../state/atomic-write.ts";
+import { withRunLockSync } from "../../state/locks.ts";
+import { loadRunManifestById, saveRunTasksCoalesced } from "../../state/state-store.ts";
 import type { TaskCheckpointState, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
-import { loadRunManifestById, saveRunTasks } from "../../state/state-store.ts";
-import { recordFromTask, upsertCrewAgent } from "../crew-agent-records.ts";
 import { logInternalError } from "../../utils/internal-error.ts";
-import { withRunLockSync } from "../../state/locks.ts";
+import { recordFromTask, upsertCrewAgent } from "../crew-agent-records.ts";
 
 export function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): TeamTaskState[] {
-	return tasks.map((task) => task.id === updated.id ? updated : task);
+	return tasks.map((task) => (task.id === updated.id ? updated : task));
 }
 
 /**
@@ -25,7 +26,12 @@ export function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): Team
  *
  * @param checkpointPhase - Optional checkpoint phase to include in the task state alongside the update.
  */
-export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks: TeamTaskState[], updated: TeamTaskState, checkpointPhase?: TaskCheckpointState["phase"]): TeamTaskState[] {
+export function persistSingleTaskUpdate(
+	manifest: TeamRunManifest,
+	fallbackTasks: TeamTaskState[],
+	updated: TeamTaskState,
+	checkpointPhase?: TaskCheckpointState["phase"],
+): TeamTaskState[] {
 	let baseMtime = 0;
 	try {
 		baseMtime = fs.statSync(manifest.tasksPath).mtimeMs;
@@ -38,12 +44,27 @@ export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks
 
 	// Build the task with optional checkpoint phase
 	const taskWithCheckpoint = checkpointPhase
-		? { ...updated, checkpoint: { phase: checkpointPhase, updatedAt: new Date().toISOString() } }
+		? {
+				...updated,
+				checkpoint: {
+					phase: checkpointPhase,
+					updatedAt: new Date().toISOString(),
+				},
+			}
 		: updated;
 
 	try {
 		return withRunLockSync(manifest, () => {
-			retryLoop: for (let attempt = 0; attempt < 100; attempt++) {
+			for (let attempt = 0; attempt < 100; attempt++) {
+				// F4: persistSingleTaskUpdate now uses saveRunTasksCoalesced below
+				// (50ms debounce window). Read-modify-write loops are unsafe under
+				// coalescing — a parallel writer's buffered write is invisible to
+				// loadRunManifestById until it actually lands. Force any pending
+				// coalesced writes to flush first so this read sees the latest
+				// durable state. Without this guard, a parallel writer could
+				// overwrite our buffered write between our load and our (async)
+				// fsync, silently losing the intermediate update.
+				flushPendingAtomicWrites();
 				const latest = loadRunManifestById(manifest.cwd, manifest.runId)?.tasks ?? fallbackTasks;
 				merged = updateTask(latest, taskWithCheckpoint);
 
@@ -58,7 +79,7 @@ export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks
 				if (currentMtime !== baseMtime) {
 					// Another writer committed — their update is in latest, re-merge on top
 					baseMtime = currentMtime;
-					continue retryLoop;
+					continue;
 				}
 
 				// No concurrent writer — check that our merged result is based on the
@@ -72,7 +93,7 @@ export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks
 				}
 				if (recheckMtime !== baseMtime) {
 					baseMtime = recheckMtime;
-					continue retryLoop;
+					continue;
 				}
 
 				// Final pre-write mtime check to catch any concurrent writer that completed
@@ -86,10 +107,10 @@ export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks
 				if (preWriteMtime !== baseMtime) {
 					// Another writer committed — retry
 					baseMtime = preWriteMtime;
-					continue retryLoop;
+					continue;
 				}
 
-				break retryLoop;
+				break;
 			}
 
 			if (merged === undefined) {
@@ -98,7 +119,12 @@ export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks
 			}
 
 			try {
-				saveRunTasks(manifest, merged);
+				// F4: coalesced write inside the withRunLockSync critical section.
+				// The mtime CAS retry loop above still guards against concurrent
+				// non-coalesced writers; the flushPendingAtomicWrites() guard at
+				// the top of the retry loop ensures reads see any other coalesced
+				// writer's flushed-before-this-call state.
+				saveRunTasksCoalesced(manifest, merged);
 			} catch (err) {
 				logInternalError("persistSingleTaskUpdate", err);
 				throw err;
@@ -113,8 +139,18 @@ export function persistSingleTaskUpdate(manifest: TeamRunManifest, fallbackTasks
 	}
 }
 
-export function checkpointTask(manifest: TeamRunManifest, tasks: TeamTaskState[], task: TeamTaskState, phase: TaskCheckpointState["phase"], childPid?: number): { task: TeamTaskState; tasks: TeamTaskState[] } {
-	const checkpoint: TaskCheckpointState = { phase, updatedAt: new Date().toISOString(), ...(childPid ? { childPid } : task.checkpoint?.childPid ? { childPid: task.checkpoint.childPid } : {}) };
+export function checkpointTask(
+	manifest: TeamRunManifest,
+	tasks: TeamTaskState[],
+	task: TeamTaskState,
+	phase: TaskCheckpointState["phase"],
+	childPid?: number,
+): { task: TeamTaskState; tasks: TeamTaskState[] } {
+	const checkpoint: TaskCheckpointState = {
+		phase,
+		updatedAt: new Date().toISOString(),
+		...(childPid ? { childPid } : task.checkpoint?.childPid ? { childPid: task.checkpoint.childPid } : {}),
+	};
 	const nextTask = { ...task, checkpoint };
 	const nextTasks = persistSingleTaskUpdate(manifest, updateTask(tasks, nextTask), nextTask);
 	try {

package/src/runtime/task-runner/tail-read.ts

@@ -5,11 +5,7 @@ import * as fs from "node:fs";
  * If the file exceeds maxBytes, reads only the last maxBytes and snaps
  * to the nearest newline boundary to avoid partial JSONL lines.
  */
-export function tailReadWithLineSnap(
-	filePath: string,
-	maxBytes: number,
-	fallbackContent: string,
-): string {
+export function tailReadWithLineSnap(filePath: string, maxBytes: number, fallbackContent: string): string {
 	if (!fs.existsSync(filePath)) return fallbackContent;
 	const stat = fs.statSync(filePath);
 	if (stat.size <= maxBytes) return fs.readFileSync(filePath, "utf-8");
@@ -23,4 +19,4 @@ export function tailReadWithLineSnap(
 	} finally {
 		fs.closeSync(fd);
 	}
-}
+}