pi-crew 0.1.46 → 0.1.49

package/src/runtime/notebook-helpers.ts

@@ -0,0 +1,90 @@
+export interface NotebookCell {
+  index: number;
+  cellType: "code" | "markdown" | "raw";
+  source: string;
+  outputs?: unknown[];
+  metadata?: Record<string, unknown>;
+}
+
+export interface Notebook {
+  cells: NotebookCell[];
+  metadata?: Record<string, unknown>;
+  nbformat: number;
+}
+
+/** Check if a file path is a notebook */
+export function isNotebookPath(filePath: string): boolean {
+  return filePath.toLowerCase().endsWith(".ipynb");
+}
+
+function normalizeSource(source: unknown): string {
+  if (typeof source === "string") return source;
+  if (Array.isArray(source)) return source.join("");
+  return "";
+}
+
+/** Parse a .ipynb JSON file into a Notebook structure */
+export function parseNotebook(content: string): Notebook {
+  let raw: unknown;
+  try {
+    raw = JSON.parse(content);
+  } catch {
+    return { cells: [], nbformat: 4 };
+  }
+  if (typeof raw !== "object" || raw === null) return { cells: [], nbformat: 4 };
+  const obj = raw as Record<string, unknown>;
+  const nbformat = typeof obj.nbformat === "number" ? obj.nbformat : 4;
+  const rawCells = Array.isArray(obj.cells) ? obj.cells : [];
+  const cells: NotebookCell[] = rawCells
+    .map((c: unknown, i: number): NotebookCell | null => {
+      if (typeof c !== "object" || c === null) return null;
+      const cell = c as Record<string, unknown>;
+      const cellType = cell.cell_type;
+      if (cellType !== "code" && cellType !== "markdown" && cellType !== "raw") return null;
+      return {
+        index: i,
+        cellType,
+        source: normalizeSource(cell.source),
+        outputs: Array.isArray(cell.outputs) ? cell.outputs : undefined,
+        metadata:
+          cell.metadata && typeof cell.metadata === "object" && !Array.isArray(cell.metadata)
+            ? (cell.metadata as Record<string, unknown>)
+            : undefined,
+      };
+    })
+    .filter((c: NotebookCell | null): c is NotebookCell => c !== null);
+  const metadata =
+    obj.metadata && typeof obj.metadata === "object" && !Array.isArray(obj.metadata)
+      ? (obj.metadata as Record<string, unknown>)
+      : undefined;
+  return { cells, metadata, nbformat };
+}
+
+/** Get a specific cell by index */
+export function getCell(notebook: Notebook, index: number): NotebookCell | undefined {
+  return notebook.cells.find((c) => c.index === index);
+}
+
+/** Update a cell's source content, returning a new Notebook */
+export function updateCell(notebook: Notebook, index: number, source: string): Notebook {
+  const cells = notebook.cells.map((c) =>
+    c.index === index ? { ...c, source } : c,
+  );
+  return { ...notebook, cells };
+}
+
+/** Serialize a Notebook back to .ipynb JSON string */
+export function serializeNotebook(notebook: Notebook): string {
+  const raw = {
+    nbformat: notebook.nbformat,
+    nbformat_minor: 5,
+    metadata: notebook.metadata ?? {},
+    cells: notebook.cells.map((c) => ({
+      cell_type: c.cellType,
+      source: c.source,
+      metadata: c.metadata ?? {},
+      ...(c.cellType === "code" ? { outputs: c.outputs ?? [], execution_count: null } : {}),
+    })),
+  };
+  return JSON.stringify(raw, null, 2) + "\n";
+}

package/src/runtime/orphan-sentinel.ts

@@ -0,0 +1,7 @@
+/**
+ * Orphan sentinel — DEPRECATED, replaced by parent-guard.ts.
+ *
+ * Keeping this file as a no-op placeholder until all references are removed.
+ * Workers now self-monitor via startParentGuard() instead of relying on
+ * an external sentinel process.
+ */

package/src/runtime/output-validator.ts

@@ -0,0 +1,187 @@
+/**
+ * Phase 4 (caveman): Output format validation for live-session workers.
+ *
+ * Validates that worker output conforms to the structured output contract
+ * for the given role. If validation fails, returns structured error info
+ * that can be used for retry or fallback.
+ *
+ * Inspired by caveman's validate.py — check structural preservation
+ * (headings, code blocks, URLs) after compression.
+ */
+
+/** Role-specific output format patterns — constructed fresh per call to avoid /g lastIndex leak */
+const ROLE_PATTERN_DEFS: Record<string, () => RegExp> = {
+	explorer: () => /^(\S+:\d+|Defs:|Refs:|Callers:|Tests:|Sites:|No match\.|totals:)/m,
+	executor: () => /^(\S+:\d+(-\d+)? — .{1,80}\.|verified:|too-big\.|needs-confirm\.|ambiguous\.|regressed\.)/m,
+	reviewer: () => /^([^:\s]+:\d+:\s+\p{Emoji_Presentation}|No issues\.|totals:)/mu,
+	"security-reviewer": () => /^([^:\s]+:\d+:\s+\p{Emoji_Presentation}|No issues\.|totals:)/mu,
+	verifier: () => /^(PASS:|FAIL:)/m,
+};
+
+/** Fresh RegExp factories for structural preservation checks (avoids /g lastIndex leak) */
+const makeUrlRe = () => /\bhttps?:\/\/[^\s<>)\]"',;]+/gi;
+const makeFencedCodeRe = () => /```[\s\S]*?```/g;
+const makeInlineCodeRe = () => /`[^`\n]+`/g;
+const makeHeadingRe = () => /^#{1,6}\s+.+/gm;
+
+export interface OutputValidationResult {
+	/** Whether the output passes validation */
+	valid: boolean;
+	/** Whether the output follows the role's contract format */
+	formatMatch: boolean;
+	/** Whether structural elements (code, URLs, headings) are preserved */
+	structurePreserved: boolean;
+	/** Specific issues found */
+	issues: string[];
+}
+
+/**
+ * Validate worker output against role-specific contract + structural preservation.
+ */
+export function validateWorkerOutput(role: string, output: string): OutputValidationResult {
+	const issues: string[] = [];
+
+	// Empty output always fails
+	if (!output || !output.trim()) {
+		return { valid: false, formatMatch: false, structurePreserved: false, issues: ["Empty output"] };
+	}
+
+	// Check role-specific format
+	const patternFactory = ROLE_PATTERN_DEFS[role];
+	const pattern = patternFactory ? patternFactory() : undefined;
+	const formatMatch = !pattern || pattern.test(output);
+	if (!formatMatch) {
+		issues.push(`Output does not match expected ${role} contract format`);
+	}
+
+	// Check structural preservation (code blocks, URLs, headings)
+	let structurePreserved = true;
+	const trimmedOutput = output.trim();
+
+	// Detect if output was truncated mid-code-block
+	const opens = (trimmedOutput.match(/```/g) ?? []).length;
+	if (opens % 2 !== 0) {
+		structurePreserved = false;
+		issues.push("Unclosed code block — output may be truncated");
+	}
+
+	// Check for malformed URLs
+	const urls = trimmedOutput.match(makeUrlRe()) ?? [];
+	for (const url of urls) {
+		if (url.endsWith(".") || url.endsWith(",")) {
+			structurePreserved = false;
+			issues.push(`URL with trailing punctuation: ${url.slice(-20)}`);
+		}
+	}
+
+	return {
+		valid: formatMatch && structurePreserved,
+		formatMatch,
+		structurePreserved,
+		issues,
+	};
+}
+
+/**
+ * Extract structured findings from reviewer output.
+ * Returns array of { file, line, severity, message } objects.
+ */
+export function parseReviewerFindings(output: string): Array<{ file: string; line: number; severity: string; message: string }> {
+	const findings: Array<{ file: string; line: number; severity: string; message: string }> = [];
+	const lines = output.split("\n");
+
+	const SEVERITY_MAP: Record<string, string> = {
+		"🔴": "bug",
+		"🟡": "risk",
+		"🔵": "nit",
+		"❓": "question",
+	};
+
+	for (const line of lines) {
+		// Match: path/to/file.ts:42: 🔴 bug: problem. fix.
+		const match = line.match(/^([^:\s]+):(\d+):\s+(\p{Emoji_Presentation}) (\w+):\s+(.+)/u);
+		if (match) {
+			findings.push({
+				file: match[1],
+				line: Number(match[2]),
+				severity: SEVERITY_MAP[match[3]] ?? match[3],
+				message: match[5].trim(),
+			});
+		}
+	}
+
+	return findings;
+}
+
+/**
+ * Extract explorer results from structured output.
+ * Returns array of { file, line, symbol, note } objects.
+ */
+export function parseExplorerResults(output: string): Array<{ file: string; line: number; symbol: string; note: string }> {
+	const results: Array<{ file: string; line: number; symbol: string; note: string }> = [];
+	const lines = output.split("\n");
+
+	for (const line of lines) {
+		// Match: path/to/file.ts:42 — `symbol` — note
+		const match = line.match(/^[- ]*(\S+):(\d+)\s*[—–-]\s*`([^`]+)`\s*[—–-]\s*(.+)/);
+		if (match) {
+			results.push({
+				file: match[1],
+				line: Number(match[2]),
+				symbol: match[3],
+				note: match[4].trim(),
+			});
+		}
+	}
+
+	return results;
+}
+
+/**
+ * Validate that compressed prose preserves structural elements from original.
+ * Returns list of specific issues (empty = valid).
+ */
+export function validateCompressionPreservation(original: string, compressed: string): string[] {
+	const issues: string[] = [];
+
+	// Check code blocks preserved
+	const origBlocks = original.match(makeFencedCodeRe()) ?? [];
+	const compBlocks = compressed.match(makeFencedCodeRe()) ?? [];
+	if (origBlocks.length !== compBlocks.length) {
+		issues.push(`Code block count: ${origBlocks.length} → ${compBlocks.length}`);
+	}
+	for (let i = 0; i < Math.min(origBlocks.length, compBlocks.length); i++) {
+		if (origBlocks[i] !== compBlocks[i]) {
+			issues.push(`Code block ${i + 1} content changed`);
+		}
+	}
+
+	// Check URLs preserved
+	const origUrls = new Set(original.match(makeUrlRe()) ?? []);
+	const compUrls = new Set(compressed.match(makeUrlRe()) ?? []);
+	for (const url of origUrls) {
+		if (!compUrls.has(url)) {
+			issues.push(`URL lost: ${url.slice(0, 60)}...`);
+		}
+	}
+
+	// Check inline code preserved
+	const origInline = original.match(makeInlineCodeRe()) ?? [];
+	const compInline = compressed.match(makeInlineCodeRe()) ?? [];
+	const origInlineSet = new Set(origInline);
+	const compInlineSet = new Set(compInline);
+	for (const code of origInlineSet) {
+		if (!compInlineSet.has(code)) {
+			issues.push(`Inline code lost: ${code}`);
+		}
+	}
+
+	// Check headings preserved
+	const origHeadings = original.match(makeHeadingRe()) ?? [];
+	const compHeadings = compressed.match(makeHeadingRe()) ?? [];
+	if (origHeadings.length !== compHeadings.length) {
+		issues.push(`Heading count: ${origHeadings.length} → ${compHeadings.length}`);
+	}
+
+	return issues;
+}

package/src/runtime/parallel-research.ts

@@ -1,44 +1,44 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
-
-export function sourcePiProjects(cwd: string): string[] {
-	const sourceDir = path.join(cwd, "Source");
-	try {
-		return fs.readdirSync(sourceDir, { withFileTypes: true })
-			.filter((entry) => entry.isDirectory() && entry.name.startsWith("pi-"))
-			.map((entry) => `Source/${entry.name}`)
-			.sort();
-	} catch {
-		return [];
-	}
-}
-
-export function chunkProjects(projects: string[], target = 6): string[][] {
-	const chunks = Array.from({ length: Math.min(Math.max(1, target), Math.max(1, projects.length)) }, () => [] as string[]);
-	projects.forEach((project, index) => chunks[index % chunks.length]!.push(project));
-	return chunks.filter((chunk) => chunk.length > 0);
-}
-
-export function expandParallelResearchWorkflow(workflow: WorkflowConfig, cwd: string): WorkflowConfig {
-	if (workflow.name !== "parallel-research") return workflow;
-	const projects = sourcePiProjects(cwd);
-	if (projects.length === 0) return workflow;
-	const chunks = chunkProjects(projects, Math.min(8, Math.max(4, Math.ceil(projects.length / 3))));
-	const exploreSteps: WorkflowStep[] = chunks.map((paths, index) => ({
-		id: `explore-shard-${index + 1}`,
-		role: "explorer",
-		parallelGroup: "explore",
-		reads: paths,
-		task: [`Explore this dynamic shard for: {goal}`, "", "Paths:", ...paths.map((item) => `- ${item}`), "", "Focus on purpose, architecture, runtime/UI patterns, package config, docs, and lessons for pi-crew."].join("\n"),
-	}));
-	return {
-		...workflow,
-		steps: [
-			{ id: "discover", role: "explorer", parallelGroup: "inventory", task: `Quickly inventory and validate ${projects.length} pi-* projects for: {goal}\n\nProjects:\n${projects.map((item) => `- ${item}`).join("\n")}\n\nDo not block shard work; summarize routing notes only.` },
-			...exploreSteps,
-			{ id: "synthesize", role: "analyst", dependsOn: exploreSteps.map((step) => step.id), task: "Synthesize all dynamic shard findings. Identify common patterns, gaps, and concrete recommendations. Use discover output if available, but prioritize completed shard outputs." },
-			{ id: "write", role: "writer", dependsOn: ["synthesize"], output: "research-summary.md", task: "Write a concise final summary with evidence, risks, and actionable next steps." },
-		],
-	};
-}
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
+
+export function sourcePiProjects(cwd: string): string[] {
+	const sourceDir = path.join(cwd, "Source");
+	try {
+		return fs.readdirSync(sourceDir, { withFileTypes: true })
+			.filter((entry) => entry.isDirectory() && entry.name.startsWith("pi-"))
+			.map((entry) => `Source/${entry.name}`)
+			.sort();
+	} catch {
+		return [];
+	}
+}
+
+export function chunkProjects(projects: string[], target = 6): string[][] {
+	const chunks = Array.from({ length: Math.min(Math.max(1, target), Math.max(1, projects.length)) }, () => [] as string[]);
+	projects.forEach((project, index) => chunks[index % chunks.length]!.push(project));
+	return chunks.filter((chunk) => chunk.length > 0);
+}
+
+export function expandParallelResearchWorkflow(workflow: WorkflowConfig, cwd: string): WorkflowConfig {
+	if (workflow.name !== "parallel-research") return workflow;
+	const projects = sourcePiProjects(cwd);
+	if (projects.length === 0) return workflow;
+	const chunks = chunkProjects(projects, Math.min(8, Math.max(4, Math.ceil(projects.length / 3))));
+	const exploreSteps: WorkflowStep[] = chunks.map((paths, index) => ({
+		id: `explore-shard-${index + 1}`,
+		role: "explorer",
+		parallelGroup: "explore",
+		reads: paths,
+		task: [`Explore this dynamic shard for: {goal}`, "", "Paths:", ...paths.map((item) => `- ${item}`), "", "Focus on purpose, architecture, runtime/UI patterns, package config, docs, and lessons for pi-crew."].join("\n"),
+	}));
+	return {
+		...workflow,
+		steps: [
+			{ id: "discover", role: "explorer", parallelGroup: "inventory", task: `Quickly inventory and validate ${projects.length} pi-* projects for: {goal}\n\nProjects:\n${projects.map((item) => `- ${item}`).join("\n")}\n\nDo not block shard work; summarize routing notes only.` },
+			...exploreSteps,
+			{ id: "synthesize", role: "analyst", dependsOn: exploreSteps.map((step) => step.id), task: "Synthesize all dynamic shard findings. Identify common patterns, gaps, and concrete recommendations. Use discover output if available, but prioritize completed shard outputs." },
+			{ id: "write", role: "writer", dependsOn: ["synthesize"], output: "research-summary.md", task: "Write a concise final summary with evidence, risks, and actionable next steps." },
+		],
+	};
+}

package/src/runtime/parallel-utils.ts

@@ -58,6 +58,63 @@ export async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item:
 	return results;
 }
 
+/**
+ * Phase 6: mapConcurrent with AbortSignal and fail-fast support.
+ * On abort: returns partial results (may contain undefined entries).
+ * On error: throws immediately (fail-fast) and cancels remaining work.
+ */
+export async function mapConcurrentWithSignal<T, R>(
+	items: T[],
+	limit: number,
+	fn: (item: T, i: number, signal: AbortSignal) => Promise<R>,
+	signal?: AbortSignal,
+): Promise<{ results: (R | undefined)[]; aborted: boolean }> {
+	const safeLimit = Math.max(1, Math.floor(limit) || 1);
+	const results: (R | undefined)[] = new Array(items.length);
+	let next = 0;
+	let aborted = false;
+
+	const abortController = new AbortController();
+	const workerSignal = signal
+		? AbortSignal.any([signal, abortController.signal])
+		: abortController.signal;
+
+	let rejectFirst: (error: unknown) => void;
+	const firstErrorPromise = new Promise<never>((_, reject) => {
+		rejectFirst = reject;
+	});
+
+	const worker = async (): Promise<void> => {
+		while (!workerSignal.aborted) {
+			const i = next++;
+			if (i >= items.length) return;
+			try {
+				results[i] = await fn(items[i], i, workerSignal);
+			} catch (error) {
+				if (!workerSignal.aborted) {
+					abortController.abort();
+					rejectFirst(error);
+					throw error;
+				}
+			}
+		}
+	};
+
+	const workers = Array.from({ length: Math.min(safeLimit, items.length) }, () => worker());
+
+	try {
+		await Promise.race([Promise.all(workers), firstErrorPromise]);
+	} catch (error) {
+		if (signal?.aborted) {
+			aborted = true;
+			return { results, aborted };
+		}
+		throw error;
+	}
+
+	return { results, aborted: signal?.aborted ?? false };
+}
+
 export interface ParallelTaskResult {
 	agent: string;
 	taskIndex?: number;

package/src/runtime/parent-guard.ts

@@ -0,0 +1,80 @@
+/**
+ * Parent liveness guard for pi-crew worker processes.
+ *
+ * Workers call `startParentGuard(parentPid)` at startup. A lightweight
+ * interval checks if the parent PID is still alive. When the parent dies
+ * (SIGKILL, crash, power loss, terminal close), the worker self-terminates
+ * immediately — no sentinel process needed.
+ *
+ * Note: `process.kill(pid, 0)` works on both Unix and Windows in Node.js
+ * for checking process existence. On Windows, it may throw for processes
+ * owned by other users (permission error), but correctly detects dead PIDs.
+ *
+ * Usage in worker entry points:
+ * ```ts
+ * const parentPid = Number(process.env.PI_CREW_PARENT_PID);
+ * if (parentPid > 0) startParentGuard(parentPid);
+ * ```
+ */
+
+const POLL_INTERVAL_MS = 3_000;
+
+let guardInterval: ReturnType<typeof setInterval> | undefined;
+
+function isPidAlive(pid: number): boolean {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+function selfTerminate(parentPid: number): never {
+	// Best-effort: try to log why we're dying
+	try {
+		if (typeof process.stderr?.write === "function") {
+			process.stderr.write(`[pi-crew] Parent process ${parentPid} is dead — self-terminating worker ${process.pid}\n`);
+		}
+	} catch {
+		// Ignore
+	}
+	process.exit(124); // 124 = "parent died" exit code
+}
+
+/**
+ * Start a lightweight poll that checks if the parent process is still alive.
+ * If the parent dies, this worker exits immediately with code 124.
+ *
+ * The interval is `unref()`'d so it doesn't keep the event loop alive
+ * on its own — but if the worker has other work (LLM calls, tool execution),
+ * the check continues running in the background.
+ */
+export function startParentGuard(parentPid: number): void {
+	if (!parentPid || !Number.isFinite(parentPid) || parentPid <= 0) return;
+
+	// Immediate check — if parent is already dead, don't even start
+	if (!isPidAlive(parentPid)) {
+		selfTerminate(parentPid);
+	}
+
+	guardInterval = setInterval(() => {
+		if (!isPidAlive(parentPid)) {
+			if (guardInterval) clearInterval(guardInterval);
+			selfTerminate(parentPid);
+		}
+	}, POLL_INTERVAL_MS);
+
+	guardInterval.unref();
+}
+
+/**
+ * Stop the parent guard. Called when the worker finishes normally
+ * and doesn't need to watch the parent anymore.
+ */
+export function stopParentGuard(): void {
+	if (guardInterval) {
+		clearInterval(guardInterval);
+		guardInterval = undefined;
+	}
+}