pi-crew 0.9.53 → 0.9.55

package/src/runtime/important-line-classifier.ts

@@ -0,0 +1,140 @@
+/**
+ * Important-Line Classifier (P0-B) — scan middle slice of a truncated value
+ * for diagnostic lines worth preserving between head and tail.
+ *
+ * Ported from Hypa's `ImportantLineClassifier.cs` (5 regexes) and the
+ * middle-scanning portion of `Stages/TruncationStage.cs:24-46`, adapted to TS
+ * (no `[GeneratedRegex]` AOT) and to pi-crew's head(75%)/tail(25%) split.
+ *
+ * Design rationale:
+ * - Patterns are intentionally OVER-INCLUSIVE. False positives preserve
+ *   harmless lines; false negatives drop critical diagnostics, which is
+ *   unacceptable (the whole point of this module). Hypa uses the same
+ *   over-inclusive design.
+ * - Patterns are evaluated against a WHOLE line, not against the raw
+ *   truncated slice, so a match at a line boundary is reliable.
+ * - The `splitWithImportantLines` helper performs the head/tail split AND
+ *   greedily picks whole important lines from the middle that fit inside
+ *   `slackFactor * maxChars` (default 15% slack). Callers compose their own
+ *   marker using the returned parts — keeping `compactString` (marker
+ *   "compacted ... chars, head+tail preserved") and `readIfSmall` (marker
+ *   "truncated ... bytes, head+tail preserved") backward-compatible when no
+ *   important lines are present.
+ */
+
+/** Diagnostic patterns. Anchored where safe to avoid matching noise. */
+export const IMPORTANT_LINE_PATTERNS: readonly RegExp[] = [
+	// error keywords — NOTE: "warning" is intentionally excluded here; it has
+	// its own case-sensitive pattern below so that the common prose word
+	// "warning" does not over-match. (Hypa does the same split.)
+	/\b(error|failed|exception|fatal|panic)\b/i,
+	// file:line diagnostic — `child-pi.ts:383:`, `App.tsx:42:`
+	/\w+\.\w+:\d+:/,
+	// HTTP 4xx / 5xx — bounded so it does not match phone numbers etc.
+	/\b[45]\d{2}\b/,
+	// k8s / linter "Warning" event (case-sensitive so prose is not matched)
+	/\bWarning\b/,
+	// compiler / linter diagnostic id — `TS2304`, `CS0246`, `ES1234`
+	/\b[A-Z]{2,4}\d{3,5}\b/,
+];
+
+/** True iff `line` matches at least one important-line pattern. */
+export function isImportantLine(line: string): boolean {
+	if (!line) return false;
+	for (const pattern of IMPORTANT_LINE_PATTERNS) {
+		if (pattern.test(line)) return true;
+	}
+	return false;
+}
+
+/**
+ * Extract up to `maxLines` important lines from `text`. Lines are split on
+ * `\n` (also handles `\r\n`). Order preserved; duplicates kept (callers may
+ * want to see the same diagnostic twice if it appears twice — that often
+ * signals a recurring failure).
+ */
+export function extractImportantLines(text: string, maxLines = 30): string[] {
+	if (!text || maxLines <= 0) return [];
+	const out: string[] = [];
+	for (const line of text.split(/\r?\n/)) {
+		if (out.length >= maxLines) break;
+		if (isImportantLine(line)) out.push(line);
+	}
+	return out;
+}
+
+export interface TruncationSplit {
+	/** The first 75% of the value (by char count), verbatim. */
+	head: string;
+	/** The last 25% of the value (by char count), verbatim. */
+	tail: string;
+	/**
+	 * Important lines from the middle slice, greedily picked (whole lines) so
+	 * the joined length fits inside `slackFactor * maxChars`. Empty when
+	 * `preserveImportant` is false OR no important lines are present OR none
+	 * fit the slack budget.
+	 */
+	importantLines: string[];
+	/** `value.length - maxChars` — chars dropped if no important lines preserved. */
+	baseDropped: number;
+}
+
+export interface SplitOptions {
+	/** When false, important-line scanning is skipped (assistant-text mode). */
+	preserveImportant?: boolean;
+	/** Hard cap on candidate lines before slack-budget selection. Default 30. */
+	maxImportantLines?: number;
+	/** Fraction of `maxChars` available for important-line content. Default 0.15. */
+	slackFactor?: number;
+}
+
+/**
+ * Split `value` into head + important-middle + tail, returning the parts.
+ * The caller is responsible for composing the final result (marker + glue)
+ * because the marker wording differs between `compactString` and
+ * `readIfSmall`.
+ *
+ * When no important lines are picked, the returned `importantLines` is `[]`
+ * and the marker wording stays bit-identical to the pre-P0-B format.
+ */
+export function splitWithImportantLines(value: string, maxChars: number, opts: SplitOptions = {}): TruncationSplit {
+	if (value.length <= maxChars) {
+		return { head: value, tail: "", importantLines: [], baseDropped: 0 };
+	}
+	const headLen = Math.floor(maxChars * 0.75);
+	const tailLen = maxChars - headLen;
+	const head = value.slice(0, headLen);
+	const tail = value.slice(value.length - tailLen);
+
+	if (opts.preserveImportant === false) {
+		return {
+			head,
+			tail,
+			importantLines: [],
+			baseDropped: value.length - maxChars,
+		};
+	}
+
+	const slackFactor = opts.slackFactor ?? 0.15;
+	const slackChars = Math.max(0, Math.floor(maxChars * slackFactor));
+	const maxCandidates = opts.maxImportantLines ?? 30;
+	const middle = value.slice(headLen, value.length - tailLen);
+	const candidates = extractImportantLines(middle, maxCandidates);
+
+	// Greedily pick whole lines that fit in the slack budget.
+	const chosen: string[] = [];
+	let used = 0;
+	for (const line of candidates) {
+		const addLen = (chosen.length > 0 ? 1 : 0) + line.length; // '\n' separator
+		if (used + addLen > slackChars) break;
+		chosen.push(line);
+		used += addLen;
+	}
+
+	return {
+		head,
+		tail,
+		importantLines: chosen,
+		baseDropped: value.length - maxChars,
+	};
+}

package/src/runtime/intercom-bridge.ts

@@ -0,0 +1,187 @@
+/**
+ * Intercom bridge — workers can escalate questions to the orchestrator.
+ *
+ * Pattern origin: pi-subagents/src/intercom-bridge.ts — contact_supervisor tool
+ * for child agents to escalate decisions, report progress, or ask questions.
+ *
+ * This module provides the message queue and correlation logic.
+ * The actual tool registration happens in task-runner.ts.
+ */
+
+// ── Types ────────────────────────────────────────────────────────────────
+
+export type IntercomUrgency = "low" | "medium" | "high" | "critical";
+export type IntercomType = "question" | "escalation" | "progress" | "block";
+
+export interface IntercomMessage {
+	type: IntercomType;
+	taskStepId: string;
+	content: string;
+	urgency: IntercomUrgency;
+	timestamp: number;
+	timeout?: number; // ms to wait for response
+}
+
+export interface IntercomResponse {
+	answer: string;
+	source: "orchestrator" | "human" | "timeout";
+	timestamp: number;
+	messageId: string;
+}
+
+// ── Message Queue ────────────────────────────────────────────────────────
+
+interface PendingMessage {
+	message: IntercomMessage;
+	id: string;
+	resolve: (response: IntercomResponse) => void;
+	timer?: ReturnType<typeof setTimeout>;
+}
+
+const MAX_QUEUE_SIZE = 100;
+
+/**
+ * In-process intercom queue for worker→orchestrator communication.
+ *
+ * Each message gets a unique ID. Callers await a response via a Promise.
+ * If no response arrives within the timeout, resolves with source="timeout".
+ */
+export class IntercomQueue {
+	private pending = new Map<string, PendingMessage>();
+	private queue: IntercomMessage[] = [];
+	/** Monotonic sequence counter — guarantees unique IDs even when many
+	 * messages enqueue within the same millisecond (avoids Date.now()
+	 * collision + weak Math.random birthday-paradox flakes under load). */
+	private seq = 0;
+
+	/**
+	 * Enqueue a message and return a promise that resolves when the
+	 * orchestrator responds (or times out).
+	 */
+	enqueue(message: IntercomMessage): Promise<IntercomResponse> {
+		if (this.pending.size >= MAX_QUEUE_SIZE) {
+			// Evict oldest
+			const firstKey = this.pending.keys().next().value;
+			if (firstKey) this.evict(firstKey, "queue_full");
+		}
+
+		const id = `icm-${Date.now().toString(36)}-${(this.seq++).toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+
+		return new Promise<IntercomResponse>((resolve) => {
+			const entry: PendingMessage = { message, id, resolve };
+
+			// Set timeout if specified
+			if (message.timeout && message.timeout > 0) {
+				entry.timer = setTimeout(() => {
+					resolve({
+						answer: "No response received within timeout",
+						source: "timeout",
+						timestamp: Date.now(),
+						messageId: id,
+					});
+					this.pending.delete(id);
+				}, message.timeout);
+				// Defense in depth: never let a pending-message timer block
+				// process exit. The timer is cleared via respond()/evict() in
+				// the normal case; .unref() ensures shutdown isn't blocked if
+				// the queue is abandoned.
+				if (entry.timer && typeof entry.timer.unref === "function") {
+					entry.timer.unref();
+				}
+			}
+
+			this.pending.set(id, entry);
+			this.queue.push({ ...message });
+		});
+	}
+
+	/**
+	 * Respond to a pending message by ID.
+	 */
+	respond(messageId: string, answer: string, source: "orchestrator" | "human" = "orchestrator"): boolean {
+		const entry = this.pending.get(messageId);
+		if (!entry) return false;
+
+		if (entry.timer) clearTimeout(entry.timer);
+
+		entry.resolve({
+			answer,
+			source,
+			timestamp: Date.now(),
+			messageId,
+		});
+
+		this.pending.delete(messageId);
+		return true;
+	}
+
+	/**
+	 * Get all pending messages (for orchestrator to process).
+	 */
+	getPending(): Array<IntercomMessage & { id: string }> {
+		return [...this.pending.entries()].map(([id, entry]) => ({
+			...entry.message,
+			id,
+		}));
+	}
+
+	/**
+	 * Number of pending messages awaiting response.
+	 */
+	get pendingCount(): number {
+		return this.pending.size;
+	}
+
+	/**
+	 * Clean up all pending messages (e.g., on run completion).
+	 */
+	clear(): void {
+		for (const [id, entry] of this.pending) {
+			this.evict(id, "run_complete");
+		}
+		this.queue = [];
+	}
+
+	private evict(id: string, reason: string): void {
+		const entry = this.pending.get(id);
+		if (!entry) return;
+
+		if (entry.timer) clearTimeout(entry.timer);
+
+		entry.resolve({
+			answer: `Message evicted: ${reason}`,
+			source: "timeout",
+			timestamp: Date.now(),
+			messageId: id,
+		});
+
+		this.pending.delete(id);
+	}
+}
+
+// ── Singleton per run ────────────────────────────────────────────────────
+
+const queues = new Map<string, IntercomQueue>();
+
+/**
+ * Get or create an intercom queue for a run.
+ */
+export function getIntercomQueue(runId: string): IntercomQueue {
+	let queue = queues.get(runId);
+	if (!queue) {
+		queue = new IntercomQueue();
+		queues.set(runId, queue);
+	}
+	return queue;
+}
+
+/**
+ * Clean up intercom queue for a completed run.
+ */
+export function cleanupIntercomQueue(runId: string): void {
+	const queue = queues.get(runId);
+	if (queue) {
+		queue.clear();
+		queues.delete(runId);
+	}
+}

package/src/runtime/iteration-hooks.ts

@@ -0,0 +1,305 @@
+/**
+ * Transparent iteration hooks — runs user-supplied before/after task scripts
+ * with structured JSON payload on stdin.
+ *
+ * Distilled from pi-autoresearch's iteration hook pattern.
+ */
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { WINDOWS_ESSENTIAL_ENV_VARS } from "../utils/env-allowlist.ts";
+import { sanitizeEnvSecrets } from "../utils/env-filter.ts";
+import { resolveShellForScript } from "../utils/resolve-shell.ts";
+import { HeadSnapStage } from "./compact-stages/index.ts";
+import { DENIED_METRIC_NAMES } from "./metric-parser.ts";
+
+/** Hook execution stage. */
+export type HookStage = "before" | "after";
+
+/** Payload sent to the hook script via stdin as JSON. */
+export interface HookPayload {
+	event: HookStage;
+	cwd: string;
+	taskId: string;
+	runId: string;
+	taskRole: string;
+	lastResult?: {
+		status: string;
+		description: string;
+		diagnostics?: Record<string, unknown>;
+	} | null;
+	session: {
+		teamName: string;
+		workflowName: string;
+		goal: string;
+		completedTasks: number;
+		totalTasks: number;
+	};
+}
+
+/** Result of executing an iteration hook. */
+export interface HookResult {
+	/** Whether the hook script was actually executed. */
+	fired: boolean;
+	/** Captured stdout (truncated to 8KB). */
+	stdout: string;
+	/** Captured stderr. */
+	stderr: string;
+	/** Exit code of the hook process. */
+	exitCode: number | null;
+	/** Whether the hook timed out. */
+	timedOut: boolean;
+	/** Wall-clock duration in milliseconds. */
+	durationMs: number;
+}
+
+/** Maximum stdout capture size in bytes (8 KB). */
+const MAX_STDOUT_BYTES = 8192;
+
+/** Hook execution timeout in milliseconds (30 seconds). */
+const HOOK_TIMEOUT_MS = 30_000;
+
+/**
+ * Validates that a hook script path is within an allowed directory.
+ * Allowed paths:
+ * - Relative paths starting with ".hooks/" (case-sensitive)
+ * - Absolute paths under $HOME/.pi/hooks/
+ * All other paths are rejected to prevent arbitrary script execution.
+ * @param hookPath - The hook script path to validate
+ * @returns true if the path is allowed, false otherwise
+ */
+export function isAllowedHookPath(hookPath: string): boolean {
+	if (!hookPath || hookPath.trim().length === 0) return false;
+	if (!path.isAbsolute(hookPath)) {
+		// Use path.posix.normalize to ensure forward-slash normalization on all platforms.
+		// On Windows, path.normalize converts .hooks/hook.sh to .hooks\hook.sh (backslash),
+		// breaking the startsWith(".hooks/") check. path.posix.normalize always uses /.
+		const normalized = path.posix.normalize(hookPath);
+		return normalized === ".hooks" || normalized.startsWith(".hooks/");
+	}
+	// Normalize to forward slashes for consistent cross-platform comparison.
+	// e.g., "C:\\Users\\runner\\.pi\\hooks\\hook.sh" matches
+	// "C:\\Users\\runner\\.pi\\hooks/hook.sh" from path.join.
+	const normalizedHookPath = hookPath.replace(/\\/g, "/");
+	const homeHooksNormalized = (process.env.HOME ?? "").replace(/\\/g, "/") + "/.pi/hooks";
+	return normalizedHookPath === homeHooksNormalized || normalizedHookPath.startsWith(homeHooksNormalized + "/");
+}
+
+/**
+ * Create a not-fired result for when the hook script is absent or not executable.
+ */
+function notFiredResult(): HookResult {
+	return {
+		fired: false,
+		stdout: "",
+		stderr: "",
+		exitCode: null,
+		timedOut: false,
+		durationMs: 0,
+	};
+}
+
+/**
+ * Check if a script path exists and is executable.
+ */
+function isScriptRunnable(scriptPath: string): boolean {
+	try {
+		if (!fs.existsSync(scriptPath)) return false;
+
+		// On Windows, X_OK is unreliable — just check F_OK (file exists).
+		// On Unix, check both F_OK and X_OK.
+		if (process.platform === "win32") {
+			fs.accessSync(scriptPath, fs.constants.F_OK);
+		} else {
+			fs.accessSync(scriptPath, fs.constants.F_OK | fs.constants.X_OK);
+		}
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Run an iteration hook script with JSON payload on stdin.
+ *
+ * Spawns `bash <script>` with the hook payload as JSON on stdin.
+ * Captures stdout (capped at 8KB) and stderr. Enforces a 30-second timeout.
+ *
+ * **Security note:** Hook paths are restricted to `.hooks/` relative paths
+ * or `$HOME/.pi/hooks/` absolute paths. All other paths are rejected before
+ * execution.
+ *
+ * @param payload - Structured hook payload
+ * @param hookScriptPath - Absolute or relative path to the hook script
+ * @returns HookResult indicating whether the hook fired and its output
+ */
+export async function runIterationHook(
+	payload: HookPayload,
+	hookScriptPath: string,
+	options?: { timeoutMs?: number },
+): Promise<HookResult> {
+	if (!isAllowedHookPath(hookScriptPath)) {
+		return {
+			fired: false,
+			stdout: "",
+			stderr: "hook path not allowed: " + hookScriptPath,
+			exitCode: null,
+			timedOut: false,
+			durationMs: 0,
+		};
+	}
+	// Resolve relative paths relative to cwd
+	const resolvedScript = path.isAbsolute(hookScriptPath) ? hookScriptPath : path.join(payload.cwd, hookScriptPath);
+	if (!isScriptRunnable(resolvedScript)) {
+		return notFiredResult();
+	}
+
+	const startTime = Date.now();
+	const stdinJson = JSON.stringify(payload);
+	const stdoutChunks: Buffer[] = [];
+	const stderrChunks: Buffer[] = [];
+
+	return new Promise<HookResult>((resolve) => {
+		const { command, args } = resolveShellForScript(resolvedScript);
+		const child = spawn(command, args, {
+			cwd: payload.cwd,
+			env: {
+				...sanitizeEnvSecrets(process.env, {
+					allowList: ["PATH", "HOME", "USER", ...WINDOWS_ESSENTIAL_ENV_VARS, "TMPDIR", "LANG", "LC_ALL", "PI_CREW_*"],
+				}),
+				PI_CREW_HOOK: "1",
+			},
+			stdio: ["pipe", "pipe", "pipe"],
+		});
+
+		let killed = false;
+		const timeoutMs = options?.timeoutMs ?? HOOK_TIMEOUT_MS;
+		const timeout = setTimeout(() => {
+			killed = true;
+			child.kill("SIGKILL");
+		}, timeoutMs);
+
+		child.stdout.on("data", (chunk: Buffer) => {
+			stdoutChunks.push(chunk);
+		});
+
+		child.stderr.on("data", (chunk: Buffer) => {
+			stderrChunks.push(chunk);
+		});
+
+		child.on("close", (code: number | null) => {
+			clearTimeout(timeout);
+			const durationMs = Date.now() - startTime;
+
+			const rawStdout = Buffer.concat(stdoutChunks);
+			// Sprint 5: refactored onto HeadSnapStage. Convert to UTF-8 string once,
+			// then apply the byte-cap stage with newline-snap so partial lines
+			// never appear in the captured preview. HeadSnapStage is byte-cap-safe
+			// (walks back partial UTF-8 sequences at the cut boundary).
+			const stdoutText = new HeadSnapStage({
+				maxBytes: MAX_STDOUT_BYTES,
+			}).apply(rawStdout.toString("utf-8"));
+
+			const rawStderr = Buffer.concat(stderrChunks);
+
+			resolve({
+				fired: true,
+				stdout: stdoutText,
+				stderr: rawStderr.toString("utf-8"),
+				exitCode: code,
+				timedOut: killed,
+				durationMs,
+			});
+		});
+
+		child.on("error", (err: Error) => {
+			clearTimeout(timeout);
+			const durationMs = Date.now() - startTime;
+			resolve({
+				fired: true,
+				stdout: "",
+				stderr: err.message,
+				exitCode: null,
+				timedOut: false,
+				durationMs,
+			});
+		});
+
+		// Write payload to stdin and close it.
+		// Handle EPIPE errors gracefully (occurs if the hook script exits before
+		// reading all of stdin, which is normal for some hook scripts on certain OS).
+		child.stdin.on("error", () => {
+			/* ignore EPIPE — hook exited early */
+		});
+		try {
+			child.stdin.write(stdinJson, "utf-8");
+			child.stdin.end();
+		} catch {
+			// ignore
+		}
+	});
+}
+
+/**
+ * Derive a steer message from the hook result.
+ *
+ * - Non-zero exit → error steer message
+ * - Timeout → timeout steer message
+ * - Empty stdout → null (no steer)
+ * - Otherwise → trimmed stdout content
+ */
+export function steerMessageFromHook(stage: HookStage, result: HookResult): string | null {
+	if (!result.fired) return null;
+
+	if (result.timedOut) {
+		return `[${stage}-hook] Hook timed out after ${result.durationMs}ms`;
+	}
+
+	if (result.exitCode !== null && result.exitCode !== 0) {
+		const stderrSnippet = result.stderr.trim().slice(0, 200);
+		return `[${stage}-hook] Hook exited with code ${result.exitCode}${stderrSnippet ? `: ${stderrSnippet}` : ""}`;
+	}
+
+	const trimmed = result.stdout.trim();
+	if (trimmed.length === 0) return null;
+
+	// Filter out prototype-polluting metric names from hook output
+	const lines = trimmed.split("\n");
+	const safeLines = lines.filter((line) => {
+		const match = /^CREW_METRIC\s+(\w+)=/.exec(line);
+		if (match) {
+			const name = match[1];
+			return !DENIED_METRIC_NAMES.has(name);
+		}
+		return true;
+	});
+
+	return safeLines.join("\n");
+}
+
+/**
+ * Build a log entry for recording hook execution in events.jsonl.
+ */
+export function hookLogEntry(stage: HookStage, result: HookResult): Record<string, unknown> {
+	const entry: Record<string, unknown> = {
+		type: "iteration-hook",
+		stage,
+		fired: result.fired,
+		durationMs: result.durationMs,
+	};
+
+	if (result.fired) {
+		entry.exitCode = result.exitCode;
+		entry.timedOut = result.timedOut;
+
+		// Include truncated stdout/stderr for diagnostics
+		if (result.stdout.length > 0) {
+			entry.stdoutPreview = result.stdout.slice(0, 512);
+		}
+		if (result.stderr.length > 0) {
+			entry.stderrPreview = result.stderr.slice(0, 512);
+		}
+	}
+
+	return entry;
+}