pi-crew 0.9.50 → 0.9.52

package/src/runtime/child-pi-streams.ts

@@ -1,296 +0,0 @@
-/**
- * child-pi-streams.ts — Stdout/stderr line parsing + observation for child Pi.
- *
- * Extracted from child-pi.ts (H-7 decomposition, step 4). The code is a verbatim
- * copy of what lived in child-pi.ts before this extraction — only the imports
- * were updated to import from child-pi-transcript.ts (compactString,
- * compactValue, appendTranscript, flushPendingTranscriptWrites) and
- * child-pi-constants.ts (MAX_*_CHARS, MAX_LINE_BUFFER_BYTES).
- */
-
-import { logInternalError } from "../utils/internal-error.ts";
-import type { ChildPiRunInput } from "./child-pi.ts";
-import { MAX_ASSISTANT_TEXT_CHARS, MAX_LINE_BUFFER_BYTES, MAX_TOOL_INPUT_CHARS, MAX_TOOL_RESULT_CHARS } from "./child-pi-constants.ts";
-import { appendTranscript, compactString, compactValue, flushPendingTranscriptWrites } from "./child-pi-transcript.ts";
-import { extractText } from "./pi-json-output.ts";
-
-function asRecord(value: unknown): Record<string, unknown> | undefined {
-	if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
-	return value as Record<string, unknown>;
-}
-
-function compactContentPart(part: unknown): unknown | undefined {
-	const record = asRecord(part);
-	if (!record) return undefined;
-	if (record.type === "text")
-		return {
-			type: "text",
-			text:
-				typeof record.text === "string"
-					? compactString(record.text, MAX_ASSISTANT_TEXT_CHARS, {
-							preserveImportant: false,
-						})
-					: "",
-		};
-	if (record.type === "toolCall")
-		return {
-			type: "toolCall",
-			name: record.name,
-			input: compactValue(typeof record.input === "string" ? compactString(record.input, MAX_TOOL_INPUT_CHARS) : record.input),
-		};
-	if (record.type === "toolResult")
-		return {
-			type: "toolResult",
-			name: record.name,
-			content: compactValue(
-				typeof record.content === "string" ? compactString(record.content, MAX_TOOL_RESULT_CHARS) : record.content,
-			),
-		};
-	return undefined;
-}
-
-function compactChildPiEvent(event: unknown): unknown | undefined {
-	const record = asRecord(event);
-	if (!record) return undefined;
-	if (record.type === "message_update") return undefined;
-	if (record.type === "tool_execution_start" || record.type === "tool_execution_end") {
-		return {
-			type: record.type,
-			toolName: record.toolName,
-			args: record.args,
-		};
-	}
-	if (record.type === "tool_result_end" || record.type === "message_end" || record.type === "message") {
-		const message = asRecord(record.message);
-		if (message?.role === "user" || message?.role === "system") return undefined;
-		const content = Array.isArray(message?.content)
-			? message.content.map(compactContentPart).filter((part) => part !== undefined)
-			: undefined;
-		return {
-			type: record.type,
-			...(typeof record.text === "string" ? { text: record.text } : {}),
-			...(message
-				? {
-						message: {
-							role: message.role,
-							...(content ? { content } : {}),
-							usage: message.usage,
-							model: message.model,
-							errorMessage: message.errorMessage,
-							stopReason: message.stopReason,
-						},
-					}
-				: {}),
-			usage: record.usage,
-			model: record.model,
-			provider: record.provider,
-			stopReason: record.stopReason,
-		};
-	}
-	return record.type ? { type: record.type } : undefined;
-}
-
-function displayTextFromCompactEvent(event: unknown): string | undefined {
-	const record = asRecord(event);
-	if (!record) return undefined;
-	if (record.type === "tool_execution_start") {
-		return typeof record.toolName === "string" ? `tool: ${record.toolName}` : "tool started";
-	}
-	if (record.type !== "message" && record.type !== "message_end") return undefined;
-	const message = asRecord(record.message);
-	if (message?.role !== undefined && message.role !== "assistant") return undefined;
-	const content = Array.isArray(message?.content) ? message.content : [];
-	const text = content
-		.flatMap((part) => {
-			const item = asRecord(part);
-			return item?.type === "text" && typeof item.text === "string" ? [item.text] : [];
-		})
-		.join("\n")
-		.trim();
-	return text || (typeof record.text === "string" ? record.text : undefined);
-}
-
-function nonJsonLineResult(line: string): {
-	persistedLine: string;
-	event?: unknown;
-	displayLine?: string;
-	json: boolean;
-} {
-	return { json: false, persistedLine: line, displayLine: line };
-}
-
-function compactChildPiLine(
-	line: string,
-	preParsed?: unknown,
-): {
-	persistedLine: string;
-	event?: unknown;
-	displayLine?: string;
-	json: boolean;
-} {
-	// OPT-PHASE2: when the caller (emitLine) already parsed the line, pass the
-	// result via preParsed to avoid a redundant JSON.parse. Standalone callers
-	// without a preParsed fall back to their own parse+catch (DRY: single
-	// compact+return path for both branches).
-	let parsed: unknown;
-	if (preParsed !== undefined) {
-		parsed = preParsed;
-	} else {
-		try {
-			parsed = JSON.parse(line);
-		} catch {
-			return nonJsonLineResult(line);
-		}
-	}
-	const compact = compactChildPiEvent(parsed);
-	return {
-		json: true,
-		event: compact,
-		persistedLine: compact ? JSON.stringify(compact) : "",
-		displayLine: displayTextFromCompactEvent(compact),
-	};
-}
-
-export class ChildPiLineObserver {
-	private buffer = "";
-	private readonly input: ChildPiRunInput;
-	/** F9: bounded ring buffer for RAW assistant-text fragments. Consumers
-	 * (getRawFinalText) only read the last element, but the legacy implementation
-	 * accumulated every fragment unconditionally, which let a verbose/long-running
-	 * worker grow this array linearly with output. We retain the last 2 entries:
-	 * the consumer needs the last; we keep the second-to-last only as a defensive
-	 * fence against a race where a final event arrives just after the consumer
-	 * read (the previous "last" is still the most-recent pre-final text in that
-	 * window). 2 is well below any plausible consumer's "tail-only" need while
-	 * bounding memory. */
-	private static readonly MAX_RAW_TEXT_EVENTS = 2;
-	private readonly rawTextEvents: string[] = [];
-	/** F9: bounded ring buffer for intermediate findings. The downstream digest
-	 * (getIntermediateFindings) slices the last 20, but the array previously grew
-	 * to 1000s of entries. We keep MAX_INTERMEDIATE_DIGEST_LINES + headroom so
-	 * the public API behaviour is preserved (still returns "last 20 lines"). */
-	private static readonly MAX_INTERMEDIATE_FINDINGS = 32;
-	private readonly intermediateFindings: string[] = [];
-
-	constructor(input: ChildPiRunInput) {
-		this.input = input;
-	}
-
-	observe(text: string): void {
-		this.buffer += text;
-		// Cap the buffer to prevent unbounded memory growth when a child process
-		// produces output without newlines (RT-F8). When exceeded, force-flush
-		// the buffer as a single line and log a warning.
-		if (this.buffer.length > MAX_LINE_BUFFER_BYTES) {
-			logInternalError(
-				"child-pi.buffer-overflow",
-				new Error(`Line buffer exceeded ${MAX_LINE_BUFFER_BYTES} bytes; force-flushing`),
-				`bufferLen=${this.buffer.length}`,
-			);
-			const line = this.buffer;
-			this.buffer = "";
-			this.emitLine(line);
-			return;
-		}
-		const lines = this.buffer.split(/\r?\n/);
-		this.buffer = lines.pop() ?? "";
-		for (const line of lines) this.emitLine(line);
-	}
-
-	flush(): Promise<void> {
-		if (this.buffer) {
-			const line = this.buffer;
-			this.buffer = "";
-			this.emitLine(line);
-		}
-		// OPT-06 follow-up: appendTranscript is fire-and-forget async, so the file
-		// may not exist on disk by the time this returns. Drain the module-scoped
-		// transcript batch buffer before resolving so callers that immediately read the
-		// transcript file (e.g. integration tests at phase4-runtime:37/:68/:103
-		// after `await observer.flush()`) see the full content.
-		return flushPendingTranscriptWrites();
-	}
-
-	/** Last non-empty RAW assistant text (mirrors {@link parsePiJsonOutput}'s
-	 *  finalText semantics but uncapped). Undefined when no assistant text was
-	 *  seen by this observer. {@link extractText} already drops empty fragments,
-	 *  so the last entry is the final assistant utterance. */
-	getRawFinalText(): string | undefined {
-		return this.rawTextEvents.length > 0 ? this.rawTextEvents[this.rawTextEvents.length - 1] : undefined;
-	}
-
-	/** #7 hardening: returns a bounded digest of intermediate findings accumulated
-	 *  during the run. This is NOT the final answer — it is a best-effort capture
-	 *  of the last assistant text or tool-result display lines before budget
-	 *  exhaustion. Only populated when getRawFinalText() would return undefined.
-	 *  @param maxChars - maximum total characters to return (default 500). */
-	getIntermediateFindings(maxChars = 500): string {
-		const MAX_INTERMEDIATE_DIGEST_LINES = 20;
-		if (this.intermediateFindings.length === 0) return "";
-		// Take the last N lines and join, then cap.
-		const lines = this.intermediateFindings.slice(-MAX_INTERMEDIATE_DIGEST_LINES);
-		const joined = lines.join("\n");
-		if (joined.length <= maxChars) return joined;
-		// Return the tail within the budget.
-		return joined.slice(-maxChars);
-	}
-
-	private emitLine(line: string): void {
-		if (!line.trim()) return;
-		// OPT-PHASE2: parse the line EXACTLY ONCE. The parsed value feeds both
-		// (a) raw assistant-text extraction for the authoritative result and
-		// (b) compaction for the telemetry transcript — previously each path
-		// called JSON.parse independently (2 parses/line). When the line is not
-		// valid JSON, parsed stays undefined and compactChildPiLine runs its own
-		// catch path to produce the json:false fallback.
-		let parsed: unknown;
-		try {
-			parsed = JSON.parse(line);
-		} catch {
-			parsed = undefined;
-		}
-		if (parsed !== undefined) {
-			const rawTexts = extractText(parsed);
-			if (rawTexts.length > 0) {
-				// F9: trim from the front if the push would exceed the cap. Slice's
-				// second arg excludes the index, so this drops the oldest entries
-				// while keeping the freshly pushed tail.
-				this.rawTextEvents.push(...rawTexts);
-				const rawOverflow = this.rawTextEvents.length - ChildPiLineObserver.MAX_RAW_TEXT_EVENTS;
-				if (rawOverflow > 0) this.rawTextEvents.splice(0, rawOverflow);
-				// Also capture raw assistant text as intermediate findings — the last raw
-				// text may be a partial answer before the worker ran out of budget.
-				const last = rawTexts[rawTexts.length - 1];
-				if (last.trim().length > 0) {
-					this.intermediateFindings.push(last.trim());
-					const findingsOverflow = this.intermediateFindings.length - ChildPiLineObserver.MAX_INTERMEDIATE_FINDINGS;
-					if (findingsOverflow > 0) this.intermediateFindings.splice(0, findingsOverflow);
-				}
-			}
-		}
-		// OPT-PHASE2: construct the non-JSON fallback directly when parsing failed,
-		// so a broken line triggers exactly ONE (failed) parse instead of two.
-		const compact = parsed !== undefined ? compactChildPiLine(line, parsed) : nonJsonLineResult(line);
-		if (compact.event !== undefined) {
-			try {
-				this.input.onJsonEvent?.(compact.event);
-			} catch (error) {
-				logInternalError("child-pi.on-json-event", error, `line=${compact.persistedLine ?? compact.displayLine ?? ""}`);
-			}
-		}
-		if (compact.persistedLine) appendTranscript(this.input, compact.persistedLine);
-		if (compact.displayLine?.trim()) {
-			try {
-				this.input.onStdoutLine?.(compact.displayLine);
-			} catch (error) {
-				logInternalError("child-pi.on-stdout-line", error, `line=${compact.displayLine}`);
-			}
-			// #7 hardening: capture display lines (tool results, stdout) as intermediate
-			// findings. This ensures we capture tool output even when no assistant text
-			// is emitted (budget exhausted on tool calls).
-			this.intermediateFindings.push(compact.displayLine!.trim());
-			const findingsOverflow = this.intermediateFindings.length - ChildPiLineObserver.MAX_INTERMEDIATE_FINDINGS;
-			if (findingsOverflow > 0) this.intermediateFindings.splice(0, findingsOverflow);
-		}
-	}
-}

package/src/runtime/child-pi-transcript.ts

@@ -1,169 +0,0 @@
-/**
- * child-pi-transcript.ts — Transcript batching module for child Pi worker output.
- *
- * Extracted from child-pi.ts (H-7 decomposition, step 1). Zero behavior change.
- *
- * Architecture:
- *   appendTranscript() validates the transcriptPath against the artifactsRoot
- *   containment boundary, redacts the line, and pushes it to a module-scoped
- *   batch buffer keyed by safePath. A debounced 50ms timer flushes the buffer
- *   in one open/write/close per path (O_NOFOLLOW | O_CREAT | O_APPEND).
- *
- * Lifecycle boundaries (ChildPiLineObserver.flush, runChildPi settle) must
- * call flushPendingTranscriptWrites() so callers that immediately read the
- * transcript file see complete content.
- *
- * Ordering: lines are appended in call order. The flush writes the joined
- * array preserving intra-batch order. Inter-batch order is not guaranteed
- * but transcript is append-only telemetry.
- */
-
-import * as fs from "node:fs";
-import { DEFAULT_CHILD_PI } from "../config/defaults.ts";
-import { logInternalError } from "../utils/internal-error.ts";
-import { redactJsonLine } from "../utils/redaction.ts";
-import { resolveRealContainedPath } from "../utils/safe-paths.ts";
-import type { ChildPiRunInput } from "./child-pi.ts";
-import { applyCompactPipeline } from "./compact-pipeline.ts";
-import { TruncationStage } from "./compact-stages/truncation-stage.ts";
-
-// ── Transcript batch buffer (OPT-PHASE3) ────────────────────────────────
-// Instead of open/write/close per line (3 syscalls × N), accumulate lines
-// in a module-scoped buffer and flush them in one open/write/close per path
-// every TRANSCRIPT_FLUSH_MS. Lifecycle boundaries (observer.flush, settle)
-// force-flush the buffer before returning so transcript reads are complete.
-const transcriptBatches = new Map<string, string[]>();
-let transcriptFlushTimer: ReturnType<typeof setTimeout> | undefined;
-const TRANSCRIPT_FLUSH_MS = 50;
-
-export function appendTranscript(input: ChildPiRunInput, line: string): void {
-	if (!input.transcriptPath) return;
-	// SECURITY FIX (Issue #1): Validate transcriptPath against artifactsRoot to prevent
-	// arbitrary file writes and symlink traversal attacks. An attacker who can influence
-	// the task graph could set transcriptPath to /etc/passwd or similar, and mkdirSync
-	// with recursive:true would create parent directories. Additionally, appendFileSync
-	// follows symlinks, potentially writing to sensitive files.
-	let safePath: string;
-	try {
-		const artifactsRoot = input.artifactsRoot ?? input.cwd;
-		safePath = resolveRealContainedPath(artifactsRoot, input.transcriptPath);
-	} catch (error) {
-		logInternalError("child-pi.transcript-path-rejected", error as Error, `transcriptPath=${input.transcriptPath}`);
-		return;
-	}
-	trackTranscriptWrite(safePath, line);
-}
-
-function scheduleTranscriptFlush(): void {
-	if (transcriptFlushTimer) return;
-	transcriptFlushTimer = setTimeout(() => {
-		transcriptFlushTimer = undefined;
-		void flushTranscriptBatches();
-	}, TRANSCRIPT_FLUSH_MS);
-	transcriptFlushTimer.unref?.();
-}
-
-async function flushTranscriptBatches(): Promise<void> {
-	const entries = [...transcriptBatches.entries()];
-	transcriptBatches.clear();
-	await Promise.allSettled(
-		entries.map(async ([safePath, lines]) => {
-			if (lines.length === 0) return;
-			const content = lines.join("");
-			try {
-				const fd = await fs.promises.open(
-					safePath,
-					fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW | fs.constants.O_CREAT | fs.constants.O_APPEND,
-					0o600,
-				);
-				try {
-					await fd.write(content, undefined, "utf-8");
-				} finally {
-					await fd.close();
-				}
-			} catch (error) {
-				logInternalError("child-pi.transcript-write-failed", error as Error, `path=${safePath}`);
-			}
-		}),
-	);
-}
-
-function trackTranscriptWrite(safePath: string, line: string): void {
-	const content = `${redactJsonLine(line)}\n`;
-	let batch = transcriptBatches.get(safePath);
-	if (!batch) {
-		batch = [];
-		transcriptBatches.set(safePath, batch);
-	}
-	batch.push(content);
-	scheduleTranscriptFlush();
-}
-
-/**
- * Drain the transcript batch buffer and await any remaining in-flight writes.
- * Called by lifecycle boundaries (ChildPiLineObserver.flush, runChildPi settle)
- * so that transcript files are complete before callers read them.
- *
- * Uses a while loop to re-check the buffer after each flush — new lines may
- * arrive during the async I/O window (trackTranscriptWrite → scheduleTranscriptFlush).
- */
-export async function flushPendingTranscriptWrites(): Promise<void> {
-	if (transcriptFlushTimer) {
-		clearTimeout(transcriptFlushTimer);
-		transcriptFlushTimer = undefined;
-	}
-	while (transcriptBatches.size > 0) {
-		await flushTranscriptBatches();
-	}
-}
-
-/**
- * Reset the module-scoped transcript batch state. Exported for test isolation
- * only — production code should never call this.
- */
-export function resetTranscriptBatchState(): void {
-	if (transcriptFlushTimer) {
-		clearTimeout(transcriptFlushTimer);
-		transcriptFlushTimer = undefined;
-	}
-	transcriptBatches.clear();
-}
-
-// ── Compaction helpers (moved from child-pi.ts; these don't need batching) ──
-
-export function compactString(
-	value: string,
-	maxChars = DEFAULT_CHILD_PI.maxCompactContentChars,
-	opts: { preserveImportant?: boolean } = {},
-): string {
-	if (value.length <= maxChars) return value;
-	const result = applyCompactPipeline(value, [
-		new TruncationStage(maxChars, {
-			preserveImportant: opts.preserveImportant,
-		}),
-	]);
-	return result.text;
-}
-
-export function compactValue(value: unknown): unknown {
-	if (typeof value === "string") return compactString(value);
-	if (Array.isArray(value)) {
-		// BUG-4: silent .slice(0, 20) lost items 21-50 with no marker.
-		if (value.length > 20) {
-			return [...value.slice(0, 20).map(compactValue), `[pi-crew truncated ${value.length - 20} entries]`];
-		}
-		return value.map(compactValue);
-	}
-	const record = asRecord(value);
-	if (!record) return value;
-	const entries = Object.entries(record);
-	const compacted: Record<string, unknown> = {};
-	for (const [key, entry] of entries.slice(0, 20)) compacted[key] = compactValue(entry);
-	if (entries.length > 20) compacted["[truncated]"] = `${entries.length - 20} entries`;
-	return compacted;
-}
-
-function asRecord(value: unknown): Record<string, unknown> | undefined {
-	if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
-	return value as Record<string, unknown>;
-}