world-model-optimizer 0.2.0__py3-none-any.whl

wmo/harness/vendor/pi-agent/src/harness/utils/shell-output.ts

@@ -0,0 +1,135 @@
+import { type ExecutionEnv, ExecutionError, err, ok, type Result, type ShellExecOptions, toError } from "../types.ts";
+import { DEFAULT_MAX_BYTES, truncateTail } from "./truncate.ts";
+
+export interface ShellCaptureOptions extends Omit<ShellExecOptions, "onStdout" | "onStderr"> {
+	onChunk?: (chunk: string) => void;
+}
+
+export interface ShellCaptureResult {
+	output: string;
+	exitCode: number | undefined;
+	cancelled: boolean;
+	truncated: boolean;
+	fullOutputPath?: string;
+}
+
+function toExecutionError(error: unknown): ExecutionError {
+	if (error instanceof ExecutionError) return error;
+	const cause = toError(error);
+	return new ExecutionError("unknown", cause.message, cause);
+}
+
+export function sanitizeBinaryOutput(str: string): string {
+	return Array.from(str)
+		.filter((char) => {
+			const code = char.codePointAt(0);
+			if (code === undefined) return false;
+			if (code === 0x09 || code === 0x0a || code === 0x0d) return true;
+			if (code <= 0x1f) return false;
+			if (code >= 0xfff9 && code <= 0xfffb) return false;
+			return true;
+		})
+		.join("");
+}
+
+export async function executeShellWithCapture(
+	env: ExecutionEnv,
+	command: string,
+	options?: ShellCaptureOptions,
+): Promise<Result<ShellCaptureResult, ExecutionError>> {
+	const outputChunks: string[] = [];
+	let outputBytes = 0;
+	const maxOutputBytes = DEFAULT_MAX_BYTES * 2;
+	const encoder = new TextEncoder();
+
+	let totalBytes = 0;
+	let fullOutputPath: string | undefined;
+	let writeChain: Promise<Result<void, ExecutionError>> = Promise.resolve(ok(undefined));
+	let captureError: ExecutionError | undefined;
+
+	const appendFullOutput = (text: string): void => {
+		if (!fullOutputPath || captureError) return;
+		const path = fullOutputPath;
+		writeChain = writeChain.then(async (previous) => {
+			if (!previous.ok) return previous;
+			const appendResult = await env.appendFile(path, text, options?.abortSignal);
+			return appendResult.ok ? ok(undefined) : err(toExecutionError(appendResult.error));
+		});
+	};
+
+	const ensureFullOutputFile = (initialContent: string): void => {
+		if (fullOutputPath || captureError) return;
+		writeChain = writeChain.then(async (previous) => {
+			if (!previous.ok) return previous;
+			const tempFile = await env.createTempFile({
+				prefix: "bash-",
+				suffix: ".log",
+				abortSignal: options?.abortSignal,
+			});
+			if (!tempFile.ok) return err(toExecutionError(tempFile.error));
+			fullOutputPath = tempFile.value;
+			const appendResult = await env.appendFile(tempFile.value, initialContent, options?.abortSignal);
+			return appendResult.ok ? ok(undefined) : err(toExecutionError(appendResult.error));
+		});
+	};
+
+	const onChunk = (chunk: string) => {
+		try {
+			totalBytes += encoder.encode(chunk).byteLength;
+			const text = sanitizeBinaryOutput(chunk).replace(/\r/g, "");
+			if (totalBytes > DEFAULT_MAX_BYTES && !fullOutputPath) {
+				ensureFullOutputFile(outputChunks.join("") + text);
+			} else {
+				appendFullOutput(text);
+			}
+			outputChunks.push(text);
+			outputBytes += text.length;
+			while (outputBytes > maxOutputBytes && outputChunks.length > 1) {
+				const removed = outputChunks.shift()!;
+				outputBytes -= removed.length;
+			}
+			options?.onChunk?.(text);
+		} catch (error) {
+			captureError = toExecutionError(error);
+		}
+	};
+
+	try {
+		const result = await env.exec(command, {
+			...(options ?? {}),
+			onStdout: onChunk,
+			onStderr: onChunk,
+		});
+		const tailOutput = outputChunks.join("");
+		const truncationResult = truncateTail(tailOutput);
+		if (truncationResult.truncated && !fullOutputPath) {
+			ensureFullOutputFile(tailOutput);
+		}
+		const writeResult = await writeChain;
+		if (!writeResult.ok) return err(writeResult.error);
+		if (captureError) return err(captureError);
+
+		if (!result.ok) {
+			if (result.error.code === "aborted" || options?.abortSignal?.aborted) {
+				return ok({
+					output: truncationResult.truncated ? truncationResult.content : tailOutput,
+					exitCode: undefined,
+					cancelled: true,
+					truncated: truncationResult.truncated,
+					fullOutputPath,
+				});
+			}
+			return err(result.error);
+		}
+		const cancelled = options?.abortSignal?.aborted ?? false;
+		return ok({
+			output: truncationResult.truncated ? truncationResult.content : tailOutput,
+			exitCode: cancelled ? undefined : result.value.exitCode,
+			cancelled,
+			truncated: truncationResult.truncated,
+			fullOutputPath,
+		});
+	} catch (error) {
+		return err(toExecutionError(error));
+	}
+}

wmo/harness/vendor/pi-agent/src/harness/utils/truncate.ts

@@ -0,0 +1,344 @@
+/**
+ * Shared truncation utilities for tool outputs.
+ *
+ * Truncation is based on two independent limits - whichever is hit first wins:
+ * - Line limit (default: 2000 lines)
+ * - Byte limit (default: 50KB)
+ *
+ * Never returns partial lines (except bash tail truncation edge case).
+ */
+
+export const DEFAULT_MAX_LINES = 2000;
+export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB
+export const GREP_MAX_LINE_LENGTH = 500; // Max chars per grep match line
+
+export interface TruncationResult {
+	/** The truncated content */
+	content: string;
+	/** Whether truncation occurred */
+	truncated: boolean;
+	/** Which limit was hit: "lines", "bytes", or null if not truncated */
+	truncatedBy: "lines" | "bytes" | null;
+	/** Total number of lines in the original content */
+	totalLines: number;
+	/** Total number of bytes in the original content */
+	totalBytes: number;
+	/** Number of complete lines in the truncated output */
+	outputLines: number;
+	/** Number of bytes in the truncated output */
+	outputBytes: number;
+	/** Whether the last line was partially truncated (only for tail truncation edge case) */
+	lastLinePartial: boolean;
+	/** Whether the first line exceeded the byte limit (for head truncation) */
+	firstLineExceedsLimit: boolean;
+	/** The max lines limit that was applied */
+	maxLines: number;
+	/** The max bytes limit that was applied */
+	maxBytes: number;
+}
+
+export interface TruncationOptions {
+	/** Maximum number of lines (default: 2000) */
+	maxLines?: number;
+	/** Maximum number of bytes (default: 50KB) */
+	maxBytes?: number;
+}
+
+interface RuntimeBuffer {
+	byteLength(content: string, encoding: "utf8"): number;
+}
+
+const runtimeBuffer = (globalThis as { Buffer?: RuntimeBuffer }).Buffer;
+const nonAsciiPattern = /[^\x00-\x7f]/;
+
+function utf8ByteLength(content: string): number {
+	if (runtimeBuffer) return runtimeBuffer.byteLength(content, "utf8");
+
+	const firstNonAscii = content.search(nonAsciiPattern);
+	if (firstNonAscii === -1) return content.length;
+
+	let bytes = firstNonAscii;
+	for (let i = firstNonAscii; i < content.length; i++) {
+		const code = content.charCodeAt(i);
+		if (code <= 0x7f) {
+			bytes += 1;
+		} else if (code <= 0x7ff) {
+			bytes += 2;
+		} else if (code >= 0xd800 && code <= 0xdbff && i + 1 < content.length) {
+			const next = content.charCodeAt(i + 1);
+			if (next >= 0xdc00 && next <= 0xdfff) {
+				bytes += 4;
+				i++;
+			} else {
+				bytes += 3;
+			}
+		} else {
+			bytes += 3;
+		}
+	}
+	return bytes;
+}
+
+function replaceUnpairedSurrogates(content: string): string {
+	let output = "";
+	for (let i = 0; i < content.length; i++) {
+		const code = content.charCodeAt(i);
+		if (code >= 0xd800 && code <= 0xdbff) {
+			if (i + 1 < content.length) {
+				const next = content.charCodeAt(i + 1);
+				if (next >= 0xdc00 && next <= 0xdfff) {
+					output += content[i] + content[i + 1];
+					i++;
+					continue;
+				}
+			}
+			output += "�";
+		} else if (code >= 0xdc00 && code <= 0xdfff) {
+			output += "�";
+		} else {
+			output += content[i];
+		}
+	}
+	return output;
+}
+
+/**
+ * Format bytes as human-readable size.
+ */
+export function formatSize(bytes: number): string {
+	if (bytes < 1024) {
+		return `${bytes}B`;
+	} else if (bytes < 1024 * 1024) {
+		return `${(bytes / 1024).toFixed(1)}KB`;
+	} else {
+		return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+	}
+}
+
+/**
+ * Truncate content from the head (keep first N lines/bytes).
+ * Suitable for file reads where you want to see the beginning.
+ *
+ * Never returns partial lines. If first line exceeds byte limit,
+ * returns empty content with firstLineExceedsLimit=true.
+ */
+export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult {
+	const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
+	const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+
+	const totalBytes = utf8ByteLength(content);
+	const lines = content.split("\n");
+	const totalLines = lines.length;
+
+	// Check if no truncation needed
+	if (totalLines <= maxLines && totalBytes <= maxBytes) {
+		return {
+			content,
+			truncated: false,
+			truncatedBy: null,
+			totalLines,
+			totalBytes,
+			outputLines: totalLines,
+			outputBytes: totalBytes,
+			lastLinePartial: false,
+			firstLineExceedsLimit: false,
+			maxLines,
+			maxBytes,
+		};
+	}
+
+	// Check if first line alone exceeds byte limit
+	const firstLineBytes = utf8ByteLength(lines[0]);
+	if (firstLineBytes > maxBytes) {
+		return {
+			content: "",
+			truncated: true,
+			truncatedBy: "bytes",
+			totalLines,
+			totalBytes,
+			outputLines: 0,
+			outputBytes: 0,
+			lastLinePartial: false,
+			firstLineExceedsLimit: true,
+			maxLines,
+			maxBytes,
+		};
+	}
+
+	// Collect complete lines that fit
+	const outputLinesArr: string[] = [];
+	let outputBytesCount = 0;
+	let truncatedBy: "lines" | "bytes" = "lines";
+
+	for (let i = 0; i < lines.length && i < maxLines; i++) {
+		const line = lines[i];
+		const lineBytes = utf8ByteLength(line) + (i > 0 ? 1 : 0); // +1 for newline
+
+		if (outputBytesCount + lineBytes > maxBytes) {
+			truncatedBy = "bytes";
+			break;
+		}
+
+		outputLinesArr.push(line);
+		outputBytesCount += lineBytes;
+	}
+
+	// If we exited due to line limit
+	if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
+		truncatedBy = "lines";
+	}
+
+	const outputContent = outputLinesArr.join("\n");
+	const finalOutputBytes = utf8ByteLength(outputContent);
+
+	return {
+		content: outputContent,
+		truncated: true,
+		truncatedBy,
+		totalLines,
+		totalBytes,
+		outputLines: outputLinesArr.length,
+		outputBytes: finalOutputBytes,
+		lastLinePartial: false,
+		firstLineExceedsLimit: false,
+		maxLines,
+		maxBytes,
+	};
+}
+
+/**
+ * Truncate content from the tail (keep last N lines/bytes).
+ * Suitable for bash output where you want to see the end (errors, final results).
+ *
+ * May return partial first line if the last line of original content exceeds byte limit.
+ */
+export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
+	const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
+	const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+
+	const totalBytes = utf8ByteLength(content);
+	const lines = content.split("\n");
+	if (lines.length > 1 && lines[lines.length - 1] === "") lines.pop();
+	const totalLines = lines.length;
+
+	// Check if no truncation needed
+	if (totalLines <= maxLines && totalBytes <= maxBytes) {
+		return {
+			content,
+			truncated: false,
+			truncatedBy: null,
+			totalLines,
+			totalBytes,
+			outputLines: totalLines,
+			outputBytes: totalBytes,
+			lastLinePartial: false,
+			firstLineExceedsLimit: false,
+			maxLines,
+			maxBytes,
+		};
+	}
+
+	// Work backwards from the end
+	const outputLinesArr: string[] = [];
+	let outputBytesCount = 0;
+	let truncatedBy: "lines" | "bytes" = "lines";
+	let lastLinePartial = false;
+
+	for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) {
+		const line = lines[i];
+		const lineBytes = utf8ByteLength(line) + (outputLinesArr.length > 0 ? 1 : 0); // +1 for newline
+
+		if (outputBytesCount + lineBytes > maxBytes) {
+			truncatedBy = "bytes";
+			// Edge case: if we haven't added ANY lines yet and this line exceeds maxBytes,
+			// take the end of the line (partial)
+			if (outputLinesArr.length === 0) {
+				const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes);
+				outputLinesArr.unshift(truncatedLine);
+				outputBytesCount = utf8ByteLength(truncatedLine);
+				lastLinePartial = true;
+			}
+			break;
+		}
+
+		outputLinesArr.unshift(line);
+		outputBytesCount += lineBytes;
+	}
+
+	// If we exited due to line limit
+	if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
+		truncatedBy = "lines";
+	}
+
+	const outputContent = outputLinesArr.join("\n");
+	const finalOutputBytes = utf8ByteLength(outputContent);
+
+	return {
+		content: outputContent,
+		truncated: true,
+		truncatedBy,
+		totalLines,
+		totalBytes,
+		outputLines: outputLinesArr.length,
+		outputBytes: finalOutputBytes,
+		lastLinePartial,
+		firstLineExceedsLimit: false,
+		maxLines,
+		maxBytes,
+	};
+}
+
+/**
+ * Truncate a string to fit within a byte limit (from the end).
+ * Handles multi-byte UTF-8 characters correctly.
+ */
+function truncateStringToBytesFromEnd(str: string, maxBytes: number): string {
+	if (maxBytes <= 0) return "";
+
+	let outputBytes = 0;
+	let start = str.length;
+	let needsReplacement = false;
+	for (let i = str.length; i > 0; ) {
+		let characterStart = i - 1;
+		const code = str.charCodeAt(characterStart);
+		let characterBytes: number;
+		let unpairedSurrogate = false;
+		if (code >= 0xdc00 && code <= 0xdfff && characterStart > 0) {
+			const previous = str.charCodeAt(characterStart - 1);
+			if (previous >= 0xd800 && previous <= 0xdbff) {
+				characterStart--;
+				characterBytes = 4;
+			} else {
+				characterBytes = 3;
+				unpairedSurrogate = true;
+			}
+		} else if (code >= 0xd800 && code <= 0xdfff) {
+			characterBytes = 3;
+			unpairedSurrogate = true;
+		} else {
+			characterBytes = code <= 0x7f ? 1 : code <= 0x7ff ? 2 : 3;
+		}
+		if (outputBytes + characterBytes > maxBytes) break;
+		outputBytes += characterBytes;
+		start = characterStart;
+		needsReplacement ||= unpairedSurrogate;
+		i = characterStart;
+	}
+
+	const output = str.slice(start);
+	return needsReplacement ? replaceUnpairedSurrogates(output) : output;
+}
+
+/**
+ * Truncate a single line to max characters, adding [truncated] suffix.
+ * Used for grep match lines.
+ */
+export function truncateLine(
+	line: string,
+	maxChars: number = GREP_MAX_LINE_LENGTH,
+): { text: string; wasTruncated: boolean } {
+	if (line.length <= maxChars) {
+		return { text: line, wasTruncated: false };
+	}
+	return { text: `${line.slice(0, maxChars)}... [truncated]`, wasTruncated: true };
+}

wmo/harness/vendor/pi-agent/src/index.ts

@@ -0,0 +1,44 @@
+// Core Agent
+export * from "./agent.ts";
+// Loop functions
+export * from "./agent-loop.ts";
+export * from "./harness/agent-harness.ts";
+export {
+	type BranchPreparation,
+	type BranchSummaryDetails,
+	type CollectEntriesResult,
+	collectEntriesForBranchSummary,
+	generateBranchSummary,
+	prepareBranchEntries,
+} from "./harness/compaction/branch-summarization.ts";
+export {
+	calculateContextTokens,
+	compact,
+	DEFAULT_COMPACTION_SETTINGS,
+	estimateContextTokens,
+	estimateTokens,
+	findCutPoint,
+	findTurnStartIndex,
+	generateSummary,
+	getLastAssistantUsage,
+	prepareCompaction,
+	serializeConversation,
+	shouldCompact,
+} from "./harness/compaction/compaction.ts";
+export * from "./harness/messages.ts";
+export * from "./harness/prompt-templates.ts";
+export * from "./harness/session/jsonl-repo.ts";
+export * from "./harness/session/memory-repo.ts";
+export * from "./harness/session/repo-utils.ts";
+export * from "./harness/session/session.ts";
+export { uuidv7 } from "./harness/session/uuid.ts";
+export * from "./harness/skills.ts";
+export * from "./harness/system-prompt.ts";
+// Harness
+export * from "./harness/types.ts";
+export * from "./harness/utils/shell-output.ts";
+export * from "./harness/utils/truncate.ts";
+// Proxy utilities
+export * from "./proxy.ts";
+// Types
+export * from "./types.ts";

wmo/harness/vendor/pi-agent/src/node.ts

@@ -0,0 +1,2 @@
+export { NodeExecutionEnv } from "./harness/env/nodejs.ts";
+export * from "./index.ts";