pi-rtk-optimizer 0.7.1 → 0.8.1

package/src/output-compactor-test.ts

@@ -6,7 +6,7 @@ import { cloneDefaultConfig, runTest } from "./test-helpers.ts";
 
 const TEST_AGENT_DIR = "/tmp/.pi/agent";
 
-mock.module("@mariozechner/pi-coding-agent", () => ({
+mock.module("@earendil-works/pi-coding-agent", () => ({
 	getAgentDir: () => TEST_AGENT_DIR,
 }));
 
@@ -75,6 +75,14 @@ function assertNoOutputEmoji(text: string): void {
 	}
 }
 
+function assertNoPartialHashlineAnchors(text: string): void {
+	for (const line of text.split(/\r?\n/)) {
+		if (/^\s*\d+\s*#[A-Za-z0-9_-]{2,32}:/.test(line)) {
+			assert.equal(line.endsWith("..."), false, `Anchor line was partially truncated: ${line}`);
+		}
+	}
+}
+
 runTest("precision read with offset keeps exact output (no source/smart/hard truncation)", () => {
 	const config = cloneDefaultConfig();
 	setReadCompaction(config, true);
@@ -168,6 +176,203 @@ runTest("normal read compacts and adds banner when read compaction is enabled",
 	assert.ok(compacted.includes("source:minimal"));
 });
 
+runTest("line-anchor read output compacts without corrupting LINE#HASH anchors", () => {
+	const config = cloneDefaultConfig();
+	setReadCompaction(config, true);
+	config.outputCompaction.sourceCodeFilteringEnabled = true;
+	config.outputCompaction.sourceCodeFiltering = "minimal";
+	config.outputCompaction.smartTruncate.enabled = true;
+	config.outputCompaction.smartTruncate.maxLines = 40;
+	config.outputCompaction.truncate.enabled = true;
+	config.outputCompaction.truncate.maxChars = 5000;
+
+	const content = Array.from({ length: 120 }, (_value, index) => {
+		const lineNumber = index + 1;
+		const sourceLine = lineNumber % 2 === 0 ? `const value${lineNumber} = ${lineNumber};` : `// comment ${lineNumber}`;
+		return `${String(lineNumber).padStart(3, " ")}#ZP:${sourceLine}`;
+	}).join("\n");
+	const result = compactToolResult(
+		{
+			toolName: "read",
+			input: { path: "sample.ts" },
+			content: [{ type: "text", text: content }],
+		},
+		config,
+	);
+
+	assert.equal(result.changed, true);
+	assert.ok(result.techniques.includes("source:minimal"));
+
+	const compacted = firstTextBlock(result.content);
+	assert.ok(compacted.startsWith("[RTK compacted output:"));
+	assert.ok(compacted.includes("source:minimal"));
+	assert.match(compacted, /\n\s*2#ZP:const value2 = 2;/);
+	assert.equal(compacted.includes("#ZP:// comment"), false);
+	assertNoPartialHashlineAnchors(compacted);
+});
+
+runTest("colon-pipe anchor read output compacts without requiring hashline extension", () => {
+	const config = cloneDefaultConfig();
+	setReadCompaction(config, true);
+	config.outputCompaction.sourceCodeFilteringEnabled = true;
+	config.outputCompaction.sourceCodeFiltering = "minimal";
+	config.outputCompaction.smartTruncate.enabled = true;
+	config.outputCompaction.smartTruncate.maxLines = 40;
+	config.outputCompaction.truncate.enabled = true;
+	config.outputCompaction.truncate.maxChars = 5000;
+
+	const content = [
+		"Read sample.ts: 120 lines",
+		"",
+		...Array.from({ length: 120 }, (_value, index) => {
+			const lineNumber = index + 1;
+			const sourceLine = lineNumber % 2 === 0 ? `const value${lineNumber} = ${lineNumber};` : `// comment ${lineNumber}`;
+			return `${lineNumber}:${(lineNumber % 256).toString(16).padStart(2, "0")}|${sourceLine}`;
+		}),
+	].join("\n");
+	const result = compactToolResult(
+		{
+			toolName: "read",
+			input: { path: "sample.ts" },
+			content: [{ type: "text", text: content }],
+		},
+		config,
+	);
+
+	assert.equal(result.changed, true);
+	assert.ok(result.techniques.includes("source:minimal"));
+
+	const compacted = firstTextBlock(result.content);
+	assert.ok(compacted.includes("Read sample.ts: 120 lines"));
+	assert.match(compacted, /\n2:02\|const value2 = 2;/);
+	assert.equal(compacted.includes("|// comment"), false);
+});
+
+runTest("compact LINEHASH pipe anchors from oh-my-pi style reads", () => {
+	const config = cloneDefaultConfig();
+	setReadCompaction(config, true);
+	config.outputCompaction.sourceCodeFilteringEnabled = true;
+	config.outputCompaction.sourceCodeFiltering = "minimal";
+	config.outputCompaction.smartTruncate.enabled = true;
+	config.outputCompaction.smartTruncate.maxLines = 40;
+	config.outputCompaction.truncate.enabled = true;
+	config.outputCompaction.truncate.maxChars = 5000;
+
+	const content = Array.from({ length: 120 }, (_value, index) => {
+		const lineNumber = index + 1;
+		const hash = lineNumber % 2 === 0 ? "sr" : "ab";
+		const sourceLine = lineNumber % 2 === 0 ? `const value${lineNumber} = ${lineNumber};` : `// comment ${lineNumber}`;
+		return `${lineNumber}${hash}|${sourceLine}`;
+	}).join("\n");
+	const result = compactToolResult(
+		{
+			toolName: "read",
+			input: { path: "sample.ts" },
+			content: [{ type: "text", text: content }],
+		},
+		config,
+	);
+
+	assert.equal(result.changed, true);
+	assert.ok(result.techniques.includes("source:minimal"));
+
+	const compacted = firstTextBlock(result.content);
+	assert.match(compacted, /\n2sr\|const value2 = 2;/);
+	assert.equal(compacted.includes("|// comment"), false);
+});
+
+runTest("compact hashline-tools file wrapper while preserving non-anchor wrapper lines", () => {
+	const config = cloneDefaultConfig();
+	setReadCompaction(config, true);
+	config.outputCompaction.sourceCodeFilteringEnabled = true;
+	config.outputCompaction.sourceCodeFiltering = "minimal";
+	config.outputCompaction.smartTruncate.enabled = true;
+	config.outputCompaction.smartTruncate.maxLines = 40;
+	config.outputCompaction.truncate.enabled = true;
+	config.outputCompaction.truncate.maxChars = 5000;
+
+	const content = [
+		"<file>",
+		...Array.from({ length: 120 }, (_value, index) => {
+			const lineNumber = index + 1;
+			const sourceLine = lineNumber % 2 === 0 ? `const value${lineNumber} = ${lineNumber};` : `// comment ${lineNumber}`;
+			return `${lineNumber}#ZM:${sourceLine}`;
+		}),
+		"",
+		"(End of file - 120 total lines)",
+		"</file>",
+	].join("\n");
+	const result = compactToolResult(
+		{
+			toolName: "read",
+			input: { path: "sample.ts" },
+			content: [{ type: "text", text: content }],
+		},
+		config,
+	);
+
+	assert.equal(result.changed, true);
+	assert.ok(result.techniques.includes("source:minimal"));
+
+	const compacted = firstTextBlock(result.content);
+	assert.ok(compacted.includes("<file>"));
+	assert.ok(compacted.includes("(End of file - 120 total lines)"));
+	assert.ok(compacted.includes("</file>"));
+	assert.match(compacted, /\n2#ZM:const value2 = 2;/);
+	assert.equal(compacted.includes("#ZM:// comment"), false);
+});
+
+runTest("anchor-safe read hard truncation preserves whole hashline anchors", () => {
+	const config = cloneDefaultConfig();
+	setReadCompaction(config, true);
+	config.outputCompaction.sourceCodeFilteringEnabled = false;
+	config.outputCompaction.smartTruncate.enabled = false;
+	config.outputCompaction.truncate.enabled = true;
+	config.outputCompaction.truncate.maxChars = 350;
+
+	const content = Array.from({ length: 120 }, (_value, index) => {
+		const lineNumber = index + 1;
+		return `${lineNumber}#ZP:const value${lineNumber} = "${"x".repeat(40)}";`;
+	}).join("\n");
+	const result = compactToolResult(
+		{
+			toolName: "read",
+			input: { path: "sample.ts" },
+			content: [{ type: "text", text: content }],
+		},
+		config,
+	);
+
+	assert.equal(result.changed, true);
+	assert.ok(result.techniques.includes("truncate"));
+
+	const compacted = firstTextBlock(result.content);
+	assert.ok(compacted.includes("anchor-safe truncate"));
+	assertNoPartialHashlineAnchors(compacted);
+});
+
+runTest("incidental single anchor-like line does not disable normal read compaction", () => {
+	const config = cloneDefaultConfig();
+	setReadCompaction(config, true);
+	config.outputCompaction.sourceCodeFilteringEnabled = true;
+	config.outputCompaction.sourceCodeFiltering = "minimal";
+	config.outputCompaction.smartTruncate.enabled = true;
+	config.outputCompaction.smartTruncate.maxLines = 40;
+
+	const content = [`1#ZP:not an anchored read`, buildReadContent(120)].join("\n");
+	const result = compactToolResult(
+		{
+			toolName: "read",
+			input: { path: "sample.ts" },
+			content: [{ type: "text", text: content }],
+		},
+		config,
+	);
+
+	assert.equal(result.changed, true);
+	assert.ok(result.techniques.includes("source:minimal") || result.techniques.includes("smart-truncate"));
+});
+
 runTest("short read output stays exact below threshold", () => {
 	const config = cloneDefaultConfig();
 	setReadCompaction(config, true);
@@ -435,7 +640,7 @@ runTest("rtk grep-style output sanitizes emoji file markers", () => {
 runTest("rtk git diff verbose summary sanitizes file markers", () => {
 	const compacted = compactBashOutput(
 		"rtk git diff -- agent/extensions/pi-mcp-adapter/package.json",
-		"agent/extensions/pi-mcp-adapter/package.json | 2 +-\n\n--- Changes ---\n\n📄 agent/extensions/pi-mcp-adapter/package.json\n  @@ -38,7 +38,7 @@\n  -    \"@mariozechner/pi-coding-agent\": \"^0.58.1\",\n",
+		"agent/extensions/pi-mcp-adapter/package.json | 2 +-\n\n--- Changes ---\n\n📄 agent/extensions/pi-mcp-adapter/package.json\n  @@ -38,7 +38,7 @@\n  -    \"@earendil-works/pi-coding-agent\": \"^0.58.1\",\n",
 	);
 
 	assert.ok(compacted.includes("--- Changes ---"));
@@ -446,7 +651,7 @@ runTest("rtk git diff verbose summary sanitizes file markers", () => {
 runTest("git diff compaction skips already-compacted RTK-shaped output", () => {
 	const compacted = compactBashOutput(
 		"git diff -- agent/extensions/pi-mcp-adapter/package.json",
-		"agent/extensions/pi-mcp-adapter/package.json | 2 +-\n\n--- Changes ---\n\n📄 agent/extensions/pi-mcp-adapter/package.json\n  @@ -38,7 +38,7 @@\n  -    \"@mariozechner/pi-coding-agent\": \"^0.58.1\",\n",
+		"agent/extensions/pi-mcp-adapter/package.json | 2 +-\n\n--- Changes ---\n\n📄 agent/extensions/pi-mcp-adapter/package.json\n  @@ -38,7 +38,7 @@\n  -    \"@earendil-works/pi-coding-agent\": \"^0.58.1\",\n",
 	);
 
 	assert.ok(compacted.includes("--- Changes ---"));

package/src/output-compactor.ts

@@ -1,4 +1,4 @@
-import { getAgentDir } from "@mariozechner/pi-coding-agent";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
 import { homedir } from "node:os";
 import { dirname, join, resolve, sep } from "node:path";
 import {
@@ -48,6 +48,24 @@ export interface ToolResultCompactionOutcome {
 	metadata?: ToolResultCompactionMetadata;
 }
 
+interface AnchoredReadLine {
+	lineNumber: number;
+	content: string;
+	originalLine: string;
+}
+
+interface AnchorSafeReadLine {
+	text: string;
+	content: string;
+}
+
+interface AnchorSafeReadParts {
+	prefixLines: string[];
+	anchoredLines: AnchoredReadLine[];
+	suffixLines: string[];
+	trailingNewline: boolean;
+}
+
 const LOSSY_TECHNIQUE_PREFIXES = [
 	"build",
 	"test",
@@ -61,6 +79,15 @@ const LOSSY_TECHNIQUE_PREFIXES = [
 
 const READ_EXACT_OUTPUT_LINE_THRESHOLD = 80;
 const READ_COMPACTION_BANNER_PREFIX = "[RTK compacted output:";
+const ANCHORED_READ_LINE_MIN_MATCHES = 2;
+const ANCHORED_READ_LINE_MIN_RATIO = 0.5;
+const ANCHORED_READ_LINE_SAMPLE_LIMIT = 200;
+const ANCHORED_READ_LINE_PATTERNS = [
+	/^\s*(?:>>>|>>|[>+\-*]+)?\s*(\d+)\s*#\s*[A-Za-z0-9_-]{2,32}:(.*)$/,
+	/^\s*(?:>>>|>>|[>+\-*]+)?\s*(\d+)\s*:\s*[A-Za-z0-9_-]{1,32}\|(.*)$/,
+	/^\s*(?:>>>|>>|[>+\-*]+)?\s*(\d+)[a-z]{2}\|(.*)$/,
+] as const;
+const ANCHORED_READ_INFORMATIONAL_LINE_PATTERN = /^\s*(?:$|<\/?file>|\.{3}|\[[^\]]+\]|Read\s+.+:\s+\d+\s+lines\b)/;
 const USER_SKILL_ROOTS = [join(getAgentDir(), "skills"), join(homedir(), ".agents", "skills")];
 
 function normalizePathForComparison(path: string): string {
@@ -135,6 +162,87 @@ function hasExplicitReadRange(input: Record<string, unknown>): boolean {
 	return input.offset !== undefined || input.limit !== undefined;
 }
 
+function splitReadLines(text: string): { lines: string[]; trailingNewline: boolean } {
+	if (!text) {
+		return { lines: [], trailingNewline: false };
+	}
+
+	const trailingNewline = text.endsWith("\n");
+	const lines = text.split(/\r?\n/);
+	if (trailingNewline) {
+		lines.pop();
+	}
+
+	return { lines, trailingNewline };
+}
+
+function joinReadLines(lines: string[], trailingNewline: boolean): string {
+	const joined = lines.join("\n");
+	return trailingNewline && joined ? `${joined}\n` : joined;
+}
+
+function parseAnchoredReadLine(line: string): AnchoredReadLine | undefined {
+	for (const pattern of ANCHORED_READ_LINE_PATTERNS) {
+		const match = line.match(pattern);
+		if (!match) {
+			continue;
+		}
+
+		const lineNumber = Number.parseInt(match[1] ?? "", 10);
+		if (!Number.isSafeInteger(lineNumber) || lineNumber <= 0) {
+			continue;
+		}
+
+		const content = match[2] ?? "";
+		return {
+			lineNumber,
+			content,
+			originalLine: line,
+		};
+	}
+
+	return undefined;
+}
+
+function parseAnchoredReadLineNumber(line: string): number | undefined {
+	return parseAnchoredReadLine(line)?.lineNumber;
+}
+
+function looksLikeAnchoredLineOutput(text: string, parseLineNumber: (line: string) => number | undefined): boolean {
+	let matchCount = 0;
+	let relevantLineCount = 0;
+	let previousMatchedLineNumber: number | undefined;
+	let hasIncreasingAnchors = false;
+
+	for (const line of splitReadLines(text).lines.slice(0, ANCHORED_READ_LINE_SAMPLE_LIMIT)) {
+		if (!ANCHORED_READ_INFORMATIONAL_LINE_PATTERN.test(line)) {
+			relevantLineCount += 1;
+		}
+
+		const lineNumber = parseLineNumber(line);
+		if (lineNumber === undefined) {
+			continue;
+		}
+
+		matchCount += 1;
+		if (previousMatchedLineNumber !== undefined && lineNumber > previousMatchedLineNumber) {
+			hasIncreasingAnchors = true;
+		}
+		previousMatchedLineNumber = lineNumber;
+	}
+
+	if (matchCount < ANCHORED_READ_LINE_MIN_MATCHES || !hasIncreasingAnchors) {
+		return false;
+	}
+
+	const ratioBase = Math.max(relevantLineCount, matchCount);
+	return matchCount / ratioBase >= ANCHORED_READ_LINE_MIN_RATIO;
+}
+
+function looksLikeAnchoredReadOutput(text: string): boolean {
+	return looksLikeAnchoredLineOutput(text, parseAnchoredReadLineNumber);
+}
+
 function shouldPreserveExactReadOutput(
 	text: string,
 	input: Record<string, unknown>,
@@ -165,6 +273,179 @@ function shouldApplyReadSourceFiltering(text: string, config: RtkIntegrationConf
 	);
 }
 
+function extractAnchoredReadParts(text: string): AnchorSafeReadParts | undefined {
+	if (!looksLikeAnchoredReadOutput(text)) {
+		return undefined;
+	}
+
+	const { lines, trailingNewline } = splitReadLines(text);
+	const parsedLines = lines.map((line) => parseAnchoredReadLine(line));
+	const firstAnchorIndex = parsedLines.findIndex((line) => line !== undefined);
+	if (firstAnchorIndex === -1) {
+		return undefined;
+	}
+
+	let lastAnchorIndex = firstAnchorIndex;
+	for (let index = parsedLines.length - 1; index >= firstAnchorIndex; index -= 1) {
+		if (parsedLines[index] !== undefined) {
+			lastAnchorIndex = index;
+			break;
+		}
+	}
+
+	const anchoredLines: AnchoredReadLine[] = [];
+	for (let index = firstAnchorIndex; index <= lastAnchorIndex; index += 1) {
+		const anchoredLine = parsedLines[index];
+		if (!anchoredLine) {
+			return undefined;
+		}
+		anchoredLines.push(anchoredLine);
+	}
+
+	return {
+		prefixLines: lines.slice(0, firstAnchorIndex),
+		anchoredLines,
+		suffixLines: lines.slice(lastAnchorIndex + 1),
+		trailingNewline,
+	};
+}
+
+function toAnchorSafeReadLines(anchoredLines: AnchoredReadLine[]): AnchorSafeReadLine[] {
+	return anchoredLines.map((line) => ({
+		text: line.originalLine,
+		content: line.content,
+	}));
+}
+
+function renderAnchorSafeReadBody(lines: AnchorSafeReadLine[]): string {
+	return lines.map((line) => line.text).join("\n");
+}
+
+function renderAnchorSafeReadText(parts: AnchorSafeReadParts, lines: AnchorSafeReadLine[]): string {
+	return joinReadLines(
+		[...parts.prefixLines, ...lines.map((line) => line.text), ...parts.suffixLines],
+		parts.trailingNewline,
+	);
+}
+
+function remapTransformedContentToAnchorSafeLines(
+	sourceLines: AnchorSafeReadLine[],
+	transformedContent: string,
+): AnchorSafeReadLine[] {
+	const transformedLines = splitReadLines(transformedContent).lines;
+	const remappedLines: AnchorSafeReadLine[] = [];
+	let searchStartIndex = 0;
+
+	for (const transformedLine of transformedLines) {
+		let matchedIndex = -1;
+		for (let index = searchStartIndex; index < sourceLines.length; index += 1) {
+			if (sourceLines[index]?.content === transformedLine) {
+				matchedIndex = index;
+				break;
+			}
+		}
+
+		if (matchedIndex === -1) {
+			remappedLines.push({
+				text: transformedLine,
+				content: transformedLine,
+			});
+			continue;
+		}
+
+		remappedLines.push(sourceLines[matchedIndex]!);
+		searchStartIndex = matchedIndex + 1;
+	}
+
+	return remappedLines;
+}
+
+function truncateAnchorSafeReadLines(lines: AnchorSafeReadLine[], maxChars: number): AnchorSafeReadLine[] {
+	if (renderAnchorSafeReadBody(lines).length <= maxChars) {
+		return lines;
+	}
+
+	const marker = "[RTK anchor-safe truncate: remaining anchored read lines omitted to preserve complete anchors]";
+	const truncatedLines: AnchorSafeReadLine[] = [];
+	let charCount = 0;
+
+	for (let index = 0; index < lines.length; index += 1) {
+		const line = lines[index]!;
+		const separatorLength = truncatedLines.length > 0 ? 1 : 0;
+		const nextCharCount = charCount + separatorLength + line.text.length;
+		const remainingAfter = lines.length - index - 1;
+		const markerLength = remainingAfter > 0 ? (nextCharCount > 0 ? 1 : 0) + marker.length : 0;
+
+		if (nextCharCount + markerLength > maxChars) {
+			const markerLine = { text: marker, content: marker };
+			return truncatedLines.length > 0 ? [...truncatedLines, markerLine] : [markerLine];
+		}
+
+		truncatedLines.push(line);
+		charCount = nextCharCount;
+	}
+
+	return truncatedLines;
+}
+
+function compactAnchoredReadText(
+	text: string,
+	filePath: string,
+	config: RtkIntegrationConfig,
+): { text: string; techniques: string[] } {
+	const parts = extractAnchoredReadParts(text);
+	if (!parts) {
+		return { text, techniques: [] };
+	}
+
+	let lines = toAnchorSafeReadLines(parts.anchoredLines);
+	const techniques: string[] = [];
+	const compaction = config.outputCompaction;
+	const language = detectLanguage(filePath);
+
+	if (
+		compaction.sourceCodeFilteringEnabled &&
+		compaction.sourceCodeFiltering !== "none" &&
+		shouldApplyReadSourceFiltering(text, config)
+	) {
+		const currentSource = lines.map((line) => line.content).join("\n");
+		const filtered = normalizeTechniqueResult(
+			filterSourceCode(currentSource, language, compaction.sourceCodeFiltering),
+			currentSource,
+		);
+		const filteredLines = remapTransformedContentToAnchorSafeLines(lines, filtered);
+		if (renderAnchorSafeReadBody(filteredLines) !== renderAnchorSafeReadBody(lines)) {
+			lines = filteredLines;
+			techniques.push(`source:${compaction.sourceCodeFiltering}`);
+		}
+	}
+
+	if (compaction.smartTruncate.enabled && lines.length > compaction.smartTruncate.maxLines) {
+		const currentSource = lines.map((line) => line.content).join("\n");
+		const compacted = smartTruncate(currentSource, compaction.smartTruncate.maxLines, language);
+		const compactedLines = remapTransformedContentToAnchorSafeLines(lines, compacted);
+		if (renderAnchorSafeReadBody(compactedLines) !== renderAnchorSafeReadBody(lines)) {
+			lines = compactedLines;
+			techniques.push("smart-truncate");
+		}
+	}
+
+	if (compaction.truncate.enabled && renderAnchorSafeReadText(parts, lines).length > compaction.truncate.maxChars) {
+		const nonBodyOverhead = renderAnchorSafeReadText(parts, []).length;
+		const bodyMaxChars = Math.max(1, compaction.truncate.maxChars - nonBodyOverhead);
+		const truncatedLines = truncateAnchorSafeReadLines(lines, bodyMaxChars);
+		if (renderAnchorSafeReadBody(truncatedLines) !== renderAnchorSafeReadBody(lines)) {
+			lines = truncatedLines;
+			techniques.push("truncate");
+		}
+	}
+
+	return {
+		text: renderAnchorSafeReadText(parts, lines),
+		techniques,
+	};
+}
+
 function formatReadCompactionBanner(techniques: string[]): string {
 	return `${READ_COMPACTION_BANNER_PREFIX} ${techniques.join(", ")}]`;
 }
@@ -285,6 +566,18 @@ function compactReadText(
 		}
 	}
 
+	if (looksLikeAnchoredReadOutput(nextText)) {
+		const anchored = compactAnchoredReadText(nextText, filePath, config);
+		nextText = anchored.text;
+		techniques.push(...anchored.techniques);
+
+		if (techniques.length > 0 && !nextText.startsWith(READ_COMPACTION_BANNER_PREFIX)) {
+			nextText = `${formatReadCompactionBanner(techniques)}\n${nextText}`;
+		}
+
+		return { text: nextText, techniques };
+	}
+
 	const language = detectLanguage(filePath);
 	// Only apply lossy source filtering when a downstream line/char safeguard would otherwise trigger.
 	if (