ultimate-pi 0.19.0 → 0.20.0

package/.pi/lib/harness-anchored-edit/anchor-state.ts

@@ -0,0 +1,320 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+import * as diff from "diff";
+
+interface TrackedDocument {
+	hashes: Uint32Array;
+	anchors: string[];
+	usedWords: Set<string>;
+	availablePool?: string[];
+}
+
+export class AnchorStateManager {
+	private static storage = new Map<string, Map<string, TrackedDocument>>();
+	private static dictionary: string[] = [];
+	private static readonly MAX_TRACKED_LINES = 50000;
+	private static readonly MAX_TRACKED_FILES = 1024;
+	private static readonly MAX_TRACKED_TASKS = 50;
+
+	private static computeHashes(lines: string[]): Uint32Array {
+		const hashes = new Uint32Array(lines.length);
+		for (let i = 0; i < lines.length; i++) {
+			const line = lines[i];
+			let h = 2166136261;
+			for (let j = 0; j < line.length; j++) {
+				h = Math.imul(h ^ line.charCodeAt(j), 16777619);
+			}
+			hashes[i] = h >>> 0;
+		}
+		return hashes;
+	}
+
+	private static getDictionary(): string[] {
+		if (AnchorStateManager.dictionary.length === 0) {
+			const dictionaryPath = path.join(
+				path.dirname(fileURLToPath(import.meta.url)),
+				".hash_anchors",
+			);
+			// do not catch errors here, we should fail loudly
+			AnchorStateManager.dictionary = fs
+				.readFileSync(dictionaryPath, "utf8")
+				.split(/\r?\n/)
+				.filter(Boolean);
+		}
+		return AnchorStateManager.dictionary;
+	}
+
+	private static refill(usedWords: Set<string>, pool: string[]) {
+		const dict = AnchorStateManager.getDictionary();
+		const dictLen = dict.length;
+		const newWords: string[] = [];
+
+		// Try to find 10,000 unique two-word combinations
+		let attempts = 0;
+		while (newWords.length < 10000 && attempts < 50000) {
+			const w1 = dict[Math.floor(Math.random() * dictLen)];
+			const w2 = dict[Math.floor(Math.random() * dictLen)];
+			const word = `${w1}${w2}`;
+			if (!usedWords.has(word)) {
+				newWords.push(word);
+			}
+			attempts++;
+		}
+
+		// Extreme fallback: three-word combinations if we are struggling
+		if (newWords.length < 100) {
+			for (let i = 0; i < 100; i++) {
+				const w1 = dict[Math.floor(Math.random() * dictLen)];
+				const w2 = dict[Math.floor(Math.random() * dictLen)];
+				const w3 = dict[Math.floor(Math.random() * dictLen)];
+				const word = `${w1}${w2}${w3}`;
+				if (!usedWords.has(word)) {
+					newWords.push(word);
+				}
+			}
+		}
+
+		// Shuffle the new batch and add to pool
+		for (let i = newWords.length - 1; i > 0; i--) {
+			const j = Math.floor(Math.random() * (i + 1));
+			[newWords[i], newWords[j]] = [newWords[j], newWords[i]];
+		}
+		pool.push(...newWords);
+	}
+
+	private static getUniqueWord(usedWords: Set<string>, pool: string[]): string {
+		while (true) {
+			if (pool.length === 0) {
+				AnchorStateManager.refill(usedWords, pool);
+			}
+
+			const word = pool.pop()!;
+			if (!usedWords.has(word)) {
+				return word;
+			}
+			// If we hit a collision (word was in usedWords but not in pool),
+			// just pop the next one.
+		}
+	}
+
+	private static getTaskState(
+		taskId = "default",
+	): Map<string, TrackedDocument> {
+		let state = AnchorStateManager.storage.get(taskId);
+		if (!state) {
+			state = new Map<string, TrackedDocument>();
+			AnchorStateManager.storage.set(taskId, state);
+
+			// Implement LRU for tasks
+			if (
+				AnchorStateManager.storage.size > AnchorStateManager.MAX_TRACKED_TASKS
+			) {
+				const oldestTaskId = AnchorStateManager.storage.keys().next().value;
+				if (oldestTaskId !== undefined) {
+					AnchorStateManager.storage.delete(oldestTaskId);
+				}
+			}
+		} else {
+			// Refresh LRU position for existing task
+			AnchorStateManager.storage.delete(taskId);
+			AnchorStateManager.storage.set(taskId, state);
+		}
+		return state;
+	}
+
+	/**
+	 * Reconciles the current file content with our saved state using Myers Diff.
+	 * Unchanged lines keep their exact word anchors. New lines get new words.
+	 */
+	public static reconcile(
+		absolutePath: string,
+		currentLines: string[],
+		taskId?: string,
+	): string[] {
+		// Safeguard for massive files
+		if (currentLines.length > AnchorStateManager.MAX_TRACKED_LINES) {
+			return currentLines.map((_, i) => `L${i + 1}`);
+		}
+
+		const state = AnchorStateManager.getTaskState(taskId);
+		const currentHashes = AnchorStateManager.computeHashes(currentLines);
+		let tracked = state.get(absolutePath);
+
+		// Fast path: if hashes are identical, nothing changed
+		if (tracked && tracked.hashes.length === currentHashes.length) {
+			let identical = true;
+			for (let i = 0; i < currentHashes.length; i++) {
+				if (tracked.hashes[i] !== currentHashes[i]) {
+					identical = false;
+					break;
+				}
+			}
+			if (identical) {
+				// Refresh LRU position
+				AnchorStateManager.updateState(absolutePath, tracked, taskId);
+				return tracked.anchors;
+			}
+		}
+
+		// First time seeing this file? Assign unique random words to every line.
+		if (!tracked) {
+			const usedWords = new Set<string>();
+			const pool = [...AnchorStateManager.getDictionary()];
+			// Initial shuffle of dictionary
+			for (let i = pool.length - 1; i > 0; i--) {
+				const j = Math.floor(Math.random() * (i + 1));
+				[pool[i], pool[j]] = [pool[j], pool[i]];
+			}
+
+			const anchors = currentLines.map(() => {
+				const w = AnchorStateManager.getUniqueWord(usedWords, pool);
+				usedWords.add(w);
+				return w;
+			});
+
+			tracked = {
+				hashes: currentHashes,
+				anchors,
+				usedWords,
+				availablePool: pool,
+			};
+			AnchorStateManager.updateState(absolutePath, tracked, taskId);
+			return anchors;
+		}
+
+		// We have history! Run Myers Diff on hashes (integers) instead of strings.
+		// Note: diffArrays accepts any array-like, but we convert Uint32Array to regular Array
+		// because jsdiff's internal comparisons are more reliable with standard Arrays.
+		const changes = diff.diffArrays(
+			Array.from(tracked.hashes),
+			Array.from(currentHashes),
+		);
+
+		const newAnchors: string[] = [];
+		const newUsedWords = new Set<string>(tracked.usedWords);
+		const pool = tracked.availablePool || [];
+		// If pool was lost (e.g. from an older version of state), initialize it
+		if (
+			pool.length === 0 &&
+			newUsedWords.size < AnchorStateManager.getDictionary().length
+		) {
+			const dict = AnchorStateManager.getDictionary();
+			for (const word of dict) {
+				if (!newUsedWords.has(word)) {
+					pool.push(word);
+				}
+			}
+			// Shuffle initial single-word pool
+			for (let i = pool.length - 1; i > 0; i--) {
+				const j = Math.floor(Math.random() * (i + 1));
+				[pool[i], pool[j]] = [pool[j], pool[i]];
+			}
+		}
+
+		let oldIdx = 0;
+
+		for (const change of changes) {
+			if (change.added) {
+				// New lines (typed by user or added by LLM) get NEW words
+				for (let i = 0; i < change.count!; i++) {
+					const word = AnchorStateManager.getUniqueWord(newUsedWords, pool);
+					newAnchors.push(word);
+					newUsedWords.add(word);
+				}
+			} else if (change.removed) {
+				// Deleted lines: We just advance the old index.
+				oldIdx += change.count!;
+			} else {
+				// Unchanged lines: CARRY OVER THE EXACT SAME WORD ANCHOR
+				for (let i = 0; i < change.count!; i++) {
+					const preservedWord = tracked.anchors[oldIdx];
+					newAnchors.push(preservedWord);
+					newUsedWords.add(preservedWord);
+					oldIdx++;
+				}
+			}
+		}
+
+		// Update the state cache
+		tracked = {
+			hashes: currentHashes,
+			anchors: newAnchors,
+			usedWords: newUsedWords,
+			availablePool: pool,
+		};
+		AnchorStateManager.updateState(absolutePath, tracked, taskId);
+		return newAnchors;
+	}
+
+	private static updateState(
+		absolutePath: string,
+		document: TrackedDocument,
+		taskId?: string,
+	) {
+		const state = AnchorStateManager.getTaskState(taskId);
+		// Implement LRU by deleting and re-inserting
+		state.delete(absolutePath);
+		state.set(absolutePath, document);
+
+		// Evict oldest if limit exceeded
+		if (state.size > AnchorStateManager.MAX_TRACKED_FILES) {
+			const oldestKey = state.keys().next().value;
+			if (oldestKey !== undefined) {
+				state.delete(oldestKey);
+			}
+		}
+	}
+
+	/**
+	 * Returns true if the file is currently being tracked.
+	 */
+	public static isTracking(absolutePath: string, taskId?: string): boolean {
+		return AnchorStateManager.getTaskState(taskId).has(absolutePath);
+	}
+
+	/**
+	 * Gets current anchors for a file if it's being tracked, otherwise returns null.
+	 */
+	public static getAnchors(
+		absolutePath: string,
+		taskId?: string,
+	): string[] | null {
+		return (
+			AnchorStateManager.getTaskState(taskId).get(absolutePath)?.anchors || null
+		);
+	}
+
+	/**
+	 * Clear state for a file (useful if needed for cleanup)
+	 */
+	public static clearState(absolutePath: string, taskId?: string) {
+		AnchorStateManager.getTaskState(taskId).delete(absolutePath);
+	}
+
+	/**
+	 * Resets all anchors for a specific task or all tasks.
+	 */
+	public static reset(taskId?: string) {
+		if (taskId) {
+			AnchorStateManager.storage.delete(taskId);
+		} else {
+			AnchorStateManager.storage.clear();
+		}
+	}
+}
+
+import { formatLineWithHash } from "./line-protocol.js";
+
+export function hashLinesStateful(
+	absolutePath: string,
+	content: string,
+	taskId?: string,
+): string {
+	if (!content) return "";
+	const lines = content.split(/\r?\n/);
+	const anchors = AnchorStateManager.reconcile(absolutePath, lines, taskId);
+	return lines
+		.map((line, index) => formatLineWithHash(line, anchors[index]!))
+		.join("\n");
+}

package/.pi/lib/harness-anchored-edit/apply-anchored-edits.ts

@@ -0,0 +1,161 @@
+import { constants } from "node:fs";
+import { access, readFile, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { AnchorStateManager } from "./anchor-state.js";
+import { EditExecutor } from "./edit-executor.js";
+import type { AnchoredEdit } from "./types.js";
+
+type EditDiffHelpers = {
+	detectLineEnding: (content: string) => "\r\n" | "\n";
+	generateDiffString: (
+		oldContent: string,
+		newContent: string,
+	) => { diff: string; firstChangedLine?: number };
+	normalizeToLF: (text: string) => string;
+	restoreLineEndings: (text: string, ending: "\r\n" | "\n") => string;
+	stripBom: (content: string) => { bom: string; text: string };
+};
+
+type FileMutationQueue = <T>(path: string, fn: () => Promise<T>) => Promise<T>;
+
+let editDiffHelpers: EditDiffHelpers | undefined;
+let fileMutationQueue: FileMutationQueue | undefined;
+
+function resolvePiDistEntry(): string {
+	const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "../../..");
+	return join(
+		repoRoot,
+		"node_modules/@earendil-works/pi-coding-agent/dist/index.js",
+	);
+}
+
+async function loadPiEditRuntime(): Promise<{
+	editDiff: EditDiffHelpers;
+	withFileMutationQueue: FileMutationQueue;
+}> {
+	if (editDiffHelpers && fileMutationQueue) {
+		return {
+			editDiff: editDiffHelpers,
+			withFileMutationQueue: fileMutationQueue,
+		};
+	}
+	const piEntry = resolvePiDistEntry();
+	const pi = await import("@earendil-works/pi-coding-agent");
+	const require = createRequire(piEntry);
+	editDiffHelpers = require(
+		join(dirname(piEntry), "core/tools/edit-diff.js"),
+	) as EditDiffHelpers;
+	fileMutationQueue = pi.withFileMutationQueue as FileMutationQueue;
+	return {
+		editDiff: editDiffHelpers,
+		withFileMutationQueue: fileMutationQueue,
+	};
+}
+
+export type ApplyAnchoredEditsResult =
+	| {
+			ok: true;
+			details: { diff: string; firstChangedLine?: number };
+			message: string;
+	  }
+	| { ok: false; error: string };
+
+export async function applyAnchoredEditsToFile(
+	absolutePath: string,
+	edits: AnchoredEdit[],
+	taskId: string,
+): Promise<ApplyAnchoredEditsResult> {
+	if (!edits.length) {
+		return {
+			ok: false,
+			error: "edit: edits must contain at least one anchored edit.",
+		};
+	}
+
+	const { editDiff, withFileMutationQueue } = await loadPiEditRuntime();
+	const {
+		detectLineEnding,
+		generateDiffString,
+		normalizeToLF,
+		restoreLineEndings,
+		stripBom,
+	} = editDiff;
+
+	return withFileMutationQueue(absolutePath, async () => {
+		try {
+			await access(absolutePath, constants.R_OK | constants.W_OK);
+		} catch (error) {
+			const code =
+				error instanceof Error && "code" in error
+					? String(error.code)
+					: String(error);
+			return {
+				ok: false,
+				error: `Could not edit file: ${absolutePath}. Error code: ${code}.`,
+			};
+		}
+
+		const buffer = await readFile(absolutePath);
+		const rawContent = buffer.toString("utf-8");
+		const { bom, text: content } = stripBom(rawContent);
+		const originalEnding = detectLineEnding(content);
+		const normalizedContent = normalizeToLF(content);
+		const lines = normalizedContent.split("\n");
+
+		const lineAnchors = AnchorStateManager.reconcile(
+			absolutePath,
+			lines,
+			taskId,
+		);
+		const executor = new EditExecutor();
+		const { resolvedEdits, failedEdits } = executor.resolveEdits(
+			edits,
+			lines,
+			lineAnchors,
+		);
+
+		if (failedEdits.length > 0) {
+			return {
+				ok: false,
+				error: failedEdits
+					.map((f) => executor.formatFailureMessage(f.edit, f.error))
+					.join("\n"),
+			};
+		}
+
+		const { finalLines } = executor.applyEdits(lines, resolvedEdits);
+		const newContent = finalLines.join("\n");
+
+		if (newContent === normalizedContent) {
+			return {
+				ok: false,
+				error:
+					"Anchored edit made no changes. Re-read the file and verify anchors and text.",
+			};
+		}
+
+		const finalContent = bom + restoreLineEndings(newContent, originalEnding);
+		await writeFile(absolutePath, finalContent, "utf-8");
+
+		const diffResult = generateDiffString(normalizedContent, newContent);
+		return {
+			ok: true,
+			details: {
+				diff: diffResult.diff,
+				firstChangedLine: diffResult.firstChangedLine,
+			},
+			message: `Successfully applied ${edits.length} anchored edit(s).`,
+		};
+	});
+}
+
+export function isAnchoredEditInput(input: unknown): boolean {
+	if (!input || typeof input !== "object") return false;
+	const edits = (input as { edits?: unknown }).edits;
+	if (!Array.isArray(edits) || edits.length === 0) return false;
+	const first = edits[0];
+	if (!first || typeof first !== "object") return false;
+	return typeof (first as { anchor?: unknown }).anchor === "string";
+}

package/.pi/lib/harness-anchored-edit/edit-executor.ts

@@ -0,0 +1,146 @@
+import { getDelimiter, splitAnchor, stripHashes } from "./line-protocol.js";
+import type {
+	AnchoredEdit,
+	FailedAnchoredEdit,
+	ResolvedAnchoredEdit,
+} from "./types.js";
+
+export class EditExecutor {
+	resolveEdits(
+		edits: AnchoredEdit[],
+		lines: string[],
+		lineAnchors: string[],
+	): {
+		resolvedEdits: ResolvedAnchoredEdit[];
+		failedEdits: FailedAnchoredEdit[];
+	} {
+		const failedEdits: FailedAnchoredEdit[] = [];
+		const resolvedEdits: ResolvedAnchoredEdit[] = [];
+		const normalized = lineAnchors.map((h) => h.trim());
+
+		for (const edit of edits) {
+			const diagnostics: string[] = [];
+			const editType = edit.edit_type ?? "replace";
+
+			const { index: lineIdx, error: startError } = this.resolveAnchor(
+				"anchor",
+				edit.anchor,
+				normalized,
+				lines,
+			);
+			if (startError) diagnostics.push(startError);
+
+			let endIdx = lineIdx;
+			if (editType === "replace") {
+				const endAnchorRaw = edit.end_anchor ?? edit.anchor;
+				const { index: resolvedEndIdx, error: endError } = this.resolveAnchor(
+					"end_anchor",
+					endAnchorRaw,
+					normalized,
+					lines,
+				);
+				if (endError) diagnostics.push(endError);
+				endIdx = resolvedEndIdx;
+			}
+
+			if (lineIdx !== -1 && endIdx !== -1 && endIdx < lineIdx) {
+				diagnostics.push(
+					"Range error: anchor must refer to a line that precedes or is the same as end_anchor.",
+				);
+			}
+
+			if (diagnostics.length > 0) {
+				failedEdits.push({ edit, error: diagnostics.join(" ") });
+			} else {
+				resolvedEdits.push({ lineIdx, endIdx, edit });
+			}
+		}
+		return { resolvedEdits, failedEdits };
+	}
+
+	resolveAnchor(
+		type: "anchor" | "end_anchor",
+		rawAnchor: string | undefined,
+		normalizedLineHashes: string[],
+		lines: string[],
+	): { index: number; error?: string } {
+		const anchorRaw = rawAnchor || "";
+		if (!anchorRaw.trim()) return { index: -1, error: `${type} is missing.` };
+
+		const { anchor: anchorName, content: providedContent } =
+			splitAnchor(anchorRaw);
+		const anchorExtractRegex = /^[A-Z][a-zA-Z]*$/;
+		if (!anchorExtractRegex.test(anchorName)) {
+			return {
+				index: -1,
+				error: `${type} is missing or incorrectly formatted. It must start with a single word followed by the delimiter (e.g., "Apple${getDelimiter()}").`,
+			};
+		}
+
+		const index = normalizedLineHashes.indexOf(anchorName);
+		if (index === -1) {
+			return {
+				index: -1,
+				error: `${type} "${anchorName}" not found in the file. Re-read the file for current anchors.`,
+			};
+		}
+
+		if (providedContent.includes("\n") || providedContent.includes("\r")) {
+			return {
+				index: -1,
+				error: `${type} "${anchorName}" must refer to a single line (Anchor${getDelimiter()}{line_text}).`,
+			};
+		}
+
+		const actualContent = lines[index];
+		if (providedContent !== actualContent) {
+			return {
+				index: -1,
+				error: `${type} "${anchorName}" line mismatch. Expected: "${actualContent}", Provided: "${providedContent}".`,
+			};
+		}
+
+		return { index };
+	}
+
+	applyEdits(
+		lines: string[],
+		resolvedEdits: ResolvedAnchoredEdit[],
+	): { finalLines: string[] } {
+		const sortedEdits = [...resolvedEdits].sort(
+			(a, b) => b.lineIdx - a.lineIdx,
+		);
+		const newLines = [...lines];
+
+		for (const { lineIdx, endIdx, edit } of sortedEdits) {
+			const editType = edit.edit_type ?? "replace";
+			const cleanText = stripHashes(edit.text || "");
+			const replacementLines = cleanText === "" ? [] : cleanText.split(/\r?\n/);
+
+			let removedInThisEdit: number;
+			let spliceIndex: number;
+
+			if (editType === "insert_after") {
+				spliceIndex = lineIdx + 1;
+				removedInThisEdit = 0;
+			} else if (editType === "insert_before") {
+				spliceIndex = lineIdx;
+				removedInThisEdit = 0;
+			} else {
+				spliceIndex = lineIdx;
+				removedInThisEdit = endIdx - lineIdx + 1;
+			}
+
+			newLines.splice(spliceIndex, removedInThisEdit, ...replacementLines);
+		}
+
+		return { finalLines: newLines };
+	}
+
+	formatFailureMessage(edit: AnchoredEdit, error?: string): string {
+		const diagnostic = error
+			? ` ${error}`
+			: " Check anchors and line content from the latest read.";
+		return `Anchored edit failed (anchor: "${edit.anchor}", end_anchor: "${edit.end_anchor ?? ""}").${diagnostic}`;
+	}
+}

package/.pi/lib/harness-anchored-edit/index.ts

@@ -0,0 +1,9 @@
+export { AnchorStateManager, hashLinesStateful } from "./anchor-state.js";
+export {
+	type ApplyAnchoredEditsResult,
+	applyAnchoredEditsToFile,
+	isAnchoredEditInput,
+} from "./apply-anchored-edits.js";
+export { ANCHOR_DELIMITER, formatLineWithHash } from "./line-protocol.js";
+export { anchoredEditTaskId } from "./task-id.js";
+export type { AnchoredEdit, AnchoredEditType } from "./types.js";

package/.pi/lib/harness-anchored-edit/line-protocol.ts

@@ -0,0 +1,38 @@
+/** Hash-anchored line protocol (from Dirac, Apache-2.0). */
+
+export const ANCHOR_DELIMITER = "§";
+
+export function getDelimiter(): string {
+	return ANCHOR_DELIMITER;
+}
+
+export function escapeRegExp(string: string): string {
+	return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+export function stripHashes(content: string): string {
+	if (!content) return "";
+	const delimiterRegex = new RegExp(
+		`\\b[A-Z][a-zA-Z]*?${escapeRegExp(ANCHOR_DELIMITER)}`,
+		"g",
+	);
+	return content.replace(delimiterRegex, "");
+}
+
+export function splitAnchor(rawAnchor: string): {
+	anchor: string;
+	content: string;
+} {
+	const delimiterIndex = rawAnchor.indexOf(ANCHOR_DELIMITER);
+	if (delimiterIndex === -1) {
+		return { anchor: rawAnchor.trim(), content: "" };
+	}
+	return {
+		anchor: rawAnchor.substring(0, delimiterIndex).trim(),
+		content: rawAnchor.substring(delimiterIndex + ANCHOR_DELIMITER.length),
+	};
+}
+
+export function formatLineWithHash(content: string, anchor: string): string {
+	return `${anchor}${ANCHOR_DELIMITER}${content}`;
+}

package/.pi/lib/harness-anchored-edit/settings.ts

@@ -0,0 +1 @@
+export { anchoredEditTaskId } from "./task-id.js";

package/.pi/lib/harness-anchored-edit/task-id.ts

@@ -0,0 +1,8 @@
+/** Per-session anchor state scope for hash reconciliation. */
+export function anchoredEditTaskId(ctx?: { sessionId?: string }): string {
+	return (
+		ctx?.sessionId?.trim() ||
+		process.env.PI_SESSION_ID?.trim() ||
+		"harness-default"
+	);
+}

package/.pi/lib/harness-anchored-edit/types.ts

@@ -0,0 +1,19 @@
+export type AnchoredEditType = "replace" | "insert_after" | "insert_before";
+
+export interface AnchoredEdit {
+	anchor: string;
+	end_anchor?: string;
+	edit_type?: AnchoredEditType;
+	text: string;
+}
+
+export interface ResolvedAnchoredEdit {
+	lineIdx: number;
+	endIdx: number;
+	edit: AnchoredEdit;
+}
+
+export interface FailedAnchoredEdit {
+	edit: AnchoredEdit;
+	error: string;
+}