pi-posthorse 0.8.0 → 0.8.2

package/src/history-entries.ts

@@ -0,0 +1,251 @@
+import {
+	isRecord,
+	isReminderType,
+	unknownArray,
+	type EntryLike,
+	type HistoryHit,
+	type MessageLike,
+	type WindowedEntry,
+} from "./contracts.ts";
+import {
+	assistantFailure,
+	excerptAround,
+	imageSummary,
+	imagesOf,
+	isRecoveryCall,
+	isRecoveryTool,
+	shortKey,
+	sourceTag,
+	textOf,
+	toolIdentity,
+} from "./message-text.ts";
+
+export function isWindow(entry: EntryLike): boolean {
+	return (
+		entry.type === "context_window" ||
+		(entry.type === "compaction" && isRecord(entry.details) && entry.details.posthorse === 1)
+	);
+}
+export function windowHandoff(entry: EntryLike | undefined): string | undefined {
+	return entry?.type === "compaction" ? entry.summary : entry?.handoff;
+}
+/** First active branch entry, including tails retained by Pi's own /compact. */
+export function contextStart(entries: readonly EntryLike[]): {
+	start: number;
+	boundary?: EntryLike;
+} {
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		if (entry.type === "context_window") {
+			return { start: i + 1, boundary: entry };
+		}
+		if (entry.type === "compaction") {
+			const kept = entries.findIndex((candidate) => candidate.id === entry.firstKeptEntryId);
+			return { start: kept >= 0 ? kept : i + 1, boundary: entry };
+		}
+	}
+	return { start: 0 };
+}
+export function contextEdits(entries: readonly EntryLike[]): Map<string, EntryLike> {
+	return new Map(
+		entries.flatMap((entry) =>
+			entry.type === "context_edit" && entry.targetId !== undefined && entry.targetId !== ""
+				? [[entry.targetId, entry]]
+				: [],
+		),
+	);
+}
+function flattenMessage(message: MessageLike): string {
+	if (message.role === "bashExecution") {
+		if (message.excludeFromContext === true) {
+			return "[bashExecution] (excluded from model context by Pi)";
+		}
+		return `[bashExecution] $ ${message.command ?? ""}\n${message.output ?? ""}`;
+	}
+	const identity =
+		message.role === "toolResult"
+			? `Tool: ${toolIdentity(message.toolName, message.namespace)}${message.toolCallId === undefined ? "" : `\nCall ID: ${message.toolCallId}`}`
+			: "";
+	const parts = [
+		textOf(message),
+		assistantFailure(message),
+		imageSummary(imagesOf(message.content)),
+		identity,
+	];
+	return `[${message.role ?? "message"}] ${parts.filter((part) => part !== "").join("\n")}`;
+}
+export function flattenEntry(entry: EntryLike): string | undefined {
+	switch (entry.type) {
+		case "message":
+			return flattenMessage(entry.message ?? {});
+		case "compaction":
+		case "branch_summary":
+			return `[${entry.type}] ${entry.summary ?? ""}`;
+		case "context_window":
+			return `[context_window] ${entry.handoff === undefined || entry.handoff === "" ? "No handoff" : `Handoff: ${entry.handoff}`}`;
+		case "custom_message":
+			return `[custom:${entry.customType ?? "unknown"}] ${contentText(entry.content)}`;
+		case "context_edit":
+			return entry.replacement === undefined || entry.replacement === null
+				? undefined
+				: `[context_edit] ${contentText(entry.replacement.content)}`;
+		case undefined:
+			return undefined;
+		default:
+			return undefined;
+	}
+}
+function contentText(content: unknown): string {
+	return [textOf({ content }), imageSummary(imagesOf(content))]
+		.filter((part) => part !== "")
+		.join("\n");
+}
+function ownAssistantText(message: MessageLike, images: WindowedEntry["images"]): string[] {
+	const originals = [""];
+	for (const part of unknownArray(message.content) ?? []) {
+		if (isRecoveryCall(part)) {
+			originals.push("");
+		} else {
+			const partText = textOf({ content: [part] });
+			const index = originals.length - 1;
+			if (partText !== "") {
+				originals[index] += `${originals[index] === "" ? "" : "\n"}${partText}`;
+			}
+		}
+	}
+	const metadata = [assistantFailure(message), imageSummary(images)]
+		.filter((part) => part !== "")
+		.join("\n");
+	const last = originals.length - 1;
+	if (metadata !== "") {
+		originals[last] += `${originals[last] === "" ? "" : "\n"}${metadata}`;
+	}
+	if (originals[0] !== "") {
+		originals[0] = `[assistant] ${originals[0]}`;
+	}
+	return originals;
+}
+function echoPriority(entry: EntryLike): 0 | 1 {
+	if (
+		entry.type === "context_window" ||
+		entry.type === "compaction" ||
+		entry.type === "branch_summary"
+	) {
+		return 1;
+	}
+	if (entry.type === "custom_message" && isReminderType(entry.customType)) {
+		return 1;
+	}
+	return entry.type === "message" &&
+		entry.message?.role === "toolResult" &&
+		isRecoveryTool(entry.message.toolName)
+		? 1
+		: 0;
+}
+function originalMatch(
+	item: WindowedEntry,
+	query: string,
+): { readonly text?: string; readonly mixed: boolean } {
+	const message = item.entry.message;
+	if (item.entry.type !== "message" || message?.role !== "assistant") {
+		return { mixed: false };
+	}
+	if (!(unknownArray(message.content) ?? []).some(isRecoveryCall)) {
+		return { mixed: false };
+	}
+	const originals = ownAssistantText(message, item.images);
+	const original = originals.find((part) => part.toLowerCase().includes(query));
+	if (original === undefined || original === "") {
+		return { mixed: true };
+	}
+	return { mixed: true, text: original === originals[0] ? original : `[assistant] ${original}` };
+}
+export function historyHit(
+	item: WindowedEntry,
+	query: string,
+	source = "",
+): HistoryHit | undefined {
+	const { entry } = item;
+	const id = entry.id;
+	if (id === undefined || id === "" || !item.text.toLowerCase().includes(query)) {
+		return undefined;
+	}
+	const original = originalMatch(item, query);
+	const text = original.text ?? item.text;
+	let priority = echoPriority(entry);
+	if (original.mixed) {
+		priority = original.text === undefined ? 1 : 0;
+	}
+	const qualifiedId = source === "" ? id : `${id}@${shortKey(source)}`;
+	const header = `${entry.timestamp ?? ""}${sourceTag(source)} [window ${item.windowId}] [${qualifiedId}] `;
+	return {
+		id: qualifiedId,
+		priority,
+		text: `${header}${excerptAround(text, text.toLowerCase().indexOf(query), 100, 400)}`,
+		headerLength: header.length,
+	};
+}
+export function entryContent(entry: EntryLike): unknown {
+	switch (entry.type) {
+		case "context_edit":
+			return entry.replacement?.content;
+		case "message":
+			return entry.message?.content;
+		case undefined:
+			return entry.content;
+		default:
+			return entry.content;
+	}
+}
+export function toWindowedEntry(
+	entry: EntryLike,
+	inheritedWindow: string,
+): WindowedEntry | undefined {
+	const id = entry.id;
+	const text = flattenEntry(entry);
+	if (id === undefined || id === "" || text === undefined || text === "") {
+		return undefined;
+	}
+	return {
+		entry,
+		windowId: isWindow(entry) ? id : inheritedWindow,
+		text,
+		images: imagesOf(entryContent(entry)),
+	};
+}
+/** Window ancestry is metadata-only; extracting text/images is a separate operation. */
+export function windowProjection(): (entry: EntryLike) => string {
+	const windows = new Map<string, string>();
+	return (entry) => {
+		const inherited = windows.get(entry.parentId ?? "") ?? "initial";
+		const windowId = isWindow(entry) ? (entry.id ?? inherited) : inherited;
+		if (entry.id !== undefined && entry.id !== "") {
+			windows.set(entry.id, windowId);
+		}
+		return windowId;
+	};
+}
+export function* reverseWindowEntries(entries: readonly EntryLike[]): Generator<WindowedEntry> {
+	// Preserve parent-relative window labels without normalizing the whole branch.
+	// ponytail: ancestry still scans metadata; a native window-aware iterator could remove that cost.
+	const windows = entries.map(windowProjection());
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const item = toWindowedEntry(entries[i], windows[i]);
+		if (item !== undefined) {
+			yield item;
+		}
+	}
+}
+export function seenText(message: MessageLike): string {
+	return [
+		textOf(message),
+		assistantFailure(message),
+		imageSummary(imagesOf(message.content)),
+		message.summary,
+		message.command,
+		message.output,
+	]
+		.filter((part) => part !== undefined && part !== "")
+		.join("\n")
+		.toLowerCase();
+}

package/src/history-pages.ts

@@ -0,0 +1,173 @@
+import {
+	MIN_PAGE_CHARS,
+	ESTIMATED_IMAGE_CHARS,
+	type HistoryHit,
+	type PolicyContext,
+	type TextResult,
+	type WindowedEntry,
+} from "./contracts.ts";
+import { excerpt, shortKey, sourceTag } from "./message-text.ts";
+import type { HistoryMatches, HistoryParams } from "./history-search.ts";
+import type { PageAccess } from "./paging.ts";
+
+type SearchPage = {
+	readonly parts: readonly string[];
+	readonly spans: readonly { readonly headerLength: number; readonly length: number }[];
+	readonly next?: string;
+};
+function cursorFor(matches: HistoryMatches, hit: HistoryHit, offset: number): string {
+	return Buffer.from(JSON.stringify([hit.id, hit.priority, offset, matches.key])).toString(
+		"base64url",
+	);
+}
+function searchFooter(next: string): string {
+	return `\n[More results; continue with cursor "${next}" and the same query/scope.]`;
+}
+function hitFragment(
+	hit: HistoryHit,
+	cursor: HistoryMatches["cursor"],
+	available: number,
+	first: boolean,
+):
+	| {
+			readonly part: string;
+			readonly end: number;
+			readonly headerLength: number;
+	  }
+	| undefined {
+	const offset = cursor?.[0] === hit.id ? cursor[2] : 0;
+	if (!first && hit.text.length - offset > available) {
+		return undefined;
+	}
+	const prefix =
+		offset > 0 || hit.text.length > available ? `[entry ${hit.id}; from char ${offset}] ` : "";
+	if (prefix.length >= available) {
+		return undefined;
+	}
+	const end = Math.min(hit.text.length, offset + available - prefix.length);
+	return {
+		part: prefix + hit.text.slice(offset, end),
+		end,
+		headerLength:
+			prefix.length + Math.max(0, Math.min(hit.headerLength - offset, end - offset)),
+	};
+}
+function searchParts(matches: HistoryMatches, chars: number): SearchPage {
+	const parts: string[] = [];
+	const spans: Array<{ headerLength: number; length: number }> = [];
+	let available = chars;
+	let next: string | undefined;
+	for (const hit of matches.hits) {
+		const fragment = hitFragment(hit, matches.cursor, available, parts.length === 0);
+		if (fragment === undefined) {
+			break;
+		}
+		parts.push(fragment.part);
+		spans.push({ headerLength: fragment.headerLength, length: fragment.part.length });
+		available -= fragment.part.length + 1;
+		next =
+			fragment.end < hit.text.length || parts.length < matches.hits.length
+				? cursorFor(matches, hit, fragment.end)
+				: undefined;
+		if (available <= 0 || parts.length >= matches.limit) {
+			break;
+		}
+	}
+	return { parts, spans, next };
+}
+export function searchPage(
+	matches: HistoryMatches,
+	params: HistoryParams,
+	host: PolicyContext,
+	pages: PageAccess,
+): TextResult {
+	const chars = pages.size(host, { offset: 0, cursor: params.cursor });
+	const reserve = Math.max(
+		0,
+		...matches.hits.map((hit) => searchFooter(cursorFor(matches, hit, hit.text.length)).length),
+	);
+	if (reserve >= chars) {
+		pages.error(host, "History entry id is too large for pagination.", params.cursor);
+	}
+	const skippedNote =
+		matches.skipped > 0 && (params.cursor === undefined || params.cursor === "")
+			? `\n[Skipped ${matches.skipped} match${matches.skipped === 1 ? "" : "es"} already in your active context.]`
+			: "";
+	const page = searchParts(matches, chars - reserve - skippedNote.length);
+	const body =
+		page.parts.length > 0
+			? page.parts.join("\n")
+			: `No history matches "${excerpt(params.query ?? "", 200)}".`;
+	const tail = skippedNote + (page.next === undefined ? "" : searchFooter(page.next));
+	return pages.result(body + tail, [], {
+		kind: "history-search",
+		entries: [...page.spans],
+		footerLength: tail.length,
+		more: page.next !== undefined,
+		skipped: matches.skipped,
+	});
+}
+function readOffsets(
+	item: WindowedEntry,
+	params: HistoryParams,
+): { readonly offset: number; readonly imageOffset: number } {
+	const offset = params.offset ?? 0;
+	const imageOffset = params.imageOffset ?? (offset === 0 ? 0 : item.images.length);
+	const id = params.id ?? "";
+	if (imageOffset > item.images.length) {
+		throw new Error(
+			`Image offset ${imageOffset} is past the end of history entry "${id}" (${item.images.length} images).`,
+		);
+	}
+	if (
+		offset > item.text.length ||
+		(offset === item.text.length && imageOffset === item.images.length)
+	) {
+		throw new Error(
+			`Offset ${offset} is past the end of history entry "${id}" (${item.text.length} chars).`,
+		);
+	}
+	return { offset, imageOffset };
+}
+function shownEntryId(item: WindowedEntry, id: string | undefined, source: string): string {
+	return source === "" ? (id ?? "") : `${item.entry.id ?? "unknown"}@${shortKey(source)}`;
+}
+export function historyReadPage(
+	item: WindowedEntry,
+	params: HistoryParams,
+	access: { readonly host: PolicyContext; readonly pages: PageAccess },
+	source = "",
+): TextResult {
+	const shownId = shownEntryId(item, params.id, source);
+	const { offset, imageOffset } = readOffsets(item, params);
+	const firstImage = imageOffset < item.images.length ? 1 : 0;
+	const chars = access.pages.size(access.host, {
+		offset,
+		imageCount: firstImage,
+		imageOffset: item.images.length > 0 ? imageOffset : undefined,
+	});
+	const imageEnd = Math.min(
+		item.images.length,
+		imageOffset + firstImage + Math.floor((chars - MIN_PAGE_CHARS) / ESTIMATED_IMAGE_CHARS),
+	);
+	const images = item.images.slice(imageOffset, imageEnd);
+	const end = Math.min(
+		item.text.length,
+		offset + chars - (images.length - firstImage) * ESTIMATED_IMAGE_CHARS,
+	);
+	const more =
+		end < item.text.length || imageEnd < item.images.length
+			? `\nMore remains; call history read with id "${shownId}" and offset ${end}${item.images.length > 0 ? ` and imageOffset ${imageEnd}` : ""}.`
+			: "";
+	const header = `${item.entry.timestamp ?? ""}${sourceTag(source)} [window ${item.windowId}] [${shownId}] [chars ${offset}-${end} of ${item.text.length}] `;
+	return access.pages.result(`${header}${item.text.slice(offset, end)}${more}`, images, {
+		kind: "history-read",
+		headerLength: header.length,
+		offset,
+		end,
+		total: item.text.length,
+		imageOffset,
+		imageEnd,
+		imageTotal: item.images.length,
+	});
+}