pi-posthorse 0.9.3 → 0.10.1

package/src/rollover.ts

@@ -14,7 +14,6 @@ import {
 import {
 	isRecord,
 	isReminderType,
-	MIN_PAGE_CHARS,
 	REMINDER_TYPE,
 	type CompactionPolicy,
 	type PolicyContext,
@@ -23,7 +22,7 @@ import {
 } from "./contracts.ts";
 import { registerContextTools } from "./context-tools.ts";
 import { safeJsonStringify } from "./message-text.ts";
-import { buildAutoHandoff } from "./recovery.ts";
+import { automaticRecovery } from "./retention.ts";
 import {
 	createReminderLookup,
 	hasReminder,
@@ -42,7 +41,7 @@ function registerGuidance(pi: ExtensionAPI, policy: SnapshotReader): void {
 		promptOptions = event.systemPromptOptions;
 	});
 	pi.on("agent_settled", () => {
-		// Older hosts skip the start hook for custom runs; do not reuse a settled run's options.
+		// Prompt options belong to one run, never to later idle context evaluation.
 		promptOptions = undefined;
 	});
 	pi.on("context_with_system", (event, ctx) => {
@@ -175,7 +174,7 @@ function sendReminder(
 	pi.sendMessage(
 		{
 			customType: REMINDER_TYPE,
-			content: `[posthorse] Checkpoint now: ${(budget.rolloverAt - tokens).toLocaleString("en-US")} tokens remain before Pi's automatic rollover line. Stop normal work, ${CHECKPOINT_STEPS}.`,
+			content: `[posthorse] Notes checkpoint: ${(budget.rolloverAt - tokens).toLocaleString("en-US")} tokens remain before Pi's automatic rollover line. Please ${CHECKPOINT_STEPS}.`,
 			display: true,
 			details: fingerprint,
 		},
@@ -232,12 +231,20 @@ function registerReminderFilter(
 function automaticDraft(
 	pi: ExtensionAPI,
 	ctx: ExtensionContext,
-	recovery: { readonly handoff: string; readonly tokensBefore: number; readonly reason: string },
+	recovery: {
+		readonly handoff: string;
+		readonly tokensBefore: number;
+		readonly reason: string;
+		readonly firstKeptEntryId?: string;
+	},
 ) {
-	// Public compaction requires an entry id. An invisible sentinel keeps no prior conversation.
-	pi.appendEntry("posthorse-boundary", {});
-	const firstKeptEntryId = ctx.sessionManager.getLeafId();
-	if (firstKeptEntryId === null || firstKeptEntryId === "") {
+	let firstKeptEntryId = recovery.firstKeptEntryId;
+	if (firstKeptEntryId === undefined) {
+		// Public compaction requires an entry id even when no prior conversation safely fits.
+		pi.appendEntry("posthorse-boundary", {});
+		firstKeptEntryId = ctx.sessionManager.getLeafId() ?? undefined;
+	}
+	if (firstKeptEntryId === undefined || firstKeptEntryId === "") {
 		throw new Error("Pi did not persist the context boundary.");
 	}
 	return {
@@ -316,17 +323,16 @@ function thresholdFallback(
 		return undefined;
 	}
 	const projected = event.context.contextEntries;
-	const limit = freshPayloadChars(host, toolTokens(), event.context.pendingMessages);
 	const ownerQuestionRegistered = pi
 		.getAllTools()
 		.some((tool) => tool.name === "ask_question" && tool.namespace === undefined);
-	const summary = buildAutoHandoff(
+	const recovery = automaticRecovery(
 		ctx.sessionManager.getBranch(),
 		projected,
-		limit,
+		{ host, toolTokens: toolTokens(), pendingMessages: event.context.pendingMessages },
 		ownerQuestionRegistered,
 	);
-	if (limit < MIN_PAGE_CHARS || summary.length > limit) {
+	if (recovery === undefined) {
 		return undefined;
 	}
 	// ponytail: turn_end drafts are eager. Later projection editors must replace/remove this draft;
@@ -336,8 +342,8 @@ function thresholdFallback(
 			...event.entries,
 			{
 				type: "compaction",
-				summary,
-				firstKeptEntryId: null,
+				summary: recovery.handoff,
+				firstKeptEntryId: recovery.firstKeptEntryId ?? null,
 				details: { posthorse: 1, reason: "threshold" },
 			},
 		],
@@ -362,17 +368,20 @@ function registerAutomaticRollover(
 			if (budgetFor(host)?.supported !== true) {
 				return;
 			}
-			const limit = freshPayloadChars(host, activeToolTokens());
 			const ownerQuestionRegistered = pi
 				.getAllTools()
 				.some((tool) => tool.name === "ask_question" && tool.namespace === undefined);
-			const handoff = buildAutoHandoff(
+			const recovery = automaticRecovery(
 				event.branchEntries,
 				ctx.sessionManager.buildSessionProjection().entries,
-				limit,
+				{
+					host,
+					toolTokens: activeToolTokens(),
+					firstKeptEntryId: event.preparation.firstKeptEntryId,
+				},
 				ownerQuestionRegistered,
 			);
-			if (limit < MIN_PAGE_CHARS || handoff.length > limit) {
+			if (recovery === undefined) {
 				throw new Error(
 					"Too little fresh context capacity for an automatic recovery record.",
 				);
@@ -381,7 +390,7 @@ function registerAutomaticRollover(
 				return { cancel: true };
 			}
 			return automaticDraft(pi, ctx, {
-				handoff,
+				...recovery,
 				tokensBefore: event.preparation.tokensBefore,
 				reason: event.reason,
 			});

package/src/session-files.ts

@@ -12,6 +12,7 @@ import {
 	type WindowedEntry,
 } from "./contracts.ts";
 import { lfLines } from "./files.ts";
+import { sessionLines, type LineMatcher, type SessionLine } from "./session-lines.ts";
 import { toWindowedEntry, windowProjection } from "./history-entries.ts";
 import { shortKey } from "./message-text.ts";
 
@@ -78,11 +79,11 @@ const ASCII_FOLDS: Readonly<Partial<Record<string, string>>> = {
 	k: "\u00e2\u0084\u00aa",
 	s: "\u00c5\u00bf",
 };
-const DOTTED_I = Buffer.from("\u0130");
-/** The read capabilities line filters need from a raw journal line. */
-export type RawLine = Readonly<Pick<Buffer, "includes" | "toString">>;
-/** Whether a raw line contains one of prefilterNeedles' needles, ignoring case like its text would. */
-export function needleLines(needles: readonly string[]): (bytes: RawLine) => boolean {
+const DOTTED_I = Buffer.from("\u0130").toString("latin1");
+/** Raw-line matcher for prefilterNeedles, ignoring case like the rendered text would. */
+export function needleLines(needles: readonly string[]): LineMatcher {
+	const overlap =
+		4 * needles.reduce((longest, needle) => Math.max(longest, needle.length), 0) + 4;
 	const others = needles.filter((needle) => needle !== "\u0130");
 	const dotted = others.length < needles.length;
 	if (others.every((needle) => /^[\0-\x7f]*$/u.test(needle))) {
@@ -97,11 +98,14 @@ export function needleLines(needles: readonly string[]): (bytes: RawLine) => boo
 				.join(""),
 		);
 		const pattern = new RegExp(latin1.join("|"), "i");
-		return (bytes) =>
-			(dotted && bytes.includes(DOTTED_I)) || pattern.test(bytes.toString("latin1"));
+		return {
+			test: (text) => (dotted && text.includes(DOTTED_I)) || pattern.test(text),
+			encoding: "latin1",
+			overlap,
+		};
 	}
 	const pattern = new RegExp(needles.map((needle) => RegExp.escape(needle)).join("|"), "iu");
-	return (bytes) => pattern.test(bytes.toString("utf8"));
+	return { test: (text) => pattern.test(text), encoding: "utf8", overlap };
 }
 /** File lists that each fit one ripgrep command line. */
 function commandGroups(paths: readonly string[]): string[][] {
@@ -210,8 +214,8 @@ function parsedEntry(line: string): EntryLike | undefined {
 	} // Pi skips malformed journal lines too.
 }
 /** Window metadata of a Pi-ordered line that cannot open a window, without parsing it. */
-function entryHead(bytes: RawLine): EntryLike | undefined {
-	const head = ENTRY_HEAD.exec(bytes.toString("utf8", 0, 512));
+function entryHead(prefix: string): EntryLike | undefined {
+	const head = ENTRY_HEAD.exec(prefix);
 	if (head === null || head[1] === "compaction" || head[1] === "context_window") {
 		return undefined;
 	}
@@ -222,31 +226,40 @@ function entryHead(bytes: RawLine): EntryLike | undefined {
 	};
 }
 /** Whether a caller may need a line; other lines only advance window ancestry. */
-type LineFilter = (bytes: RawLine, entry: EntryLike) => boolean;
+type LineFilter = (entry: EntryLike) => boolean;
+async function readEntry(
+	line: SessionLine,
+	wanted: LineFilter,
+): Promise<{ readonly entry: EntryLike; readonly selected: boolean } | undefined> {
+	const head = entryHead(line.head);
+	if (head !== undefined && (!line.matches || !wanted(head))) {
+		return { entry: head, selected: false };
+	}
+	const entry = parsedEntry(await line.text());
+	return entry === undefined
+		? undefined
+		: {
+				entry,
+				selected: head !== undefined || (line.matches && wanted(entry)),
+			};
+}
 export async function* sessionWindowEntries(
 	file: string,
 	signal?: AbortSignal,
 	wanted: LineFilter = () => true,
+	matcher?: LineMatcher,
 ): AsyncGenerator<WindowedEntry> {
 	if (!existsSync(file)) {
 		return;
 	}
 	const project = windowProjection();
-	for await (const bytes of lfLines(file, signal, 1024 * 1024)) {
-		const head = entryHead(bytes);
-		if (head !== undefined && !wanted(bytes, head)) {
-			project(head);
+	for await (const line of sessionLines(file, signal, matcher)) {
+		const candidate = await readEntry(line, wanted);
+		if (candidate === undefined) {
 			continue;
 		}
-		const entry = parsedEntry(bytes.toString("utf8"));
-		if (entry === undefined) {
-			continue;
-		}
-		const windowId = project(entry);
-		const item =
-			head !== undefined || wanted(bytes, entry)
-				? toWindowedEntry(entry, windowId)
-				: undefined;
+		const windowId = project(candidate.entry);
+		const item = candidate.selected ? toWindowedEntry(candidate.entry, windowId) : undefined;
 		if (item !== undefined) {
 			yield item;
 		}

package/src/session-lines.ts

@@ -0,0 +1,108 @@
+import { open, type FileHandle } from "node:fs/promises";
+import { StringDecoder } from "node:string_decoder";
+import { lfChunks } from "./files.ts";
+
+export type LineMatcher = {
+	readonly test: (text: string) => boolean;
+	readonly encoding: "latin1" | "utf8";
+	/** Enough characters to include every case-fold alternative of any needle. */
+	readonly overlap: number;
+};
+export type SessionLine = {
+	readonly head: string;
+	readonly matches: boolean;
+	readonly text: () => Promise<string>;
+};
+type LineRange = {
+	readonly start: number;
+	readonly end: number;
+	readonly head: string;
+	readonly matches: boolean;
+	readonly bytes?: Readonly<Pick<Buffer, "toString">>;
+};
+async function* lineRanges(
+	file: string,
+	handle: Readonly<Pick<FileHandle, "createReadStream">>,
+	signal: AbortSignal | undefined,
+	matcher: LineMatcher | undefined,
+): AsyncGenerator<LineRange> {
+	let start = 0;
+	let position = 0;
+	let head = Buffer.alloc(0);
+	let tail = "";
+	let decoder = new StringDecoder(matcher?.encoding);
+	let matches = matcher === undefined;
+	for await (const { bytes, end } of lfChunks(file, signal, 1024 * 1024, handle)) {
+		if (head.length < 512) {
+			head = Buffer.concat([head, bytes.subarray(0, 512 - head.length)]);
+		}
+		if (!matches && matcher !== undefined) {
+			const window = tail + decoder.write(bytes) + (end ? decoder.end() : "");
+			matches = matcher.test(window);
+			tail = window.slice(-matcher.overlap);
+		}
+		position += bytes.length;
+		if (end) {
+			yield {
+				start,
+				end: position,
+				head: head.toString("utf8"),
+				matches,
+				bytes: position - start === bytes.length ? bytes : undefined,
+			};
+			position++;
+			start = position;
+			head = Buffer.alloc(0);
+			tail = "";
+			decoder = new StringDecoder(matcher?.encoding);
+			matches = matcher === undefined;
+		}
+	}
+}
+async function rangeText(
+	handle: Readonly<Pick<FileHandle, "read">>,
+	line: LineRange,
+	signal?: AbortSignal,
+): Promise<string> {
+	signal?.throwIfAborted();
+	if (line.bytes !== undefined) {
+		return line.bytes.toString("utf8");
+	}
+	const buffer = Buffer.alloc(line.end - line.start);
+	let offset = 0;
+	while (offset < buffer.length) {
+		signal?.throwIfAborted();
+		// Positioned reads may be short; each continues the same selected record.
+		// oxlint-disable-next-line no-await-in-loop
+		const { bytesRead } = await handle.read(
+			buffer,
+			offset,
+			buffer.length - offset,
+			line.start + offset,
+		);
+		if (bytesRead === 0) {
+			throw new Error("Session file changed while reading a history entry.");
+		}
+		offset += bytesRead;
+	}
+	return buffer.toString("utf8");
+}
+/** Irrelevant lines retain only their prefix, byte range and needle overlap, not their payload. */
+export async function* sessionLines(
+	file: string,
+	signal?: AbortSignal,
+	matcher?: LineMatcher,
+): AsyncGenerator<SessionLine> {
+	const handle = await open(file, "r");
+	try {
+		for await (const line of lineRanges(file, handle, signal, matcher)) {
+			yield {
+				head: line.head,
+				matches: line.matches,
+				text: () => rangeText(handle, line, signal),
+			};
+		}
+	} finally {
+		await handle.close();
+	}
+}