pi-posthorse 0.10.17 → 0.10.19

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pi-posthorse",
-	"version": "0.10.17",
+	"version": "0.10.19",
 	"type": "module",
 	"description": "Posthorse: fresh context, same journey. No-summary context rollover for Pi, with sparse budget reminders, handoffs, notes, and history.",
 	"keywords": [

package/src/note-provenance.ts

@@ -7,6 +7,7 @@ import { parseSessionLine } from "./session-files.ts";
 import { sessionNoteTouches } from "./note-evidence.ts";
 import { mapBatch, readManifest, snapshot, type SnapshotNote } from "./note-snapshots.ts";
 import {
+	addressIn,
 	addressOf,
 	bindingsOf,
 	canonicalPath,
@@ -130,7 +131,7 @@ export async function resolveReference(
 		}
 		visited.add(directory);
 		if (existsSync(notePath(current, path))) {
-			return { ...current, path };
+			return addressIn(current, path);
 		}
 		// A manifest includes later adoptions; the journal includes later writes and deletions.
 		const binding: NoteBinding | undefined =
@@ -204,7 +205,7 @@ function seedCandidates(
 			) {
 				selected.set(seed.path, {
 					...seed,
-					source: { root: binding.root, scope: binding.scope, path: seed.path },
+					source: addressIn(binding, seed.path),
 					cleared: false,
 				});
 			}

package/src/note-scopes.ts

@@ -83,6 +83,13 @@ export function forkScope(owner: NoteOwner): string {
 export function scopeDirectory(scope: NoteScope): string {
 	return join(scope.root, scope.scope);
 }
+/**
+ * An address is stored in the journal, and a scope value is often a whole binding. Build an
+ * address here, never by spreading a scope, so it holds the two location fields and nothing else.
+ */
+export function addressIn(scope: NoteScope, path: string): NoteAddress {
+	return { root: scope.root, scope: scope.scope, path };
+}
 export function sameScope(left: NoteScope, right: NoteScope): boolean {
 	return left.root === right.root && left.scope === right.scope;
 }
@@ -278,7 +285,7 @@ export function addressOf(path: string, scopes: readonly NoteScope[]): NoteAddre
 				? undefined
 				: candidates.find((item) => item.directory === match.directory);
 		if (match !== undefined && candidate !== undefined) {
-			return { ...candidate.scope, path: match.path };
+			return addressIn(candidate.scope, match.path);
 		}
 		if (
 			scopes.some(

package/src/note-snapshots.ts

@@ -18,6 +18,7 @@ import { withFileMutationQueue } from "@earendil-works/pi-coding-agent";
 import { isRecord } from "./contracts.ts";
 import { publishFile } from "./files.ts";
 import {
+	addressIn,
 	notePath,
 	parseBinding,
 	sameOwner,
@@ -127,7 +128,19 @@ async function copyNote(note: SnapshotNote, destination: string): Promise<CopyRe
 		}
 		await utimes(destination, before.atimeMs / 1000, before.mtimeMs / 1000);
 		const [digest, destInfo] = await Promise.all([fileDigest(destination), stat(destination)]);
-		return { kind: "verified", seed: { ...note, digest, identity: fileIdentity(destInfo) } };
+		return {
+			kind: "verified",
+			// Journals written by 0.10.4 through 0.10.17 hold whole bindings in seed addresses.
+			// A new seed keeps each location only, so that data is not copied into this fork.
+			seed: {
+				path: note.path,
+				source: addressIn(note.source, note.source.path),
+				lineage: addressIn(note.lineage, note.lineage.path),
+				digest,
+				identity: fileIdentity(destInfo),
+				anchors: note.anchors,
+			},
+		};
 	} catch (error) {
 		return {
 			kind: "unverified",

package/src/note-store.ts

@@ -13,6 +13,7 @@ import {
 import {
 	NOTES_BINDING,
 	NOTES_TOUCH,
+	addressIn,
 	bindingsOf,
 	canonicalPath,
 	inside,
@@ -154,7 +155,7 @@ export class SessionNotes {
 		if (options.scope !== undefined) {
 			const scope = await this.selectScope(options.scope);
 			return {
-				address: { ...scope, path: relative },
+				address: addressIn(scope, relative),
 				path: notePath(scope, relative),
 				writable: sameScope(scope, this.binding),
 			};
@@ -162,11 +163,7 @@ export class SessionNotes {
 		const reference = this.owns(relative)
 			? undefined
 			: await this.#reference(relative, options.signal);
-		const address = reference ?? {
-			root: this.binding.root,
-			scope: this.binding.scope,
-			path: relative,
-		};
+		const address = reference ?? addressIn(this.binding, relative);
 		return { address, path: notePath(address, relative), writable: reference === undefined };
 	}
 	async adopt(path: string, call: NoteCall): Promise<Adoption> {

package/src/recovery-inputs.ts

@@ -0,0 +1,255 @@
+import { AUTO_HANDOFF_PREFIX, isRecord, type EntryLike, type ProjectedEntry } from "./contracts.ts";
+import { contextEdits, contextStart, windowHandoff } from "./history-entries.ts";
+import { toolCalls } from "./message-text.ts";
+import {
+	formatRecord,
+	MAX_RECOVERY_RECORD_CHARS,
+	recoveryRecord,
+	type RecoveryRecord,
+} from "./recovery-records.ts";
+
+/** One input's room as a share of the whole record: 4,000 of the 20,000 characters a record may have. */
+const RECORD_SHARE = 5;
+
+function projectedCurrent(
+	entries: readonly EntryLike[],
+	projected: readonly ProjectedEntry[],
+	current: readonly EntryLike[] = entries.slice(contextStart(entries).start),
+	retained: ReadonlySet<string> = new Set(),
+): EntryLike[] {
+	const edits = contextEdits(entries);
+	const omittedIds = new Set(
+		projected
+			.filter(({ messages }) => messages.length === 0)
+			.map(({ sourceEntry }) => sourceEntry.id),
+	);
+	return current.flatMap((entry) => {
+		const id = entry.id ?? "";
+		if (retained.has(id) || omittedIds.has(id)) {
+			return [];
+		}
+		const edit = edits.get(id);
+		if (edit?.replacement === null) {
+			return [];
+		}
+		if (edit?.replacement === undefined) {
+			return [entry];
+		}
+		if (entry.type === "message") {
+			return [
+				{
+					...entry,
+					id: edit.id,
+					message: { ...entry.message, content: edit.replacement.content },
+				},
+			];
+		}
+		if (entry.type === "custom_message") {
+			return [{ ...entry, id: edit.id, content: edit.replacement.content }];
+		}
+		return [entry];
+	});
+}
+function ownerRecords(
+	entries: readonly EntryLike[],
+	projected: readonly ProjectedEntry[],
+	ownerQuestionRegistered: boolean,
+): RecoveryRecord[] {
+	const calls = new Map(
+		projected.flatMap(({ messages }) =>
+			messages.flatMap((message) =>
+				message.role === "assistant"
+					? toolCalls(message.content).map((call) => [call.id, call] as const)
+					: [],
+			),
+		),
+	);
+	return entries.flatMap((entry) => {
+		const call = calls.get(entry.message?.toolCallId);
+		const record = recoveryRecord(
+			entry,
+			ownerQuestionRegistered &&
+				call?.name === "ask_question" &&
+				call.namespace === undefined,
+		);
+		return record === undefined ? [] : [record];
+	});
+}
+function anchors(records: readonly RecoveryRecord[]): {
+	fixed: RecoveryRecord[];
+	others: RecoveryRecord[];
+} {
+	const firstOwner = records.find((record) => record.label === "owner input") ?? records[0];
+	const latestOwner = [...records].reverse().find((record) => record.kind === "owner");
+	const latestOverall = records.at(-1);
+	const selected = new Set([firstOwner, latestOwner, latestOverall]);
+	return {
+		fixed: records.filter((record) => selected.has(record)),
+		others: records.filter((record) => !selected.has(record)),
+	};
+}
+function automaticBoundary(entry: EntryLike | undefined): boolean {
+	if (
+		entry === undefined ||
+		windowHandoff(entry)?.trim().startsWith(AUTO_HANDOFF_PREFIX) !== true
+	) {
+		return false;
+	}
+	return (
+		entry.type === "context_window" ||
+		(isRecord(entry.details) &&
+			entry.details.posthorse === 1 &&
+			(entry.details.reason === "threshold" || entry.details.reason === "overflow"))
+	);
+}
+export function automaticChainStart(entries: readonly EntryLike[]): number {
+	let prefix = entries;
+	let window = contextStart(prefix);
+	while (automaticBoundary(window.boundary) && window.boundary !== undefined) {
+		prefix = prefix.slice(0, prefix.indexOf(window.boundary));
+		window = contextStart(prefix);
+	}
+	return window.start;
+}
+function windowOwners(
+	entries: readonly EntryLike[],
+	projected: readonly ProjectedEntry[],
+	window: readonly EntryLike[],
+	ownerQuestionRegistered: boolean,
+): RecoveryRecord[] {
+	const restored = projectedCurrent(entries, projected, window);
+	const historical = restored.map((sourceEntry) => ({
+		sourceEntry,
+		messages: sourceEntry.message === undefined ? [] : [sourceEntry.message],
+	}));
+	return ownerRecords(restored, historical, ownerQuestionRegistered).filter(
+		(record) => record.kind === "owner",
+	);
+}
+/** The windows before the current one, newest first, as far back as automatic boundaries reach. */
+function* earlierWindows(entries: readonly EntryLike[]): Generator<{
+	readonly closedBy: EntryLike;
+	readonly entries: readonly EntryLike[];
+	readonly openedBy: EntryLike | undefined;
+}> {
+	let prefix = entries;
+	let boundary = contextStart(prefix).boundary;
+	while (boundary !== undefined && automaticBoundary(boundary)) {
+		prefix = prefix.slice(0, prefix.indexOf(boundary));
+		const window = contextStart(prefix);
+		yield {
+			closedBy: boundary,
+			entries: prefix.slice(window.start),
+			openedBy: window.boundary,
+		};
+		boundary = window.boundary;
+	}
+}
+/** A window's owner inputs that still fit the room, taken newest first and returned oldest first. */
+function newestThatFit(
+	owners: readonly RecoveryRecord[],
+	room: number,
+	seen: Set<string>,
+): { readonly records: readonly RecoveryRecord[]; readonly used: number } {
+	const records: RecoveryRecord[] = [];
+	let used = 0;
+	for (const owner of owners.toReversed()) {
+		if (used >= room) {
+			break;
+		}
+		if (seen.has(owner.id)) {
+			continue;
+		}
+		const record = {
+			...owner,
+			label: `${owner.label}; carried from an earlier window, possibly already handled; not current progress`,
+		};
+		seen.add(owner.id);
+		records.unshift(record);
+		used += formatRecord(record, Number.MAX_SAFE_INTEGER).length;
+	}
+	return { records, used };
+}
+/**
+ * Earlier windows give a record the owner's most recent inputs, newest first, until they fill the
+ * room of one input. Owner text the current window already holds counts against that room, so a
+ * short follow-up keeps the task before it and a long new request stands alone.
+ */
+function recoveryAncestry(
+	entries: readonly EntryLike[],
+	projected: readonly ProjectedEntry[],
+	ownerQuestionRegistered: boolean,
+	options: { readonly current: readonly RecoveryRecord[]; readonly maxChars: number },
+): {
+	readonly checkpoint?: EntryLike | undefined;
+	readonly carried: readonly RecoveryRecord[];
+	readonly skipped: readonly string[];
+} {
+	const held = options.current
+		.filter((record) => record.kind === "owner")
+		.reduce((chars, record) => chars + record.text.length, 0);
+	let room =
+		Math.min(MAX_RECOVERY_RECORD_CHARS, Math.floor(options.maxChars / RECORD_SHARE)) - held;
+	const seen = new Set(options.current.map((record) => record.id));
+	const carried: RecoveryRecord[] = [];
+	const skipped: EntryLike[] = [];
+	let ownerWindowFound = false;
+	let reached = contextStart(entries).boundary;
+	for (const window of earlierWindows(entries)) {
+		if (ownerWindowFound && room <= 0) {
+			break;
+		}
+		if (!ownerWindowFound) {
+			skipped.push(window.closedBy);
+		}
+		const owners = windowOwners(entries, projected, window.entries, ownerQuestionRegistered);
+		ownerWindowFound ||= owners.length > 0;
+		const fit = newestThatFit(owners, room, seen);
+		carried.unshift(...fit.records);
+		room -= fit.used;
+		reached = window.openedBy;
+	}
+	return {
+		checkpoint: automaticBoundary(reached) ? undefined : reached,
+		carried,
+		skipped: skipped.map((boundary) => boundary.id?.slice(0, 120) ?? "unknown"),
+	};
+}
+/** The inputs a record quotes in full (oldest first), the ones it only lists, and where its walk back ended. */
+export function recoveryInputs(
+	entries: readonly EntryLike[],
+	projected: readonly ProjectedEntry[],
+	options: {
+		readonly ownerQuestionRegistered: boolean;
+		readonly retained: ReadonlySet<string>;
+		readonly maxChars: number;
+	},
+): {
+	readonly fixed: readonly RecoveryRecord[];
+	readonly others: readonly RecoveryRecord[];
+	readonly checkpoint?: EntryLike | undefined;
+	readonly skipped: readonly string[];
+} {
+	const { ownerQuestionRegistered, retained, maxChars } = options;
+	const { carried, checkpoint, skipped } = recoveryAncestry(
+		entries,
+		projected,
+		ownerQuestionRegistered,
+		{
+			current: ownerRecords(
+				projectedCurrent(entries, projected),
+				projected,
+				ownerQuestionRegistered,
+			),
+			maxChars,
+		},
+	);
+	const { fixed, others } = anchors(
+		ownerRecords(
+			projectedCurrent(entries, projected, undefined, retained),
+			projected,
+			ownerQuestionRegistered,
+		),
+	);
+	return { fixed: [...carried, ...fixed], others, checkpoint, skipped };
+}

package/src/recovery-records.ts

@@ -1,6 +1,7 @@
 import { isReminderType, type EntryLike } from "./contracts.ts";
 import { boundedBlock, imageSummary, imagesOf, textOf } from "./message-text.ts";
 
+export const MAX_RECOVERY_RECORD_CHARS = 4_000;
 export type RecoveryRecord = {
 	readonly id: string;
 	readonly timestamp: string;

package/src/recovery.ts

@@ -1,10 +1,11 @@
-import { AUTO_HANDOFF_PREFIX, isRecord, type EntryLike, type ProjectedEntry } from "./contracts.ts";
+import { AUTO_HANDOFF_PREFIX, type EntryLike, type ProjectedEntry } from "./contracts.ts";
 import { contextEdits, contextStart, windowHandoff } from "./history-entries.ts";
-import { boundedBlock, excerpt, toolCalls } from "./message-text.ts";
+import { boundedBlock, excerpt } from "./message-text.ts";
+import { automaticChainStart, recoveryInputs } from "./recovery-inputs.ts";
 import {
 	formatRecord,
 	indexLine,
-	recoveryRecord,
+	MAX_RECOVERY_RECORD_CHARS,
 	type RecoveryRecord,
 } from "./recovery-records.ts";
 import type { NotesRecovery } from "./note-evidence.ts";
@@ -16,7 +17,6 @@ import {
 	type RecoveryEntry,
 } from "./recovery-results.ts";
 
-const MAX_RECOVERY_RECORD_CHARS = 4_000;
 const MAX_RECOVERY_RESULT_CHARS = 1_500;
 const HANDOFF_OVERHEAD_RESERVE = 1_000;
 const MAX_NOTES_SECTION_CHARS = 1_200;
@@ -69,142 +69,6 @@ function priorCheckpoint(entry: EntryLike | undefined, limit: number): string |
 	const header = `[older checkpoint; possibly stale | context-window entry ${id}]`;
 	return boundedBlock(header, handoff, limit);
 }
-function projectedCurrent(
-	entries: readonly EntryLike[],
-	projected: readonly ProjectedEntry[],
-	current: readonly EntryLike[] = entries.slice(contextStart(entries).start),
-	retained: ReadonlySet<string> = new Set(),
-): EntryLike[] {
-	const edits = contextEdits(entries);
-	const omittedIds = new Set(
-		projected
-			.filter(({ messages }) => messages.length === 0)
-			.map(({ sourceEntry }) => sourceEntry.id),
-	);
-	return current.flatMap((entry) => {
-		const id = entry.id ?? "";
-		if (retained.has(id) || omittedIds.has(id)) {
-			return [];
-		}
-		const edit = edits.get(id);
-		if (edit?.replacement === null) {
-			return [];
-		}
-		if (edit?.replacement === undefined) {
-			return [entry];
-		}
-		if (entry.type === "message") {
-			return [
-				{
-					...entry,
-					id: edit.id,
-					message: { ...entry.message, content: edit.replacement.content },
-				},
-			];
-		}
-		if (entry.type === "custom_message") {
-			return [{ ...entry, id: edit.id, content: edit.replacement.content }];
-		}
-		return [entry];
-	});
-}
-function ownerRecords(
-	entries: readonly EntryLike[],
-	projected: readonly ProjectedEntry[],
-	ownerQuestionRegistered: boolean,
-): RecoveryRecord[] {
-	const calls = new Map(
-		projected.flatMap(({ messages }) =>
-			messages.flatMap((message) =>
-				message.role === "assistant"
-					? toolCalls(message.content).map((call) => [call.id, call] as const)
-					: [],
-			),
-		),
-	);
-	return entries.flatMap((entry) => {
-		const call = calls.get(entry.message?.toolCallId);
-		const record = recoveryRecord(
-			entry,
-			ownerQuestionRegistered &&
-				call?.name === "ask_question" &&
-				call.namespace === undefined,
-		);
-		return record === undefined ? [] : [record];
-	});
-}
-function anchors(records: readonly RecoveryRecord[]): {
-	fixed: RecoveryRecord[];
-	others: RecoveryRecord[];
-} {
-	const firstOwner = records.find((record) => record.label === "owner input") ?? records[0];
-	const latestOwner = [...records].reverse().find((record) => record.kind === "owner");
-	const latestOverall = records.at(-1);
-	const selected = new Set([firstOwner, latestOwner, latestOverall]);
-	return {
-		fixed: records.filter((record) => selected.has(record)),
-		others: records.filter((record) => !selected.has(record)),
-	};
-}
-function automaticBoundary(entry: EntryLike | undefined): boolean {
-	if (
-		entry === undefined ||
-		windowHandoff(entry)?.trim().startsWith(AUTO_HANDOFF_PREFIX) !== true
-	) {
-		return false;
-	}
-	return (
-		entry.type === "context_window" ||
-		(isRecord(entry.details) &&
-			entry.details.posthorse === 1 &&
-			(entry.details.reason === "threshold" || entry.details.reason === "overflow"))
-	);
-}
-function automaticChainStart(entries: readonly EntryLike[]): number {
-	let prefix = entries;
-	let window = contextStart(prefix);
-	while (automaticBoundary(window.boundary) && window.boundary !== undefined) {
-		prefix = prefix.slice(0, prefix.indexOf(window.boundary));
-		window = contextStart(prefix);
-	}
-	return window.start;
-}
-function recoveryAncestry(
-	entries: readonly EntryLike[],
-	projected: readonly ProjectedEntry[],
-	ownerQuestionRegistered: boolean,
-	retained: ReadonlySet<string>,
-): {
-	readonly checkpoint?: EntryLike | undefined;
-	readonly carried: readonly RecoveryRecord[];
-	readonly skipped: readonly string[];
-} {
-	let prefix = entries;
-	let boundary = contextStart(prefix).boundary;
-	const skipped: string[] = [];
-	while (automaticBoundary(boundary) && boundary !== undefined) {
-		skipped.push(boundary.id?.slice(0, 120) ?? "unknown");
-		prefix = prefix.slice(0, prefix.indexOf(boundary));
-		const window = contextStart(prefix);
-		const restored = projectedCurrent(entries, projected, prefix.slice(window.start));
-		const historical = restored.map((sourceEntry) => ({
-			sourceEntry,
-			messages: sourceEntry.message === undefined ? [] : [sourceEntry.message],
-		}));
-		const owners = ownerRecords(restored, historical, ownerQuestionRegistered).filter(
-			(record) => record.kind === "owner",
-		);
-		if (owners.length > 0) {
-			return {
-				checkpoint: automaticBoundary(window.boundary) ? undefined : window.boundary,
-				carried: anchors(owners.filter((record) => !retained.has(record.id))).fixed,
-				skipped,
-			};
-		}
-		boundary = window.boundary;
-	}
-	return { checkpoint: boundary, carried: [], skipped };
-}
 function skippedRecords(ids: readonly string[], limit: number): string | undefined {
 	const remaining = ids.values();
 	const first = remaining.next();
@@ -346,34 +210,20 @@ function renderIndex(records: readonly RecoveryRecord[], available: number): str
 		...listed,
 	].join("\n");
 }
-function selectedInputs(
-	entries: readonly EntryLike[],
-	projected: readonly ProjectedEntry[],
-	options: {
-		readonly ownerQuestionRegistered: boolean;
-		readonly retained: ReadonlySet<string>;
-		readonly carried: readonly RecoveryRecord[];
-	},
-): { readonly fixed: readonly RecoveryRecord[]; readonly others: readonly RecoveryRecord[] } {
-	const { ownerQuestionRegistered, retained, carried } = options;
-	const current = projectedCurrent(entries, projected);
-	const allRecords = ownerRecords(current, projected, ownerQuestionRegistered);
-	const records = ownerRecords(
-		projectedCurrent(entries, projected, undefined, retained),
-		projected,
-		ownerQuestionRegistered,
-	);
-	const { fixed, others } = anchors(records);
-	if (!allRecords.some((record) => record.kind === "owner")) {
-		fixed.unshift(
-			...carried.map((record) =>
-				Object.assign({}, record, {
-					label: `${record.label}; carried from an earlier window, possibly already handled; not current progress`,
-				}),
-			),
-		);
+/** The per-block limit at which the blocks fit the budget together; short blocks leave room to long ones. */
+function sharedLimit(lengths: readonly number[], budget: number): number {
+	let remaining = budget;
+	const ascending = lengths
+		.map((length) => Math.min(length, MAX_RECOVERY_RECORD_CHARS))
+		.toSorted((a, b) => a - b);
+	for (const [index, length] of ascending.entries()) {
+		const share = Math.floor(remaining / (ascending.length - index));
+		if (length > share) {
+			return share;
+		}
+		remaining -= length;
 	}
-	return { fixed, others };
+	return MAX_RECOVERY_RECORD_CHARS;
 }
 function rolloverPreambleWithNotes(
 	entries: readonly EntryLike[],
@@ -417,13 +267,12 @@ export function buildAutoHandoff(
 	},
 ): string {
 	const { ownerQuestionRegistered, retainedEntryIds: retained = new Set<string>() } = options;
-	const history = recoveryAncestry(entries, projected, ownerQuestionRegistered, retained);
-	const priorWindow = history.checkpoint;
-	const { fixed, others } = selectedInputs(entries, projected, {
+	const { fixed, others, ...history } = recoveryInputs(entries, projected, {
 		ownerQuestionRegistered,
 		retained,
-		carried: history.carried,
+		maxChars,
 	});
+	const priorWindow = history.checkpoint;
 	const emptyInputHeader =
 		retained.size > 0
 			? "No discarded owner or visible coordination inputs were selected for this recovery record."
@@ -453,19 +302,18 @@ export function buildAutoHandoff(
 		header,
 		Math.max(0, maxChars - joined(minimum).length - HANDOFF_OVERHEAD_RESERVE),
 	);
-	const handoff = windowHandoff(priorWindow)?.trim();
-	const count =
-		fixed.length +
-		(handoff === undefined || handoff === "" ? 0 : 1) +
-		(history.skipped.length > 0 ? 1 : 0);
 	const fixedBudget = Math.max(
 		0,
 		maxChars - joined([preamble, inputHeader, plan.bare]).length - HANDOFF_OVERHEAD_RESERVE,
 	);
-	const fixedLimit =
-		count > 0
-			? Math.min(MAX_RECOVERY_RECORD_CHARS, Math.floor(fixedBudget / count))
-			: MAX_RECOVERY_RECORD_CHARS;
+	const fixedLimit = sharedLimit(
+		[
+			...fixed.map((record) => formatRecord(record, Number.MAX_SAFE_INTEGER)),
+			priorCheckpoint(priorWindow, Number.MAX_SAFE_INTEGER),
+			skippedRecords(history.skipped, Number.MAX_SAFE_INTEGER),
+		].flatMap((block) => (block === undefined ? [] : [block.length])),
+		fixedBudget,
+	);
 	const prior = joined([
 		priorCheckpoint(priorWindow, fixedLimit),
 		skippedRecords(history.skipped, fixedLimit),