pi-posthorse 0.7.0 → 0.8.1

package/src/rollover.ts

@@ -0,0 +1,330 @@
+import type {
+	BoundaryResult,
+	ExtensionAPI,
+	ExtensionContext,
+	TurnEndEvent,
+} from "@earendil-works/pi-coding-agent";
+import {
+	budgetFor,
+	buildGuidance,
+	CHECKPOINT_STEPS,
+	freshPayloadChars,
+	type Budget,
+} from "./budget.ts";
+import {
+	isRecord,
+	isReminderType,
+	MIN_PAGE_CHARS,
+	REMINDER_TYPE,
+	type CompactionPolicy,
+	type PolicyContext,
+	type ReminderFingerprint,
+} from "./contracts.ts";
+import { registerContextTools } from "./context-tools.ts";
+import { safeJsonStringify } from "./message-text.ts";
+import { buildAutoHandoff } from "./recovery.ts";
+import {
+	createReminderLookup,
+	hasReminder,
+	reminderMatches,
+	type ReminderLookup,
+} from "./reminders.ts";
+
+export type SnapshotReader = (
+	ctx: ExtensionContext,
+	livePolicy?: CompactionPolicy,
+) => PolicyContext;
+const REMINDER_BUFFER_TOKENS = 32_000;
+function registerGuidance(pi: ExtensionAPI, policy: SnapshotReader): void {
+	let promptOptions: { readonly forceSystemPrompt?: string } | undefined;
+	pi.on("before_agent_start", (event) => {
+		promptOptions = event.systemPromptOptions;
+	});
+	pi.on("context_with_system", (event, ctx) => {
+		const guidance = buildGuidance(policy(ctx));
+		const system =
+			event.messages.findLast(
+				(message) =>
+					message.role === "system" && typeof message.sections?.posthorse === "string",
+			) ?? event.messages.find((message) => message.role === "system");
+		if (system === undefined || system.role !== "system") {
+			return;
+		}
+		if (promptOptions?.forceSystemPrompt !== undefined) {
+			// ponytail: Pi applies forced prompts after this hook; keep request-local guidance
+			// until the host projects forced text before context_with_system.
+			event.messages.splice(1, 0, {
+				role: "custom",
+				customType: "posthorse-guidance",
+				content: guidance,
+				display: false,
+				timestamp: 0,
+			});
+		} else {
+			system.sections = {
+				...system.sections,
+				posthorse: `<posthorse>\n${guidance}\n</posthorse>`,
+			};
+		}
+	});
+}
+function explicitReset(
+	event: TurnEndEvent,
+	ctx: ExtensionContext,
+	summary: string,
+	limit: () => number,
+): BoundaryResult | undefined {
+	if (event.outcome === "aborted" || ctx.signal?.aborted === true) {
+		return undefined;
+	}
+	if (summary.length > limit()) {
+		return {
+			entries: [
+				...event.entries,
+				{
+					type: "custom_message",
+					customType: "posthorse-reset-deferred",
+					display: true,
+					content:
+						"Posthorse reset deferred: queued messages leave too little room for the requested handoff. The current context is unchanged; process those messages, then request a shorter handoff.",
+				},
+			],
+		};
+	}
+	return {
+		entries: [
+			...event.entries,
+			{
+				type: "compaction",
+				summary,
+				firstKeptEntryId: null,
+				details: { posthorse: 1, reason: "explicit" },
+			},
+		],
+		continue: true,
+	};
+}
+function reminderBudget(
+	host: PolicyContext,
+): { readonly budget: Budget; readonly tokens: number } | undefined {
+	const usage = host.getContextUsage();
+	if (usage === undefined || usage.tokens === null || usage.contextWindow <= 0) {
+		return undefined;
+	}
+	const budget = budgetFor(host, usage.contextWindow);
+	if (
+		budget === undefined ||
+		!budget.enabled ||
+		!budget.supported ||
+		usage.tokens >= budget.rolloverAt
+	) {
+		return undefined;
+	}
+	const buffer = Math.min(REMINDER_BUFFER_TOKENS, Math.floor(budget.usable * 0.1));
+	return usage.tokens < budget.rolloverAt - buffer ? undefined : { budget, tokens: usage.tokens };
+}
+function sendReminder(
+	pi: ExtensionAPI,
+	ctx: ExtensionContext,
+	available: { readonly budget: Budget; readonly tokens: number },
+	reminders: ReminderLookup,
+): void {
+	const { budget, tokens } = available;
+	const state = reminders.read(ctx);
+	const fingerprint: ReminderFingerprint = {
+		windowId: state.windowId,
+		contextWindow: budget.contextWindow,
+		reserveTokens: budget.reserveTokens,
+	};
+	if (hasReminder(state.reminders, fingerprint)) {
+		return;
+	}
+	pi.sendMessage(
+		{
+			customType: REMINDER_TYPE,
+			content: `[posthorse] Checkpoint now: ${(budget.rolloverAt - tokens).toLocaleString("en-US")} tokens remain before Pi's automatic rollover line. Stop normal work, ${CHECKPOINT_STEPS}.`,
+			display: true,
+			details: fingerprint,
+		},
+		{ deliverAs: "steer" },
+	);
+}
+function registerReminderFilter(
+	pi: ExtensionAPI,
+	policy: SnapshotReader,
+	reminders: ReminderLookup,
+): void {
+	pi.on("context", (event, ctx) => {
+		if (
+			!event.messages.some(
+				(message) => message.role === "custom" && isReminderType(message.customType),
+			)
+		) {
+			return;
+		}
+		const marker = event.messages.find(
+			(message) => message.role === "custom" && message.customType === "context-window",
+		);
+		let windowId: unknown;
+		if (marker?.role === "custom") {
+			windowId = isRecord(marker.details) ? marker.details.windowId : undefined;
+		} else {
+			windowId = reminders.read(ctx).windowId;
+		}
+		if (typeof windowId !== "string") {
+			return;
+		}
+		const host = policy(ctx);
+		const budget = budgetFor(host);
+		const fingerprint: ReminderFingerprint = {
+			windowId,
+			contextWindow: budget?.contextWindow,
+			reserveTokens: budget?.reserveTokens,
+		};
+		const enabled = host.getCompactionSettings().enabled;
+		const stale = (message: {
+			readonly role: string;
+			readonly customType?: string;
+			readonly details?: unknown;
+		}) =>
+			message.role === "custom" &&
+			isReminderType(message.customType) &&
+			(!enabled || !reminderMatches(message.details, fingerprint));
+		if (event.messages.some(stale)) {
+			return { messages: event.messages.filter((message) => !stale(message)) };
+		}
+		return;
+	});
+}
+function automaticDraft(
+	pi: ExtensionAPI,
+	ctx: ExtensionContext,
+	recovery: { readonly handoff: string; readonly tokensBefore: number; readonly reason: string },
+) {
+	// Public compaction requires an entry id. An invisible sentinel keeps no prior conversation.
+	pi.appendEntry("posthorse-boundary", {});
+	const firstKeptEntryId = ctx.sessionManager.getLeafId();
+	if (firstKeptEntryId === null || firstKeptEntryId === "") {
+		throw new Error("Pi did not persist the context boundary.");
+	}
+	return {
+		compaction: {
+			summary: recovery.handoff,
+			firstKeptEntryId,
+			tokensBefore: recovery.tokensBefore,
+			details: { posthorse: 1, reason: recovery.reason },
+		},
+	};
+}
+function reportAutomaticFailure(ctx: ExtensionContext, error: unknown): void {
+	const detail = error instanceof Error ? error.message : safeJsonStringify(error);
+	const message = `Posthorse automatic rollover cancelled: ${detail}`;
+	if (ctx.hasUI) {
+		ctx.ui.notify(message, "error");
+	} else {
+		console.error(message);
+	}
+}
+function registerAutomaticRollover(
+	pi: ExtensionAPI,
+	policy: SnapshotReader,
+	activeToolTokens: () => number,
+): void {
+	pi.on("session_before_compact", (event, ctx) => {
+		if (event.reason === "manual") {
+			return;
+		}
+		const cancelled = () => event.signal.aborted || ctx.signal?.aborted === true;
+		if (cancelled()) {
+			return { cancel: true };
+		}
+		try {
+			const host = policy(ctx, event.preparation.settings);
+			if (budgetFor(host)?.supported !== true) {
+				return;
+			}
+			const limit = freshPayloadChars(host, activeToolTokens());
+			const ownerQuestionRegistered = pi
+				.getAllTools()
+				.some((tool) => tool.name === "ask_question" && tool.namespace === undefined);
+			const handoff = buildAutoHandoff(
+				event.branchEntries,
+				ctx.sessionManager.buildSessionProjection().entries,
+				limit,
+				ownerQuestionRegistered,
+			);
+			if (limit < MIN_PAGE_CHARS || handoff.length > limit) {
+				throw new Error(
+					"Too little fresh context capacity for an automatic recovery record.",
+				);
+			}
+			if (cancelled()) {
+				return { cancel: true };
+			}
+			return automaticDraft(pi, ctx, {
+				handoff,
+				tokensBefore: event.preparation.tokensBefore,
+				reason: event.reason,
+			});
+		} catch (error) {
+			// Hook errors are swallowed by Pi, which would run its summarizer; cancel instead.
+			if (!cancelled()) {
+				reportAutomaticFailure(ctx, error);
+			}
+			return { cancel: true };
+		}
+	});
+}
+/** Owns per-turn reset requests and leaf-certified reminder state. Pi owns persistence and scheduling. */
+export function registerRollover(
+	pi: ExtensionAPI,
+	policy: SnapshotReader,
+	activeToolTokens: () => number,
+	resetPages: () => void,
+): void {
+	const resetRequests = new Map<string, string>();
+	const reminders = createReminderLookup();
+	const resetTurn = () => {
+		resetRequests.clear();
+		resetPages();
+	};
+	pi.on("turn_start", resetTurn);
+	pi.on("session_start", () => {
+		resetTurn();
+		reminders.reset();
+	});
+	pi.on("session_tree", () => {
+		reminders.reset();
+	});
+	registerGuidance(pi, policy);
+	registerReminderFilter(pi, policy, reminders);
+	registerAutomaticRollover(pi, policy, activeToolTokens);
+	pi.on("turn_end", (event, ctx) => {
+		const requested = event.toolResults.find(
+			(result) => result.toolName === "new_context" && resetRequests.has(result.toolCallId),
+		);
+		const summary =
+			requested === undefined ? undefined : resetRequests.get(requested.toolCallId);
+		resetRequests.clear();
+		const host = policy(ctx);
+		if (
+			event.message.role === "assistant" &&
+			(event.message.stopReason === "error" || event.message.stopReason === "aborted")
+		) {
+			return;
+		}
+		if (summary !== undefined && !event.toolResults.some((result) => result.isError)) {
+			return explicitReset(event, ctx, summary, () =>
+				freshPayloadChars(host, activeToolTokens(), event.context.pendingMessages),
+			);
+		}
+		const available = reminderBudget(host);
+		if (available !== undefined) {
+			sendReminder(pi, ctx, available, reminders);
+		}
+		return;
+	});
+	registerContextTools(pi, policy, activeToolTokens, (id, handoff) => {
+		resetRequests.set(id, handoff);
+	});
+}

package/src/session-files.ts

@@ -0,0 +1,259 @@
+import { execFile } from "node:child_process";
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join, relative, resolve, sep } from "node:path";
+
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import {
+	isRecord,
+	optionalString,
+	type EntryLike,
+	type MessageLike,
+	type WindowedEntry,
+} from "./contracts.ts";
+import { lfLines } from "./files.ts";
+import { windowProjection } from "./history-entries.ts";
+import { shortKey } from "./message-text.ts";
+
+function execFileText(
+	file: string,
+	args: readonly string[],
+	signal?: AbortSignal,
+): Promise<string> {
+	return new Promise((resolveOutput, reject) => {
+		execFile(file, args, { signal, maxBuffer: 64 * 1024 * 1024 }, (error, stdout) => {
+			if (error !== null) {
+				const failure: Error = error;
+				reject(failure);
+			} else {
+				resolveOutput(stdout);
+			}
+		});
+	});
+}
+/** Raw JSONL needles must cover every match in normalized text; otherwise scan all files. */
+export function prefilterNeedles(query: string): string[] | undefined {
+	if (/["\\[\](){}:$\p{Cc}]/u.test(query) || query.trim() !== query) {
+		return undefined;
+	}
+	if (
+		["call id", "execution arguments", "no handoff", "excluded from model context by pi"].some(
+			(phrase) => phrase.includes(query),
+		)
+	) {
+		return undefined;
+	}
+	const image =
+		query.includes(",") ||
+		/^\d+ ?i?m?a?g?e?s?$/.test(query) ||
+		["images", "unknown type"].some((phrase) => phrase.includes(query));
+	return image ? [query, '"type":"image"'] : [query];
+}
+export async function filesContaining(
+	dir: string,
+	needles: readonly string[],
+	signal?: AbortSignal,
+): Promise<Set<string> | undefined> {
+	const args = [
+		"--files-with-matches",
+		"--fixed-strings",
+		"--ignore-case",
+		"--no-ignore",
+		"--hidden",
+		"--no-messages",
+		"--glob",
+		"*.jsonl",
+		...needles.flatMap((needle) => ["--regexp", needle]),
+		dir,
+	];
+	for (const rg of [join(getAgentDir(), "bin", "rg"), "rg"]) {
+		try {
+			// Try the host-managed binary first; PATH is a fallback only when it is unavailable.
+			// oxlint-disable-next-line no-await-in-loop
+			const stdout = await execFileText(rg, args, signal);
+			return new Set(
+				stdout
+					.split("\n")
+					.filter((file) => file !== "")
+					.map((file) => resolve(file)),
+			);
+		} catch (error) {
+			const code = isRecord(error) ? error.code : undefined;
+			if (code === 1) {
+				return new Set();
+			}
+			if (code === "ENOENT") {
+				continue;
+			}
+			signal?.throwIfAborted();
+			return undefined;
+		}
+	}
+	return;
+}
+function messageFrom(value: unknown): MessageLike | undefined {
+	if (!isRecord(value)) {
+		return undefined;
+	}
+	return {
+		role: optionalString(value.role),
+		stopReason: optionalString(value.stopReason),
+		errorMessage: optionalString(value.errorMessage),
+		content: value.content,
+		summary: optionalString(value.summary),
+		toolName: optionalString(value.toolName),
+		namespace: optionalString(value.namespace),
+		toolCallId: optionalString(value.toolCallId),
+		isError: value.isError === true,
+		command: optionalString(value.command),
+		output: optionalString(value.output),
+		excludeFromContext: value.excludeFromContext === true,
+	};
+}
+function parsedEntry(line: string): EntryLike | undefined {
+	try {
+		const value: unknown = JSON.parse(line);
+		if (!isRecord(value)) {
+			return undefined;
+		}
+		const replacement = isRecord(value.replacement)
+			? { content: value.replacement.content }
+			: undefined;
+		return {
+			type: optionalString(value.type),
+			id: optionalString(value.id),
+			parentId: value.parentId === null ? null : optionalString(value.parentId),
+			timestamp: optionalString(value.timestamp),
+			message: messageFrom(value.message),
+			summary: optionalString(value.summary),
+			customType: optionalString(value.customType),
+			content: value.content,
+			details: value.details,
+			display: value.display === true,
+			handoff: optionalString(value.handoff),
+			firstKeptEntryId: optionalString(value.firstKeptEntryId),
+			targetId: optionalString(value.targetId),
+			replacement: value.replacement === null ? null : replacement,
+		};
+	} catch {
+		return undefined;
+	} // Pi skips malformed journal lines too.
+}
+export async function* sessionWindowEntries(
+	file: string,
+	signal?: AbortSignal,
+): AsyncGenerator<WindowedEntry> {
+	if (!existsSync(file)) {
+		return;
+	}
+	const project = windowProjection();
+	for await (const line of lfLines(file, signal)) {
+		const entry = parsedEntry(line);
+		if (entry === undefined) {
+			continue;
+		}
+		const item = project(entry);
+		if (item !== undefined) {
+			yield item;
+		}
+	}
+}
+function sessionFiles(dir: string): string[] {
+	if (!existsSync(dir)) {
+		return [];
+	}
+	return readdirSync(dir, { recursive: true, withFileTypes: true })
+		.filter(
+			(entry) =>
+				entry.isFile() && entry.name.endsWith(".jsonl") && !entry.name.includes(".intent."),
+		)
+		.map((entry) => join(entry.parentPath, entry.name));
+}
+async function sessionHeader(
+	file: string,
+	signal?: AbortSignal,
+): Promise<{ readonly cwd?: string } | undefined> {
+	for await (const line of lfLines(file, signal)) {
+		try {
+			const value: unknown = JSON.parse(line);
+			if (value === null) {
+				continue;
+			}
+			return isRecord(value) && value.type === "session" && typeof value.id === "string"
+				? { cwd: optionalString(value.cwd) }
+				: undefined;
+		} catch {
+			/* Pi skips malformed lines before the first parsed entry. */
+		}
+	}
+	return;
+}
+function parentSession(
+	file: string,
+	dir: string,
+	hasFile: (file: string) => boolean,
+): string | undefined {
+	const parts = relative(dir, file).split(sep);
+	for (let i = parts.length - 1; i > 0; i--) {
+		const candidate = `${join(dir, ...parts.slice(0, i))}.jsonl`;
+		if (hasFile(candidate)) {
+			return candidate;
+		}
+	}
+	return;
+}
+export type SessionScope = {
+	readonly dir: string;
+	readonly cwd: string;
+	readonly currentFile?: string;
+	readonly signal?: AbortSignal;
+	readonly fileKey?: string;
+	readonly candidates?: (file: string) => boolean;
+};
+/** Scope memoization is owned by one enumeration; nested runs inherit their parent's project. */
+function membership(
+	scope: SessionScope,
+	hasFile: (file: string) => boolean,
+): (file: string) => Promise<boolean> {
+	const visible = new Map<string, boolean>();
+	const belongs = async (file: string): Promise<boolean> => {
+		const known = visible.get(file);
+		if (known !== undefined) {
+			return known;
+		}
+		const header = await sessionHeader(file, scope.signal);
+		if (header === undefined) {
+			visible.set(file, false);
+			return false;
+		}
+		const parent = parentSession(file, scope.dir, hasFile);
+		const project =
+			header.cwd !== undefined && header.cwd !== "" && resolve(header.cwd) === scope.cwd;
+		const allowed =
+			file === scope.currentFile || (parent === undefined ? project : await belongs(parent));
+		visible.set(file, allowed);
+		return allowed;
+	};
+	return belongs;
+}
+/** Newest modified project session files, optionally limited by file key or prefilter. */
+export async function* scopedSessionFiles(scope: SessionScope): AsyncGenerator<string> {
+	const files = sessionFiles(scope.dir);
+	const fileSet = new Set(files);
+	const belongs = membership(scope, (file) => fileSet.has(file));
+	const selected = files
+		.filter(
+			(file) =>
+				(scope.candidates === undefined || scope.candidates(resolve(file))) &&
+				(scope.fileKey === undefined ||
+					scope.fileKey.startsWith(shortKey(relative(scope.dir, file)))),
+		)
+		.map((file) => ({ file, mtime: statSync(file).mtimeMs }))
+		.sort((a, b) => b.mtime - a.mtime);
+	for (const { file } of selected) {
+		// Preserve newest-first traversal while closing each header stream before opening another.
+		// oxlint-disable-next-line no-await-in-loop
+		if (await belongs(file)) {
+			yield file;
+		}
+	}
+}