pi-observational-memory 2.4.3 → 3.0.1

package/src/hooks/compaction-hook.ts

@@ -1,396 +1,51 @@
-import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
-import { Text } from "@mariozechner/pi-tui";
-import { debugLog, withDebugLogContext } from "../debug-log.js";
-import { resolveTurnLimits } from "../config.js";
-import {
-	collectObservationsByCoverage,
-	findLastCompactionIndex,
-	gapRawEntries,
-	getMemoryState,
-} from "../branch.js";
-import {
-	REFLECTOR_MAX_PASSES,
-	coverageTagCounts,
-	migrateLegacyReflections,
-	observationPoolTokens,
-	renderSummary,
-	runPruner,
-	runReflector,
-	type CoverageTagCounts,
-	type PrunerResult,
-	type ReflectorStats,
-} from "../compaction.js";
-import { observationsToPromptLines, runObserver } from "../observer.js";
-import { CompactionProgressTracker } from "../progress.js";
-import type { Runtime } from "../runtime.js";
-import { serializeSourceAddressedBranchEntries } from "../serialize.js";
-import { estimateStringTokens } from "../tokens.js";
-import {
-	OBSERVATION_CUSTOM_TYPE,
-	reflectionToPromptLine,
-	type MemoryDetailsV4,
-	type MemoryReflection,
-	type ObservationEntryData,
-	type ObservationRecord,
-} from "../types.js";
-
-function plural(count: number, singular: string, pluralForm = `${singular}s`): string {
-	return `${count.toLocaleString()} ${count === 1 ? singular : pluralForm}`;
-}
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 
-function formatCoverageCounts(counts: CoverageTagCounts): string {
-	return `${counts.uncited.toLocaleString()}/${counts.cited.toLocaleString()}/${counts.reinforced.toLocaleString()} uncited/cited/reinforced`;
-}
+import type { Runtime } from "../runtime.js";
+import { buildCompactionProjection, renderSummary, type Entry } from "../session-ledger/index.js";
 
-function formatReflectorStats(stats: ReflectorStats): string {
-	const failed = stats.failedPass === undefined ? "" : `, failed pass ${stats.failedPass}`;
-	return `reflector ${plural(stats.toolCalls, "tool call")}, +${stats.added.toLocaleString()} added, ${stats.merged.toLocaleString()} merged, ${stats.promoted.toLocaleString()} promoted, ${stats.duplicates.toLocaleString()} duplicate/no-op, ${stats.unsupported.toLocaleString()} unsupported${failed}`;
-}
+const DEFAULT_OBSERVATIONS_POOL_MAX_TOKENS = 20_000;
 
-function formatPrunerStats(result: PrunerResult): string {
-	return `pruner dropped ${plural(result.droppedIds.length, "observation")} in ${plural(result.passes.length, "pass", "passes")}, stop: ${result.stopReason}`;
+function observationsPoolMaxTokens(runtime: Runtime): number {
+	const value = (runtime.config as { observationsPoolMaxTokens?: unknown }).observationsPoolMaxTokens;
+	return typeof value === "number" && Number.isFinite(value) && value > 0
+		? value
+		: DEFAULT_OBSERVATIONS_POOL_MAX_TOKENS;
 }
 
 export function registerCompactionHook(pi: ExtensionAPI, runtime: Runtime): void {
-	pi.on("session_before_compact", async (event, ctx) => {
+	pi.on("session_before_compact", async (event: any, ctx: any) => {
 		if (runtime.compactHookInFlight) {
-			if (ctx.hasUI) ctx.ui.notify(
-				"Observational memory: another compaction is already in progress; cancelling duplicate",
-				"warning",
-			);
+			if (ctx.hasUI) {
+				ctx.ui.notify(
+					"Observational memory: another compaction is already in progress; cancelling duplicate",
+					"warning",
+				);
+			}
 			return { cancel: true };
 		}
+
 		runtime.compactHookInFlight = true;
-		const progress = new CompactionProgressTracker();
-		const WIDGET_NAME = "om_compact_progress";
-		let clearWidget = () => {};
 		try {
 			runtime.ensureConfig(ctx.cwd);
-			const runId = `compaction-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
-			return await withDebugLogContext({ enabled: runtime.config.debugLog === true, cwd: ctx.cwd, runId }, async () => {
-			const { preparation, branchEntries, signal } = event;
+			const { preparation, branchEntries } = event;
 			const { firstKeptEntryId, tokensBefore } = preparation;
-
-			// Capture ctx properties synchronously — after multiple awaits below,
-			// the extension ctx may become stale (e.g. after session replacement/reload).
-			const hasUI = ctx.hasUI;
-			const ui = ctx.ui;
-			const turnLimits = resolveTurnLimits(runtime.config);
-			debugLog("compaction.start", {
+			const projection = buildCompactionProjection(
+				branchEntries as Entry[],
 				firstKeptEntryId,
-				tokensBefore,
-				branchEntryCount: branchEntries.length,
-				reflectionThresholdTokens: runtime.config.reflectionThresholdTokens,
-				turnLimits,
-				legacyCompactionMaxToolCalls: runtime.config.compactionMaxToolCalls,
-			});
-
-			const resolved = await runtime.resolveModel(ctx as any);
-			if (!resolved.ok) {
-				debugLog("compaction.model_unavailable", { reason: resolved.reason });
-				if (hasUI) ui?.notify(
-					`Observational memory: cannot compact — ${resolved.reason}. ` +
-					"Fix the model/API key and try /compact manually.",
-					"error",
-				);
-				return { cancel: true };
-			}
-			runtime.resolveFailureNotified = false;
-
-			const updateWidget = () => {
-				if (!hasUI || !ui) return;
-				if (!progress.getPhase()) {
-					ui.setWidget(WIDGET_NAME, undefined);
-					return;
-				}
-				ui.setWidget(WIDGET_NAME, (_tui: any, theme: any) => {
-					return new Text(
-						progress.formatWidget(theme),
-						0, 0,
-					);
-				});
-			};
-			clearWidget = () => {
-				if (hasUI && ui) ui.setWidget(WIDGET_NAME, undefined);
-			};
-
-			let entries = branchEntries as Parameters<typeof getMemoryState>[0];
-
-			if (runtime.observerPromise) {
-				try { await runtime.observerPromise; } catch { /* already notified via launchObserverTask */ }
-				// In-flight observer may have appended a new observation entry during the await;
-				// refresh from sessionManager so gap computation and coverage collection see it
-				entries = ctx.sessionManager.getBranch() as typeof entries;
-			}
-
-			const memoryState = getMemoryState(entries);
-			debugLog("compaction.memory_state", {
-				committedObservations: memoryState.committedObs.length,
-				pendingObservations: memoryState.pendingObs.length,
-				reflections: memoryState.reflections.length,
-			});
-
-			let gapObservationData: ObservationEntryData | null = null;
-			const gap = gapRawEntries(entries, firstKeptEntryId);
-			if (gap.length > 0) {
-				const { text: gapChunk, sourceEntryIds } = serializeSourceAddressedBranchEntries(gap);
-				if (gapChunk.trim() && sourceEntryIds.length > 0) {
-					const gapFromId = gap[0].id;
-					const gapUpToId = gap[gap.length - 1].id;
-					const priorObservationLines = observationsToPromptLines([
-						...memoryState.committedObs,
-						...memoryState.pendingObs,
-					]);
-					const gapTokenEstimate = estimateStringTokens(gapChunk);
-					debugLog("compaction.sync_catchup.start", {
-						gapEntryCount: gap.length,
-						sourceEntryIds,
-						gapFromId,
-						gapUpToId,
-						tokenEstimate: gapTokenEstimate,
-					});
-					if (hasUI) ui?.notify(
-						`Observational memory: sync catch-up observer running on ~${gapTokenEstimate.toLocaleString()}-token gap`,
-						"info",
-					);
-					progress.setPhase("observer", 1, 1);
-					updateWidget();
-					runtime.observerInFlight = true;
-					const gapCall = runObserver({
-						model: resolved.model as any,
-						apiKey: resolved.apiKey,
-						headers: resolved.headers,
-						priorReflections: memoryState.reflections.map(reflectionToPromptLine),
-						priorObservations: priorObservationLines,
-						chunk: gapChunk,
-						allowedSourceEntryIds: sourceEntryIds,
-						signal,
-						maxTurns: turnLimits.observerMaxTurnsPerRun,
-						thinkingLevel: runtime.config.thinkingLevel,
-					});
-					const gapPromise: Promise<void> = gapCall.then(() => undefined, () => undefined);
-					runtime.observerPromise = gapPromise;
-					try {
-						const records = await gapCall;
-						if (records && records.length > 0) {
-							const observationTokens = records.reduce((sum, r) => sum + estimateStringTokens(r.content), 0);
-							gapObservationData = {
-								records,
-								coversFromId: gapFromId,
-								coversUpToId: gapUpToId,
-								tokenCount: observationTokens,
-							};
-							debugLog("compaction.sync_catchup.records", {
-								count: records.length,
-								observationTokens,
-								coversFromId: gapFromId,
-								coversUpToId: gapUpToId,
-								records,
-							});
-							pi.appendEntry(OBSERVATION_CUSTOM_TYPE, gapObservationData);
-							if (hasUI && ui) ui.notify(
-								`Observational memory: sync catch-up recorded ${records.length} observation${records.length === 1 ? "" : "s"} (~${observationTokens.toLocaleString()} tokens)`,
-								"info",
-							);
-						} else if (hasUI && ui) {
-							debugLog("compaction.sync_catchup.empty", { gapEntryCount: gap.length });
-							ui.notify(
-								"Observational memory: sync catch-up observer returned empty — proceeding with compaction",
-								"warning",
-							);
-						}
-					} catch (error) {
-						const msg = error instanceof Error ? error.message : String(error);
-						debugLog("compaction.sync_catchup.error", { gapEntryCount: gap.length, errorMessage: msg });
-						if (hasUI && ui) ui.notify(
-							`Observational memory: sync catch-up observer failed: ${msg}. Cancelling compaction — ${gap.length} unobserved raw entries would be pruned without coverage. Try /compact again.`,
-							"warning",
-						);
-						return { cancel: true };
-					} finally {
-						runtime.observerInFlight = false;
-						if (runtime.observerPromise === gapPromise) runtime.observerPromise = null;
-					}
-				}
-			}
-
-			const priorCompactionIdx = findLastCompactionIndex(entries);
-			const priorFirstKeptEntryId = priorCompactionIdx >= 0 ? entries[priorCompactionIdx].firstKeptEntryId : undefined;
-			const deltaObservationData = collectObservationsByCoverage(entries, priorFirstKeptEntryId, firstKeptEntryId);
-			if (gapObservationData) deltaObservationData.push(gapObservationData);
-			debugLog("compaction.delta", {
-				priorFirstKeptEntryId,
-				firstKeptEntryId,
-				deltaObservationEntries: deltaObservationData.length,
-				deltaObservationRecords: deltaObservationData.reduce((sum, data) => sum + data.records.length, 0),
-				gapObservationRecords: gapObservationData?.records.length ?? 0,
-			});
-
-			if (deltaObservationData.length === 0) {
-				// No new observations since last compaction. If we have existing memory,
-				// carry it forward in a no-op compaction so it survives Pi's compaction.
-				// If there is truly nothing (no prior memory either), cancel.
-				if (memoryState.committedObs.length === 0 && memoryState.reflections.length === 0) {
-					debugLog("compaction.no_delta_cancel", {
-						committedObservations: memoryState.committedObs.length,
-						pendingObservations: memoryState.pendingObs.length,
-						reflections: memoryState.reflections.length,
-					});
-					if (hasUI) {
-						ui?.notify(
-							`Observational memory: nothing to compact yet — ${plural(memoryState.committedObs.length, "committed observation")} and ${plural(memoryState.pendingObs.length, "pending observation")}; no eligible delta before compact boundary`,
-							"warning",
-						);
-					}
-					return { cancel: true };
-				}
-
-				// Carry forward existing memory without running reflector/pruner
-				const workingReflections: MemoryReflection[] = migrateLegacyReflections(memoryState.reflections);
-				debugLog("compaction.no_delta_carry_forward", {
-					observations: memoryState.committedObs.length,
-					reflections: workingReflections.length,
-				});
-				const summary = renderSummary(workingReflections, memoryState.committedObs);
-				const details: MemoryDetailsV4 = {
-					type: "observational-memory",
-					version: 4,
-					observations: memoryState.committedObs,
-					reflections: workingReflections,
-				};
-				if (hasUI) ui?.notify(
-					`Observational memory: no new observations — carrying forward ${memoryState.committedObs.length} observation${memoryState.committedObs.length === 1 ? "" : "s"}, ${workingReflections.length} reflection${workingReflections.length === 1 ? "" : "s"}`,
-					"info",
-				);
-				return {
-					compaction: {
-						summary,
-						firstKeptEntryId,
-						tokensBefore,
-						details,
-					},
-				};
-			}
-
-			const workingReflections: MemoryReflection[] = migrateLegacyReflections(memoryState.reflections);
-			const workingObservations: ObservationRecord[] = [
-				...memoryState.committedObs,
-				...deltaObservationData.flatMap((d) => d.records),
-			];
-
-			const observationTokens = observationPoolTokens(workingObservations);
-			debugLog("compaction.reflect_prune.gate", {
-				observationTokens,
-				reflectionThresholdTokens: runtime.config.reflectionThresholdTokens,
-				willRun: observationTokens >= runtime.config.reflectionThresholdTokens,
-				workingObservations: workingObservations.length,
-				workingReflections: workingReflections.length,
-			});
-
-			let finalReflections = workingReflections;
-			let finalObservations = workingObservations;
-
-			if (observationTokens >= runtime.config.reflectionThresholdTokens) {
-				try {
-					debugLog("compaction.reflect_prune.start", {
-						workingObservations: workingObservations.length,
-						workingReflections: workingReflections.length,
-						observationTokens,
-					});
-					if (hasUI) ui?.notify("Observational memory: running reflector + pruner...", "info");
-					progress.setPhase("reflector", 1, REFLECTOR_MAX_PASSES);
-					progress.setStartingCounts(workingReflections.length, workingObservations.length);
-					updateWidget();
-					const coverageBefore = coverageTagCounts(workingReflections, workingObservations);
-					const reflectorResult = await runReflector(
-						{ model: resolved.model as any, apiKey: resolved.apiKey, headers: resolved.headers, signal, onEvent: (event) => { progress.onEvent(event); updateWidget(); }, maxTurns: turnLimits.reflectorMaxTurnsPerPass, thinkingLevel: runtime.config.thinkingLevel },
-						workingReflections,
-						workingObservations,
-						(pass, max) => { progress.setPhase("reflector", pass, max); updateWidget(); },
-					);
-					finalReflections = reflectorResult.reflections;
-					const coverageAfter = coverageTagCounts(finalReflections, workingObservations);
-					debugLog("compaction.reflector.result", {
-						stats: reflectorResult.stats,
-						coverageBefore,
-						coverageAfter,
-						beforeReflections: workingReflections.length,
-						afterReflections: finalReflections.length,
-					});
-
-					const prunerResult = await runPruner(
-						{ model: resolved.model as any, apiKey: resolved.apiKey, headers: resolved.headers, signal, onEvent: (event) => { progress.onEvent(event); updateWidget(); }, maxTurns: turnLimits.prunerMaxTurnsPerPass, thinkingLevel: runtime.config.thinkingLevel },
-						finalReflections,
-						workingObservations,
-						runtime.config.reflectionThresholdTokens,
-						(pass, max) => { progress.setPhase("pruner", pass, max); updateWidget(); },
-					);
-					finalObservations = prunerResult.observations;
-					debugLog("compaction.pruner.result", {
-						stopReason: prunerResult.stopReason,
-						fellBack: prunerResult.fellBack,
-						droppedIds: prunerResult.droppedIds,
-						passes: prunerResult.passes,
-						beforeObservations: workingObservations.length,
-						afterObservations: finalObservations.length,
-					});
-					updateWidget();
-					if (hasUI) {
-						ui?.notify(
-							`Observational memory: diagnostics — ${formatReflectorStats(reflectorResult.stats)}; coverage ${formatCoverageCounts(coverageBefore)} → ${formatCoverageCounts(coverageAfter)}; ${formatPrunerStats(prunerResult)}`,
-							"info",
-						);
-					}
-					if (prunerResult.fellBack && hasUI) {
-						ui?.notify(
-							"Observational memory: pruner run failed; kept observation set unchanged",
-							"warning",
-						);
-					}
-				} catch (error) {
-					const msg = error instanceof Error ? error.message : String(error);
-					debugLog("compaction.reflect_prune.error", { errorMessage: msg });
-					if (hasUI) ui?.notify(`Observational memory: reflect/prune failed: ${msg}`, "warning");
-				}
-			}
-
-			const summary = renderSummary(finalReflections, finalObservations);
-
-			if (finalObservations.length === 0) {
-				throw new Error("invariant violated: finalObservations empty after delta guard");
-			}
-
-			const details: MemoryDetailsV4 = {
-				type: "observational-memory",
-				version: 4,
-				observations: finalObservations,
-				reflections: finalReflections,
-			};
-			debugLog("compaction.result", {
-				finalObservations: finalObservations.length,
-				finalReflections: finalReflections.length,
-				firstKeptEntryId,
-				tokensBefore,
-			});
-
-			if (hasUI) ui?.notify(
-				`Observational memory: compaction assembled — ${finalObservations.length} observation${finalObservations.length === 1 ? "" : "s"}, ${finalReflections.length} reflection${finalReflections.length === 1 ? "" : "s"}`,
-				"info",
+				{ observationsPoolMaxTokens: observationsPoolMaxTokens(runtime) },
 			);
+			const summary = renderSummary(projection.reflections, projection.observations);
 
 			return {
 				compaction: {
 					summary,
 					firstKeptEntryId,
 					tokensBefore,
-					details,
+					details: projection.details,
 				},
 			};
-			});
 		} finally {
 			runtime.compactHookInFlight = false;
-			progress.clear();
-			clearWidget();
 		}
 	});
-}
+}

package/src/hooks/compaction-trigger.ts

@@ -1,5 +1,5 @@
-import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
-import { rawTokensSinceLastCompaction } from "../branch.js";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { rawTokensSinceLastCompaction, type Entry } from "../session-ledger/index.js";
 import type { Runtime } from "../runtime.js";
 
 /**
@@ -11,7 +11,7 @@ const RETRYABLE_ERROR_RE =
 	/overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i;
 
 export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
-	pi.on("agent_end", (event, ctx) => {
+	pi.on("agent_end", (event: any, ctx: any) => {
 		runtime.ensureConfig(ctx.cwd);
 		if (runtime.config.passive === true) return;
 		if (runtime.compactInFlight) return;
@@ -31,9 +31,9 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 			return;
 		}
 
-		const entries = ctx.sessionManager.getBranch() as Parameters<typeof rawTokensSinceLastCompaction>[0];
+		const entries = ctx.sessionManager.getBranch() as Entry[];
 		const tokens = rawTokensSinceLastCompaction(entries);
-		if (tokens < runtime.config.compactionThresholdTokens) return;
+		if (tokens < runtime.config.compactAfterTokens) return;
 
 		// Capture ctx properties synchronously — the setTimeout + async work below
 		// may outlive the extension ctx (stale after session replacement/reload).
@@ -46,31 +46,22 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 		);
 
 		runtime.compactInFlight = true;
-		setTimeout(async () => {
-			if (runtime.observerPromise) {
-				try {
-					await runtime.observerPromise;
-				} catch {
-					// errors already surfaced via launchObserverTask
-				}
-			}
-			// After awaiting observerPromise, ctx may be stale.
-			// Use captured hasUI/ui for notification; wrap ctx access in try/catch.
+		setTimeout(() => {
 			try {
 				if (!ctx.isIdle()) {
 					runtime.compactInFlight = false;
 					if (hasUI) ui?.notify(
-						"Observational memory: compaction deferred — agent became busy after observer wait",
+						"Observational memory: compaction deferred — agent became busy before compaction",
 						"info",
 					);
 					return;
 				}
-				const currentEntries = ctx.sessionManager.getBranch() as Parameters<typeof rawTokensSinceLastCompaction>[0];
+				const currentEntries = ctx.sessionManager.getBranch() as Entry[];
 				const currentTokens = rawTokensSinceLastCompaction(currentEntries);
-				if (currentTokens < runtime.config.compactionThresholdTokens) {
+				if (currentTokens < runtime.config.compactAfterTokens) {
 					runtime.compactInFlight = false;
 					if (hasUI) ui?.notify(
-						"Observational memory: compaction skipped — another compaction already ran during observer wait",
+						"Observational memory: compaction skipped — another compaction already ran before deferred compaction",
 						"info",
 					);
 					return;
@@ -80,7 +71,7 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 						runtime.compactInFlight = false;
 						if (hasUI) ui?.notify("Observational memory: compaction complete", "info");
 					},
-					onError: (error) => {
+					onError: (error: { message: string }) => {
 						runtime.compactInFlight = false;
 						if (error.message === "Compaction cancelled") {
 							// We already notified the user with the real reason before returning { cancel: true }.
@@ -96,4 +87,4 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 			}
 		}, 0);
 	});
-}
+}