pi-observational-memory 2.4.1 → 2.4.3

package/src/compaction.ts

@@ -1,7 +1,9 @@
-import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@mariozechner/pi-agent-core";
-import { Type, type Message, type Model } from "@mariozechner/pi-ai";
-import type { Static } from "@sinclair/typebox";
+import { agentLoop, type AgentContext, type AgentEvent, type AgentLoopConfig, type AgentTool } from "@mariozechner/pi-agent-core";
+import { Type, type Message, type Model, type ModelThinkingLevel } from "@mariozechner/pi-ai";
+import type { Static } from "typebox";
+import { debugLog, isDebugLogEnabled } from "./debug-log.js";
 import { hashId } from "./ids.js";
+import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "./model-budget.js";
 import { observationsToPromptLines } from "./observer.js";
 import { buildPrunerPassGuidance, buildReflectorPassGuidance, CONTEXT_USAGE_INSTRUCTIONS, PRUNER_SYSTEM, REFLECTOR_SYSTEM } from "./prompts.js";
 import { truncateRecordContent } from "./serialize.js";
@@ -9,12 +11,12 @@ import { estimateStringTokens } from "./tokens.js";
 import { reflectionContent, reflectionToPromptLine } from "./types.js";
 import type { MemoryReflection, ObservationRecord, ReflectionRecord } from "./types.js";
 
-const REFLECTOR_MAX_PASSES = 3;
-const PRUNER_MAX_PASSES = 5;
+export const REFLECTOR_MAX_PASSES = 2;
+export const PRUNER_MAX_PASSES = 2;
 const PRUNER_TARGET_RATIO = 0.8;
 
-function observationPoolTokens(observations: ObservationRecord[]): number {
-	return observations.reduce((sum, o) => sum + estimateStringTokens(o.content), 0);
+export function observationPoolTokens(observations: ObservationRecord[]): number {
+	return estimateStringTokens(observationsToPromptLines(observations).join("\n"));
 }
 
 interface LlmArgs {
@@ -23,6 +25,9 @@ interface LlmArgs {
 	headers?: Record<string, string>;
 	signal?: AbortSignal;
 	agentLoop?: typeof agentLoop;
+	onEvent?: (event: import("@mariozechner/pi-agent-core").AgentEvent) => void;
+	maxTurns?: number;
+	thinkingLevel?: ModelThinkingLevel;
 }
 
 function joinReflectionsOrEmpty(items: MemoryReflection[]): string {
@@ -33,8 +38,123 @@ function joinObservationsOrEmpty(items: ObservationRecord[]): string {
 	return items.length ? observationsToPromptLines(items).join("\n") : "(none yet)";
 }
 
+function summarizeContentTypes(content: unknown): string | string[] {
+	if (!Array.isArray(content)) return typeof content;
+	return content.map((block) => {
+		if (block && typeof block === "object" && "type" in block) {
+			const type = (block as { type?: unknown }).type;
+			return typeof type === "string" ? type : typeof type;
+		}
+		return typeof block;
+	});
+}
+
+function summarizeAgentMessage(message: unknown): Record<string, unknown> {
+	if (!message || typeof message !== "object") return { type: typeof message };
+	const record = message as Record<string, unknown>;
+	const summary: Record<string, unknown> = {
+		role: typeof record.role === "string" ? record.role : "unknown",
+	};
+	if ("api" in record && typeof record.api === "string") summary.api = record.api;
+	if ("provider" in record && typeof record.provider === "string") summary.provider = record.provider;
+	if ("model" in record && typeof record.model === "string") summary.model = record.model;
+	if ("stopReason" in record && typeof record.stopReason === "string") summary.stopReason = record.stopReason;
+	if ("errorMessage" in record && typeof record.errorMessage === "string") summary.errorMessage = record.errorMessage;
+	if ("toolName" in record && typeof record.toolName === "string") summary.toolName = record.toolName;
+	if ("isError" in record && typeof record.isError === "boolean") summary.isError = record.isError;
+	if ("content" in record) summary.contentTypes = summarizeContentTypes(record.content);
+	return summary;
+}
+
+function finalAssistantSummary(messages: unknown): Record<string, unknown> | undefined {
+	if (!Array.isArray(messages)) return undefined;
+	for (let i = messages.length - 1; i >= 0; i--) {
+		const message = messages[i];
+		if (message && typeof message === "object" && (message as { role?: unknown }).role === "assistant") {
+			return summarizeAgentMessage(message);
+		}
+	}
+	return undefined;
+}
+
+function summarizeToolResults(toolResults: unknown): Record<string, unknown>[] {
+	if (!Array.isArray(toolResults)) return [];
+	return toolResults.map(summarizeAgentMessage);
+}
+
+function summarizeObjectKeys(value: unknown): string[] | undefined {
+	return value && typeof value === "object" ? Object.keys(value as Record<string, unknown>).sort() : undefined;
+}
+
+function logAgentLoopEvent(scope: "reflector" | "pruner", pass: number, event: AgentEvent): void {
+	switch (event.type) {
+		case "agent_start":
+		case "turn_start":
+			debugLog(`${scope}.agent_loop.${event.type}`, { pass });
+			return;
+		case "message_start":
+		case "message_end":
+			debugLog(`${scope}.agent_loop.${event.type}`, { pass, message: summarizeAgentMessage(event.message) });
+			return;
+		case "message_update":
+			debugLog(`${scope}.agent_loop.message_update`, {
+				pass,
+				message: summarizeAgentMessage(event.message),
+				assistantEventType: event.assistantMessageEvent.type,
+			});
+			return;
+		case "turn_end":
+			debugLog(`${scope}.agent_loop.turn_end`, {
+				pass,
+				message: summarizeAgentMessage(event.message),
+				toolResultCount: event.toolResults.length,
+				toolResults: summarizeToolResults(event.toolResults),
+			});
+			return;
+		case "agent_end":
+			debugLog(`${scope}.agent_loop.agent_end`, {
+				pass,
+				messageCount: event.messages.length,
+				finalAssistant: finalAssistantSummary(event.messages),
+			});
+			return;
+		case "tool_execution_start":
+			debugLog(`${scope}.agent_loop.tool_execution_start`, {
+				pass,
+				toolCallId: event.toolCallId,
+				toolName: event.toolName,
+				argsKeys: summarizeObjectKeys(event.args),
+			});
+			return;
+		case "tool_execution_update":
+			debugLog(`${scope}.agent_loop.tool_execution_update`, {
+				pass,
+				toolCallId: event.toolCallId,
+				toolName: event.toolName,
+				argsKeys: summarizeObjectKeys(event.args),
+				partialResultKeys: summarizeObjectKeys(event.partialResult),
+			});
+			return;
+		case "tool_execution_end":
+			debugLog(`${scope}.agent_loop.tool_execution_end`, {
+				pass,
+				toolCallId: event.toolCallId,
+				toolName: event.toolName,
+				isError: event.isError,
+				resultKeys: summarizeObjectKeys(event.result),
+			});
+			return;
+	}
+}
+
 export type ObservationCoverageTag = "uncited" | "cited" | "reinforced";
 
+export interface CoverageTagCounts {
+	uncited: number;
+	cited: number;
+	reinforced: number;
+}
+
 export function deriveObservationCoverageTags(
 	reflections: MemoryReflection[],
 	observations: ObservationRecord[],
@@ -57,6 +177,18 @@ export function deriveObservationCoverageTags(
 	return tags;
 }
 
+export function coverageTagCounts(
+	reflections: MemoryReflection[],
+	observations: ObservationRecord[],
+): CoverageTagCounts {
+	const tags = deriveObservationCoverageTags(reflections, observations);
+	const counts: CoverageTagCounts = { uncited: 0, cited: 0, reinforced: 0 };
+	for (const observation of observations) {
+		counts[tags.get(observation.id) ?? "uncited"]++;
+	}
+	return counts;
+}
+
 export function renderObservationsForPrunerPrompt(
 	observations: ObservationRecord[],
 	coverageTags: ReadonlyMap<string, ObservationCoverageTag>,
@@ -120,13 +252,13 @@ const RecordReflectionsSchema = Type.Object({
 			supportingObservationIds: Type.Array(
 				Type.String({
 					pattern: "^[a-f0-9]{12}$",
-					description: "Exact observation id from the current-observations list that supports this reflection.",
+					description: "Exact observation id from the current-observations list whose durable meaning is captured by this reflection.",
 				}),
 				{
 					minItems: 1,
 					description:
-						"Smallest exact set of current observation ids that directly support this reflection. " +
-						"Use only ids shown in the current observations list; never invent ids.",
+						"Current observation ids whose durable meaning is captured by this reflection and can be treated as covered active-memory detail. " +
+						"Do not include observations whose unique exact detail or current task state is not captured. Use only ids shown in the current observations list; never invent ids.",
 				},
 			),
 		}),
@@ -179,6 +311,83 @@ export interface ApplyReflectionProposalsResult {
 	unsupported: number;
 }
 
+export interface ReflectorPassStats {
+	pass: number;
+	toolCalls: number;
+	accepted: number;
+	added: number;
+	merged: number;
+	promoted: number;
+	duplicates: number;
+	unsupported: number;
+	failed: boolean;
+}
+
+export interface ReflectorStats {
+	passes: ReflectorPassStats[];
+	toolCalls: number;
+	accepted: number;
+	added: number;
+	merged: number;
+	promoted: number;
+	duplicates: number;
+	unsupported: number;
+	failedPass?: number;
+}
+
+export interface ReflectorResult {
+	reflections: MemoryReflection[];
+	stats: ReflectorStats;
+}
+
+function emptyReflectorPassStats(pass: number): ReflectorPassStats {
+	return {
+		pass,
+		toolCalls: 0,
+		accepted: 0,
+		added: 0,
+		merged: 0,
+		promoted: 0,
+		duplicates: 0,
+		unsupported: 0,
+		failed: false,
+	};
+}
+
+function addReflectionProposalStats(target: ReflectorPassStats, result: ApplyReflectionProposalsResult): void {
+	target.toolCalls++;
+	target.accepted += result.accepted;
+	target.added += result.added;
+	target.merged += result.merged;
+	target.promoted += result.promoted;
+	target.duplicates += result.duplicates;
+	target.unsupported += result.unsupported;
+}
+
+function aggregateReflectorStats(passes: ReflectorPassStats[]): ReflectorStats {
+	const stats: ReflectorStats = {
+		passes,
+		toolCalls: 0,
+		accepted: 0,
+		added: 0,
+		merged: 0,
+		promoted: 0,
+		duplicates: 0,
+		unsupported: 0,
+	};
+	for (const pass of passes) {
+		stats.toolCalls += pass.toolCalls;
+		stats.accepted += pass.accepted;
+		stats.added += pass.added;
+		stats.merged += pass.merged;
+		stats.promoted += pass.promoted;
+		stats.duplicates += pass.duplicates;
+		stats.unsupported += pass.unsupported;
+		if (pass.failed && stats.failedPass === undefined) stats.failedPass = pass.pass;
+	}
+	return stats;
+}
+
 function reflectorPassContext(pass: number): ReflectorPassContext {
 	return {
 		pass,
@@ -307,16 +516,26 @@ async function runReflectorPass(
 	reflections: MemoryReflection[],
 	observations: ObservationRecord[],
 	passContext: ReflectorPassContext,
-): Promise<{ reflections: MemoryReflection[]; failed: boolean }> {
+): Promise<{ reflections: MemoryReflection[]; stats: ReflectorPassStats }> {
 	const allowedObservationIds = observations.map((o) => o.id);
 	let currentReflections = reflections;
+	const stats = emptyReflectorPassStats(passContext.pass);
+	let consecutiveEmptyCalls = 0;
+	debugLog("reflector.pass.start", {
+		pass: passContext.pass,
+		maxPasses: passContext.maxPasses,
+		minSupportingObservationIds: passContext.minSupportingObservationIds,
+		reflectionCount: reflections.length,
+		observationCount: observations.length,
+		observationIds: isDebugLogEnabled() ? allowedObservationIds : undefined,
+	});
 
 	const recordTool: AgentTool<typeof RecordReflectionsSchema> = {
 		name: "record_reflections",
 		label: "Record reflections",
 		description:
-			"Record a batch of reflections crystallized from the observation pool. " +
-			"May be called multiple times. Stop calling when nothing more is stable enough to crystallize for this pass, " +
+			"Record a batch of reflections crystallized from the observation pool, with supporting ids for observations whose durable meaning is captured. " +
+			"May be called multiple times. Stop calling when nothing more is stable enough to crystallize or strengthen for this pass, " +
 			"then emit a short plain-text confirmation.",
 		parameters: RecordReflectionsSchema,
 		execute: async (_id, params: RecordReflectionsArgs) => {
@@ -327,6 +546,12 @@ async function runReflectorPass(
 				passContext,
 			);
 			currentReflections = result.reflections;
+			addReflectionProposalStats(stats, result);
+			if (result.accepted === 0) {
+				consecutiveEmptyCalls++;
+			} else {
+				consecutiveEmptyCalls = 0;
+			}
 			const parts: string[] = [];
 			parts.push(`Accepted ${result.accepted} reflection proposal${result.accepted === 1 ? "" : "s"}.`);
 			if (result.added) parts.push(`${result.added} new.`);
@@ -339,6 +564,20 @@ async function runReflectorPass(
 				);
 			}
 			parts.push("Call record_reflections again if more should be crystallized for this pass; otherwise stop and emit a short plain-text confirmation.");
+			debugLog("reflector.tool_call", {
+				pass: passContext.pass,
+				accepted: result.accepted,
+				added: result.added,
+				merged: result.merged,
+				promoted: result.promoted,
+				duplicates: result.duplicates,
+				unsupported: result.unsupported,
+				currentReflectionCount: currentReflections.length,
+				proposals: params.reflections.map((reflection: ReflectionProposal) => ({
+					content: reflection.content,
+					supportingObservationIds: reflection.supportingObservationIds,
+				})),
+			});
 			return {
 				content: [{ type: "text", text: parts.join(" ") }],
 				details: result,
@@ -356,7 +595,7 @@ ${joinObservationsOrEmpty(observations)}
 REFLECTOR PASS GUIDANCE:
 ${passGuidance}
 
-Crystallize long-lived reflections from the full observation pool for this pass. Call record_reflections with batches of reflection proposals, each with the exact supporting observation ids. You may call the tool multiple times as you reason through the pool. To strengthen or promote an existing reflection, repeat the exact existing reflection content with valid supporting observation ids. Do not lightly reword existing reflections. When done, stop calling the tool and emit a short plain-text confirmation.`;
+Crystallize long-lived reflections from the full observation pool for this pass. Call record_reflections with batches of reflection proposals, each with supporting observation ids whose durable meaning is captured by that reflection. You may call the tool multiple times as you reason through the pool. To strengthen or promote an existing reflection, repeat the exact existing reflection content with additional valid supporting observation ids. Do not lightly reword existing reflections. Do not attach observations whose unique exact detail or current task state is not captured with equivalent fidelity. When done, stop calling the tool and emit a short plain-text confirmation.`;
 
 	const prompts: Message[] = [
 		{
@@ -373,50 +612,114 @@ Crystallize long-lived reflections from the full observation pool for this pass.
 	};
 
 	const reasoning = (args.model as { reasoning?: unknown }).reasoning;
+	const thinkingLevel = args.thinkingLevel ?? "low";
+	const effectiveMaxTurns = args.maxTurns && args.maxTurns > 0 ? args.maxTurns : undefined;
+	let turnCount = 0;
+
 	const config: AgentLoopConfig = {
 		model: args.model as any,
 		apiKey: args.apiKey,
 		headers: args.headers,
-		maxTokens: 4096,
+		maxTokens: boundedMaxTokens(args.model, AGENT_LOOP_MAX_TOKENS),
 		convertToLlm: (msgs) => msgs as Message[],
 		toolExecution: "sequential",
-		...(reasoning ? { reasoning: "high" as const } : {}),
+		...(reasoning && thinkingLevel !== "off" ? { reasoning: thinkingLevel } : {}),
+		shouldStopAfterTurn: () => {
+			turnCount++;
+			if (effectiveMaxTurns !== undefined && turnCount >= effectiveMaxTurns) return true;
+			if (consecutiveEmptyCalls >= 2) return true;
+			return false;
+		},
 	};
 
+	let firstEventSeen = false;
 	try {
+		debugLog("reflector.agent_loop.before_call", { pass: passContext.pass });
 		const loop = args.agentLoop ?? agentLoop;
 		const stream = loop(prompts, context, config, args.signal);
-		for await (const _event of stream) {
-			// Drain events; the tool's execute already updates reflections.
+		debugLog("reflector.agent_loop.stream_created", { pass: passContext.pass });
+		for await (const event of stream) {
+			if (!firstEventSeen) {
+				firstEventSeen = true;
+				debugLog("reflector.agent_loop.first_event", { pass: passContext.pass, type: event.type });
+			}
+			logAgentLoopEvent("reflector", passContext.pass, event);
+			args.onEvent?.(event);
 		}
 		await stream.result();
-	} catch {
-		return { reflections: currentReflections, failed: true };
+		debugLog("reflector.pass.result", { pass: passContext.pass, stats, reflectionCount: currentReflections.length });
+	} catch (error) {
+		stats.failed = true;
+		debugLog("reflector.agent_loop.error", {
+			pass: passContext.pass,
+			firstEventSeen,
+			errorMessage: error instanceof Error ? error.message : String(error),
+		});
+		return { reflections: currentReflections, stats };
 	}
 
-	return { reflections: currentReflections, failed: false };
+	return { reflections: currentReflections, stats };
 }
 
 export async function runReflector(
 	args: LlmArgs,
 	reflections: MemoryReflection[],
 	observations: ObservationRecord[],
-): Promise<MemoryReflection[]> {
+	onPassStart?: (pass: number, maxPasses: number) => void,
+): Promise<ReflectorResult> {
+	debugLog("reflector.start", {
+		reflectionCount: reflections.length,
+		observationCount: observations.length,
+		observations: isDebugLogEnabled()
+			? observations.map((observation) => ({
+				id: observation.id,
+				timestamp: observation.timestamp,
+				relevance: observation.relevance,
+				content: observation.content,
+				sourceEntryIds: observation.sourceEntryIds,
+			}))
+			: undefined,
+	});
 	let currentReflections = reflections;
+	const passes: ReflectorPassStats[] = [];
 
 	for (let pass = 1; pass <= REFLECTOR_MAX_PASSES; pass++) {
+		onPassStart?.(pass, REFLECTOR_MAX_PASSES);
 		const result = await runReflectorPass(args, currentReflections, observations, reflectorPassContext(pass));
 		currentReflections = result.reflections;
-		if (result.failed) break;
+		passes.push(result.stats);
+		if (result.stats.failed) break;
 	}
 
-	return currentReflections;
+	const result = { reflections: currentReflections, stats: aggregateReflectorStats(passes) };
+	debugLog("reflector.result", {
+		stats: result.stats,
+		reflectionCount: result.reflections.length,
+		reflections: isDebugLogEnabled()
+			? result.reflections.map((reflection) => typeof reflection === "string" ? { legacyString: true, content: reflection } : reflection)
+			: undefined,
+	});
+	return result;
+}
+
+export type PrunerStopReason = "empty" | "under_target" | "fell_back" | "zero_drops" | "max_passes";
+
+export interface PrunerPassStats {
+	pass: number;
+	poolTokens: number;
+	targetTokens: number;
+	deltaTokens: number;
+	dropped: number;
+	remaining: number;
+	fellBack: boolean;
 }
 
 export interface PrunerResult {
 	observations: ObservationRecord[];
 	droppedIds: string[];
 	fellBack: boolean;
+	passes: PrunerPassStats[];
+	stopReason: PrunerStopReason;
 }
 
 const DropObservationsSchema = Type.Object({
@@ -460,6 +763,17 @@ async function runPrunerPass(
 ): Promise<PrunerPassResult> {
 	const idSet = new Set(observations.map((o) => o.id));
 	const dropped = new Set<string>();
+	let consecutiveEmptyCalls = 0;
+	debugLog("pruner.pass.start", {
+		pass: passContext.pass,
+		maxPasses: passContext.maxPasses,
+		poolTokens: passContext.poolTokens,
+		targetTokens: passContext.targetTokens,
+		deltaTokens: passContext.deltaTokens,
+		observationCount: observations.length,
+		reflectionCount: reflections.length,
+		observationIds: isDebugLogEnabled() ? observations.map((observation) => observation.id) : undefined,
+	});
 
 	const dropTool: AgentTool<typeof DropObservationsSchema> = {
 		name: "drop_observations",
@@ -484,6 +798,11 @@ async function runPrunerPass(
 				dropped.add(id);
 				valid.push(id);
 			}
+			if (valid.length === 0) {
+				consecutiveEmptyCalls++;
+			} else {
+				consecutiveEmptyCalls = 0;
+			}
 			const remaining = idSet.size - dropped.size;
 			const parts: string[] = [];
 			parts.push(`Dropped ${valid.length} observation${valid.length === 1 ? "" : "s"}.`);
@@ -491,6 +810,15 @@ async function runPrunerPass(
 			if (already.length) parts.push(`Already dropped: ${already.join(", ")}.`);
 			parts.push(`Remaining kept: ${remaining} of ${idSet.size}.`);
 			parts.push("Call drop_observations again if more should be removed; otherwise stop and emit a short plain-text confirmation.");
+			debugLog("pruner.tool_call", {
+				pass: passContext.pass,
+				requestedIds: params.ids,
+				dropped: valid,
+				unknown,
+				already,
+				remaining,
+				reason: params.reason,
+			});
 			return {
 				content: [{ type: "text", text: parts.join(" ") }],
 				details: { dropped: valid, unknown, already, remaining },
@@ -532,29 +860,68 @@ Decide which observations to remove from the kept set. Call drop_observations wi
 	};
 
 	const reasoning = (args.model as { reasoning?: unknown }).reasoning;
+	const thinkingLevel = args.thinkingLevel ?? "low";
+	const effectiveMaxTurns = args.maxTurns && args.maxTurns > 0 ? args.maxTurns : undefined;
+	let turnCount = 0;
+
 	const config: AgentLoopConfig = {
 		model: args.model as any,
 		apiKey: args.apiKey,
 		headers: args.headers,
-		maxTokens: 2048,
+		maxTokens: boundedMaxTokens(args.model, AGENT_LOOP_MAX_TOKENS),
 		convertToLlm: (msgs) => msgs as Message[],
 		toolExecution: "sequential",
-		...(reasoning ? { reasoning: "high" as const } : {}),
+		...(reasoning && thinkingLevel !== "off" ? { reasoning: thinkingLevel } : {}),
+		shouldStopAfterTurn: () => {
+			turnCount++;
+			if (effectiveMaxTurns !== undefined && turnCount >= effectiveMaxTurns) return true;
+			if (consecutiveEmptyCalls >= 2) return true;
+			return false;
+		},
 	};
 
+	let agentLoopCalled = false;
+	let streamCreated = false;
+	let firstEventSeen = false;
 	try {
+		debugLog("pruner.agent_loop.before_call", { pass: passContext.pass });
 		const loop = args.agentLoop ?? agentLoop;
+		agentLoopCalled = true;
 		const stream = loop(prompts, context, config, args.signal);
-		for await (const _event of stream) {
-			// Drain events; the tool's execute already records drops.
+		streamCreated = true;
+		debugLog("pruner.agent_loop.stream_created", { pass: passContext.pass });
+		for await (const event of stream) {
+			if (!firstEventSeen) {
+				firstEventSeen = true;
+				debugLog("pruner.agent_loop.first_event", { pass: passContext.pass, type: event.type });
+			}
+			logAgentLoopEvent("pruner", passContext.pass, event);
+			args.onEvent?.(event);
 		}
 		await stream.result();
-	} catch {
+	} catch (error) {
+		debugLog("pruner.agent_loop.error", {
+			pass: passContext.pass,
+			agentLoopCalled,
+			streamCreated,
+			firstEventSeen,
+			errorMessage: error instanceof Error ? error.message : String(error),
+		});
 		return { kept: observations, droppedIds: [], fellBack: true };
 	}
 
 	const kept = observations.filter((o) => !dropped.has(o.id));
-	return { kept, droppedIds: Array.from(dropped), fellBack: false };
+	const droppedIds = Array.from(dropped);
+	debugLog("pruner.pass.result", {
+		pass: passContext.pass,
+		droppedIds,
+		dropped: droppedIds.length,
+		remaining: kept.length,
+		agentLoopCalled,
+		streamCreated,
+		firstEventSeen,
+	});
+	return { kept, droppedIds, fellBack: false };
 }
 
 export async function runPruner(
@@ -562,21 +929,44 @@ export async function runPruner(
 	reflections: MemoryReflection[],
 	observations: ObservationRecord[],
 	budgetTokens: number,
+	onPassStart?: (pass: number, maxPasses: number) => void,
 ): Promise<PrunerResult> {
+	debugLog("pruner.start", {
+		reflectionCount: reflections.length,
+		observationCount: observations.length,
+		budgetTokens,
+	});
 	if (observations.length === 0) {
-		return { observations: [], droppedIds: [], fellBack: false };
+		const result: PrunerResult = { observations: [], droppedIds: [], fellBack: false, passes: [], stopReason: "empty" };
+		debugLog("pruner.result", {
+			stopReason: result.stopReason,
+			fellBack: result.fellBack,
+			droppedIds: result.droppedIds,
+			dropped: result.droppedIds.length,
+			passes: result.passes,
+			finalObservationCount: result.observations.length,
+		});
+		return result;
 	}
 
 	const target = Math.max(1, Math.floor(budgetTokens * PRUNER_TARGET_RATIO));
-	let pool = observations;
 	const coverageTags = deriveObservationCoverageTags(reflections, observations);
+	let pool = observations;
+
 	const allDropped: string[] = [];
+	const passes: PrunerPassStats[] = [];
 	let fellBack = false;
+	let stopReason: PrunerStopReason | undefined;
 
 	for (let pass = 1; pass <= PRUNER_MAX_PASSES; pass++) {
 		const poolTokens = observationPoolTokens(pool);
-		if (poolTokens <= target) break;
+		if (poolTokens <= target) {
+			stopReason = "under_target";
+			debugLog("pruner.under_target", { pass, poolTokens, targetTokens: target, observationCount: pool.length });
+			break;
+		}
 
+		onPassStart?.(pass, PRUNER_MAX_PASSES);
 		const deltaTokens = poolTokens - target;
 		const result = await runPrunerPass(args, reflections, pool, {
 			poolTokens,
@@ -586,18 +976,41 @@ export async function runPruner(
 			maxPasses: PRUNER_MAX_PASSES,
 			coverageTags,
 		});
+		passes.push({
+			pass,
+			poolTokens,
+			targetTokens: target,
+			deltaTokens,
+			dropped: result.droppedIds.length,
+			remaining: result.kept.length,
+			fellBack: result.fellBack,
+		});
 
 		if (result.fellBack) {
 			fellBack = true;
+			stopReason = "fell_back";
+			break;
+		}
+		if (result.droppedIds.length === 0) {
+			stopReason = "zero_drops";
 			break;
 		}
-		if (result.droppedIds.length === 0) break;
 
 		pool = result.kept;
 		allDropped.push(...result.droppedIds);
 	}
 
-	return { observations: pool, droppedIds: allDropped, fellBack };
+	stopReason ??= observationPoolTokens(pool) <= target ? "under_target" : "max_passes";
+	const result = { observations: pool, droppedIds: allDropped, fellBack, passes, stopReason };
+	debugLog("pruner.result", {
+		stopReason: result.stopReason,
+		fellBack: result.fellBack,
+		droppedIds: result.droppedIds,
+		dropped: result.droppedIds.length,
+		passes: result.passes,
+		finalObservationCount: result.observations.length,
+	});
+	return result;
 }
 
 export function renderSummary(reflections: MemoryReflection[], observations: ObservationRecord[]): string {