pi-observational-memory 3.0.2 → 3.0.4

package/README.md

@@ -208,6 +208,8 @@ A typical config:
     "observeAfterTokens": 10000,
     "reflectAfterTokens": 20000,
     "compactAfterTokens": 81000,
+    "compactAfterTokensMode": "calibrated",
+    "compactAfterTokensRatio": 0.68,
     "observationsPoolMaxTokens": 20000,
     "observationsPoolTargetTokens": 10000,
     "agentMaxTurns": 16,
@@ -216,6 +218,7 @@ A typical config:
       "id": "google/gemma-4-31b-it",
       "thinking": "low"
     },
+    "showWorkerNotifications": true,
     "passive": false,
     "debugLog": false
   }
@@ -224,17 +227,61 @@ A typical config:
 
 Most users can start with the defaults and tune only if they have a specific reason.
 
+### Scaling compaction to the model's context window
+
+By default `compactAfterTokensMode` is `"calibrated"`, so the proactive
+compaction trigger uses the fixed `compactAfterTokens` estimated source-entry
+threshold (81,000 by default). This preserves the pre-PR #40 compaction metric
+for typical ~128K–200K context models.
+
+On a large-context model (e.g. 1M tokens) the calibrated default preempts
+compaction at ~81K, wasting most of the window. Switch to `"ratio"` mode to let
+the trigger scale with the active model's `contextWindow`:
+
+```json
+{
+  "observational-memory": {
+    "compactAfterTokens": 81000,
+    "compactAfterTokensMode": "ratio",
+    "compactAfterTokensRatio": 0.5
+  }
+}
+```
+
+In ratio mode the effective threshold is
+`floor(model.contextWindow * compactAfterTokensRatio)` (clamped to a minimum of
+1). With the example above, a 1,000,000-token window compacts after about
+500,000 estimated source-entry tokens after the latest compaction boundary; a
+200,000-token window uses about 100,000. The threshold counts source entries,
+not Pi's system prompt, tool schemas, or provider accounting. Pi's native
+window-pressure compaction remains independent.
+
+`compactAfterTokensRatio` is user-tunable precisely because **context window ≠
+attention**. Some models advertise a large window but degrade at long range; set
+a lower ratio (e.g. `0.4`) to compact earlier on those, or a higher ratio
+(e.g. `0.7`) on models that stay sharp. The default ratio is `0.68`.
+
+`compactAfterTokens` is always retained as the fallback: in `"calibrated"`
+mode it is the threshold directly, and in `"ratio"` mode it is used whenever
+the active model's `contextWindow` is unavailable (undefined, 0, or negative),
+so compaction still triggers safely. `/om:status` shows the resolved threshold
+on the `Next compaction` line regardless of mode.
+
 ### Defaults
 
 | Setting                     | Default       | Meaning                                                                                           |
 | --------------------------- | ------------- | ------------------------------------------------------------------------------------------------- |
 | `observeAfterTokens`        | `10000`       | Raw/source token threshold for observation runs.                                                  |
+| `observerChunkMaxTokens`    | derived       | Max estimated tokens serialized into one observer chunk (minimum `256`). Unset: `floor(contextWindow * 0.2)` of the resolved memory model, or `60000` when the window is unknown. Larger backlogs drain oldest-first; a single over-budget source is sent as a marked head/tail excerpt while the original source remains in the session ledger. |
 | `reflectAfterTokens`        | `20000`       | Raw/source token threshold for reflection runs; successful reflection creates dropper opportunities. |
-| `compactAfterTokens`        | `81000`       | Raw/source token threshold for proactive auto-compaction.                                         |
+| `compactAfterTokens`        | `81000`       | Estimated source-entry threshold for proactive auto-compaction, counted after the latest compaction boundary. |
+| `compactAfterTokensMode`    | `"calibrated"`| `"calibrated"` uses `compactAfterTokens` directly. `"ratio"` scales the source-entry threshold by the active model's `contextWindow`. |
+| `compactAfterTokensRatio`   | `0.68`        | In `"ratio"` mode, the threshold is `floor(contextWindow * ratio)`. Tunable because large windows do not always mean strong long-range attention. Must be in `(0, 1)`. |
 | `observationsPoolMaxTokens` | `20000`       | Observation-token budget used for compaction full-fold pressure.                                  |
 | `observationsPoolTargetTokens` | half of max | Active observation target used by post-reflection dropper maintenance.                            |
 | `agentMaxTurns`             | `16`          | Shared turn cap for background memory-agent loops.                                                |
 | `model`                     | session model | Optional memory-worker model override: `{ provider, id, thinking }`.                              |
+| `showWorkerNotifications`   | `true`        | Shows routine observer, reflector, and dropper progress notifications. Warnings and errors are unaffected. |
 | `passive`                   | `false`       | Disables proactive background observation, reflection, maintenance, and auto-compaction triggers. |
 | `debugLog`                  | `false`       | Writes opt-in per-session extension debug events to Pi's agent directory.                         |
 
@@ -246,9 +293,12 @@ Valid `model.thinking` values are:
 * `medium`
 * `high`
 * `xhigh`
+* `max`
 
 If no `model` is configured, memory workers use the session model.
 
+Set `showWorkerNotifications` to `false` to hide routine worker start and completion messages (including deliberate-empty observer info messages). Model fallback/unavailability, worker failures (including observer stream errors), compaction notifications, and explicit `/om:*` command output remain visible.
+
 `observationsPoolMaxTokens` and `observationsPoolTargetTokens` intentionally describe different pools. Max tokens control when compaction performs a full fold over visible memory. Target tokens control the folded active observation pool that the dropper maintains after successful reflection. If the target is omitted, it defaults to half of max.
 
 Dropper pruning balances age, relevance, and reflection coverage. Relevance is importance/resistance, not a permanent active-memory pin: `critical` observations require the strongest evidence but can be dropped when they are older and safely represented by reflections, superseded by newer memory, redundant, or obsolete. Dropper input annotates each active observation with deterministic coverage evidence: `none`, `partial`, or `strong`; coverage guides model judgment and is not an automatic drop rule. Dropping removes observations from active memory, not ledger history.
@@ -299,6 +349,12 @@ The high-level lifecycle:
 
 The important part: compaction does not need to rethink the whole session from scratch.
 
+The proactive compaction threshold counts estimated source-entry tokens after
+the latest compaction boundary. It includes source entries retained by
+`firstKeptEntryId` and newer source entries, while memory ledger entries and
+compaction metadata contribute zero. `/om:status` uses the same metric. Pi's
+own window-pressure compaction remains independent.
+
 ---
 
 ## Current V3 behavior

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-observational-memory",
-  "version": "3.0.2",
+  "version": "3.0.4",
   "description": "Observational memory extension for pi — cache-friendly tiered compaction with observations and reflections.",
   "type": "module",
   "license": "MIT",
@@ -40,10 +40,10 @@
     "@earendil-works/pi-tui": "*"
   },
   "devDependencies": {
-    "@earendil-works/pi-agent-core": "^0.74.0",
-    "@earendil-works/pi-ai": "^0.74.0",
-    "@earendil-works/pi-coding-agent": "^0.74.0",
-    "@earendil-works/pi-tui": "^0.74.0",
+    "@earendil-works/pi-agent-core": "^0.81.0",
+    "@earendil-works/pi-ai": "^0.81.0",
+    "@earendil-works/pi-coding-agent": "^0.81.0",
+    "@earendil-works/pi-tui": "^0.81.0",
     "@types/node": "^22.0.0",
     "typebox": "^1.1.38",
     "typescript": "^5.6.0",

package/src/agents/dropper/agent.ts

@@ -1,9 +1,11 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { Type } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import type { Static } from "typebox";
 import { debugLog } from "../../debug-log.js";
 import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
+import { logAgentStreamError } from "../stream-errors.js";
 import { reflectionToSummaryLine, type Observation, type Reflection } from "../../session-ledger/index.js";
 import { DROPPER_SYSTEM } from "./prompts.js";
 import {
@@ -37,7 +39,7 @@ export type { CoverageSummaryByRelevance, CoverageTransitionSummaryByRelevance,
 
 interface RunDropperArgs {
 	model: Model<any>;
-	apiKey: string;
+	apiKey?: string;
 	headers?: Record<string, string>;
 	reflections: Reflection[];
 	observations: Observation[];
@@ -249,9 +251,10 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
 	};
 
 	const loop = args.agentLoop ?? agentLoop;
-	const stream = loop(prompts, context, config, signal);
-	for await (const _event of stream) {
+	const stream = loop(prompts, context, config, signal, streamSimple);
+	for await (const event of stream) {
 		// Tool execution collects candidate ids.
+		logAgentStreamError("dropper", event);
 	}
 	await stream.result();
 	const droppedIds = selectDropCandidates(proposedDropIds, observations, maxDropsAllowed, reflections);

package/src/agents/dropper/pool.ts

@@ -1,3 +1,4 @@
+import { observationLineTokenCount } from "../../tokens.js";
 import type { Observation } from "../../session-ledger/index.js";
 
 export type ObservationPoolMetrics = {
@@ -12,8 +13,11 @@ export type ObservationPoolMetrics = {
 	ready: boolean;
 };
 
-export function observationTokenSum(observations: readonly { tokenCount: number }[]): number {
-	return observations.reduce((sum, observation) => sum + observation.tokenCount, 0);
+export function observationTokenSum(observations: readonly Observation[]): number {
+	// Count the full rendered line (id + timestamp + relevance + content), not
+	// bare content: the pool budget caps how much observation text is re-rendered
+	// into future contexts, and every line carries metadata overhead.
+	return observations.reduce((sum, observation) => sum + observationLineTokenCount(observation), 0);
 }
 
 export function observationPoolFullness(observationTokens: number, targetTokens: number): number {

package/src/agents/observer/agent.ts

@@ -1,17 +1,19 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { Type } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";
+import { logAgentStreamError } from "../stream-errors.js";
 import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
 import { OBSERVER_SYSTEM } from "./prompts.js";
 import { nowTimestamp, truncateRecordContent } from "../../serialize.js";
 import type { Observation, Relevance } from "../../session-ledger/index.js";
-import { estimateStringTokens } from "../../tokens.js";
+import { observationLineTokenCount } from "../../tokens.js";
 
 interface RunObserverArgs {
 	model: Model<any>;
-	apiKey: string;
+	apiKey?: string;
 	headers?: Record<string, string>;
 	priorReflections: string[];
 	priorObservations: string[];
@@ -60,6 +62,21 @@ const RecordObservationsSchema = Type.Object({
 
 type RecordObservationsArgs = Static<typeof RecordObservationsSchema>;
 
+/**
+ * Thrown when the agent loop ends with an API/stream failure (`stopReason`
+ * `"error"`/`"aborted"`) without recording anything. agent-core returns such
+ * runs normally, so without this the caller cannot tell a hard failure from a
+ * deliberate empty result (#32).
+ */
+export class ObserverStreamError extends Error {
+	readonly stopReason: string;
+	constructor(stopReason: string, errorMessage?: string) {
+		super(`observer stream ended with stopReason "${stopReason}"${errorMessage ? `: ${errorMessage}` : ""}`);
+		this.name = "ObserverStreamError";
+		this.stopReason = stopReason;
+	}
+}
+
 function joinOrEmpty(items: string[]): string {
 	return items.length ? items.join("\n") : "(none yet)";
 }
@@ -118,7 +135,12 @@ export async function runObserver(args: RunObserverArgs): Promise<Observation[]
 					timestamp: obs.timestamp,
 					relevance: obs.relevance as Relevance,
 					sourceEntryIds,
-					tokenCount: estimateStringTokens(content),
+					tokenCount: observationLineTokenCount({
+						id,
+						timestamp: obs.timestamp,
+						relevance: obs.relevance,
+						content,
+					}),
 				});
 				added++;
 			}
@@ -186,12 +208,23 @@ ${conversation}`;
 	};
 
 	const loop = args.agentLoop ?? agentLoop;
-	const stream = loop(prompts, context, config, signal);
-	for await (const _event of stream) {
+	const stream = loop(prompts, context, config, signal, streamSimple);
+	let streamError: { stopReason: string; errorMessage?: string } | undefined;
+	for await (const event of stream) {
 		// Drain events; the tool's execute already collects records.
+		logAgentStreamError("observer", event);
+		// Watch for a terminal API/stream failure so it is not conflated with
+		// a deliberate empty result.
+		const message = (event as { message?: { role?: string; stopReason?: string; errorMessage?: string } }).message;
+		if (message?.role === "assistant" && (message.stopReason === "error" || message.stopReason === "aborted")) {
+			streamError = { stopReason: message.stopReason, errorMessage: message.errorMessage };
+		}
 	}
 	await stream.result();
 
-	if (accumulated.size === 0) return undefined;
+	if (accumulated.size === 0) {
+		if (streamError) throw new ObserverStreamError(streamError.stopReason, streamError.errorMessage);
+		return undefined;
+	}
 	return Array.from(accumulated.values());
 }

package/src/agents/reflector/agent.ts

@@ -1,9 +1,11 @@
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
 import { Type } from "@earendil-works/pi-ai";
+import { streamSimple } from "@earendil-works/pi-ai/compat";
 import type { Static } from "typebox";
 import { debugLog } from "../../debug-log.js";
 import { hashId } from "../../ids.js";
+import { logAgentStreamError } from "../stream-errors.js";
 import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
 import { truncateRecordContent } from "../../serialize.js";
 import { REFLECTOR_SYSTEM } from "./prompts.js";
@@ -19,7 +21,7 @@ import {
 
 interface RunReflectorArgs {
 	model: Model<any>;
-	apiKey: string;
+	apiKey?: string;
 	headers?: Record<string, string>;
 	reflections: Reflection[];
 	observations: Observation[];
@@ -182,9 +184,10 @@ export async function runReflector(args: RunReflectorArgs): Promise<Reflection[]
 	};
 
 	const loop = args.agentLoop ?? agentLoop;
-	const stream = loop(prompts, context, config, signal);
-	for await (const _event of stream) {
+	const stream = loop(prompts, context, config, signal, streamSimple);
+	for await (const event of stream) {
 		// Tool execution collects records.
+		logAgentStreamError("reflector", event);
 	}
 	await stream.result();
 	const acceptedReflections = Array.from(accumulated.values());

package/src/agents/stream-errors.ts

@@ -0,0 +1,22 @@
+import type { AgentEvent } from "@earendil-works/pi-agent-core";
+import { debugLog } from "../debug-log.js";
+
+/**
+ * Surface LLM failures from an agent-loop event stream.
+ *
+ * When the underlying LLM call fails, the loop ends the stream with a final
+ * assistant message whose stopReason is "error" (or "aborted") — no exception
+ * is thrown. Without this hook the drain loops treat such runs exactly like
+ * "the model chose not to call the tool", which hides the real cause
+ * (rate limits, oversized prompts, auth failures, ...) from the debug log.
+ */
+export function logAgentStreamError(stage: "observer" | "reflector" | "dropper", event: AgentEvent): void {
+	if (event.type !== "message_end") return;
+	const message = event.message;
+	if (message.role !== "assistant") return;
+	if (message.stopReason !== "error" && message.stopReason !== "aborted") return;
+	debugLog(`${stage}.stream_error`, {
+		stopReason: message.stopReason,
+		errorMessage: message.errorMessage,
+	});
+}

package/src/commands/status.ts

@@ -1,5 +1,6 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { observationPoolMetrics } from "../agents/dropper/pool.js";
+import { resolveCompactAfterTokens } from "../config.js";
 import type { Runtime } from "../runtime.js";
 import {
 	diffProjection,
@@ -61,6 +62,8 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
 			const obsProgress = rawTokensSinceObservationCoverage(entries);
 			const reflectionProgress = rawTokensSinceReflectionCoverage(entries);
 			const compactionProgress = rawTokensSinceLastCompaction(entries);
+			const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : undefined;
+			const compactThreshold = resolveCompactAfterTokens(runtime.config, contextWindow);
 
 			const passiveLines = runtime.config.passive === true
 				? [
@@ -79,7 +82,7 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
 				"── Activity ──",
 				`Next observation: ~${obsProgress.toLocaleString()} / ${runtime.config.observeAfterTokens.toLocaleString()} tokens (${pct(obsProgress, runtime.config.observeAfterTokens)}%)`,
 				`Next reflection:  ~${reflectionProgress.toLocaleString()} / ${runtime.config.reflectAfterTokens.toLocaleString()} tokens (${pct(reflectionProgress, runtime.config.reflectAfterTokens)}%)`,
-				`Next compaction:  ~${compactionProgress.toLocaleString()} / ${runtime.config.compactAfterTokens.toLocaleString()} tokens (${pct(compactionProgress, runtime.config.compactAfterTokens)}%)`,
+				`Next compaction:  ~${compactionProgress.toLocaleString()} / ${compactThreshold.toLocaleString()} estimated source tokens (${pct(compactionProgress, compactThreshold)}%)`,
 				`Visible observation pool: ~${visibleObservationTokens.toLocaleString()} / ${runtime.config.observationsPoolMaxTokens.toLocaleString()} tokens (${pct(visibleObservationTokens, runtime.config.observationsPoolMaxTokens)}%)`,
 				`Active observation pool: ~${activeObservationPool.observationTokens.toLocaleString()} / ${runtime.config.observationsPoolTargetTokens.toLocaleString()} target tokens (${pct(activeObservationPool.observationTokens, runtime.config.observationsPoolTargetTokens)}%)`,
 				`Reflection pool:         ~${visibleReflectionTokens.toLocaleString()} tokens`,

package/src/config.ts

@@ -9,14 +9,44 @@ export interface ConfiguredModel {
 	thinking?: ModelThinkingLevel;
 }
 
+/**
+ * How `compactAfterTokens` is interpreted.
+ *
+ * - `"calibrated"` (default): use the static `compactAfterTokens` value directly.
+ *   Backwards-compatible with all existing V3 configs.
+ *
+ * - `"ratio"`: compute the effective threshold as
+ *   `floor(model.contextWindow * compactAfterTokensRatio)`. This auto-scales the
+ *   proactive compaction trigger to the active model's context window, so a 1M
+ *   context model is not preempted at the same 81K threshold as a 128K model.
+ *
+ *   Some models advertise a large context window but lose attention at long
+ *   range; users can lower `compactAfterTokensRatio` to compact earlier on such
+ *   models without giving up the window on models that stay sharp.
+ *
+ *   When the active model's `contextWindow` is unavailable (undefined, 0, or
+ *   negative), ratio mode falls back to the calibrated `compactAfterTokens`
+ *   value so compaction still triggers safely.
+ */
+export type CompactAfterTokensMode = "calibrated" | "ratio";
+
 export interface Config {
 	observeAfterTokens: number;
 	reflectAfterTokens: number;
+	/**
+	 * Maximum estimated source tokens serialized into a single observer chunk.
+	 * Unset (default) derives the cap from the resolved memory model's context
+	 * window; see {@link resolveObserverChunkMaxTokens}.
+	 */
+	observerChunkMaxTokens?: number;
 	compactAfterTokens: number;
+	compactAfterTokensMode: CompactAfterTokensMode;
+	compactAfterTokensRatio: number;
 	observationsPoolMaxTokens: number;
 	observationsPoolTargetTokens: number;
 	agentMaxTurns: number;
 	model?: ConfiguredModel;
+	showWorkerNotifications: boolean;
 	passive: boolean;
 	debugLog: boolean;
 }
@@ -25,14 +55,78 @@ export const DEFAULTS: Config = {
 	observeAfterTokens: 10_000,
 	reflectAfterTokens: 20_000,
 	compactAfterTokens: 81_000,
+	compactAfterTokensMode: "calibrated",
+	compactAfterTokensRatio: 0.68,
 	observationsPoolMaxTokens: 20_000,
 	observationsPoolTargetTokens: 10_000,
 	agentMaxTurns: 16,
+	showWorkerNotifications: true,
 	passive: false,
 	debugLog: false,
 };
 
-export const THINKING_LEVEL_VALUES: readonly ModelThinkingLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh"] as const;
+export const COMPACT_AFTER_TOKENS_MODE_VALUES: readonly CompactAfterTokensMode[] = ["calibrated", "ratio"] as const;
+
+/**
+ * Resolve the effective proactive-compaction token threshold for the given
+ * config and active model context window.
+ *
+ * In `"calibrated"` mode this is always `config.compactAfterTokens`.
+ *
+ * In `"ratio"` mode this is `floor(contextWindow * compactAfterTokensRatio)`
+ * (clamped to a minimum of 1) when `contextWindow` is a positive number, and
+ * falls back to `config.compactAfterTokens` otherwise.
+ */
+export function resolveCompactAfterTokens(config: Config, contextWindow: number | undefined): number {
+	if (config.compactAfterTokensMode === "ratio" && typeof contextWindow === "number" && contextWindow > 0) {
+		return Math.max(1, Math.floor(contextWindow * config.compactAfterTokensRatio));
+	}
+	return config.compactAfterTokens;
+}
+
+export const THINKING_LEVEL_VALUES: readonly ModelThinkingLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh", "max"] as const;
+
+/** Observer chunk cap used when no config is set and the model's context window is unknown. */
+export const OBSERVER_CHUNK_FALLBACK_MAX_TOKENS = 60_000;
+
+/** Smallest useful observer chunk: enough for labels, omission markers, and source context. */
+export const OBSERVER_CHUNK_MIN_TOKENS = 256;
+
+/**
+ * Fraction of the memory model's context window used for the derived observer
+ * chunk cap. Chunk sizes are estimated at ~4 chars/token, which can undercount
+ * real tokens by up to ~4x on non-ASCII content, so 0.2 keeps even the worst
+ * case at ~80% of the window with room left for the system prompt, prior
+ * memory, and the response.
+ */
+export const OBSERVER_CHUNK_CONTEXT_RATIO = 0.2;
+
+/**
+ * Resolve the maximum estimated tokens the observer serializes into one chunk.
+ *
+ * An explicit `observerChunkMaxTokens` config value always wins. Otherwise the
+ * cap is `floor(contextWindow * OBSERVER_CHUNK_CONTEXT_RATIO)` for the resolved
+ * memory model, falling back to {@link OBSERVER_CHUNK_FALLBACK_MAX_TOKENS} when
+ * the context window is unavailable.
+ *
+ * Without a cap, a backlog that outgrows the model's context window (e.g.
+ * after repeated observer failures, or when the extension is enabled mid-way
+ * into a long session) makes every observer call fail, so coverage never
+ * advances and the session can never recover. With the cap, oversized backlogs
+ * are drained oldest-first across successive runs.
+ */
+export function resolveObserverChunkMaxTokens(config: Config, contextWindow: number | undefined): number {
+	if (config.observerChunkMaxTokens !== undefined && config.observerChunkMaxTokens > 0) {
+		return Math.max(OBSERVER_CHUNK_MIN_TOKENS, config.observerChunkMaxTokens);
+	}
+	if (typeof contextWindow === "number" && Number.isFinite(contextWindow) && contextWindow > 0) {
+		return Math.max(
+			OBSERVER_CHUNK_MIN_TOKENS,
+			Math.floor(contextWindow * OBSERVER_CHUNK_CONTEXT_RATIO),
+		);
+	}
+	return OBSERVER_CHUNK_FALLBACK_MAX_TOKENS;
+}
 
 const SETTINGS_KEY = "observational-memory";
 const PASSIVE_ENV = "PI_OBSERVATIONAL_MEMORY_PASSIVE";
@@ -54,6 +148,19 @@ function isThinkingLevel(value: unknown): value is ModelThinkingLevel {
 	return typeof value === "string" && (THINKING_LEVEL_VALUES as readonly string[]).includes(value);
 }
 
+function isCompactAfterTokensMode(value: unknown): value is CompactAfterTokensMode {
+	return typeof value === "string" && (COMPACT_AFTER_TOKENS_MODE_VALUES as readonly string[]).includes(value);
+}
+
+/**
+ * A valid ratio is a finite number strictly between 0 and 1.
+ * 0 would never trigger; >= 1 would compact at/after the full window with no
+ * room left for the response.
+ */
+function validRatioOrUndefined(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isFinite(value) && value > 0 && value < 1 ? value : undefined;
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === "object" && value !== null;
 }
@@ -77,6 +184,7 @@ function normalizeSettingsConfig(value: Record<string, unknown>): Partial<Config
 	const numberKeys = [
 		"observeAfterTokens",
 		"reflectAfterTokens",
+		"observerChunkMaxTokens",
 		"compactAfterTokens",
 		"observationsPoolMaxTokens",
 		"observationsPoolTargetTokens",
@@ -86,6 +194,12 @@ function normalizeSettingsConfig(value: Record<string, unknown>): Partial<Config
 		const normalizedValue = positiveIntegerOrUndefined(value[key]);
 		if (normalizedValue !== undefined) normalized[key] = normalizedValue;
 	}
+	if (isCompactAfterTokensMode(value.compactAfterTokensMode)) {
+		normalized.compactAfterTokensMode = value.compactAfterTokensMode;
+	}
+	const ratio = validRatioOrUndefined(value.compactAfterTokensRatio);
+	if (ratio !== undefined) normalized.compactAfterTokensRatio = ratio;
+	if (typeof value.showWorkerNotifications === "boolean") normalized.showWorkerNotifications = value.showWorkerNotifications;
 	if (typeof value.passive === "boolean") normalized.passive = value.passive;
 	if (typeof value.debugLog === "boolean") normalized.debugLog = value.debugLog;
 	const model = normalizeModel(value.model);

package/src/hooks/compaction-trigger.ts

@@ -1,4 +1,5 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { resolveCompactAfterTokens } from "../config.js";
 import { rawTokensSinceLastCompaction, type Entry } from "../session-ledger/index.js";
 import type { Runtime } from "../runtime.js";
 
@@ -31,9 +32,12 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 			return;
 		}
 
-		const entries = ctx.sessionManager.getBranch() as Entry[];
-		const tokens = rawTokensSinceLastCompaction(entries);
-		if (tokens < runtime.config.compactAfterTokens) return;
+		const entries = ctx.sessionManager?.getBranch?.() as Entry[] | undefined;
+		if (!entries) return;
+		const progress = rawTokensSinceLastCompaction(entries);
+		const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : undefined;
+		const threshold = resolveCompactAfterTokens(runtime.config, contextWindow);
+		if (progress < threshold) return;
 
 		// Capture ctx properties synchronously — the setTimeout + async work below
 		// may outlive the extension ctx (stale after session replacement/reload).
@@ -41,7 +45,7 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 		const ui = ctx.ui;
 
 		if (hasUI) ui?.notify(
-			`Observational memory: compaction threshold reached (~${tokens.toLocaleString()} tokens); triggering compaction`,
+			`Observational memory: compaction threshold reached (~${progress.toLocaleString()} estimated source tokens); triggering compaction`,
 			"info",
 		);
 
@@ -56,9 +60,13 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 					);
 					return;
 				}
-				const currentEntries = ctx.sessionManager.getBranch() as Entry[];
-				const currentTokens = rawTokensSinceLastCompaction(currentEntries);
-				if (currentTokens < runtime.config.compactAfterTokens) {
+				const currentEntries = ctx.sessionManager?.getBranch?.() as Entry[] | undefined;
+				if (!currentEntries) {
+					runtime.compactInFlight = false;
+					return;
+				}
+				const currentProgress = rawTokensSinceLastCompaction(currentEntries);
+				if (currentProgress < threshold) {
 					runtime.compactInFlight = false;
 					if (hasUI) ui?.notify(
 						"Observational memory: compaction skipped — another compaction already ran before deferred compaction",

package/src/hooks/consolidation-trigger.ts

@@ -1,10 +1,11 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { runDropper } from "../agents/dropper/agent.js";
 import { observationPoolMetrics } from "../agents/dropper/pool.js";
-import { runObserver } from "../agents/observer/agent.js";
+import { ObserverStreamError, runObserver } from "../agents/observer/agent.js";
 import { runReflector } from "../agents/reflector/agent.js";
 import { debugLog, withDebugLogContext } from "../debug-log.js";
-import { type ResolveResult, type Runtime } from "../runtime.js";
+import { resolveObserverChunkMaxTokens } from "../config.js";
+import type { ResolveResult, Runtime } from "../runtime.js";
 import { serializeSourceAddressedBranchEntries } from "../serialize.js";
 import {
 	OM_OBSERVATIONS_DROPPED,
@@ -20,11 +21,14 @@ import {
 	latestCoverageIndex,
 	latestCoverageMarkerId,
 	observationToSummaryLine,
+	realTokensSinceAnchor,
 	rawTokensSinceObservationCoverage,
 	rawTokensSinceReflectionCoverage,
 	reflectionToSummaryLine,
 	type Entry,
+	type Observation,
 	type Reflection,
+	type V3MemoryCustomType,
 } from "../session-ledger/index.js";
 
 type ResolvedModel = Extract<ResolveResult, { ok: true }>;
@@ -35,6 +39,7 @@ type ConsolidationCtx = {
 	ui?: { notify: (message: string, type?: "warning" | "info" | "error") => void };
 	model: unknown;
 	modelRegistry: any;
+	getContextUsage?: () => { tokens?: number | null; contextWindow?: number } | undefined;
 	sessionManager: {
 		getBranch: () => unknown;
 		getSessionId?: () => string;
@@ -69,9 +74,43 @@ function mergeReflections(existing: Reflection[], additional: Reflection[]): Ref
 	return merged;
 }
 
-function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
-	return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
-		|| rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens;
+/**
+ * Real current context tokens from the session (provider-reported usage, the
+ * same basis the footer percentage uses). Falls back to undefined when the
+ * host pi lacks getContextUsage or the count is unknown (e.g. right after a
+ * compaction, before the next valid assistant response).
+ */
+function realContextTokens(ctx: ConsolidationCtx): number | undefined {
+	const usage = typeof ctx.getContextUsage === "function" ? ctx.getContextUsage() : undefined;
+	const tokens = usage?.tokens;
+	return typeof tokens === "number" && Number.isFinite(tokens) ? tokens : undefined;
+}
+
+function stageDue(
+	entries: Entry[],
+	runtime: Runtime,
+	currentTokens: number | undefined,
+	customType: V3MemoryCustomType,
+	rawEstimateFn: (entries: Entry[]) => number,
+	threshold: number,
+): boolean {
+	if (currentTokens !== undefined) {
+		const real = realTokensSinceAnchor(entries, customType, currentTokens);
+		if (real !== undefined) return real >= threshold;
+	}
+	// Real delta unmeasurable (no usage baseline, or accounting basis changed) or
+	// old pi host without getContextUsage — fall back to the raw estimate, which
+	// self-limits after coverage and cannot over-fire or starve.
+	return rawEstimateFn(entries) >= threshold;
+}
+
+function anyStageDue(entries: Entry[], runtime: Runtime, currentTokens: number | undefined): boolean {
+	return stageDue(entries, runtime, currentTokens, OM_OBSERVATIONS_RECORDED, rawTokensSinceObservationCoverage, runtime.config.observeAfterTokens)
+		|| stageDue(entries, runtime, currentTokens, OM_REFLECTIONS_RECORDED, rawTokensSinceReflectionCoverage, runtime.config.reflectAfterTokens);
+}
+
+function shouldNotifyWorker(runtime: Runtime, ctx: ConsolidationCtx): boolean {
+	return runtime.config.showWorkerNotifications && ctx.hasUI;
 }
 
 function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "observer" | "reflector" | "dropper") => Promise<ResolvedModel | undefined> {
@@ -121,7 +160,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 	if (runtime.consolidationInFlight) return;
 
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	if (!anyStageDue(entries, runtime)) return;
+	if (!anyStageDue(entries, runtime, realContextTokens(ctx))) return;
 
 	const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
 	const consolidationCtx: ConsolidationCtx = {
@@ -130,6 +169,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 		ui: ctx.ui,
 		model: ctx.model,
 		modelRegistry: ctx.modelRegistry,
+		getContextUsage: ctx.getContextUsage,
 		sessionManager: ctx.sessionManager,
 	};
 
@@ -185,27 +225,79 @@ async function runObserverStage(
 	resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
 ): Promise<StageOutcome> {
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	const tokens = rawTokensSinceObservationCoverage(entries);
+	const currentTokens = realContextTokens(ctx);
+	const real = currentTokens !== undefined ? realTokensSinceAnchor(entries, OM_OBSERVATIONS_RECORDED, currentTokens) : undefined;
+	const tokens = real !== undefined ? real : rawTokensSinceObservationCoverage(entries); // fallback: no usage baseline / basis change
 	if (tokens < runtime.config.observeAfterTokens) return "continue";
 
+	const sessionMetadata = debugSessionMetadata(ctx);
+	const sessionIdentity = sessionMetadata.sessionId ?? sessionMetadata.sessionFile;
+	const coverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
+
+	// Deliberate-empty backoff (#23): an intentional "nothing to record" verdict
+	// must not re-fire the observer every turn over the same span. Retry only
+	// after another observeAfterTokens worth of new source tokens arrives, and
+	// drop the backoff as soon as coverage advances.
+	const backoff = runtime.observerEmptyBackoff;
+	if (backoff) {
+		if (
+			sessionIdentity !== backoff.sessionIdentity
+			|| coverageId !== backoff.coverageId
+			|| tokens >= backoff.tokensAtEmpty + runtime.config.observeAfterTokens
+		) {
+			runtime.observerEmptyBackoff = undefined;
+		} else {
+			debugLog("observer.empty_backoff", { tokens, resumeAtTokens: backoff.tokensAtEmpty + runtime.config.observeAfterTokens });
+			return "continue";
+		}
+	}
+
+	// Resolve the model before building the chunk: the default chunk cap
+	// derives from the resolved model's context window.
+	const resolved = await resolveModel("observer");
+	if (!resolved) return "abort";
+
 	const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
-	const chunkEntries = sourceEntriesAfter(entries, lastCoverageIdx);
-	const coversUpToId = chunkEntries.at(-1)?.id;
+	const backlogEntries = sourceEntriesAfter(entries, lastCoverageIdx);
+
+	// Budget the text that is actually sent to the observer, including source
+	// labels and rendered message content. Complete entries are kept intact.
+	// Only a first entry that cannot fit by itself is represented by a clearly
+	// marked head/tail excerpt; the original ledger entry remains untouched.
+	const contextWindow = (resolved.model as { contextWindow?: number }).contextWindow;
+	const maxChunkTokens = resolveObserverChunkMaxTokens(runtime.config, contextWindow);
+	const {
+		text: chunk,
+		sourceEntryIds,
+		estimatedTokens: chunkTokens,
+		truncatedSourceEntryIds,
+	} = serializeSourceAddressedBranchEntries(backlogEntries, { maxTokens: maxChunkTokens });
+	if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
+	const coversUpToId = sourceEntryIds.at(-1);
 	if (!coversUpToId) return "continue";
 
-	const { text: chunk, sourceEntryIds } = serializeSourceAddressedBranchEntries(chunkEntries);
-	if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
+	if (sourceEntryIds.length < backlogEntries.length || truncatedSourceEntryIds.length > 0) {
+		debugLog("observer.chunk_capped", {
+			maxChunkTokens,
+			backlogEntries: backlogEntries.length,
+			backlogTokens: tokens,
+			chunkEntries: sourceEntryIds.length,
+			chunkTokens,
+			truncatedSourceEntryIds,
+		});
+	}
 
 	const memory = fullProjection(entries);
 	const priorReflections = memory.reflections.map(reflectionToSummaryLine);
 	const priorObservations = memory.observations.map(observationToSummaryLine);
 
-	if (ctx.hasUI) ctx.ui?.notify(
-		`Observational memory: observer running on ~${tokens.toLocaleString()}-token chunk`,
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+		`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk`,
 		"info",
 	);
 	debugLog("observer.start", {
 		tokens,
+		chunkTokens,
 		coversUpToId,
 		sourceEntryIds,
 		sourceEntryCount: sourceEntryIds.length,
@@ -213,28 +305,40 @@ async function runObserverStage(
 		priorObservations: priorObservations.length,
 	});
 
-	const resolved = await resolveModel("observer");
-	if (!resolved) return "abort";
-
-	const observations = await runObserver({
-		model: resolved.model as any,
-		apiKey: resolved.apiKey,
-		headers: resolved.headers,
-		priorReflections,
-		priorObservations,
-		chunk,
-		allowedSourceEntryIds: sourceEntryIds,
-		maxTurns: runtime.config.agentMaxTurns,
-		thinkingLevel: runtime.config.model?.thinking ?? "low",
-	});
+	let observations: Observation[] | undefined;
+	try {
+		observations = await runObserver({
+			model: resolved.model as any,
+			apiKey: resolved.apiKey,
+			headers: resolved.headers,
+			priorReflections,
+			priorObservations,
+			chunk,
+			allowedSourceEntryIds: sourceEntryIds,
+			maxTurns: runtime.config.agentMaxTurns,
+			thinkingLevel: runtime.config.model?.thinking ?? "low",
+		});
+	} catch (error) {
+		if (error instanceof ObserverStreamError) {
+			// API/stream failure is not a clean empty (#32): surface it as a real
+			// failure instead of the "no observations" path. Coverage stays put.
+			runtime.recordConsolidationStageError(ctx, "observer", error);
+			return "abort";
+		}
+		throw error;
+	}
 	if (!observations || observations.length === 0) {
+		// Deliberate empty: routine info, not a warning, and back off re-fires
+		// over the same span (#23).
 		debugLog("observer.empty", { coversUpToId });
-		if (ctx.hasUI) ctx.ui?.notify(
-			"Observational memory: observer returned no observations",
-			"warning",
+		runtime.observerEmptyBackoff = { sessionIdentity, coverageId, tokensAtEmpty: tokens };
+		if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+			"Observational memory: observer found nothing new in this chunk (coverage unchanged; will retry later)",
+			"info",
 		);
 		return "continue";
 	}
+	runtime.observerEmptyBackoff = undefined;
 
 	const data = buildObservationsRecordedData(observations, coversUpToId);
 	if (!data) return "continue";
@@ -245,7 +349,7 @@ async function runObserverStage(
 	});
 	appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
 	debugLog("observer.appended", { count: observations.length, coversUpToId });
-	if (ctx.hasUI) ctx.ui?.notify(
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
 		`Observational memory: ${observations.length} observation${observations.length === 1 ? "" : "s"} recorded`,
 		"info",
 	);
@@ -259,13 +363,15 @@ async function runReflectorStage(
 	resolveModel: (stage: "reflector") => Promise<ResolvedModel | undefined>,
 ): Promise<ReflectorStageResult> {
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	const reflectionTokens = rawTokensSinceReflectionCoverage(entries);
+	const currentTokens = realContextTokens(ctx);
+	const real = currentTokens !== undefined ? realTokensSinceAnchor(entries, OM_REFLECTIONS_RECORDED, currentTokens) : undefined;
+	const reflectionTokens = real !== undefined ? real : rawTokensSinceReflectionCoverage(entries); // fallback: no usage baseline / basis change
 	if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
 
 	const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
 	if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
 
-	if (ctx.hasUI) ctx.ui?.notify(
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
 		`Observational memory: reflector running (~${reflectionTokens.toLocaleString()} tokens)`,
 		"info",
 	);
@@ -337,7 +443,7 @@ async function runDropperStage(
 		maxDropsAllowed: metrics.maxDropsAllowed,
 	});
 
-	if (ctx.hasUI) ctx.ui?.notify(
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
 		`Observational memory: dropper running after reflection — active observation pool ~${metrics.observationTokens.toLocaleString()} / ${metrics.targetTokens.toLocaleString()} target tokens (${Math.round(metrics.fullness * 100).toLocaleString()}%)`,
 		"info",
 	);

package/src/runtime.ts

@@ -1,9 +1,26 @@
 import { type Config, DEFAULTS, loadConfig } from "./config.js";
 
 export type ResolveResult =
-	| { ok: true; model: unknown; apiKey: string; headers?: Record<string, string> }
+	| { ok: true; model: unknown; apiKey?: string; headers?: Record<string, string> }
 	| { ok: false; reason: string };
 
+/**
+ * Mirrors pi's own request-auth acceptance rule (`AgentSession._getRequiredRequestAuth`):
+ * resolved auth is usable when it carries an apiKey OR at least one header value.
+ * OAuth providers (kimi-coding, xai, openai-codex, anthropic OAuth, …) authenticate via
+ * `toAuth()` returning `{ headers: { Authorization: "Bearer …" } }` with no apiKey, and
+ * pi-ai providers accept a caller-supplied Authorization header in place of an apiKey.
+ */
+function hasUsableAuth(auth: { apiKey?: unknown; headers?: unknown }): boolean {
+	if (typeof auth.apiKey === "string" && auth.apiKey.length > 0) return true;
+	if (auth.headers && typeof auth.headers === "object") {
+		return Object.values(auth.headers as Record<string, unknown>).some(
+			(value) => typeof value === "string" && value.length > 0,
+		);
+	}
+	return false;
+}
+
 type NotifyLevel = "warning" | "info" | "error";
 type Notify = (message: string, type?: NotifyLevel) => void;
 export type ConsolidationPhase = "observer" | "reflector" | "dropper";
@@ -32,6 +49,12 @@ export class Runtime {
 	lastObserverError: string | undefined;
 	lastReflectorError: string | undefined;
 	lastDropperError: string | undefined;
+	/** Deliberate-empty backoff (#23): skip observer re-fires over the same span until enough new tokens arrive. */
+	observerEmptyBackoff: {
+		sessionIdentity: string | undefined;
+		coverageId: string | undefined;
+		tokensAtEmpty: number;
+	} | undefined;
 
 	ensureConfig(cwd: string): void {
 		if (this.configLoaded) return;
@@ -54,11 +77,15 @@ export class Runtime {
 		}
 		if (!model) return { ok: false, reason: "no model available (session has no model and no observational-memory model configured)" };
 		const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
-		if (!auth.ok || !auth.apiKey) {
-			const provider = (model as { provider?: string }).provider ?? "unknown";
-			return { ok: false, reason: `no API key for provider "${provider}"` };
+		const provider = (model as { provider?: string }).provider ?? "unknown";
+		if (!auth.ok || !hasUsableAuth(auth)) {
+			const isOAuth = ctx.modelRegistry.isUsingOAuth?.(model) === true;
+			const reason = isOAuth
+				? `authentication failed for provider "${provider}" — OAuth credentials may have expired; run '/login ${provider}' to re-authenticate`
+				: `no API key or auth headers for provider "${provider}"`;
+			return { ok: false, reason };
 		}
-		return { ok: true, model, apiKey: auth.apiKey as string, headers: auth.headers as Record<string, string> | undefined };
+		return { ok: true, model, apiKey: auth.apiKey as string | undefined, headers: auth.headers as Record<string, string> | undefined };
 	}
 
 	launchConsolidationTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> {

package/src/serialize.ts

@@ -1,4 +1,5 @@
 import type { Message, TextContent, ToolResultMessage } from "@earendil-works/pi-ai";
+import { estimateStringTokens } from "./tokens.js";
 
 function pad(n: number): string {
 	return n.toString().padStart(2, "0");
@@ -160,23 +161,78 @@ export function serializeBranchEntries(entries: RenderableEntry[]): string {
 export type SourceAddressedSerialization = {
 	text: string;
 	sourceEntryIds: string[];
+	estimatedTokens: number;
+	truncatedSourceEntryIds: string[];
 };
 
+export type SourceAddressedSerializationOptions = {
+	/** Maximum estimated tokens in the final source-addressed text. */
+	maxTokens?: number;
+};
+
+const SOURCE_OMISSION_MARKER =
+	"\n\n[… middle omitted: source exceeds observer input budget; original source remains in the session ledger …]\n\n";
+
+function truncateSourceBlockToTokenBudget(label: string, rendered: string, maxTokens: number): string | undefined {
+	const required = `${label}\n${SOURCE_OMISSION_MARKER}`;
+	if (estimateStringTokens(required) > maxTokens) return undefined;
+	const full = `${label}\n${rendered}`;
+	if (estimateStringTokens(full) <= maxTokens) return full;
+	const maxChars = Math.max(1, maxTokens * 4);
+	const fixed = `${label}\n${SOURCE_OMISSION_MARKER}`;
+	const retainedChars = maxChars - fixed.length;
+	const headChars = Math.ceil(retainedChars / 2);
+	const tailChars = retainedChars - headChars;
+	return `${label}\n${rendered.slice(0, headChars)}${SOURCE_OMISSION_MARKER}${tailChars > 0 ? rendered.slice(-tailChars) : ""}`;
+}
+
 function isSourceRenderableEntry(entry: RenderableEntry): boolean {
 	return entry.type === "message" || entry.type === "custom_message" || entry.type === "branch_summary";
 }
 
-export function serializeSourceAddressedBranchEntries(entries: RenderableEntry[]): SourceAddressedSerialization {
+/**
+ * Serialize complete source entries up to the token budget. If the first entry
+ * alone exceeds the budget, include a clearly marked head/tail excerpt so one
+ * pathological tool result cannot permanently block observation coverage.
+ * The original ledger entry is never modified and remains recallable by id.
+ */
+export function serializeSourceAddressedBranchEntries(
+	entries: RenderableEntry[],
+	options: SourceAddressedSerializationOptions = {},
+): SourceAddressedSerialization {
 	const blocks: string[] = [];
 	const sourceEntryIds: string[] = [];
+	const truncatedSourceEntryIds: string[] = [];
+	let estimatedTokens = 0;
+
 	for (const entry of entries) {
 		if (!entry.id || !isSourceRenderableEntry(entry)) continue;
 		const rendered = serializeBranchEntries([entry]);
 		if (!rendered.trim()) continue;
+		const label = `[Source entry id: ${entry.id}]`;
+		const block = `${label}\n${rendered}`;
+		const separator = blocks.length > 0 ? "\n\n" : "";
+		const blockTokens = estimateStringTokens(`${separator}${block}`);
+		const maxTokens = options.maxTokens;
+
+		if (maxTokens !== undefined && estimatedTokens + blockTokens > maxTokens) {
+			if (blocks.length > 0) break;
+			const excerpt = truncateSourceBlockToTokenBudget(label, rendered, maxTokens);
+			if (!excerpt) break;
+			blocks.push(excerpt);
+			sourceEntryIds.push(entry.id);
+			truncatedSourceEntryIds.push(entry.id);
+			estimatedTokens = estimateStringTokens(excerpt);
+			break;
+		}
+
+		blocks.push(block);
 		sourceEntryIds.push(entry.id);
-		blocks.push(`[Source entry id: ${entry.id}]\n${rendered}`);
+		estimatedTokens += blockTokens;
 	}
-	return { text: blocks.join("\n\n"), sourceEntryIds };
+
+	const text = blocks.join("\n\n");
+	return { text, sourceEntryIds, estimatedTokens: estimateStringTokens(text), truncatedSourceEntryIds };
 }
 
 function renderRecallMessage(entry: RenderableEntry): string | null {

package/src/session-ledger/progress.ts

@@ -117,6 +117,106 @@ export function findLastCompactionIndex(entries: Entry[]): number {
 	return -1;
 }
 
+// ==== Real (provider-reported) token accounting ====
+//
+// These helpers measure context growth from provider-reported usage for the
+// observation and reflection coverage clocks. Automatic compaction keeps its
+// separate raw source-entry clock because its setting counts ledger entries.
+
+type UsageLike = {
+	totalTokens?: number;
+	input?: number;
+	output?: number;
+	cacheRead?: number;
+	cacheWrite?: number;
+};
+
+export function contextTokensFromUsage(usage: unknown): number | undefined {
+	if (!usage || typeof usage !== "object") return undefined;
+	const u = usage as UsageLike;
+	const total = typeof u.totalTokens === "number" && Number.isFinite(u.totalTokens) && u.totalTokens > 0 ? u.totalTokens : undefined;
+	if (total !== undefined) return total;
+	const parts = [u.input, u.output, u.cacheRead, u.cacheWrite];
+	if (parts.every((p) => typeof p === "number" && Number.isFinite(p))) {
+		const sum = parts.reduce<number>((acc, p) => acc + (p ?? 0), 0);
+		return sum > 0 ? sum : undefined;
+	}
+	return undefined;
+}
+
+function validAssistantContextTokens(entry: Entry): number | undefined {
+	if (entry.type !== "message" || !entry.message || typeof entry.message !== "object") return undefined;
+	const msg = entry.message as { role?: string; stopReason?: string; usage?: unknown };
+	if (msg.role !== "assistant" || msg.stopReason === "aborted" || msg.stopReason === "error") return undefined;
+	return contextTokensFromUsage(msg.usage);
+}
+
+/**
+ * Real context tokens right after a compaction anchor.
+ *
+ * Only usage from an assistant that responded AFTER the compaction is a valid
+ * post-compaction baseline: pi's own docs state the last assistant usage
+ * before/at a compaction reflects the PRE-compaction context size. The usage
+ * carried on the compaction entry itself is the summary-generation call's
+ * usage (pre-compaction scale, a different LLM call), so it is deliberately
+ * NOT used as a baseline.
+ */
+export function realContextTokensAfterCompaction(entries: Entry[], compactionIdx: number): number | undefined {
+	for (let i = compactionIdx + 1; i < entries.length; i++) {
+		const t = validAssistantContextTokens(entries[i]);
+		if (t !== undefined) return t;
+	}
+	return undefined;
+}
+
+/**
+ * Real context tokens at the time observation coverage ended: last valid
+ * assistant usage at/before the covered entry. Returns undefined when no valid
+ * usage exists (e.g. an error/abort storm) — callers must fall back to the
+ * raw estimate rather than measuring from zero, which would otherwise read the
+ * full context as "growth" and re-fire stages every turn.
+ */
+export function realContextTokensAtCoverage(entries: Entry[], coverageIdx: number): number | undefined {
+	for (let i = coverageIdx; i >= 0; i--) {
+		const t = validAssistantContextTokens(entries[i]);
+		if (t !== undefined) return t;
+	}
+	return undefined;
+}
+
+/**
+ * Real context growth since the most recent anchor (a compaction, or the given
+ * coverage marker), measured from provider-reported usage.
+ *
+ * Returns undefined when the baseline cannot be measured reliably — no usage
+ * at/after the anchor, or the current context is SMALLER than the baseline
+ * (accounting basis changed, e.g. a mid-session model/provider switch that
+ * counts usage differently). Callers must fall back to the raw estimate in
+ * that case; clamping a stale baseline to 0 would starve the stage forever,
+ * and measuring from zero would over-fire it.
+ */
+export function realTokensSinceAnchor(
+	entries: Entry[],
+	customType: V3MemoryCustomType | undefined,
+	currentContextTokens: number,
+): number | undefined {
+	const coverageIdx = customType ? latestCoverageIndex(entries, customType) : -1;
+	const compactionIdx = findLastCompactionIndex(entries);
+	if (compactionIdx > coverageIdx) {
+		const baseline = realContextTokensAfterCompaction(entries, compactionIdx);
+		if (baseline === undefined) return undefined;
+		const delta = currentContextTokens - baseline;
+		return delta >= 0 ? delta : undefined;
+	}
+	if (coverageIdx >= 0) {
+		const baseline = realContextTokensAtCoverage(entries, coverageIdx);
+		if (baseline === undefined) return undefined;
+		const delta = currentContextTokens - baseline;
+		return delta >= 0 ? delta : undefined;
+	}
+	return Math.max(0, currentContextTokens);
+}
+
 export function rawTokensSinceLastCompaction(entries: Entry[]): number {
 	const compactionIndex = findLastCompactionIndex(entries);
 	if (compactionIndex === -1) return rawTokensAfterIndex(entries, -1);

package/src/tokens.ts

@@ -4,6 +4,24 @@ export function estimateStringTokens(text: string): number {
 	return Math.ceil(text.length / 4);
 }
 
+/**
+ * Estimate the rendered footprint of an observation line as it appears in
+ * summaries / pool listings: "[id] YYYY-MM-DD HH:MM [relevance] content".
+ * Pool budgets that only count bare content undercount every line's
+ * metadata overhead (id + timestamp + relevance tags), so the configured
+ * pool target was reached later than the rendered memory actually allowed.
+ */
+export function observationLineTokenCount(observation: {
+	id: string;
+	timestamp: string;
+	relevance: string;
+	content: string;
+}): number {
+	return estimateStringTokens(
+		`[${observation.id}] ${observation.timestamp} [${observation.relevance}] ${observation.content}`,
+	);
+}
+
 export function estimateEntryTokens(entry: { type: string; message?: unknown; content?: unknown; summary?: unknown }): number {
 	if (entry.type === "message" && entry.message) {
 		return estimateMessageTokens(entry.message as Parameters<typeof estimateMessageTokens>[0]);