pi-blackhole 0.3.4 → 0.3.5

package/src/core/settings.ts

@@ -1,40 +1,9 @@
 /**
- * Pi-vcc settings — reads from unified pi-blackhole/pi-blackhole-config.json.
+ * Scaffold the pi-blackhole config file on disk.
  *
- * Upstream: https://github.com/sting8k/pi-vcc (src/core/settings.ts)
- * Modified by pi-vcc-om: loadSettings wraps loadUnifiedConfig.
+ * Only holds scaffoldSettings(); config loading/parsing happens in unified-config.ts.
  */
-import { loadUnifiedConfig, scaffoldConfig } from "./unified-config.js";
-
-export interface PiVccSettings {
-	/** @deprecated Use compactionEngine instead. */
-	overrideDefaultCompaction: boolean;
-	/** @deprecated Use compaction instead. */
-	noAutoCompact: boolean;
-	/** @deprecated Use compaction + memory instead. */
-	passive: boolean;
-	/** Write debug snapshots to /tmp/pi-blackhole-debug.json. */
-	debug: boolean;
-	/** Unified compaction control: "auto" | "manual" | "off". */
-	compaction: "auto" | "manual" | "off";
-	/** Compaction engine: "blackhole" | "pi-default". */
-	compactionEngine: "blackhole" | "pi-default";
-	/** Visible tail behavior: "pi-default" | "minimal". */
-	tailBehavior: "pi-default" | "minimal";
-}
-
-export function loadSettings(): PiVccSettings {
-	const config = loadUnifiedConfig(process.cwd());
-	return {
-		overrideDefaultCompaction: config.overrideDefaultCompaction ?? false,
-		noAutoCompact: config.noAutoCompact ?? false,
-		passive: config.passive ?? false,
-		debug: config.debug,
-		compaction: config.compaction,
-		compactionEngine: config.compactionEngine,
-		tailBehavior: config.tailBehavior,
-	};
-}
+import { scaffoldConfig } from "./unified-config.js";
 
 export function scaffoldSettings(): void {
 	scaffoldConfig();

package/src/core/summarize.ts

@@ -27,11 +27,19 @@ const sectionOf = (text: string, header: string): string => {
   const start = text.indexOf(tag);
   if (start < 0) return "";
   const after = text.slice(start);
-  // Find next section header or separator
+  // Find next section header (must start at line boundary to avoid matching in content)
   const nextSection = HEADER_NAMES
     .filter((h) => h !== header)
-    .map((h) => after.indexOf(`[${h}]`))
-    .filter((n) => n > 0);
+    .map((h) => {
+      // Escape the header name for regex safety
+      const escaped = h.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      const re = new RegExp(`(?:^|\\n)\\[${escaped}\\]`);
+      const m = after.match(re);
+      if (!m) return -1;
+      // m.index points to \n (or 0); advance past it to the [
+      return m.index! + (m[0].startsWith("\n") ? 1 : 0);
+    })
+    .filter((n) => n >= 0);
   const nextSep = after.indexOf("\n\n---\n\n");
   const candidates = [...nextSection, ...(nextSep > 0 ? [nextSep] : [])].sort((a, b) => a - b);
   const end = candidates[0];
@@ -93,6 +101,8 @@ const mergeFileLines = (prev: string, fresh: string): string => {
 
   // Dedup: if already in Modified, drop from Created (file existed before)
   for (const p of merged.Modified) merged.Created.delete(p);
+  // Also remove Read entries that also appear in Modified (same file read+edited)
+  for (const p of merged.Modified) merged.Read.delete(p);
 
   const cap = (set: Set<string>, limit: number) => {
     const arr = [...set];
@@ -154,9 +164,7 @@ export const compile = (input: CompileInput): string => {
   prev = prev ? stripOMContent(prev) : undefined;
   const merged = prev ? mergePrevious(prev, fresh) : fresh;
   if (!merged) return "";
-  return wrapLongLines(merged);
-  // RECALL_NOTE is now part of OM_FOOTER in render-summary.ts, appended
-  // by the before-compact hook after observations/reflections.
+  return wrapLongLines(merged + SEPARATOR + RECALL_NOTE);
 };
 
 const stripRecallNote = (text: string): string => {
@@ -168,19 +176,28 @@ const stripRecallNote = (text: string): string => {
 };
 
 /**
- * Strip OM content (## Reflections, ## Observations, and the
- * CONTEXT_USAGE_INSTRUCTIONS preamble) from a previous compaction summary.
+ * Strip OM content and recall-guidance footers from a previous compaction summary.
+ *
+ * OM content (## Reflections / ## Observations + instructions) is appended after
+ * compile() by the before-compact hook, so it must be stripped from the previous
+ * summary to prevent compounding across compactions. The fresh OM projection is
+ * re-rendered each time.
  *
- * OM content is appended after compile() by the before-compact hook, so it
- * must be stripped from the previous summary to prevent compounding across
- * compactions. The fresh OM projection is re-rendered each time.
+ * Also strips the basic recall-guidance footer that is always appended when OM
+ * is off or has no entries.
  */
 const stripOMContent = (text: string): string => {
   // Remove everything from "## Reflections" or "## Observations" onward,
   // plus the instructions preamble that precedes them.
   // The preamble starts with "These are condensed memories from earlier in this session."
-  const reflIdx = text.indexOf("## Reflections");
-  const obsIdx = text.indexOf("## Observations");
+  // Use line-start anchoring to avoid matching inside conversation content
+  const reflMatch = text.match(/^## Reflections/m);
+  const reflIdx = reflMatch ? reflMatch.index! : -1;
+  const obsMatch = text.match(/^## Observations/m);
+  const obsIdx = obsMatch ? obsMatch.index! : -1;
+
+  // Also detect the basic recall-guidance footer (no observation preamble)
+  const basicFooterIdx = text.indexOf("Use `recall` with an id to retrieve original context, or `#N:path` drill-down");
 
   // Find the start of OM content: either the instructions preamble or the first section header
   let stripFrom = -1;
@@ -197,6 +214,9 @@ const stripOMContent = (text: string): string => {
     } else if (minSectionIdx < Infinity) {
       stripFrom = minSectionIdx;
     }
+  } else if (basicFooterIdx >= 0) {
+    // Strip the basic recall-guidance footer (no observations/reflections present)
+    stripFrom = basicFooterIdx;
   }
 
   if (stripFrom < 0) return text;

package/src/core/tool-args.ts

@@ -1,5 +1,8 @@
+/** Canonical list of path-like argument keys across all Pi tools. */
+export const PATH_KEYS = ["path", "file_path", "filePath", "file"] as const;
+
 export const extractPath = (args: Record<string, unknown>): string | null => {
-  for (const key of ["path", "file_path", "filePath", "file"]) {
+  for (const key of PATH_KEYS) {
     if (typeof args[key] === "string") return args[key] as string;
   }
   return null;

package/src/core/unified-config.ts

@@ -117,6 +117,10 @@ export interface UnifiedConfig {
 	/** Fallback models for dropper, tried in order after primary model fails. */
 	dropperFallbackModels?: OmModelConfig[];
 
+	/** When false, skip session model fallback when all OM model candidates are exhausted.
+	 *  Default true for backward compatibility. */
+	sessionFallback?: boolean;
+
 	/** @deprecated Use compaction instead. */
 	noAutoCompact?: boolean;
 	/** @deprecated Use compaction + memory instead. */
@@ -131,6 +135,7 @@ export interface UnifiedConfig {
 
 export const DEFAULTS: UnifiedConfig = {
 	debug: false,
+	sessionFallback: true,
 
 	// New config surface
 	compaction: "auto",
@@ -225,15 +230,18 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
 	if (typeof raw.debug === "boolean") c.debug = raw.debug;
 
 	// Booleans — om
+	if (typeof raw.sessionFallback === "boolean") c.sessionFallback = raw.sessionFallback;
 	if (typeof raw.noAutoCompact === "boolean") c.noAutoCompact = raw.noAutoCompact;
 	if (typeof raw.passive === "boolean") c.passive = raw.passive;
 	if (typeof raw.memory === "boolean") c.memory = raw.memory;
 	if (typeof raw.debugLog === "boolean") c.debugLog = raw.debugLog;
 
-	// Positive integers
+	// Numeric fields — use nonNegativeInt for observerPreambleMaxTokens (0 = auto)
 	const numKeys = ["observeAfterTokens", "reflectAfterTokens", "compactAfterTokens", "observationsPoolMaxTokens", "observationsPoolTargetTokens", "reflectorInputMaxTokens", "dropperInputMaxTokens", "observerChunkMaxTokens", "observerPreambleMaxTokens", "agentMaxTurns"] as const;
 	for (const k of numKeys) {
-		const v = positiveInt(raw[k]);
+		// observerPreambleMaxTokens accepts 0 (auto-compute); everything else must be > 0
+		const validator = k === "observerPreambleMaxTokens" ? nonNegativeInt : positiveInt;
+		const v = validator(raw[k]);
 		if (v !== undefined) (c as Record<string, unknown>)[k] = v;
 	}
 
@@ -262,8 +270,9 @@ function parseConfig(raw: Record<string, unknown>): Partial<UnifiedConfig> {
 
 /**
  * Migrate legacy config knobs to new unified surface.
- * Runs once at load time; old keys are removed from the parsed object.
- * Does NOT mutate the on-disk config file.
+ * Operates on the in-memory parsed object each time loadUnifiedConfig() is called;
+ * old keys are removed from this copy. Does NOT mutate the on-disk config file.
+ * Idempotent — safe to call repeatedly.
  */
 function migrateOldKnobs(parsed: Record<string, unknown>): void {
 	// Only run if new keys are absent AND old keys are present
@@ -361,13 +370,23 @@ export function loadUnifiedConfig(cwd: string): UnifiedConfig {
 
 	// Env override — new compaction surface
 	const envCompaction = process.env.PI_BLACKHOLE_COMPACTION;
-	if (envCompaction !== undefined && isCompaction(envCompaction.trim().toLowerCase())) {
-		parsed.compaction = envCompaction.trim().toLowerCase() as "auto" | "manual" | "off";
+	if (envCompaction !== undefined) {
+		const trimmed = envCompaction.trim().toLowerCase();
+		if (isCompaction(trimmed)) {
+			parsed.compaction = trimmed as "auto" | "manual" | "off";
+		} else {
+			console.warn(`blackhole: invalid PI_BLACKHOLE_COMPACTION value "${envCompaction}"; ignoring`);
+		}
 	}
 
 	const envCompactionEngine = process.env.PI_BLACKHOLE_COMPACTION_ENGINE;
-	if (envCompactionEngine !== undefined && isCompactionEngine(envCompactionEngine.trim().toLowerCase())) {
-		parsed.compactionEngine = envCompactionEngine.trim().toLowerCase() as "blackhole" | "pi-default";
+	if (envCompactionEngine !== undefined) {
+		const trimmed = envCompactionEngine.trim().toLowerCase();
+		if (isCompactionEngine(trimmed)) {
+			parsed.compactionEngine = trimmed as "blackhole" | "pi-default";
+		} else {
+			console.warn(`blackhole: invalid PI_BLACKHOLE_COMPACTION_ENGINE value "${envCompactionEngine}"; ignoring`);
+		}
 	}
 
 	// Merge defaults then override
@@ -445,24 +464,7 @@ export function scaffoldConfig(): void {
 	}
 }
 
-// ── Toggle helpers ───────────────────────────────────────────────────────────
-
-/** Cycle compaction: auto → manual → off → auto */
-export function toggleCompaction(current: "auto" | "manual" | "off"): "auto" | "manual" | "off" {
-	const cycle: Array<"auto" | "manual" | "off"> = ["auto", "manual", "off"];
-	const idx = cycle.indexOf(current);
-	return cycle[(idx + 1) % cycle.length];
-}
-
-/** Toggle compactionEngine: blackhole ↔ pi-default */
-export function toggleCompactionEngine(current: "blackhole" | "pi-default"): "blackhole" | "pi-default" {
-	return current === "blackhole" ? "pi-default" : "blackhole";
-}
 
-/** Toggle tailBehavior: pi-default ↔ minimal */
-export function toggleTailBehavior(current: "pi-default" | "minimal"): "pi-default" | "minimal" {
-	return current === "pi-default" ? "minimal" : "pi-default";
-}
 
 // ── Migration detection ───────────────────────────────────────────────────────
 

package/src/extract/commits.ts

@@ -6,8 +6,9 @@ interface CommitInfo {
 }
 
 const COMMIT_MSG_RE = /git\s+commit[^\n]*?-m\s+(?:"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)'|\$?'((?:[^'\\]|\\.)*)')/;
-// Match short hash from git output: "[branch hash]" or "main hash" or 7-12 hex
-const HASH_RE = /\b([0-9a-f]{7,12})\b/;
+// Match short hash from git output — only as fallback after bracket/range patterns fail.
+// Requires 8+ hex chars to reduce false positives from random hex in tool output.
+const HASH_RE = /\b([0-9a-f]{8,12})\b/;
 
 const firstLineOf = (text: string): string => {
   const line = text.split(/\\n|\n/)[0] ?? "";

package/src/extract/files.ts

@@ -16,16 +16,18 @@ const FILE_WRITE_TOOLS = new Set([
   "MultiEdit",
 ]);
 
-const FILE_CREATE_TOOLS = new Set([
-  "Write", "write", "write_file",
-]);
+// Pi never exposes a "createdFiles" field — Write operations are tracked as modified.
+// FILE_CREATE_TOOLS kept as empty set for forward-compat if Pi adds a creation signal.
+const FILE_CREATE_TOOLS = new Set<string>();
 
 /**
  * Find the longest common directory prefix among absolute paths.
  * Returns "" if fewer than 2 absolute paths or no meaningful common prefix.
  */
 const longestCommonDirPrefix = (paths: string[]): string => {
-  const abs = paths.filter((p) => p.startsWith("/"));
+  // Normalize backslashes (Windows) to forward slashes for uniform comparison
+  const normalized = paths.map((p) => p.replace(/\\/g, "/"));
+  const abs = normalized.filter((p) => p.startsWith("/") || /^[A-Za-z]:\//.test(p));
   if (abs.length < 2) return "";
   const split = abs.map((p) => p.split("/"));
   const min = Math.min(...split.map((s) => s.length));

package/src/hooks/before-compact.ts

@@ -289,8 +289,10 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
     }
 
     // Determine effective tail behavior for buildOwnCut
+    // Both /blackhole and auto-triggered default to "minimal" (aggressive cut);
+    // users can opt into "pi-default" (gentler) by setting tailBehavior in config.
     const effectiveTailBehavior = isPiVcc
-      ? (omRuntime.config.tailBehavior ?? "minimal")   // /blackhole: minimal by default
+      ? (omRuntime.config.tailBehavior ?? "minimal")
       : (omRuntime.config.tailBehavior ?? "minimal");
 
     trace("before_compact.tail_behavior", {
@@ -451,7 +453,7 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
     omRuntime.compactWasPiVcc = isPiVcc;
 
     // ── Inject observational-memory content ───────────────────────────
-    let omContent = "";
+    let omContent: string;
     let omDetails: Record<string, unknown> | undefined;
     trace("before_compact.om_injection", { memoryEnabled: omRuntime.config.memory !== false });
     if (omRuntime.config.memory !== false) {
@@ -462,11 +464,13 @@ export const registerBeforeCompactHook = (pi: ExtensionAPI, omRuntime: Runtime)
     );
       omContent = renderSummary(projection.reflections, projection.observations);
       omDetails = projection.details;
+    } else {
+      omContent = renderSummary([], []);
     }
 
     return {
       compaction: {
-        summary: omContent ? summary + "\n\n" + omContent : summary,
+        summary: summary + "\n\n" + omContent,
         details: { ...details, "om.folded": omDetails },
         tokensBefore: preparation.tokensBefore,
         firstKeptEntryId,

package/src/om/agents/dropper/agent.ts

@@ -7,6 +7,7 @@
  */
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
+import { createBridgeStreamFn } from "../../provider-stream.js";
 import { streamSimple } from "@earendil-works/pi-ai";
 import { Type } from "typebox";
 import type { Static } from "typebox";
@@ -287,13 +288,7 @@ export async function runDropper(args: RunDropperArgs): Promise<string[] | undef
 
 	const loop = args.agentLoop ?? agentLoop;
 	// ── Bridge stream function ──
-	const PROVIDER_STREAMS_KEY = Symbol.for("pi-blackhole:provider-streams");
-	const bridgeStreamFn = (model: any, ctx: any, opts: any) => {
-		const providerStreams: Map<string, Function> | undefined = (globalThis as any)[PROVIDER_STREAMS_KEY];
-		if (!providerStreams) return streamSimple(model, ctx, opts);
-		const customFn = model?.api ? providerStreams.get(model.api) : undefined;
-		return customFn ? customFn(model, ctx, opts) : streamSimple(model, ctx, opts);
-	};
+	const bridgeStreamFn = createBridgeStreamFn(streamSimple);
 	const streamFn = args.streamFn ?? bridgeStreamFn;
 	const stream = loop(prompts, context, config, signal, streamFn);
 	let agentError: string | undefined;

package/src/om/agents/observer/agent.ts

@@ -8,6 +8,7 @@
  */
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
+import { createBridgeStreamFn } from "../../provider-stream.js";
 import { streamSimple } from "@earendil-works/pi-ai";
 import { Type } from "typebox";
 import type { Static } from "typebox";
@@ -85,13 +86,19 @@ export function normalizeSourceEntryIds(
 	const allowedOrder = new Map<string, number>();
 	for (let i = 0; i < allowedSourceEntryIds.length; i++) allowedOrder.set(allowedSourceEntryIds[i], i);
 
+	// Filter out invalid/unknown IDs instead of rejecting the entire batch.
+	// Matches the dropper's normalizeDropObservationIds pattern: one hallucinated
+	// ID from the LLM should not discard valid observations.
 	const seen = new Set<string>();
+	const valid: string[] = [];
 	for (const id of sourceEntryIds) {
-		if (!allowedOrder.has(id)) return undefined;
+		if (!allowedOrder.has(id)) continue;
+		if (seen.has(id)) continue;
 		seen.add(id);
+		valid.push(id);
 	}
-	if (seen.size === 0) return undefined;
-	return Array.from(seen).sort((a, b) => (allowedOrder.get(a) ?? 0) - (allowedOrder.get(b) ?? 0));
+	if (valid.length === 0) return undefined;
+	return valid.sort((a, b) => (allowedOrder.get(a) ?? 0) - (allowedOrder.get(b) ?? 0));
 }
 
 /** Result returned by runObserver when no observations are recorded. */
@@ -226,16 +233,7 @@ ${conversation}`;
 	// Consolidation agents run via jiti (moduleCache: false) which creates a separate
 	// pi-ai instance whose apiProviderRegistry lacks custom providers registered by
 	// other extensions (e.g., claude-bridge). The bridge looks up streamSimple functions
-	// from a Symbol.for() global that index.ts populates during provider registration.
-	// If no custom provider is found, it falls back to the jiti-loaded streamSimple
-	// which handles all built-in providers correctly.
-	const PROVIDER_STREAMS_KEY = Symbol.for("pi-blackhole:provider-streams");
-	const bridgeStreamFn = (model: any, ctx: any, opts: any) => {
-		const providerStreams: Map<string, Function> | undefined = (globalThis as any)[PROVIDER_STREAMS_KEY];
-		if (!providerStreams) return streamSimple(model, ctx, opts);
-		const customFn = model?.api ? providerStreams.get(model.api) : undefined;
-		return customFn ? customFn(model, ctx, opts) : streamSimple(model, ctx, opts);
-	};
+	const bridgeStreamFn = createBridgeStreamFn(streamSimple);
 	const streamFn = args.streamFn ?? bridgeStreamFn;
 	const stream = loop(prompts, context, config, signal, streamFn);
 	let agentError: string | undefined;

package/src/om/agents/reflector/agent.ts

@@ -7,6 +7,7 @@
  */
 import { agentLoop, type AgentContext, type AgentLoopConfig, type AgentTool } from "@earendil-works/pi-agent-core";
 import type { Message, Model, ModelThinkingLevel } from "@earendil-works/pi-ai";
+import { createBridgeStreamFn } from "../../provider-stream.js";
 import { streamSimple } from "@earendil-works/pi-ai";
 import { Type } from "typebox";
 import type { Static } from "typebox";
@@ -150,13 +151,7 @@ export async function runReflector(args: RunReflectorArgs): Promise<Reflection[]
 
 	const loop = args.agentLoop ?? agentLoop;
 	// ── Bridge stream function ──
-	const PROVIDER_STREAMS_KEY = Symbol.for("pi-blackhole:provider-streams");
-	const bridgeStreamFn = (model: any, ctx: any, opts: any) => {
-		const providerStreams: Map<string, Function> | undefined = (globalThis as any)[PROVIDER_STREAMS_KEY];
-		if (!providerStreams) return streamSimple(model, ctx, opts);
-		const customFn = model?.api ? providerStreams.get(model.api) : undefined;
-		return customFn ? customFn(model, ctx, opts) : streamSimple(model, ctx, opts);
-	};
+	const bridgeStreamFn = createBridgeStreamFn(streamSimple);
 	const streamFn = args.streamFn ?? bridgeStreamFn;
 	const stream = loop(prompts, context, config, signal, streamFn);
 	let agentError: string | undefined;

package/src/om/compaction-trigger.ts

@@ -2,14 +2,7 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { rawTokensSinceLastCompaction, type Entry } from "./ledger/index.js";
 import type { Runtime } from "./runtime.js";
 import { debugLog } from "./debug-log.js";
-
-/**
- * Regex matching Pi's internal retryable error detection.
- * When the last assistant message in agent_end has stopReason "error" matching this pattern,
- * Pi will auto-retry — we must not trigger compaction between attempts.
- */
-const RETRYABLE_ERROR_RE =
-	/overloaded|provider.?returned.?error|rate.?limit|too many requests|429|500|502|503|504|service.?unavailable|server.?error|internal.?error|network.?error|connection.?error|connection.?refused|connection.?lost|websocket.?closed|websocket.?error|other side closed|fetch failed|upstream.?connect|reset before headers|socket hang up|ended without|http2 request did not get a response|timed? out|timeout|terminated|retry delay/i;
+import { RETRYABLE_ERROR_RE } from "./retryable-error.js";
 
 export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
 	pi.on("agent_end", (event: any, ctx: any) => {

package/src/om/configure-overlay.ts

@@ -8,6 +8,7 @@
 
 import { visibleWidth } from "./key-matcher.js";
 import { matchesKey, decodeKittyPrintable } from "@earendil-works/pi-tui";
+import { DEFAULTS } from "../core/unified-config.js";
 import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 
@@ -49,6 +50,8 @@ const FIELDS: FieldDef[] = [
 	// ── Observational Memory ──
 	{ key: "memory", label: "Observational memory", type: "boolean", section: "Observational Memory",
 		helpText: "Enable OM workers (observer, reflector, dropper) and content injection" },
+	{ key: "sessionFallback", label: "Session model fallback", type: "boolean", section: "Observational Memory",
+		helpText: "off=skip stage when all OM models fail, instead of falling back to the main coding model" },
 	{ key: "observeAfterTokens", label: "Observer threshold", type: "number", section: "Observational Memory",
 		helpText: "Tokens accumulated since last observer run before triggering next observe" },
 	{ key: "reflectAfterTokens", label: "Reflect + dropper threshold", type: "number", section: "Observational Memory",
@@ -139,9 +142,10 @@ export function createConfigureOverlay(
 		raw = {};
 	}
 
+	const defaults = DEFAULTS as unknown as Record<string, unknown>;
 	const fields: FieldState[] = FIELDS.map((def) => ({
 		def,
-		value: formatValue(def, raw[def.key]),
+		value: formatValue(def, def.key in raw ? raw[def.key] : defaults[def.key]),
 		editing: false,
 		cursor: 0,
 	}));

package/src/om/consolidation.ts

@@ -15,7 +15,7 @@ import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
 import type { ConfiguredModel } from "./config.js";
 import { debugLog, withDebugLogContext } from "./debug-log.js";
 import { type ResolveResult, type Runtime } from "./runtime.js";
-import { isRetryableError } from "./cooldown.js";
+import { isRetryableError } from "./retryable-error.js";
 import { effectiveContextWindow } from "./model-budget.js";
 import { serializeSourceAddressedBranchEntries } from "./serialize.js";
 
@@ -27,6 +27,7 @@ import {
 	savePendingReflection,
 	savePendingDropped,
 	isObservationChunkPending,
+	PendingOMState,
 } from "./pending.js";
 import {
 	OM_OBSERVATIONS_DROPPED,
@@ -98,6 +99,8 @@ function capSourceEntriesToTokens(entries: Entry[], maxTokens: number): Entry[]
 	for (let i = entries.length - 1; i >= 0; i--) {
 		const entry = entries[i];
 		let chars = 0;
+		// Tokenize all entry types, not just "message": custom_message and
+		// branch_summary entries also consume observer context window.
 		if (entry.type === "message" && entry.message) {
 			const msg = entry.message as any;
 			if (typeof msg.content === "string") chars = msg.content.length;
@@ -106,9 +109,25 @@ function capSourceEntriesToTokens(entries: Entry[], maxTokens: number): Entry[]
 					if (block.text) chars += block.text.length;
 				}
 			}
+		} else if (entry.type === "custom" && (entry.customType === OM_OBSERVATIONS_RECORDED || entry.customType === OM_REFLECTIONS_RECORDED || entry.customType === OM_OBSERVATIONS_DROPPED)) {
+			// Custom entries carry structured data — estimate from JSON serialization
+			chars = String(JSON.stringify(entry.data ?? {})).length;
+		} else if (entry.summary) {
+			chars = String(entry.summary).length;
 		}
 		const estTokens = Math.ceil(chars / 4);
 		if (totalTokens + estTokens > maxTokens && kept.length > 0) break;
+		// Remove the `kept.length > 0` guard? No — keep the guard but allow
+		// the first entry to be dropped only if it exceeds maxTokens alone.
+		// (The guard against empty kept list prevents dropping the first entry
+		// when later entries are small; but a single oversized entry should
+		// still be included to avoid losing the newest data entirely.)
+		if (totalTokens + estTokens > maxTokens && kept.length === 0) {
+			// First (newest) entry exceeds maxTokens alone — include it anyway
+			// to avoid data loss, but don't add more.
+			kept.unshift(entry);
+			break;
+		}
 		kept.unshift(entry);
 		totalTokens += estTokens;
 	}
@@ -140,7 +159,7 @@ function mergeReflections(existing: Reflection[], additional: Reflection[]): Ref
  * should still be served as "new" on subsequent runs.
  */
 function pendingObservationsCreatedAfter(
-	pending: any,
+	pending: PendingOMState,
 	entries: Entry[],
 	afterCoversUpToId: string | undefined,
 ): Observation[] {
@@ -315,8 +334,6 @@ async function runObserverStage(
 	// of processing everything including the compaction summary.
 	const effectiveStart = lastCoverageIdx >= 0 ? lastCoverageIdx : findLastCompactionIndex(entries);
 	let chunkEntries = sourceEntriesAfter(entries, effectiveStart);
-	const coversUpToId = chunkEntries.at(-1)?.id;
-	if (!coversUpToId) return "continue";
 
 	// Cap observer input to observerChunkMaxTokens (newest-to-oldest)
 	const maxChunkTokens = runtime.config.observerChunkMaxTokens;
@@ -324,6 +341,10 @@ async function runObserverStage(
 		chunkEntries = capSourceEntriesToTokens(chunkEntries, maxChunkTokens);
 	}
 
+	// coversUpToId must point to the LAST entry AFTER capping, not before
+	const coversUpToId = chunkEntries.at(-1)?.id;
+	if (!coversUpToId) return "continue";
+
 	const { text: chunk, sourceEntryIds } = serializeSourceAddressedBranchEntries(chunkEntries);
 	if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
 	const chunkTokens = Math.ceil(chunk.length / 4);
@@ -390,8 +411,9 @@ async function runObserverStage(
 		const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "observer"), stageFallbacks: stageFallbackModels(runtime, "observer") });
 
 		// Check if estimated input fits in model's context window
+		// Use actual chunk tokens (already computed) instead of the configured cap
 		const effectiveObsCtx = effectiveContextWindow(resolved.model as any, stageModelForThinking);
-		const observerEstimatedInput = runtime.config.observerChunkMaxTokens + AGENT_LOOP_RESERVE;
+		const observerEstimatedInput = chunkTokens + AGENT_LOOP_RESERVE;
 		if (observerEstimatedInput > effectiveObsCtx) {
 			debugLog("observer.context_window_exceeded", { estimatedInput: observerEstimatedInput, effectiveCtx: effectiveObsCtx, model: `${(resolved.model as any).provider}/${(resolved.model as any).id}` });
 			runtime.recordRetryableError(stageModelForThinking, new Error(`context window ${effectiveObsCtx} too small for estimated input ${observerEstimatedInput}`), "observer");
@@ -474,7 +496,7 @@ async function runReflectorStage(
 ): Promise<ReflectorStageResult> {
 	const sessionId = ctx.sessionManager.getSessionId();
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	let reflectionTokens: number;
+	let reflectionTokens = 0;
 	let observationCoverageId: string | undefined;
 	if (runtime.config.noAutoCompact) {
 		const pending = readPendingState(sessionId);
@@ -523,8 +545,7 @@ async function runReflectorStage(
 		// Adjust accumulated for pending coverage in noAutoCompact mode
 		let effectiveReflectionTokens = reflectionTokens;
 		if (runtime.config.noAutoCompact) {
-			const pending = readPendingState(sessionId);
-			if (pending.reflection?.coversUpToId) {
+			if (pending?.reflection?.coversUpToId) {
 				const idx = entryIndexForId(entries, pending.reflection.coversUpToId);
 				if (idx >= 0) effectiveReflectionTokens = rawTokensAfterIndex(entries, idx);
 			}
@@ -535,8 +556,9 @@ async function runReflectorStage(
 		const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "reflector"), stageFallbacks: stageFallbackModels(runtime, "reflector") });
 
 		// Check if estimated input fits in model's context window
+		// Use actual computed input size (new items + summary budget) instead of cap
 		const effectiveRefCtx = effectiveContextWindow(resolved.model as any, stageModelForThinking);
-		const reflectorEstimatedInput = runtime.config.reflectorInputMaxTokens + AGENT_LOOP_RESERVE;
+		const reflectorEstimatedInput = reflectorInputTokens + AGENT_LOOP_RESERVE;
 		if (reflectorEstimatedInput > effectiveRefCtx) {
 			debugLog("reflector.context_window_exceeded", { estimatedInput: reflectorEstimatedInput, effectiveCtx: effectiveRefCtx, model: `${(resolved.model as any).provider}/${(resolved.model as any).id}` });
 			runtime.recordRetryableError(stageModelForThinking, new Error(`context window ${effectiveRefCtx} too small for estimated input ${reflectorEstimatedInput}`), "reflector");
@@ -614,7 +636,7 @@ async function runDropperStage(
 ): Promise<StageOutcome> {
 	const sessionId = ctx.sessionManager.getSessionId();
 	const entries = ctx.sessionManager.getBranch() as Entry[];
-	let dropTokens: number;
+	let dropTokens = 0;
 	let observationCoverageId: string | undefined;
 	if (runtime.config.noAutoCompact) {
 		const pending = readPendingState(sessionId);
@@ -661,8 +683,7 @@ async function runDropperStage(
 		// Adjust accumulated for pending coverage in noAutoCompact mode
 		let effectiveDropTokens = dropTokens;
 		if (runtime.config.noAutoCompact) {
-			const pending = readPendingState(sessionId);
-			if (pending.dropped?.coversUpToId) {
+			if (pending?.dropped?.coversUpToId) {
 				const idx = entryIndexForId(entries, pending.dropped.coversUpToId);
 				if (idx >= 0) effectiveDropTokens = rawTokensAfterIndex(entries, idx);
 			}
@@ -692,8 +713,9 @@ async function runDropperStage(
 			const stageModelForThinking = runtime.findCandidateConfig(resolved.model, { model: ctx.model, modelRegistry: ctx.modelRegistry, hasUI: ctx.hasUI, ui: ctx.ui, stageModel: stageModelConfig(runtime, "dropper"), stageFallbacks: stageFallbackModels(runtime, "dropper") });
 
 			// Check if estimated input fits in model's context window
+			// Use actual computed input size (new observations + summary budget) instead of cap
 			const effectiveDropCtx = effectiveContextWindow(resolved.model as any, stageModelForThinking);
-			const dropperEstimatedInput = runtime.config.dropperInputMaxTokens + AGENT_LOOP_RESERVE;
+			const dropperEstimatedInput = dropperInputTokens + AGENT_LOOP_RESERVE;
 			if (dropperEstimatedInput > effectiveDropCtx) {
 				debugLog("dropper.context_window_exceeded", { estimatedInput: dropperEstimatedInput, effectiveCtx: effectiveDropCtx, model: `${(resolved.model as any).provider}/${(resolved.model as any).id}` });
 				runtime.recordRetryableError(stageModelForThinking, new Error(`context window ${effectiveDropCtx} too small for estimated input ${dropperEstimatedInput}`), "dropper");