pi-observational-memory-extension 0.2.0 → 0.2.1

package/CHANGELOG.md

@@ -23,6 +23,19 @@ When preparing a new release:
 
 ---
 
+## [0.2.1] - 2026-06-23
+
+### Fixed
+- **status**: Restored the colored OM footer metrics after reloads and session restarts.
+- **status**: Prevented failed OM states from permanently disabling future observation attempts.
+- **status**: Clamped footer percentages to readable `100%+` overflow instead of noisy `999%` values.
+- **observer**: Forced/block-after observation now processes bounded chunks instead of sending the entire backlog to the Observer.
+- **observer**: Automatically recovers from oversized stale backlogs by retaining the latest tail-window and recording a diagnostic observation.
+- **observer**: Treats empty observer output and stale reload races as non-fatal skips that advance the cursor safely.
+- **ui**: Added resilient status bootstrapping on `session_start`, `agent_start`, and `turn_start`.
+- **recovery**: Shows an explicit OM footer error if state recovery fails instead of leaving the status area empty.
+- **recovery**: Removed stdout/stderr recovery logging so internal extension diagnostics do not leak into chat transcripts.
+
 ## [0.2.0] - 2026-06-23
 
 ### Added

package/extensions/index.ts

@@ -20,6 +20,8 @@ const DEFAULT_REFLECTION_MODEL = "google/gemini-2.5-flash";
 const OBSERVATION_THRESHOLD = 30_000;
 const REFLECTION_THRESHOLD = 40_000;
 const OBSERVER_MAX_BATCH_TOKENS = 10_000;
+const MAX_PENDING_BACKLOG_TOKENS = OBSERVER_MAX_BATCH_TOKENS * 12;
+const RETAIN_PENDING_TAIL_TOKENS = OBSERVER_MAX_BATCH_TOKENS * 3;
 const BUFFER_TOKENS_RATIO = 0.2;
 const BUFFER_ACTIVATION_RATIO = 0.8;
 const DEFAULT_BLOCK_AFTER_MULTIPLIER = 1.2;
@@ -496,9 +498,8 @@ export default function (pi: ExtensionAPI) {
   });
 
   pi.on("session_start", async (_event, ctx) => {
-    await ensureState(ctx);
-    await refreshCounts(ctx);
-    updateStatus(ctx);
+    await bootstrapStatus(ctx, "session_start");
+    if (runtime.statusTimer) clearInterval(runtime.statusTimer);
     runtime.statusTimer = setInterval(() => updateStatus(ctx), 1000);
 
     if (!checkedForUpdate) {
@@ -509,6 +510,14 @@ export default function (pi: ExtensionAPI) {
     }
   });
 
+  pi.on("agent_start", async (_event, ctx) => {
+    await bootstrapStatus(ctx, "agent_start");
+  });
+
+  pi.on("turn_start", async (_event, ctx) => {
+    await bootstrapStatus(ctx, "turn_start");
+  });
+
   pi.on("session_shutdown", async (_event, ctx) => {
     if (runtime.statusTimer) clearInterval(runtime.statusTimer);
     runtime.currentOperation?.abort();
@@ -810,7 +819,13 @@ async function safeBoundary(ctx: any, boundary: string): Promise<void> {
   if (isStale) return;
 
   const state = await ensureState(ctx);
-  if (!state.enabled || state.status === "failed") return;
+  if (!state.enabled) return;
+  if (state.status === "failed") {
+    state.status = "idle";
+    state.lastError = undefined;
+    state.operationLock = undefined;
+    await saveState(state);
+  }
   await refreshCounts(ctx);
   updateStatus(ctx);
   if (runtime.currentOperation) return;
@@ -871,6 +886,7 @@ async function ensureState(ctx: any): Promise<PiOMRecord> {
   runtime.debugDir = debugDir;
   let state: PiOMRecord | undefined;
   let statePath = legacyStatePath;
+  let recoveryWarning: string | undefined;
   if (existsSync(legacyStatePath)) {
     try {
       state = normalizeState(JSON.parse(await readFile(legacyStatePath, "utf8")), { sessionId, sessionFile, cwd: ctx.cwd });
@@ -880,14 +896,14 @@ async function ensureState(ctx: any): Promise<PiOMRecord> {
         try {
           state = normalizeState(JSON.parse(await readFile(backupPath, "utf8")), { sessionId, sessionFile, cwd: ctx.cwd });
         } catch {
-          console.error(`[OM] Error: Failed to parse state and backup for ${legacyStatePath}. Falling back to default empty state.`);
+          recoveryWarning = `Recovered from corrupted OM state and backup at ${legacyStatePath}; moved them to .corrupted files.`;
           try {
             await rename(legacyStatePath, `${legacyStatePath}.corrupted`);
             await rename(backupPath, `${backupPath}.corrupted`);
           } catch {}
         }
       } else {
-        console.error(`[OM] Error: Failed to parse state for ${legacyStatePath} (no backup). Falling back to default empty state.`);
+        recoveryWarning = `Recovered from corrupted OM state at ${legacyStatePath}; moved it to a .corrupted file.`;
         try {
           await rename(legacyStatePath, `${legacyStatePath}.corrupted`);
         } catch {}
@@ -896,6 +912,7 @@ async function ensureState(ctx: any): Promise<PiOMRecord> {
   }
   if (!state) {
     state = createDefaultState({ sessionId, sessionFile, cwd: ctx.cwd });
+    if (recoveryWarning) state.lastError = recoveryWarning;
   }
   statePath = statePathFor(ctx.cwd, state.scope, sessionId);
   if (statePath !== legacyStatePath && existsSync(statePath)) {
@@ -1162,14 +1179,57 @@ async function refreshCounts(ctx: any): Promise<void> {
   const state = await ensureState(ctx);
   const branch = ctx.sessionManager.getBranch() as SessionEntry[];
   const pending = entriesAfter(branch, state.lastObservedEntryId).filter(isMessageLikeEntry);
-  state.pendingMessageTokens = estimateEntriesTokens(pending);
+  const pendingTokens = estimateEntriesTokens(pending);
+  if (pendingTokens > MAX_PENDING_BACKLOG_TOKENS) {
+    const retained = takeTailEntriesUpTo(pending, RETAIN_PENDING_TAIL_TOKENS);
+    const firstRetained = retained[0];
+    const firstRetainedIndex = firstRetained ? pending.findIndex(e => e.id === firstRetained.id) : -1;
+    const cursor = firstRetainedIndex > 0 ? pending[firstRetainedIndex - 1] : undefined;
+    if (cursor?.id) state.lastObservedEntryId = cursor.id;
+    const retainedTokens = estimateEntriesTokens(retained);
+    const skippedTokens = Math.max(0, pendingTokens - retainedTokens);
+    const note = `OM recovery: skipped ~${formatTokens(skippedTokens)} tokens of stale unobserved backlog after reload/corruption recovery; retained latest ~${formatTokens(retainedTokens)} tokens for observation.`;
+    state.observations = suppressDuplicateObservations(state.observations, `- ${note}`);
+    state.lastOperation = { type: "observation", startedAt: new Date().toISOString(), endedAt: new Date().toISOString(), inputTokens: 0, outputTokens: estimateTokens(note) };
+    state.lastError = note;
+    state.pendingMessageTokens = retainedTokens;
+  } else {
+    state.pendingMessageTokens = pendingTokens;
+  }
   state.observationTokens = estimateTokens(state.observations);
   await saveState(state);
 }
 
+async function bootstrapStatus(ctx: any, reason: string): Promise<void> {
+  try {
+    const state = await ensureState(ctx);
+    updateStatus(ctx);
+    try {
+      await refreshCounts(ctx);
+    } catch (error) {
+      state.lastError = `OM status refresh failed during ${reason}: ${errorMessage(error)}`;
+      await saveState(state);
+    }
+    updateStatus(ctx);
+  } catch (error) {
+    try {
+      ctx.ui.setStatus("om", `\x1b[31mOM error\x1b[0m \x1b[2m${truncateToWidth(errorMessage(error), 80, "…")}\x1b[0m`);
+    } catch {
+      // Context might be stale or UI disposed
+    }
+  }
+}
+
 function updateStatus(ctx: any): void {
   const state = runtime.state;
-  if (!state) return;
+  if (!state) {
+    try {
+      ctx.ui.setStatus("om", "\x1b[2;37mOM loading…\x1b[0m");
+    } catch {
+      // Context might be stale or UI disposed
+    }
+    return;
+  }
   try {
     ctx.ui.setStatus("om", formatShortStatusColored(state));
   } catch (e) {
@@ -1192,7 +1252,7 @@ async function observeNow(ctx: any, opts: { force: boolean; reason: string; sign
   const branch = ctx.sessionManager.getBranch() as SessionEntry[];
   const candidates = entriesAfter(branch, state.lastObservedEntryId).filter(isMessageLikeEntry);
   if (candidates.length === 0 && !opts.manualText) return;
-  const selected = opts.force ? candidates : selectEntriesForObservation(candidates, state);
+  const selected = opts.force ? selectEntriesForForcedObservation(candidates) : selectEntriesForObservation(candidates, state);
   if (selected.length === 0 && !opts.manualText) return;
   const observerInput = opts.manualText
     ? { text: opts.manualText, attachmentParts: [] as any[], omissions: [] as string[] }
@@ -1218,9 +1278,21 @@ async function observeNow(ctx: any, opts: { force: boolean; reason: string; sign
     await writeDebug(ctx, "observation", { startedAt, reason: opts.reason, inputText, attachmentOmissions: observerInput.omissions, attachmentCount: observerInput.attachmentParts.length, rawOutput: result.rawOutput, parsed: result });
     await saveState(state);
   } catch (error) {
+    const message = errorMessage(error);
+    const last = selected.at(-1);
+    if (!opts.manualText && (message.includes("Observer returned no observations") || message.includes("ctx is stale"))) {
+      if (last?.id) state.lastObservedEntryId = last.id;
+      state.pendingMessageTokens = Math.max(0, state.pendingMessageTokens - estimateEntriesTokens(selected));
+      state.operationLock = undefined;
+      state.status = "idle";
+      state.lastError = undefined;
+      state.lastOperation = { type: "observation", startedAt, endedAt: new Date().toISOString(), inputTokens: estimateTokens(inputText), error: `Non-fatal observer skip: ${message}` };
+      await saveState(state);
+      return;
+    }
     state.operationLock = undefined;
     state.status = "failed";
-    state.lastError = `Observational Memory Observer failed: ${errorMessage(error)}`;
+    state.lastError = `Observational Memory Observer failed: ${message}`;
     state.lastOperation = { type: "observation", startedAt, endedAt: new Date().toISOString(), inputTokens: estimateTokens(inputText), error: state.lastError };
     await saveState(state);
     throw new Error(state.lastError);
@@ -1737,6 +1809,10 @@ function selectEntriesForObservation(entries: SessionEntry[], state: PiOMRecord)
   return takeEntriesUpTo(entries, Math.min(target, OBSERVER_MAX_BATCH_TOKENS));
 }
 
+function selectEntriesForForcedObservation(entries: SessionEntry[]): SessionEntry[] {
+  return takeEntriesUpTo(entries, OBSERVER_MAX_BATCH_TOKENS);
+}
+
 function takeEntriesUpTo(entries: SessionEntry[], targetTokens: number): SessionEntry[] {
   const selected: SessionEntry[] = [];
   let total = 0;
@@ -1748,6 +1824,18 @@ function takeEntriesUpTo(entries: SessionEntry[], targetTokens: number): Session
   return selected;
 }
 
+function takeTailEntriesUpTo(entries: SessionEntry[], targetTokens: number): SessionEntry[] {
+  const selected: SessionEntry[] = [];
+  let total = 0;
+  for (let i = entries.length - 1; i >= 0; i--) {
+    const entry = entries[i];
+    selected.unshift(entry);
+    total += estimateEntryTokens(entry);
+    if (total >= targetTokens) break;
+  }
+  return selected;
+}
+
 function entriesAfter(branch: SessionEntry[], id?: string): SessionEntry[] {
   if (!id) return branch;
   const idx = branch.findIndex(e => e.id === id);
@@ -2001,8 +2089,8 @@ function responseText(response: any): string {
 
 function formatShortStatus(state: PiOMRecord): string {
   if (!state.enabled) return "om: disabled";
-  const msgPct = Math.min(999, Math.round((state.pendingMessageTokens / effectiveObservationThreshold(state)) * 100));
-  const memPct = Math.min(999, Math.round((state.observationTokens / state.thresholds.reflection) * 100));
+  const msgPct = boundedPercent(state.pendingMessageTokens, effectiveObservationThreshold(state));
+  const memPct = boundedPercent(state.observationTokens, state.thresholds.reflection);
   const buffer = state.buffered.observations.length ? ` | buf ${state.buffered.observations.length} ↓${formatTokens(state.buffered.observations.reduce((s, c) => s + c.messageTokens, 0))}` : "";
   const err = state.status === "failed" && state.lastError ? ` | ${truncateText(state.lastError, 50).replace(/\n/g, " ")}` : "";
   return `om: ${state.status} | msg ${formatTokens(state.pendingMessageTokens)}/${formatTokens(effectiveObservationThreshold(state))} ${msgPct}% | mem ${formatTokens(state.observationTokens)}/${formatTokens(state.thresholds.reflection)} ${memPct}%${buffer}${err}`;
@@ -2011,16 +2099,16 @@ function formatShortStatus(state: PiOMRecord): string {
 function formatShortStatusColored(state: PiOMRecord): string {
   if (!state.enabled) return "\x1b[2;31mom: disabled\x1b[0m";
   
-  const msgPct = Math.min(999, Math.round((state.pendingMessageTokens / effectiveObservationThreshold(state)) * 100));
-  const memPct = Math.min(999, Math.round((state.observationTokens / state.thresholds.reflection) * 100));
+  const msgPct = boundedPercent(state.pendingMessageTokens, effectiveObservationThreshold(state));
+  const memPct = boundedPercent(state.observationTokens, state.thresholds.reflection);
   
   let statusColor = "\x1b[32m"; // green for idle
   if (state.status === "failed") statusColor = "\x1b[91m";
   else if (state.status !== "idle") statusColor = "\x1b[93m"; // yellow for active ops
 
   // Highlight percentages
-  const msgPctStr = msgPct >= 80 ? `\x1b[1;91m${msgPct}%\x1b[0m` : msgPct >= 50 ? `\x1b[1;93m${msgPct}%\x1b[0m` : `\x1b[1;32m${msgPct}%\x1b[0m`;
-  const memPctStr = memPct >= 80 ? `\x1b[1;91m${memPct}%\x1b[0m` : memPct >= 50 ? `\x1b[1;93m${memPct}%\x1b[0m` : `\x1b[1;32m${memPct}%\x1b[0m`;
+  const msgPctStr = formatPercentColored(msgPct);
+  const memPctStr = formatPercentColored(memPct);
 
   const buffer = state.buffered.observations.length 
     ? ` \x1b[2;37m|\x1b[0m buf \x1b[1;36m${state.buffered.observations.length}\x1b[0m \x1b[36m↓${formatTokens(state.buffered.observations.reduce((s, c) => s + c.messageTokens, 0))}\x1b[0m` 
@@ -2033,6 +2121,18 @@ function formatShortStatusColored(state: PiOMRecord): string {
   return `om: ${statusColor}${state.status}\x1b[0m \x1b[2;37m|\x1b[0m msg \x1b[1;33m${formatTokens(state.pendingMessageTokens)}\x1b[0m/\x1b[2;37m${formatTokens(effectiveObservationThreshold(state))}\x1b[0m [${msgPctStr}] \x1b[2;37m|\x1b[0m mem \x1b[1;36m${formatTokens(state.observationTokens)}\x1b[0m/\x1b[2;37m${formatTokens(state.thresholds.reflection)}\x1b[0m [${memPctStr}]${buffer}${err}`;
 }
 
+function boundedPercent(value: number, total: number): number {
+  if (!Number.isFinite(value) || !Number.isFinite(total) || total <= 0) return 0;
+  return Math.min(100, Math.max(0, Math.round((value / total) * 100)));
+}
+
+function formatPercentColored(percent: number): string {
+  const suffix = percent >= 100 ? "%+" : "%";
+  if (percent >= 80) return `\x1b[1;91m${percent}${suffix}\x1b[0m`;
+  if (percent >= 50) return `\x1b[1;93m${percent}${suffix}\x1b[0m`;
+  return `\x1b[1;32m${percent}${suffix}\x1b[0m`;
+}
+
 function parsePositiveIntSetting(key: string, value: string): number {
   const n = Number(value);
   if (!Number.isFinite(n) || n <= 0) throw new Error(`OM setting ${key} must be a positive number`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-observational-memory-extension",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Mastra-style Observational Memory extension for Pi compaction and runtime context.",
   "license": "MIT",
   "author": "Nikita Nosov <20nik.nosov21@gmail.com>",