pi-mega-compact 0.11.3 → 0.11.5

package/extensions/mega-events/agent-handlers.ts

@@ -18,11 +18,13 @@ import { isMegaCache } from "../../src/game/scoring.js";
 import { resolveRepoRoot } from "../mega-config.js";
 import {
 	classifyError,
+	classifyErrorDetailed,
 	errorRetryBackoffMs,
 	extractErrorSignature,
 	isKnownRetryableTransient,
 } from "./error-classifier.js";
 import { safeSendUserMessage } from "./send-safe.js";
+import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
 import { vectorStats } from "../../src/vectorStore.js";
 
 /** Register agent/turn tracking event handlers. */
@@ -187,7 +189,10 @@ export function registerAgentHandlers(
 							setTimeout(() => {
 								try {
 									if (runtime.rt.sessionId !== liveSid) return; // session reset
-									const since2 = now - (runtime.rt.lastNativeCompactAt ?? 0);
+									// IMPORTANT: recompute Date.now() inside the timeout.
+									// `now` above is the durable-trim handler's timestamp and can be
+									// ~500ms stale by the time this callback runs.
+									const since2 = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
 									if (
 										runtime.rt.lastNativeCompactAt !== stamp &&
 										since2 < cooldownMs
@@ -265,6 +270,16 @@ export function registerAgentHandlers(
 		runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
 		runtime.snapshot(ctx);
 
+		// S53: consume staged recall blocks ONLY if they were actually injected
+		// into a view this turn. If no context event fired (edge: turn ended
+		// before any LLM call), recallInjectedThisTurn stays false and the blocks
+		// remain staged for the next turn's first context event.
+		if (config.recallTailInject && runtime.rt.recallInjectedThisTurn) {
+			runtime.pendingRecallBlock = undefined;
+			runtime.pendingMemoryRecallBlock = undefined;
+			runtime.rt.recallInjectedThisTurn = false;
+		}
+
 		// S43 (per-turn tracking): record one turn row with the cached metrics so
 		// the turn layer is queryable + forkable. Best-effort + non-fatal: a write
 		// failure never breaks the agent loop.
@@ -393,6 +408,7 @@ export function registerAgentHandlers(
 				// S28 handles; nothing for S38 to do here.
 			} else {
 				const category = classifyError(event.message);
+				const detail = classifyErrorDetailed(event.message);
 				// R3: stateful poisoned-context signal — repeated identical error text
 				// across consecutive turns. The classifier is stateless, so this
 				// upgrade happens here. A 'transient' (or 'poisoned-context') turn
@@ -430,6 +446,12 @@ export function registerAgentHandlers(
 							// (error-classifier.ts) so the two never drift.
 							if (!isKnownRetryableTransient(errSig)) {
 								effectiveCategory = "poisoned-context";
+							} else {
+								runtime.logger.info("repeat-upgrade-declined", {
+									sessionId: runtime.rt.sessionId,
+									signature: (errSig || '').slice(0, 500),
+									repeatCount: runtime.rt.errorTextRepeatCount,
+								});
 							}
 						}
 					}
@@ -445,6 +467,9 @@ export function registerAgentHandlers(
 					runtime.rt.consecutiveErrors = 0; // S38.6: circuit-breaker reset on success
 					// R4: a successful assistant turn consumes any queued nudge.
 					runtime.rt.retryNudgePending = false;
+					// R10: reset outage advisory so a recovered-then-flapping
+					// provider re-advises once per episode.
+					runtime.rt.providerOutageAdvised = false;
 				} else if (effectiveCategory === "compaction-noop") {
 					// (4) pi race / manual compact catch — NOT retryable. The compaction
 					// already succeeded via pi's native path; retrying would race again
@@ -540,22 +565,31 @@ export function registerAgentHandlers(
 					runtime.rt.consecutiveErrors++; // still counts toward the circuit breaker
 					runtime.rt.poisonedCount++; // R7: dashboard counter
 					const sig = errSig || "unknown";
-					// (a) dashboard + log
+					// (a) dashboard + log — R11: include signal + rawText for forensics
 					runtime.dashboard.event("poisoned_context", {
 						signature: sig,
 						repeatCount: runtime.rt.errorTextRepeatCount,
 						turnIndex: event.turnIndex,
 						sessionId: runtime.rt.sessionId,
+						signal: detail.signal,
+						rawText: (errSig || '').slice(0, 500),
+						...(detail.httpStatus !== undefined ? { httpStatus: detail.httpStatus } : {}),
 					});
 					runtime.logger.warn("poisoned-context", {
 						sessionId: runtime.rt.sessionId,
 						turnIndex: event.turnIndex,
 						signature: sig,
 						repeatCount: runtime.rt.errorTextRepeatCount,
+						signal: detail.signal,
+						rawText: (errSig || '').slice(0, 500),
+						...(detail.httpStatus !== undefined ? { httpStatus: detail.httpStatus } : {}),
 					});
-					// (b) one-per-session advise message (throttled by poisonedAdviseSent)
-					if (!runtime.rt.poisonedAdviseSent) {
-						runtime.rt.poisonedAdviseSent = true;
+					// (b) throttle flag: set unconditionally so the advise fires at
+					// most once per session regardless of channel.
+					runtime.rt.poisonedAdviseSent = true;
+					// Legacy flag-OFF path: user-role advise message.
+					// Default ON (advisoryChannel=true): dashboard-only, no user message.
+					if (!config.advisoryChannel) {
 						// PREVENT-PI-003: user-role sendUserMessage only.
 						await safeSendUserMessage(
 							pi,
@@ -607,6 +641,11 @@ export function registerAgentHandlers(
 					}
 					// S38.6: circuit-breaker — stop retrying after too many consecutive errors.
 					runtime.rt.consecutiveErrors++;
+					// R10: send calm "provider outage" advisory once per episode.
+					await maybeSendProviderOutageAdvisory(
+						effectiveCategory, runtime, pi, config,
+						{ signal: detail.signal, rawText: (errSig || '').slice(0, 500) },
+					);
 					if (runtime.rt.consecutiveErrors > config.maxConsecutiveErrors) {
 						runtime.dashboard.event("error_retry_circuit_open", {
 							consecutive: runtime.rt.consecutiveErrors,

package/extensions/mega-events/compact-handlers.ts

@@ -69,6 +69,7 @@ async function nudgeResume(pi: ExtensionAPI, runtime: MegaRuntime): Promise<void
 		const now = Date.now();
 		if (now >= runtime.resumeNudgeUntil) {
 			runtime.resumeNudgeUntil = now + 30_000;
+			runtime.rt.extensionInitiatedTurn = true; // R13: suppress self-classification
 			await safeSendUserMessage(
 				pi,
 				"[mega-compact] continue from the compacted context above.",

package/extensions/mega-events/context-handler.ts

@@ -13,10 +13,8 @@ import type {
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import {
 	openStore,
-	appendRawTranscript,
 	writeCheckpointEpoch,
 	type CheckpointEpoch,
-	type RawTranscriptRow,
 } from "../../src/store/sqlite.js";
 import { epochIdFor } from "../../src/mirror/epoch.js";
 import { autoCompactCheck } from "../../src/compact.js";
@@ -37,84 +35,8 @@ import {
 	pressureRatio,
 	type MegaConfig,
 } from "../mega-config.js";
-import { computeContentDigest } from "../../src/dedup/digest.js";
-
-/**
- * Recursively canonicalize a value for deterministic JSON serialization:
- * - Objects: keys sorted alphabetically, values recursively canonicalized
- * - Arrays: elements recursively canonicalized (array ORDER is preserved)
- * - Primitives: returned as-is
- *
- * F5 fix: shallow-sorted JSON.stringify(content, Object.keys(content).sort())
- * omitted nested keys not in the top-level keys array and only sorted one level.
- * This recursively sorts every object at every depth so semantically-equal
- * differently-ordered content hashes identically.
- */
-function canonicalize(value: unknown): unknown {
-	if (value === null || value === undefined) return value;
-	if (typeof value !== "object") return value;
-	if (Array.isArray(value)) return value.map(canonicalize);
-	// Plain object: sort keys, recursively canonicalize values.
-	const sorted = Object.keys(value as Record<string, unknown>).sort();
-	const out: Record<string, unknown> = {};
-	for (const k of sorted) {
-		out[k] = canonicalize((value as Record<string, unknown>)[k]);
-	}
-	return out;
-}
-
-/**
- * Convert a pi AgentMessage to a RawTranscriptRow for the DB mirror.
- * content_bytes is canonical JSON (sorted keys, recursive) for deterministic
- * storage; contentHash is the canonical digest (normalize + hash) that matches
- * what dedupTranscript computes so the two pipelines use the same linkage key.
- *
- * F4 fix: the hash key MUST match on both sides of the append/dedup split.
- * Using computeContentDigest here (normalize → hash) ensures content_ref set
- * by dedupTranscript always resolves to an existing dedup_mirror row even for
- * whitespace/case-variant content. F5 fix: recursive canonicalize replaces the
- * broken shallow-sorted JSON.stringify replacer.
- */
-function toRawTranscriptRow(
-	msg: AgentMessage,
-	sessionId: string,
-	epochId: string,
-	currentTurn?: number,
-): RawTranscriptRow | null {
-	// Narrow to Message union (has content + timestamp).
-	const m = msg as {
-		role?: string;
-		content?: unknown;
-		timestamp?: number;
-		toolName?: string;
-	};
-	const content = m.content;
-	if (content == null || content === "") return null;
-	// Canonical bytes: string content → normalize for consistent byte content;
-	// object content → recursive canonicalize then JSON (no replacer).
-	const contentBytes =
-		typeof content === "string"
-			? content // F5: strings are primitives; their byte content is fixed
-			: JSON.stringify(canonicalize(content));
-	// F4 fix: use the same digest as dedupTranscript so contentHash is consistent
-	// on both sides of the append/dedup split. computeContentDigest normalizes
-	// (strip ANSI, NFC, case-fold, collapse whitespace) then hashes → the same
-	// key is stored in raw_transcript.content_hash AND used as the dedup link.
-	const { contentHash } = computeContentDigest(contentBytes);
-	return {
-		contentHash,
-		sessionId,
-		seq: 0, // assigned by appendRawTranscript (COALESCE(MAX(seq),0)+1)
-		role: m.role ?? "unknown",
-		contentBytes,
-		toolName: m.toolName ?? null,
-		messageTimestamp: m.timestamp ?? null,
-		checkpointEpoch: epochId,
-		// S50: label the row with the turn that produced it (per-turn dedup /
-		// compression-by-turn metrics). Null when the writer omits it (back-compat).
-		turnIndex: currentTurn ?? null,
-	};
-}
+import { appendMirrorMessages } from "./mirror-append.js";
+import { stagedForTail, withRecallTail } from "./recall-tail.js";
 
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(
@@ -140,6 +62,16 @@ export function registerContextHandler(
 		const usage = ctx.getContextUsage();
 		const pct = usage?.percent;
 		const messages = event.messages;
+		// S53: helper to inject the staged recall/memory block as a user-role
+		// tail message at any view-return point. Returns undefined when nothing
+		// is staged (or the flag is OFF) so the caller falls through to its
+		// normal return. The F3 mirror append (above the gates) runs on the
+		// REAL transcript before any view is built, so the injected tail never
+		// reaches raw_transcript (PREVENT-PI: append-only, view-only).
+		const tailResult = (msgs?: readonly AgentMessage[]) => {
+			if (!stagedForTail(runtime, config)) return undefined;
+			return { messages: withRecallTail(msgs ?? messages, runtime, config) };
+		};
 		// Always track context for the dashboard/widget, even when auto is off.
 		// (v0.8 regression: !config.auto gate sat above this, leaving ctx stats
 		// null -> widget '?% / ?/?' when auto disabled. Track first, THEN gate.)
@@ -162,30 +94,35 @@ export function registerContextHandler(
 		runtime.lastCtxPercent = pct ?? null;
 		runtime.lastCtxWindow = usage?.contextWindow ?? 0;
 		runtime.snapshot(ctx);
-		if (!config.auto) return;
+		if (!config.auto) {
+			const tailed = tailResult();
+			if (tailed) return tailed;
+			return;
+		}
 
 		const view = viewForFallback ?? runtime.engineView(messages);
 
-		// S27 DB-mirror: append ALL incoming messages to raw_transcript.
-		// Runs BEFORE fast-gate so every message is captured, even if we
-		// don't compact this turn. Append is idempotent (content_hash PK).
-		if (config.dbMirror) {
-			try {
-				const db = openStore(runtime.currentStateDir);
-				const epochId = epochIdFor(runtime.rt.sessionId);
-				for (const msg of messages) {
-					const raw = toRawTranscriptRow(
-						msg,
+		// S27 DB-mirror: append incoming messages to raw_transcript.
+			// Runs BEFORE fast-gate so every message is captured, even if we
+			// don't compact this turn. Append is idempotent (content_hash PK).
+			// F3: high-water mark (mirror-append.ts) skips already-processed
+			// messages on subsequent events. On fork/rewind (shorter list or
+			// boundary hash mismatch) the mark is dropped, falling back to a
+			// full reprocess.
+			if (config.dbMirror) {
+				try {
+					const db = openStore(runtime.currentStateDir);
+					appendMirrorMessages(
+						db,
+						messages,
 						runtime.rt.sessionId,
-						epochId,
+						epochIdFor(runtime.rt.sessionId),
 						runtime.currentTurn,
 					);
-					if (raw) appendRawTranscript(db, raw);
+				} catch (e) {
+					runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
 				}
-			} catch (e) {
-				runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
 			}
-		}
 
 		// S29 FAST GATE: drive the auto-trigger off the context % (the number the
 		// menu bar shows), NOT the token count — the model under-reports tokens,
@@ -205,18 +142,18 @@ export function registerContextHandler(
 			// custom tier OR tiered-but-pct-unavailable → token gate (S27 fallback).
 			if (currentTokens < runtime.effectiveThreshold) {
 				runtime.diagCtxFastGate++;
-				return;
+				return tailResult() ?? undefined;
 			}
 			const check = autoCompactCheck(currentTokens, runtime.effectiveThreshold); // SERVER-STYLE CONFIRM (local)
 			if (!check.shouldCompact) {
 				runtime.diagCtxNoCompact++;
-				return;
+				return tailResult() ?? undefined;
 			}
 			gatePassed = true;
 		}
 		if (!gatePassed) {
 			runtime.diagCtxFastGate++;
-			return;
+			return tailResult() ?? undefined;
 		}
 
 		// D.2: Replay MUST be exempt from debounce — replay is free (no compute,
@@ -249,9 +186,8 @@ export function registerContextHandler(
 				runtime.snapshot(ctx);
 				// v0.8.7: shallow-copy the cached summary so pi's transformContext can't
 				// mutate the shared reference across replays (audit P3).
-				return {
-					messages: [{ ...runtime.trimCache.summaryAgentMsg }, ...recent],
-				};
+				const replayView = [{ ...runtime.trimCache.summaryAgentMsg }, ...recent];
+				return tailResult(replayView) ?? { messages: replayView };
 			}
 			// else: context grew enough → fall through to re-compact (cache is stale)
 		}
@@ -261,7 +197,7 @@ export function registerContextHandler(
 		const now = Date.now();
 		if (now < runtime.debounceUntil) {
 			runtime.diagCtxDebounce++;
-			return;
+			return tailResult() ?? undefined;
 		}
 		runtime.debounceUntil = now + 2000;
 
@@ -290,11 +226,10 @@ export function registerContextHandler(
 				runtime.diagLiveTrimFires++;
 				runtime.diagLiveTrimReplays++;
 				runtime.snapshot(ctx);
-				return {
-					messages: [{ ...runtime.trimCache.summaryAgentMsg }, ...recent],
-				};
+				const skipView = [{ ...runtime.trimCache.summaryAgentMsg }, ...recent];
+				return tailResult(skipView) ?? { messages: skipView };
 			}
-			return;
+			return tailResult() ?? undefined;
 		}
 
 		// S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
@@ -460,7 +395,7 @@ export function registerContextHandler(
 					viewLen: view.length,
 					anchorUserMessages,
 				});
-				return; // unsafe / below anchor floor — no trim this call
+				return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
 			}
 			const summaryMsg = liveTrimSummaryMessage({
 				compactedFrom: ran.result.compactedFrom,
@@ -515,10 +450,10 @@ export function registerContextHandler(
 				ctxPct: pct,
 				ctxTokens: usage?.tokens ?? null,
 			});
-			return { messages: [summaryAgentMsg, ...recent] };
+			return tailResult([summaryAgentMsg, ...recent]) ?? { messages: [summaryAgentMsg, ...recent] };
 		} catch {
 			runtime.diagCtxThrown++;
-			return; // non-fatal: no trim this call; the next context event retries
+			return tailResult() ?? undefined; // non-fatal: no trim this call; the next context event retries
 		}
 	});
 }

package/extensions/mega-events/context-hwm.ts

@@ -0,0 +1,85 @@
+/**
+ * mega-events/context-hwm.ts — high-water mark for DB-mirror raw_transcript
+ * incremental append (F3 sprint).
+ *
+ * Stores the last-processed seq + content_hash per session so the context
+ * event only appends the tail on subsequent events. On fork/rewind (shorter
+ * message list or hash mismatch at the boundary), the mark is dropped so the
+ * next event falls back to a full reprocess.
+ *
+ * Non-fatal: every function logs and swallows errors; the agent loop is never
+ * broken (repo invariant).
+ *
+ * All queries parameterized (PREVENT-002); no `any` (PREVENT-011).
+ */
+import type { DatabaseSync } from "node:sqlite";
+
+export interface ContextHwm {
+  sessionId: string;
+  lastSeq: number;
+  lastContentHash: string;
+}
+
+const TABLE_DDL = `CREATE TABLE IF NOT EXISTS context_hwm (
+  session_id        TEXT PRIMARY KEY,
+  last_seq          INTEGER NOT NULL,
+  last_content_hash TEXT NOT NULL
+)`;
+
+/** Ensure the context_hwm table exists. Idempotent. */
+export function ensureHwmTable(db: DatabaseSync): void {
+  try {
+    db.exec(TABLE_DDL);
+  } catch {
+    // non-fatal
+  }
+}
+
+/** Read the high-water mark for a session. Returns undefined if none. */
+export function readHwm(
+  db: DatabaseSync,
+  sessionId: string,
+): ContextHwm | undefined {
+  try {
+    const row = db
+      .prepare(
+        "SELECT session_id, last_seq, last_content_hash FROM context_hwm WHERE session_id = ?",
+      )
+      .get(sessionId) as
+      | { session_id: string; last_seq: number; last_content_hash: string }
+      | undefined;
+    if (!row) return undefined;
+    return {
+      sessionId: row.session_id,
+      lastSeq: Number(row.last_seq),
+      lastContentHash: row.last_content_hash,
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+/** Write (INSERT OR REPLACE) the high-water mark for a session. */
+export function writeHwm(
+  db: DatabaseSync,
+  hwm: ContextHwm,
+): void {
+  try {
+    db.prepare(
+      "INSERT OR REPLACE INTO context_hwm (session_id, last_seq, last_content_hash) VALUES (?, ?, ?)",
+    ).run(hwm.sessionId, hwm.lastSeq, hwm.lastContentHash);
+  } catch {
+    // non-fatal
+  }
+}
+
+/** Drop the high-water mark for a session (invalidation). */
+export function dropHwm(db: DatabaseSync, sessionId: string): void {
+  try {
+    db.prepare("DELETE FROM context_hwm WHERE session_id = ?").run(sessionId);
+  } catch {
+    // non-fatal
+  }
+}
+
+export const TEST_ONLY = { TABLE_DDL };