pi-mega-compact 0.11.4 → 0.11.5

package/dist/extensions/mega-events/mirror-append.js

@@ -0,0 +1,107 @@
+import { appendRawTranscript } from "../../src/store/sqlite.js";
+import { computeContentDigest } from "../../src/dedup/digest.js";
+import { ensureHwmTable, readHwm, writeHwm, dropHwm, } from "./context-hwm.js";
+/**
+ * Append incoming messages to raw_transcript with HWM incremental support.
+ *
+ * On fork/rewind (shorter message list or boundary hash mismatch), the HWM
+ * is dropped and all messages are reprocessed.
+ */
+export function appendMirrorMessages(db, messages, sessionId, epochId, currentTurn) {
+    ensureHwmTable(db);
+    const hwm = readHwm(db, sessionId);
+    let tailStart = 0;
+    let needDrop = false;
+    if (hwm) {
+        if (messages.length <= hwm.lastSeq) {
+            // Session was shortened (fork/rewind/clear) — full reprocess.
+            needDrop = true;
+        }
+        else {
+            // Verify boundary integrity: hash of message at hwm.lastSeq-1
+            // must match the stored hash.
+            const boundaryMsg = messages[hwm.lastSeq - 1];
+            const boundaryHash = contentHashOf(boundaryMsg.content);
+            if (boundaryHash === hwm.lastContentHash) {
+                tailStart = hwm.lastSeq;
+            }
+            else {
+                needDrop = true;
+            }
+        }
+    }
+    if (needDrop) {
+        dropHwm(db, sessionId);
+    }
+    // Process tail (or all if tailStart === 0).
+    let rowsProcessed = 0;
+    for (let i = tailStart; i < messages.length; i++) {
+        const raw = toRawRow(messages[i], sessionId, epochId, currentTurn);
+        if (raw) {
+            appendRawTranscript(db, raw);
+            rowsProcessed++;
+        }
+    }
+    // Write new HWM.
+    if (messages.length > 0) {
+        const lastMsg = messages[messages.length - 1];
+        writeHwm(db, {
+            sessionId,
+            lastSeq: messages.length,
+            lastContentHash: contentHashOf(lastMsg.content),
+        });
+    }
+    return { rowsProcessed };
+}
+/** Single rule for hashing message content on BOTH sides of the HWM
+ *  (write + boundary check) — a mismatch here silently defeats the
+ *  incremental path (every event would full-reprocess). null/undefined → "". */
+function contentHashOf(content) {
+    if (content == null)
+        return "";
+    const bytes = typeof content === "string"
+        ? content
+        : JSON.stringify(canonicalize(content));
+    return computeContentDigest(bytes).contentHash;
+}
+/**
+ * Canonicalize a value for deterministic JSON serialization (keys sorted).
+ * This is the same logic as the inline function in context-handler.ts.
+ */
+function canonicalize(value) {
+    if (value === null || value === undefined)
+        return value;
+    if (typeof value !== "object")
+        return value;
+    if (Array.isArray(value))
+        return value.map(canonicalize);
+    const sorted = Object.keys(value).sort();
+    const out = {};
+    for (const k of sorted) {
+        out[k] = canonicalize(value[k]);
+    }
+    return out;
+}
+/** Convert an AgentMessage to a RawTranscriptRow (same logic as toRawTranscriptRow in context-handler.ts). */
+function toRawRow(msg, sessionId, epochId, currentTurn) {
+    const m = msg;
+    const content = m.content;
+    if (content == null || content === "")
+        return null;
+    const contentBytes = typeof content === "string"
+        ? content
+        : JSON.stringify(canonicalize(content));
+    return {
+        contentHash: computeContentDigest(contentBytes).contentHash,
+        sessionId,
+        seq: 0, // assigned by appendRawTranscript
+        role: m.role ?? "unknown",
+        contentBytes,
+        toolName: m.toolName ?? null,
+        messageTimestamp: m.timestamp ?? null,
+        checkpointEpoch: epochId,
+        turnIndex: currentTurn ?? null,
+    };
+}
+// Re-export for convenience when the handler only needs the table ensure.
+export { ensureHwmTable, readHwm, writeHwm, dropHwm };

package/dist/extensions/mega-events/outage-advisor.js

@@ -49,5 +49,11 @@ export async function maybeSendProviderOutageAdvisory(effectiveCategory, runtime
         ...(detail?.signal !== undefined ? { signal: detail.signal } : {}),
         ...(detail?.rawText !== undefined ? { rawText: detail.rawText } : {}),
     });
-    await safeSendUserMessage(pi, `[mega-compact] the provider is having issues (${runtime.rt.consecutiveErrors} consecutive failures — timeouts/5xx/rate-limits). Retries are bounded and continue automatically; your context is fine — do NOT clear or reset it. Work resumes as soon as the provider recovers.`);
+    // Advisory is dashboard-only (events tab + log). No user-visible message —
+    // injecting into the conversation scares users and triggers false-positive
+    // re-classification of the resulting turn (2026-07-31 incident).
+    if (!config.advisoryChannel) {
+        // Legacy path (flag OFF): user-role message injection.
+        await safeSendUserMessage(pi, `[mega-compact] the provider is having issues (${runtime.rt.consecutiveErrors} consecutive failures — timeouts/5xx/rate-limits). Retries are bounded and continue automatically; your context is fine — do NOT clear or reset it. Work resumes as soon as the provider recovers.`);
+    }
 }

package/dist/extensions/mega-events/perf-handler.js

@@ -1,10 +1,44 @@
 import { recordPerfSample } from "../../src/store/sqlite.js";
+import { classifyPrefixBreak } from "../../src/prefix-break.js";
 /** Narrow a turn_end message to its usage block when it is an assistant msg. */
 function usageOf(msg) {
     if (msg.role !== "assistant" || !msg.usage)
         return null;
     return msg.usage;
 }
+/**
+ * Attempt to record a prefix_break sample when a significant cache_hit_pct
+ * drop is detected between two consecutive turns.
+ *
+ * Drop threshold: 20 percentage points.  When triggered, calls
+ * classifyPrefixBreak with the three candidate event timestamps from the
+ * runtime and records a `prefix_break` sample.  Best-effort + non-fatal.
+ *
+ * PREVENT-PI-004: guardrails-allow PREVENT-PI-004: local SQLite write only.
+ * PREVENT-002: SQL is fully parameterized via recordPerfSample.
+ */
+function tryRecordPrefixBreak(stateDir, prevHitPct, currHitPct, breakAt, runtime) {
+    try {
+        if (prevHitPct - currHitPct < 20)
+            return; // no significant drop
+        const events = {
+            lastRecallAt: runtime.rt.lastInjectAt, // S53: inject fires alongside recall
+            lastCompactAt: runtime.rt.lastCompactAt,
+            lastInjectAt: runtime.rt.lastInjectAt,
+        };
+        const { cause, confidence } = classifyPrefixBreak(breakAt, events);
+        recordPerfSample(stateDir, "prefix_break", 0, {
+            cause,
+            confidence,
+            prevHitPct,
+            currHitPct,
+            breakAt,
+        }); // guardrails-allow PREVENT-PI-004: local SQLite write (loopback perf instrumentation)
+    }
+    catch {
+        /* non-fatal: prefix-break detection never blocks the agent loop */
+    }
+}
 /** Register perf instrumentation handlers + start the 5s cpu/mem interval. */
 export function registerPerfHandler(pi, runtime) {
     // turn_start: record the wall-clock start of the turn. Using Date.now() (not
@@ -23,6 +57,7 @@ export function registerPerfHandler(pi, runtime) {
     });
     // turn_end: compute turn latency + TPS + cache hit % from the assistant
     // message's usage block. One perf_samples row per metric per turn.
+    // S53A: tracks prevHitPct across turns to detect prefix breaks.
     pi.on("turn_end", async (event) => {
         try {
             if (runtime.perfTurnStart > 0) {
@@ -36,7 +71,22 @@ export function registerPerfHandler(pi, runtime) {
                     recordPerfSample(runtime.currentStateDir, "tps", u.output / durSec, { outputTokens: u.output });
                     const denom = u.cacheRead + u.input + u.cacheWrite;
                     const hitPct = denom > 0 ? (u.cacheRead / denom) * 100 : 0;
-                    recordPerfSample(runtime.currentStateDir, "cache_hit_pct", hitPct, { input: u.input, cacheRead: u.cacheRead, cacheWrite: u.cacheWrite });
+                    recordPerfSample(runtime.currentStateDir, "cache_hit_pct", hitPct, {
+                        input: u.input,
+                        cacheRead: u.cacheRead,
+                        cacheWrite: u.cacheWrite,
+                        modelName: runtime.currentModel?.modelName,
+                        modelId: runtime.currentModel?.modelId,
+                    });
+                    // S53A: detect significant cache hit % drop → record prefix_break.
+                    // Only trigger when we have a previous value AND the drop is >20pp.
+                    if (runtime.rt._prevCacheHitPct != null &&
+                        runtime.rt._prevCacheHitPct - hitPct >= 20) {
+                        const breakAt = runtime.perfTurnStart;
+                        tryRecordPrefixBreak(runtime.currentStateDir, runtime.rt._prevCacheHitPct, hitPct, breakAt, runtime);
+                    }
+                    // Stamp for the next turn's comparison (even when no break fires).
+                    runtime.rt._prevCacheHitPct = hitPct;
                 }
             }
         }

package/dist/extensions/mega-events/recall-tail.js

@@ -0,0 +1,59 @@
+/**
+ * True when recall blocks are staged AND the feature flag is ON. Called by the
+ * context handler to decide whether to call withRecallTail.
+ */
+export function stagedForTail(runtime, config) {
+    if (!config.recallTailInject)
+        return false;
+    return (runtime.pendingRecallBlock != null ||
+        runtime.pendingMemoryRecallBlock != null);
+}
+/**
+ * Append staged recall/memory blocks as a single user-role message at the very
+ * tail of the messages array. Returns a NEW array (does not mutate input).
+ *
+ * Sets `runtime.recallInjectedThisTurn = true` when it actually appends, so the
+ * turn_end handler (agent-handlers.ts) consumes the staged blocks only if they
+ * reached a view this turn. If a turn ends with no context event, the flag
+ * stays false and the blocks remain staged for the next turn.
+ *
+ * Stable timestamp (runtime.perfTurnStart ?? Date.now()) so replays within the
+ * turn are byte-identical (mirrors the v0.8.6 summaryAgentMsg rationale).
+ *
+ * Non-fatal: on any failure, returns messages unmodified and does NOT set the
+ * injected flag. PREVENT-PI-003: the injected message has role "user", never
+ * "system".
+ */
+export function withRecallTail(messages, runtime, config) {
+    if (!config.recallTailInject)
+        return messages;
+    try {
+        const blocks = [];
+        if (runtime.pendingRecallBlock != null) {
+            blocks.push(runtime.pendingRecallBlock);
+        }
+        if (runtime.pendingMemoryRecallBlock != null) {
+            blocks.push(runtime.pendingMemoryRecallBlock);
+        }
+        if (blocks.length === 0)
+            return messages;
+        const tailText = blocks.join("\n\n");
+        const timestamp = runtime.perfTurnStart ?? // stable within a turn
+            Date.now();
+        const tailMsg = {
+            role: "user",
+            content: tailText,
+            timestamp,
+        };
+        runtime.rt.recallInjectedThisTurn = true;
+        // Append at the very tail — never splits toolCall/toolResult.
+        // PREVENT-PI-001/002: a single message appended after a complete
+        // transcript prefix cannot split a toolCall/toolResult pair.
+        return [...messages, tailMsg];
+    }
+    catch {
+        // Non-fatal: leave blocks staged (flag stays unset) and return the
+        // unmodified view. The context event still proceeds normally.
+        return messages;
+    }
+}

package/dist/extensions/mega-events/session-handlers.js

@@ -149,14 +149,21 @@ export function registerSessionHandlers(pi, runtime, config) {
         catch {
             /* non-fatal: intent polling never breaks the agent loop */
         }
-        const cpBlock = runtime.pendingRecallBlock;
-        const memBlock = runtime.pendingMemoryRecallBlock;
-        if (!cpBlock && !memBlock)
-            return;
-        runtime.pendingRecallBlock = undefined;
-        runtime.pendingMemoryRecallBlock = undefined;
-        const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
-        return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
+        // S53: when recallTailInject is ON, the staged blocks are injected as a
+        // user-role tail message by the context handler (recall-tail.ts), NOT
+        // here. This prepend path is the legacy behavior (flag OFF = byte-
+        // identical pre-sprint). captureModel + rewind-intent consumption above
+        // stay unconditional regardless of the flag.
+        if (!config.recallTailInject) {
+            const cpBlock = runtime.pendingRecallBlock;
+            const memBlock = runtime.pendingMemoryRecallBlock;
+            if (!cpBlock && !memBlock)
+                return;
+            runtime.pendingRecallBlock = undefined;
+            runtime.pendingMemoryRecallBlock = undefined;
+            const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
+            return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
+        }
     });
     pi.on("session_shutdown", async (_event, ctx) => {
         runtime.setStatus(ctx, undefined);

package/dist/extensions/mega-runtime/reset-runtime.js

@@ -24,6 +24,9 @@ export function resetRuntimeImpl(self, sessionId) {
         dedupAttempts: 0,
         tokensSaved: 0,
         lastCompactAt: null,
+        lastRecallAt: null,
+        lastInjectAt: null,
+        _prevCacheHitPct: null,
         lastNativeCompactAt: null,
         compactCount: 0,
         recallInjections: 0,
@@ -40,7 +43,9 @@ export function resetRuntimeImpl(self, sessionId) {
         poisonedAdviseSent: false,
         providerOutageAdvised: false,
         poisonedCompactSignatures: new Set(),
+        recallInjectedThisTurn: false,
         poisonedCount: 0,
+        extensionInitiatedTurn: false,
     };
     self.trimCache = null; // v0.8.6: never replay a stale trim into a new session
     self.statusKey = undefined;

package/dist/extensions/mega-runtime/runtime.js

@@ -49,6 +49,9 @@ export class MegaRuntime {
         dedupAttempts: 0,
         tokensSaved: 0,
         lastCompactAt: null,
+        lastRecallAt: null,
+        lastInjectAt: null,
+        _prevCacheHitPct: null,
         lastNativeCompactAt: null,
         compactCount: 0,
         recallInjections: 0,
@@ -65,7 +68,10 @@ export class MegaRuntime {
         poisonedAdviseSent: false,
         providerOutageAdvised: false,
         poisonedCompactSignatures: new Set(),
+        // S53: tracks whether withRecallTail injected this turn (consumed at turn_end).
+        recallInjectedThisTurn: false,
         poisonedCount: 0,
+        extensionInitiatedTurn: false,
     };
     // v0.8.6 cache-stability: the cached live-trim view for the current
     // compaction epoch. Set after a fresh runCompact + computeLiveTrimCut, and

package/dist/src/embedder.js

@@ -49,20 +49,60 @@ function fnv1a(str) {
  * Default embedder: character 3-gram bag-of-counts, hashed into a fixed-dim
  * vector, L2-normalized. Captures local lexical/structure overlap well enough
  * for checkpoint relevance ranking.
+ *
+ * S53B: includes a 256-entry (configurable) FIFO cache keyed by FNV-1a(text).
+ * Each cache entry holds a DEFENSIVE COPY of the embedding to prevent mutation
+ * from corrupting stored vectors. Set MEGACOMPACT_EMBED_CACHE=0 to disable.
  */
 export class TrigramEmbedder {
     dim;
     seed;
+    /** @type {Map<string, Vector>} */
+    _cache;
+    _cacheSize;
+    _hits = 0;
+    _misses = 0;
     constructor(dim = 512, seed = 0x9e3779b9) {
         this.dim = dim;
         this.seed = seed >>> 0;
+        const cap = Number(process.env.MEGACOMPACT_EMBED_CACHE ?? "256");
+        this._cacheSize = cap > 0 ? Math.floor(cap) : 0;
+        this._cache = new Map();
+    }
+    /** Cache statistics — always { hits: 0, misses: 0 } when cache is disabled. */
+    getEmbedCacheStats() {
+        return { hits: this._hits, misses: this._misses };
     }
     embed(text) {
+        if (this._cacheSize === 0) {
+            return this._embedRaw(text);
+        }
+        const key = fnv1a(text).toString(36);
+        const cached = this._cache.get(key);
+        if (cached !== undefined) {
+            this._hits++;
+            // Defensive copy: caller mutating the returned vector must not corrupt cache.
+            return cached.slice();
+        }
+        this._misses++;
+        const vec = this._embedRaw(text);
+        // Store a defensive copy (slice) so caller mutation cannot corrupt stored value.
+        this._cache.set(key, vec.slice());
+        // FIFO eviction: remove the oldest entry when at capacity.
+        if (this._cache.size > this._cacheSize) {
+            // Map insertion order is guaranteed — first key is the oldest.
+            const firstKey = this._cache.keys().next().value;
+            if (firstKey !== undefined)
+                this._cache.delete(firstKey);
+        }
+        return vec;
+    }
+    /** Raw embed computation (no cache logic). Public so tests can bypass cache. */
+    _embedRaw(text) {
         const vec = new Array(this.dim).fill(0);
         const norm = text.toLowerCase().replace(/\s+/g, " ");
         if (norm.length === 0)
             return l2Normalize(vec);
-        // Whole-string + word + char-trigram signals.
         vec[fnv1a(norm) % this.dim] += 1;
         for (const word of norm.split(" ")) {
             if (word.length === 0)
@@ -74,7 +114,6 @@ export class TrigramEmbedder {
                 vec[idx] += 1;
             }
         }
-        // Edge: very short tokens still get a slot.
         if (norm.length < 3)
             vec[fnv1a(norm) % this.dim] += 1;
         return l2Normalize(vec);

package/dist/src/memoryStats.js

@@ -0,0 +1,75 @@
+/**
+ * memoryStats.ts — aggregate memory-store statistics (S53B).
+ *
+ * Reads from the durable SQLite memories table to produce:
+ *   - totalMemories: all memories in the store
+ *   - memoriesInLast30Days: memories created within the last 30 days
+ *   - topStableMemories: top-N memories by recall count (desc)
+ *   - avgRecallScore: mean recallCount across all memories
+ *
+ * recallCount = 1 if last_recalled_at IS NOT NULL, else 0.
+ * This is the durable signal from recallMemory() — a memory that was
+ * recalled at least once is "stable" (still relevant).  The exact recall
+ * frequency is tracked via turn_recall rows (source='memory') for
+ * cross-session analysis, but the memories table provides a reliable
+ * on-store signal that is always present without a schema migration.
+ *
+ * Real-store only (PREVENT-PI-004). No mocks. No network.
+ */
+import { openStore } from "./store/sqlite/utils.js";
+/**
+ * Aggregate memory statistics from the SQLite store.
+ * Always returns a valid MemoryStatsResult; never throws.
+ */
+export async function memoryStats(stateDir, opts = {}) {
+    const topN = opts.topN ?? 5;
+    try {
+        const db = openStore(stateDir);
+        // ── totalMemories ──────────────────────────────────────────────────────
+        const totalRow = db
+            .prepare("SELECT COUNT(*) AS n FROM memories")
+            .get();
+        const totalMemories = totalRow.n;
+        // ── memoriesInLast30Days ───────────────────────────────────────────────
+        const thirtyDaysAgo = Math.floor(Date.now() / 1000) - 30 * 86_400;
+        const recentRow = db
+            .prepare("SELECT COUNT(*) AS n FROM memories WHERE created_at >= ?")
+            .get(thirtyDaysAgo);
+        const memoriesInLast30Days = recentRow.n;
+        // ── topStableMemories + avgRecallScore ─────────────────────────────────
+        // recallCount: 1 if last_recalled_at IS NOT NULL (recalled at least once),
+        //             0 otherwise.  Stable memories score high.
+        const topRows = db
+            .prepare(`SELECT
+           m.id,
+           m.content         AS text,
+           m.last_recalled_at AS lastRecalledAt,
+           CASE WHEN m.last_recalled_at IS NOT NULL THEN 1 ELSE 0 END AS recallCount
+         FROM memories m
+         ORDER BY recallCount DESC, m.id ASC
+         LIMIT ?`)
+            .all(topN);
+        // avgRecallScore = (count of recalled memories) / totalMemories.
+        // This represents the fraction of memories that are "stable" (recalled at least once).
+        const recalledRow = db
+            .prepare("SELECT COUNT(*) AS n FROM memories WHERE last_recalled_at IS NOT NULL")
+            .get();
+        const avgRecallScore = totalMemories > 0 ? recalledRow.n / totalMemories : 0;
+        const topStableMemories = topRows.map((r) => ({
+            id: r.id,
+            text: r.text ?? "",
+            recallCount: r.recallCount ?? 0,
+            lastRecalledAt: r.lastRecalledAt ?? null,
+        }));
+        return { totalMemories, memoriesInLast30Days, topStableMemories, avgRecallScore };
+    }
+    catch {
+        // Non-fatal: return zeros rather than propagating an error.
+        return {
+            totalMemories: 0,
+            memoriesInLast30Days: 0,
+            topStableMemories: [],
+            avgRecallScore: 0,
+        };
+    }
+}

package/dist/src/prefix-break.js

@@ -0,0 +1,43 @@
+/**
+ * prefix-break.ts — S53A prefix-break classification.
+ *
+ * Classifies WHY the provider cache prefix broke on a given turn by comparing
+ * the break timestamp against the three candidate event timestamps with a
+ * configurable tolerance window.
+ *
+ * The classification is best-effort (non-fatal): null timestamps are absent,
+ * and a break outside all tolerance windows is classified as 'other'.
+ *
+ * Pi-agnostic: no pi runtime types, no network.
+ */
+/**
+ * Classify why the provider cache prefix broke at `breakTimestamp`.
+ *
+ * @param breakTimestamp  Wall-clock ms of the cache hit % drop sample.
+ * @param events          Recent event timestamps from the runtime.
+ * @param toleranceMs     Max ms gap to consider a match (default 2000).
+ * @returns Classification result with cause + confidence.
+ *
+ * Priority: recall > compaction > inject.  Confidence = 1 − (gap / (2·tolerance)),
+ * capped at [0.5, 1.0].  Returns 'other' when no event is within tolerance.
+ */
+export function classifyPrefixBreak(breakTimestamp, events, toleranceMs = 2000) {
+    // Ordered by priority (recall wins if multiple match).
+    const candidates = [
+        { cause: "recall", ts: events.lastRecallAt },
+        { cause: "compaction", ts: events.lastCompactAt },
+        { cause: "inject", ts: events.lastInjectAt },
+    ];
+    for (const { cause, ts } of candidates) {
+        if (ts == null)
+            continue;
+        const gap = Math.abs(breakTimestamp - ts);
+        if (gap <= toleranceMs) {
+            // confidence = 1 − (gap / (2·tolerance)) ∈ [0.5, 1.0]
+            // exact match (gap=0) → 1.0;  at boundary (gap=tolerance) → 0.5
+            const confidence = Math.max(0.5, 1 - gap / (2 * toleranceMs));
+            return { cause, confidence };
+        }
+    }
+    return { cause: "other", confidence: 0 };
+}

package/dist/src/pricing.js

@@ -34,7 +34,7 @@ export const MODEL_INPUT_RATES = {
     "gemini-2.0-flash": 0.1e-6,
     "deepseek-chat": 0.14e-6,
     "deepseek-reasoner": 0.55e-6,
-    "codestral": 0.3e-6,
+    codestral: 0.3e-6,
     "mistral-large": 2e-6,
     "llama-3.1-405b": 1.33e-6,
     "llama-3.1-70b": 0.8e-6,