pi-mega-compact 0.11.8 → 0.11.10

package/dist/extensions/dashboard-server/gather-debug-keys.js

@@ -0,0 +1,18 @@
+export const MEGACOMPACT_ENV_KEYS = [
+    "MEGACOMPACT_AUTO",
+    "MEGACOMPACT_THRESHOLD_TOKENS",
+    "MEGACOMPACT_TIER",
+    "MEGACOMPACT_DBMIRROR",
+    "MEGACOMPACT_MESSAGE_SEPARATION",
+    "MEGACOMPACT_CACHE_STRIPING",
+    "MEGACOMPACT_LEGACY_DURABLE_TRIM",
+    "MEGACOMPACT_RACE_GUARD_STRICT",
+    "MEGACOMPACT_RECALL_TAIL_INJECT",
+    "MEGACOMPACT_PGLITE_DISABLED",
+    "MEGACOMPACT_WIKI_REBUILD_EVERY",
+    "MEGACOMPACT_WIKI_SEED_FROM_TURNS",
+    "MEGACOMPACT_MEMORY_GRAPH_SEED_TURNS",
+    "MEGACOMPACT_MEMORY_GRAPH_SEED_TIER",
+    "MEGACOMPACT_MEMORY_GRAPH_SEED_BATCH",
+    "MEGACOMPACT_MEMORY_GRAPH_SEED_MAX",
+];

package/dist/extensions/dashboard-server/gather-debug.js

@@ -0,0 +1,129 @@
+/**
+ * gather-debug.ts — assemble a diagnostic bundle for bug reports.
+ *
+ * POST /api/maintenance/gather-debug → DebugBundleResponse. Reads local
+ * SQLite + FS only (PREVENT-PI-004: zero network). Each section is non-fatal:
+ * a read failure records `{ error }` for that section and continues, so the
+ * bundle always returns 200 with whatever could be gathered.
+ */
+import { readFileSync, existsSync } from "node:fs";
+import { createRequire } from "node:module";
+import { defaultMetricsPath, defaultEventsPath } from "../../src/monitoring.js";
+import { ownVersion } from "../mega-runtime/helpers.js";
+import { MEGACOMPACT_ENV_KEYS } from "./gather-debug-keys.js";
+/** Event-name patterns that indicate a critical/compaction class of error. */
+const CRITICAL_PATTERN = /critical|overflow|already[.\s-]?compacted|compaction[.\s-]?failed|stream[.\s-]?death|poisoned|prefix_stability/i;
+const require = createRequire(import.meta.url);
+/** Lazy store opener (mirrors routes-maintenance.ts's lazyStoreUtils). */
+function openStoreDb(ctx) {
+    const mod = require("../../src/store/sqlite/utils.js");
+    return mod.openStore(ctx.stateDir);
+}
+/** Lazy maintenance loaders (mirrors routes-maintenance.ts's lazyMaintenance). */
+function loadMaintenance() {
+    return require("../../src/store/sqlite/maintenance.js");
+}
+/** Send the debug bundle as JSON. Always 200 (partial on read failure). */
+export function sendGatherDebug(res, ctx) {
+    const result = {
+        builtAt: Date.now(),
+        version: ownVersion(),
+        stateDir: ctx.stateDir,
+        config: {},
+        schemaHealth: { error: "not fetched" },
+        storeStats: { error: "not fetched" },
+        dashboardSnapshot: { error: "not fetched" },
+        recentEvents: [],
+        criticalEvents: [],
+    };
+    // 1. Config flags (no secrets — MEGACOMPACT_* only)
+    for (const key of MEGACOMPACT_ENV_KEYS) {
+        result.config[key] = process.env[key] ?? "<default>";
+    }
+    // 2. Schema health
+    try {
+        const maint = loadMaintenance();
+        const integrity = maint.integrityCheck(ctx.stateDir);
+        const db = openStoreDb(ctx);
+        const metaRow = db
+            .prepare("SELECT value FROM meta WHERE key = ?")
+            .get("schema_version");
+        const schemaVersion = metaRow ? Number(metaRow.value) : 0;
+        const fkRows = db.prepare("PRAGMA foreign_key_check").all();
+        result.schemaHealth = {
+            type: "schema-health",
+            schemaVersion,
+            integrity,
+            fkCheck: fkRows.map((r) => JSON.stringify(r)),
+            columns: [],
+            healthy: true,
+        };
+    }
+    catch (e) {
+        result.schemaHealth = { error: e instanceof Error ? e.message : String(e) };
+    }
+    // 3. Store stats
+    try {
+        const maint = loadMaintenance();
+        const raw = maint.getDbStats(ctx.stateDir);
+        result.storeStats = {
+            type: "db-stats",
+            tables: Object.entries(raw.tableCounts).map(([table, rowCount]) => ({ table, rowCount })),
+            storage: {
+                files: { dbBytes: raw.dbBytes, walBytes: raw.walBytes, shmBytes: raw.shmBytes },
+                pageSize: raw.pageSize,
+                pageCount: raw.pageCount,
+                freelistPages: raw.freelistPages,
+            },
+        };
+    }
+    catch (e) {
+        result.storeStats = { error: e instanceof Error ? e.message : String(e) };
+    }
+    // 4. Dashboard snapshot
+    try {
+        const path = defaultMetricsPath(ctx.stateDir);
+        if (existsSync(path)) {
+            const raw = readFileSync(path, "utf-8");
+            result.dashboardSnapshot = JSON.parse(raw);
+        }
+        else {
+            result.dashboardSnapshot = { error: "no snapshot" };
+        }
+    }
+    catch (e) {
+        result.dashboardSnapshot = { error: e instanceof Error ? e.message : String(e) };
+    }
+    // 5. Recent + critical events (tail ~200 lines of events.log)
+    try {
+        const path = defaultEventsPath(ctx.stateDir);
+        if (existsSync(path)) {
+            const lines = readFileSync(path, "utf-8").split("\n").filter(Boolean);
+            for (const line of lines.slice(-200)) {
+                try {
+                    const ev = JSON.parse(line);
+                    result.recentEvents.push(ev);
+                    if (CRITICAL_PATTERN.test(line))
+                        result.criticalEvents.push(ev);
+                }
+                catch {
+                    continue; // skip unparseable line (PREVENT-001)
+                }
+            }
+        }
+    }
+    catch (e) {
+        result.recentEvents = [{ error: e instanceof Error ? e.message : String(e) }];
+    }
+    // Send (local helper to avoid importing sendJson from a non-exporting module)
+    try {
+        const body = JSON.stringify(result);
+        res.writeHead(200, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+        res.end(body);
+    }
+    catch {
+        // last resort — never leave the response hanging
+        res.writeHead(500, { "Content-Type": "application/json" }); // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+        res.end('{"error":"failed to serialize debug bundle"}');
+    }
+}

package/dist/extensions/dashboard-server/routes-maintenance.js

@@ -11,6 +11,7 @@
  * SQLite only; zero network calls.
  */
 import { createRequire } from "node:module";
+import { sendGatherDebug } from "./gather-debug.js";
 // ---------------------------------------------------------------------------
 // Lazy imports (via createRequire)
 // ---------------------------------------------------------------------------
@@ -48,6 +49,11 @@ export function handleMaintenance(req, res, ctx) {
         handleMaintenanceAction(req, res, ctx);
         return true;
     }
+    // ── POST /api/maintenance/gather-debug ──────────────────────────
+    if (method === "POST" && url === "/api/maintenance/gather-debug") {
+        sendGatherDebug(res, ctx);
+        return true;
+    }
     // ── GET /api/maintenance ──────────────────────────────────────────
     if (method === "GET" && url === "/api/maintenance") {
         sendDbStats(res, ctx);

package/dist/extensions/mega-config.js

@@ -168,6 +168,7 @@ export function loadConfig() {
         windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
         recallTailInject: envBool("MEGACOMPACT_RECALL_TAIL_INJECT", true),
         messageSeparation: envBool("MEGACOMPACT_MESSAGE_SEPARATION", false),
+        cacheStriping: envBool("MEGACOMPACT_CACHE_STRIPING", false),
         tuiWidget: envBool("MEGACOMPACT_TUI_WIDGET", true),
         ragQueryReformulation: envBool("MEGACOMPACT_QUERY_REFORMULATION", false),
         ragTieredRouter: envBool("MEGACOMPACT_TIERED_ROUTER", false),

package/dist/extensions/mega-events/agent-handlers.js

@@ -9,6 +9,9 @@ import { classifyError, classifyErrorDetailed, errorRetryBackoffMs, extractError
 import { safeSendUserMessage } from "./send-safe.js";
 import { maybeSendProviderOutageAdvisory } from "./outage-advisor.js";
 import { vectorStats } from "../../src/vectorStore.js";
+import { defaultEmbedder } from "../../src/embedder.js";
+import { storeTopicEmbedding, loadTopicEmbedding, detectTopicShift, } from "./separated-prompt.js";
+import { refreshStripeAssignments as writeStripeAssignments, } from "../../src/cache-stripe-impl.js";
 /** Register agent/turn tracking event handlers. */
 export function registerAgentHandlers(pi, runtime, config) {
     // ---- Agent tracking for real-time widget + status-line updates ---------
@@ -130,10 +133,21 @@ export function registerAgentHandlers(pi, runtime, config) {
                     // context-handler.ts:258-287 so both call sites stay in sync.
                     const cooldownMs = config.raceGuardStrict ? 30_000 : 10_000;
                     const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
+                    // CRITICAL-OVER ESCAPE HATCH: when context is critically over the
+                    // window (>= 90%), force the durable ctx.compact() even if
+                    // piCompactWouldNoop says it would no-op ("Already compacted").
+                    // Without this, the on-disk transcript never truncates and the
+                    // session stays overflowed on every resume — the "Already
+                    // compacted" + overflow death-spiral (2026-08-01 incident). The
+                    // cooldown still applies (we don't spam compact), but the no-op
+                    // gate is bypassed when critical. pi's ctx.compact() may throw
+                    // "Already compacted" in the truly-no-op case — wrapped in
+                    // try/catch so the throw is non-fatal.
+                    const criticalOver = (runtime.lastCtxPercent ?? 0) >= 90;
                     if (sinceCompact < cooldownMs) {
                         runtime.diagAgentEndDurableSkipRecent++;
                     }
-                    else if (!piCompactWouldNoop(ctx)) {
+                    else if (criticalOver || !piCompactWouldNoop(ctx)) {
                         runtime.debounceUntil = now + 2000;
                         runtime.diagAgentEndDurable++;
                         runtime.logger.info("agent-end-durable-trigger", {
@@ -150,6 +164,7 @@ export function registerAgentHandlers(pi, runtime, config) {
                             // would re-check before it lands.
                             const stamp = runtime.rt.lastNativeCompactAt;
                             const liveSid = runtime.rt.sessionId;
+                            const liveCritical = criticalOver;
                             setTimeout(() => {
                                 try {
                                     if (runtime.rt.sessionId !== liveSid)
@@ -161,19 +176,26 @@ export function registerAgentHandlers(pi, runtime, config) {
                                     if (runtime.rt.lastNativeCompactAt !== stamp &&
                                         since2 < cooldownMs)
                                         return;
-                                    if (piCompactWouldNoop(ctx))
+                                    // CRITICAL-OVER ESCAPE HATCH: bypass the no-op gate when
+                                    // critical (captured at handler time) so the durable trim fires.
+                                    if (!liveCritical && piCompactWouldNoop(ctx))
                                         return;
                                     ctx.compact({
                                         customInstructions: undefined,
-                                    }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated.
+                                    }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; deferred + re-validated. May throw "Already compacted" when the escape hatch forces it — caught below.
                                 }
                                 catch {
-                                    /* non-fatal */
+                                    /* non-fatal: "Already compacted" throws are expected when the critical-over hatch forces a no-op compact */
                                 }
                             }, 500);
                         }
                         else {
-                            ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastNativeCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
+                            try {
+                                ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. May throw "Already compacted" when the critical-over escape hatch forces it — caught.
+                            }
+                            catch {
+                                /* non-fatal */
+                            }
                         }
                         didDurableTrim = true;
                     }
@@ -712,5 +734,52 @@ export function registerAgentHandlers(pi, runtime, config) {
         catch {
             /* non-fatal: a classifier/retry failure never breaks the agent loop */
         }
+        // P3.5: topic-shift stripe refresh. When MEGACOMPACT_CACHE_STRIPING is ON,
+        // embed the current turn, detect topic shift vs. the previous turn, and
+        // refresh cache stripe assignments when shifted (or first turn). Best-effort:
+        // failures log + continue, never break the agent loop.
+        if (config.cacheStriping) {
+            try {
+                const stateDir = runtime.currentStateDir;
+                const sessionId = runtime.rt.sessionId;
+                const currentTurn = event.turnIndex;
+                const embedder = defaultEmbedder();
+                // (a) Extract text from the assistant's response for topic embedding.
+                let textToEmbed = "";
+                const msg = event.message;
+                if (msg.role === "assistant" && Array.isArray(msg.content)) {
+                    for (const part of msg.content) {
+                        if ("text" in part && typeof part.text === "string") {
+                            textToEmbed += part.text + " ";
+                        }
+                    }
+                }
+                textToEmbed = textToEmbed.trim() || `turn-${currentTurn}`;
+                const embedding = new Float32Array(embedder.embed(textToEmbed));
+                storeTopicEmbedding(stateDir, sessionId, currentTurn, embedding);
+                // (b) Load previous turn's embedding and detect topic shift.
+                const prevEmb = currentTurn > 0
+                    ? loadTopicEmbedding(stateDir, sessionId, currentTurn - 1)
+                    : null;
+                const shifted = currentTurn === 0 || detectTopicShift(embedding, prevEmb);
+                // (c) If shifted (or first turn), refresh stripe assignments (WRITE path).
+                let chunkCount = 0;
+                if (shifted) {
+                    chunkCount = writeStripeAssignments(stateDir, undefined, embedder, (detail) => runtime.logger.info("stripe_refresh_detail", {
+                        detail,
+                        turnIndex: currentTurn,
+                    }));
+                }
+                // (d) Log the refresh outcome.
+                runtime.logger.info("stripe_refresh", {
+                    turnIndex: currentTurn,
+                    shifted,
+                    chunkCount,
+                });
+            }
+            catch {
+                /* non-fatal: stripe refresh never breaks the agent loop */
+            }
+        }
     });
 }

package/dist/extensions/mega-events/context-handler.js

@@ -12,7 +12,21 @@ import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
 import { pressureFromPct, pressureRatio, } from "../mega-config.js";
 import { appendMirrorMessages } from "./mirror-append.js";
 import { stagedForTail, withRecallTail } from "./recall-tail.js";
-import { buildSeparatedPrompt } from "./separated-prompt.js";
+import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "./separated-prompt.js";
+/** Best-effort text extraction from an AgentMessage for analytics/logging.
+ *  AgentMessage is a discriminated union; .content exists only on some
+ *  variants (user/assistant/toolResult/custom). Narrow by role, never assume. */
+function messageContentText(m) {
+    const c = m.content;
+    if (typeof c === "string")
+        return c;
+    if (Array.isArray(c)) {
+        return c
+            .map((b) => (b && typeof b === "object" && "text" in b ? String(b.text ?? "") : ""))
+            .join(" ");
+    }
+    return "";
+}
 /** Register the context event handler (live-trim auto-trigger). */
 export function registerContextHandler(pi, runtime, config) {
     // ---- Auto-trigger: live trim (compact and continue) + native durable ----
@@ -41,14 +55,32 @@ export function registerContextHandler(pi, runtime, config) {
         // reaches raw_transcript (PREVENT-PI: append-only, view-only).
         const tailResult = (msgs) => {
             const base = msgs ?? messages;
-            if (!stagedForTail(runtime, config) && !config.messageSeparation)
+            if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping)
                 return undefined;
             let result = stagedForTail(runtime, config)
                 ? withRecallTail(base, runtime, config)
                 : base;
-            if (config.messageSeparation) {
+            if (config.cacheStriping) {
+                result = buildCacheOptimizedPrompt(result);
+            }
+            else if (config.messageSeparation) {
                 result = buildSeparatedPrompt(result);
             }
+            // P2.5: log prefix stability (fire-and-forget, non-fatal).
+            // tailResult is sync, so use .then().catch() on the dynamic import.
+            if (result.length > 1) {
+                import("../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
+                    const stableScore = computeStabilityScore({ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 }, result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })));
+                    runtime.logger.info("prefix_stability", {
+                        stableScore: Number.isFinite(stableScore) ? stableScore : 0,
+                        prefixMessages: result.length,
+                        separation: config.messageSeparation ? "v2" : "off",
+                        striping: config.cacheStriping ? "v3" : "off",
+                    });
+                }).catch(() => {
+                    // Non-fatal: stability logging is best-effort.
+                });
+            }
             return { messages: result };
         };
         // Always track context for the dashboard/widget, even when auto is off.
@@ -89,6 +121,30 @@ export function registerContextHandler(pi, runtime, config) {
             try {
                 const db = openStore(runtime.currentStateDir);
                 appendMirrorMessages(db, messages, runtime.rt.sessionId, epochIdFor(runtime.rt.sessionId), runtime.currentTurn);
+                // P2.2: populate conversation_thread + tool_results tables for
+                // prompt-cache analytics and durable separation. The live-array
+                // separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
+                // tailResult above) is sufficient for the prompt-construction path;
+                // these DB writes persist the split for post-hoc analysis, dashboard
+                // queries, and future readers. Non-fatal — failure here never breaks
+                // the agent loop (PREVENT-PI-004: zero network, local SQLite only).
+                {
+                    const sid = runtime.rt.sessionId;
+                    const turn = runtime.currentTurn;
+                    const now = Date.now();
+                    const threadStmt = db.prepare("INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
+                    const toolStmt = db.prepare("INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
+                    for (const m of messages) {
+                        const role = m.role;
+                        const content = messageContentText(m);
+                        if (role === "user" || role === "assistant") {
+                            threadStmt.run(sid, role, content, turn, now);
+                        }
+                        else if (role === "toolResult" || role === "bashExecution") {
+                            toolStmt.run(sid, role, content, turn, now);
+                        }
+                    }
+                }
             }
             catch (e) {
                 runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
@@ -184,7 +240,7 @@ export function registerContextHandler(pi, runtime, config) {
             if (runtime.trimCache &&
                 runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
                 runtime.trimCache.cut <= messages.length) {
-                const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002
+                const recent = messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
                 runtime.diagLiveTrimFires++;
                 runtime.diagLiveTrimReplays++;
                 runtime.snapshot(ctx);
@@ -401,6 +457,13 @@ export function registerContextHandler(pi, runtime, config) {
                 compactedFrom: ran.result.compactedFrom,
                 summary: ran.result.summary,
                 anchorUserMessages,
+                // CRITICAL-OVER ESCAPE HATCH: when context is at/over ~90% of the
+                // window, relief takes priority over the anchor floor. Without this,
+                // computeLiveTrimCut bails to null (can't satisfy the floor) and the
+                // model is fed a raw overflow that errors every turn — the
+                // "Already compacted" + overflow death-spiral (2026-08-01 incident).
+                // A thin anchor is recoverable; an overflowed session is not.
+                criticalOver: (pct ?? 0) >= 90,
             });
             if (cut === null) {
                 runtime.diagCtxCutNull++;
@@ -409,6 +472,7 @@ export function registerContextHandler(pi, runtime, config) {
                     compactedFrom: ran.result.compactedFrom,
                     viewLen: view.length,
                     anchorUserMessages,
+                    criticalOver: (pct ?? 0) >= 90,
                 });
                 return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
             }

package/dist/extensions/mega-runtime/snapshot.js

@@ -131,7 +131,7 @@ export function computeMegaSnapshot(p) {
         cachePct,
         megaCacheFlare: p.megaCacheFlare,
         megaCacheFlarePct: p.megaCacheFlarePct,
-        perTurnCacheHitPct: p.perTurnCacheHitPct ?? cachePct,
+        perTurnCacheHitPct: p.perTurnCacheHitPct,
         levelUpFlare: p.levelUpFlare,
         achievementFlare: p.achievementFlare,
         achievementFlareTitles: p.achievementFlareTitles,

package/dist/extensions/mega-trim.js

@@ -48,10 +48,20 @@ export function computeLiveTrimCut(view, opts) {
         if (cut > 1) {
             const finalRecent = view.slice(cut);
             if (finalRecent.filter((m) => m.role === "user").length < opts.anchorUserMessages) {
+                // CRITICAL-OVER ESCAPE HATCH: when context is critically over the window,
+                // return the boundary-safe cut even though it's below the anchor floor.
+                // Bailing to null here would feed the model the raw overflow and trap the
+                // session in an unrecoverable error loop ("Already compacted" + overflow).
+                // A thin anchor is recoverable; an overflowed session is not.
+                if (opts.criticalOver && cut > 0)
+                    return cut;
                 return null; // cannot satisfy the floor without dropping too much — retry next call
             }
         }
         else {
+            // Same escape hatch for the cut > 1 false branch.
+            if (opts.criticalOver && cut > 0)
+                return cut;
             return null;
         }
     }

package/dist/src/config/dedup.js

@@ -64,6 +64,7 @@ export function loadDedupConfig() {
         RAPTOR_MULTILEVEL_ENABLED: envBool("MEGACOMPACT_RAPTOR_MULTILEVEL", true),
         RAPTOR_LEVEL_WEIGHTS: envNumArray("MEGACOMPACT_RAPTOR_LEVEL_WEIGHTS", [1.0, 0.9, 0.8, 0.7, 0.5]),
         RAPTOR_LEAF_EXPANSION: envBool("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", true),
+        RAPTOR_INCREMENTAL: envBool("MEGACOMPACT_RAPTOR_INCREMENTAL", true),
         RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
         RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
         RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),