pi-mega-compact 0.17.1 → 0.18.0

package/dist/config/vector-cortex.js

@@ -63,6 +63,16 @@ export const VC0C_ENABLED = () => sprintFlag("MEGACOMPACT_VC0C");
  * emit seam, the minhash-v2 backfill seam and the downgrade-export seam.
  */
 export const VC1C_ENABLED = () => sprintFlag("MEGACOMPACT_VC1C");
+/**
+ * VC2A — offline model runtime and asset decision (ModelManifestV1 /
+ * EncoderRuntime).
+ * Default ON. `MEGACOMPACT_VC2A=0` disables and is byte-identical to the
+ * predecessor (mode C: no asset manifest is read/verified, the encoder runtime
+ * idles in mode C, zero `vector_cortex_encoder_*` emissions; the trigram/lexical
+ * paths are unchanged). The real consumers are the encoder emit seam and the
+ * encoder runtime's A/B/C selection.
+ */
+export const VC2A_ENABLED = () => sprintFlag("MEGACOMPACT_VC2A");
 // ---------------------------------------------------------------------------
 // Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
 // Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.

package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js

@@ -138,6 +138,7 @@ export const SETTINGS = [
             boolDirect("MEGACOMPACT_VC1B", "VC1B Occurrence Ledger + Tool Identity", "Neutral occurrence ledger (LedgerReader/Writer/Admin + CompatJournalV1): per-session monotonic seq, tool result references one earlier call, uniqueness by (eventId,digest) only, and the M2 copy-validate-switch downgrade journal. OFF = mode C, ledger unwritten, byte-identical.", true),
             boolDirect("MEGACOMPACT_VC0C", "VC0C Live Safety Envelope", "TriadResult/Breaker live circuit breaker (60s window, 20 attempts, 30s cooldown, 3 probes, 5min healthy residence) + durable spool before provider invocation; manual reset clears cooldown but never evidence. OFF = mode C, unchanged transcript, byte-identical.", true),
             boolDirect("MEGACOMPACT_VC1C", "VC1C Cross-Language Conformance v2", "FixtureManifestV2 canonical manifest validator + DowngradeReport deterministic downgrade export + MinHashV2 exact big-integer signatures and the M4 copy/validate/switch minhash-v2 migration (seed table frozen, cross-language byte-exact). OFF = mode C, v1 sync dedup scan unchanged, byte-identical.", true),
+            boolDirect("MEGACOMPACT_VC2A", "VC2A Offline Model Runtime", "ModelManifestV1 digest-before-load ONNX runtime (opset17/batch1/max512) + asset-free trigram demotion. Asset path assets/vector-cortex/encoder-v1 is immutable/digest-pinned. OFF = mode C, byte-identical to predecessor.", true),
         ],
     },
     {

package/dist/extensions/mega-events/context-handler/dbMirrorAppend.js

@@ -0,0 +1,63 @@
+import { openStore } from "../../../src/store/sqlite.js";
+import { appendMirrorMessages } from "../mirror-append.js";
+import { appendMessagesToLedger } from "../../mega-runtime/vector-cortex-ledger.js";
+import { epochIdFor } from "../../../src/mirror/epoch.js";
+import { messageContentText } from "./messageText.js";
+/**
+ * Append incoming messages to the DB mirror (raw_transcript + thread/tool
+ * tables) and the v2 ledger. Gated on config.dbMirror for the mirror; the VC1B
+ * ledger append is flag-gated inside appendMessagesToLedger (flag-OFF opens no
+ * DB, byte-identical to the predecessor). Non-fatal end-to-end.
+ */
+export function appendMirrorAndLedger(runtime, config, messages) {
+    // S27 DB-mirror: append incoming messages to raw_transcript.
+    // Runs BEFORE fast-gate so every message is captured, even if we
+    // don't compact this turn. Append is idempotent (content_hash PK).
+    // F3: high-water mark (mirror-append.ts) skips already-processed
+    // messages on subsequent events. On fork/rewind (shorter list or
+    // boundary hash mismatch) the mark is dropped, falling back to a
+    // full reprocess.
+    if (config.dbMirror) {
+        try {
+            const db = openStore(runtime.currentStateDir);
+            appendMirrorMessages(db, messages, runtime.rt.sessionId, epochIdFor(runtime.rt.sessionId), runtime.currentTurn);
+            // P2.2: populate conversation_thread + tool_results tables for
+            // prompt-cache analytics and durable separation. The live-array
+            // separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
+            // tailResult) is sufficient for the prompt-construction path;
+            // these DB writes persist the split for post-hoc analysis, dashboard
+            // queries, and future readers. Non-fatal — failure here never breaks
+            // the agent loop (PREVENT-PI-004: zero network, local SQLite only).
+            {
+                const sid = runtime.rt.sessionId;
+                const turn = runtime.currentTurn;
+                const now = Date.now();
+                const threadStmt = db.prepare("INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
+                const toolStmt = db.prepare("INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
+                for (const m of messages) {
+                    const role = m.role;
+                    const content = messageContentText(m);
+                    if (role === "user" || role === "assistant") {
+                        threadStmt.run(sid, role, content, turn, now);
+                    }
+                    else if (role === "toolResult" || role === "bashExecution") {
+                        toolStmt.run(sid, role, content, turn, now);
+                    }
+                }
+            }
+        }
+        catch (e) {
+            runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
+        }
+    }
+    // VC1B (S1): canonical messages -> v2 ledger occurrences. Flag-OFF opens
+    // no DB (byte-identical predecessor); non-fatal. onFailure surfaces
+    // per-append rejections (e.g. EVT_SEQ_REGRESSION on rewind/fork) as
+    // structured warnings rather than swallowing them silently.
+    try {
+        appendMessagesToLedger(runtime.currentStateDir, runtime.rt.sessionId, messages, runtime.logger);
+    }
+    catch (e) {
+        runtime.logger.warn("vc1b-ledger-append-fail", { error: String(e) });
+    }
+}

package/dist/extensions/mega-events/context-handler/gateCheck.js

@@ -0,0 +1,59 @@
+import { resolveModelThreshold, DEFAULT_SAFETY_MARGIN_PCT, DEFAULT_FIRE_POINT_PCT, } from "../../../src/store/sqlite.js";
+import { autoCompactCheck } from "../../../src/compact.js";
+/**
+ * Evaluate whether the current context warrants compaction. Returns a tailed
+ * view ("return") when the gate does not pass, or "proceed" with the resolved
+ * per-model threshold (reused by the live-trim token-budget tail cap).
+ */
+export function evaluateGate(runtime, config, opts) {
+    const pct = opts.pct;
+    const currentTokens = opts.currentTokens;
+    const tailResult = opts.tailResult;
+    // S52 / v0.16.1: per-model threshold override. The user can tune the
+    // fire point + safety margin PER MODEL (different providers' models range
+    // 8K-1M+ context, so one global tier % is wrong). Falls back to env/default
+    // when no override row exists. Computed once here + reused in the tail cap
+    // below; the lookup is a single SQLite PK hit (cheap; cached after the
+    // first read in a session).
+    const modelIdForThreshold = runtime.currentModel?.modelId ?? null;
+    const perModelThreshold = resolveModelThreshold(modelIdForThreshold, {
+        safetyMarginFallback: DEFAULT_SAFETY_MARGIN_PCT,
+        firePointFallback: config.tierPct != null
+            ? Math.round(config.tierPct * 100)
+            : DEFAULT_FIRE_POINT_PCT,
+        stateDir: runtime.currentStateDir,
+    });
+    // S29 FAST GATE: `custom` (absolute MEGACOMPACT_THRESHOLD_TOKENS,
+    // tierPct null) is an explicit opt-out of percent scaling — it keeps the
+    // token gate. When pct is unavailable (window unknown / a model that
+    // doesn't report percent) a tiered config falls back to the token gate
+    // (S27 boot-fallback guarantee) instead of skipping compaction — a
+    // percent-only gate would regress that.
+    let gatePassed = false;
+    if (config.tierPct != null && pct != null) {
+        // Per-model override is a % (10-90); tierPct is a fraction (0.1-1.0).
+        // Prefer the override; fall back to autoPctTrigger + tierPct.
+        const tierPctFraction = config.autoPctTrigger ?? config.tierPct;
+        const perModelFraction = perModelThreshold.firePointPct / 100;
+        const firePct = modelIdForThreshold != null ? perModelFraction : tierPctFraction;
+        gatePassed = pct / 100 >= firePct;
+    }
+    else {
+        // custom tier OR tiered-but-pct-unavailable → token gate (S27 fallback).
+        if (currentTokens < runtime.effectiveThreshold) {
+            runtime.diagCtxFastGate++;
+            return { kind: "return", view: tailResult() ?? undefined };
+        }
+        const check = autoCompactCheck(currentTokens, runtime.effectiveThreshold); // SERVER-STYLE CONFIRM (local)
+        if (!check.shouldCompact) {
+            runtime.diagCtxNoCompact++;
+            return { kind: "return", view: tailResult() ?? undefined };
+        }
+        gatePassed = true;
+    }
+    if (!gatePassed) {
+        runtime.diagCtxFastGate++;
+        return { kind: "return", view: tailResult() ?? undefined };
+    }
+    return { kind: "proceed", perModelThreshold };
+}

package/dist/extensions/mega-events/context-handler/liveTrim.js

@@ -0,0 +1,178 @@
+import { estimateBlockTokens, estimateMessageTokens, } from "../../../src/tokens.js";
+import { computeLiveTrimCut, liveTrimSummaryMessage } from "../../mega-trim.js";
+import { messageContentText } from "./messageText.js";
+/**
+ * Reconstruct the live-trim window (summary + recent anchor) for this LLM
+ * call. Returns the tailed view, or undefined when no trim is safe this call.
+ */
+export function buildLiveTrimView(runtime, config, ctx, opts) {
+    const { messages, view, pct, currentTokens, usageTokens, pressure, ran, perModelThreshold, tailResult, } = opts;
+    // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
+    // Non-destructive: pi keeps the real transcript; only this LLM call sees the
+    // trimmed window. We compute the cut on the engine view (pure, tested) then
+    // slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
+    // mirroring dropCompactedRange) and prepend a user-role summary message.
+    // A build failure or unsafe cut returns nothing (no trim this call — the
+    // next context event retries). The anchor floor is read live from env (the
+    // config value is the cached default) so it can be tuned per-test / per-run
+    // without reloading the module.
+    try {
+        const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
+        const anchorUserMessages = anchorEnv != null &&
+            anchorEnv !== "" &&
+            Number.isFinite(Number(anchorEnv))
+            ? Number(anchorEnv)
+            : config.anchorUserMessages;
+        const cut = computeLiveTrimCut(view, {
+            compactedFrom: ran.result.compactedFrom,
+            summary: ran.result.summary,
+            anchorUserMessages,
+            // CRITICAL-OVER ESCAPE HATCH: when context is at/over ~90% of the
+            // window, relief takes priority over the anchor floor. Without this,
+            // computeLiveTrimCut bails to null (can't satisfy the floor) and the
+            // model is fed a raw overflow that errors every turn — the
+            // "Already compacted" + overflow death-spiral (2026-08-01 incident).
+            // A thin anchor is recoverable; an overflowed session is not.
+            //
+            // CRITICAL: pct is null for OpenAI-compatible providers that don't
+            // report usage.percent (e.g. neuralwatt). Without the token-pressure
+            // fallback the hatch never armed → cut=null → raw overflow → 400
+            // "conversation too long even after compaction" (2026-08-03 incident
+            // on glm-5.2-short, 200K window). Now also fires on pressure >= 0.9
+            // (token-basis) so the hatch arms regardless of whether the provider
+            // reports pct.
+            criticalOver: (pct ?? 0) >= 90 || pressure >= 0.9,
+        });
+        if (cut === null) {
+            runtime.diagCtxCutNull++;
+            runtime.logger.info("live-trim-skip", {
+                sessionId: runtime.rt.sessionId,
+                compactedFrom: ran.result.compactedFrom,
+                viewLen: view.length,
+                anchorUserMessages,
+                criticalOver: (pct ?? 0) >= 90,
+            });
+            return tailResult() ?? undefined; // unsafe / below anchor floor — no trim this call
+        }
+        const summaryMsg = liveTrimSummaryMessage({
+            compactedFrom: ran.result.compactedFrom,
+            summary: ran.result.summary,
+            anchorUserMessages: config.anchorUserMessages,
+        });
+        // Synthesize a user-role AgentMessage carrying the compacted summary.
+        const summaryAgentMsg = {
+            role: "user",
+            content: summaryMsg.text,
+            // v0.8.6: stable timestamp across the epoch (NOT Date.now()) so the
+            // summary message bytes — and thus the KV-cache prefix — don't drift
+            // on every replay within the same compaction epoch.
+            timestamp: runtime.rt.lastCompactAt ?? Date.now(),
+        };
+        const recentRaw = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
+        // FIX 2 (2026-08-03 incident): TOKEN-BUDGET CAP on the live-trim view.
+        // Compaction fires at tier% of the window (140K for a 200K window),
+        // but a SINGLE turn can inject a huge tool output (file read, bash) that
+        // jumps context from 139K → 199K+ before the next gate fires. When that
+        // happens [summary + preserved tail] can STILL exceed the model window,
+        // and the provider rejects with 400 "conversation too long even after
+        // compaction". The anchor floor (PREVENT-PI-001) keeps ≥N user messages
+        // but has NO token cap, so a 2-message tail of two 80K bash outputs sails
+        // right past the window.
+        //
+        // Cap: when the model context window is known, reserve room for the
+        // summary + the model's max output tokens + a 10% safety margin, then
+        // drop oldest preserved messages from the front of `recentRaw` until the
+        // tail fits. Never drops below the FINAL message (always keep the latest
+        // turn so the agent can respond). This is a last-resort HARD cap — it
+        // only fires when the preserved tail alone is oversized, which is rare.
+        const ctxWindow = runtime.lastCtxWindow;
+        // Reuse the per-model threshold resolved at the gate (single lookup).
+        const modelThreshold = perModelThreshold;
+        // Reserve room for output tokens. Use the model's reported max output
+        // when known; fall back to 10% of the window (scales with any model —
+        // 20K for a 200K window, 100K for a 1M window) so we never let the
+        // preserved tail eat the model's output budget when maxTokens is unknown.
+        const maxOutput = runtime.currentModel?.maxTokens && runtime.currentModel.maxTokens > 0
+            ? runtime.currentModel.maxTokens
+            : Math.ceil(ctxWindow * 0.1);
+        let recent = recentRaw;
+        if (ctxWindow > 0 && recentRaw.length > 1) {
+            const summaryTokens = estimateBlockTokens(summaryMsg.text);
+            // Reserve: summary + max output + per-model safety margin (0-20%).
+            const safetyMargin = Math.ceil(ctxWindow * (modelThreshold.safetyMarginPct / 100));
+            const budget = ctxWindow - maxOutput - safetyMargin - summaryTokens;
+            if (budget > 0) {
+                // Walk recent from the front, dropping oldest first until the
+                // remaining tail fits. Use the AgentMessage→engine-text estimate via
+                // messageContentText (already imported) + estimateMessageTokens.
+                let tailTokens = 0;
+                for (let i = recentRaw.length - 1; i >= 0; i--) {
+                    const m = recentRaw[i];
+                    tailTokens += estimateMessageTokens({
+                        text: messageContentText(m),
+                    });
+                    if (tailTokens > budget) {
+                        // Keep from i+1 onward; but never fewer than the final message.
+                        const startIdx = Math.min(i + 1, recentRaw.length - 1);
+                        if (startIdx > 0) {
+                            recent = recentRaw.slice(startIdx);
+                            runtime.logger.warn("live-trim-tail-cap", {
+                                sessionId: runtime.rt.sessionId,
+                                dropped: startIdx,
+                                tailTokens,
+                                safetyMarginPct: modelThreshold.safetyMarginPct,
+                                budget,
+                                ctxWindow,
+                            });
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+        // v0.8.6: cache the trim view so subsequent gated calls in this epoch
+        // replay it verbatim (stabilizing the KV-cache prefix) instead of
+        // regenerating a fresh summary + sentinel every fire.
+        runtime.trimCache = {
+            // v0.8.7: key the replay cache on the STABLE epoch signal
+            // (rt.lastCheckpointId) instead of ran.result.checkpointId, which is
+            // dedup-volatile: on a re-compact that dedups onto a DIFFERENT existing
+            // checkpoint, result.checkpointId is the matched id (engine.ts:188) while
+            // lastCheckpointId is only updated on a genuinely new checkpoint
+            // (compact.ts:100-104). Keying on result.checkpointId would make
+            // trimCache.checkpointId != rt.lastCheckpointId forever after that
+            // dedup fire, disabling replay for the rest of the epoch (the
+            // alternating cache-miss that 0.8.6 meant to fix). Prefer the stable
+            // signal; fall back to result.checkpointId then the epoch timestamp
+            // only for the no-checkpoint edge case.
+            checkpointId: runtime.rt.lastCheckpointId ??
+                ran.result.checkpointId ??
+                `epoch-${runtime.rt.lastCompactAt ?? Date.now()}`,
+            cut,
+            summaryAgentMsg,
+            ctxPct: pct ?? null,
+            ctxTokens: currentTokens,
+        };
+        runtime.snapshot(ctx);
+        // DIAG (team-run relief): confirm the live trim actually fires + how big
+        // the window still is. The return is non-durable (per-LLM-call only), so
+        // this is the signal that the model is being fed a compacted view while
+        // the on-disk transcript + context meter keep growing.
+        runtime.diagLiveTrimFires++;
+        runtime.logger.info("live-trim", {
+            sessionId: runtime.rt.sessionId,
+            inputMsgs: messages.length,
+            outputMsgs: recent.length + 1,
+            compactedFrom: cut,
+            ctxPct: pct,
+            ctxTokens: usageTokens,
+        });
+        return (tailResult([summaryAgentMsg, ...recent]) ?? {
+            messages: [summaryAgentMsg, ...recent],
+        });
+    }
+    catch {
+        runtime.diagCtxThrown++;
+        return tailResult() ?? undefined; // non-fatal: no trim this call; the next context event retries
+    }
+}

package/dist/extensions/mega-events/context-handler/pipelineRun.js

@@ -0,0 +1,37 @@
+import { runCompact } from "../../mega-pipeline.js";
+import { pressureFromPct, pressureRatio } from "../../mega-config.js";
+/**
+ * Invoke the compaction pipeline with adaptive pressure. Returns a tailed view
+ * ("return") when compaction skipped, or "proceed" with the non-skipped result
+ * and the computed pressure (consumed by live-trim's critical-over hatch).
+ */
+export function invokePipeline(pi, runtime, config, ctx, opts) {
+    // Adaptive compression (Fix E): scale compression strength + keepFrom depth
+    // with how close we are to the model context limit. Null-safe: when the
+    // token-fallback path ran (pct unavailable) use the token-basis pressure
+    // (the same basis the runtime `pressure` getter uses for custom/no-window).
+    const pressure = opts.pct != null
+        ? pressureFromPct(opts.pct)
+        : pressureRatio(opts.currentTokens, runtime.effectiveThreshold);
+    const ran = runCompact(pi, runtime, config, ctx, opts.messages, {
+        compressionPressure: pressure,
+    });
+    // D.3: skip paths fall back to replay instead of returning empty.
+    // If runCompact skipped and we have a valid trimCache, replay it
+    // (free stability win) — otherwise defer to the next event.
+    if (ran.skipped) {
+        runtime.diagCtxRunSkipped++;
+        if (runtime.trimCache &&
+            runtime.trimCache.checkpointId === runtime.rt.lastCheckpointId &&
+            runtime.trimCache.cut <= opts.messages.length) {
+            const recent = opts.messages.slice(runtime.trimCache.cut); // guardrails-allow PREVENT-PI-002: cached `cut` was sanitized by computeLiveTrimCut (src/boundary.ts); replayed verbatim, transcript only grows within an epoch.
+            runtime.diagLiveTrimFires++;
+            runtime.diagLiveTrimReplays++;
+            runtime.snapshot(ctx);
+            const skipView = [{ ...runtime.trimCache.summaryAgentMsg }, ...recent];
+            return { kind: "return", view: opts.tailResult(skipView) ?? { messages: skipView } };
+        }
+        return { kind: "return", view: opts.tailResult() ?? undefined };
+    }
+    return { kind: "proceed", ran, pressure };
+}