pi-mega-compact 0.20.22 → 0.20.24

package/dist/extensions/mega-events/context-handler/dbMirrorAppend.js

@@ -3,6 +3,21 @@ import { appendMirrorMessages } from "../mirror-append.js";
 import { appendMessagesToLedger } from "../../mega-runtime/vector-cortex-ledger.js";
 import { epochIdFor } from "../../../src/mirror/epoch.js";
 import { messageContentText } from "./messageText.js";
+/**
+ * Best-effort tool_call_id for the tool_results insert. The toolResult variant
+ * carries a top-level toolCallId (read via an `unknown`-narrowed cast — never
+ * reach into `.content`, which is variant-specific and requires narrowing).
+ * bashExecution has no toolCallId, so fall back to a stable synthetic id keyed
+ * on (turn, index) to satisfy the NOT NULL column. No `any` (PREVENT-011).
+ */
+function toolCallIdOf(m, fallback) {
+    if (m.role === "toolResult") {
+        const id = m.toolCallId;
+        if (typeof id === "string" && id.length > 0)
+            return id;
+    }
+    return fallback;
+}
 /**
  * Append incoming messages to the DB mirror (raw_transcript + thread/tool
  * tables) and the v2 ledger. Gated on config.dbMirror for the mirror; the VC1B
@@ -26,22 +41,35 @@ export function appendMirrorAndLedger(runtime, config, messages) {
             // separation (buildSeparatedPrompt / buildCacheOptimizedPrompt in
             // tailResult) is sufficient for the prompt-construction path;
             // these DB writes persist the split for post-hoc analysis, dashboard
-            // queries, and future readers. Non-fatal — failure here never breaks
-            // the agent loop (PREVENT-PI-004: zero network, local SQLite only).
-            {
+            // queries, and future readers. Gated on (messageSeparation ||
+            // cacheStriping) so flag-OFF remains byte-identical to the
+            // predecessor — when both flags are OFF the live prompt is never
+            // separated, and growing these tables would be dead state.
+            // Non-fatal — failure here never breaks the agent loop
+            // (PREVENT-PI-004: zero network, local SQLite only).
+            if (config.messageSeparation || config.cacheStriping) {
                 const sid = runtime.rt.sessionId;
                 const turn = runtime.currentTurn;
                 const now = Date.now();
                 const threadStmt = db.prepare("INSERT OR IGNORE INTO conversation_thread (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
-                const toolStmt = db.prepare("INSERT OR IGNORE INTO tool_results (conversation_id, role, content, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
-                for (const m of messages) {
+                // Schema (plan-v2.ts) is (conversation_id, tool_call_id,
+                // tool_result, turn_index, timestamp) — NOT role/content.
+                const toolStmt = db.prepare("INSERT OR IGNORE INTO tool_results (conversation_id, tool_call_id, tool_result, turn_index, timestamp) VALUES (?, ?, ?, ?, ?)");
+                const toolHas = db.prepare("SELECT 1 FROM tool_results WHERE conversation_id = ? AND turn_index = ? AND tool_call_id = ? AND tool_result = ? LIMIT 1");
+                const threadHas = db.prepare("SELECT 1 FROM conversation_thread WHERE conversation_id = ? AND turn_index = ? AND role = ? AND content = ? LIMIT 1");
+                for (const [idx, m] of messages.entries()) {
                     const role = m.role;
                     const content = messageContentText(m);
                     if (role === "user" || role === "assistant") {
-                        threadStmt.run(sid, role, content, turn, now);
+                        if (threadHas.get(sid, turn, role, content) == null) {
+                            threadStmt.run(sid, role, content, turn, now);
+                        }
                     }
                     else if (role === "toolResult" || role === "bashExecution") {
-                        toolStmt.run(sid, role, content, turn, now);
+                        const toolCallId = toolCallIdOf(m, `bash:${turn}:${idx}`);
+                        if (toolHas.get(sid, turn, toolCallId, content) == null) {
+                            toolStmt.run(sid, toolCallId, content, turn, now);
+                        }
                     }
                 }
             }

package/dist/extensions/mega-events/context-handler/pipelineRun.js

@@ -1,11 +1,39 @@
 import { runCompact } from "../../mega-pipeline.js";
 import { pressureFromPct, pressureRatio } from "../../mega-config.js";
+import { recordCompactLatency } from "../../mega-runtime/vc-observer.js";
+import { decideLivePath } from "../../mega-runtime/vector-cortex-live.js";
+import { VC5C_ENABLED } from "../../../src/config/vector-cortex.js";
 /**
  * Invoke the compaction pipeline with adaptive pressure. Returns a tailed view
  * ("return") when compaction skipped, or "proceed" with the non-skipped result
  * and the computed pressure (consumed by live-trim's critical-over hatch).
  */
 export function invokePipeline(pi, runtime, config, ctx, opts) {
+    // VC5C: emit the rollout decision per compact event (observability seam).
+    // vcGate/vcHardFaults are not yet declared on MegaRuntime.rt — cast to the
+    // rollout shapes. Best-effort + non-fatal; decision does NOT gate behavior yet.
+    if (VC5C_ENABLED()) {
+        try {
+            const decision = decideLivePath(runtime.rt.sessionId, {
+                emit: runtime.appendEvent.bind(runtime),
+                currentGate: runtime.rt.vcGate ?? 0,
+                hardFaults: runtime.rt
+                    .vcHardFaults ?? [],
+            });
+            runtime.appendEvent("vector_cortex_rollout_decision", {
+                sessionId: runtime.rt.sessionId,
+                vcActive: decision.vcActive,
+                forcedPreVc: decision.forcedPreVc,
+                mode: decision.mode,
+                bucket: decision.bucket,
+                gateIndex: decision.gateIndex,
+                promotionBlocked: decision.promotionBlocked,
+            });
+        }
+        catch {
+            /* non-fatal: rollout decision emission never breaks compaction */
+        }
+    }
     // Adaptive compression (Fix E): scale compression strength + keepFrom depth
     // with how close we are to the model context limit. Null-safe: when the
     // token-fallback path ran (pct unavailable) use the token-basis pressure
@@ -13,9 +41,14 @@ export function invokePipeline(pi, runtime, config, ctx, opts) {
     const pressure = opts.pct != null
         ? pressureFromPct(opts.pct)
         : pressureRatio(opts.currentTokens, runtime.effectiveThreshold);
+    const t0 = Date.now();
     const ran = runCompact(pi, runtime, config, ctx, opts.messages, {
         compressionPressure: pressure,
     });
+    // VC0A: record compact latency on the eval observer (mode A) on every
+    // outcome so the dashboard histogram reflects real data. No-op when the
+    // observer is absent (flag off / construction failure).
+    recordCompactLatency(runtime, Date.now() - t0, runtime.rt.sessionId, runtime.rt.compactCount);
     // D.3: skip paths fall back to replay instead of returning empty.
     // If runCompact skipped and we have a valid trimCache, replay it
     // (free stability win) — otherwise defer to the next event.

package/dist/extensions/mega-events/context-handler/tailResult.js

@@ -1,6 +1,8 @@
 import { stagedForTail, withRecallTail } from "../recall-tail.js";
 import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "../separated-prompt.js";
 import { messageContentText } from "./messageText.js";
+import { computeContentDigest } from "../../../src/dedup/digest.js";
+const lastTurnPrefix = new WeakMap();
 /**
  * Build the tail injection closure. Returns undefined when nothing is staged
  * (or the flags are OFF) so the caller falls through to its normal return.
@@ -25,20 +27,51 @@ export function buildTailResult(runtime, config, messages) {
         else if (config.messageSeparation) {
             result = buildSeparatedPrompt(result);
         }
-        // P2.5: log prefix stability (fire-and-forget, non-fatal).
-        // tailResult is sync, so use .then().catch() on the dynamic import.
+        // P2.5: log cross-turn stable-prefix length (cache-hit proxy). After the
+        // prompt is built (separated or cache-optimized), fingerprint each leading
+        // message and count how many are byte-identical to the previous turn's
+        // prompt, in order. A high stablePrefix = the provider KV-cache prefix is
+        // re-used (cache hit). Fire-and-forget + non-fatal.
+        //
+        // tailResult may run several times per turn (gate return / replay /
+        // debounce / live-trim), so the sessionId+turn guard measures once per
+        // turn — compare against the PREVIOUS turn's stored footprint, then store
+        // this turn's for the next comparison. Cross-session compares are skipped
+        // via the stored sessionId check (belt-and-suspenders; the WeakMap entry
+        // is scoped to this runtime, which owns the session).
         if (result.length > 1) {
-            import("../../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
-                const stableScore = computeStabilityScore({ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 }, result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })));
-                runtime.logger.info("prefix_stability", {
-                    stableScore: Number.isFinite(stableScore) ? stableScore : 0,
-                    prefixMessages: result.length,
-                    separation: config.messageSeparation ? "v2" : "off",
-                    striping: config.cacheStriping ? "v3" : "off",
-                });
-            }).catch(() => {
+            try {
+                const sessionId = runtime.rt.sessionId;
+                const turn = runtime.currentTurn;
+                const prev = lastTurnPrefix.get(runtime);
+                const isNewTurn = !prev || prev.sessionId !== sessionId || prev.turn !== turn;
+                if (isNewTurn) {
+                    const fingerprints = result.map((m) => 
+                    // Role prefix disambiguates identical text across variants
+                    // (roles are a fixed enum — no realistic fingerprint collision).
+                    computeContentDigest(`${m.role}|${messageContentText(m)}`).contentHash);
+                    let stablePrefix = 0;
+                    if (prev && prev.sessionId === sessionId) {
+                        for (let i = 0; i < fingerprints.length; i++) {
+                            if (i >= prev.fingerprints.length ||
+                                prev.fingerprints[i] !== fingerprints[i]) {
+                                break;
+                            }
+                            stablePrefix++;
+                        }
+                    }
+                    lastTurnPrefix.set(runtime, { sessionId, turn, fingerprints });
+                    runtime.logger.info("prefix_stability", {
+                        stablePrefix,
+                        totalMessages: result.length,
+                        separation: config.messageSeparation ? "v2" : "off",
+                        striping: config.cacheStriping ? "v3" : "off",
+                    });
+                }
+            }
+            catch {
                 // Non-fatal: stability logging is best-effort.
-            });
+            }
         }
         return { messages: result };
     };

package/dist/extensions/mega-runtime/dashboard-snapshot.js

@@ -17,6 +17,7 @@ export function buildDashboardSnapshot(ctx) {
         tier: ctx.pressureBand,
         presetTier: ctx.config.tier,
         pressure: ctx.pressure,
+        vcObserverSamples: ctx.vcObserverSamples ?? 0,
         config: {
             fastGatePct: ctx.config.fastGatePct,
             thresholdTokens: ctx.effectiveThreshold,
@@ -73,17 +74,17 @@ export function buildDashboardSnapshot(ctx) {
         repo: ctx.repo,
         compression: {
             session: {
-                tokensIn: ctx.rt.tokensSaved + (ctx.st.totalTokenEstimate - ctx.st.originalTokens),
+                tokensIn: ctx.st.originalTokens,
                 tokensOut: ctx.st.totalTokenEstimate,
                 tokensFreed: ctx.rt.tokensSaved,
-                compressionPct: ctx.rt.tokensSaved / Math.max(1, ctx.rt.tokensSaved + (ctx.st.totalTokenEstimate - ctx.st.originalTokens)),
+                compressionPct: ctx.rt.tokensSaved / Math.max(1, ctx.st.originalTokens),
                 dedupPct: ctx.rt.dedupAttempts > 0 ? ctx.rt.dedupSkips / ctx.rt.dedupAttempts : 0,
             },
             repo: {
-                tokensIn: ctx.repo.tokensSaved + (ctx.repo.totalTokenEstimate - ctx.repo.originalTokens),
+                tokensIn: ctx.repo.originalTokens,
                 tokensOut: ctx.repo.totalTokenEstimate,
                 tokensFreed: ctx.repo.tokensSaved,
-                compressionPct: ctx.repo.tokensSaved / Math.max(1, ctx.repo.tokensSaved + (ctx.repo.totalTokenEstimate - ctx.repo.originalTokens)),
+                compressionPct: ctx.repo.tokensSaved / Math.max(1, ctx.repo.originalTokens),
                 dedupPct: ctx.repo.dedupAttempts > 0 ? ctx.repo.dedupCollapsed / ctx.repo.dedupAttempts : 0,
             },
         },
@@ -96,7 +97,9 @@ export function buildDashboardSnapshot(ctx) {
             // per-session in rt and there's no cumulative counter yet.
             total: ctx.repo.dedupCollapsed,
             sessionTokensSaved: ctx.rt.cacheHitTokens,
-            totalTokensSaved: ctx.repo.dedupCollapsed > 0 ? ctx.repo.dedupCollapsed * 100 : 0,
+            // Cumulative tokens saved across all sessions in this repo —
+            // was a placeholder (dedupCollapsed * 100).
+            totalTokensSaved: ctx.repo.tokensSaved,
         },
         compacts: {
             session: ctx.rt.compactCount,
@@ -109,7 +112,7 @@ export function buildDashboardSnapshot(ctx) {
             },
             cacheHit: {
                 sessionSec: ctx.rt.cacheHitTokens / 1000,
-                totalSec: (ctx.repo.dedupCollapsed * 100) / 1000,
+                totalSec: ctx.repo.tokensSaved / 1000,
             },
         },
         model: ctx.currentModel

package/dist/extensions/mega-runtime/runtime-snapshot.js

@@ -22,6 +22,7 @@ import { latestModelSnapshot, readLatestCacheHitPct, recordPerfSample, recordSes
 import { resolveRepoRoot, } from "../mega-config.js";
 import { computeMegaSnapshot } from "./snapshot.js";
 import { buildDashboardSnapshot } from "./dashboard-snapshot.js";
+import { getVcObserver } from "./vc-observer.js";
 import { materialSigImpl, embedderNameImpl, driftStatusImpl, getTurnLevelImpl, } from "./runtime-helpers.js";
 // -------------------------------------------------------------- snapshotImpl
 /** Collect live state and write it to disk (+ paint the above-editor widget).
@@ -85,6 +86,7 @@ export function snapshotImpl(self, ctx) {
         pressureBand: self.pressureBand,
         pressure: self.pressure,
         effectiveThreshold: self.effectiveThreshold,
+        vcObserverSamples: getVcObserver(self)?.rows().length ?? 0,
         statusKey: self.statusKey,
         lastCtxTokens: self.lastCtxTokens,
         lastCtxPercent: self.lastCtxPercent,

package/dist/extensions/mega-runtime/vc-observer.js

@@ -0,0 +1,98 @@
+/**
+ * vc-observer.ts — VC0A mode-A eval observer wiring for MegaRuntime.
+ *
+ * Bridges the pure in-memory EvalObserver (src/vector-cortex/eval/observer.ts)
+ * into the production runtime: every recorded compact/recall latency sample is
+ * both emitted to events.log (via MegaRuntime.appendEvent) and appended to the
+ * redacted evaluation.jsonl (via appendEvalRow) so the Vector Cortex dashboard
+ * "Observer" histogram reads real data instead of an empty file.
+ *
+ * All wiring is gated on VC0A_ENABLED() (MEGACOMPACT_VC0A). Flag-off and any
+ * construction/persistence error are non-fatal (PREVENT-PI-004: local FS only,
+ * no network) and degrade to a null observer — byte-identical to the
+ * predecessor (mode C). PREVENT-011: no `any` type.
+ *
+ * Follows the capture-model.ts / append-event.ts pattern: a context interface +
+ * free functions + a thin delegate. No import-time side effects — merely
+ * importing these functions is inert regardless of the flag.
+ */
+import { VC0A_ENABLED } from "../../src/config.js";
+import { createEvalObserver } from "../../src/vector-cortex/eval/observer.js";
+import { appendEvalRow } from "../../src/vector-cortex/eval/persist.js";
+// ------------------------------------------------------------- construction
+/**
+ * Build the production-wired eval observer (mode A) if the VC0A flag is on.
+ * Returns null when the flag is off or on any construction error (non-fatal).
+ * The returned observer is a thin, best-effort observability seam that never
+ * breaks the agent loop.
+ */
+export function createVcObserver(self) {
+    if (!VC0A_ENABLED())
+        return null;
+    try {
+        return createEvalObserver({
+            emit: self.appendEvent.bind(self),
+            persist: (sample) => appendEvalRow(self.currentStateDir, [sample]),
+        });
+    }
+    catch {
+        /* non-fatal observability — never break the agent loop */
+        return null;
+    }
+}
+// ---------------------------------------------------------------- weak cache
+/**
+ * Module-level observer cache keyed by the runtime object. Avoids adding a
+ * `vcObserver` field to MegaRuntime (keeping runtime.ts under its soft limit
+ * — no field, no constructor line, no import bloat). One observer per runtime
+ * instance, lazily built on the first latency recording.
+ */
+const observerCache = new WeakMap();
+/** Lazily create + cache the observer for the given runtime context. */
+export function getVcObserver(self) {
+    let obs = observerCache.get(self);
+    if (obs === undefined) {
+        obs = createVcObserver(self);
+        observerCache.set(self, obs);
+    }
+    return obs;
+}
+// ---------------------------------------------------------------- reporting
+/** Record a compact latency sample on the observer (no-op when absent). */
+export function recordCompactLatency(self, elapsedMs, sessionId, epoch) {
+    const obs = getVcObserver(self);
+    if (obs == null)
+        return;
+    try {
+        obs.record({
+            session: sessionId,
+            seq: epoch,
+            event: "compact",
+            value: elapsedMs,
+            unit: "ms",
+            mode: "A",
+        });
+    }
+    catch {
+        /* non-fatal observability */
+    }
+}
+/** Record a recall latency sample on the observer (no-op when absent). */
+export function recordRecallLatency(self, elapsedMs, sessionId, epoch) {
+    const obs = getVcObserver(self);
+    if (obs == null)
+        return;
+    try {
+        obs.record({
+            session: sessionId,
+            seq: epoch,
+            event: "recall",
+            value: elapsedMs,
+            unit: "ms",
+            mode: "A",
+        });
+    }
+    catch {
+        /* non-fatal observability */
+    }
+}