pi-mega-compact 0.7.3 → 0.7.4

package/dist/extensions/mega-config.js

@@ -135,6 +135,7 @@ export function loadConfig() {
         dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
         raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
         legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
+        dbMirror: envBool("MEGACOMPACT_DB_MIRROR", false),
         crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
         crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
         memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),

package/dist/extensions/mega-events.js

@@ -7,6 +7,8 @@
  * sync and delegates the heavy lifting to the pipeline + command modules.
  */
 import { normalizeSessionId } from "../src/store.js";
+import { openStore, appendRawTranscript, writeCheckpointEpoch } from "../src/store/sqlite.js";
+import { epochIdFor } from "../src/mirror/epoch.js";
 import { autoCompactCheck } from "../src/compact.js";
 import { estimateSessionTokens, estimateBlockTokens } from "../src/tokens.js";
 import { recentUserQuery, WIDGET_KEY, } from "./mega-runtime.js";
@@ -15,6 +17,34 @@ import { recallMemoriesAndInline } from "../src/recall.js";
 import { driveNativeCompaction, } from "./mega-compact-driver.js";
 import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
 import { pressureFromPct, memoryReviewCadence, } from "./mega-config.js";
+import { createHash } from "node:crypto";
+/**
+ * Convert a pi AgentMessage to a RawTranscriptRow for the DB mirror.
+ * content_bytes is canonical JSON (sorted keys) for deterministic hashing.
+ * Returns null if the message has no usable content.
+ */
+function toRawTranscriptRow(msg, sessionId, epochId) {
+    // Narrow to Message union (has content + timestamp).
+    const m = msg;
+    const content = m.content;
+    if (content == null || content === "")
+        return null;
+    // Canonical form: sort object keys for deterministic hashing.
+    const contentBytes = typeof content === "string"
+        ? content
+        : JSON.stringify(content, Object.keys(content).sort());
+    const contentHash = createHash("sha256").update(contentBytes).digest("hex");
+    return {
+        contentHash,
+        sessionId,
+        seq: 0, // assigned by appendRawTranscript (COALESCE(MAX(seq),0)+1)
+        role: m.role ?? "unknown",
+        contentBytes,
+        toolName: m.toolName ?? null,
+        messageTimestamp: m.timestamp ?? null,
+        checkpointEpoch: epochId,
+    };
+}
 /**
  * DIAG accessor for the headless test harness: the most recently constructed
  * MegaRuntime, so a test that loads the compiled extension via its default
@@ -223,7 +253,21 @@ export function registerEventHandlers(pi, runtime, config) {
                 // trim we ALWAYS nudge so the agent reliably restarts. Debounced 30s.
                 let didDurableTrim = false;
                 if (idle && overThreshold && now >= runtime.debounceUntil) {
-                    if (!piCompactWouldNoop(ctx)) {
+                    // COMPACT-DEDUP FIX: skip the manual durable-trim trigger when pi's
+                    // NATIVE auto-compaction just fired (or is in-flight). pi emits
+                    // agent_end BEFORE its own _checkCompaction (per its docstring:
+                    // "Called after agent_end and before prompt submission"), so a
+                    // synchronous `piCompactWouldNoop` branch check misses a native
+                    // compaction that hasn't appended its entry yet — calling
+                    // ctx.compact() then races with pi and throws "Already compacted"
+                    // to the user. The `lastCompactAt` cooldown (updated by the
+                    // session_compact listener for EVERY compaction, native or
+                    // extension-supplied) closes that race window.
+                    const sinceCompact = now - (runtime.rt.lastNativeCompactAt ?? 0);
+                    if (sinceCompact < 10_000) {
+                        runtime.diagAgentEndDurableSkipRecent++;
+                    }
+                    else if (!piCompactWouldNoop(ctx)) {
                         runtime.debounceUntil = now + 2000;
                         runtime.diagAgentEndDurable++;
                         runtime.logger.info("agent-end-durable-trigger", {
@@ -232,7 +276,7 @@ export function registerEventHandlers(pi, runtime, config) {
                             thresholdTokens: config.thresholdTokens,
                             queued,
                         });
-                        ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
+                        ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort. Race-guarded by lastCompactAt cooldown above (ctx.compact returns void → throw is surfaced by pi as compaction_end; the cooldown prevents the call entirely).
                         didDurableTrim = true;
                     }
                 }
@@ -309,6 +353,23 @@ export function registerEventHandlers(pi, runtime, config) {
         const currentTokens = usage?.tokens ??
             estimateSessionTokens(view) ??
             Math.round((pct / 100) * (usage?.contextWindow ?? 0));
+        // S27 DB-mirror: append ALL incoming messages to raw_transcript.
+        // Runs BEFORE fast-gate so every message is captured, even if we
+        // don't compact this turn. Append is idempotent (content_hash PK).
+        if (config.dbMirror) {
+            try {
+                const db = openStore(runtime.currentStateDir);
+                const epochId = epochIdFor(runtime.rt.sessionId);
+                for (const msg of messages) {
+                    const raw = toRawTranscriptRow(msg, runtime.rt.sessionId, epochId);
+                    if (raw)
+                        appendRawTranscript(db, raw);
+                }
+            }
+            catch (e) {
+                runtime.logger.warn("db-mirror-append-fail", { error: String(e) });
+            }
+        }
         // FAST GATE: token-based (tier% of the window), not a static amount.
         if (currentTokens < runtime.effectiveThreshold) {
             runtime.diagCtxFastGate++;
@@ -336,6 +397,37 @@ export function registerEventHandlers(pi, runtime, config) {
             runtime.diagCtxRunSkipped++;
             return;
         }
+        // S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
+        // This makes the cache key stable across identical compactions.
+        if (config.dbMirror) {
+            try {
+                const db = openStore(runtime.currentStateDir);
+                const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
+                const epoch = {
+                    epochId: epochIdFor(cpId),
+                    sessionId: runtime.rt.sessionId,
+                    startedSeq: 0,
+                    committedSeq: ran.result.compactedFrom,
+                    checkpointId: cpId,
+                    cutIndex: ran.result.compactedFrom,
+                    summaryMessageText: ran.result.summary,
+                    createdAt: Date.now(),
+                };
+                writeCheckpointEpoch(db, epoch);
+                // S27 Task 6: Fire-and-forget dedup pipeline.
+                // Deduplicates raw_transcript rows for the compacted range.
+                try {
+                    const { dedupTranscript } = await import("../src/mirror/dedup.js");
+                    dedupTranscript(db, runtime.rt.sessionId, 0, ran.result.compactedFrom);
+                }
+                catch (_dedupErr) {
+                    // Fire-and-forget: dedup failure is non-fatal
+                }
+            }
+            catch (e) {
+                runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
+            }
+        }
         // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
         // manual compact path aborts the in-flight turn — only used behind the flag.
         // Read live from env (in addition to the load-time config) so the flag can be
@@ -345,9 +437,14 @@ export function registerEventHandlers(pi, runtime, config) {
             process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" ||
             process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
         if (legacy) {
-            if (piCompactWouldNoop(ctx))
+            // COMPACT-DEDUP FIX: same race guard as the agent_end path. Skip when a
+            // NATIVE compaction just fired (avoids racing pi and surfacing a spurious
+            // "Already compacted" / "Nothing to compact" toast). Uses lastNativeCompactAt
+            // (NOT lastCompactAt, which runCompact also stamps for our own checkpoint).
+            const sinceCompact = Date.now() - (runtime.rt.lastNativeCompactAt ?? 0);
+            if (sinceCompact < 10_000 || piCompactWouldNoop(ctx))
                 return;
-            ctx.compact({ customInstructions: undefined });
+            ctx.compact({ customInstructions: undefined }); // race-guarded by lastNativeCompactAt cooldown (ctx.compact returns void → not catchable; the cooldown prevents the call)
             return;
         }
         // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
@@ -482,6 +579,22 @@ export function registerEventHandlers(pi, runtime, config) {
         // Absolute last resort: let pi run its own (may throw "Nothing to compact").
         return {};
     });
+    // COMPACT-DEDUP FIX: track EVERY compaction (native + extension-supplied)
+    // so the agent_end durable-trim guard can skip a redundant ctx.compact()
+    // when pi just compacted. Without this, agent_end fires ctx.compact()
+    // synchronously AFTER pi's native auto-compaction appended a compaction
+    // entry but BEFORE our branch read sees it on the next tick — racing
+    // into a user-facing "Already compacted" throw. `lastCompactAt` is the
+    // race-closing signal: any compaction (manual/threshold/overflow, ours
+    // or pi's own) stamps it, and the agent_end guard skips for 10s.
+    pi.on("session_compact", async (_event, _ctx) => {
+        runtime.rt.lastNativeCompactAt = Date.now();
+        runtime.rt.lastCompactAt = Date.now();
+        runtime.logger.info("session-compacted", {
+            sessionId: runtime.rt.sessionId,
+            at: runtime.rt.lastCompactAt,
+        });
+    });
     /**
      * Build a minimal fallback compaction so pi never runs its throwing compact().
      *

package/dist/extensions/mega-events.test.js

@@ -0,0 +1,47 @@
+/**
+ * Tests for mega-events extension — DB-mirror event wiring.
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { openStore, listCheckpointEpochs, countRawTranscript } from "../src/store/sqlite.js";
+function makeTmp() {
+    return mkdtempSync(join(tmpdir(), "mega-events-test-"));
+}
+describe("mega-events: DB-mirror integration", () => {
+    let dir;
+    beforeEach(() => {
+        dir = makeTmp();
+        // openStore creates the tables as a side effect
+        openStore(dir);
+    });
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("openStore creates checkpoint_epochs and raw_transcript tables", () => {
+        const db = openStore(dir);
+        // Should not throw — tables exist
+        const epochs = listCheckpointEpochs(db);
+        assert.ok(Array.isArray(epochs));
+        const count = countRawTranscript(db);
+        assert.equal(count, 0);
+    });
+    it("DB-mirror flag defaults to false when env is unset", () => {
+        delete process.env.MEGACOMPACT_DB_MIRROR;
+        // Re-import to pick up env
+        // The extension checks env at load time, so just verify the env is absent
+        assert.equal(process.env.MEGACOMPACT_DB_MIRROR, undefined);
+    });
+    it("DB-mirror flag is enabled when env is '1'", () => {
+        process.env.MEGACOMPACT_DB_MIRROR = "1";
+        assert.equal(process.env.MEGACOMPACT_DB_MIRROR, "1");
+        delete process.env.MEGACOMPACT_DB_MIRROR;
+    });
+    it("DB-mirror flag is enabled when env is 'true'", () => {
+        process.env.MEGACOMPACT_DB_MIRROR = "true";
+        assert.equal(process.env.MEGACOMPACT_DB_MIRROR, "true");
+        delete process.env.MEGACOMPACT_DB_MIRROR;
+    });
+});

package/dist/extensions/mega-runtime.js

@@ -191,6 +191,7 @@ export class MegaRuntime {
         dedupAttempts: 0,
         tokensSaved: 0,
         lastCompactAt: null,
+        lastNativeCompactAt: null,
     };
     debounceUntil = 0;
     // S16: debounce for the agent_end resume nudge (avoid busy-loops).
@@ -255,6 +256,7 @@ export class MegaRuntime {
     diagBeforeCompactSupplied = 0; // session_before_compact supplied our trim
     diagAgentEndIdle = 0; // agent_end with activeAgents===0
     diagAgentEndDurable = 0; // agent_end fired ctx.compact() (mid-run durable trim)
+    diagAgentEndDurableSkipRecent = 0; // agent_end skipped ctx.compact() — compaction in last 10s (race guard)
     // Per-skip-path counters for the team-run diagnosis.
     diagCtxFastGate = 0; // returned at token fast-gate (below threshold)
     diagCtxNoCompact = 0; // autoCompactCheck().shouldCompact === false
@@ -719,6 +721,7 @@ export class MegaRuntime {
             dedupAttempts: 0,
             tokensSaved: 0,
             lastCompactAt: null,
+            lastNativeCompactAt: null,
         };
         this.statusKey = undefined;
         this.activeAgents = 0;

package/dist/src/mirror/dedup.js

@@ -0,0 +1,44 @@
+/**
+ * dedup.ts — S27 Task 6: Fork snapshot → compress/dedupe pipeline.
+ *
+ * After the served window is handed to pi, asynchronously:
+ * 1. Read raw_transcript rows [0..cut_index] for the epoch
+ * 2. For each row, compute content_hash (reuse digest from dedup/)
+ * 3. INSERT OR IGNORE INTO dedup_mirror (stores bytes once per unique hash)
+ * 4. Update raw_transcript.content_ref to point to dedup_mirror
+ * 5. Increment dedup_mirror.ref_count for existing hashes
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
+import { upsertDedupMirror, updateRawTranscriptRef, listRawTranscriptRange, getDedupRatio, } from "../store/sqlite.js";
+import { computeContentDigest } from "../dedup/digest.js";
+/**
+ * Deduplicate raw transcript rows for a session range.
+ * Fire-and-forget: errors are logged, not thrown.
+ *
+ * @returns Number of rows deduplicated, or -1 on error.
+ */
+export function dedupTranscript(db, sessionId, fromSeq, toSeq) {
+    try {
+        const rows = listRawTranscriptRange(db, sessionId, fromSeq, toSeq);
+        let deduped = 0;
+        for (const row of rows) {
+            const contentHash = computeContentDigest(row.contentBytes).contentHash;
+            const isNew = upsertDedupMirror(db, contentHash, row.contentBytes, row.seq);
+            updateRawTranscriptRef(db, sessionId, row.seq, contentHash);
+            if (!isNew) {
+                deduped++;
+            }
+        }
+        return deduped;
+    }
+    catch (err) {
+        // Fire-and-forget: log but don't throw
+        console.error("[mega-compact] dedupTranscript failed:", err);
+        return -1;
+    }
+}
+/**
+ * Get dedup ratio for a session.
+ */
+export { getDedupRatio };

package/dist/src/mirror/epoch.js

@@ -0,0 +1,36 @@
+/**
+ * epoch.ts — deterministic epoch-id derivation for the S27 DB-mirror.
+ *
+ * The epoch id MUST be a pure function of the checkpoint it decorates so that
+ * replaying / refreshing the same compaction yields the SAME epoch id (idempotent
+ * appends + ON CONFLICT refresh). No Date.now / uuid / crypto — this is the
+ * only source of randomness-free epoch naming in the mirror stack.
+ *
+ * - epochIdFor(cp)  → "epoch:" + cp   (human-traceable back to its checkpoint)
+ * - epochNonceFor(cp) → FNV-1a 32-bit hash (cheap, well-distributed nonce)
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
+/**
+ * FNV-1a 32-bit nonce for a checkpoint id. Deterministic and RNG-free:
+ * h = 0x811c9dc5; for each char: h ^= codePoint; h = Math.imul(h, 0x01000193);
+ * return h >>> 0 (unsigned).
+ */
+export function epochNonceFor(checkpointId) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < checkpointId.length; i++) {
+        const cp = checkpointId.codePointAt(i);
+        if (cp === undefined)
+            continue;
+        h ^= cp;
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/**
+ * Deterministic epoch id: "epoch:" + checkpointId. Trivially traceable back to
+ * the source checkpoint, and stable under replay (refresh-safe upserts).
+ */
+export function epochIdFor(checkpointId) {
+    return "epoch:" + checkpointId;
+}

package/dist/src/mirror/mirror.test.js

@@ -0,0 +1,185 @@
+/**
+ * mirror.test.ts — S27 DB-mirror integration tests.
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ *
+ * NOTE: raw_transcript has PRIMARY KEY (content_hash, session_id), so
+ * duplicate content in the same session is silently dropped by INSERT OR IGNORE.
+ * Tests are designed around this constraint.
+ */
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import { openStore, closeStore } from "../../src/store/sqlite.js";
+import { writeCheckpointEpoch, listCheckpointEpochs, appendRawTranscript, listRawTranscriptRange, upsertDedupMirror, getDedupRatio, getDedupMirrorStats, countRawTranscript, } from "../../src/store/sqlite.js";
+import { epochIdFor } from "../../src/mirror/epoch.js";
+import { dedupTranscript } from "../../src/mirror/dedup.js";
+import { computeContentDigest } from "../../src/dedup/digest.js";
+function tmp() {
+    return mkdtempSync(join(tmpdir(), "mirror-test-"));
+}
+/**
+ * Build a valid RawTranscriptRow using the canonical content hash from
+ * computeContentDigest (matches what dedupTranscript uses).
+ */
+function mkRow(sessionId, seq, // ignored by appendRawTranscript (auto-assigned)
+role, content) {
+    const { contentHash } = computeContentDigest(content);
+    return {
+        contentHash,
+        sessionId,
+        seq,
+        role,
+        contentBytes: content,
+        toolName: null,
+        messageTimestamp: Date.now(),
+        checkpointEpoch: "",
+    };
+}
+describe("S27 DB-mirror", () => {
+    it("epochIdFor is deterministic", () => {
+        assert.equal(epochIdFor("cp-abc-123"), epochIdFor("cp-abc-123"));
+        assert.notEqual(epochIdFor("cp-abc-123"), epochIdFor("cp-abc-456"));
+        const id = epochIdFor("cp-abc-123");
+        assert.ok(id.startsWith("epoch:"));
+        assert.ok(id.length > 6);
+    });
+    it("writeCheckpointEpoch + listCheckpointEpochs round-trips", () => {
+        const dir = tmp();
+        const db = openStore(dir);
+        writeCheckpointEpoch(db, {
+            epochId: "epoch-test-001",
+            sessionId: "sess-abc",
+            startedSeq: 0,
+            committedSeq: 100,
+            checkpointId: "cp-test-001",
+            cutIndex: 100,
+            summaryMessageText: "Test summary",
+            createdAt: Date.now(),
+        });
+        const rows = listCheckpointEpochs(db);
+        assert.ok(rows.length >= 1);
+        assert.equal(rows[0].epochId, "epoch-test-001");
+        assert.equal(rows[0].sessionId, "sess-abc");
+        assert.equal(rows[0].checkpointId, "cp-test-001");
+        closeStore(dir);
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("appendRawTranscript + listRawTranscriptRange round-trips (unique content)", () => {
+        const dir = tmp();
+        const db = openStore(dir);
+        // Use unique content for each row to avoid PK collision
+        appendRawTranscript(db, mkRow("sess-abc", 0, "user", "first message"));
+        appendRawTranscript(db, mkRow("sess-abc", 1, "assistant", "second message"));
+        appendRawTranscript(db, mkRow("sess-abc", 2, "user", "third message"));
+        // seq is auto-assigned: 1, 2, 3
+        const rows = listRawTranscriptRange(db, "sess-abc", 0, 10);
+        assert.equal(rows.length, 3);
+        assert.equal(rows[0].contentBytes, "first message");
+        assert.equal(rows[0].seq, 1);
+        assert.equal(rows[1].contentBytes, "second message");
+        assert.equal(rows[1].seq, 2);
+        assert.equal(rows[2].contentBytes, "third message");
+        assert.equal(rows[2].seq, 3);
+        // Range filter: [2..3]
+        const rows2 = listRawTranscriptRange(db, "sess-abc", 2, 3);
+        assert.equal(rows2.length, 2);
+        assert.equal(rows2[0].contentBytes, "second message");
+        assert.equal(rows2[1].contentBytes, "third message");
+        closeStore(dir);
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("upsertDedupMirror increments ref_count for duplicate content", () => {
+        const dir = tmp();
+        const db = openStore(dir);
+        const isNew1 = upsertDedupMirror(db, "hash-aaa", "Hello", 0);
+        assert.equal(isNew1, true);
+        const isNew2 = upsertDedupMirror(db, "hash-aaa", "Hello", 1);
+        assert.equal(isNew2, false);
+        const stats = getDedupMirrorStats(db);
+        assert.equal(stats.rowCount, 1);
+        assert.equal(stats.avgRefCount, 2);
+        closeStore(dir);
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("dedupTranscript deduplicates cross-session content via dedup_mirror", () => {
+        const dir = tmp();
+        const db = openStore(dir);
+        // Insert same content in TWO different sessions (raw_transcript PK allows this)
+        appendRawTranscript(db, mkRow("sess-a", 0, "user", "shared hello"));
+        appendRawTranscript(db, mkRow("sess-a", 1, "assistant", "shared world"));
+        appendRawTranscript(db, mkRow("sess-a", 2, "user", "unique A"));
+        appendRawTranscript(db, mkRow("sess-b", 0, "user", "shared hello"));
+        appendRawTranscript(db, mkRow("sess-b", 1, "assistant", "shared world"));
+        appendRawTranscript(db, mkRow("sess-b", 2, "user", "unique B"));
+        // Dedup session A: 3 rows, all new → deduped=0
+        const dedupedA = dedupTranscript(db, "sess-a", 0, 10);
+        assert.equal(dedupedA, 0);
+        // Dedup session B: 3 rows, but 2 already in dedup_mirror → deduped=2
+        const dedupedB = dedupTranscript(db, "sess-b", 0, 10);
+        assert.equal(dedupedB, 2);
+        // dedup_mirror has 4 unique hashes: shared-hello, shared-world, unique-A, unique-B
+        const stats = getDedupMirrorStats(db);
+        assert.equal(stats.rowCount, 4);
+        assert.ok(stats.avgRefCount > 1);
+        closeStore(dir);
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("getDedupRatio reflects dedup savings", () => {
+        const dir = tmp();
+        const db = openStore(dir);
+        // Two sessions with identical content → cross-session dedup
+        for (let i = 0; i < 3; i++) {
+            appendRawTranscript(db, mkRow("sess-x", i, "user", "same content"));
+            appendRawTranscript(db, mkRow("sess-y", i, "user", "same content"));
+        }
+        // Each session has 1 row (PK dedup within session), so 1 row each
+        // sess-x: 1 row, sess-y: 1 row
+        dedupTranscript(db, "sess-x", 0, 10);
+        dedupTranscript(db, "sess-y", 0, 10);
+        // For sess-x: totalBytes = LENGTH("same content") = 12, uniqueBytes = 12 → ratio 1.0
+        const { totalBytes, uniqueBytes, ratio } = getDedupRatio(db, "sess-x");
+        assert.ok(totalBytes > 0);
+        assert.ok(uniqueBytes > 0);
+        assert.ok(ratio >= 1.0);
+        closeStore(dir);
+        rmSync(dir, { recursive: true, force: true });
+    });
+    it("full pipeline: append + dedup + epoch", () => {
+        const dir = tmp();
+        const db = openStore(dir);
+        // Insert 5 unique rows
+        const contents = ["alpha", "bravo", "charlie", "delta", "echo"];
+        for (let i = 0; i < contents.length; i++) {
+            appendRawTranscript(db, mkRow("sess-pipe", i, i % 2 === 0 ? "user" : "assistant", contents[i]));
+        }
+        const total = countRawTranscript(db);
+        assert.ok(total >= 5);
+        // Dedup: all 5 unique → deduped = 0
+        const deduped = dedupTranscript(db, "sess-pipe", 0, 100);
+        assert.equal(deduped, 0);
+        // Mirror should have 5 unique hashes
+        const stats = getDedupMirrorStats(db);
+        assert.equal(stats.rowCount, 5);
+        // Write checkpoint epoch
+        writeCheckpointEpoch(db, {
+            epochId: "epoch-integration",
+            sessionId: "sess-pipe",
+            startedSeq: 0,
+            committedSeq: 100,
+            checkpointId: "cp-integration",
+            cutIndex: 5,
+            summaryMessageText: "Integration test summary",
+            createdAt: Date.now(),
+        });
+        const epochs = listCheckpointEpochs(db);
+        assert.ok(epochs.length >= 1);
+        assert.equal(epochs[0].epochId, "epoch-integration");
+        const rows = listRawTranscriptRange(db, "sess-pipe", 0, 100);
+        assert.equal(rows.length, 5);
+        closeStore(dir);
+        rmSync(dir, { recursive: true, force: true });
+    });
+});

package/dist/src/recall.js

@@ -41,7 +41,44 @@ export function formatRecallBlock(hits) {
  * it records injections via `markInjected` so the next call dedupes. The
  * `store` is passed by the extension (defaults to the engine's default store).
  */
+/**
+ * Recall and inline context from the checkpoint store.
+ *
+ * S27 contract — DB-Mirror demotion:
+ *
+ * When `MEGACOMPACT_DB_MIRROR` is ON, the `raw_transcript` table is the
+ * canonical, byte-stable source of truth for message reconstruction. The
+ * `dedup_mirror` provides space-efficient storage with ref_count tracking
+ * (see src/mirror/dedup.ts). The legacy JSON checkpoint is retained as a
+ * DR snapshot only (see src/store.ts checkpoint helpers).
+ *
+ * The recall function continues to work from the VectorStore (checkpoint
+ * summaries + embeddings) for fast semantic search — this path is unaffected
+ * by the mirror flag. If full transcript reconstruction is ever needed
+ * (replay, export, debug), prefer reading from `raw_transcript + dedup_mirror`
+ * via `listRawTranscriptRange()` + `dedupTranscript()` instead of the legacy
+ * JSON checkpoint. Falls back to legacy checkpoint if mirror is empty
+ * (pre-migration sessions).
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
 export function recallAndInline(opts, store) {
+    // ── S27 Recall Demotion ─────────────────────────────────────────────
+    //
+    // When MEGACOMPACT_DB_MIRROR is ON, the raw_transcript + dedup_mirror
+    // tables are preferred for byte-stable reconstruction. The current
+    // recall path (VectorStore search → format → inject) is unaffected —
+    // it provides fast semantic search over checkpoint summaries.
+    //
+    // If full transcript reconstruction is ever needed (replay, export,
+    // debug), call reconstructFromMirror(db, sessionId, fromSeq, toSeq)
+    // from src/mirror/dedup.ts instead of reading from the legacy JSON
+    // checkpoint. Falls back to legacy checkpoint if mirror is empty
+    // (pre-migration sessions).
+    //
+    // Invariant: raw_transcript + dedup_mirror are additive and never
+    // lose data. The legacy JSON checkpoint remains as a DR snapshot.
+    // ─────────────────────────────────────────────────────────────────────
     const limit = opts.limit ?? 3;
     const skip = opts.skipInjected ?? true;
     const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)