pi-mega-compact 0.7.3 → 0.7.5

package/dist/src/recall.js

@@ -41,7 +41,44 @@ export function formatRecallBlock(hits) {
  * it records injections via `markInjected` so the next call dedupes. The
  * `store` is passed by the extension (defaults to the engine's default store).
  */
+/**
+ * Recall and inline context from the checkpoint store.
+ *
+ * S27 contract — DB-Mirror demotion:
+ *
+ * When `MEGACOMPACT_DB_MIRROR` is ON, the `raw_transcript` table is the
+ * canonical, byte-stable source of truth for message reconstruction. The
+ * `dedup_mirror` provides space-efficient storage with ref_count tracking
+ * (see src/mirror/dedup.ts). The legacy JSON checkpoint is retained as a
+ * DR snapshot only (see src/store.ts checkpoint helpers).
+ *
+ * The recall function continues to work from the VectorStore (checkpoint
+ * summaries + embeddings) for fast semantic search — this path is unaffected
+ * by the mirror flag. If full transcript reconstruction is ever needed
+ * (replay, export, debug), prefer reading from `raw_transcript + dedup_mirror`
+ * via `listRawTranscriptRange()` + `dedupTranscript()` instead of the legacy
+ * JSON checkpoint. Falls back to legacy checkpoint if mirror is empty
+ * (pre-migration sessions).
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
 export function recallAndInline(opts, store) {
+    // ── S27 Recall Demotion ─────────────────────────────────────────────
+    //
+    // When MEGACOMPACT_DB_MIRROR is ON, the raw_transcript + dedup_mirror
+    // tables are preferred for byte-stable reconstruction. The current
+    // recall path (VectorStore search → format → inject) is unaffected —
+    // it provides fast semantic search over checkpoint summaries.
+    //
+    // If full transcript reconstruction is ever needed (replay, export,
+    // debug), call reconstructFromMirror(db, sessionId, fromSeq, toSeq)
+    // from src/mirror/dedup.ts instead of reading from the legacy JSON
+    // checkpoint. Falls back to legacy checkpoint if mirror is empty
+    // (pre-migration sessions).
+    //
+    // Invariant: raw_transcript + dedup_mirror are additive and never
+    // lose data. The legacy JSON checkpoint remains as a DR snapshot.
+    // ─────────────────────────────────────────────────────────────────────
     const limit = opts.limit ?? 3;
     const skip = opts.skipInjected ?? true;
     const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)

package/dist/src/store/sqlite.dbmaint.test.js

@@ -0,0 +1,158 @@
+/**
+ * Tests for S27 Task 10 DB maintenance primitives.
+ *
+ * Covers getDbStats, pruneOldRows, checkpointWal, vacuumDb, integrityCheck,
+ * reconcileDedupMirror, autoMaintain. All against a tmp SQLite store.
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { openStore, appendRawTranscript, writeCheckpointEpoch, upsertDedupMirror, updateRawTranscriptRef, getDbStats, pruneOldRows, checkpointWal, vacuumDb, integrityCheck, reconcileDedupMirror, autoMaintain, } from "./sqlite.js";
+function makeTmp() {
+    return mkdtempSync(join(tmpdir(), "dbmaint-test-"));
+}
+function makeRow(overrides = {}) {
+    return {
+        contentHash: `hash-${Math.random().toString(36).slice(2, 10)}`,
+        sessionId: "sess-1",
+        seq: 0,
+        role: "user",
+        contentBytes: "hello world",
+        toolName: null,
+        messageTimestamp: Date.now(),
+        checkpointEpoch: "epoch-1",
+        ...overrides,
+    };
+}
+function makeEpoch(overrides = {}) {
+    return {
+        epochId: `epoch-${Math.random().toString(36).slice(2, 10)}`,
+        sessionId: "sess-1",
+        startedSeq: 0,
+        committedSeq: 10,
+        checkpointId: "cp-1",
+        cutIndex: 10,
+        summaryMessageText: "test summary",
+        createdAt: Date.now(),
+        ...overrides,
+    };
+}
+describe("DB maintenance primitives (S27 Task 10)", () => {
+    let dir;
+    let db;
+    beforeEach(() => {
+        dir = makeTmp();
+        db = openStore(dir);
+    });
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+    describe("getDbStats", () => {
+        it("returns zero counts on an empty store", () => {
+            const s = getDbStats(dir);
+            assert.equal(s.tableCounts.raw_transcript, 0);
+            assert.equal(s.tableCounts.checkpoint_epochs, 0);
+            assert.equal(s.tableCounts.dedup_mirror, 0);
+            // main DB file exists (schema was written)
+            assert.ok(s.dbBytes > 0);
+            assert.ok(s.pageSize > 0);
+            assert.ok(s.pageCount > 0);
+        });
+        it("counts rows after inserts", () => {
+            appendRawTranscript(db, makeRow());
+            appendRawTranscript(db, makeRow());
+            const s = getDbStats(dir);
+            assert.equal(s.tableCounts.raw_transcript, 2);
+        });
+    });
+    describe("pruneOldRows", () => {
+        it("deletes nothing when all rows are recent", () => {
+            appendRawTranscript(db, makeRow({ messageTimestamp: Date.now() }));
+            const r = pruneOldRows(dir, 30);
+            assert.equal(r.affected, 0);
+        });
+        it("deletes raw_transcript rows older than the cutoff", () => {
+            const oldTs = Date.now() - 31 * 86_400_000;
+            appendRawTranscript(db, makeRow({ messageTimestamp: oldTs }));
+            appendRawTranscript(db, makeRow({ messageTimestamp: Date.now() }));
+            const r = pruneOldRows(dir, 30);
+            assert.equal(r.affected, 1);
+            assert.equal(getDbStats(dir).tableCounts.raw_transcript, 1);
+        });
+        it("deletes checkpoint_epochs older than the cutoff", () => {
+            const oldTs = Date.now() - 40 * 86_400_000;
+            writeCheckpointEpoch(db, makeEpoch({ createdAt: oldTs }));
+            writeCheckpointEpoch(db, makeEpoch({ createdAt: Date.now() }));
+            const r = pruneOldRows(dir, 30);
+            assert.equal(r.affected, 1);
+            assert.equal(getDbStats(dir).tableCounts.checkpoint_epochs, 1);
+        });
+    });
+    describe("integrityCheck", () => {
+        it("returns ['ok'] on a healthy DB", () => {
+            const lines = integrityCheck(dir);
+            assert.deepEqual(lines, ["ok"]);
+        });
+    });
+    describe("checkpointWal", () => {
+        it("runs without error and reports checkpointed frames", () => {
+            const r = checkpointWal(dir);
+            assert.ok(r.summary.includes("wal_checkpoint(TRUNCATE)"));
+        });
+    });
+    describe("vacuumDb", () => {
+        it("rebuilds the DB file", () => {
+            appendRawTranscript(db, makeRow());
+            const r = vacuumDb(dir);
+            assert.ok(r.summary.includes("VACUUM"));
+            // DB file still exists after vacuum
+            assert.ok(statSync(join(dir, "sqlite.db")).size > 0);
+        });
+    });
+    describe("reconcileDedupMirror", () => {
+        it("fixes ref_count drift and deletes orphans", () => {
+            // Insert a dedup_mirror row with ref_count = 5 (drift: actual refs = 1).
+            const hash = "hash-recon-1";
+            upsertDedupMirror(db, hash, "content-bytes-1", 0);
+            // Force a drift: bump ref_count without a matching raw_transcript row.
+            db.prepare("UPDATE dedup_mirror SET ref_count = 5 WHERE content_hash = ?").run(hash);
+            // Insert one raw_transcript row pointing at it. appendRawTranscript auto-
+            // assigns seq = MAX(seq)+1 = 1 for the first row in this session.
+            appendRawTranscript(db, makeRow({ contentHash: "rt-recon-1" }));
+            updateRawTranscriptRef(db, "sess-1", 1, hash);
+            const r = reconcileDedupMirror(dir);
+            // ref_count should now be 1 (one raw_transcript row points at it).
+            const dm = db
+                .prepare("SELECT ref_count FROM dedup_mirror WHERE content_hash = ?")
+                .get(hash);
+            assert.equal(dm?.ref_count, 1);
+            assert.equal(r.fixedRefCount, 1);
+        });
+        it("deletes orphan dedup_mirror rows (no raw_transcript refs)", () => {
+            // Insert a dedup_mirror row with ref_count > 0 but NO raw_transcript ref.
+            const hash = "hash-orphan-1";
+            upsertDedupMirror(db, hash, "orphan-content", 0);
+            db.prepare("UPDATE dedup_mirror SET ref_count = 3 WHERE content_hash = ?").run(hash);
+            const r = reconcileDedupMirror(dir);
+            const exists = db
+                .prepare("SELECT 1 FROM dedup_mirror WHERE content_hash = ?")
+                .get(hash);
+            assert.equal(exists, undefined);
+            assert.ok(r.orphansDeleted > 0);
+        });
+    });
+    describe("autoMaintain", () => {
+        it("runs best-effort and returns a summary string", () => {
+            appendRawTranscript(db, makeRow());
+            const result = autoMaintain(dir);
+            assert.ok(typeof result === "string");
+            assert.ok(result.startsWith("auto-maintain:"));
+        });
+        it("reports nothing to do on a fresh empty DB", () => {
+            const result = autoMaintain(dir);
+            assert.equal(result, "auto-maintain: nothing to do");
+        });
+    });
+});

package/dist/src/store/sqlite.dbmirror.test.js

@@ -0,0 +1,175 @@
+/**
+ * Tests for DB-mirror raw_transcript + checkpoint_epochs tables.
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { openStore, appendRawTranscript, listRawTranscriptRange, writeCheckpointEpoch, readCheckpointEpoch, getActiveEpochForSession, listCheckpointEpochs, countRawTranscript, } from "./sqlite.js";
+function makeTmp() {
+    return mkdtempSync(join(tmpdir(), "dbmirror-test-"));
+}
+function makeRow(overrides = {}) {
+    return {
+        contentHash: `hash-${Math.random().toString(36).slice(2, 10)}`,
+        sessionId: "sess-1",
+        seq: 0,
+        role: "user",
+        contentBytes: "hello world",
+        toolName: null,
+        messageTimestamp: null,
+        checkpointEpoch: "epoch-1",
+        ...overrides,
+    };
+}
+function makeEpoch(overrides = {}) {
+    return {
+        epochId: `epoch-${Math.random().toString(36).slice(2, 10)}`,
+        sessionId: "sess-1",
+        startedSeq: 0,
+        committedSeq: 10,
+        checkpointId: "cp-1",
+        cutIndex: 10,
+        summaryMessageText: "test summary",
+        createdAt: Date.now(),
+        ...overrides,
+    };
+}
+describe("DB mirror", () => {
+    let dir;
+    let db;
+    beforeEach(() => {
+        dir = makeTmp();
+        db = openStore(dir);
+    });
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+    describe("appendRawTranscript", () => {
+        it("inserts a row and auto-assigns seq", () => {
+            const row = makeRow();
+            appendRawTranscript(db, row);
+            const count = countRawTranscript(db);
+            assert.equal(count, 1);
+            const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+            assert.equal(rows.length, 1);
+            assert.equal(rows[0].contentHash, row.contentHash);
+            assert.equal(rows[0].role, "user");
+        });
+        it("is idempotent on content_hash PK — no duplicate rows", () => {
+            const row = makeRow({ contentHash: "fixed-hash" });
+            appendRawTranscript(db, row);
+            appendRawTranscript(db, row); // duplicate
+            const count = countRawTranscript(db);
+            assert.equal(count, 1);
+        });
+        it("increments seq across different messages in same session", () => {
+            appendRawTranscript(db, makeRow({ contentHash: "h1", sessionId: "s1" }));
+            appendRawTranscript(db, makeRow({ contentHash: "h2", sessionId: "s1" }));
+            appendRawTranscript(db, makeRow({ contentHash: "h3", sessionId: "s1" }));
+            const count = countRawTranscript(db);
+            assert.equal(count, 3);
+        });
+        it("stores tool_name and message_timestamp when provided", () => {
+            appendRawTranscript(db, makeRow({
+                contentHash: "tool-row",
+                role: "toolResult",
+                toolName: "bash",
+                messageTimestamp: 1234567890,
+            }));
+            const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+            assert.equal(rows[0].toolName, "bash");
+            assert.equal(rows[0].messageTimestamp, 1234567890);
+        });
+        it("stores checkpoint_epoch", () => {
+            appendRawTranscript(db, makeRow({ checkpointEpoch: "ep-42" }));
+            const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+            assert.equal(rows[0].checkpointEpoch, "ep-42");
+        });
+    });
+    describe("writeCheckpointEpoch + readCheckpointEpoch", () => {
+        it("round-trips a checkpoint epoch", () => {
+            const epoch = makeEpoch();
+            writeCheckpointEpoch(db, epoch);
+            const got = readCheckpointEpoch(db, epoch.epochId);
+            assert.ok(got);
+            assert.equal(got.epochId, epoch.epochId);
+            assert.equal(got.sessionId, epoch.sessionId);
+            assert.equal(got.committedSeq, epoch.committedSeq);
+            assert.equal(got.checkpointId, epoch.checkpointId);
+            assert.equal(got.cutIndex, epoch.cutIndex);
+            assert.equal(got.summaryMessageText, epoch.summaryMessageText);
+        });
+        it("is idempotent on epochId PK", () => {
+            const epoch = makeEpoch({ epochId: "ep-dup" });
+            writeCheckpointEpoch(db, epoch);
+            writeCheckpointEpoch(db, epoch); // duplicate
+            // Should not throw, still one row
+            const got = readCheckpointEpoch(db, "ep-dup");
+            assert.ok(got);
+        });
+        it("returns null for unknown epochId", () => {
+            const got = readCheckpointEpoch(db, "nonexistent");
+            assert.equal(got, null);
+        });
+    });
+    describe("getActiveEpochForSession", () => {
+        it("returns the most recent epoch for a session", () => {
+            writeCheckpointEpoch(db, makeEpoch({
+                epochId: "ep-old",
+                sessionId: "s1",
+                createdAt: 1000,
+            }));
+            writeCheckpointEpoch(db, makeEpoch({
+                epochId: "ep-new",
+                sessionId: "s1",
+                createdAt: 2000,
+            }));
+            const got = getActiveEpochForSession(db, "s1");
+            assert.ok(got);
+            assert.equal(got.epochId, "ep-new");
+        });
+        it("returns null if no epochs exist for session", () => {
+            const got = getActiveEpochForSession(db, "no-such-session");
+            assert.equal(got, null);
+        });
+    });
+    describe("checkpointEpochs()", () => {
+        it("returns all rows", () => {
+            writeCheckpointEpoch(db, makeEpoch({ epochId: "ep-a" }));
+            writeCheckpointEpoch(db, makeEpoch({ epochId: "ep-b" }));
+            const rows = listCheckpointEpochs(db);
+            assert.equal(rows.length, 2);
+        });
+    });
+    describe("integration: raw_transcript + checkpoint_epochs", () => {
+        it("full flow: append messages, write epoch, query both", () => {
+            // Append 5 messages
+            for (let i = 1; i <= 5; i++) {
+                appendRawTranscript(db, makeRow({
+                    contentHash: `msg-${i}`,
+                    sessionId: "s1",
+                    checkpointEpoch: "ep-1",
+                }));
+            }
+            // Write epoch
+            writeCheckpointEpoch(db, makeEpoch({
+                epochId: "ep-1",
+                sessionId: "s1",
+                committedSeq: 5,
+                cutIndex: 5,
+            }));
+            // Verify transcript rows
+            const transcripts = listRawTranscriptRange(db, "s1", 0, 999999);
+            assert.equal(transcripts.length, 5);
+            for (const t of transcripts) {
+                assert.equal(t.checkpointEpoch, "ep-1");
+            }
+            // Verify epoch
+            const epoch = readCheckpointEpoch(db, "ep-1");
+            assert.ok(epoch);
+            assert.equal(epoch.committedSeq, 5);
+        });
+    });
+});