pi-mega-compact 0.7.2 → 0.7.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",

package/src/mirror/dedup.ts

@@ -0,0 +1,57 @@
+/**
+ * dedup.ts — S27 Task 6: Fork snapshot → compress/dedupe pipeline.
+ *
+ * After the served window is handed to pi, asynchronously:
+ * 1. Read raw_transcript rows [0..cut_index] for the epoch
+ * 2. For each row, compute content_hash (reuse digest from dedup/)
+ * 3. INSERT OR IGNORE INTO dedup_mirror (stores bytes once per unique hash)
+ * 4. Update raw_transcript.content_ref to point to dedup_mirror
+ * 5. Increment dedup_mirror.ref_count for existing hashes
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
+
+import type { DatabaseSync } from "node:sqlite";
+import {
+  upsertDedupMirror,
+  updateRawTranscriptRef,
+  listRawTranscriptRange,
+  getDedupRatio,
+} from "../store/sqlite.js";
+import { computeContentDigest } from "../dedup/digest.js";
+
+/**
+ * Deduplicate raw transcript rows for a session range.
+ * Fire-and-forget: errors are logged, not thrown.
+ *
+ * @returns Number of rows deduplicated, or -1 on error.
+ */
+export function dedupTranscript(
+  db: DatabaseSync,
+  sessionId: string,
+  fromSeq: number,
+  toSeq: number,
+): number {
+  try {
+    const rows = listRawTranscriptRange(db, sessionId, fromSeq, toSeq);
+    let deduped = 0;
+    for (const row of rows) {
+      const contentHash = computeContentDigest(row.contentBytes).contentHash;
+      const isNew = upsertDedupMirror(db, contentHash, row.contentBytes, row.seq);
+      updateRawTranscriptRef(db, sessionId, row.seq, contentHash);
+      if (!isNew) {
+        deduped++;
+      }
+    }
+    return deduped;
+  } catch (err) {
+    // Fire-and-forget: log but don't throw
+    console.error("[mega-compact] dedupTranscript failed:", err);
+    return -1;
+  }
+}
+
+/**
+ * Get dedup ratio for a session.
+ */
+export { getDedupRatio };

package/src/mirror/epoch.ts

@@ -0,0 +1,37 @@
+/**
+ * epoch.ts — deterministic epoch-id derivation for the S27 DB-mirror.
+ *
+ * The epoch id MUST be a pure function of the checkpoint it decorates so that
+ * replaying / refreshing the same compaction yields the SAME epoch id (idempotent
+ * appends + ON CONFLICT refresh). No Date.now / uuid / crypto — this is the
+ * only source of randomness-free epoch naming in the mirror stack.
+ *
+ * - epochIdFor(cp)  → "epoch:" + cp   (human-traceable back to its checkpoint)
+ * - epochNonceFor(cp) → FNV-1a 32-bit hash (cheap, well-distributed nonce)
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
+
+/**
+ * FNV-1a 32-bit nonce for a checkpoint id. Deterministic and RNG-free:
+ * h = 0x811c9dc5; for each char: h ^= codePoint; h = Math.imul(h, 0x01000193);
+ * return h >>> 0 (unsigned).
+ */
+export function epochNonceFor(checkpointId: string): number {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < checkpointId.length; i++) {
+    const cp = checkpointId.codePointAt(i);
+    if (cp === undefined) continue;
+    h ^= cp;
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/**
+ * Deterministic epoch id: "epoch:" + checkpointId. Trivially traceable back to
+ * the source checkpoint, and stable under replay (refresh-safe upserts).
+ */
+export function epochIdFor(checkpointId: string): string {
+  return "epoch:" + checkpointId;
+}

package/src/mirror/mirror.test.ts

@@ -0,0 +1,240 @@
+/**
+ * mirror.test.ts — S27 DB-mirror integration tests.
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ *
+ * NOTE: raw_transcript has PRIMARY KEY (content_hash, session_id), so
+ * duplicate content in the same session is silently dropped by INSERT OR IGNORE.
+ * Tests are designed around this constraint.
+ */
+
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import type { DatabaseSync } from "node:sqlite";
+import { openStore, closeStore } from "../../src/store/sqlite.js";
+import {
+  writeCheckpointEpoch,
+  listCheckpointEpochs,
+  appendRawTranscript,
+  listRawTranscriptRange,
+  upsertDedupMirror,
+  getDedupRatio,
+  getDedupMirrorStats,
+  countRawTranscript,
+} from "../../src/store/sqlite.js";
+import { epochIdFor } from "../../src/mirror/epoch.js";
+import { dedupTranscript } from "../../src/mirror/dedup.js";
+import { computeContentDigest } from "../../src/dedup/digest.js";
+
+function tmp(): string {
+  return mkdtempSync(join(tmpdir(), "mirror-test-"));
+}
+
+/**
+ * Build a valid RawTranscriptRow using the canonical content hash from
+ * computeContentDigest (matches what dedupTranscript uses).
+ */
+function mkRow(
+  sessionId: string,
+  seq: number, // ignored by appendRawTranscript (auto-assigned)
+  role: "user" | "assistant",
+  content: string,
+) {
+  const { contentHash } = computeContentDigest(content);
+  return {
+    contentHash,
+    sessionId,
+    seq,
+    role,
+    contentBytes: content,
+    toolName: null as string | null,
+    messageTimestamp: Date.now() as number | null,
+    checkpointEpoch: "",
+  };
+}
+
+describe("S27 DB-mirror", () => {
+  it("epochIdFor is deterministic", () => {
+    assert.equal(epochIdFor("cp-abc-123"), epochIdFor("cp-abc-123"));
+    assert.notEqual(epochIdFor("cp-abc-123"), epochIdFor("cp-abc-456"));
+    const id = epochIdFor("cp-abc-123");
+    assert.ok(id.startsWith("epoch:"));
+    assert.ok(id.length > 6);
+  });
+
+  it("writeCheckpointEpoch + listCheckpointEpochs round-trips", () => {
+    const dir = tmp();
+    const db: DatabaseSync = openStore(dir);
+
+    writeCheckpointEpoch(db, {
+      epochId: "epoch-test-001",
+      sessionId: "sess-abc",
+      startedSeq: 0,
+      committedSeq: 100,
+      checkpointId: "cp-test-001",
+      cutIndex: 100,
+      summaryMessageText: "Test summary",
+      createdAt: Date.now(),
+    });
+
+    const rows = listCheckpointEpochs(db);
+    assert.ok(rows.length >= 1);
+    assert.equal(rows[0].epochId, "epoch-test-001");
+    assert.equal(rows[0].sessionId, "sess-abc");
+    assert.equal(rows[0].checkpointId, "cp-test-001");
+
+    closeStore(dir);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("appendRawTranscript + listRawTranscriptRange round-trips (unique content)", () => {
+    const dir = tmp();
+    const db: DatabaseSync = openStore(dir);
+
+    // Use unique content for each row to avoid PK collision
+    appendRawTranscript(db, mkRow("sess-abc", 0, "user", "first message"));
+    appendRawTranscript(db, mkRow("sess-abc", 1, "assistant", "second message"));
+    appendRawTranscript(db, mkRow("sess-abc", 2, "user", "third message"));
+
+    // seq is auto-assigned: 1, 2, 3
+    const rows = listRawTranscriptRange(db, "sess-abc", 0, 10);
+    assert.equal(rows.length, 3);
+    assert.equal(rows[0].contentBytes, "first message");
+    assert.equal(rows[0].seq, 1);
+    assert.equal(rows[1].contentBytes, "second message");
+    assert.equal(rows[1].seq, 2);
+    assert.equal(rows[2].contentBytes, "third message");
+    assert.equal(rows[2].seq, 3);
+
+    // Range filter: [2..3]
+    const rows2 = listRawTranscriptRange(db, "sess-abc", 2, 3);
+    assert.equal(rows2.length, 2);
+    assert.equal(rows2[0].contentBytes, "second message");
+    assert.equal(rows2[1].contentBytes, "third message");
+
+    closeStore(dir);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("upsertDedupMirror increments ref_count for duplicate content", () => {
+    const dir = tmp();
+    const db: DatabaseSync = openStore(dir);
+
+    const isNew1 = upsertDedupMirror(db, "hash-aaa", "Hello", 0);
+    assert.equal(isNew1, true);
+
+    const isNew2 = upsertDedupMirror(db, "hash-aaa", "Hello", 1);
+    assert.equal(isNew2, false);
+
+    const stats = getDedupMirrorStats(db);
+    assert.equal(stats.rowCount, 1);
+    assert.equal(stats.avgRefCount, 2);
+
+    closeStore(dir);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("dedupTranscript deduplicates cross-session content via dedup_mirror", () => {
+    const dir = tmp();
+    const db: DatabaseSync = openStore(dir);
+
+    // Insert same content in TWO different sessions (raw_transcript PK allows this)
+    appendRawTranscript(db, mkRow("sess-a", 0, "user", "shared hello"));
+    appendRawTranscript(db, mkRow("sess-a", 1, "assistant", "shared world"));
+    appendRawTranscript(db, mkRow("sess-a", 2, "user", "unique A"));
+    appendRawTranscript(db, mkRow("sess-b", 0, "user", "shared hello"));
+    appendRawTranscript(db, mkRow("sess-b", 1, "assistant", "shared world"));
+    appendRawTranscript(db, mkRow("sess-b", 2, "user", "unique B"));
+
+    // Dedup session A: 3 rows, all new → deduped=0
+    const dedupedA = dedupTranscript(db, "sess-a", 0, 10);
+    assert.equal(dedupedA, 0);
+
+    // Dedup session B: 3 rows, but 2 already in dedup_mirror → deduped=2
+    const dedupedB = dedupTranscript(db, "sess-b", 0, 10);
+    assert.equal(dedupedB, 2);
+
+    // dedup_mirror has 4 unique hashes: shared-hello, shared-world, unique-A, unique-B
+    const stats = getDedupMirrorStats(db);
+    assert.equal(stats.rowCount, 4);
+    assert.ok(stats.avgRefCount > 1);
+
+    closeStore(dir);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("getDedupRatio reflects dedup savings", () => {
+    const dir = tmp();
+    const db: DatabaseSync = openStore(dir);
+
+    // Two sessions with identical content → cross-session dedup
+    for (let i = 0; i < 3; i++) {
+      appendRawTranscript(db, mkRow("sess-x", i, "user", "same content"));
+      appendRawTranscript(db, mkRow("sess-y", i, "user", "same content"));
+    }
+    // Each session has 1 row (PK dedup within session), so 1 row each
+    // sess-x: 1 row, sess-y: 1 row
+
+    dedupTranscript(db, "sess-x", 0, 10);
+    dedupTranscript(db, "sess-y", 0, 10);
+
+    // For sess-x: totalBytes = LENGTH("same content") = 12, uniqueBytes = 12 → ratio 1.0
+    const { totalBytes, uniqueBytes, ratio } = getDedupRatio(db, "sess-x");
+    assert.ok(totalBytes > 0);
+    assert.ok(uniqueBytes > 0);
+    assert.ok(ratio >= 1.0);
+
+    closeStore(dir);
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("full pipeline: append + dedup + epoch", () => {
+    const dir = tmp();
+    const db: DatabaseSync = openStore(dir);
+
+    // Insert 5 unique rows
+    const contents = ["alpha", "bravo", "charlie", "delta", "echo"];
+    for (let i = 0; i < contents.length; i++) {
+      appendRawTranscript(
+        db,
+        mkRow("sess-pipe", i, i % 2 === 0 ? "user" : "assistant", contents[i]),
+      );
+    }
+
+    const total = countRawTranscript(db);
+    assert.ok(total >= 5);
+
+    // Dedup: all 5 unique → deduped = 0
+    const deduped = dedupTranscript(db, "sess-pipe", 0, 100);
+    assert.equal(deduped, 0);
+
+    // Mirror should have 5 unique hashes
+    const stats = getDedupMirrorStats(db);
+    assert.equal(stats.rowCount, 5);
+
+    // Write checkpoint epoch
+    writeCheckpointEpoch(db, {
+      epochId: "epoch-integration",
+      sessionId: "sess-pipe",
+      startedSeq: 0,
+      committedSeq: 100,
+      checkpointId: "cp-integration",
+      cutIndex: 5,
+      summaryMessageText: "Integration test summary",
+      createdAt: Date.now(),
+    });
+
+    const epochs = listCheckpointEpochs(db);
+    assert.ok(epochs.length >= 1);
+    assert.equal(epochs[0].epochId, "epoch-integration");
+
+    const rows = listRawTranscriptRange(db, "sess-pipe", 0, 100);
+    assert.equal(rows.length, 5);
+
+    closeStore(dir);
+    rmSync(dir, { recursive: true, force: true });
+  });
+});

package/src/recall.ts

@@ -84,10 +84,48 @@ export function formatRecallBlock(hits: SearchHit[]): string {
  * it records injections via `markInjected` so the next call dedupes. The
  * `store` is passed by the extension (defaults to the engine's default store).
  */
+/**
+ * Recall and inline context from the checkpoint store.
+ *
+ * S27 contract — DB-Mirror demotion:
+ *
+ * When `MEGACOMPACT_DB_MIRROR` is ON, the `raw_transcript` table is the
+ * canonical, byte-stable source of truth for message reconstruction. The
+ * `dedup_mirror` provides space-efficient storage with ref_count tracking
+ * (see src/mirror/dedup.ts). The legacy JSON checkpoint is retained as a
+ * DR snapshot only (see src/store.ts checkpoint helpers).
+ *
+ * The recall function continues to work from the VectorStore (checkpoint
+ * summaries + embeddings) for fast semantic search — this path is unaffected
+ * by the mirror flag. If full transcript reconstruction is ever needed
+ * (replay, export, debug), prefer reading from `raw_transcript + dedup_mirror`
+ * via `listRawTranscriptRange()` + `dedupTranscript()` instead of the legacy
+ * JSON checkpoint. Falls back to legacy checkpoint if mirror is empty
+ * (pre-migration sessions).
+ *
+ * Pi-agnostic: no pi runtime imports (src/ invariant).
+ */
 export function recallAndInline(
   opts: RecallInjectOptions,
   store: Pick<VectorStore, "search" | "wasInjected" | "markInjected">,
 ): RecallInjectResult {
+  // ── S27 Recall Demotion ─────────────────────────────────────────────
+  //
+  // When MEGACOMPACT_DB_MIRROR is ON, the raw_transcript + dedup_mirror
+  // tables are preferred for byte-stable reconstruction. The current
+  // recall path (VectorStore search → format → inject) is unaffected —
+  // it provides fast semantic search over checkpoint summaries.
+  //
+  // If full transcript reconstruction is ever needed (replay, export,
+  // debug), call reconstructFromMirror(db, sessionId, fromSeq, toSeq)
+  // from src/mirror/dedup.ts instead of reading from the legacy JSON
+  // checkpoint. Falls back to legacy checkpoint if mirror is empty
+  // (pre-migration sessions).
+  //
+  // Invariant: raw_transcript + dedup_mirror are additive and never
+  // lose data. The legacy JSON checkpoint remains as a DR snapshot.
+  // ─────────────────────────────────────────────────────────────────────
+
   const limit = opts.limit ?? 3;
   const skip = opts.skipInjected ?? true;
   const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)

package/src/store/sqlite.dbmirror.test.ts

@@ -0,0 +1,219 @@
+/**
+ * Tests for DB-mirror raw_transcript + checkpoint_epochs tables.
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import {
+  openStore,
+  appendRawTranscript,
+  listRawTranscriptRange,
+  writeCheckpointEpoch,
+  readCheckpointEpoch,
+  getActiveEpochForSession,
+  listCheckpointEpochs,
+  countRawTranscript,
+  type RawTranscriptRow,
+  type CheckpointEpoch,
+} from "./sqlite.js";
+
+function makeTmp(): string {
+  return mkdtempSync(join(tmpdir(), "dbmirror-test-"));
+}
+
+function makeRow(overrides: Partial<RawTranscriptRow> = {}): RawTranscriptRow {
+  return {
+    contentHash: `hash-${Math.random().toString(36).slice(2, 10)}`,
+    sessionId: "sess-1",
+    seq: 0,
+    role: "user",
+    contentBytes: "hello world",
+    toolName: null,
+    messageTimestamp: null,
+    checkpointEpoch: "epoch-1",
+    ...overrides,
+  };
+}
+
+function makeEpoch(overrides: Partial<CheckpointEpoch> = {}): CheckpointEpoch {
+  return {
+    epochId: `epoch-${Math.random().toString(36).slice(2, 10)}`,
+    sessionId: "sess-1",
+    startedSeq: 0,
+    committedSeq: 10,
+    checkpointId: "cp-1",
+    cutIndex: 10,
+    summaryMessageText: "test summary",
+    createdAt: Date.now(),
+    ...overrides,
+  };
+}
+
+describe("DB mirror", () => {
+  let dir: string;
+  let db: ReturnType<typeof openStore>;
+
+  beforeEach(() => {
+    dir = makeTmp();
+    db = openStore(dir);
+  });
+
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  describe("appendRawTranscript", () => {
+    it("inserts a row and auto-assigns seq", () => {
+      const row = makeRow();
+      appendRawTranscript(db, row);
+      const count = countRawTranscript(db);
+      assert.equal(count, 1);
+      const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+      assert.equal(rows.length, 1);
+      assert.equal(rows[0].contentHash, row.contentHash);
+      assert.equal(rows[0].role, "user");
+    });
+
+    it("is idempotent on content_hash PK — no duplicate rows", () => {
+      const row = makeRow({ contentHash: "fixed-hash" });
+      appendRawTranscript(db, row);
+      appendRawTranscript(db, row); // duplicate
+      const count = countRawTranscript(db);
+      assert.equal(count, 1);
+    });
+
+    it("increments seq across different messages in same session", () => {
+      appendRawTranscript(db, makeRow({ contentHash: "h1", sessionId: "s1" }));
+      appendRawTranscript(db, makeRow({ contentHash: "h2", sessionId: "s1" }));
+      appendRawTranscript(db, makeRow({ contentHash: "h3", sessionId: "s1" }));
+      const count = countRawTranscript(db);
+      assert.equal(count, 3);
+    });
+
+    it("stores tool_name and message_timestamp when provided", () => {
+      appendRawTranscript(
+        db,
+        makeRow({
+          contentHash: "tool-row",
+          role: "toolResult",
+          toolName: "bash",
+          messageTimestamp: 1234567890,
+        }),
+      );
+      const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+      assert.equal(rows[0].toolName, "bash");
+      assert.equal(rows[0].messageTimestamp, 1234567890);
+    });
+
+    it("stores checkpoint_epoch", () => {
+      appendRawTranscript(db, makeRow({ checkpointEpoch: "ep-42" }));
+      const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+      assert.equal(rows[0].checkpointEpoch, "ep-42");
+    });
+  });
+
+  describe("writeCheckpointEpoch + readCheckpointEpoch", () => {
+    it("round-trips a checkpoint epoch", () => {
+      const epoch = makeEpoch();
+      writeCheckpointEpoch(db, epoch);
+      const got = readCheckpointEpoch(db, epoch.epochId);
+      assert.ok(got);
+      assert.equal(got.epochId, epoch.epochId);
+      assert.equal(got.sessionId, epoch.sessionId);
+      assert.equal(got.committedSeq, epoch.committedSeq);
+      assert.equal(got.checkpointId, epoch.checkpointId);
+      assert.equal(got.cutIndex, epoch.cutIndex);
+      assert.equal(got.summaryMessageText, epoch.summaryMessageText);
+    });
+
+    it("is idempotent on epochId PK", () => {
+      const epoch = makeEpoch({ epochId: "ep-dup" });
+      writeCheckpointEpoch(db, epoch);
+      writeCheckpointEpoch(db, epoch); // duplicate
+      // Should not throw, still one row
+      const got = readCheckpointEpoch(db, "ep-dup");
+      assert.ok(got);
+    });
+
+    it("returns null for unknown epochId", () => {
+      const got = readCheckpointEpoch(db, "nonexistent");
+      assert.equal(got, null);
+    });
+  });
+
+  describe("getActiveEpochForSession", () => {
+    it("returns the most recent epoch for a session", () => {
+      writeCheckpointEpoch(
+        db,
+        makeEpoch({
+          epochId: "ep-old",
+          sessionId: "s1",
+          createdAt: 1000,
+        }),
+      );
+      writeCheckpointEpoch(
+        db,
+        makeEpoch({
+          epochId: "ep-new",
+          sessionId: "s1",
+          createdAt: 2000,
+        }),
+      );
+      const got = getActiveEpochForSession(db, "s1");
+      assert.ok(got);
+      assert.equal(got.epochId, "ep-new");
+    });
+
+    it("returns null if no epochs exist for session", () => {
+      const got = getActiveEpochForSession(db, "no-such-session");
+      assert.equal(got, null);
+    });
+  });
+
+  describe("checkpointEpochs()", () => {
+    it("returns all rows", () => {
+      writeCheckpointEpoch(db, makeEpoch({ epochId: "ep-a" }));
+      writeCheckpointEpoch(db, makeEpoch({ epochId: "ep-b" }));
+      const rows = listCheckpointEpochs(db);
+      assert.equal(rows.length, 2);
+    });
+  });
+
+  describe("integration: raw_transcript + checkpoint_epochs", () => {
+    it("full flow: append messages, write epoch, query both", () => {
+      // Append 5 messages
+      for (let i = 1; i <= 5; i++) {
+        appendRawTranscript(
+          db,
+          makeRow({
+            contentHash: `msg-${i}`,
+            sessionId: "s1",
+            checkpointEpoch: "ep-1",
+          }),
+        );
+      }
+      // Write epoch
+      writeCheckpointEpoch(
+        db,
+        makeEpoch({
+          epochId: "ep-1",
+          sessionId: "s1",
+          committedSeq: 5,
+          cutIndex: 5,
+        }),
+      );
+      // Verify transcript rows
+      const transcripts = listRawTranscriptRange(db, "s1", 0, 999999);
+      assert.equal(transcripts.length, 5);
+      for (const t of transcripts) {
+        assert.equal(t.checkpointEpoch, "ep-1");
+      }
+      // Verify epoch
+      const epoch = readCheckpointEpoch(db, "ep-1");
+      assert.ok(epoch);
+      assert.equal(epoch.committedSeq, 5);
+    });
+  });
+});