pi-mega-compact 0.8.24 → 0.8.26

package/src/store/sqlite/dedup-mirror.ts

@@ -21,6 +21,9 @@ export interface DedupMirrorRowDB {
 /**
  * Upsert a row into dedup_mirror. If the hash already exists, increment ref_count.
  * Returns true if this was a NEW unique content (first insert), false if it was a duplicate.
+ *
+ * F3 fix: uses INSERT ... ON CONFLICT DO UPDATE (single atomic statement) instead of
+ * a check-then-act race-prone SELECT + UPDATE/INSERT sequence.
  */
 export function upsertDedupMirror(
   db: DatabaseSync,
@@ -29,29 +32,35 @@ export function upsertDedupMirror(
   seq: number,
 ): boolean {
   const now = Date.now();
-  const existing = db
-    .prepare(`SELECT content_hash FROM dedup_mirror WHERE content_hash = @hash`)
-    .get({ "@hash": contentHash }) as { content_hash: string } | undefined;
-  if (existing) {
-    db.prepare(`UPDATE dedup_mirror SET ref_count = ref_count + 1 WHERE content_hash = @hash`).run({
-      "@hash": contentHash,
-    });
-    return false;
-  }
-  db.prepare(
+  // Atomic upsert: on conflict, increment ref_count in-place (no check-then-act
+  // race). RETURNING ref_count distinguishes the two paths in one statement:
+  // inserted rows report ref_count=1, conflict-updated rows report ref_count>1.
+  const row = db.prepare(
     `INSERT INTO dedup_mirror (content_hash, content_bytes, ref_count, first_seen_seq, created_at)
-     VALUES (@hash, @bytes, 1, @seq, @now)`,
-  ).run({
+     VALUES (@hash, @bytes, 1, @seq, @now)
+     ON CONFLICT(content_hash) DO UPDATE SET
+       ref_count = ref_count + 1,
+       content_bytes = excluded.content_bytes
+     RETURNING ref_count`,
+  ).get({
     "@hash": contentHash,
     "@bytes": contentBytes,
     "@seq": seq,
     "@now": now,
-  });
-  return true;
+  }) as { ref_count: number } | undefined;
+  return (row?.ref_count ?? 1) === 1;
 }
 
 /**
  * Get dedup ratio for a session: total bytes vs unique bytes.
+ *
+ * F2 fix: both total and unique bytes are now scoped to the session, via a JOIN
+ * of raw_transcript.content_ref → dedup_mirror. The ratio is meaningful: how much
+ * smaller the session's storage footprint is compared to naive inline storage.
+ *
+ * NOTE: for sessions with NO dedup pipeline runs yet (all content_ref NULL),
+ * uniqueBytes falls back to the raw_transcript bytes (ratio=1), which is correct
+ * since nothing has been deduplicated yet.
  */
 export function getDedupRatio(
   db: DatabaseSync,
@@ -64,12 +73,20 @@ export function getDedupRatio(
        WHERE session_id = @session_id`,
     )
     .get({ "@session_id": sessionId }) as { total: number };
+  // F2 fix: session-scoped unique bytes via JOIN on content_ref.
+  // A row contributes its dedup_mirror bytes exactly once even when content_ref
+  // is NULL (fallback: use the raw_transcript bytes for that row, which is
+  // accurate when dedup hasn't run yet for the session).
   const uniqueRow = db
     .prepare(
-      `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS unique_bytes
-       FROM dedup_mirror`,
+      `SELECT COALESCE(SUM(LENGTH(
+        COALESCE(dm.content_bytes, rt.content_bytes)
+      )), 0) AS unique_bytes
+       FROM raw_transcript rt
+       LEFT JOIN dedup_mirror dm ON rt.content_ref = dm.content_hash
+       WHERE rt.session_id = @session_id`,
     )
-    .get() as { unique_bytes: number };
+    .get({ "@session_id": sessionId }) as { unique_bytes: number };
   const totalBytes = totalRow.total;
   const uniqueBytes = uniqueRow.unique_bytes;
   const ratio = uniqueBytes > 0 ? totalBytes / uniqueBytes : 1;
@@ -111,4 +128,4 @@ export function updateRawTranscriptRef(
     "@sid": sessionId,
     "@seq": seq,
   });
-}
+}

package/src/store/sqlite/maintenance.ts

@@ -36,7 +36,7 @@ const DB_TABLE_NAMES = [
   "checkpoint_epochs",
   "dedup_mirror",
   "memories",
-  "dedup_stats",
+  "meta",
   "daily_log",
 ] as const;
 
@@ -136,7 +136,7 @@ export function pruneOldRows(stateDir: string = getStateDir(), daysOld = 30): Ma
   // dedup_mirror: cascade-delete orphan rows whose ref_count has dropped to 0
   // after the raw_transcript deletes. Safe even if FK is off (raw_transcript has
   // no FK to dedup_mirror; ref_count is maintained by the dedup pipeline).
-  const delDedup = db.prepare(`DELETE FROM dedup_mirror WHERE ref_count <= 0`).run() as {
+  const delDedup = db.prepare(`DELETE FROM dedup_mirror WHERE content_hash NOT IN (SELECT DISTINCT content_ref FROM raw_transcript WHERE content_ref IS NOT NULL)`).run() as {
     changes?: number;
   } | undefined;
   const dedupDeleted = delDedup?.changes ?? 0;

package/src/store/sqlite/mechanical-fix.test.ts

@@ -0,0 +1,162 @@
+/**
+ * mechanical-fix.test.ts — focused unit tests for the SQLite-side mechanical-fix
+ * batch: safeJson (utils.ts), storeStats numeric MAX + dedup_status filter
+ * (stats.ts), addTokensSaved CAST AS REAL (meta.ts), evictMemoryLru named
+ * @param bindings (memories.ts). Pi-agnostic; uses isolated state dirs (G7).
+ */
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import { safeJson, closeStore } from "./utils.js";
+import { storeStats } from "./stats.js";
+import { upsertCheckpoint, setDedupStatus } from "./checkpoints.js";
+import { addTokensSaved, getTokensSaved } from "./meta.js";
+import { addMemory, listMemories } from "./memories.js";
+import type { StoredCheckpoint } from "../../store.js";
+
+function makeCp(id: string, sessionId: string): StoredCheckpoint {
+  return {
+    checkpointId: id,
+    sessionId,
+    summary: `summary-${id}`,
+    keyDecisions: [],
+    nextSteps: [],
+    filesModified: [],
+    tokenEstimate: 100,
+    regionHash: `hash-${id}`,
+    embedding: [],
+    timestamp: Date.now(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// safeJson (utils.ts)
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: safeJson (utils.ts)", () => {
+  it("returns fallback on null, undefined, empty, and corrupt JSON", () => {
+    assert.deepEqual(safeJson<string[]>(null, []), []);
+    assert.deepEqual(safeJson<string[]>(undefined, []), []);
+    assert.deepEqual(safeJson<string[]>("", []), []);
+    assert.deepEqual(safeJson<string[]>("not json", ["fb"]), ["fb"]);
+    assert.deepEqual(safeJson<string[]>('["a","b"]', []), ["a", "b"]);
+    assert.equal(safeJson<number>(null, 42), 42);
+    assert.deepEqual(safeJson<Record<string, number>>('{"k":1}', {}), { k: 1 });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// storeStats (stats.ts) — numeric MAX(id) + dedup_status filter
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: storeStats (stats.ts)", () => {
+  let dir: string;
+  before(() => {
+    dir = mkdtempSync(join(tmpdir(), "mc-mech-stats-"));
+    process.env.MEGACOMPACT_STATE_DIR = dir;
+  });
+  after(() => {
+    closeStore(dir);
+    delete process.env.MEGACOMPACT_STATE_DIR;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("returns numerically-max checkpoint id (not lexicographic) for 100+ checkpoints", () => {
+    const sid = "sess_stats";
+    // Insert chkpt_001..chkpt_100 (100 checkpoints) + chkpt_999 + chkpt_1000.
+    // Numeric max = 1000 → "chkpt_1000"; lexicographic max would be "chkpt_999"
+    // (because '9' > '1' at position 6 when comparing 8 vs 9-char strings).
+    for (let i = 1; i <= 100; i++) {
+      upsertCheckpoint(makeCp(`chkpt_${String(i).padStart(3, "0")}`, sid), dir);
+    }
+    upsertCheckpoint(makeCp("chkpt_999", sid), dir);
+    upsertCheckpoint(makeCp("chkpt_1000", sid), dir);
+
+    const s = storeStats(sid, dir);
+    assert.equal(s.checkpointCount, 102);
+    assert.equal(s.lastCheckpointId, "chkpt_1000");
+    assert.equal(s.lastSummary, "summary-chkpt_1000");
+  });
+
+  it("excludes dedup_status='removed' rows from counts", () => {
+    const sid = "sess_dedup";
+    upsertCheckpoint(makeCp("chkpt_001", sid), dir);
+    upsertCheckpoint(makeCp("chkpt_002", sid), dir);
+    setDedupStatus("chkpt_001", sid, "removed", dir);
+    const s = storeStats(sid, dir);
+    assert.equal(s.checkpointCount, 1);
+    assert.equal(s.lastCheckpointId, "chkpt_002");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// addTokensSaved (meta.ts) — CAST AS REAL preserves fractional values
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: addTokensSaved (meta.ts)", () => {
+  let dir: string;
+  before(() => {
+    dir = mkdtempSync(join(tmpdir(), "mc-mech-meta-"));
+    process.env.MEGACOMPACT_STATE_DIR = dir;
+  });
+  after(() => {
+    closeStore(dir);
+    delete process.env.MEGACOMPACT_STATE_DIR;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("preserves fractional values (CAST AS REAL, not INTEGER)", () => {
+    addTokensSaved(10.5, dir);
+    assert.equal(getTokensSaved(dir), 10.5);
+    addTokensSaved(10.5, dir);
+    // With INTEGER: CAST("10.5" AS INTEGER)=10, 10+10.5=20.5.
+    // With REAL: CAST("10.5" AS REAL)=10.5, 10.5+10.5=21.
+    assert.equal(getTokensSaved(dir), 21);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// evictMemoryLru (memories.ts) — named @param bindings
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: evictMemoryLru (memories.ts)", () => {
+  let dir: string;
+  before(() => {
+    dir = mkdtempSync(join(tmpdir(), "mc-mech-mem-"));
+    process.env.MEGACOMPACT_STATE_DIR = dir;
+    process.env.MEGACOMPACT_MEMORY_MAX_ROWS = "3";
+  });
+  after(() => {
+    closeStore(dir);
+    delete process.env.MEGACOMPACT_STATE_DIR;
+    delete process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("evicts oldest rows past cap (repo-scoped, @repo named binding)", () => {
+    const repo = "test-repo";
+    for (let i = 0; i < 5; i++) {
+      addMemory({ content: `memory-${i}` }, repo, dir);
+    }
+    const memories = listMemories(repo, 100, dir);
+    assert.equal(memories.length, 3);
+    // Oldest two (memory-0, memory-1) evicted; memory-2, 3, 4 survive.
+    const contents = memories.map((m) => m.content).sort();
+    assert.deepEqual(contents, ["memory-2", "memory-3", "memory-4"]);
+  });
+
+  it("handles null-repo scope (repo IS NULL branch, @over only)", () => {
+    for (let i = 0; i < 4; i++) {
+      addMemory({ content: `nullmem-${i}` }, null, dir);
+    }
+    // listMemories(null, ...) returns ALL rows (no repo filter); narrow to
+    // null-repo rows to verify the IS NULL eviction branch in isolation.
+    const nullMemories = listMemories(null, 100, dir).filter((m) => m.repo === null);
+    assert.equal(nullMemories.length, 3);
+    // Oldest (nullmem-0) evicted.
+    const contents = nullMemories.map((m) => m.content).sort();
+    assert.deepEqual(contents, ["nullmem-1", "nullmem-2", "nullmem-3"]);
+  });
+});

package/src/store/sqlite/memories.ts

@@ -53,10 +53,10 @@ function evictMemoryLru(repo: string | null, stateDir: string): void {
   const maxRows = memoryMaxRows();
   // SQLite `= NULL` is never true, so the null-repo scope (memories are
   // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
-  const where = repo == null ? "repo IS NULL" : "repo = ?";
+  const where = repo == null ? "repo IS NULL" : "repo = @repo";
   const countRow = repo == null
     ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
-    : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
+    : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get({ "@repo": repo });
   const count = (countRow as { n: number }).n;
   const over = count - maxRows;
   if (over <= 0) return;
@@ -67,10 +67,10 @@ function evictMemoryLru(repo: string | null, stateDir: string): void {
     `DELETE FROM memories WHERE ${where} AND id IN (
        SELECT id FROM memories WHERE ${where}
        ORDER BY COALESCE(last_referenced, last_recalled_at, created_at) ASC, id ASC
-       LIMIT ?
+       LIMIT @over
      )`;
-  if (repo == null) db.prepare(sql).run(over);
-  else db.prepare(sql).run(repo, repo, over);
+  if (repo == null) db.prepare(sql).run({ "@over": over });
+  else db.prepare(sql).run({ "@over": over, "@repo": repo });
 }
 
 export interface MemoryRecord {

package/src/store/sqlite/meta.ts

@@ -33,7 +33,7 @@ export function addTokensSaved(delta: number, stateDir: string = getStateDir()):
   const db = openStore(stateDir);
   db.prepare(
     `INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
-     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
+     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS REAL) + ? AS TEXT)`,
   ).run(String(delta), delta);
 }
 

package/src/store/sqlite/raptor.test.ts

@@ -0,0 +1,139 @@
+/**
+ * raptor.test.ts — regression tests for RAPTOR node persistence.
+ *
+ * Covers:
+ *   1. saveRaptorTree atomicity (withTx + openStore nesting safety): rebuilding
+ *      a DIFFERENT tree for the same session leaves no stale nodes from the
+ *      first tree.
+ *   2. listRaptorNodes robustness: corrupt `children` JSON in a row does not
+ *      throw and that row's children come back as [].
+ *
+ * No network. Uses node:sqlite directly via openStore.
+ */
+
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openStore, closeStore } from "./utils.js";
+import { saveRaptorTree, listRaptorNodes, type StoredRaptorNode } from "./raptor.js";
+
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "mc-raptor-persist-"));
+});
+
+afterEach(() => {
+  closeStore(tmpDir);
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+/** Build a simple tree map with the minimal shape saveRaptorTree expects. */
+function makeTree(nodeSpecs: { id: string; level: number; children: string[] }[]) {
+  const nodes = new Map();
+  for (const spec of nodeSpecs) {
+    nodes.set(spec.id, {
+      id: spec.id,
+      level: spec.level,
+      parentId: spec.level === 0 ? null : "root",
+      children: spec.children,
+      summary: `summary for ${spec.id}`,
+      embedding: [1, 0, 0],
+      qualityMarker: "low",
+      tokenEstimate: 10,
+    });
+  }
+  return { nodes };
+}
+
+// ── Test 1: rebuild + save a different tree → no stale nodes ──────────────────
+
+test("saveRaptorTree: rebuilding a different tree for the same session leaves no stale nodes", () => {
+  const sid = "sess_replace";
+  const builtAt1 = 1000;
+  const builtAt2 = 2000;
+
+  // Tree A: nodes alpha, beta, root.
+  const treeA = makeTree([
+    { id: "alpha", level: 1, children: ["leaf_0", "leaf_1"] },
+    { id: "beta", level: 1, children: ["leaf_2", "leaf_3"] },
+    { id: "root", level: 2, children: ["leaf_0", "leaf_1", "leaf_2", "leaf_3"] },
+  ]);
+
+  saveRaptorTree(sid, treeA, builtAt1, tmpDir);
+
+  const afterA = listRaptorNodes(sid, tmpDir);
+  assert.equal(afterA.length, 3, "tree A has 3 nodes");
+  assert.deepEqual(
+    afterA.map((n) => n.id).sort(),
+    ["alpha", "beta", "root"],
+    "tree A node ids match",
+  );
+
+  // Tree B: completely different nodes gamma, delta, root.
+  const treeB = makeTree([
+    { id: "gamma", level: 1, children: ["leaf_4", "leaf_5"] },
+    { id: "delta", level: 1, children: ["leaf_6", "leaf_7"] },
+    { id: "root", level: 2, children: ["leaf_4", "leaf_5", "leaf_6", "leaf_7"] },
+  ]);
+
+  saveRaptorTree(sid, treeB, builtAt2, tmpDir);
+
+  const afterB = listRaptorNodes(sid, tmpDir);
+  assert.equal(afterB.length, 3, "tree B has 3 nodes (stale A nodes gone)");
+
+  // No stale node ids from tree A survive.
+  const idsAfterB = new Set(afterB.map((n) => n.id));
+  assert.ok(!idsAfterB.has("alpha"), "stale node alpha is gone");
+  assert.ok(!idsAfterB.has("beta"), "stale node beta is gone");
+  assert.ok(idsAfterB.has("gamma"), "new node gamma present");
+  assert.ok(idsAfterB.has("delta"), "new node delta present");
+  assert.ok(idsAfterB.has("root"), "root present (upserted, not duplicated)");
+
+  // builtAt was updated to the new timestamp.
+  for (const n of afterB) {
+    assert.equal(n.builtAt, builtAt2, `node ${n.id} builtAt updated to tree B timestamp`);
+  }
+});
+
+// ── Test 2: corrupt children JSON → no throw, children = [] ──────────────────
+
+test("listRaptorNodes: corrupt children JSON does not throw and returns []", () => {
+  const sid = "sess_corrupt";
+  const builtAt = 5000;
+
+  const tree = makeTree([
+    { id: "good", level: 1, children: ["leaf_0", "leaf_1"] },
+    { id: "bad", level: 1, children: ["leaf_2", "leaf_3"] },
+    { id: "root", level: 2, children: ["leaf_0", "leaf_1", "leaf_2", "leaf_3"] },
+  ]);
+
+  saveRaptorTree(sid, tree, builtAt, tmpDir);
+
+  // Corrupt the `children` JSON for the "bad" row directly in SQLite.
+  const db = openStore(tmpDir);
+  db.prepare(
+    "UPDATE raptor_nodes SET children = ? WHERE session_id = ? AND id = ?",
+  ).run("{not valid json", sid, "bad");
+
+  // listRaptorNodes must not throw.
+  let nodes: StoredRaptorNode[];
+  assert.doesNotThrow(() => {
+    nodes = listRaptorNodes(sid, tmpDir);
+  }, "listRaptorNodes must not throw on corrupt children JSON");
+
+  const badNode = nodes!.find((n) => n.id === "bad");
+  assert.ok(badNode, "bad node still present");
+  assert.deepEqual(badNode!.children, [], "corrupt children parsed to []");
+
+  // The good node is unaffected.
+  const goodNode = nodes!.find((n) => n.id === "good");
+  assert.ok(goodNode, "good node present");
+  assert.deepEqual(
+    goodNode!.children,
+    ["leaf_0", "leaf_1"],
+    "good node children intact",
+  );
+});