pi-mega-compact 0.8.23 → 0.8.25

package/src/store/sqlite/mechanical-fix.test.ts

@@ -0,0 +1,162 @@
+/**
+ * mechanical-fix.test.ts — focused unit tests for the SQLite-side mechanical-fix
+ * batch: safeJson (utils.ts), storeStats numeric MAX + dedup_status filter
+ * (stats.ts), addTokensSaved CAST AS REAL (meta.ts), evictMemoryLru named
+ * @param bindings (memories.ts). Pi-agnostic; uses isolated state dirs (G7).
+ */
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import { safeJson, closeStore } from "./utils.js";
+import { storeStats } from "./stats.js";
+import { upsertCheckpoint, setDedupStatus } from "./checkpoints.js";
+import { addTokensSaved, getTokensSaved } from "./meta.js";
+import { addMemory, listMemories } from "./memories.js";
+import type { StoredCheckpoint } from "../../store.js";
+
+function makeCp(id: string, sessionId: string): StoredCheckpoint {
+  return {
+    checkpointId: id,
+    sessionId,
+    summary: `summary-${id}`,
+    keyDecisions: [],
+    nextSteps: [],
+    filesModified: [],
+    tokenEstimate: 100,
+    regionHash: `hash-${id}`,
+    embedding: [],
+    timestamp: Date.now(),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// safeJson (utils.ts)
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: safeJson (utils.ts)", () => {
+  it("returns fallback on null, undefined, empty, and corrupt JSON", () => {
+    assert.deepEqual(safeJson<string[]>(null, []), []);
+    assert.deepEqual(safeJson<string[]>(undefined, []), []);
+    assert.deepEqual(safeJson<string[]>("", []), []);
+    assert.deepEqual(safeJson<string[]>("not json", ["fb"]), ["fb"]);
+    assert.deepEqual(safeJson<string[]>('["a","b"]', []), ["a", "b"]);
+    assert.equal(safeJson<number>(null, 42), 42);
+    assert.deepEqual(safeJson<Record<string, number>>('{"k":1}', {}), { k: 1 });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// storeStats (stats.ts) — numeric MAX(id) + dedup_status filter
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: storeStats (stats.ts)", () => {
+  let dir: string;
+  before(() => {
+    dir = mkdtempSync(join(tmpdir(), "mc-mech-stats-"));
+    process.env.MEGACOMPACT_STATE_DIR = dir;
+  });
+  after(() => {
+    closeStore(dir);
+    delete process.env.MEGACOMPACT_STATE_DIR;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("returns numerically-max checkpoint id (not lexicographic) for 100+ checkpoints", () => {
+    const sid = "sess_stats";
+    // Insert chkpt_001..chkpt_100 (100 checkpoints) + chkpt_999 + chkpt_1000.
+    // Numeric max = 1000 → "chkpt_1000"; lexicographic max would be "chkpt_999"
+    // (because '9' > '1' at position 6 when comparing 8 vs 9-char strings).
+    for (let i = 1; i <= 100; i++) {
+      upsertCheckpoint(makeCp(`chkpt_${String(i).padStart(3, "0")}`, sid), dir);
+    }
+    upsertCheckpoint(makeCp("chkpt_999", sid), dir);
+    upsertCheckpoint(makeCp("chkpt_1000", sid), dir);
+
+    const s = storeStats(sid, dir);
+    assert.equal(s.checkpointCount, 102);
+    assert.equal(s.lastCheckpointId, "chkpt_1000");
+    assert.equal(s.lastSummary, "summary-chkpt_1000");
+  });
+
+  it("excludes dedup_status='removed' rows from counts", () => {
+    const sid = "sess_dedup";
+    upsertCheckpoint(makeCp("chkpt_001", sid), dir);
+    upsertCheckpoint(makeCp("chkpt_002", sid), dir);
+    setDedupStatus("chkpt_001", sid, "removed", dir);
+    const s = storeStats(sid, dir);
+    assert.equal(s.checkpointCount, 1);
+    assert.equal(s.lastCheckpointId, "chkpt_002");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// addTokensSaved (meta.ts) — CAST AS REAL preserves fractional values
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: addTokensSaved (meta.ts)", () => {
+  let dir: string;
+  before(() => {
+    dir = mkdtempSync(join(tmpdir(), "mc-mech-meta-"));
+    process.env.MEGACOMPACT_STATE_DIR = dir;
+  });
+  after(() => {
+    closeStore(dir);
+    delete process.env.MEGACOMPACT_STATE_DIR;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("preserves fractional values (CAST AS REAL, not INTEGER)", () => {
+    addTokensSaved(10.5, dir);
+    assert.equal(getTokensSaved(dir), 10.5);
+    addTokensSaved(10.5, dir);
+    // With INTEGER: CAST("10.5" AS INTEGER)=10, 10+10.5=20.5.
+    // With REAL: CAST("10.5" AS REAL)=10.5, 10.5+10.5=21.
+    assert.equal(getTokensSaved(dir), 21);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// evictMemoryLru (memories.ts) — named @param bindings
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: evictMemoryLru (memories.ts)", () => {
+  let dir: string;
+  before(() => {
+    dir = mkdtempSync(join(tmpdir(), "mc-mech-mem-"));
+    process.env.MEGACOMPACT_STATE_DIR = dir;
+    process.env.MEGACOMPACT_MEMORY_MAX_ROWS = "3";
+  });
+  after(() => {
+    closeStore(dir);
+    delete process.env.MEGACOMPACT_STATE_DIR;
+    delete process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("evicts oldest rows past cap (repo-scoped, @repo named binding)", () => {
+    const repo = "test-repo";
+    for (let i = 0; i < 5; i++) {
+      addMemory({ content: `memory-${i}` }, repo, dir);
+    }
+    const memories = listMemories(repo, 100, dir);
+    assert.equal(memories.length, 3);
+    // Oldest two (memory-0, memory-1) evicted; memory-2, 3, 4 survive.
+    const contents = memories.map((m) => m.content).sort();
+    assert.deepEqual(contents, ["memory-2", "memory-3", "memory-4"]);
+  });
+
+  it("handles null-repo scope (repo IS NULL branch, @over only)", () => {
+    for (let i = 0; i < 4; i++) {
+      addMemory({ content: `nullmem-${i}` }, null, dir);
+    }
+    // listMemories(null, ...) returns ALL rows (no repo filter); narrow to
+    // null-repo rows to verify the IS NULL eviction branch in isolation.
+    const nullMemories = listMemories(null, 100, dir).filter((m) => m.repo === null);
+    assert.equal(nullMemories.length, 3);
+    // Oldest (nullmem-0) evicted.
+    const contents = nullMemories.map((m) => m.content).sort();
+    assert.deepEqual(contents, ["nullmem-1", "nullmem-2", "nullmem-3"]);
+  });
+});

package/src/store/sqlite/memories.ts

@@ -53,10 +53,10 @@ function evictMemoryLru(repo: string | null, stateDir: string): void {
   const maxRows = memoryMaxRows();
   // SQLite `= NULL` is never true, so the null-repo scope (memories are
   // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
-  const where = repo == null ? "repo IS NULL" : "repo = ?";
+  const where = repo == null ? "repo IS NULL" : "repo = @repo";
   const countRow = repo == null
     ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
-    : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
+    : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get({ "@repo": repo });
   const count = (countRow as { n: number }).n;
   const over = count - maxRows;
   if (over <= 0) return;
@@ -67,10 +67,10 @@ function evictMemoryLru(repo: string | null, stateDir: string): void {
     `DELETE FROM memories WHERE ${where} AND id IN (
        SELECT id FROM memories WHERE ${where}
        ORDER BY COALESCE(last_referenced, last_recalled_at, created_at) ASC, id ASC
-       LIMIT ?
+       LIMIT @over
      )`;
-  if (repo == null) db.prepare(sql).run(over);
-  else db.prepare(sql).run(repo, repo, over);
+  if (repo == null) db.prepare(sql).run({ "@over": over });
+  else db.prepare(sql).run({ "@over": over, "@repo": repo });
 }
 
 export interface MemoryRecord {

package/src/store/sqlite/meta.ts

@@ -33,7 +33,7 @@ export function addTokensSaved(delta: number, stateDir: string = getStateDir()):
   const db = openStore(stateDir);
   db.prepare(
     `INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
-     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
+     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS REAL) + ? AS TEXT)`,
   ).run(String(delta), delta);
 }
 

package/src/store/sqlite/raptor.test.ts

@@ -0,0 +1,139 @@
+/**
+ * raptor.test.ts — regression tests for RAPTOR node persistence.
+ *
+ * Covers:
+ *   1. saveRaptorTree atomicity (withTx + openStore nesting safety): rebuilding
+ *      a DIFFERENT tree for the same session leaves no stale nodes from the
+ *      first tree.
+ *   2. listRaptorNodes robustness: corrupt `children` JSON in a row does not
+ *      throw and that row's children come back as [].
+ *
+ * No network. Uses node:sqlite directly via openStore.
+ */
+
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openStore, closeStore } from "./utils.js";
+import { saveRaptorTree, listRaptorNodes, type StoredRaptorNode } from "./raptor.js";
+
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "mc-raptor-persist-"));
+});
+
+afterEach(() => {
+  closeStore(tmpDir);
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+/** Build a simple tree map with the minimal shape saveRaptorTree expects. */
+function makeTree(nodeSpecs: { id: string; level: number; children: string[] }[]) {
+  const nodes = new Map();
+  for (const spec of nodeSpecs) {
+    nodes.set(spec.id, {
+      id: spec.id,
+      level: spec.level,
+      parentId: spec.level === 0 ? null : "root",
+      children: spec.children,
+      summary: `summary for ${spec.id}`,
+      embedding: [1, 0, 0],
+      qualityMarker: "low",
+      tokenEstimate: 10,
+    });
+  }
+  return { nodes };
+}
+
+// ── Test 1: rebuild + save a different tree → no stale nodes ──────────────────
+
+test("saveRaptorTree: rebuilding a different tree for the same session leaves no stale nodes", () => {
+  const sid = "sess_replace";
+  const builtAt1 = 1000;
+  const builtAt2 = 2000;
+
+  // Tree A: nodes alpha, beta, root.
+  const treeA = makeTree([
+    { id: "alpha", level: 1, children: ["leaf_0", "leaf_1"] },
+    { id: "beta", level: 1, children: ["leaf_2", "leaf_3"] },
+    { id: "root", level: 2, children: ["leaf_0", "leaf_1", "leaf_2", "leaf_3"] },
+  ]);
+
+  saveRaptorTree(sid, treeA, builtAt1, tmpDir);
+
+  const afterA = listRaptorNodes(sid, tmpDir);
+  assert.equal(afterA.length, 3, "tree A has 3 nodes");
+  assert.deepEqual(
+    afterA.map((n) => n.id).sort(),
+    ["alpha", "beta", "root"],
+    "tree A node ids match",
+  );
+
+  // Tree B: completely different nodes gamma, delta, root.
+  const treeB = makeTree([
+    { id: "gamma", level: 1, children: ["leaf_4", "leaf_5"] },
+    { id: "delta", level: 1, children: ["leaf_6", "leaf_7"] },
+    { id: "root", level: 2, children: ["leaf_4", "leaf_5", "leaf_6", "leaf_7"] },
+  ]);
+
+  saveRaptorTree(sid, treeB, builtAt2, tmpDir);
+
+  const afterB = listRaptorNodes(sid, tmpDir);
+  assert.equal(afterB.length, 3, "tree B has 3 nodes (stale A nodes gone)");
+
+  // No stale node ids from tree A survive.
+  const idsAfterB = new Set(afterB.map((n) => n.id));
+  assert.ok(!idsAfterB.has("alpha"), "stale node alpha is gone");
+  assert.ok(!idsAfterB.has("beta"), "stale node beta is gone");
+  assert.ok(idsAfterB.has("gamma"), "new node gamma present");
+  assert.ok(idsAfterB.has("delta"), "new node delta present");
+  assert.ok(idsAfterB.has("root"), "root present (upserted, not duplicated)");
+
+  // builtAt was updated to the new timestamp.
+  for (const n of afterB) {
+    assert.equal(n.builtAt, builtAt2, `node ${n.id} builtAt updated to tree B timestamp`);
+  }
+});
+
+// ── Test 2: corrupt children JSON → no throw, children = [] ──────────────────
+
+test("listRaptorNodes: corrupt children JSON does not throw and returns []", () => {
+  const sid = "sess_corrupt";
+  const builtAt = 5000;
+
+  const tree = makeTree([
+    { id: "good", level: 1, children: ["leaf_0", "leaf_1"] },
+    { id: "bad", level: 1, children: ["leaf_2", "leaf_3"] },
+    { id: "root", level: 2, children: ["leaf_0", "leaf_1", "leaf_2", "leaf_3"] },
+  ]);
+
+  saveRaptorTree(sid, tree, builtAt, tmpDir);
+
+  // Corrupt the `children` JSON for the "bad" row directly in SQLite.
+  const db = openStore(tmpDir);
+  db.prepare(
+    "UPDATE raptor_nodes SET children = ? WHERE session_id = ? AND id = ?",
+  ).run("{not valid json", sid, "bad");
+
+  // listRaptorNodes must not throw.
+  let nodes: StoredRaptorNode[];
+  assert.doesNotThrow(() => {
+    nodes = listRaptorNodes(sid, tmpDir);
+  }, "listRaptorNodes must not throw on corrupt children JSON");
+
+  const badNode = nodes!.find((n) => n.id === "bad");
+  assert.ok(badNode, "bad node still present");
+  assert.deepEqual(badNode!.children, [], "corrupt children parsed to []");
+
+  // The good node is unaffected.
+  const goodNode = nodes!.find((n) => n.id === "good");
+  assert.ok(goodNode, "good node present");
+  assert.deepEqual(
+    goodNode!.children,
+    ["leaf_0", "leaf_1"],
+    "good node children intact",
+  );
+});

package/src/store/sqlite/raptor.ts

@@ -2,106 +2,160 @@
  * raptor.ts — Sprint 13 RAPTOR node persistence.
  */
 import { getStateDir, normalizeSessionId } from "../../store.js";
-import { openStore, jsonText, encodeEmbedding, decodeEmbedding } from "./utils.js";
+import {
+	openStore,
+	withTx,
+	jsonText,
+	encodeEmbedding,
+	decodeEmbedding,
+} from "./utils.js";
 
 export interface StoredRaptorNode {
-  id: string;
-  sessionId: string;
-  level: number;
-  parentId: string | null;
-  children: string[];
-  summary: string;
-  embedding: number[];
-  qualityMarker: string;
-  tokenEstimate: number;
-  /** S25: epoch ms when the tree containing this node was built. */
-  builtAt: number;
+	id: string;
+	sessionId: string;
+	level: number;
+	parentId: string | null;
+	children: string[];
+	summary: string;
+	embedding: number[];
+	qualityMarker: string;
+	tokenEstimate: number;
+	/** S25: epoch ms when the tree containing this node was built. */
+	builtAt: number;
 }
 
 /** Persist a single RAPTOR node (upsert by (session_id, id)). */
-export function upsertRaptorNode(node: StoredRaptorNode, stateDir: string = getStateDir()): void {
-  const db = openStore(stateDir);
-  db.prepare(
-    `INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
-     VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+export function upsertRaptorNode(
+	node: StoredRaptorNode,
+	stateDir: string = getStateDir(),
+): void {
+	const db = openStore(stateDir);
+	db.prepare(
+		`INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
+     VALUES(@id, @session_id, @level, @parent_id, @children, @summary, @embedding_blob, @quality_marker, @token_estimate, @built_at)
      ON CONFLICT(session_id, id) DO UPDATE SET
        level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
        summary=excluded.summary, embedding_blob=excluded.embedding_blob,
        quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate,
        built_at=excluded.built_at`,
-  ).run(
-    node.id,
-    node.sessionId,
-    node.level,
-    node.parentId,
-    jsonText(node.children),
-    node.summary,
-    encodeEmbedding(node.embedding),
-    node.qualityMarker,
-    node.tokenEstimate,
-    node.builtAt,
-  );
+	).run({
+		id: node.id,
+		session_id: node.sessionId,
+		level: node.level,
+		parent_id: node.parentId,
+		children: jsonText(node.children),
+		summary: node.summary,
+		embedding_blob: encodeEmbedding(node.embedding),
+		quality_marker: node.qualityMarker,
+		token_estimate: node.tokenEstimate,
+		built_at: node.builtAt,
+	});
 }
 
 /** Persist an entire built RAPTOR tree for a session (shadow or live). */
 export function saveRaptorTree(
-  sessionId: string,
-  tree: {
-    nodes: Map<string, {
-      id: string;
-      level: number;
-      parentId: string | null;
-      children: string[];
-      summary: string;
-      embedding: number[];
-      qualityMarker: string;
-      tokenEstimate: number;
-    }>
-  },
-  builtAt: number,
-  stateDir: string = getStateDir(),
+	sessionId: string,
+	tree: {
+		nodes: Map<
+			string,
+			{
+				id: string;
+				level: number;
+				parentId: string | null;
+				children: string[];
+				summary: string;
+				embedding: number[];
+				qualityMarker: string;
+				tokenEstimate: number;
+			}
+		>;
+	},
+	builtAt: number,
+	stateDir: string = getStateDir(),
 ): void {
-  for (const node of tree.nodes.values()) {
-    upsertRaptorNode(
-      {
-        id: node.id,
-        sessionId,
-        level: node.level,
-        parentId: node.parentId,
-        children: node.children,
-        summary: node.summary,
-        embedding: node.embedding,
-        qualityMarker: node.qualityMarker,
-        tokenEstimate: node.tokenEstimate,
-        builtAt,
-      },
-      stateDir,
-    );
-  }
+	const nsid = normalizeSessionId(sessionId);
+	const db = openStore(stateDir);
+	withTx(db, () => {
+		db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(nsid);
+		for (const node of tree.nodes.values()) {
+			upsertRaptorNode(
+				{
+					id: node.id,
+					sessionId: nsid,
+					level: node.level,
+					parentId: node.parentId,
+					children: node.children,
+					summary: node.summary,
+					embedding: node.embedding,
+					qualityMarker: node.qualityMarker,
+					tokenEstimate: node.tokenEstimate,
+					builtAt,
+				},
+				stateDir,
+			);
+		}
+	});
+}
+
+/** Safe JSON array parse — returns [] on corrupt input. */
+function safeJsonArray(raw: unknown): string[] {
+	if (typeof raw !== "string" || !raw) return [];
+	try { return JSON.parse(raw) as string[]; }
+	catch { return []; }
 }
 
 /** Load all RAPTOR nodes for a session. */
-export function listRaptorNodes(sessionId: string, stateDir: string = getStateDir()): StoredRaptorNode[] {
-  const db = openStore(stateDir);
-  const rows = db
-    .prepare("SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC")
-    .all(normalizeSessionId(sessionId)) as any[];
-  return rows.map((row) => ({
-    id: row.id,
-    sessionId: row.session_id,
-    level: row.level,
-    parentId: row.parent_id ?? null,
-    children: row.children ? JSON.parse(row.children) : [],
-    summary: row.summary ?? "",
-    embedding: decodeEmbedding(row.embedding_blob),
-    qualityMarker: row.quality_marker ?? "low",
-    tokenEstimate: row.token_estimate ?? 0,
-    builtAt: Number(row.built_at ?? 0),
-  }));
+export function listRaptorNodes(
+	sessionId: string,
+	stateDir: string = getStateDir(),
+): StoredRaptorNode[] {
+	const db = openStore(stateDir);
+	const rows = db
+		.prepare(
+			"SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC",
+		)
+		.all(normalizeSessionId(sessionId)) as any[];
+	return rows.map((row) => ({
+		id: row.id,
+		sessionId: row.session_id,
+		level: row.level,
+		parentId: row.parent_id ?? null,
+		children: safeJsonArray(row.children),
+		summary: row.summary ?? "",
+		embedding: decodeEmbedding(row.embedding_blob),
+		qualityMarker: row.quality_marker ?? "low",
+		tokenEstimate: row.token_estimate ?? 0,
+		builtAt: Number(row.built_at ?? 0),
+	}));
 }
 
 /** Delete all RAPTOR nodes for a session (rollback/cleanup). */
-export function clearRaptorNodes(sessionId: string, stateDir: string = getStateDir()): void {
-  const db = openStore(stateDir);
-  db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
+export function clearRaptorNodes(
+	sessionId: string,
+	stateDir: string = getStateDir(),
+): void {
+	const db = openStore(stateDir);
+	db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(
+		normalizeSessionId(sessionId),
+	);
+}
+
+/**
+ * S25: newest built_at for a session's RAPTOR tree (0 if no nodes exist).
+ * Cheap indexed MAX query used by raptorSearchHits to validate cache freshness
+ * without rehydrating the full node Map.
+ */
+export function maxRaptorNodeBuiltAt(
+	sessionId: string,
+	stateDir: string = getStateDir(),
+): number {
+	const db = openStore(stateDir);
+	const row = db
+		.prepare(
+			`SELECT MAX(built_at) AS max_built FROM raptor_nodes WHERE session_id = ?`,
+		)
+		.get(normalizeSessionId(sessionId)) as
+		| { max_built: number | null }
+		| undefined;
+	return row?.max_built ?? 0;
 }