pi-mega-compact 0.4.0

package/src/store/integrity.ts

@@ -0,0 +1,65 @@
+/**
+ * integrity.ts — post-backfill / audit integrity checks (Sprint 10).
+ *
+ * Two checks (QA #1 / QA #14 spirit, re-mapped locally):
+ *   1. Sentinel vs recomputed: the `session_state.stored_region_hashes` set must
+ *      equal the set of `region_hash` values recomputed from `context_chunks`.
+ *      A mismatch flags a tampered / stale sentinel (so the dedup sentinel can't
+ *      miss a real duplicate).
+ *   2. Orphan id detection: any `injected_checkpoint_ids` entry that does not
+ *      correspond to a real `context_chunks.id` is orphaned and flagged.
+ *
+ * Pure read-only verification — never mutates; returns a structured report.
+ * SQLite is the source of truth; no network (PREVENT-PI-004).
+ */
+
+import { openStore, listCheckpoints, loadSessionState } from "./sqlite.js";
+import { getStateDir, normalizeSessionId } from "../store.js";
+
+export interface IntegrityReport {
+  sessionId: string;
+  ok: boolean;
+  storedRegionHashes: number;
+  recomputedRegionHashes: number;
+  regionHashMismatch: boolean;
+  orphanInjectedIds: string[];
+}
+
+/** Verify one session's sentinel set + injected-id integrity. */
+export function checkSessionIntegrity(
+  sessionId: string,
+  stateDir: string = getStateDir(),
+): IntegrityReport {
+  openStore(stateDir); // ensure schema is initialized for this state dir
+  const sid = normalizeSessionId(sessionId);
+  const state = loadSessionState(sessionId, stateDir);
+  const checkpoints = listCheckpoints(sessionId, stateDir);
+
+  // Recompute the region-hash set from the checkpoint rows (source of truth).
+  const recomputed = new Set(checkpoints.map((c) => c.regionHash).filter(Boolean));
+  const stored = new Set(state.storedRegionHashes);
+  const regionHashMismatch =
+    recomputed.size !== stored.size || [...recomputed].some((h) => !stored.has(h));
+
+  // Orphan injected ids: referenced but no matching checkpoint.
+  const validIds = new Set(checkpoints.map((c) => c.checkpointId));
+  const orphanInjectedIds = state.injectedCheckpointIds.filter((id) => !validIds.has(id));
+
+  return {
+    sessionId: sid,
+    ok: !regionHashMismatch && orphanInjectedIds.length === 0,
+    storedRegionHashes: stored.size,
+    recomputedRegionHashes: recomputed.size,
+    regionHashMismatch,
+    orphanInjectedIds,
+  };
+}
+
+/** Check every session present in the store. */
+export function checkAllIntegrity(stateDir: string = getStateDir()): IntegrityReport[] {
+  const db = openStore(stateDir);
+  const rows = db.prepare("SELECT DISTINCT session_id FROM context_chunks").all() as {
+    session_id: string;
+  }[];
+  return rows.map((r) => checkSessionIntegrity(r.session_id, stateDir));
+}

package/src/store/migrate.test.ts

@@ -0,0 +1,158 @@
+/**
+ * migrate + recall integration test — Sprint 8 acceptance proofs.
+ *
+ * 1. Migration is lossless: a v0.1.0 `<sess>.checkpoints.json.gz` roundtrips
+ *    into SQLite with checkpoint count + regionHash set identical, and the JSON
+ *    file is retained as a DR snapshot.
+ * 2. Cross-process recall: compact in one VectorStore, then recall via a FRESH
+ *    VectorStore over the SAME stateDir (re-opens the same sqlite.db file) — the
+ *    checkpoint must reappear. Mirrors Sprint 6.1's durability requirement.
+ *
+ * Uses MEGACOMPACT_STATE_DIR overrides; never the real user state dir.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../vectorStore.js";
+import { writeGzJson } from "../store.js";
+import type { StoredCheckpoint } from "../store.js";
+import { migrateJsonToSqlite, readLegacyCheckpointFile } from "../store/migrate.js";
+import { listCheckpoints, closeStore } from "../store/sqlite.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-migrate-"));
+let counter = 0;
+function stateDir() {
+  return join(baseTmp, `run-${counter++}`);
+}
+function msgVec(): number[] {
+  // Deterministic 8-dim vector.
+  return [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8];
+}
+function fakeCheckpoints(sessionId: string): StoredCheckpoint[] {
+  return [
+    {
+      checkpointId: "chkpt_001",
+      sessionId,
+      summary: "Investigated the vector store and added a cosine helper.",
+      topicSummary: "Added cosine similarity helper to vector store.",
+      summaryHash: "a1b2c3d4e5f6a7b8",
+      keyDecisions: ["use linear scan"],
+      nextSteps: ["add tests"],
+      filesModified: ["src/vectorStore.ts"],
+      tokenEstimate: 1200,
+      regionHash: "r1",
+      embedding: msgVec(),
+      timestamp: 1,
+    },
+    {
+      checkpointId: "chkpt_002",
+      sessionId,
+      summary: "Refactored the recall path to dedupe against the window.",
+      topicSummary: "Recall dedup against injected set.",
+      summaryHash: "b2c3d4e5f6a7b8c9",
+      keyDecisions: [],
+      nextSteps: [],
+      filesModified: ["src/recall.ts"],
+      tokenEstimate: 900,
+      regionHash: "r2",
+      embedding: msgVec().map((v) => v + 0.01),
+      timestamp: 2,
+    },
+  ];
+}
+
+test("migration: v0.1.0 JSON checkpoints migrate losslessly into SQLite", () => {
+  const dir = stateDir();
+  const sid = "sess_migrate_lossless";
+  // Write a legacy JSON checkpoint file (the format v0.1.0 shipped).
+  writeGzJson(join(dir, `${sid}.checkpoints.json.gz`), fakeCheckpoints(sid));
+
+  const result = migrateJsonToSqlite(dir);
+  assert.equal(result.sessionsScanned, 1);
+  assert.equal(result.checkpointsMigrated, 2);
+  assert.equal(result.alreadyPresent, 0);
+
+  // SQLite now has both checkpoints, regionHash preserved.
+  const migrated = listCheckpoints(sid, dir);
+  assert.equal(migrated.length, 2, "both checkpoints present");
+  assert.ok(migrated.every((c) => c.regionHash && c.regionHash.length > 0), "regionHash preserved");
+  assert.deepEqual(migrated.map((c) => c.checkpointId).sort(), ["chkpt_001", "chkpt_002"]);
+
+  // content_hash columns populated (needed by Sprint 9).
+  const legacy = readLegacyCheckpointFile(sid, dir);
+  assert.equal(legacy.length, 2, "legacy file intact (DR snapshot retained)");
+  assert.ok(existsSync(join(dir, `${sid}.checkpoints.json.gz`)), "JSON DR snapshot retained");
+
+  closeStore(dir);
+});
+
+test("migration: re-running is idempotent (no duplicates)", () => {
+  const dir = stateDir();
+  const sid = "sess_migrate_idem";
+  writeGzJson(join(dir, `${sid}.checkpoints.json.gz`), fakeCheckpoints(sid));
+  migrateJsonToSqlite(dir);
+  const r2 = migrateJsonToSqlite(dir);
+  assert.equal(r2.checkpointsMigrated, 0, "nothing new migrated");
+  assert.equal(r2.alreadyPresent, 2, "both counted as already present");
+  assert.equal(listCheckpoints(sid, dir).length, 2);
+  closeStore(dir);
+});
+
+test("cross-process recall: fresh VectorStore over same dir recalls prior checkpoint", () => {
+  const dir = stateDir();
+  const sid = "sess_xproc";
+
+  // Process A: compact a checkpoint into the store.
+  const a = new VectorStore({ stateDir: dir });
+  const added = a.add({
+    sessionId: sid,
+    summary: "Cross-process recall proof: persisted in process A.",
+    topicSummary: "Persisted checkpoint in process A.",
+    regionText: "cross process recall proof session A write path",
+    tokenEstimate: 500,
+    timestamp: 1,
+  });
+  assert.equal(added.deduped, false);
+  assert.equal(added.checkpoint.checkpointId, "chkpt_001");
+
+  // Force a clean reopen (simulates a new process opening the same file).
+  closeStore(dir);
+
+  // Process B: brand-new VectorStore, same stateDir.
+  const b = new VectorStore({ stateDir: dir });
+  const hits = b.search(sid, "cross process recall proof", 5);
+  assert.equal(hits.length, 1, "checkpoint survives cross-process reopen");
+  assert.equal(hits[0].checkpoint.checkpointId, "chkpt_001");
+  assert.ok(hits[0].score > 0.5, "recall is relevant");
+
+  closeStore(dir);
+});
+
+test("cross-process recall: injected state persists across reopen", () => {
+  const dir = stateDir();
+  const sid = "sess_xproc_inj";
+  const a = new VectorStore({ stateDir: dir });
+  const added = a.add({
+    sessionId: sid,
+    summary: "A checkpoint to inject and remember across processes.",
+    topicSummary: "Injected checkpoint.",
+    regionText: "injected state persists across process reopen test",
+    timestamp: 1,
+  });
+  a.markInjected(sid, added.checkpoint.checkpointId);
+  assert.equal(a.wasInjected(sid, added.checkpoint.checkpointId), true);
+
+  closeStore(dir);
+
+  const b = new VectorStore({ stateDir: dir });
+  assert.equal(b.wasInjected(sid, added.checkpoint.checkpointId), true, "injection remembered");
+
+  closeStore(dir);
+});
+
+test("cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/store/migrate.ts

@@ -0,0 +1,108 @@
+/**
+ * migrate.ts — Sprint 8: bring v0.1.0 JSON checkpoint files into SQLite.
+ *
+ * Reads every `<sessionId>.checkpoints.json.gz` in the state dir, computes the
+ * dedup-tier columns (content_hash / content_hash2 / normalized_text) that
+ * Sprints 9-12 match on, and upserts into context_chunks idempotently
+ * (ON CONFLICT id DO NOTHING — re-running is a no-op). The JSON files are kept
+ * as disaster-recovery snapshots; they are never deleted.
+ *
+ * Runs on first VectorStore construction (auto-migrate) and is also exposed for
+ * the integration test to call explicitly.
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { getStateDir, readGzJson, normalizeSessionId } from "../store.js";
+import type { StoredCheckpoint } from "../store.js";
+import { openStore, upsertCheckpoint, listCheckpoints } from "./sqlite.js";
+
+/** Scan a state dir for v0.1.0 checkpoint JSON files. */
+function legacyCheckpointFiles(stateDir: string): string[] {
+  if (!existsSync(stateDir)) return [];
+  return readdirSync(stateDir).filter((f) => f.endsWith(".checkpoints.json.gz"));
+}
+
+/** Derive the normalized text + two content hashes for the dedup tiers. */
+export function deriveContentHashes(cp: StoredCheckpoint): {
+  normalizedText: string;
+  contentHash: string;
+  contentHash2: string;
+} {
+  // L0/L1 basis: the human summary (whitespace-normalized).
+  const normalized = (cp.summary ?? "").replace(/\s+/g, " ").trim();
+  const contentHash = createHash("sha256").update(normalized).digest("hex");
+  // L2 basis: summary + extractive topic summary (catches paraphrased topics).
+  const basis2 = `${normalized}\n${cp.topicSummary ?? ""}`.trim();
+  const contentHash2 = createHash("sha256").update(basis2).digest("hex");
+  return { normalizedText: normalized, contentHash, contentHash2 };
+}
+
+/** Read a single legacy checkpoint file (lossless: returns every stored field). */
+export function readLegacyCheckpointFile(sessionId: string, stateDir: string = getStateDir()): StoredCheckpoint[] {
+  const file = join(stateDir, `${normalizeSessionId(sessionId)}.checkpoints.json.gz`);
+  return readGzJson<StoredCheckpoint[]>(file, []);
+}
+
+export interface MigrationResult {
+  sessionsScanned: number;
+  checkpointsMigrated: number;
+  alreadyPresent: number;
+}
+
+/**
+ * Migrate all legacy JSON checkpoint files in `stateDir` into SQLite.
+ * Idempotent — safe to call repeatedly. Does not delete JSON files.
+ */
+export function migrateJsonToSqlite(stateDir: string = getStateDir()): MigrationResult {
+  openStore(stateDir); // ensures schema exists
+  const files = legacyCheckpointFiles(stateDir);
+  let sessionsScanned = 0;
+  let migrated = 0;
+  let alreadyPresent = 0;
+
+  for (const file of files) {
+    // File name shape: <sessionId>.checkpoints.json.gz
+    const sessionId = file.replace(/\.checkpoints\.json\.gz$/, "");
+    const cps = readLegacyCheckpointFile(sessionId, stateDir);
+    if (cps.length === 0) continue;
+    sessionsScanned++;
+
+    const existing = new Set(listCheckpoints(sessionId, stateDir).map((c) => c.checkpointId));
+    for (const cp of cps) {
+      if (existing.has(cp.checkpointId)) {
+        alreadyPresent++;
+        continue;
+      }
+      const { normalizedText, contentHash, contentHash2 } = deriveContentHashes(cp);
+      upsertCheckpoint(
+        { ...cp, summary: cp.summary ?? "", regionHash: cp.regionHash ?? "" },
+        stateDir,
+      );
+      // Persist the extra dedup columns (upsertCheckpoint sets them null).
+      setContentHashes(cp.checkpointId, contentHash, contentHash2, normalizedText, stateDir);
+      migrated++;
+    }
+  }
+
+  return { sessionsScanned, checkpointsMigrated: migrated, alreadyPresent };
+}
+
+// Direct column update for the computed hashes (kept out of upsertCheckpoint's
+// hot path so the common write doesn't pay for hashing).
+function setContentHashes(
+  checkpointId: string,
+  contentHash: string,
+  contentHash2: string,
+  normalizedText: string,
+  stateDir: string,
+): void {
+  const Database = openStore(stateDir);
+  Database.prepare(
+    `UPDATE context_chunks
+       SET content_hash = @ch, content_hash2 = @ch2,
+           content_hash_version = 1, normalized_text = @nt
+     WHERE id = @id`,
+  ).run({ id: checkpointId, ch: contentHash, ch2: contentHash2, nt: normalizedText });
+}

package/src/store/sprint10.test.ts

@@ -0,0 +1,182 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../vectorStore.js";
+import { normalize } from "../dedup/normalize.js";
+import { openBloom, closeBloom } from "./bloom.js";
+import { backfillContentHashes, isBackfillComplete } from "./backfill.js";
+import { checkSessionIntegrity, checkAllIntegrity } from "./integrity.js";
+import { openStore, closeStore, upsertCheckpoint, saveSessionState, loadSessionState } from "./sqlite.js";
+import type { StoredCheckpoint } from "../store.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-s10-"));
+
+let counter = 0;
+function store(opts: { dedupSim?: number } = {}) {
+  const dir = join(baseTmp, `run-${counter++}`);
+  return { s: new VectorStore({ dedupSim: opts.dedupSim ?? 0.9, stateDir: dir }), dir };
+}
+
+// --- L0 normalization upgrade (case/whitespace/ANSI collapse) --------------
+
+test("Sprint 10 L0: case/whitespace/ANSI variants dedup to one row", () => {
+  const { s } = store();
+  const variants = ["user reviewed the auth module and merged the PR", "  USER   REVIEWED the AUTH module and MERGED the PR ", "USER REVIEWED THE AUTH MODULE AND MERGED THE PR"];
+  let added = 0;
+  for (const v of variants) {
+    const r = s.add({ sessionId: "sess_norm", summary: "x", regionText: v, timestamp: added + 1 });
+    if (!r.deduped) added++;
+  }
+  assert.equal(s.list("sess_norm").length, 1);
+});
+
+test("normalize case-folds so Foo/foo/FOO are equal", () => {
+  assert.equal(normalize("Foo Bar"), normalize("foo bar"));
+  assert.equal(normalize("FOO BAR"), normalize("foo bar"));
+});
+
+test("normalize strips ANSI before hashing", () => {
+  assert.equal(normalize("err\x1b[31m fatal\x1b[0m"), normalize("err fatal"));
+});
+
+// --- Bloom accelerator ------------------------------------------------------
+
+test("bloom miss short-circuits the scan; hit is confirmed by query", () => {
+  const { s, dir } = store();
+  const raw = "the region text under test for bloom behavior";
+  s.add({ sessionId: "sess_bloom", summary: "x", regionText: raw, timestamp: 1 });
+  const bloom = openBloom(dir);
+  // content_hash for `raw` is present → hit path must confirm via query and dedup.
+  assert.equal(bloom.maybeHas("this-key-is-not-present"), false); // definitive miss
+  const r2 = s.add({ sessionId: "sess_bloom", summary: "x", regionText: raw, timestamp: 2 });
+  assert.equal(r2.deduped, true);
+  assert.equal(r2.reason, "contentHash");
+});
+
+test("bloom persists to disk and reloads warm", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const s1 = new VectorStore({ stateDir: dir });
+  s1.add({ sessionId: "sess_persist", summary: "x", regionText: "persisted region", timestamp: 1 });
+  closeBloom(dir); // evict cache so the next open reads from disk
+  const s2 = new VectorStore({ stateDir: dir });
+  const r = s2.add({ sessionId: "sess_persist", summary: "x", regionText: "persisted region", timestamp: 2 });
+  assert.equal(r.deduped, true);
+});
+
+// --- Atomic write + QA #13 timeout degrade ---------------------------------
+
+test("duplicate add is idempotent (no partial rows, single checkpoint)", () => {
+  const { s } = store();
+  const raw = "idempotent region content for atomicity check";
+  const a = s.add({ sessionId: "sess_atom", summary: "x", regionText: raw, timestamp: 1 });
+  const b = s.add({ sessionId: "sess_atom", summary: "x", regionText: raw, timestamp: 2 });
+  assert.equal(a.deduped, false);
+  assert.equal(b.deduped, true);
+  assert.equal(s.list("sess_atom").length, 1);
+});
+
+// --- Backfill orchestrator -------------------------------------------------
+
+test("backfill populates null content_hash rows and is idempotent", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  openStore(dir);
+  // Seed two rows with null content_hash via direct upsert (simulating legacy data).
+  const cp = (id: string, summary: string): StoredCheckpoint => ({
+    checkpointId: id,
+    sessionId: "sess_bf",
+    summary,
+    keyDecisions: [],
+    nextSteps: [],
+    filesModified: [],
+    tokenEstimate: 0,
+    regionHash: "r",
+    embedding: [0, 0, 0],
+    timestamp: 1,
+  });
+  upsertCheckpoint(cp("chkpt_001", "alpha summary"), dir);
+  upsertCheckpoint(cp("chkpt_002", "beta summary"), dir);
+
+  const r1 = backfillContentHashes(dir);
+  assert.equal(r1.updated, 2);
+  assert.equal(r1.processed, 2);
+  assert.equal(isBackfillComplete(dir), true);
+
+  // Second run is a no-op (idempotent): no rows left to process.
+  const r2 = backfillContentHashes(dir);
+  assert.equal(r2.processed, 0);
+});
+
+test("backfill resolves a content_hash collision keeping the oldest row", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  openStore(dir);
+  // Both rows share the same normalized summary → same content_hash.
+  const cp = (id: string): StoredCheckpoint => ({
+    checkpointId: id,
+    sessionId: "sess_dup",
+    summary: "identical normalized summary text",
+    keyDecisions: [],
+    nextSteps: [],
+    filesModified: [],
+    tokenEstimate: 0,
+    regionHash: "r",
+    embedding: [0, 0, 0],
+    timestamp: 1,
+  });
+  upsertCheckpoint(cp("chkpt_001"), dir);
+  upsertCheckpoint(cp("chkpt_002"), dir);
+  const r = backfillContentHashes(dir);
+  assert.equal(r.duplicatesResolved, 1);
+  assert.equal(r.updated, 1);
+});
+
+// --- Integrity checks ------------------------------------------------------
+
+test("integrity flags a tampered storedRegionHashes set", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ stateDir: dir });
+  s.add({ sessionId: "sess_int", summary: "x", regionText: "integrity region one", timestamp: 1 });
+  s.add({ sessionId: "sess_int", summary: "y", regionText: "integrity region two", timestamp: 2 });
+  const okReport = checkSessionIntegrity("sess_int", dir);
+  assert.equal(okReport.ok, true); // consistent after normal adds
+
+  // Simulate tampering: the sentinel stores a region hash the checkpoints lack.
+  const db = openStore(dir);
+  db.prepare(
+    "UPDATE session_state SET stored_region_hashes = ? WHERE session_id = ?",
+  ).run(JSON.stringify(["deadbeefcafe0000"]), "sess_int");
+  const tampered = checkSessionIntegrity("sess_int", dir);
+  assert.equal(tampered.ok, false);
+  assert.equal(tampered.regionHashMismatch, true);
+});
+
+test("integrity detects an orphan injectedCheckpointId", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ stateDir: dir });
+  s.add({ sessionId: "sess_orphan", summary: "x", regionText: "orphan region", timestamp: 1 });
+  // Manually inject a dangling id into session state (SQLite-backed).
+  const st = loadSessionState("sess_orphan", dir);
+  st.injectedCheckpointIds.push("chkpt_999");
+  saveSessionState("sess_orphan", st, dir);
+  const report = checkSessionIntegrity("sess_orphan", dir);
+  assert.equal(report.ok, false);
+  assert.deepEqual(report.orphanInjectedIds, ["chkpt_999"]);
+});
+
+test("checkAllIntegrity covers every session", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ stateDir: dir });
+  s.add({ sessionId: "sess_a", summary: "x", regionText: "a region", timestamp: 1 });
+  s.add({ sessionId: "sess_b", summary: "y", regionText: "b region", timestamp: 1 });
+  const reports = checkAllIntegrity(dir);
+  assert.equal(reports.length, 2);
+  assert.ok(reports.every((r: { ok: boolean }) => r.ok));
+});
+
+// --- cleanup ---------------------------------------------------------------
+
+test("Sprint 10 cleanup", () => {
+  closeStore(baseTmp);
+  rmSync(baseTmp, { recursive: true, force: true });
+});