pi-mega-compact 0.4.4 → 0.4.6

package/dist/src/store/bloom.js

@@ -0,0 +1,114 @@
+/**
+ * bloom.ts — local bloom-filter accelerator for the L0 content-hash dedup tier
+ * (Sprint 10).
+ *
+ * ACCELERATOR ONLY (QA #2 spirit, re-mapped locally): a bloom filter has zero
+ * false negatives — a MISS truly means "this content_hash is not present", so we
+ * can skip the full SQLite scan on the happy path. A HIT is only a candidate and
+ * MUST be confirmed by a SELECT against SQLite, which remains the source of truth
+ * (PREVENT-PI-004: in-process, no network; SQLite owns durability).
+ *
+ * The filter is an in-memory `bloom-filters` Map persisted to
+ * `STATE_DIR/bloom.json.gz` so a fresh VectorStore over the same dir reuses the
+ * warm filter instead of rebuilding from a scan.
+ */
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { getStateDir } from "../store.js";
+import { compressSmart, decompressSmart } from "../store.js";
+// Fixed bit-array size + hash count sized for a 1K-checkpoint fixture at <1% FP
+// (m ≈ -n*ln(p)/ln(2)^2). 8 KiB bits → ~8192 bits, k=7 → well under 1% at 1K.
+const BITS = 8192;
+const HASHES = 7;
+const STORAGE_MARK = 0x42; // 'B' — marks a persisted bloom blob (not versioned)
+function fnv1a(data, seed) {
+    let h = 0x811c9dc5 ^ seed;
+    for (let i = 0; i < data.length; i++) {
+        h ^= data[i];
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+export class BloomFilter {
+    bits;
+    constructor(bits) {
+        this.bits = bits ?? new Uint8Array(BITS);
+    }
+    indices(key) {
+        const data = Buffer.from(key, "utf-8");
+        const idx = [];
+        for (let i = 0; i < HASHES; i++) {
+            // Double-hashing (Kirsch–Mitzenmacher) to derive k independent positions.
+            const h1 = fnv1a(data, 0x9e3779b1 * i);
+            const h2 = fnv1a(data, 0x85ebca77 * (i + 1));
+            idx.push((h1 + i * h2) % BITS);
+        }
+        return idx;
+    }
+    add(key) {
+        for (const i of this.indices(key))
+            this.bits[i >> 3] |= 1 << (i & 7);
+    }
+    /** A miss is definitive (zero false negatives): false ⇒ definitely absent. */
+    maybeHas(key) {
+        for (const i of this.indices(key)) {
+            if ((this.bits[i >> 3] & (1 << (i & 7))) === 0)
+                return false;
+        }
+        return true;
+    }
+    toBuffer() {
+        return Buffer.concat([Buffer.from([STORAGE_MARK]), Buffer.from(this.bits)]);
+    }
+    /** Raw bit array (for persistence). */
+    bytes() {
+        return this.bits;
+    }
+    static fromBuffer(buf) {
+        if (buf.length >= 1 && buf[0] === STORAGE_MARK) {
+            return new BloomFilter(Uint8Array.from(buf.subarray(1)));
+        }
+        // Legacy/compressed form: best-effort decompress.
+        try {
+            const raw = decompressSmart(buf);
+            return new BloomFilter(Uint8Array.from(raw));
+        }
+        catch {
+            return new BloomFilter();
+        }
+    }
+}
+const cache = new Map();
+/** Load (or lazily create + cache) the bloom filter for a state dir. */
+export function openBloom(stateDir = getStateDir()) {
+    const existing = cache.get(stateDir);
+    if (existing)
+        return existing;
+    const path = join(stateDir, "bloom.json.gz");
+    let filter = new BloomFilter();
+    if (existsSync(path)) {
+        try {
+            filter = BloomFilter.fromBuffer(readFileSync(path));
+        }
+        catch {
+            filter = new BloomFilter();
+        }
+    }
+    cache.set(stateDir, filter);
+    return filter;
+}
+/** Persist the bloom filter to disk (additive — does not clear the cache). */
+export function saveBloom(stateDir = getStateDir()) {
+    const filter = cache.get(stateDir);
+    if (!filter)
+        return;
+    if (!existsSync(stateDir))
+        mkdirSync(stateDir, { recursive: true });
+    // Compress the raw bit array for a smaller, versioned-on-disk footprint.
+    const blob = compressSmart(Buffer.from(filter.bytes()));
+    writeFileSync(join(stateDir, "bloom.json.gz"), blob);
+}
+/** Evict the cached filter (test teardown only). */
+export function closeBloom(stateDir) {
+    cache.delete(stateDir);
+}

package/dist/src/store/compression.js

@@ -0,0 +1,177 @@
+/**
+ * compression.ts — versioned, size-adaptive compression for checkpoint blobs.
+ *
+ * Extracted from store.ts (Sprint 8). Two coordinated compressors:
+ *
+ *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
+ *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
+ *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *
+ *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
+ *     Optional, used for DR-export / large-blob paths where an await is fine.
+ *
+ * FORMAT-VERSION PROBLEM (root cause of Sprint 8):
+ *   store.ts shipped `0x03` = brotli (legacy single-tag format). PLAN.md reassigns
+ *   `0x03` → zstd, which would corrupt every existing checkpoint file. We fix this
+ *   with a 2-byte magic header on the NEW format so the tag byte is namespaced and
+ *   can never collide with legacy payloads:
+ *
+ *     NEW (versioned):  0xEC 0x01 [TIER_TAG] [payload]
+ *     LEGACY single-tag: [TIER_TAG] [payload]   (tags 0x00..0x03)
+ *     LEGACY untagged:   0x1f ...               (plain gzip magic)
+ *
+ * `0xEC` is chosen because it collides with no zlib output: gzip magic is 0x1f,
+ * brotli streams start 0xCE/0xCF, zlib/deflate streams start 0x78/0x05/0x03.
+ * decompressSmart detects the magic first, so all three eras roundtrip together.
+ */
+import { gzipSync, gunzipSync, brotliCompressSync, brotliDecompressSync, constants as zlibConstants, } from "node:zlib";
+import zstd from "@mongodb-js/zstd";
+// --- Versioned format markers ----------------------------------------------
+const MAGIC_HI = 0xec;
+const MAGIC_LO = 0x01; // format version 1
+// Tier tags (only meaningful inside the 0xEC 0x01 versioned frame).
+const TAG_RAW = 0x00; // no compression (< 512 bytes)
+const TAG_GZIP_1 = 0x01; // gzip level 1 (fast, 512B–4KB)
+const TAG_GZIP_6 = 0x02; // gzip level 6 (default, 4KB–32KB)
+const TAG_BROTLI_4 = 0x05; // brotli level 4 (> 32KB, best text ratio, sync)
+// Reserved for the async zstd helper (see compressZstd). Not used by the sync path.
+const TAG_ZSTD_3 = 0x03;
+const TAG_ZSTD_9 = 0x04;
+/** Gzip magic byte — used to detect legacy untagged files. */
+const GZIP_MAGIC = 0x1f;
+const SIZE_TINY = 512;
+const SIZE_SMALL = 4096;
+const SIZE_MEDIUM = 32768;
+function header(ver, tag) {
+    return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
+}
+/**
+ * Compress synchronously using the best zlib tier for the payload size.
+ *
+ * Tiers (all synchronous — no network, no async, PREVENT-PI-004):
+ *   < 512 B  → raw         (tag 0x00)
+ *   512B–4KB → gzip level 1 (tag 0x01)
+ *   4KB–32KB → gzip level 6 (tag 0x02)
+ *   > 32 KB  → brotli 4     (tag 0x05)
+ *
+ * Writes the versioned header so readers disambiguate from legacy blobs.
+ */
+export function compressSmart(data) {
+    const len = data.length;
+    if (len < SIZE_TINY) {
+        return Buffer.concat([header(1, TAG_RAW), data]);
+    }
+    if (len < SIZE_SMALL) {
+        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+    }
+    if (len < SIZE_MEDIUM) {
+        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+    }
+    const compressed = brotliCompressSync(data, {
+        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+    });
+    return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
+}
+/** True when `buf` is a versioned-format blob (0xEC 0x01 …). */
+export function isVersioned(buf) {
+    return buf.length >= 2 && buf[0] === MAGIC_HI && buf[1] === MAGIC_LO;
+}
+export function detectFormat(buf) {
+    if (isVersioned(buf))
+        return "versioned";
+    if (buf[0] === GZIP_MAGIC)
+        return "legacy-gzip";
+    // Legacy single-tag: first byte is a known legacy tag.
+    if (buf[0] === 0x00 || buf[0] === 0x01 || buf[0] === 0x02 || buf[0] === 0x03) {
+        return "legacy-tag";
+    }
+    return "unknown";
+}
+/**
+ * Decompress a buffer written by `compressSmart` (versioned) OR any legacy
+ * format still on disk (legacy single-tag, legacy untagged gzip). SYNCHRONOUS.
+ *
+ * Throws on zstd blobs — those must go through the async `decompressZstd`,
+ * because zstd decompression cannot be awaited inside this sync path.
+ */
+export function decompressSmart(buf) {
+    if (buf.length === 0)
+        return buf;
+    // New versioned format — dispatch on the namespaced tier tag.
+    if (isVersioned(buf)) {
+        const tag = buf[3];
+        const payload = buf.subarray(4);
+        switch (tag) {
+            case TAG_RAW:
+                return payload;
+            case TAG_GZIP_1:
+            case TAG_GZIP_6:
+                return gunzipSync(payload);
+            case TAG_BROTLI_4:
+                return brotliDecompressSync(payload);
+            case TAG_ZSTD_3:
+            case TAG_ZSTD_9:
+                throw new Error("decompressSmart cannot read zstd blobs (async only) — use decompressZstd");
+            default:
+                throw new Error(`decompressSmart: unknown versioned tier tag 0x${tag.toString(16)}`);
+        }
+    }
+    // Legacy untagged gzip file (old writeGzJson with no tag byte).
+    if (buf[0] === GZIP_MAGIC) {
+        return gunzipSync(buf);
+    }
+    // Legacy single-tag format (store.ts v0.1.0): tags 0x00..0x03.
+    const tag = buf[0];
+    const payload = buf.subarray(1);
+    switch (tag) {
+        case 0x00: // TAG_RAW (legacy)
+            return payload;
+        case 0x01: // TAG_GZIP_1 (legacy)
+        case 0x02: // TAG_GZIP_6 (legacy)
+            return gunzipSync(payload);
+        case 0x03: // TAG_BROTLI (legacy) — the very collision this format fixes
+            return brotliDecompressSync(payload);
+        default:
+            // Unknown legacy tag — last-ditch try plain gzip.
+            return gunzipSync(buf);
+    }
+}
+// --- Optional async zstd path (DR export / large blobs) --------------------
+// Self-describing: own 2-byte marker so it never routes through decompressSmart.
+const ZSTD_MAGIC_HI = 0x5a; // 'Z'
+const ZSTD_MAGIC_LO = 0x53; // 'S'
+async function compressZstdWithLevel(data, level) {
+    const compressed = await zstd.compress(data, level);
+    return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
+}
+/** Compress with zstd level 3 (fast, balanced). Async. */
+export function compressZstd(data) {
+    return compressZstdWithLevel(data, 3);
+}
+/** Compress with zstd level 9 (max ratio). Async. */
+export function compressZstdMax(data) {
+    return compressZstdWithLevel(data, 9);
+}
+/** True when a buffer is a zstd-compressed blob from this helper. */
+export function isZstd(buf) {
+    return buf.length >= 2 && buf[0] === ZSTD_MAGIC_HI && buf[1] === ZSTD_MAGIC_LO;
+}
+/** Decompress a zstd blob produced by compressZstd/compressZstdMax. Async. */
+export async function decompressZstd(buf) {
+    if (buf.length === 0)
+        return buf;
+    if (!isZstd(buf)) {
+        throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
+    }
+    return zstd.decompress(buf.subarray(2));
+}
+/**
+ * Decompress anything we can WITHOUT awaiting: versioned + legacy zlib formats.
+ * zstd blobs are detected and reported (not thrown blindly) so callers can
+ * decide whether to await decompressZstd.
+ */
+export function decompressSyncAuto(buf) {
+    if (isZstd(buf))
+        return { data: buf, isZstd: true };
+    return { data: decompressSmart(buf), isZstd: false };
+}

package/dist/src/store/compression.test.js

@@ -0,0 +1,67 @@
+/**
+ * compression.test.ts — versioned compression tiers + backward compatibility.
+ *
+ * Proves Sprint 8's root-cause fix: the 0x03 tag collision is impossible because
+ * new blobs carry a 2-byte version magic, and legacy blobs (untagged gzip, legacy
+ * single-tag incl. the old 0x03=brotli) still decompress.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { gzipSync, brotliCompressSync } from "node:zlib";
+import { compressSmart, decompressSmart, compressZstd, decompressZstd, isVersioned, detectFormat, decompressSyncAuto, } from "./compression.js";
+const buf = (s, n) => Buffer.from(s.repeat(n));
+test("versioned format: all size tiers roundtrip and are versioned", () => {
+    const cases = [
+        { label: "tiny", data: buf("x", 100), expectTag: 0x00 },
+        { label: "small", data: buf("a", 600), expectTag: 0x01 },
+        { label: "medium", data: buf("b", 8000), expectTag: 0x02 },
+        { label: "large", data: buf("c", 40000), expectTag: 0x05 },
+    ];
+    for (const { label, data, expectTag } of cases) {
+        const c = compressSmart(data);
+        // New 2-byte magic header present.
+        assert.ok(isVersioned(c), `${label}: should be versioned (0xEC 0x01)`);
+        assert.equal(c[0], 0xec, `${label}: magic hi`);
+        assert.equal(c[1], 0x01, `${label}: magic lo (version)`);
+        assert.equal(c[2], 0x01, `${label}: format version 1`);
+        assert.equal(c[3], expectTag, `${label}: tier tag ${expectTag.toString(16)}`);
+        // Roundtrips exactly.
+        assert.deepEqual(decompressSmart(c), data, `${label}: roundtrip`);
+    }
+});
+test("large tier actually compresses better than raw (brotli 0x05)", () => {
+    const data = buf("this is a long summary of a coding session. ", 900);
+    const c = compressSmart(data);
+    assert.ok(c.length < data.length, "compressed smaller than raw");
+    assert.equal(c[3], 0x05, "tag is brotli-4");
+    assert.deepEqual(decompressSmart(c), data);
+});
+test("legacy untagged gzip (0x1f magic) still decompresses", () => {
+    const data = Buffer.from(JSON.stringify({ legacy: true }));
+    const legacyGzip = gzipSync(data); // no tag byte, starts with 0x1f
+    assert.equal(legacyGzip[0], 0x1f, "gzip magic present");
+    assert.equal(detectFormat(legacyGzip), "legacy-gzip");
+    assert.deepEqual(JSON.parse(decompressSmart(legacyGzip).toString()), { legacy: true });
+});
+test("legacy single-tag 0x03=brotli (the collision case) still decompresses", () => {
+    const data = buf("legacy brotli payload ", 200);
+    // Reconstruct the EXACT v0.1.0 legacy brotli frame: tag 0x03 + brotli payload.
+    const legacy = Buffer.concat([Buffer.from([0x03]), brotliCompressSync(data)]);
+    assert.equal(detectFormat(legacy), "legacy-tag");
+    assert.deepEqual(decompressSmart(legacy), data, "legacy 0x03 brotli roundtrips");
+});
+test("detectFormat classifies all eras", () => {
+    assert.equal(detectFormat(compressSmart(buf("q", 700))), "versioned");
+    assert.equal(detectFormat(gzipSync(buf("q", 10))), "legacy-gzip");
+    assert.equal(detectFormat(Buffer.from([0x00, 1, 2, 3])), "legacy-tag");
+    assert.equal(detectFormat(Buffer.from([0x99, 0x88])), "unknown");
+});
+test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
+    const data = buf("zstd dr export payload ", 1500);
+    const c = await compressZstd(data);
+    assert.ok(c.length < data.length, "zstd compresses");
+    // decompressSyncAuto reports zstd without throwing (caller awaits decompressZstd).
+    const auto = decompressSyncAuto(c);
+    assert.equal(auto.isZstd, true, "flagged as zstd");
+    assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
+});

package/dist/src/store/integrity.js

@@ -0,0 +1,44 @@
+/**
+ * integrity.ts — post-backfill / audit integrity checks (Sprint 10).
+ *
+ * Two checks (QA #1 / QA #14 spirit, re-mapped locally):
+ *   1. Sentinel vs recomputed: the `session_state.stored_region_hashes` set must
+ *      equal the set of `region_hash` values recomputed from `context_chunks`.
+ *      A mismatch flags a tampered / stale sentinel (so the dedup sentinel can't
+ *      miss a real duplicate).
+ *   2. Orphan id detection: any `injected_checkpoint_ids` entry that does not
+ *      correspond to a real `context_chunks.id` is orphaned and flagged.
+ *
+ * Pure read-only verification — never mutates; returns a structured report.
+ * SQLite is the source of truth; no network (PREVENT-PI-004).
+ */
+import { openStore, listCheckpoints, loadSessionState } from "./sqlite.js";
+import { getStateDir, normalizeSessionId } from "../store.js";
+/** Verify one session's sentinel set + injected-id integrity. */
+export function checkSessionIntegrity(sessionId, stateDir = getStateDir()) {
+    openStore(stateDir); // ensure schema is initialized for this state dir
+    const sid = normalizeSessionId(sessionId);
+    const state = loadSessionState(sessionId, stateDir);
+    const checkpoints = listCheckpoints(sessionId, stateDir);
+    // Recompute the region-hash set from the checkpoint rows (source of truth).
+    const recomputed = new Set(checkpoints.map((c) => c.regionHash).filter(Boolean));
+    const stored = new Set(state.storedRegionHashes);
+    const regionHashMismatch = recomputed.size !== stored.size || [...recomputed].some((h) => !stored.has(h));
+    // Orphan injected ids: referenced but no matching checkpoint.
+    const validIds = new Set(checkpoints.map((c) => c.checkpointId));
+    const orphanInjectedIds = state.injectedCheckpointIds.filter((id) => !validIds.has(id));
+    return {
+        sessionId: sid,
+        ok: !regionHashMismatch && orphanInjectedIds.length === 0,
+        storedRegionHashes: stored.size,
+        recomputedRegionHashes: recomputed.size,
+        regionHashMismatch,
+        orphanInjectedIds,
+    };
+}
+/** Check every session present in the store. */
+export function checkAllIntegrity(stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const rows = db.prepare("SELECT DISTINCT session_id FROM context_chunks").all();
+    return rows.map((r) => checkSessionIntegrity(r.session_id, stateDir));
+}

package/dist/src/store/migrate.js

@@ -0,0 +1,79 @@
+/**
+ * migrate.ts — Sprint 8: bring v0.1.0 JSON checkpoint files into SQLite.
+ *
+ * Reads every `<sessionId>.checkpoints.json.gz` in the state dir, computes the
+ * dedup-tier columns (content_hash / content_hash2 / normalized_text) that
+ * Sprints 9-12 match on, and upserts into context_chunks idempotently
+ * (ON CONFLICT id DO NOTHING — re-running is a no-op). The JSON files are kept
+ * as disaster-recovery snapshots; they are never deleted.
+ *
+ * Runs on first VectorStore construction (auto-migrate) and is also exposed for
+ * the integration test to call explicitly.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { getStateDir, readGzJson, normalizeSessionId } from "../store.js";
+import { openStore, upsertCheckpoint, listCheckpoints } from "./sqlite.js";
+/** Scan a state dir for v0.1.0 checkpoint JSON files. */
+function legacyCheckpointFiles(stateDir) {
+    if (!existsSync(stateDir))
+        return [];
+    return readdirSync(stateDir).filter((f) => f.endsWith(".checkpoints.json.gz"));
+}
+/** Derive the normalized text + two content hashes for the dedup tiers. */
+export function deriveContentHashes(cp) {
+    // L0/L1 basis: the human summary (whitespace-normalized).
+    const normalized = (cp.summary ?? "").replace(/\s+/g, " ").trim();
+    const contentHash = createHash("sha256").update(normalized).digest("hex");
+    // L2 basis: summary + extractive topic summary (catches paraphrased topics).
+    const basis2 = `${normalized}\n${cp.topicSummary ?? ""}`.trim();
+    const contentHash2 = createHash("sha256").update(basis2).digest("hex");
+    return { normalizedText: normalized, contentHash, contentHash2 };
+}
+/** Read a single legacy checkpoint file (lossless: returns every stored field). */
+export function readLegacyCheckpointFile(sessionId, stateDir = getStateDir()) {
+    const file = join(stateDir, `${normalizeSessionId(sessionId)}.checkpoints.json.gz`);
+    return readGzJson(file, []);
+}
+/**
+ * Migrate all legacy JSON checkpoint files in `stateDir` into SQLite.
+ * Idempotent — safe to call repeatedly. Does not delete JSON files.
+ */
+export function migrateJsonToSqlite(stateDir = getStateDir()) {
+    openStore(stateDir); // ensures schema exists
+    const files = legacyCheckpointFiles(stateDir);
+    let sessionsScanned = 0;
+    let migrated = 0;
+    let alreadyPresent = 0;
+    for (const file of files) {
+        // File name shape: <sessionId>.checkpoints.json.gz
+        const sessionId = file.replace(/\.checkpoints\.json\.gz$/, "");
+        const cps = readLegacyCheckpointFile(sessionId, stateDir);
+        if (cps.length === 0)
+            continue;
+        sessionsScanned++;
+        const existing = new Set(listCheckpoints(sessionId, stateDir).map((c) => c.checkpointId));
+        for (const cp of cps) {
+            if (existing.has(cp.checkpointId)) {
+                alreadyPresent++;
+                continue;
+            }
+            const { normalizedText, contentHash, contentHash2 } = deriveContentHashes(cp);
+            upsertCheckpoint({ ...cp, summary: cp.summary ?? "", regionHash: cp.regionHash ?? "" }, stateDir);
+            // Persist the extra dedup columns (upsertCheckpoint sets them null).
+            setContentHashes(cp.checkpointId, contentHash, contentHash2, normalizedText, stateDir);
+            migrated++;
+        }
+    }
+    return { sessionsScanned, checkpointsMigrated: migrated, alreadyPresent };
+}
+// Direct column update for the computed hashes (kept out of upsertCheckpoint's
+// hot path so the common write doesn't pay for hashing).
+function setContentHashes(checkpointId, contentHash, contentHash2, normalizedText, stateDir) {
+    const Database = openStore(stateDir);
+    Database.prepare(`UPDATE context_chunks
+       SET content_hash = @ch, content_hash2 = @ch2,
+           content_hash_version = 1, normalized_text = @nt
+     WHERE id = @id`).run({ id: checkpointId, ch: contentHash, ch2: contentHash2, nt: normalizedText });
+}

package/dist/src/store/migrate.test.js

@@ -0,0 +1,139 @@
+/**
+ * migrate + recall integration test — Sprint 8 acceptance proofs.
+ *
+ * 1. Migration is lossless: a v0.1.0 `<sess>.checkpoints.json.gz` roundtrips
+ *    into SQLite with checkpoint count + regionHash set identical, and the JSON
+ *    file is retained as a DR snapshot.
+ * 2. Cross-process recall: compact in one VectorStore, then recall via a FRESH
+ *    VectorStore over the SAME stateDir (re-opens the same sqlite.db file) — the
+ *    checkpoint must reappear. Mirrors Sprint 6.1's durability requirement.
+ *
+ * Uses MEGACOMPACT_STATE_DIR overrides; never the real user state dir.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../vectorStore.js";
+import { writeGzJson } from "../store.js";
+import { migrateJsonToSqlite, readLegacyCheckpointFile } from "../store/migrate.js";
+import { listCheckpoints, closeStore } from "../store/sqlite.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-migrate-"));
+let counter = 0;
+function stateDir() {
+    return join(baseTmp, `run-${counter++}`);
+}
+function msgVec() {
+    // Deterministic 8-dim vector.
+    return [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8];
+}
+function fakeCheckpoints(sessionId) {
+    return [
+        {
+            checkpointId: "chkpt_001",
+            sessionId,
+            summary: "Investigated the vector store and added a cosine helper.",
+            topicSummary: "Added cosine similarity helper to vector store.",
+            summaryHash: "a1b2c3d4e5f6a7b8",
+            keyDecisions: ["use linear scan"],
+            nextSteps: ["add tests"],
+            filesModified: ["src/vectorStore.ts"],
+            tokenEstimate: 1200,
+            regionHash: "r1",
+            embedding: msgVec(),
+            timestamp: 1,
+        },
+        {
+            checkpointId: "chkpt_002",
+            sessionId,
+            summary: "Refactored the recall path to dedupe against the window.",
+            topicSummary: "Recall dedup against injected set.",
+            summaryHash: "b2c3d4e5f6a7b8c9",
+            keyDecisions: [],
+            nextSteps: [],
+            filesModified: ["src/recall.ts"],
+            tokenEstimate: 900,
+            regionHash: "r2",
+            embedding: msgVec().map((v) => v + 0.01),
+            timestamp: 2,
+        },
+    ];
+}
+test("migration: v0.1.0 JSON checkpoints migrate losslessly into SQLite", () => {
+    const dir = stateDir();
+    const sid = "sess_migrate_lossless";
+    // Write a legacy JSON checkpoint file (the format v0.1.0 shipped).
+    writeGzJson(join(dir, `${sid}.checkpoints.json.gz`), fakeCheckpoints(sid));
+    const result = migrateJsonToSqlite(dir);
+    assert.equal(result.sessionsScanned, 1);
+    assert.equal(result.checkpointsMigrated, 2);
+    assert.equal(result.alreadyPresent, 0);
+    // SQLite now has both checkpoints, regionHash preserved.
+    const migrated = listCheckpoints(sid, dir);
+    assert.equal(migrated.length, 2, "both checkpoints present");
+    assert.ok(migrated.every((c) => c.regionHash && c.regionHash.length > 0), "regionHash preserved");
+    assert.deepEqual(migrated.map((c) => c.checkpointId).sort(), ["chkpt_001", "chkpt_002"]);
+    // content_hash columns populated (needed by Sprint 9).
+    const legacy = readLegacyCheckpointFile(sid, dir);
+    assert.equal(legacy.length, 2, "legacy file intact (DR snapshot retained)");
+    assert.ok(existsSync(join(dir, `${sid}.checkpoints.json.gz`)), "JSON DR snapshot retained");
+    closeStore(dir);
+});
+test("migration: re-running is idempotent (no duplicates)", () => {
+    const dir = stateDir();
+    const sid = "sess_migrate_idem";
+    writeGzJson(join(dir, `${sid}.checkpoints.json.gz`), fakeCheckpoints(sid));
+    migrateJsonToSqlite(dir);
+    const r2 = migrateJsonToSqlite(dir);
+    assert.equal(r2.checkpointsMigrated, 0, "nothing new migrated");
+    assert.equal(r2.alreadyPresent, 2, "both counted as already present");
+    assert.equal(listCheckpoints(sid, dir).length, 2);
+    closeStore(dir);
+});
+test("cross-process recall: fresh VectorStore over same dir recalls prior checkpoint", () => {
+    const dir = stateDir();
+    const sid = "sess_xproc";
+    // Process A: compact a checkpoint into the store.
+    const a = new VectorStore({ stateDir: dir });
+    const added = a.add({
+        sessionId: sid,
+        summary: "Cross-process recall proof: persisted in process A.",
+        topicSummary: "Persisted checkpoint in process A.",
+        regionText: "cross process recall proof session A write path",
+        tokenEstimate: 500,
+        timestamp: 1,
+    });
+    assert.equal(added.deduped, false);
+    assert.equal(added.checkpoint.checkpointId, "chkpt_001");
+    // Force a clean reopen (simulates a new process opening the same file).
+    closeStore(dir);
+    // Process B: brand-new VectorStore, same stateDir.
+    const b = new VectorStore({ stateDir: dir });
+    const hits = b.search(sid, "cross process recall proof", 5);
+    assert.equal(hits.length, 1, "checkpoint survives cross-process reopen");
+    assert.equal(hits[0].checkpoint.checkpointId, "chkpt_001");
+    assert.ok(hits[0].score > 0.5, "recall is relevant");
+    closeStore(dir);
+});
+test("cross-process recall: injected state persists across reopen", () => {
+    const dir = stateDir();
+    const sid = "sess_xproc_inj";
+    const a = new VectorStore({ stateDir: dir });
+    const added = a.add({
+        sessionId: sid,
+        summary: "A checkpoint to inject and remember across processes.",
+        topicSummary: "Injected checkpoint.",
+        regionText: "injected state persists across process reopen test",
+        timestamp: 1,
+    });
+    a.markInjected(sid, added.checkpoint.checkpointId);
+    assert.equal(a.wasInjected(sid, added.checkpoint.checkpointId), true);
+    closeStore(dir);
+    const b = new VectorStore({ stateDir: dir });
+    assert.equal(b.wasInjected(sid, added.checkpoint.checkpointId), true, "injection remembered");
+    closeStore(dir);
+});
+test("cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});