pi-mega-compact 0.4.4 → 0.4.6

package/dist/src/store/sqlite.js

@@ -0,0 +1,574 @@
+/**
+ * sqlite.ts — Sprint 8 storage backbone (the "one store").
+ *
+ * Replaces the per-session gzipped-JSON checkpoint files with a single local
+ * SQLite database (better-sqlite3, in-process, FS-backed, ZERO network calls —
+ * honors PREVENT-PI-004). Chosen over PGlite because PGlite is async-only in
+ * every published version, and VectorStore (engine.ts / recall.ts / the
+ * extension) is fully synchronous — adopting PGlite would have cascaded async
+ * through the whole call chain. SQLite keeps every VectorStore signature sync.
+ *
+ * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
+ * / pg_trgm-equivalent verification). The default cosine path stays a linear
+ * scan over `embedding_blob` (checkpoint counts are small, no ANN index needed).
+ *
+ * All queries are parameterized (PREVENT-002) — never string-concatenated.
+ */
+import Database from "better-sqlite3";
+import { existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { getStateDir } from "../store.js";
+import { normalizeSessionId } from "../store.js";
+const SCHEMA_VERSION = 1;
+/** Encode a float vector as a little-endian Float32 BLOB for cosine scanning. */
+function encodeEmbedding(v) {
+    const buf = Buffer.allocUnsafe(v.length * 4);
+    for (let i = 0; i < v.length; i++)
+        buf.writeFloatLE(v[i] ?? 0, i * 4);
+    return buf;
+}
+/** Decode a Float32 BLOB back to a number[]. */
+function decodeEmbedding(buf) {
+    if (!buf || buf.length === 0)
+        return [];
+    const n = buf.length / 4;
+    const out = new Array(n);
+    for (let i = 0; i < n; i++)
+        out[i] = buf.readFloatLE(i * 4);
+    return out;
+}
+function jsonText(v) {
+    return JSON.stringify(v ?? []);
+}
+// In-process cache so the same stateDir reuses one connection (and so a fresh
+// VectorStore over the same dir shares the open DB). Cross-process durability
+// comes from reopening the same file path — proven by the integration test.
+const cache = new Map();
+/** Open (or reuse) the SQLite store for a state dir. */
+export function openStore(stateDir = getStateDir()) {
+    const existing = cache.get(stateDir);
+    if (existing)
+        return existing;
+    if (!existsSync(stateDir))
+        mkdirSync(stateDir, { recursive: true });
+    const db = new Database(join(stateDir, "sqlite.db"));
+    db.pragma("journal_mode = WAL");
+    db.pragma("foreign_keys = ON");
+    initSchema(db);
+    cache.set(stateDir, db);
+    return db;
+}
+function initSchema(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS context_chunks (
+      id                  TEXT NOT NULL,
+      session_id         TEXT NOT NULL,
+      region_hash        TEXT,
+      content_hash       TEXT,
+      content_hash2      TEXT,
+      content_hash_version INTEGER,
+      normalized_text    TEXT,
+      summary            TEXT,
+      topic_summary      TEXT,
+      summary_hash       TEXT,
+      key_decisions      TEXT,           -- JSON array
+      next_steps         TEXT,           -- JSON array
+      files_modified     TEXT,           -- JSON array
+      embedding_blob     BLOB,           -- float32 vector
+      token_estimate     INTEGER,
+      original_token_estimate INTEGER,    -- dropped region size (tokens saved = orig − stored)
+      timestamp          INTEGER,
+      dedup_status       TEXT DEFAULT 'active',
+      compressed_original BLOB           -- optional DR copy
+    );
+    -- Primary key is (session_id, id): checkpoint ids are unique per session
+    -- (chkpt_001 per session), not globally, so a bare id PK would collide
+    -- across sessions on the nextCheckpointId sequence.
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_chunks_pk
+      ON context_chunks(session_id, id);
+    CREATE INDEX IF NOT EXISTS idx_chunks_session ON context_chunks(session_id);
+    CREATE INDEX IF NOT EXISTS idx_chunks_region ON context_chunks(region_hash);
+    CREATE INDEX IF NOT EXISTS idx_chunks_content ON context_chunks(content_hash);
+    -- Partial UNIQUE (QA #1): null content_hash rows never violate the constraint;
+    -- ON CONFLICT DO NOTHING makes backfill + L0 inserts safe.
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_content_hash
+      ON context_chunks(session_id, content_hash) WHERE content_hash IS NOT NULL;
+
+    -- Sprint 11: MinHash signature + LSH bucket tables for L1 near-dup dedup.
+    CREATE TABLE IF NOT EXISTS minhash_signatures (
+      chunk_id          TEXT NOT NULL,
+      session_id        TEXT NOT NULL,
+      signature_version INTEGER NOT NULL,
+      signatures        TEXT NOT NULL,   -- JSON array of 256 uint32
+      PRIMARY KEY (chunk_id, signature_version)
+    );
+    CREATE INDEX IF NOT EXISTS idx_minhash_session ON minhash_signatures(session_id);
+
+    CREATE TABLE IF NOT EXISTS dedup_lsh_buckets (
+      bucket_key        TEXT NOT NULL,
+      chunk_id          TEXT NOT NULL,
+      session_id        TEXT NOT NULL,
+      signature_version INTEGER NOT NULL,
+      PRIMARY KEY (bucket_key, chunk_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_lsh_bucket ON dedup_lsh_buckets(bucket_key, session_id);
+
+    CREATE TABLE IF NOT EXISTS session_state (
+      session_id               TEXT PRIMARY KEY,
+      injected_checkpoint_ids TEXT,      -- JSON array
+      stored_region_hashes    TEXT       -- JSON array
+    );
+
+    CREATE TABLE IF NOT EXISTS meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT
+    );
+
+    -- Sprint 13 (RAPTOR): hierarchical summary tree nodes. children are a JSON
+    -- array of child node ids (or raw leaf ids at the bottom); embedding_blob
+    -- is the node centroid. Additive; retrieval ignores this table until
+    -- Sprint 14 promotes RAPTOR out of shadow mode.
+    CREATE TABLE IF NOT EXISTS raptor_nodes (
+      id            TEXT NOT NULL,
+      session_id   TEXT NOT NULL,
+      level        INTEGER NOT NULL,
+      parent_id    TEXT,
+      children     TEXT,           -- JSON array of child ids
+      summary      TEXT,
+      embedding_blob BLOB,         -- float32 centroid
+      quality_marker TEXT DEFAULT 'low',
+      token_estimate INTEGER,
+      PRIMARY KEY (session_id, id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_raptor_session ON raptor_nodes(session_id);
+    CREATE INDEX IF NOT EXISTS idx_raptor_parent ON raptor_nodes(parent_id);
+
+    -- Foundation for future features (resume sessions, daily log, lessons
+    -- learned). Scaffolded now so all store data lives in SQLite from day one;
+    -- population is minimal (touchSession / logDaily on compact) and the full
+    -- UI/recall for these lands in later sprints.
+
+    -- Per-session registry (resume + per-repo session history).
+    CREATE TABLE IF NOT EXISTS sessions (
+      session_id    TEXT PRIMARY KEY,
+      repo          TEXT,
+      started_at    INTEGER,
+      ended_at      INTEGER,
+      last_compacted_at INTEGER,
+      status        TEXT DEFAULT 'active'
+    );
+
+    -- Append-only daily activity log (the "daily log" feature seed).
+    CREATE TABLE IF NOT EXISTS daily_log (
+      id            INTEGER PRIMARY KEY AUTOINCREMENT,
+      day           TEXT NOT NULL,     -- YYYY-MM-DD
+      session_id    TEXT,
+      event         TEXT,             -- e.g. 'compact'
+      detail        TEXT,
+      tokens_saved  INTEGER DEFAULT 0,
+      ts            INTEGER
+    );
+    CREATE INDEX IF NOT EXISTS idx_daily_log_day ON daily_log(day);
+
+    -- Lessons learned (future recall/browse feature seed).
+    CREATE TABLE IF NOT EXISTS lessons (
+      id            INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id    TEXT,
+      repo          TEXT,
+      lesson        TEXT,
+      ts            INTEGER
+    );
+
+    -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
+    CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
+      id UNINDEXED,
+      normalized_text,
+      tokenize='trigram'
+    );
+  `);
+    // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
+    // pre-existing table, so new columns added to context_chunks after a store was
+    // first created (e.g. original_token_estimate in v0.4.2) must be ALTERed in for
+    // databases created by an older version — otherwise repoStats()/upsert crash
+    // with "no such column" and the extension fails to load. Additive only.
+    ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
+    const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
+    if (!v) {
+        db.prepare("INSERT INTO meta(key, value) VALUES(?, ?)").run("schema_version", String(SCHEMA_VERSION));
+    }
+}
+/**
+ * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
+ * exist. Idempotent: checks PRAGMA table_info first, so it is safe to run on
+ * every open. Table/column/decl are code-controlled constants (never user
+ * input), so the unavoidable identifier interpolation here does not violate
+ * PREVENT-002 (no external data reaches this SQL).
+ */
+function ensureColumn(db, table, column, decl) {
+    const cols = db.prepare(`PRAGMA table_info(${table})`).all();
+    if (cols.some((c) => c.name === column))
+        return;
+    db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
+}
+/** Read a string-valued meta key (or undefined). Used for cumulative counters. */
+export function getMeta(key, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
+    return row?.value;
+}
+/**
+ * Cumulative "tokens saved" — the sum of stored checkpoint token estimates across
+ * all compactions in this store (one per repo). Persisted in the SQLite `meta`
+ * table so it survives session restarts and travels with the repo's state dir,
+ * mirroring how `storageDedupRate` is cumulative. Incremented in VectorStore.add()
+ * when a new (non-deduped) checkpoint is persisted.
+ */
+export function getTokensSaved(stateDir = getStateDir()) {
+    const raw = getMeta("tokens_saved", stateDir);
+    const n = raw == null ? 0 : Number(raw);
+    return Number.isFinite(n) ? n : 0;
+}
+/** Add `delta` (>=0) to the cumulative tokens-saved counter. */
+export function addTokensSaved(delta, stateDir = getStateDir()) {
+    if (!(delta > 0))
+        return;
+    const db = openStore(stateDir);
+    db.prepare(`INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
+     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`).run(String(delta), delta);
+}
+/** Read a store-wide integer counter from the meta table (0 if absent). */
+export function getMetaNumber(key, stateDir = getStateDir()) {
+    const raw = getMeta(key, stateDir);
+    const n = raw == null ? 0 : Number(raw);
+    return Number.isFinite(n) ? n : 0;
+}
+/** Atomically add `delta` to an integer meta counter. */
+function incMeta(key, delta, stateDir = getStateDir()) {
+    if (!(delta > 0))
+        return;
+    const db = openStore(stateDir);
+    db.prepare(`INSERT INTO meta(key, value) VALUES(?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`).run(key, String(delta), delta);
+}
+/** Read the cumulative store-wide dedup counters. */
+export function getDedupStats(stateDir = getStateDir()) {
+    return {
+        attempts: getMetaNumber("dedup_attempts", stateDir),
+        deduped: getMetaNumber("deduped", stateDir),
+    };
+}
+/** Increment the store-wide dedup counters for one add() call. */
+export function bumpDedupStats(deduped, stateDir = getStateDir()) {
+    incMeta("dedup_attempts", 1, stateDir);
+    if (deduped)
+        incMeta("deduped", 1, stateDir);
+}
+// --- Future-feature foundation (resume sessions / daily log / lessons) -------
+// Scaffolded tables + minimal helpers so all store data lives in SQLite from
+// day one. Full UI/recall for these lands in later sprints.
+/** Upsert a `sessions` row (resume + per-repo session history). */
+export function touchSession(sessionId, repo, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const existing = db
+        .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
+        .get(sid);
+    const now = Math.floor(Date.now() / 1000);
+    if (!existing) {
+        db.prepare(`INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
+       VALUES(?, ?, ?, ?, 'active')`).run(sid, repo ?? null, now, now);
+    }
+    else {
+        db.prepare("UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?").run(now, repo ?? null, sid);
+    }
+}
+/** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
+export function logDaily(sessionId, event, detail, tokensSaved, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const day = new Date().toISOString().slice(0, 10);
+    const now = Math.floor(Date.now() / 1000);
+    db.prepare(`INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
+     VALUES(?, ?, ?, ?, ?, ?)`).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
+}
+/** Append a `lessons` entry (future lessons-learned browse/recall). */
+export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const now = Math.floor(Date.now() / 1000);
+    db.prepare(`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
+}
+/** Map a DB row to the public StoredCheckpoint shape. */
+function rowToCheckpoint(row) {
+    return {
+        checkpointId: row.id,
+        sessionId: row.session_id,
+        summary: row.summary ?? "",
+        topicSummary: row.topic_summary ?? undefined,
+        summaryHash: row.summary_hash ?? undefined,
+        keyDecisions: row.key_decisions ? JSON.parse(row.key_decisions) : [],
+        nextSteps: row.next_steps ? JSON.parse(row.next_steps) : [],
+        filesModified: row.files_modified ? JSON.parse(row.files_modified) : [],
+        tokenEstimate: row.token_estimate ?? 0,
+        originalTokenEstimate: row.original_token_estimate ?? undefined,
+        regionHash: row.region_hash ?? "",
+        contentHash: row.content_hash ?? undefined,
+        contentHash2: row.content_hash2 ?? undefined,
+        contentHashVersion: row.content_hash_version ?? undefined,
+        normalizedText: row.normalized_text ?? undefined,
+        compressedOriginal: row.compressed_original ?? undefined,
+        embedding: decodeEmbedding(row.embedding_blob),
+        timestamp: Number(row.timestamp ?? 0),
+        dedupStatus: row.dedup_status ?? undefined,
+    };
+}
+/** Insert or replace a checkpoint (idempotent by id). */
+export function upsertCheckpoint(cp, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(cp.sessionId);
+    const tx = db.transaction(() => {
+        db.prepare(`INSERT INTO context_chunks
+        (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
+         normalized_text, summary, topic_summary, summary_hash,
+         key_decisions, next_steps, files_modified, embedding_blob,
+         token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
+       VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
+               @normalized_text, @summary, @topic_summary, @summary_hash,
+               @key_decisions, @next_steps, @files_modified, @embedding_blob,
+               @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
+       ON CONFLICT(session_id, id) DO UPDATE SET
+         summary=excluded.summary,
+         topic_summary=excluded.topic_summary,
+         summary_hash=excluded.summary_hash,
+         key_decisions=excluded.key_decisions,
+         next_steps=excluded.next_steps,
+         files_modified=excluded.files_modified,
+         embedding_blob=excluded.embedding_blob,
+         token_estimate=excluded.token_estimate,
+         original_token_estimate=excluded.original_token_estimate,
+         timestamp=excluded.timestamp,
+         dedup_status=excluded.dedup_status,
+         compressed_original=excluded.compressed_original`).run({
+            id: cp.checkpointId,
+            sid,
+            region_hash: cp.regionHash ?? null,
+            content_hash: cp.contentHash ?? null,
+            content_hash2: cp.contentHash2 ?? null,
+            content_hash_version: cp.contentHashVersion ?? null,
+            normalized_text: cp.normalizedText ?? null,
+            summary: cp.summary ?? "",
+            topic_summary: cp.topicSummary ?? null,
+            summary_hash: cp.summaryHash ?? null,
+            key_decisions: jsonText(cp.keyDecisions),
+            next_steps: jsonText(cp.nextSteps),
+            files_modified: jsonText(cp.filesModified),
+            embedding_blob: encodeEmbedding(cp.embedding ?? []),
+            token_estimate: cp.tokenEstimate ?? 0,
+            original_token_estimate: cp.originalTokenEstimate ?? null,
+            timestamp: cp.timestamp ?? 0,
+            dedup_status: "active",
+            compressed_original: cp.compressedOriginal ?? null,
+        });
+        // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
+        // Store normalized_text (the L1 verify key); fall back to summary for rows
+        // that predate normalized_text population.
+        db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
+        db.prepare("INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)").run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
+    });
+    tx();
+}
+// --- Sprint 11: MinHash signatures + LSH buckets --------------------------
+/** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
+export function upsertMinhashSignature(chunkId, sessionId, signatureVersion, signatures, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    db.prepare(`INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
+     VALUES(?, ?, ?, ?)
+     ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
+       session_id=excluded.session_id, signatures=excluded.signatures`).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
+}
+/** Persist LSH bucket memberships for a chunk (one row per bucket key). */
+export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKeys, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
+    const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
+    const tx = db.transaction(() => {
+        del.run(chunkId);
+        for (const key of bucketKeys)
+            ins.run(key, chunkId, sid, signatureVersion);
+    });
+    tx();
+}
+/**
+ * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
+ * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
+ * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
+ */
+export function lshCandidateChunks(bucketKeys, sessionId, excludeChunkId, stateDir = getStateDir(), limit = 100) {
+    if (bucketKeys.length === 0)
+        return [];
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const placeholders = bucketKeys.map(() => "?").join(",");
+    const rows = db
+        .prepare(`SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
+       WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
+       LIMIT ?`)
+        .all(...bucketKeys, sid, excludeChunkId, limit);
+    return rows.map((r) => r.chunk_id);
+}
+/** All checkpoints for a session, sorted by id. */
+export function listCheckpoints(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const rows = db
+        .prepare("SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC")
+        .all(sid);
+    return rows.map(rowToCheckpoint);
+}
+/** Next sequential checkpoint id (chkpt_001 …) for a session. */
+export function nextCheckpointId(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const row = db
+        .prepare("SELECT MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS n FROM context_chunks WHERE session_id = ?")
+        .get(sid);
+    const next = (row.n ?? 0) + 1;
+    return `chkpt_${String(next).padStart(3, "0")}`;
+}
+/** True if a checkpoint id already exists for a session. */
+export function hasCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const row = db
+        .prepare("SELECT 1 FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
+        .get(normalizeSessionId(sessionId), checkpointId);
+    return row !== undefined;
+}
+/** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
+export function setDedupStatus(checkpointId, sessionId, status, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    db.prepare("UPDATE context_chunks SET dedup_status = ? WHERE id = ? AND session_id = ?").run(status, checkpointId, normalizeSessionId(sessionId));
+}
+// --- Session state (injection tracking) ------------------------------------
+function loadSessionStateRow(sid, db) {
+    const row = db.prepare("SELECT * FROM session_state WHERE session_id = ?").get(sid);
+    if (!row) {
+        return { injectedCheckpointIds: [], storedRegionHashes: [] };
+    }
+    return {
+        injectedCheckpointIds: row.injected_checkpoint_ids ? JSON.parse(row.injected_checkpoint_ids) : [],
+        storedRegionHashes: row.stored_region_hashes ? JSON.parse(row.stored_region_hashes) : [],
+    };
+}
+export function loadSessionState(sessionId, stateDir = getStateDir()) {
+    return loadSessionStateRow(normalizeSessionId(sessionId), openStore(stateDir));
+}
+export function saveSessionState(sessionId, state, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    db.prepare(`INSERT INTO session_state(session_id, injected_checkpoint_ids, stored_region_hashes)
+     VALUES(@sid, @inj, @reg)
+     ON CONFLICT(session_id) DO UPDATE SET
+       injected_checkpoint_ids=excluded.injected_checkpoint_ids,
+       stored_region_hashes=excluded.stored_region_hashes`).run({
+        sid,
+        inj: jsonText(state.injectedCheckpointIds),
+        reg: jsonText(state.storedRegionHashes),
+    });
+}
+export function storeStats(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const sid = normalizeSessionId(sessionId);
+    const row = db
+        .prepare(`SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
+              MAX(id) AS lastId
+       FROM context_chunks WHERE session_id = ?`)
+        .get(sid);
+    let lastSummary;
+    if (row.lastId) {
+        const s = db.prepare("SELECT summary FROM context_chunks WHERE id = ?").get(row.lastId);
+        lastSummary = s?.summary;
+    }
+    return {
+        checkpointCount: row.c,
+        totalTokenEstimate: row.tok,
+        lastCheckpointId: row.lastId ?? undefined,
+        lastSummary,
+    };
+}
+export function repoStats(stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const row = db
+        .prepare(`SELECT COUNT(*) AS c, COALESCE(SUM(token_estimate),0) AS tok,
+              COALESCE(SUM(original_token_estimate),0) AS orig,
+              COUNT(DISTINCT session_id) AS sessions
+       FROM context_chunks WHERE dedup_status != 'removed'`)
+        .get();
+    const ds = getDedupStats(stateDir);
+    return {
+        checkpointCount: row.c,
+        totalTokenEstimate: row.tok,
+        originalTokens: row.orig,
+        sessionCount: row.sessions,
+        tokensSaved: getMetaNumber("tokens_saved", stateDir),
+        dedupAttempts: ds.attempts,
+        dedupCollapsed: ds.deduped,
+        storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
+    };
+}
+/** Close and evict a cached connection (test teardown only). */
+export function closeStore(stateDir) {
+    const db = cache.get(stateDir);
+    if (db) {
+        db.close();
+        cache.delete(stateDir);
+    }
+}
+/** Persist a single RAPTOR node (upsert by (session_id, id)). */
+export function upsertRaptorNode(node, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    db.prepare(`INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate)
+     VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(session_id, id) DO UPDATE SET
+       level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
+       summary=excluded.summary, embedding_blob=excluded.embedding_blob,
+       quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate`).run(node.id, node.sessionId, node.level, node.parentId, jsonText(node.children), node.summary, encodeEmbedding(node.embedding), node.qualityMarker, node.tokenEstimate);
+}
+/** Persist an entire built RAPTOR tree for a session (shadow or live). */
+export function saveRaptorTree(sessionId, tree, stateDir = getStateDir()) {
+    for (const node of tree.nodes.values()) {
+        upsertRaptorNode({
+            id: node.id,
+            sessionId,
+            level: node.level,
+            parentId: node.parentId,
+            children: node.children,
+            summary: node.summary,
+            embedding: node.embedding,
+            qualityMarker: node.qualityMarker,
+            tokenEstimate: node.tokenEstimate,
+        }, stateDir);
+    }
+}
+/** Load all RAPTOR nodes for a session. */
+export function listRaptorNodes(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const rows = db
+        .prepare("SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC")
+        .all(normalizeSessionId(sessionId));
+    return rows.map((row) => ({
+        id: row.id,
+        sessionId: row.session_id,
+        level: row.level,
+        parentId: row.parent_id ?? null,
+        children: row.children ? JSON.parse(row.children) : [],
+        summary: row.summary ?? "",
+        embedding: decodeEmbedding(row.embedding_blob),
+        qualityMarker: row.quality_marker ?? "low",
+        tokenEstimate: row.token_estimate ?? 0,
+    }));
+}
+/** Delete all RAPTOR nodes for a session (rollback/cleanup). */
+export function clearRaptorNodes(sessionId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
+}

package/dist/src/store.js

@@ -0,0 +1,115 @@
+/**
+ * store.ts — persistence primitives for checkpoints + session state.
+ *
+ * Mirrors memory-mcp session_context.py: sessions normalize to `sess_xxx`,
+ * checkpoints are sequential `chkpt_001` per session. State lives under
+ * ~/.pi/agent/extensions/mega-compact/ as gzipped JSON.
+ */
+import { randomBytes } from "node:crypto";
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { compressSmart, decompressSmart } from "./store/compression.js";
+// Re-export the compression primitives from their extracted home so existing
+// imports (`import { compressSmart } from "./store.js"`) keep working.
+export { compressSmart, decompressSmart, compressZstd, compressZstdMax, decompressZstd, isVersioned, isZstd, detectFormat, decompressSyncAuto, } from "./store/compression.js";
+/**
+ * State directory. Read lazily (per call) so tests can redirect it via
+ * MEGACOMPACT_STATE_DIR without re-importing the module. Defaults to
+ * ~/.pi/agent/extensions/mega-compact/.
+ */
+export function getStateDir() {
+    return process.env.MEGACOMPACT_STATE_DIR ?? join(homedir(), ".pi", "agent", "extensions", "mega-compact");
+}
+/** Normalize an arbitrary session id to the `sess_xxx` form (port of memory-mcp). */
+export function normalizeSessionId(sessionId) {
+    if (!sessionId)
+        return `sess_${randomBytes(8).toString("hex")}`;
+    if (sessionId.startsWith("sess_"))
+        return sessionId;
+    if (sessionId.length >= 32 && sessionId.includes("-")) {
+        return `sess_${sessionId.replace(/-/g, "").slice(0, 16)}`;
+    }
+    return `sess_${sessionId}`;
+}
+// ---------------------------------------------------------------------------
+// JSON persistence (DR snapshots + migration source).
+// Compression lives in ./store/compression.ts; compressSmart/decompressSmart
+// are imported above and re-exported for backward-compatible call sites.
+// ---------------------------------------------------------------------------
+/**
+ * Read smart-compressed JSON, returning `fallback` on missing/corrupt file.
+ *
+ * Backward-compatible: detects legacy gzip files (magic byte 0x1f) and
+ * decompresses them correctly alongside new tagged files.
+ */
+export function readGzJson(path, fallback) {
+    try {
+        if (!existsSync(path))
+            return fallback;
+        const buf = readFileSync(path);
+        const out = decompressSmart(buf);
+        return JSON.parse(out.toString("utf-8"));
+    }
+    catch {
+        return fallback;
+    }
+}
+/** Write JSON with dynamic compression; creates parent dirs. */
+export function writeGzJson(path, data) {
+    mkdirSync(join(path, ".."), { recursive: true });
+    const jsonBuf = Buffer.from(JSON.stringify(data), "utf-8");
+    const compressed = compressSmart(jsonBuf);
+    writeFileSync(path, compressed);
+}
+/** Append a checkpoint to the per-session checkpoint file (gzipped). */
+export function appendCheckpoint(cp, stateDir = getStateDir()) {
+    const file = join(stateDir, `${cp.sessionId}.checkpoints.json.gz`);
+    const existing = readGzJson(file, []);
+    existing.push(cp);
+    writeGzJson(file, existing);
+}
+/** All checkpoints for a session (across branches). */
+export function listCheckpoints(sessionId, stateDir = getStateDir()) {
+    const file = join(stateDir, `${normalizeSessionId(sessionId)}.checkpoints.json.gz`);
+    return readGzJson(file, []);
+}
+/**
+ * Rewrite ALL checkpoints for a session (in-place update).
+ *
+ * Used by VectorStore when summaryHash or contentSimilarity dedup updates an
+ * existing checkpoint's timestamp/metadata instead of creating a new one.
+ */
+export function rewriteCheckpoints(sessionId, checkpoints, stateDir = getStateDir()) {
+    const file = join(stateDir, `${normalizeSessionId(sessionId)}.checkpoints.json.gz`);
+    writeGzJson(file, checkpoints);
+}
+/**
+ * Generate the next sequential checkpoint id for a session (chkpt_001 ...).
+ */
+export function nextCheckpointId(sessionId, stateDir = getStateDir()) {
+    const list = listCheckpoints(sessionId, stateDir);
+    const max = list.reduce((m, c) => {
+        const n = parseInt(c.checkpointId.replace("chkpt_", ""), 10);
+        return Number.isFinite(n) && n > m ? n : m;
+    }, 0);
+    return `chkpt_${String(max + 1).padStart(3, "0")}`;
+}
+/** Load mutable session state (created on demand). */
+export function loadSessionState(sessionId, stateDir = getStateDir()) {
+    const file = join(stateDir, `${normalizeSessionId(sessionId)}.state.json.gz`);
+    return readGzJson(file, {
+        injectedCheckpointIds: [],
+        storedRegionHashes: [],
+    });
+}
+export function saveSessionState(sessionId, state, stateDir = getStateDir()) {
+    const file = join(stateDir, `${normalizeSessionId(sessionId)}.state.json.gz`);
+    writeGzJson(file, state);
+}
+/**
+ * Cumulative store-wide dedup accounting now lives in the SQLite `meta` table
+ * (see store/sqlite.ts: getDedupStats / bumpDedupStats). All store stats are
+ * SQLite-backed so they survive session restarts and travel with the repo's
+ * state dir. The legacy JSON `dedup-stats.json` path was removed.
+ */