pi-mega-compact 0.4.21 → 0.4.23

package/dist/src/recall.js

@@ -14,6 +14,8 @@
  * extension decides where it lands.
  */
 import { recall as searchRecall } from "./engine.js";
+import { estimateBlockTokens } from "./tokens.js";
+import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
 /** Wrap a recall block so the model reads it as restored compacted context. */
 export function formatRecallBlock(hits) {
     if (hits.length === 0)
@@ -38,18 +40,42 @@ export function formatRecallBlock(hits) {
 export function recallAndInline(opts, store) {
     const limit = opts.limit ?? 3;
     const skip = opts.skipInjected ?? true;
+    const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)
+    const doWindowDedupe = opts.windowDedupe ?? false;
+    const dedupSim = opts.dedupSim ?? 0.9;
     const { hits } = searchRecall({ sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false }, store);
-    // Shared dedup: drop checkpoints already injected this session, then mark the
-    // survivors so repeated triggers are free. (Cosine near-dup collapse already
-    // happened inside store.search.)
+    // Precompute live-window embeddings once for inline dedupe (Fix C). Trigram
+    // embedder is local + cheap; never a network call (PREVENT-PI-004).
+    let liveEmbeddings = [];
+    if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+        const embedder = defaultEmbedder();
+        liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+    }
+    // Shared dedup + bounded/inline block assembly. We build the block
+    // incrementally so the token cap can stop mid-stream (Fix C).
     const toInject = [];
+    const parts = [];
+    let blockTokens = 0;
     for (const h of hits) {
         if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
             continue;
+        // Inline dedupe: skip a hit already resident in the live window (Fix C).
+        if (doWindowDedupe && liveEmbeddings.length > 0) {
+            const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+            if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
+                continue;
+        }
+        const part = formatRecallBlock([h]);
+        const partTokens = estimateBlockTokens(part);
+        // Token cap: never push a chunk that would overrun the ceiling.
+        if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
+            break;
+        parts.push(part);
         toInject.push(h);
+        blockTokens += partTokens;
         store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
     }
-    const block = formatRecallBlock(toInject);
+    const block = parts.join("\n");
     const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
     return {
         toInject,

package/dist/src/recall.test.js

@@ -45,6 +45,34 @@ test("recallAndInline empty when store has nothing for query", () => {
     assert.equal(r.empty, true);
     assert.equal(r.block, "");
 });
+test("Fix C: recallMaxTokens caps the injected block", () => {
+    const s = store();
+    // Three distinct checkpoints so we can observe the cap bite mid-stream.
+    compactSession({ sessionId: SESS, messages: [msg("user", "alpha module wiring and bootstrap sequence"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "beta module config and env resolution"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "gamma module shutdown and cleanup hooks"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 3 }, s);
+    // A ceiling of 100 tokens fits the first checkpoint (~82) but stops before the
+    // second (~163 cumulative) — proving the cap bites mid-stream.
+    const r = recallAndInline({ sessionId: SESS, query: "module wiring config shutdown", limit: 5, source: "command", recallMaxTokens: 100, skipInjected: false }, s);
+    assert.ok(r.toInject.length >= 1, "at least one injected under the cap");
+    assert.ok(r.toInject.length < 3, "cap prevented all three from injecting");
+    assert.ok(r.block.length > 0, "block non-empty");
+});
+test("Fix C: inline dedupe drops a hit already resident in the live window", () => {
+    const s = store();
+    const resident = "alpha module wiring and bootstrap sequence";
+    compactSession({ sessionId: SESS, messages: [msg("user", resident), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "omega module telemetry and tracing spans"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+    // Baseline: with dedupe OFF, both checkpoints are candidates.
+    const rNoDedup = recallAndInline({ sessionId: SESS, query: "module wiring telemetry", limit: 5, source: "command", skipInjected: false }, s);
+    // The live window contains the exact summary of the first checkpoint — as it
+    // would be if a prior recall already injected it. Inline dedupe must drop it
+    // (strictly fewer injected than the no-dedupe baseline).
+    const residentSummary = rNoDedup.toInject[0].checkpoint.summary;
+    const rDedup = recallAndInline({ sessionId: SESS, query: "module wiring telemetry", limit: 5, source: "command", skipInjected: false, windowDedupe: true, liveWindow: [residentSummary], dedupSim: 0.9 }, s);
+    assert.ok(rDedup.toInject.length <= rNoDedup.toInject.length, "dedupe never adds hits");
+    assert.ok(rDedup.toInject.length < rNoDedup.toInject.length, "inline dedupe dropped a resident hit");
+});
 test("cleanup", () => {
     rmSync(baseTmp, { recursive: true, force: true });
 });

package/dist/src/store/backfill.js

@@ -19,7 +19,7 @@ import { openStore } from "./sqlite.js";
 import { computeContentDigest } from "../dedup/digest.js";
 import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
 import { lshBands } from "../dedup/l1-lsh.js";
-import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree } from "./sqlite.js";
+import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree, withTx } from "./sqlite.js";
 import { buildRaptorTree } from "../dedup/raptor/tree.js";
 import { defaultEmbedder } from "../embedder.js";
 import { getStateDir } from "../store.js";
@@ -64,7 +64,7 @@ export function backfillContentHashes(stateDir = getStateDir()) {
     let processed = 0;
     let lastSid = start.lastSid;
     let lastId = start.lastId;
-    const tx = db.transaction((rows) => {
+    function applyRows(rows) {
         const lookup = db.prepare("SELECT id FROM context_chunks WHERE session_id = ? AND content_hash = ? AND content_hash2 = ? AND id != ? LIMIT 1");
         const update = db.prepare(`UPDATE context_chunks
        SET content_hash=?, content_hash2=?, content_hash_version=?, normalized_text=?,
@@ -87,9 +87,9 @@ export function backfillContentHashes(stateDir = getStateDir()) {
             lastId = row.id;
             processed++;
         }
-    });
+    }
     if (pending.length > 0) {
-        tx(pending);
+        withTx(db, () => applyRows(pending));
         db.prepare("INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved) VALUES('content_hashes',?,?,?,?) ON CONFLICT(name) DO UPDATE SET last_session_id=excluded.last_session_id, last_id=excluded.last_id, updated=excluded.updated, duplicates_resolved=excluded.duplicates_resolved").run(lastSid, lastId, updated, duplicatesResolved);
     }
     if (THROTTLE_MS > 0) {
@@ -135,7 +135,7 @@ export function backfillPhase(phase, sessionId, stateDir, opts = {}) {
     let cursor = lastId ?? undefined;
     for (let i = Math.max(0, startIndex); i < all.length; i += batchSize) {
         const batch = all.slice(i, i + batchSize);
-        const tx = db.transaction(() => {
+        withTx(db, () => {
             for (const cp of batch) {
                 const sig = minhashSignature(cp.normalizedText ?? cp.summary ?? "");
                 if (sig.length === NUM_HASHES) {
@@ -148,7 +148,6 @@ export function backfillPhase(phase, sessionId, stateDir, opts = {}) {
                 processed++;
             }
         });
-        tx();
         savePhaseCursor(db, phase, cursor ?? null, processed);
         batches++;
         if (THROTTLE_MS > 0) {

package/dist/src/store/compression.js

@@ -5,7 +5,7 @@
  *
  *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
  *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
- *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *     node:sqlite replaced PGlite precisely to avoid an async cascade).
  *
  *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
  *     Optional, used for DR-export / large-blob paths where an await is fine.
@@ -25,7 +25,13 @@
  * decompressSmart detects the magic first, so all three eras roundtrip together.
  */
 import { gzipSync, gunzipSync, brotliCompressSync, brotliDecompressSync, constants as zlibConstants, } from "node:zlib";
-import zstd from "@mongodb-js/zstd";
+// zstd is loaded lazily (see compressZstdWithLevel / decompressZstd). It is an
+// OPTIONAL async DR-export dependency: its native addon (`zstd.node`) is not in
+// the npm tarball and may be absent on a clean/allowScripts-blocked install, so
+// a static import here would crash the whole extension at load time. Lazy
+// import keeps the extension loadable even when the binary is missing; the DR
+// path throws a clear error only if it is actually used. (Fix A.)
+// import zstd from "@mongodb-js/zstd";
 // --- Versioned format markers ----------------------------------------------
 const MAGIC_HI = 0xec;
 const MAGIC_LO = 0x01; // format version 1
@@ -45,6 +51,12 @@ const SIZE_MEDIUM = 32768;
 function header(ver, tag) {
     return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
 }
+/** Clamp a value to the [0, 1] range (pressure bands). */
+function clamp01(n) {
+    if (Number.isNaN(n))
+        return 0;
+    return n < 0 ? 0 : n > 1 ? 1 : n;
+}
 /**
  * Compress synchronously using the best zlib tier for the payload size.
  *
@@ -54,21 +66,36 @@ function header(ver, tag) {
  *   4KB–32KB → gzip level 6 (tag 0x02)
  *   > 32 KB  → brotli 4     (tag 0x05)
  *
- * Writes the versioned header so readers disambiguate from legacy blobs.
+ * `pressure` (0–1, optional) escalates the brotli quality for the large tier
+ * when the session is near its context limit — the "variable compression as we
+ * approach the limit" design (Fix E). Low/undefined pressure keeps brotli-4;
+ * high pressure pushes toward brotli-11. Stays fully synchronous (brotli-11 is
+ * sync via brotliCompressSync) so the sync `add()` contract is preserved; zstd
+ * is reserved for the async DR-export path only. Same versioned header/tags for
+ * every pressure, so decompressSmart is unaffected.
  */
-export function compressSmart(data) {
+export function compressSmart(data, pressure = 0) {
+    const p = clamp01(pressure);
     const len = data.length;
     if (len < SIZE_TINY) {
         return Buffer.concat([header(1, TAG_RAW), data]);
     }
     if (len < SIZE_SMALL) {
-        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+        // Small tier: escalate gzip level 1 → 9 with context pressure (Fix E) so
+        // the "variable compression as we approach the limit" dial bites for
+        // short sessions too, not just the >32KB brotli tier.
+        const level = Math.max(1, Math.min(9, Math.round(1 + 8 * p)));
+        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level })]);
     }
     if (len < SIZE_MEDIUM) {
-        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+        // Medium tier: escalate gzip level 6 → 9 with context pressure (Fix E).
+        const level = Math.max(6, Math.min(9, Math.round(6 + 3 * p)));
+        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level })]);
     }
+    // Large tier: escalate brotli quality 4 → 11 with context pressure (Fix E).
+    const quality = Math.max(4, Math.min(11, Math.round(4 + 7 * p)));
     const compressed = brotliCompressSync(data, {
-        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: quality },
     });
     return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
 }
@@ -141,6 +168,17 @@ export function decompressSmart(buf) {
 const ZSTD_MAGIC_HI = 0x5a; // 'Z'
 const ZSTD_MAGIC_LO = 0x53; // 'S'
 async function compressZstdWithLevel(data, level) {
+    // Lazy import: the native addon may be absent (clean/allowScripts install).
+    // Throws a clear, actionable error instead of a load-time crash.
+    let zstd;
+    try {
+        zstd = await import("@mongodb-js/zstd");
+    }
+    catch {
+        throw new Error("zstd is not available — the @mongodb-js/zstd native addon (zstd.node) " +
+            "was not built. Run the extension's native install step (or allow npm " +
+            "install scripts) to enable DR-export compression.");
+    }
     const compressed = await zstd.compress(data, level);
     return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
 }
@@ -163,6 +201,8 @@ export async function decompressZstd(buf) {
     if (!isZstd(buf)) {
         throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
     }
+    // Lazy import (see compressZstdWithLevel for rationale).
+    const zstd = await import("@mongodb-js/zstd");
     return zstd.decompress(buf.subarray(2));
 }
 /**

package/dist/src/store/compression.test.js

@@ -65,3 +65,51 @@ test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
     assert.equal(auto.isZstd, true, "flagged as zstd");
     assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
 });
+test("module loads without a top-level zstd import (Fix A: no load crash)", async () => {
+    // The extension must load even when the @mongodb-js/zstd native addon is
+    // absent (clean/allowScripts-blocked install). The dynamic import() lives
+    // inside the helpers, so importing this module must never throw.
+    const mod = await import("./compression.js");
+    assert.equal(typeof mod.compressSmart, "function", "compressSmart exported");
+    assert.equal(typeof mod.compressZstd, "function", "compressZstd exported");
+    // The real invariant: no STATIC `import ... from "@mongodb-js/zstd"` at the
+    // top level (that's what crashed the whole extension). zstd must be loaded
+    // lazily inside the helpers only. Check the source text.
+    const { readFileSync } = await import("node:fs");
+    const { join } = await import("node:path");
+    // Tests run with cwd at repo root (`node --test`), so resolve the source.
+    const src = readFileSync(join(process.cwd(), "src/store/compression.ts"), "utf-8");
+    const staticImport = /^import\s+.+\s+from\s+["']@mongodb-js\/zstd["'];?$/m;
+    assert.equal(staticImport.test(src), false, "no static top-level import of @mongodb-js/zstd (would crash load if binary absent)");
+    assert.ok(src.includes('await import("@mongodb-js/zstd")'), "zstd is loaded lazily via dynamic import() inside the helpers");
+});
+test("compressSmart escalates brotli quality with pressure (Fix E)", () => {
+    // Large (>32KB) payloads hit the brotli tier; higher pressure → brotli-11
+    // → smaller output than the default brotli-4, and still decodes.
+    const words = Array.from({ length: 6000 }, (_, i) => "word" + ((i * 2654435761) % 9973));
+    const big = Buffer.from(words.join(" "));
+    const low = compressSmart(big, 0);
+    const high = compressSmart(big, 1);
+    assert.equal(isVersioned(low), true, "versioned header preserved at p=0");
+    assert.equal(isVersioned(high), true, "versioned header preserved at p=1");
+    assert.ok(high.length < low.length, "high pressure compresses smaller");
+    assert.deepEqual(decompressSmart(low), big, "p=0 roundtrip");
+    assert.deepEqual(decompressSmart(high), big, "p=1 roundtrip");
+    // Small payloads ignore pressure (gzip tier) but still roundtrip.
+    const small = buf("hello world ", 300);
+    assert.deepEqual(decompressSmart(compressSmart(small, 1)), small, "small ignores pressure");
+    // pressure out of range is clamped (no throw, still versioned + decodable).
+    assert.deepEqual(decompressSmart(compressSmart(big, 5)), big, "over-pressure clamped");
+    assert.deepEqual(decompressSmart(compressSmart(big, -1)), big, "under-pressure clamped");
+});
+test("pressureFromPct + preserveRecentForPressure scale with context (Fix E)", async () => {
+    const { pressureFromPct, preserveRecentForPressure } = await import("../config.js");
+    assert.equal(pressureFromPct(50), 0.5, "pct→pressure");
+    assert.equal(pressureFromPct(null), 0, "null pct → 0");
+    assert.equal(pressureFromPct(150), 1, "pct clamped");
+    // low pressure keeps preserveRecent; high pressure compacts deeper (min floor).
+    assert.equal(preserveRecentForPressure(0, 4, 2), 4, "p=0 → preserveRecent");
+    assert.equal(preserveRecentForPressure(1, 4, 2), 2, "p=1 → preserveRecentMin");
+    assert.equal(preserveRecentForPressure(0.5, 4, 2), 3, "p=0.5 → interpolates");
+    assert.ok(preserveRecentForPressure(1, 4, 2) >= 2, "never below floor");
+});

package/dist/src/store/sqlite.js

@@ -2,11 +2,13 @@
  * sqlite.ts — Sprint 8 storage backbone (the "one store").
  *
  * Replaces the per-session gzipped-JSON checkpoint files with a single local
- * SQLite database (better-sqlite3, in-process, FS-backed, ZERO network calls —
- * honors PREVENT-PI-004). Chosen over PGlite because PGlite is async-only in
- * every published version, and VectorStore (engine.ts / recall.ts / the
- * extension) is fully synchronous — adopting PGlite would have cascaded async
- * through the whole call chain. SQLite keeps every VectorStore signature sync.
+ * SQLite database (node:sqlite — the Node built-in, in-process, FS-backed,
+ * ZERO network calls — honors PREVENT-PI-004). No native build and no install
+ * scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
+ * native binary could not be built under pi, which crashed every `pi update
+ * --extensions`). node:sqlite is synchronous, so every VectorStore signature
+ * stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
+ * real HNSW indexing (Slice 2 of the dual-backend plan).
  *
  * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
  * / pg_trgm-equivalent verification). The default cosine path stays a linear
@@ -14,7 +16,7 @@
  *
  * All queries are parameterized (PREVENT-002) — never string-concatenated.
  */
-import Database from "better-sqlite3";
+import { DatabaseSync } from "node:sqlite";
 import { existsSync, mkdirSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
@@ -28,14 +30,17 @@ function encodeEmbedding(v) {
         buf.writeFloatLE(v[i] ?? 0, i * 4);
     return buf;
 }
-/** Decode a Float32 BLOB back to a number[]. */
+/** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
+ *  Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass — both
+ *  work). */
 function decodeEmbedding(buf) {
     if (!buf || buf.length === 0)
         return [];
+    const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
     const n = buf.length / 4;
     const out = new Array(n);
     for (let i = 0; i < n; i++)
-        out[i] = buf.readFloatLE(i * 4);
+        out[i] = dv.getFloat32(i * 4, true);
     return out;
 }
 function jsonText(v) {
@@ -52,9 +57,9 @@ export function openStore(stateDir = getStateDir()) {
         return existing;
     if (!existsSync(stateDir))
         mkdirSync(stateDir, { recursive: true });
-    const db = new Database(join(stateDir, "sqlite.db"));
-    db.pragma("journal_mode = WAL");
-    db.pragma("foreign_keys = ON");
+    const db = new DatabaseSync(join(stateDir, "sqlite.db"));
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA foreign_keys = ON");
     initSchema(db);
     cache.set(stateDir, db);
     return db;
@@ -91,10 +96,10 @@ export function openIndexStore(indexDir = getIndexDir()) {
         return indexCache;
     if (!existsSync(indexDir))
         mkdirSync(indexDir, { recursive: true });
-    const db = new Database(join(indexDir, "index.sqlite"));
-    db.pragma("journal_mode = WAL");
-    db.pragma("busy_timeout = 3000"); // tolerate brief cross-process write contention
-    db.exec(`
+    const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
+    iddb.exec("PRAGMA journal_mode = WAL");
+    iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
+    iddb.exec(`
     CREATE TABLE IF NOT EXISTS repo_registry (
       repo_root                 TEXT PRIMARY KEY,
       display_name              TEXT,
@@ -114,9 +119,9 @@ export function openIndexStore(indexDir = getIndexDir()) {
     );
     CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
   `);
-    indexCache = db;
+    indexCache = iddb;
     indexCacheDir = indexDir;
-    return db;
+    return iddb;
 }
 /**
  * Upsert a repo's aggregate stats into the global index. Called on repo-switch
@@ -535,6 +540,24 @@ function mapMemoryRow(row) {
         lastRecalledAt: row.last_recalled_at ?? null,
     };
 }
+/**
+ * Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
+ * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
+ * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
+ * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
+ */
+export function withTx(db, fn) {
+    db.exec("SAVEPOINT mc_tx");
+    try {
+        fn();
+        db.exec("RELEASE mc_tx");
+    }
+    catch (e) {
+        db.exec("ROLLBACK TO mc_tx");
+        db.exec("RELEASE mc_tx");
+        throw e;
+    }
+}
 /** Map a DB row to the public StoredCheckpoint shape. */
 function rowToCheckpoint(row) {
     return {
@@ -553,7 +576,9 @@ function rowToCheckpoint(row) {
         contentHash2: row.content_hash2 ?? undefined,
         contentHashVersion: row.content_hash_version ?? undefined,
         normalizedText: row.normalized_text ?? undefined,
-        compressedOriginal: row.compressed_original ?? undefined,
+        // node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
+        // (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
+        compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
         embedding: decodeEmbedding(row.embedding_blob),
         timestamp: Number(row.timestamp ?? 0),
         dedupStatus: row.dedup_status ?? undefined,
@@ -563,7 +588,7 @@ function rowToCheckpoint(row) {
 export function upsertCheckpoint(cp, stateDir = getStateDir()) {
     const db = openStore(stateDir);
     const sid = normalizeSessionId(cp.sessionId);
-    const tx = db.transaction(() => {
+    withTx(db, () => {
         db.prepare(`INSERT INTO context_chunks
         (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
          normalized_text, summary, topic_summary, summary_hash,
@@ -586,25 +611,25 @@ export function upsertCheckpoint(cp, stateDir = getStateDir()) {
          timestamp=excluded.timestamp,
          dedup_status=excluded.dedup_status,
          compressed_original=excluded.compressed_original`).run({
-            id: cp.checkpointId,
-            sid,
-            region_hash: cp.regionHash ?? null,
-            content_hash: cp.contentHash ?? null,
-            content_hash2: cp.contentHash2 ?? null,
-            content_hash_version: cp.contentHashVersion ?? null,
-            normalized_text: cp.normalizedText ?? null,
-            summary: cp.summary ?? "",
-            topic_summary: cp.topicSummary ?? null,
-            summary_hash: cp.summaryHash ?? null,
-            key_decisions: jsonText(cp.keyDecisions),
-            next_steps: jsonText(cp.nextSteps),
-            files_modified: jsonText(cp.filesModified),
-            embedding_blob: encodeEmbedding(cp.embedding ?? []),
-            token_estimate: cp.tokenEstimate ?? 0,
-            original_token_estimate: cp.originalTokenEstimate ?? null,
-            timestamp: cp.timestamp ?? 0,
-            dedup_status: "active",
-            compressed_original: cp.compressedOriginal ?? null,
+            "@id": cp.checkpointId,
+            "@sid": sid,
+            "@region_hash": cp.regionHash ?? null,
+            "@content_hash": cp.contentHash ?? null,
+            "@content_hash2": cp.contentHash2 ?? null,
+            "@content_hash_version": cp.contentHashVersion ?? null,
+            "@normalized_text": cp.normalizedText ?? null,
+            "@summary": cp.summary ?? "",
+            "@topic_summary": cp.topicSummary ?? null,
+            "@summary_hash": cp.summaryHash ?? null,
+            "@key_decisions": jsonText(cp.keyDecisions),
+            "@next_steps": jsonText(cp.nextSteps),
+            "@files_modified": jsonText(cp.filesModified),
+            "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
+            "@token_estimate": cp.tokenEstimate ?? 0,
+            "@original_token_estimate": cp.originalTokenEstimate ?? null,
+            "@timestamp": cp.timestamp ?? 0,
+            "@dedup_status": "active",
+            "@compressed_original": cp.compressedOriginal ?? null,
         });
         // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
         // Store normalized_text (the L1 verify key); fall back to summary for rows
@@ -612,7 +637,6 @@ export function upsertCheckpoint(cp, stateDir = getStateDir()) {
         db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
         db.prepare("INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)").run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
     });
-    tx();
 }
 // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
 /** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
@@ -630,12 +654,11 @@ export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKey
     const sid = normalizeSessionId(sessionId);
     const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
     const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
-    const tx = db.transaction(() => {
+    withTx(db, () => {
         del.run(chunkId);
         for (const key of bucketKeys)
             ins.run(key, chunkId, sid, signatureVersion);
     });
-    tx();
 }
 /**
  * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the

package/dist/src/store.test.js

@@ -109,6 +109,25 @@ test("compression tier: GZIP-1 and GZIP-6 tiers produce valid, smaller output",
     assert.ok(big.length < Buffer.from("compress me ".repeat(1800)).length, "GZIP-6 output smaller than input");
     assert.deepEqual(decompressSmart(big), Buffer.from("compress me ".repeat(1800)));
 });
+test("Fix E: pressure escalates gzip tier strength (sync, no zstd)", () => {
+    const small = Buffer.from("compress me ".repeat(200)); // 512B–4KB band
+    const medium = Buffer.from("compress me ".repeat(1800)); // 4KB–32KB band
+    // Low pressure → cheap levels (gzip-1 / gzip-6).
+    const lowSmall = compressSmart(small, 0);
+    const lowMedium = compressSmart(medium, 0);
+    assert.equal(lowSmall[3], 0x01, "small tier tag");
+    assert.equal(lowMedium[3], 0x02, "medium tier tag");
+    assert.ok(decompressSmart(lowSmall).equals(small), "low-pressure small roundtrips");
+    assert.ok(decompressSmart(lowMedium).equals(medium), "low-pressure medium roundtrips");
+    // High pressure → stronger levels (gzip-9 / gzip-9); tag unchanged, output
+    // must still decode to the exact original (versioned header preserved).
+    const highSmall = compressSmart(small, 1);
+    const highMedium = compressSmart(medium, 1);
+    assert.equal(highSmall[3], 0x01, "tag unchanged under pressure");
+    assert.equal(highMedium[3], 0x02, "tag unchanged under pressure");
+    assert.ok(decompressSmart(highSmall).equals(small), "high-pressure small roundtrips");
+    assert.ok(decompressSmart(highMedium).equals(medium), "high-pressure medium roundtrips");
+});
 // ---------------------------------------------------------------------------
 // normalizeSessionId
 // ---------------------------------------------------------------------------

package/dist/src/vectorStore.js

@@ -20,6 +20,8 @@ import { mmrRerank } from "./dedup/mmr.js";
 import { topK } from "./dedup/topk.js";
 import { openBloom, saveBloom } from "./store/bloom.js";
 import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
+import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
+import { stagedExpansion } from "./dedup/raptor/retrieval.js";
 import { migrateJsonToSqlite } from "./store/migrate.js";
 /** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
 export const L2_ENABLED = true;
@@ -242,7 +244,7 @@ export class VectorStore {
             contentHash2: digest.contentHash2,
             contentHashVersion: digest.contentHashVersion,
             normalizedText: digest.normalizedText,
-            compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8")),
+            compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8"), input.compressionPressure),
             embedding,
             timestamp: input.timestamp,
         };
@@ -339,6 +341,27 @@ export class VectorStore {
         // (Sprint 14 flag), returning the plain relevance-ranked window instead.
         if (!this.cfg.L2_ENABLED)
             return window.slice(0, k);
+        // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
+        // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
+        // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
+        // O(n) flat leaves, tightening the block at read time.
+        if (this.cfg.RAPTOR_ENABLED) {
+            const raptorHits = this.raptorSearchHits(sid, query, k);
+            if (raptorHits.length > 0) {
+                const merged = [...window];
+                for (const rh of raptorHits) {
+                    if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
+                        merged.push(rh);
+                    }
+                }
+                const mmrItems = merged.map((h) => ({
+                    item: h,
+                    vector: h.checkpoint.embedding,
+                    relevance: h.score,
+                }));
+                return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
+            }
+        }
         const mmrItems = window.map((h) => ({
             item: h,
             vector: h.checkpoint.embedding,
@@ -347,6 +370,38 @@ export class VectorStore {
         const ranked = mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
         return ranked;
     }
+    /**
+     * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
+     * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
+     * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
+     */
+    raptorSearchHits(sid, query, k) {
+        try {
+            const tree = rehydrateRaptorTree(sid, this.stateDir);
+            if (!tree || !tree.rootId)
+                return [];
+            const leafIds = stagedExpansion(query, tree, {
+                embedder: this.embedder,
+                k,
+                topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
+                mmrLambda: this.cfg.MMR_LAMBDA,
+            });
+            if (leafIds.length === 0)
+                return [];
+            const all = listCheckpoints(sid, this.stateDir).filter((cp) => cp.dedupStatus !== "removed");
+            const qv = this.embedder.embed(query);
+            const hits = [];
+            for (const id of leafIds) {
+                const cp = all.find((c) => c.checkpointId === id);
+                if (cp)
+                    hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
+            }
+            return hits;
+        }
+        catch {
+            return [];
+        }
+    }
     /**
      * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
      * lower-quality row of any pair scoring cosine > `threshold` as

package/extensions/DASHBOARD.md

@@ -2,8 +2,8 @@
 
 A lightweight local web dashboard for monitoring mega-compact's live state — compactions, context usage, checkpoints, and recall hits.
 
-Uses Node built-in modules (`http`, `fs`, `path`) plus `better-sqlite3` (the
-project's one-store DB backend) to read the machine-wide multi-repo index.
+Uses Node built-in modules (`http`, `fs`, `path`, `node:sqlite` — the project's
+one-store DB backend) to read the machine-wide multi-repo index.
 
 ## Quick Start
 
@@ -74,7 +74,7 @@ The server runs as a detached child process, independent of the pi session. It:
 - Auto-discovers the state directory from the `port.pid` file
 - Cleans up stale `port.pid` files from dead processes
 - Supports `SIGTERM`/`SIGINT` for graceful shutdown
-- Serves static HTML; reads the multi-repo index from SQLite (`better-sqlite3`)
+- Serves static HTML; reads the multi-repo index from SQLite (`node:sqlite`)
 
 ## Browser UI