pi-mega-compact 0.4.20 → 0.4.23

package/dist/src/store/compression.js

@@ -5,7 +5,7 @@
  *
  *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
  *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
- *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *     node:sqlite replaced PGlite precisely to avoid an async cascade).
  *
  *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
  *     Optional, used for DR-export / large-blob paths where an await is fine.
@@ -25,7 +25,13 @@
  * decompressSmart detects the magic first, so all three eras roundtrip together.
  */
 import { gzipSync, gunzipSync, brotliCompressSync, brotliDecompressSync, constants as zlibConstants, } from "node:zlib";
-import zstd from "@mongodb-js/zstd";
+// zstd is loaded lazily (see compressZstdWithLevel / decompressZstd). It is an
+// OPTIONAL async DR-export dependency: its native addon (`zstd.node`) is not in
+// the npm tarball and may be absent on a clean/allowScripts-blocked install, so
+// a static import here would crash the whole extension at load time. Lazy
+// import keeps the extension loadable even when the binary is missing; the DR
+// path throws a clear error only if it is actually used. (Fix A.)
+// import zstd from "@mongodb-js/zstd";
 // --- Versioned format markers ----------------------------------------------
 const MAGIC_HI = 0xec;
 const MAGIC_LO = 0x01; // format version 1
@@ -45,6 +51,12 @@ const SIZE_MEDIUM = 32768;
 function header(ver, tag) {
     return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
 }
+/** Clamp a value to the [0, 1] range (pressure bands). */
+function clamp01(n) {
+    if (Number.isNaN(n))
+        return 0;
+    return n < 0 ? 0 : n > 1 ? 1 : n;
+}
 /**
  * Compress synchronously using the best zlib tier for the payload size.
  *
@@ -54,21 +66,36 @@ function header(ver, tag) {
  *   4KB–32KB → gzip level 6 (tag 0x02)
  *   > 32 KB  → brotli 4     (tag 0x05)
  *
- * Writes the versioned header so readers disambiguate from legacy blobs.
+ * `pressure` (0–1, optional) escalates the brotli quality for the large tier
+ * when the session is near its context limit — the "variable compression as we
+ * approach the limit" design (Fix E). Low/undefined pressure keeps brotli-4;
+ * high pressure pushes toward brotli-11. Stays fully synchronous (brotli-11 is
+ * sync via brotliCompressSync) so the sync `add()` contract is preserved; zstd
+ * is reserved for the async DR-export path only. Same versioned header/tags for
+ * every pressure, so decompressSmart is unaffected.
  */
-export function compressSmart(data) {
+export function compressSmart(data, pressure = 0) {
+    const p = clamp01(pressure);
     const len = data.length;
     if (len < SIZE_TINY) {
         return Buffer.concat([header(1, TAG_RAW), data]);
     }
     if (len < SIZE_SMALL) {
-        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+        // Small tier: escalate gzip level 1 → 9 with context pressure (Fix E) so
+        // the "variable compression as we approach the limit" dial bites for
+        // short sessions too, not just the >32KB brotli tier.
+        const level = Math.max(1, Math.min(9, Math.round(1 + 8 * p)));
+        return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level })]);
     }
     if (len < SIZE_MEDIUM) {
-        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+        // Medium tier: escalate gzip level 6 → 9 with context pressure (Fix E).
+        const level = Math.max(6, Math.min(9, Math.round(6 + 3 * p)));
+        return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level })]);
     }
+    // Large tier: escalate brotli quality 4 → 11 with context pressure (Fix E).
+    const quality = Math.max(4, Math.min(11, Math.round(4 + 7 * p)));
     const compressed = brotliCompressSync(data, {
-        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+        params: { [zlibConstants.BROTLI_PARAM_QUALITY]: quality },
     });
     return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
 }
@@ -141,6 +168,17 @@ export function decompressSmart(buf) {
 const ZSTD_MAGIC_HI = 0x5a; // 'Z'
 const ZSTD_MAGIC_LO = 0x53; // 'S'
 async function compressZstdWithLevel(data, level) {
+    // Lazy import: the native addon may be absent (clean/allowScripts install).
+    // Throws a clear, actionable error instead of a load-time crash.
+    let zstd;
+    try {
+        zstd = await import("@mongodb-js/zstd");
+    }
+    catch {
+        throw new Error("zstd is not available — the @mongodb-js/zstd native addon (zstd.node) " +
+            "was not built. Run the extension's native install step (or allow npm " +
+            "install scripts) to enable DR-export compression.");
+    }
     const compressed = await zstd.compress(data, level);
     return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
 }
@@ -163,6 +201,8 @@ export async function decompressZstd(buf) {
     if (!isZstd(buf)) {
         throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
     }
+    // Lazy import (see compressZstdWithLevel for rationale).
+    const zstd = await import("@mongodb-js/zstd");
     return zstd.decompress(buf.subarray(2));
 }
 /**

package/dist/src/store/compression.test.js

@@ -65,3 +65,51 @@ test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
     assert.equal(auto.isZstd, true, "flagged as zstd");
     assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
 });
+test("module loads without a top-level zstd import (Fix A: no load crash)", async () => {
+    // The extension must load even when the @mongodb-js/zstd native addon is
+    // absent (clean/allowScripts-blocked install). The dynamic import() lives
+    // inside the helpers, so importing this module must never throw.
+    const mod = await import("./compression.js");
+    assert.equal(typeof mod.compressSmart, "function", "compressSmart exported");
+    assert.equal(typeof mod.compressZstd, "function", "compressZstd exported");
+    // The real invariant: no STATIC `import ... from "@mongodb-js/zstd"` at the
+    // top level (that's what crashed the whole extension). zstd must be loaded
+    // lazily inside the helpers only. Check the source text.
+    const { readFileSync } = await import("node:fs");
+    const { join } = await import("node:path");
+    // Tests run with cwd at repo root (`node --test`), so resolve the source.
+    const src = readFileSync(join(process.cwd(), "src/store/compression.ts"), "utf-8");
+    const staticImport = /^import\s+.+\s+from\s+["']@mongodb-js\/zstd["'];?$/m;
+    assert.equal(staticImport.test(src), false, "no static top-level import of @mongodb-js/zstd (would crash load if binary absent)");
+    assert.ok(src.includes('await import("@mongodb-js/zstd")'), "zstd is loaded lazily via dynamic import() inside the helpers");
+});
+test("compressSmart escalates brotli quality with pressure (Fix E)", () => {
+    // Large (>32KB) payloads hit the brotli tier; higher pressure → brotli-11
+    // → smaller output than the default brotli-4, and still decodes.
+    const words = Array.from({ length: 6000 }, (_, i) => "word" + ((i * 2654435761) % 9973));
+    const big = Buffer.from(words.join(" "));
+    const low = compressSmart(big, 0);
+    const high = compressSmart(big, 1);
+    assert.equal(isVersioned(low), true, "versioned header preserved at p=0");
+    assert.equal(isVersioned(high), true, "versioned header preserved at p=1");
+    assert.ok(high.length < low.length, "high pressure compresses smaller");
+    assert.deepEqual(decompressSmart(low), big, "p=0 roundtrip");
+    assert.deepEqual(decompressSmart(high), big, "p=1 roundtrip");
+    // Small payloads ignore pressure (gzip tier) but still roundtrip.
+    const small = buf("hello world ", 300);
+    assert.deepEqual(decompressSmart(compressSmart(small, 1)), small, "small ignores pressure");
+    // pressure out of range is clamped (no throw, still versioned + decodable).
+    assert.deepEqual(decompressSmart(compressSmart(big, 5)), big, "over-pressure clamped");
+    assert.deepEqual(decompressSmart(compressSmart(big, -1)), big, "under-pressure clamped");
+});
+test("pressureFromPct + preserveRecentForPressure scale with context (Fix E)", async () => {
+    const { pressureFromPct, preserveRecentForPressure } = await import("../config.js");
+    assert.equal(pressureFromPct(50), 0.5, "pct→pressure");
+    assert.equal(pressureFromPct(null), 0, "null pct → 0");
+    assert.equal(pressureFromPct(150), 1, "pct clamped");
+    // low pressure keeps preserveRecent; high pressure compacts deeper (min floor).
+    assert.equal(preserveRecentForPressure(0, 4, 2), 4, "p=0 → preserveRecent");
+    assert.equal(preserveRecentForPressure(1, 4, 2), 2, "p=1 → preserveRecentMin");
+    assert.equal(preserveRecentForPressure(0.5, 4, 2), 3, "p=0.5 → interpolates");
+    assert.ok(preserveRecentForPressure(1, 4, 2) >= 2, "never below floor");
+});

package/dist/src/store/sqlite.js

@@ -2,11 +2,13 @@
  * sqlite.ts — Sprint 8 storage backbone (the "one store").
  *
  * Replaces the per-session gzipped-JSON checkpoint files with a single local
- * SQLite database (better-sqlite3, in-process, FS-backed, ZERO network calls —
- * honors PREVENT-PI-004). Chosen over PGlite because PGlite is async-only in
- * every published version, and VectorStore (engine.ts / recall.ts / the
- * extension) is fully synchronous — adopting PGlite would have cascaded async
- * through the whole call chain. SQLite keeps every VectorStore signature sync.
+ * SQLite database (node:sqlite — the Node built-in, in-process, FS-backed,
+ * ZERO network calls — honors PREVENT-PI-004). No native build and no install
+ * scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
+ * native binary could not be built under pi, which crashed every `pi update
+ * --extensions`). node:sqlite is synchronous, so every VectorStore signature
+ * stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
+ * real HNSW indexing (Slice 2 of the dual-backend plan).
  *
  * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
  * / pg_trgm-equivalent verification). The default cosine path stays a linear
@@ -14,7 +16,7 @@
  *
  * All queries are parameterized (PREVENT-002) — never string-concatenated.
  */
-import Database from "better-sqlite3";
+import { DatabaseSync } from "node:sqlite";
 import { existsSync, mkdirSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
@@ -28,14 +30,17 @@ function encodeEmbedding(v) {
         buf.writeFloatLE(v[i] ?? 0, i * 4);
     return buf;
 }
-/** Decode a Float32 BLOB back to a number[]. */
+/** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
+ *  Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass — both
+ *  work). */
 function decodeEmbedding(buf) {
     if (!buf || buf.length === 0)
         return [];
+    const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
     const n = buf.length / 4;
     const out = new Array(n);
     for (let i = 0; i < n; i++)
-        out[i] = buf.readFloatLE(i * 4);
+        out[i] = dv.getFloat32(i * 4, true);
     return out;
 }
 function jsonText(v) {
@@ -52,9 +57,9 @@ export function openStore(stateDir = getStateDir()) {
         return existing;
     if (!existsSync(stateDir))
         mkdirSync(stateDir, { recursive: true });
-    const db = new Database(join(stateDir, "sqlite.db"));
-    db.pragma("journal_mode = WAL");
-    db.pragma("foreign_keys = ON");
+    const db = new DatabaseSync(join(stateDir, "sqlite.db"));
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA foreign_keys = ON");
     initSchema(db);
     cache.set(stateDir, db);
     return db;
@@ -91,10 +96,10 @@ export function openIndexStore(indexDir = getIndexDir()) {
         return indexCache;
     if (!existsSync(indexDir))
         mkdirSync(indexDir, { recursive: true });
-    const db = new Database(join(indexDir, "index.sqlite"));
-    db.pragma("journal_mode = WAL");
-    db.pragma("busy_timeout = 3000"); // tolerate brief cross-process write contention
-    db.exec(`
+    const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
+    iddb.exec("PRAGMA journal_mode = WAL");
+    iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
+    iddb.exec(`
     CREATE TABLE IF NOT EXISTS repo_registry (
       repo_root                 TEXT PRIMARY KEY,
       display_name              TEXT,
@@ -114,9 +119,9 @@ export function openIndexStore(indexDir = getIndexDir()) {
     );
     CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
   `);
-    indexCache = db;
+    indexCache = iddb;
     indexCacheDir = indexDir;
-    return db;
+    return iddb;
 }
 /**
  * Upsert a repo's aggregate stats into the global index. Called on repo-switch
@@ -359,6 +364,20 @@ function initSchema(db) {
       ts            INTEGER
     );
 
+    -- Durable "save to memory" store (taken over from memory extensions).
+    -- One row per saved memory; scoped by repo so memory travels with the
+    -- clone. All params are parameterized (PREVENT-002).
+    CREATE TABLE IF NOT EXISTS memories (
+      id                INTEGER PRIMARY KEY AUTOINCREMENT,
+      repo              TEXT,
+      kind              TEXT DEFAULT 'note',   -- note | fact | decision | preference
+      content           TEXT NOT NULL,
+      tags              TEXT,                   -- JSON array of strings
+      created_at        INTEGER,
+      last_recalled_at  INTEGER
+    );
+    CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
+
     -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
     CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
       id UNINDEXED,
@@ -476,6 +495,69 @@ export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
     const now = Math.floor(Date.now() / 1000);
     db.prepare(`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
 }
+/** Save a memory to the current repo's store. Returns the new row id. */
+export function addMemory(memory, repo, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const now = Math.floor(Date.now() / 1000);
+    const res = db
+        .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
+       VALUES(?, ?, ?, ?, ?, NULL)`)
+        .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
+    return Number(res.lastInsertRowid);
+}
+/** List recent memories for a repo (or all repos when repo is null). */
+export function listMemories(repo, limit = 50, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const rows = repo
+        ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
+        : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
+    return rows.map(mapMemoryRow);
+}
+/** Substring search across content + tags. */
+export function searchMemories(query, repo = null, limit = 50, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const like = `%${query}%`;
+    const rows = repo
+        ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
+        : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
+    return rows.map(mapMemoryRow);
+}
+/** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
+export function recallMemory(id, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const now = Math.floor(Date.now() / 1000);
+    const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
+    return res.changes > 0;
+}
+function mapMemoryRow(row) {
+    return {
+        id: row.id,
+        repo: row.repo ?? null,
+        kind: row.kind ?? "note",
+        content: row.content ?? "",
+        tags: row.tags ? JSON.parse(row.tags) : [],
+        createdAt: row.created_at ?? 0,
+        lastRecalledAt: row.last_recalled_at ?? null,
+    };
+}
+/**
+ * Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
+ * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
+ * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
+ * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
+ */
+export function withTx(db, fn) {
+    db.exec("SAVEPOINT mc_tx");
+    try {
+        fn();
+        db.exec("RELEASE mc_tx");
+    }
+    catch (e) {
+        db.exec("ROLLBACK TO mc_tx");
+        db.exec("RELEASE mc_tx");
+        throw e;
+    }
+}
 /** Map a DB row to the public StoredCheckpoint shape. */
 function rowToCheckpoint(row) {
     return {
@@ -494,7 +576,9 @@ function rowToCheckpoint(row) {
         contentHash2: row.content_hash2 ?? undefined,
         contentHashVersion: row.content_hash_version ?? undefined,
         normalizedText: row.normalized_text ?? undefined,
-        compressedOriginal: row.compressed_original ?? undefined,
+        // node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
+        // (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
+        compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
         embedding: decodeEmbedding(row.embedding_blob),
         timestamp: Number(row.timestamp ?? 0),
         dedupStatus: row.dedup_status ?? undefined,
@@ -504,7 +588,7 @@ function rowToCheckpoint(row) {
 export function upsertCheckpoint(cp, stateDir = getStateDir()) {
     const db = openStore(stateDir);
     const sid = normalizeSessionId(cp.sessionId);
-    const tx = db.transaction(() => {
+    withTx(db, () => {
         db.prepare(`INSERT INTO context_chunks
         (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
          normalized_text, summary, topic_summary, summary_hash,
@@ -527,25 +611,25 @@ export function upsertCheckpoint(cp, stateDir = getStateDir()) {
          timestamp=excluded.timestamp,
          dedup_status=excluded.dedup_status,
          compressed_original=excluded.compressed_original`).run({
-            id: cp.checkpointId,
-            sid,
-            region_hash: cp.regionHash ?? null,
-            content_hash: cp.contentHash ?? null,
-            content_hash2: cp.contentHash2 ?? null,
-            content_hash_version: cp.contentHashVersion ?? null,
-            normalized_text: cp.normalizedText ?? null,
-            summary: cp.summary ?? "",
-            topic_summary: cp.topicSummary ?? null,
-            summary_hash: cp.summaryHash ?? null,
-            key_decisions: jsonText(cp.keyDecisions),
-            next_steps: jsonText(cp.nextSteps),
-            files_modified: jsonText(cp.filesModified),
-            embedding_blob: encodeEmbedding(cp.embedding ?? []),
-            token_estimate: cp.tokenEstimate ?? 0,
-            original_token_estimate: cp.originalTokenEstimate ?? null,
-            timestamp: cp.timestamp ?? 0,
-            dedup_status: "active",
-            compressed_original: cp.compressedOriginal ?? null,
+            "@id": cp.checkpointId,
+            "@sid": sid,
+            "@region_hash": cp.regionHash ?? null,
+            "@content_hash": cp.contentHash ?? null,
+            "@content_hash2": cp.contentHash2 ?? null,
+            "@content_hash_version": cp.contentHashVersion ?? null,
+            "@normalized_text": cp.normalizedText ?? null,
+            "@summary": cp.summary ?? "",
+            "@topic_summary": cp.topicSummary ?? null,
+            "@summary_hash": cp.summaryHash ?? null,
+            "@key_decisions": jsonText(cp.keyDecisions),
+            "@next_steps": jsonText(cp.nextSteps),
+            "@files_modified": jsonText(cp.filesModified),
+            "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
+            "@token_estimate": cp.tokenEstimate ?? 0,
+            "@original_token_estimate": cp.originalTokenEstimate ?? null,
+            "@timestamp": cp.timestamp ?? 0,
+            "@dedup_status": "active",
+            "@compressed_original": cp.compressedOriginal ?? null,
         });
         // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
         // Store normalized_text (the L1 verify key); fall back to summary for rows
@@ -553,7 +637,6 @@ export function upsertCheckpoint(cp, stateDir = getStateDir()) {
         db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
         db.prepare("INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)").run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
     });
-    tx();
 }
 // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
 /** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
@@ -571,12 +654,11 @@ export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKey
     const sid = normalizeSessionId(sessionId);
     const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
     const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
-    const tx = db.transaction(() => {
+    withTx(db, () => {
         del.run(chunkId);
         for (const key of bucketKeys)
             ins.run(key, chunkId, sid, signatureVersion);
     });
-    tx();
 }
 /**
  * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the

package/dist/src/store.test.js

@@ -109,6 +109,25 @@ test("compression tier: GZIP-1 and GZIP-6 tiers produce valid, smaller output",
     assert.ok(big.length < Buffer.from("compress me ".repeat(1800)).length, "GZIP-6 output smaller than input");
     assert.deepEqual(decompressSmart(big), Buffer.from("compress me ".repeat(1800)));
 });
+test("Fix E: pressure escalates gzip tier strength (sync, no zstd)", () => {
+    const small = Buffer.from("compress me ".repeat(200)); // 512B–4KB band
+    const medium = Buffer.from("compress me ".repeat(1800)); // 4KB–32KB band
+    // Low pressure → cheap levels (gzip-1 / gzip-6).
+    const lowSmall = compressSmart(small, 0);
+    const lowMedium = compressSmart(medium, 0);
+    assert.equal(lowSmall[3], 0x01, "small tier tag");
+    assert.equal(lowMedium[3], 0x02, "medium tier tag");
+    assert.ok(decompressSmart(lowSmall).equals(small), "low-pressure small roundtrips");
+    assert.ok(decompressSmart(lowMedium).equals(medium), "low-pressure medium roundtrips");
+    // High pressure → stronger levels (gzip-9 / gzip-9); tag unchanged, output
+    // must still decode to the exact original (versioned header preserved).
+    const highSmall = compressSmart(small, 1);
+    const highMedium = compressSmart(medium, 1);
+    assert.equal(highSmall[3], 0x01, "tag unchanged under pressure");
+    assert.equal(highMedium[3], 0x02, "tag unchanged under pressure");
+    assert.ok(decompressSmart(highSmall).equals(small), "high-pressure small roundtrips");
+    assert.ok(decompressSmart(highMedium).equals(medium), "high-pressure medium roundtrips");
+});
 // ---------------------------------------------------------------------------
 // normalizeSessionId
 // ---------------------------------------------------------------------------

package/dist/src/vectorStore.js

@@ -20,6 +20,8 @@ import { mmrRerank } from "./dedup/mmr.js";
 import { topK } from "./dedup/topk.js";
 import { openBloom, saveBloom } from "./store/bloom.js";
 import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
+import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
+import { stagedExpansion } from "./dedup/raptor/retrieval.js";
 import { migrateJsonToSqlite } from "./store/migrate.js";
 /** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
 export const L2_ENABLED = true;
@@ -242,7 +244,7 @@ export class VectorStore {
             contentHash2: digest.contentHash2,
             contentHashVersion: digest.contentHashVersion,
             normalizedText: digest.normalizedText,
-            compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8")),
+            compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8"), input.compressionPressure),
             embedding,
             timestamp: input.timestamp,
         };
@@ -339,6 +341,27 @@ export class VectorStore {
         // (Sprint 14 flag), returning the plain relevance-ranked window instead.
         if (!this.cfg.L2_ENABLED)
             return window.slice(0, k);
+        // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
+        // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
+        // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
+        // O(n) flat leaves, tightening the block at read time.
+        if (this.cfg.RAPTOR_ENABLED) {
+            const raptorHits = this.raptorSearchHits(sid, query, k);
+            if (raptorHits.length > 0) {
+                const merged = [...window];
+                for (const rh of raptorHits) {
+                    if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
+                        merged.push(rh);
+                    }
+                }
+                const mmrItems = merged.map((h) => ({
+                    item: h,
+                    vector: h.checkpoint.embedding,
+                    relevance: h.score,
+                }));
+                return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
+            }
+        }
         const mmrItems = window.map((h) => ({
             item: h,
             vector: h.checkpoint.embedding,
@@ -347,6 +370,38 @@ export class VectorStore {
         const ranked = mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
         return ranked;
     }
+    /**
+     * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
+     * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
+     * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
+     */
+    raptorSearchHits(sid, query, k) {
+        try {
+            const tree = rehydrateRaptorTree(sid, this.stateDir);
+            if (!tree || !tree.rootId)
+                return [];
+            const leafIds = stagedExpansion(query, tree, {
+                embedder: this.embedder,
+                k,
+                topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
+                mmrLambda: this.cfg.MMR_LAMBDA,
+            });
+            if (leafIds.length === 0)
+                return [];
+            const all = listCheckpoints(sid, this.stateDir).filter((cp) => cp.dedupStatus !== "removed");
+            const qv = this.embedder.embed(query);
+            const hits = [];
+            for (const id of leafIds) {
+                const cp = all.find((c) => c.checkpointId === id);
+                if (cp)
+                    hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
+            }
+            return hits;
+        }
+        catch {
+            return [];
+        }
+    }
     /**
      * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
      * lower-quality row of any pair scoring cosine > `threshold` as

package/extensions/DASHBOARD.md

@@ -2,8 +2,8 @@
 
 A lightweight local web dashboard for monitoring mega-compact's live state — compactions, context usage, checkpoints, and recall hits.
 
-Uses Node built-in modules (`http`, `fs`, `path`) plus `better-sqlite3` (the
-project's one-store DB backend) to read the machine-wide multi-repo index.
+Uses Node built-in modules (`http`, `fs`, `path`, `node:sqlite` — the project's
+one-store DB backend) to read the machine-wide multi-repo index.
 
 ## Quick Start
 
@@ -74,7 +74,7 @@ The server runs as a detached child process, independent of the pi session. It:
 - Auto-discovers the state directory from the `port.pid` file
 - Cleans up stale `port.pid` files from dead processes
 - Supports `SIGTERM`/`SIGINT` for graceful shutdown
-- Serves static HTML; reads the multi-repo index from SQLite (`better-sqlite3`)
+- Serves static HTML; reads the multi-repo index from SQLite (`node:sqlite`)
 
 ## Browser UI