pi-mega-compact 0.4.20 → 0.4.23

package/src/recall.ts

@@ -16,6 +16,8 @@
 
 import { recall as searchRecall } from "./engine.js";
 import type { SearchHit, VectorStore } from "./vectorStore.js";
+import { estimateBlockTokens } from "./tokens.js";
+import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
 
 export type RecallSource = "resume" | "command" | "sentinel";
 
@@ -26,6 +28,16 @@ export interface RecallInjectOptions {
   source: RecallSource;
   /** Skip checkpoints already injected this session (recall dedup). */
   skipInjected?: boolean;
+  /** Token ceiling for the re-injected block (Fix C). Recall stops adding once
+   *  the block would exceed this, so the read path can never net-inflate. */
+  recallMaxTokens?: number;
+  /** Inline-dedupe hits against the live window (Fix C): drop a hit whose
+   *  summary is ≥ `dedupSim` similar to a live message. */
+  windowDedupe?: boolean;
+  /** Live window text (from the session manager) used for inline dedupe. */
+  liveWindow?: string[];
+  /** Similarity threshold for inline dedupe (defaults to 0.9). */
+  dedupSim?: number;
 }
 
 export interface RecallInjectResult {
@@ -70,23 +82,50 @@ export function recallAndInline(
 ): RecallInjectResult {
   const limit = opts.limit ?? 3;
   const skip = opts.skipInjected ?? true;
+  const maxTokens = opts.recallMaxTokens ?? 0; // 0 = unbounded (legacy behavior)
+  const doWindowDedupe = opts.windowDedupe ?? false;
+  const dedupSim = opts.dedupSim ?? 0.9;
 
   const { hits } = searchRecall(
     { sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false },
     store as VectorStore,
   );
 
-  // Shared dedup: drop checkpoints already injected this session, then mark the
-  // survivors so repeated triggers are free. (Cosine near-dup collapse already
-  // happened inside store.search.)
+  // Precompute live-window embeddings once for inline dedupe (Fix C). Trigram
+  // embedder is local + cheap; never a network call (PREVENT-PI-004).
+  let liveEmbeddings: number[][] = [];
+  if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+    const embedder = defaultEmbedder();
+    liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+  }
+
+  // Shared dedup + bounded/inline block assembly. We build the block
+  // incrementally so the token cap can stop mid-stream (Fix C).
   const toInject: SearchHit[] = [];
+  const parts: string[] = [];
+  let blockTokens = 0;
+
   for (const h of hits) {
     if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
+
+    // Inline dedupe: skip a hit already resident in the live window (Fix C).
+    if (doWindowDedupe && liveEmbeddings.length > 0) {
+      const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+      if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
+    }
+
+    const part = formatRecallBlock([h]);
+    const partTokens = estimateBlockTokens(part);
+    // Token cap: never push a chunk that would overrun the ceiling.
+    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+
+    parts.push(part);
     toInject.push(h);
+    blockTokens += partTokens;
     store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
   }
 
-  const block = formatRecallBlock(toInject);
+  const block = parts.join("\n");
   const report = toInject.map(
     (h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
   );

package/src/store/backfill.ts

@@ -16,12 +16,12 @@
  * SQLite is the source of truth; this touches no network (PREVENT-PI-004).
  */
 
-import type { Database } from "better-sqlite3";
+import type { DatabaseSync } from "node:sqlite";
 import { openStore } from "./sqlite.js";
 import { computeContentDigest } from "../dedup/digest.js";
 import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
 import { lshBands } from "../dedup/l1-lsh.js";
-import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree } from "./sqlite.js";
+import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree, withTx } from "./sqlite.js";
 import { buildRaptorTree, type Leaf } from "../dedup/raptor/tree.js";
 import type { Embedder } from "../embedder.js";
 import { defaultEmbedder } from "../embedder.js";
@@ -45,7 +45,7 @@ interface PhaseProgressRow {
   processed: number;
 }
 
-function ensureProgressTable(db: Database): void {
+function ensureProgressTable(db: DatabaseSync): void {
   db.exec(`
     CREATE TABLE IF NOT EXISTS backfill_progress (
       name TEXT PRIMARY KEY,
@@ -57,7 +57,7 @@ function ensureProgressTable(db: Database): void {
   `);
 }
 
-function progress(db: Database): { lastSid: string | null; lastId: string | null; updated: number; dups: number } {
+function progress(db: DatabaseSync): { lastSid: string | null; lastId: string | null; updated: number; dups: number } {
   const row = db
     .prepare("SELECT last_session_id, last_id, updated, duplicates_resolved FROM backfill_progress WHERE name='content_hashes'")
     .get() as { last_session_id: string | null; last_id: string | null; updated: number; duplicates_resolved: number } | undefined;
@@ -91,7 +91,7 @@ export function backfillContentHashes(stateDir: string = getStateDir()): Backfil
   let lastSid = start.lastSid;
   let lastId = start.lastId;
 
-  const tx = db.transaction((rows: { id: string; session_id: string; summary: string }[]) => {
+  function applyRows(rows: { id: string; session_id: string; summary: string }[]): void {
     const lookup = db.prepare(
       "SELECT id FROM context_chunks WHERE session_id = ? AND content_hash = ? AND content_hash2 = ? AND id != ? LIMIT 1",
     );
@@ -123,10 +123,10 @@ export function backfillContentHashes(stateDir: string = getStateDir()): Backfil
       lastId = row.id;
       processed++;
     }
-  });
+  }
 
   if (pending.length > 0) {
-    tx(pending);
+    withTx(db, () => applyRows(pending));
     db.prepare(
       "INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved) VALUES('content_hashes',?,?,?,?) ON CONFLICT(name) DO UPDATE SET last_session_id=excluded.last_session_id, last_id=excluded.last_id, updated=excluded.updated, duplicates_resolved=excluded.duplicates_resolved",
     ).run(lastSid, lastId, updated, duplicatesResolved);
@@ -150,7 +150,7 @@ export function isBackfillComplete(stateDir: string = getStateDir()): boolean {
 
 // ---- Sprint 14: L1 / L2 / RAPTOR phase backfill (resumable) ---------------
 
-function phaseCursor(db: Database, phase: BackfillPhase): { lastId: string | null; processed: number } {
+function phaseCursor(db: DatabaseSync, phase: BackfillPhase): { lastId: string | null; processed: number } {
   ensureProgressTable(db);
   const row = db
     .prepare("SELECT last_id, updated AS processed FROM backfill_progress WHERE name = ?")
@@ -158,7 +158,7 @@ function phaseCursor(db: Database, phase: BackfillPhase): { lastId: string | nul
   return { lastId: row?.last_id ?? null, processed: row?.processed ?? 0 };
 }
 
-function savePhaseCursor(db: Database, phase: BackfillPhase, lastId: string | null, processed: number): void {
+function savePhaseCursor(db: DatabaseSync, phase: BackfillPhase, lastId: string | null, processed: number): void {
   db.prepare(
     `INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved)
      VALUES(?, NULL, ?, ?, 0)
@@ -201,7 +201,7 @@ export function backfillPhase(
 
   for (let i = Math.max(0, startIndex); i < all.length; i += batchSize) {
     const batch = all.slice(i, i + batchSize);
-    const tx = db.transaction(() => {
+    withTx(db, () => {
       for (const cp of batch) {
         const sig = minhashSignature(cp.normalizedText ?? cp.summary ?? "");
         if (sig.length === NUM_HASHES) {
@@ -217,7 +217,6 @@ export function backfillPhase(
         processed++;
       }
     });
-    tx();
     savePhaseCursor(db, phase, cursor ?? null, processed);
     batches++;
     if (THROTTLE_MS > 0) { const end = Date.now() + THROTTLE_MS; while (Date.now() < end) { /* throttle */ } }

package/src/store/compression.test.ts

@@ -81,3 +81,61 @@ test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
   assert.equal(auto.isZstd, true, "flagged as zstd");
   assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
 });
+
+test("module loads without a top-level zstd import (Fix A: no load crash)", async () => {
+  // The extension must load even when the @mongodb-js/zstd native addon is
+  // absent (clean/allowScripts-blocked install). The dynamic import() lives
+  // inside the helpers, so importing this module must never throw.
+  const mod = await import("./compression.js");
+  assert.equal(typeof mod.compressSmart, "function", "compressSmart exported");
+  assert.equal(typeof mod.compressZstd, "function", "compressZstd exported");
+  // The real invariant: no STATIC `import ... from "@mongodb-js/zstd"` at the
+  // top level (that's what crashed the whole extension). zstd must be loaded
+  // lazily inside the helpers only. Check the source text.
+  const { readFileSync } = await import("node:fs");
+  const { join } = await import("node:path");
+  // Tests run with cwd at repo root (`node --test`), so resolve the source.
+  const src = readFileSync(join(process.cwd(), "src/store/compression.ts"), "utf-8");
+  const staticImport = /^import\s+.+\s+from\s+["']@mongodb-js\/zstd["'];?$/m;
+  assert.equal(
+    staticImport.test(src),
+    false,
+    "no static top-level import of @mongodb-js/zstd (would crash load if binary absent)",
+  );
+  assert.ok(
+    src.includes('await import("@mongodb-js/zstd")'),
+    "zstd is loaded lazily via dynamic import() inside the helpers",
+  );
+});
+
+test("compressSmart escalates brotli quality with pressure (Fix E)", () => {
+  // Large (>32KB) payloads hit the brotli tier; higher pressure → brotli-11
+  // → smaller output than the default brotli-4, and still decodes.
+  const words = Array.from({ length: 6000 }, (_, i) => "word" + ((i * 2654435761) % 9973));
+  const big = Buffer.from(words.join(" "));
+  const low = compressSmart(big, 0);
+  const high = compressSmart(big, 1);
+  assert.equal(isVersioned(low), true, "versioned header preserved at p=0");
+  assert.equal(isVersioned(high), true, "versioned header preserved at p=1");
+  assert.ok(high.length < low.length, "high pressure compresses smaller");
+  assert.deepEqual(decompressSmart(low), big, "p=0 roundtrip");
+  assert.deepEqual(decompressSmart(high), big, "p=1 roundtrip");
+  // Small payloads ignore pressure (gzip tier) but still roundtrip.
+  const small = buf("hello world ", 300);
+  assert.deepEqual(decompressSmart(compressSmart(small, 1)), small, "small ignores pressure");
+  // pressure out of range is clamped (no throw, still versioned + decodable).
+  assert.deepEqual(decompressSmart(compressSmart(big, 5)), big, "over-pressure clamped");
+  assert.deepEqual(decompressSmart(compressSmart(big, -1)), big, "under-pressure clamped");
+});
+
+test("pressureFromPct + preserveRecentForPressure scale with context (Fix E)", async () => {
+  const { pressureFromPct, preserveRecentForPressure } = await import("../config.js");
+  assert.equal(pressureFromPct(50), 0.5, "pct→pressure");
+  assert.equal(pressureFromPct(null), 0, "null pct → 0");
+  assert.equal(pressureFromPct(150), 1, "pct clamped");
+  // low pressure keeps preserveRecent; high pressure compacts deeper (min floor).
+  assert.equal(preserveRecentForPressure(0, 4, 2), 4, "p=0 → preserveRecent");
+  assert.equal(preserveRecentForPressure(1, 4, 2), 2, "p=1 → preserveRecentMin");
+  assert.equal(preserveRecentForPressure(0.5, 4, 2), 3, "p=0.5 → interpolates");
+  assert.ok(preserveRecentForPressure(1, 4, 2) >= 2, "never below floor");
+});

package/src/store/compression.ts

@@ -5,7 +5,7 @@
  *
  *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
  *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
- *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *     node:sqlite replaced PGlite precisely to avoid an async cascade).
  *
  *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
  *     Optional, used for DR-export / large-blob paths where an await is fine.
@@ -32,7 +32,13 @@ import {
   brotliDecompressSync,
   constants as zlibConstants,
 } from "node:zlib";
-import zstd from "@mongodb-js/zstd";
+// zstd is loaded lazily (see compressZstdWithLevel / decompressZstd). It is an
+// OPTIONAL async DR-export dependency: its native addon (`zstd.node`) is not in
+// the npm tarball and may be absent on a clean/allowScripts-blocked install, so
+// a static import here would crash the whole extension at load time. Lazy
+// import keeps the extension loadable even when the binary is missing; the DR
+// path throws a clear error only if it is actually used. (Fix A.)
+// import zstd from "@mongodb-js/zstd";
 
 // --- Versioned format markers ----------------------------------------------
 const MAGIC_HI = 0xec;
@@ -59,6 +65,12 @@ function header(ver: number, tag: number): Buffer {
   return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
 }
 
+/** Clamp a value to the [0, 1] range (pressure bands). */
+function clamp01(n: number): number {
+  if (Number.isNaN(n)) return 0;
+  return n < 0 ? 0 : n > 1 ? 1 : n;
+}
+
 /**
  * Compress synchronously using the best zlib tier for the payload size.
  *
@@ -68,21 +80,36 @@ function header(ver: number, tag: number): Buffer {
  *   4KB–32KB → gzip level 6 (tag 0x02)
  *   > 32 KB  → brotli 4     (tag 0x05)
  *
- * Writes the versioned header so readers disambiguate from legacy blobs.
+ * `pressure` (0–1, optional) escalates the brotli quality for the large tier
+ * when the session is near its context limit — the "variable compression as we
+ * approach the limit" design (Fix E). Low/undefined pressure keeps brotli-4;
+ * high pressure pushes toward brotli-11. Stays fully synchronous (brotli-11 is
+ * sync via brotliCompressSync) so the sync `add()` contract is preserved; zstd
+ * is reserved for the async DR-export path only. Same versioned header/tags for
+ * every pressure, so decompressSmart is unaffected.
  */
-export function compressSmart(data: Buffer): Buffer {
+export function compressSmart(data: Buffer, pressure = 0): Buffer {
+  const p = clamp01(pressure);
   const len = data.length;
   if (len < SIZE_TINY) {
     return Buffer.concat([header(1, TAG_RAW), data]);
   }
   if (len < SIZE_SMALL) {
-    return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+    // Small tier: escalate gzip level 1 → 9 with context pressure (Fix E) so
+    // the "variable compression as we approach the limit" dial bites for
+    // short sessions too, not just the >32KB brotli tier.
+    const level = Math.max(1, Math.min(9, Math.round(1 + 8 * p)));
+    return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level })]);
   }
   if (len < SIZE_MEDIUM) {
-    return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+    // Medium tier: escalate gzip level 6 → 9 with context pressure (Fix E).
+    const level = Math.max(6, Math.min(9, Math.round(6 + 3 * p)));
+    return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level })]);
   }
+  // Large tier: escalate brotli quality 4 → 11 with context pressure (Fix E).
+  const quality = Math.max(4, Math.min(11, Math.round(4 + 7 * p)));
   const compressed = brotliCompressSync(data, {
-    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: quality },
   });
   return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
 }
@@ -164,6 +191,18 @@ const ZSTD_MAGIC_HI = 0x5a; // 'Z'
 const ZSTD_MAGIC_LO = 0x53; // 'S'
 
 async function compressZstdWithLevel(data: Buffer, level: number): Promise<Buffer> {
+  // Lazy import: the native addon may be absent (clean/allowScripts install).
+  // Throws a clear, actionable error instead of a load-time crash.
+  let zstd: typeof import("@mongodb-js/zstd");
+  try {
+    zstd = await import("@mongodb-js/zstd");
+  } catch {
+    throw new Error(
+      "zstd is not available — the @mongodb-js/zstd native addon (zstd.node) " +
+        "was not built. Run the extension's native install step (or allow npm " +
+        "install scripts) to enable DR-export compression.",
+    );
+  }
   const compressed = await zstd.compress(data, level);
   return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
 }
@@ -189,6 +228,8 @@ export async function decompressZstd(buf: Buffer): Promise<Buffer> {
   if (!isZstd(buf)) {
     throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
   }
+  // Lazy import (see compressZstdWithLevel for rationale).
+  const zstd = await import("@mongodb-js/zstd");
   return zstd.decompress(buf.subarray(2));
 }