pi-mega-compact 0.4.20 → 0.4.23

package/src/store/sqlite.ts

@@ -2,11 +2,13 @@
  * sqlite.ts — Sprint 8 storage backbone (the "one store").
  *
  * Replaces the per-session gzipped-JSON checkpoint files with a single local
- * SQLite database (better-sqlite3, in-process, FS-backed, ZERO network calls —
- * honors PREVENT-PI-004). Chosen over PGlite because PGlite is async-only in
- * every published version, and VectorStore (engine.ts / recall.ts / the
- * extension) is fully synchronous — adopting PGlite would have cascaded async
- * through the whole call chain. SQLite keeps every VectorStore signature sync.
+ * SQLite database (node:sqlite — the Node built-in, in-process, FS-backed,
+ * ZERO network calls — honors PREVENT-PI-004). No native build and no install
+ * scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
+ * native binary could not be built under pi, which crashed every `pi update
+ * --extensions`). node:sqlite is synchronous, so every VectorStore signature
+ * stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
+ * real HNSW indexing (Slice 2 of the dual-backend plan).
  *
  * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
  * / pg_trgm-equivalent verification). The default cosine path stays a linear
@@ -15,7 +17,7 @@
  * All queries are parameterized (PREVENT-002) — never string-concatenated.
  */
 
-import Database from "better-sqlite3";
+import { DatabaseSync } from "node:sqlite";
 import { existsSync, mkdirSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
@@ -31,12 +33,15 @@ function encodeEmbedding(v: number[]): Buffer {
   for (let i = 0; i < v.length; i++) buf.writeFloatLE(v[i] ?? 0, i * 4);
   return buf;
 }
-/** Decode a Float32 BLOB back to a number[]. */
-function decodeEmbedding(buf: Buffer | null | undefined): number[] {
+/** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
+ *  Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass — both
+ *  work). */
+function decodeEmbedding(buf: Uint8Array | null | undefined): number[] {
   if (!buf || buf.length === 0) return [];
+  const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
   const n = buf.length / 4;
   const out = new Array<number>(n);
-  for (let i = 0; i < n; i++) out[i] = buf.readFloatLE(i * 4);
+  for (let i = 0; i < n; i++) out[i] = dv.getFloat32(i * 4, true);
   return out;
 }
 
@@ -47,17 +52,17 @@ function jsonText(v: unknown): string {
 // In-process cache so the same stateDir reuses one connection (and so a fresh
 // VectorStore over the same dir shares the open DB). Cross-process durability
 // comes from reopening the same file path — proven by the integration test.
-const cache = new Map<string, Database.Database>();
+const cache = new Map<string, DatabaseSync>();
 
 /** Open (or reuse) the SQLite store for a state dir. */
-export function openStore(stateDir: string = getStateDir()): Database.Database {
+export function openStore(stateDir: string = getStateDir()): DatabaseSync {
   const existing = cache.get(stateDir);
   if (existing) return existing;
 
   if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
-  const db = new Database(join(stateDir, "sqlite.db"));
-  db.pragma("journal_mode = WAL");
-  db.pragma("foreign_keys = ON");
+  const db = new DatabaseSync(join(stateDir, "sqlite.db"));
+  db.exec("PRAGMA journal_mode = WAL");
+  db.exec("PRAGMA foreign_keys = ON");
   initSchema(db);
   cache.set(stateDir, db);
   return db;
@@ -87,17 +92,17 @@ export function getIndexDir(): string {
   }
 }
 
-let indexCache: Database.Database | undefined;
+let indexCache: DatabaseSync | undefined;
 let indexCacheDir: string | undefined;
 
 /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
-export function openIndexStore(indexDir: string = getIndexDir()): Database.Database {
+export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
   if (indexCache && indexCacheDir === indexDir) return indexCache;
   if (!existsSync(indexDir)) mkdirSync(indexDir, { recursive: true });
-  const db = new Database(join(indexDir, "index.sqlite"));
-  db.pragma("journal_mode = WAL");
-  db.pragma("busy_timeout = 3000"); // tolerate brief cross-process write contention
-  db.exec(`
+  const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
+  iddb.exec("PRAGMA journal_mode = WAL");
+  iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
+  iddb.exec(`
     CREATE TABLE IF NOT EXISTS repo_registry (
       repo_root                 TEXT PRIMARY KEY,
       display_name              TEXT,
@@ -117,9 +122,9 @@ export function openIndexStore(indexDir: string = getIndexDir()): Database.Datab
     );
     CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
   `);
-  indexCache = db;
+  indexCache = iddb;
   indexCacheDir = indexDir;
-  return db;
+  return iddb;
 }
 
 /** One row of the global repo registry (multi-repo dashboard source). */
@@ -276,7 +281,7 @@ export function closeIndexStore(): void {
   }
 }
 
-function initSchema(db: Database.Database): void {
+function initSchema(db: DatabaseSync): void {
   db.exec(`
     CREATE TABLE IF NOT EXISTS context_chunks (
       id                  TEXT NOT NULL,
@@ -415,6 +420,20 @@ function initSchema(db: Database.Database): void {
       ts            INTEGER
     );
 
+    -- Durable "save to memory" store (taken over from memory extensions).
+    -- One row per saved memory; scoped by repo so memory travels with the
+    -- clone. All params are parameterized (PREVENT-002).
+    CREATE TABLE IF NOT EXISTS memories (
+      id                INTEGER PRIMARY KEY AUTOINCREMENT,
+      repo              TEXT,
+      kind              TEXT DEFAULT 'note',   -- note | fact | decision | preference
+      content           TEXT NOT NULL,
+      tags              TEXT,                   -- JSON array of strings
+      created_at        INTEGER,
+      last_recalled_at  INTEGER
+    );
+    CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
+
     -- FTS5 trigram virtual table (Sprint 9+ pg_trgm-equivalent verification).
     CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_trgm USING fts5(
       id UNINDEXED,
@@ -443,7 +462,7 @@ function initSchema(db: Database.Database): void {
  * input), so the unavoidable identifier interpolation here does not violate
  * PREVENT-002 (no external data reaches this SQL).
  */
-function ensureColumn(db: Database.Database, table: string, column: string, decl: string): void {
+function ensureColumn(db: DatabaseSync, table: string, column: string, decl: string): void {
   const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
   if (cols.some((c) => c.name === column)) return;
   db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
@@ -582,6 +601,94 @@ export function addLesson(
   ).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
 }
 
+// --- Durable memory (save-to-memory takeover) ---------------------------------
+// One SQLite store for user-saved memories, scoped by repo. Mirrors the
+// lessons/sessions pattern: all state lives in SQLite from day one.
+
+export interface MemoryRecord {
+  id: number;
+  repo: string | null;
+  kind: string;
+  content: string;
+  tags: string[];
+  createdAt: number;
+  lastRecalledAt: number | null;
+}
+
+/** Save a memory to the current repo's store. Returns the new row id. */
+export function addMemory(
+  memory: { kind?: string; content: string; tags?: string[] },
+  repo: string | null,
+  stateDir: string = getStateDir(),
+): number {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  const res = db
+    .prepare(
+      `INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
+       VALUES(?, ?, ?, ?, ?, NULL)`,
+    )
+    .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
+  return Number(res.lastInsertRowid);
+}
+
+/** List recent memories for a repo (or all repos when repo is null). */
+export function listMemories(repo: string | null, limit = 50, stateDir: string = getStateDir()): MemoryRecord[] {
+  const db = openStore(stateDir);
+  const rows = repo
+    ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
+    : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
+  return (rows as any[]).map(mapMemoryRow);
+}
+
+/** Substring search across content + tags. */
+export function searchMemories(query: string, repo: string | null = null, limit = 50, stateDir: string = getStateDir()): MemoryRecord[] {
+  const db = openStore(stateDir);
+  const like = `%${query}%`;
+  const rows = repo
+    ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
+    : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
+  return (rows as any[]).map(mapMemoryRow);
+}
+
+/** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
+export function recallMemory(id: number, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
+  return res.changes > 0;
+}
+
+function mapMemoryRow(row: any): MemoryRecord {
+  return {
+    id: row.id,
+    repo: row.repo ?? null,
+    kind: row.kind ?? "note",
+    content: row.content ?? "",
+    tags: row.tags ? JSON.parse(row.tags) : [],
+    createdAt: row.created_at ?? 0,
+    lastRecalledAt: row.last_recalled_at ?? null,
+  };
+}
+
+/**
+ * Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
+ * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
+ * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
+ * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
+ */
+export function withTx(db: DatabaseSync, fn: () => void): void {
+  db.exec("SAVEPOINT mc_tx");
+  try {
+    fn();
+    db.exec("RELEASE mc_tx");
+  } catch (e) {
+    db.exec("ROLLBACK TO mc_tx");
+    db.exec("RELEASE mc_tx");
+    throw e;
+  }
+}
+
 /** Map a DB row to the public StoredCheckpoint shape. */
 function rowToCheckpoint(row: any): StoredCheckpoint {
   return {
@@ -600,7 +707,9 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
     contentHash2: row.content_hash2 ?? undefined,
     contentHashVersion: row.content_hash_version ?? undefined,
     normalizedText: row.normalized_text ?? undefined,
-    compressedOriginal: row.compressed_original ?? undefined,
+    // node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
+    // (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
+    compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
     embedding: decodeEmbedding(row.embedding_blob),
     timestamp: Number(row.timestamp ?? 0),
     dedupStatus: row.dedup_status ?? undefined,
@@ -611,7 +720,7 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
 export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getStateDir()): void {
   const db = openStore(stateDir);
   const sid = normalizeSessionId(cp.sessionId);
-  const tx = db.transaction(() => {
+  withTx(db, () => {
     db.prepare(
       `INSERT INTO context_chunks
         (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
@@ -636,25 +745,25 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
          dedup_status=excluded.dedup_status,
          compressed_original=excluded.compressed_original`,
     ).run({
-      id: cp.checkpointId,
-      sid,
-      region_hash: cp.regionHash ?? null,
-      content_hash: cp.contentHash ?? null,
-      content_hash2: cp.contentHash2 ?? null,
-      content_hash_version: cp.contentHashVersion ?? null,
-      normalized_text: cp.normalizedText ?? null,
-      summary: cp.summary ?? "",
-      topic_summary: cp.topicSummary ?? null,
-      summary_hash: cp.summaryHash ?? null,
-      key_decisions: jsonText(cp.keyDecisions),
-      next_steps: jsonText(cp.nextSteps),
-      files_modified: jsonText(cp.filesModified),
-      embedding_blob: encodeEmbedding(cp.embedding ?? []),
-      token_estimate: cp.tokenEstimate ?? 0,
-      original_token_estimate: cp.originalTokenEstimate ?? null,
-      timestamp: cp.timestamp ?? 0,
-      dedup_status: "active",
-      compressed_original: cp.compressedOriginal ?? null,
+      "@id": cp.checkpointId,
+      "@sid": sid,
+      "@region_hash": cp.regionHash ?? null,
+      "@content_hash": cp.contentHash ?? null,
+      "@content_hash2": cp.contentHash2 ?? null,
+      "@content_hash_version": cp.contentHashVersion ?? null,
+      "@normalized_text": cp.normalizedText ?? null,
+      "@summary": cp.summary ?? "",
+      "@topic_summary": cp.topicSummary ?? null,
+      "@summary_hash": cp.summaryHash ?? null,
+      "@key_decisions": jsonText(cp.keyDecisions),
+      "@next_steps": jsonText(cp.nextSteps),
+      "@files_modified": jsonText(cp.filesModified),
+      "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
+      "@token_estimate": cp.tokenEstimate ?? 0,
+      "@original_token_estimate": cp.originalTokenEstimate ?? null,
+      "@timestamp": cp.timestamp ?? 0,
+      "@dedup_status": "active",
+      "@compressed_original": cp.compressedOriginal ?? null,
     });
 
     // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
@@ -665,7 +774,6 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
       "INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)",
     ).run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
   });
-  tx();
 }
 
 // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
@@ -702,11 +810,10 @@ export function insertLshBuckets(
   const ins = db.prepare(
     "INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)",
   );
-  const tx = db.transaction(() => {
+  withTx(db, () => {
     del.run(chunkId);
     for (const key of bucketKeys) ins.run(key, chunkId, sid, signatureVersion);
   });
-  tx();
 }
 
 /**
@@ -780,7 +887,7 @@ export function setDedupStatus(
 
 // --- Session state (injection tracking) ------------------------------------
 
-function loadSessionStateRow(sid: string, db: Database.Database): SessionState {
+function loadSessionStateRow(sid: string, db: DatabaseSync): SessionState {
   const row = db.prepare("SELECT * FROM session_state WHERE session_id = ?").get(sid) as any;
   if (!row) {
     return { injectedCheckpointIds: [], storedRegionHashes: [] };

package/src/store.test.ts

@@ -130,6 +130,28 @@ test("compression tier: GZIP-1 and GZIP-6 tiers produce valid, smaller output",
   assert.deepEqual(decompressSmart(big), Buffer.from("compress me ".repeat(1800)));
 });
 
+test("Fix E: pressure escalates gzip tier strength (sync, no zstd)", () => {
+  const small = Buffer.from("compress me ".repeat(200)); // 512B–4KB band
+  const medium = Buffer.from("compress me ".repeat(1800)); // 4KB–32KB band
+
+  // Low pressure → cheap levels (gzip-1 / gzip-6).
+  const lowSmall = compressSmart(small, 0);
+  const lowMedium = compressSmart(medium, 0);
+  assert.equal(lowSmall[3], 0x01, "small tier tag");
+  assert.equal(lowMedium[3], 0x02, "medium tier tag");
+  assert.ok(decompressSmart(lowSmall).equals(small), "low-pressure small roundtrips");
+  assert.ok(decompressSmart(lowMedium).equals(medium), "low-pressure medium roundtrips");
+
+  // High pressure → stronger levels (gzip-9 / gzip-9); tag unchanged, output
+  // must still decode to the exact original (versioned header preserved).
+  const highSmall = compressSmart(small, 1);
+  const highMedium = compressSmart(medium, 1);
+  assert.equal(highSmall[3], 0x01, "tag unchanged under pressure");
+  assert.equal(highMedium[3], 0x02, "tag unchanged under pressure");
+  assert.ok(decompressSmart(highSmall).equals(small), "high-pressure small roundtrips");
+  assert.ok(decompressSmart(highMedium).equals(medium), "high-pressure medium roundtrips");
+});
+
 // ---------------------------------------------------------------------------
 // normalizeSessionId
 // ---------------------------------------------------------------------------

package/src/vectorStore.ts

@@ -38,6 +38,8 @@ import {
   repoStats as repoStatsFromStore,
   dataInvariantStats,
 } from "./store/sqlite.js";
+import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
+import { stagedExpansion } from "./dedup/raptor/retrieval.js";
 import { migrateJsonToSqlite } from "./store/migrate.js";
 
 export interface SearchHit {
@@ -66,6 +68,9 @@ export interface AddInput {
    *  Lets the UI render live per-tier progress during compaction. Never awaited;
    *  must be cheap. Optional for back-compat. */
   onTier?: (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void;
+  /** Context-window pressure (0–1) — escalates the stored checkpoint's sync
+   *  compression strength (Fix E). Optional; defaults to 0 (brotli-4). */
+  compressionPressure?: number;
 }
 
 /** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
@@ -318,7 +323,10 @@ export class VectorStore {
       contentHash2: digest.contentHash2,
       contentHashVersion: digest.contentHashVersion,
       normalizedText: digest.normalizedText,
-      compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8")),
+      compressedOriginal: compressSmart(
+        Buffer.from(input.regionText, "utf-8"),
+        input.compressionPressure,
+      ),
       embedding,
       timestamp: input.timestamp,
     };
@@ -432,6 +440,29 @@ export class VectorStore {
     // MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
     // (Sprint 14 flag), returning the plain relevance-ranked window instead.
     if (!this.cfg.L2_ENABLED) return window.slice(0, k);
+
+    // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
+    // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
+    // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
+    // O(n) flat leaves, tightening the block at read time.
+    if (this.cfg.RAPTOR_ENABLED) {
+      const raptorHits = this.raptorSearchHits(sid, query, k);
+      if (raptorHits.length > 0) {
+        const merged: SearchHit[] = [...window];
+        for (const rh of raptorHits) {
+          if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
+            merged.push(rh);
+          }
+        }
+        const mmrItems: MmrItem<SearchHit>[] = merged.map((h) => ({
+          item: h,
+          vector: h.checkpoint.embedding,
+          relevance: h.score,
+        }));
+        return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
+      }
+    }
+
     const mmrItems: MmrItem<SearchHit>[] = window.map((h) => ({
       item: h,
       vector: h.checkpoint.embedding,
@@ -441,6 +472,37 @@ export class VectorStore {
     return ranked;
   }
 
+  /**
+   * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
+   * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
+   * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
+   */
+  private raptorSearchHits(sid: string, query: string, k: number): SearchHit[] {
+    try {
+      const tree = rehydrateRaptorTree(sid, this.stateDir);
+      if (!tree || !tree.rootId) return [];
+      const leafIds = stagedExpansion(query, tree, {
+        embedder: this.embedder,
+        k,
+        topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
+        mmrLambda: this.cfg.MMR_LAMBDA,
+      });
+      if (leafIds.length === 0) return [];
+      const all = listCheckpoints(sid, this.stateDir).filter(
+        (cp) => cp.dedupStatus !== "removed",
+      );
+      const qv = this.embedder.embed(query);
+      const hits: SearchHit[] = [];
+      for (const id of leafIds) {
+        const cp = all.find((c) => c.checkpointId === id);
+        if (cp) hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
+      }
+      return hits;
+    } catch {
+      return [];
+    }
+  }
+
   /**
    * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
    * lower-quality row of any pair scoring cosine > `threshold` as