pi-mega-compact 0.8.23 → 0.8.25

package/dist/src/vector-search.js

@@ -14,10 +14,13 @@ import { cosineSimilarity } from "./embedder.js";
 import { normalizeSessionId } from "./store.js";
 import { mmrRerank } from "./dedup/mmr.js";
 import { topK } from "./dedup/topk.js";
-import { listCheckpoints, getCheckpoint, maxCheckpointTimestamp, } from "./store/sqlite.js";
+import { listCheckpoints, getCheckpoint, maxCheckpointTimestamp, maxRaptorNodeBuiltAt, } from "./store/sqlite.js";
 import { initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
 import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
 import { stagedExpansion } from "./dedup/raptor/retrieval.js";
+import { multilevelRetrieval } from "./dedup/raptor/multilevel.js";
+import { Logger } from "./log.js";
+const logger = new Logger();
 // ---------------------------------------------------------------------------
 // raptorSearchHits — internal helper (NOT exported)
 // ---------------------------------------------------------------------------
@@ -25,8 +28,13 @@ import { stagedExpansion } from "./dedup/raptor/retrieval.js";
  * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
  * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
  * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
+ *
+ * `checkpoints` is the caller's already-loaded, dedup-status-filtered checkpoint
+ * list — pass it in so the flat-search path and the RAPTOR merge path share ONE
+ * `listCheckpoints` call per `vectorSearch` (eliminates a redundant full scan +
+ * N-row hydration at 500+ checkpoints — QA perf review).
  */
-function raptorSearchHits(store, sid, query, k) {
+function raptorSearchHits(store, sid, query, k, checkpoints) {
     const t0 = Date.now();
     try {
         const stateDir = store.stateDir;
@@ -38,17 +46,93 @@ function raptorSearchHits(store, sid, query, k) {
         // RAPTOR_SHADOW_MODE is anything other than "false".
         if (isShadowMode())
             return [];
-        const tree = rehydrateRaptorTree(sid, stateDir);
+        // S25 gate (b): freshness + fallback guards with per-session cache.
+        // The cache avoids O(n·leaves) rehydrate on every search when the tree
+        // hasn't changed. Freshness is validated via maxRaptorNodeBuiltAt (cheap
+        // indexed MAX query) — if the persisted builtAt matches the cached builtAt,
+        // the tree hasn't been rebuilt and can be served from cache.
+        const cacheKey = `${stateDir}::${sid}`;
+        const cached = store.raptorCache.get(cacheKey);
+        const latestBuiltAt = maxRaptorNodeBuiltAt(sid, stateDir);
+        let tree;
+        if (cached && cached.builtAt === latestBuiltAt && latestBuiltAt > 0) {
+            tree = cached.tree;
+        }
+        else {
+            tree = rehydrateRaptorTree(sid, stateDir);
+            if (tree && tree.rootId && !tree.timedOut) {
+                store.raptorCache.set(cacheKey, { tree, builtAt: tree.builtAt ?? 0 });
+            }
+            else {
+                store.raptorCache.delete(cacheKey);
+            }
+        }
         if (!tree || !tree.rootId)
             return [];
-        // S25 gate (b): freshness + fallback guards. Skip a tree built before the
-        // newest checkpoint (stale → may reference trimmed/deduped leaves) or one
-        // whose root is a budget-exhausted extractive fallback (level 99).
-        if (tree.timedOut)
-            return [];
+        // Freshness: skip persisted tree if the session has been compacted since
+        // the tree was built. The next runRaptor will rebuild.
         const maxTs = maxCheckpointTimestamp(sid, stateDir);
         if (tree.builtAt && tree.builtAt < maxTs)
             return [];
+        // S25: skip timedOut extractive-fallback trees (level === 99).
+        if (tree.timedOut)
+            return [];
+        // Use the caller's already-loaded checkpoint list (no second scan).
+        const all = checkpoints;
+        const qv = embedder.embed(query);
+        const hits = [];
+        // S42B: multi-level retrieval — score all tree levels (leaves + internal
+        // clusters), expand cluster hits to their leaf descendants, dedup overlaps,
+        // and MMR-rerank. Default ON (RAPTOR_MULTILEVEL_ENABLED). When OFF, the
+        // leaf-only stagedExpansion path below is used (identical to pre-S42B).
+        if (cfg.RAPTOR_MULTILEVEL_ENABLED) {
+            const mlHits = multilevelRetrieval(query, tree, {
+                embedder,
+                levelWeights: cfg.RAPTOR_LEVEL_WEIGHTS,
+                leafExpansion: cfg.RAPTOR_LEAF_EXPANSION,
+                maxLeafExpansion: cfg.RAPTOR_MAX_LEAF_EXPANSION,
+                k,
+                mmrLambda: cfg.MMR_LAMBDA,
+            });
+            if (mlHits.length === 0)
+                return [];
+            for (const mh of mlHits) {
+                // Leaf hits: nodeId IS the checkpointId — hydrate from the stored list.
+                const cp = all.find((c) => c.checkpointId === mh.nodeId);
+                if (cp) {
+                    hits.push({ checkpoint: cp, score: mh.score });
+                    continue;
+                }
+                // Orphaned leaf (checkpoint missing from the stored list, e.g. SemDeDup
+                // removed) — skip; synthesizing it would yield an empty cluster block.
+                if (mh.isLeaf)
+                    continue;
+                // Cluster hit: synthesize a SearchHit so the recall block can surface
+                // the hierarchical summary (not a stored checkpoint). The minimal
+                // StoredCheckpoint carries the node's centroid embedding for any
+                // downstream cosine / MMR the caller does.
+                hits.push({
+                    checkpoint: {
+                        checkpointId: mh.nodeId,
+                        sessionId: sid,
+                        summary: mh.summary,
+                        keyDecisions: [],
+                        nextSteps: [],
+                        filesModified: [],
+                        tokenEstimate: 0,
+                        regionHash: `raptor:${mh.nodeId}`,
+                        embedding: mh.embedding,
+                        timestamp: tree.builtAt ?? Date.now(),
+                    },
+                    score: mh.score,
+                    raptorSummary: mh.summary,
+                    raptorLevel: mh.level,
+                });
+            }
+            record("RAPTOR", hits.length > 0 ? "new" : "mark_only", `ml=${mlHits.length}`, Date.now() - t0);
+            return hits;
+        }
+        // S42B: flag OFF — leaf-only stagedExpansion (pre-S42B production path).
         const leafIds = stagedExpansion(query, tree, {
             embedder,
             k,
@@ -57,20 +141,24 @@ function raptorSearchHits(store, sid, query, k) {
         });
         if (leafIds.length === 0)
             return [];
-        const all = listCheckpoints(sid, stateDir).filter((cp) => cp.dedupStatus !== "removed");
-        const qv = embedder.embed(query);
-        const hits = [];
         for (const id of leafIds) {
             const cp = all.find((c) => c.checkpointId === id);
             if (cp)
-                hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
+                hits.push({
+                    checkpoint: cp,
+                    score: cosineSimilarity(qv, cp.embedding),
+                });
         }
         // S25 monitoring: emit a raptor_serve decision so canary.ts can track
         // p95 latency + the tier's live traffic (non-fatal, best-effort).
         record("RAPTOR", hits.length > 0 ? "new" : "mark_only", `leaves=${leafIds.length}`, Date.now() - t0);
         return hits;
     }
-    catch {
+    catch (e) {
+        logger.warn("raptor_search_error", {
+            error: String(e),
+            sessionId: sid,
+        });
         return [];
     }
 }
@@ -109,7 +197,7 @@ export function vectorSearch(store, sessionId, query, k = 3) {
     // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
     // O(n) flat leaves, tightening the block at read time.
     if (cfg.RAPTOR_ENABLED) {
-        const rh = raptorSearchHits(store, sid, query, k);
+        const rh = raptorSearchHits(store, sid, query, k, checkpoints);
         if (rh.length > 0) {
             const merged = [...window];
             for (const h of rh) {
@@ -176,7 +264,11 @@ export async function vectorSearchAsync(store, sessionId, query, k = 3, opts = {
         const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
         if (cp && cp.dedupStatus !== "removed") {
             const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
-            hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
+            hydrated.push({
+                checkpoint: cp,
+                score: h.score,
+                repoId: crossRepo ? h.repoId : undefined,
+            });
         }
     }
     if (hydrated.length === 0)

package/dist/src/vectorStore.js

@@ -9,11 +9,11 @@
  */
 import { createHash } from "node:crypto";
 import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
-import { loadDedupConfig } from "./config/dedup.js";
+import { loadDedupConfig, } from "./config/dedup.js";
 import { logDecision } from "./monitoring.js";
 import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
 import { computeContentDigest } from "./dedup/digest.js";
-import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "./dedup/l1-minhash.js";
+import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES, } from "./dedup/l1-minhash.js";
 import { lshBands } from "./dedup/l1-lsh.js";
 import { isNearDuplicate } from "./dedup/l1-verify.js";
 import { openBloom, saveBloom } from "./store/bloom.js";
@@ -47,6 +47,10 @@ export class VectorStore {
      * global index keys on repoId so recall can span repos.
      */
     repoId;
+    /** S25: per-session cached rehydrated RaptorTree. Keyed by sessionId.
+     *  Freshness-validated via maxRaptorNodeBuiltAt on each lookup — a cheap
+     *  indexed MAX query replaces the O(n·leaves) rehydrate on every search. */
+    raptorCache = new Map();
     constructor(opts = {}) {
         this.embedder = opts.embedder ?? defaultEmbedder();
         this.stateDir = opts.stateDir ?? getStateDir();
@@ -125,7 +129,11 @@ export class VectorStore {
                     bumpDedupStats(true, this.stateDir);
                     // Deduped: whole original region discarded, nothing new stored.
                     addTokensSaved(origTokens, this.stateDir);
-                    const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
+                    const r = {
+                        checkpoint: contentMatch,
+                        deduped: true,
+                        reason: "contentHash",
+                    };
                     this.record("L0", "deduped", "contentHash", Date.now() - t0);
                     onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
                     return r;
@@ -143,7 +151,11 @@ export class VectorStore {
                     bumpDedupStats(true, this.stateDir);
                     // Deduped: whole original region discarded, nothing new stored.
                     addTokensSaved(origTokens, this.stateDir);
-                    const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
+                    const r = {
+                        checkpoint: regionMatch,
+                        deduped: true,
+                        reason: "regionHash",
+                    };
                     this.record("L0", "deduped", "regionHash", Date.now() - t0);
                     onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
                     return r;
@@ -167,7 +179,11 @@ export class VectorStore {
                     bumpDedupStats(true, this.stateDir);
                     // Deduped: whole original region discarded, nothing new stored.
                     addTokensSaved(origTokens, this.stateDir);
-                    const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
+                    const r = {
+                        checkpoint: summaryMatch,
+                        deduped: true,
+                        reason: "summaryHash",
+                    };
                     this.record("L0", "deduped", "summaryHash", Date.now() - t0);
                     onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
                     return r;
@@ -226,14 +242,26 @@ export class VectorStore {
                     bumpDedupStats(true, this.stateDir);
                     // Deduped: whole original region discarded, nothing new stored.
                     addTokensSaved(origTokens, this.stateDir);
-                    const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
+                    const r = {
+                        checkpoint: nearest.checkpoint,
+                        deduped: true,
+                        reason: "contentSimilarity",
+                    };
                     this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
-                    onTier?.({ tier: "L2", status: "deduped", detail: nearest.sim.toFixed(2) });
+                    onTier?.({
+                        tier: "L2",
+                        status: "deduped",
+                        detail: nearest.sim.toFixed(2),
+                    });
                     return r;
                 }
                 markOnly = "L2";
             }
-            onTier?.({ tier: "L2", status: "passed", detail: `best ${nearest.sim.toFixed(2)}` });
+            onTier?.({
+                tier: "L2",
+                status: "passed",
+                detail: `best ${nearest.sim.toFixed(2)}`,
+            });
         }
         // 4. Genuinely new — create checkpoint
         const checkpointId = nextCheckpointId(sessionId, this.stateDir);

package/dist/src/wordpiece.js

@@ -0,0 +1,129 @@
+/**
+ * wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
+ *
+ * Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
+ * implements the standard uncased BERT preprocessing + greedy longest-match
+ * WordPiece segmentation. No native dependency, no network — the vocab file is a
+ * local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
+ *
+ * This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
+ * use: lowercase, strip accents, split on whitespace + punctuation, then
+ * WordPiece each token with the `##` continuation convention. Special tokens
+ * [CLS]/[SEP] are added by the caller's encode().
+ */
+import { readFileSync, existsSync } from "node:fs";
+const UNK = "[UNK]";
+const CLS = "[CLS]";
+const SEP = "[SEP]";
+const PAD = "[PAD]";
+const MAX_INPUT_CHARS_PER_WORD = 200;
+export class WordPieceTokenizer {
+    vocab;
+    clsId;
+    sepId;
+    padId;
+    unkId;
+    constructor(vocab) {
+        this.vocab = vocab;
+        this.clsId = vocab.get(CLS) ?? 101;
+        this.sepId = vocab.get(SEP) ?? 102;
+        this.padId = vocab.get(PAD) ?? 0;
+        this.unkId = vocab.get(UNK) ?? 100;
+    }
+    /** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
+    static fromVocabFile(path) {
+        if (!existsSync(path)) {
+            throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
+        }
+        const lines = readFileSync(path, "utf-8").split("\n");
+        const vocab = new Map();
+        for (let i = 0; i < lines.length; i++) {
+            const tok = lines[i].replace(/\r$/, "");
+            if (tok.length > 0 || i < lines.length - 1)
+                vocab.set(tok, i);
+        }
+        return new WordPieceTokenizer(vocab);
+    }
+    /** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
+    basicTokenize(text) {
+        // NFD + strip combining marks (accent removal), then lowercase.
+        const cleaned = text
+            .normalize("NFD")
+            .replace(/[̀-ͯ]/g, "")
+            .toLowerCase();
+        const tokens = [];
+        let buf = "";
+        const flush = () => {
+            if (buf.length > 0) {
+                tokens.push(buf);
+                buf = "";
+            }
+        };
+        for (const ch of cleaned) {
+            if (/\s/.test(ch)) {
+                flush();
+            }
+            else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
+                // Punctuation becomes its own token.
+                flush();
+                tokens.push(ch);
+            }
+            else {
+                buf += ch;
+            }
+        }
+        flush();
+        return tokens;
+    }
+    /** Greedy longest-match WordPiece for a single word. */
+    wordpiece(word) {
+        if (word.length > MAX_INPUT_CHARS_PER_WORD)
+            return [UNK];
+        const pieces = [];
+        let start = 0;
+        while (start < word.length) {
+            let end = word.length;
+            let cur = null;
+            while (start < end) {
+                let sub = word.slice(start, end);
+                if (start > 0)
+                    sub = "##" + sub;
+                if (this.vocab.has(sub)) {
+                    cur = sub;
+                    break;
+                }
+                end--;
+            }
+            if (cur === null)
+                return [UNK]; // any unmatchable piece → whole word is UNK
+            pieces.push(cur);
+            start = end;
+        }
+        return pieces;
+    }
+    /** Tokenize text into WordPiece token strings (no special tokens). */
+    tokenize(text) {
+        const out = [];
+        for (const word of this.basicTokenize(text)) {
+            for (const piece of this.wordpiece(word))
+                out.push(piece);
+        }
+        return out;
+    }
+    /**
+     * Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
+     * attention_mask is all 1s (no padding for single-sequence inference).
+     */
+    encode(text, maxLen = 256) {
+        const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
+        const inputIds = [this.clsId];
+        for (const p of pieces)
+            inputIds.push(this.vocab.get(p) ?? this.unkId);
+        inputIds.push(this.sepId);
+        return {
+            inputIds,
+            attentionMask: inputIds.map(() => 1),
+            tokenTypeIds: inputIds.map(() => 0),
+        };
+    }
+}