pi-mega-compact 0.8.21 → 0.8.22

package/dist/src/vector-read.js

@@ -0,0 +1,168 @@
+/**
+ * vector-read.ts — free functions for VectorStore read-only operations.
+ *
+ * Extracted from vectorStore.ts (PR0 split) to bring it under 500 lines.
+ * All functions take `store: VectorStore` as first param and access store
+ * fields/props directly via type-cast to access private fields (acceptable
+ * since these fields were effectively public via the original methods).
+ *
+ * Call sites in src/ and extensions/ are rewritten from `store.stats(sid)`
+ * → `vectorStats(store, sid)`.
+ */
+import { cosineSimilarity } from "./embedder.js";
+import { listCheckpoints, loadSessionState, saveSessionState, setDedupStatus, getDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
+import { normalizeSessionId } from "./store.js";
+import { computeRegionHash } from "./vectorStore.js";
+// ---------------------------------------------------------------------------
+// Cosine
+// ---------------------------------------------------------------------------
+/** Convenience wrapper for raw vector cosine similarity (exposed for tests). */
+export function vectorSimilarity(_store, a, b) {
+    return cosineSimilarity(a, b);
+}
+// ---------------------------------------------------------------------------
+// SemDeDup
+// ---------------------------------------------------------------------------
+/**
+ * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
+ * lower-quality row of any pair scoring cosine > `threshold` as
+ * `dedup_status='removed'` (kept, not deleted — retrieval excludes it). Keeps
+ * the row with the higher `tokenEstimate` (more context preserved). Runs as a
+ * single scan; idempotent (re-running skips already-removed rows).
+ *
+ * Returns the number of rows marked removed.
+ */
+export function vectorSemDedup(store, sessionId, threshold) {
+    const sid = normalizeSessionId(sessionId);
+    const stateDir = store.stateDir;
+    const cfg = store.cfg;
+    const thr = threshold ?? cfg.SEMDEDUP_COSINE;
+    const cps = listCheckpoints(sid, stateDir).filter((c) => c.dedupStatus !== "removed");
+    let removed = 0;
+    for (let i = 0; i < cps.length; i++) {
+        for (let j = i + 1; j < cps.length; j++) {
+            const a = cps[i];
+            const b = cps[j];
+            if (a.dedupStatus === "removed" || b.dedupStatus === "removed")
+                continue;
+            if (cosineSimilarity(a.embedding, b.embedding) > thr) {
+                const keep = a.tokenEstimate >= b.tokenEstimate ? a : b;
+                const drop = keep === a ? b : a;
+                setDedupStatus(drop.checkpointId, sid, "removed", stateDir);
+                drop.dedupStatus = "removed";
+                removed++;
+            }
+        }
+    }
+    return removed;
+}
+// ---------------------------------------------------------------------------
+// Dedup sentinel
+// ---------------------------------------------------------------------------
+/**
+ * Dedup sentinel check: has this region already been stored/represented?
+ * Consulted by both the persist path and the recall/inline path.
+ */
+export function vectorDedupe(store, sessionId, regionHashOrText, isText = false) {
+    const stateDir = store.stateDir;
+    const sid = normalizeSessionId(sessionId);
+    const hash = isText
+        ? computeRegionHash(regionHashOrText)
+        : regionHashOrText;
+    const state = loadSessionState(sid, stateDir);
+    if (state.storedRegionHashes.includes(hash))
+        return true;
+    return listCheckpoints(sid, stateDir).some((c) => c.regionHash === hash);
+}
+// ---------------------------------------------------------------------------
+// Injection tracking
+// ---------------------------------------------------------------------------
+/** Mark a checkpoint as injected into the window (recall dedup). */
+export function vectorMarkInjected(store, sessionId, checkpointId) {
+    const stateDir = store.stateDir;
+    const sid = normalizeSessionId(sessionId);
+    const state = loadSessionState(sid, stateDir);
+    if (!state.injectedCheckpointIds.includes(checkpointId)) {
+        state.injectedCheckpointIds.push(checkpointId);
+        saveSessionState(sid, state, stateDir);
+    }
+}
+/** True if this checkpoint was already injected this session. */
+export function vectorWasInjected(store, sessionId, checkpointId) {
+    const stateDir = store.stateDir;
+    const state = loadSessionState(normalizeSessionId(sessionId), stateDir);
+    return state.injectedCheckpointIds.includes(checkpointId);
+}
+// ---------------------------------------------------------------------------
+// List & TopSimilar
+// ---------------------------------------------------------------------------
+/** All checkpoints for a session (sorted by checkpointId). */
+export function vectorList(store, sessionId) {
+    const stateDir = store.stateDir;
+    return listCheckpoints(normalizeSessionId(sessionId), stateDir);
+}
+/**
+ * Return the n most similar checkpoints to the current (most recent) checkpoint
+ * by cosine similarity. Returns fewer than n if the session has fewer checkpoints.
+ * The current checkpoint itself is excluded from results.
+ */
+export function vectorTopSimilar(store, sessionId, n) {
+    const stateDir = store.stateDir;
+    const sid = normalizeSessionId(sessionId);
+    const checkpoints = listCheckpoints(sid, stateDir);
+    if (checkpoints.length <= 1)
+        return [];
+    const ordered = [...checkpoints].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
+    const current = ordered[ordered.length - 1];
+    const scored = ordered
+        .filter((cp) => cp.checkpointId !== current.checkpointId)
+        .map((cp) => ({
+        checkpoint: cp,
+        score: cosineSimilarity(current.embedding, cp.embedding),
+    }))
+        .sort((a, b) => b.score - a.score);
+    return scored.slice(0, n);
+}
+/**
+ * Store statistics for status reporting / logging. Returns counts + the last
+ * (highest-numbered) checkpoint, or nulls when the session is empty.
+ */
+export function vectorStats(store, sessionId) {
+    const stateDir = store.stateDir;
+    const sid = normalizeSessionId(sessionId);
+    const cps = listCheckpoints(sid, stateDir);
+    const state = loadSessionState(sid, stateDir);
+    const ordered = [...cps].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
+    const last = ordered[ordered.length - 1];
+    const injected = state.injectedCheckpointIds.length;
+    const ds = getDedupStats(stateDir);
+    const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
+    const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
+    const sessionSaved = cps.reduce((s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)), 0);
+    return {
+        checkpointCount: cps.length,
+        totalTokenEstimate: sessionTok,
+        lastCheckpointId: last?.checkpointId,
+        lastSummary: last?.summary,
+        injectedCount: injected,
+        dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
+        storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
+        tokensSaved: sessionSaved,
+        originalTokens: sessionOrig,
+        dedupAttempts: ds.attempts,
+        dedupCollapsed: ds.deduped,
+    };
+}
+/**
+ * Repo-wide stats — aggregates every session in this store (one per repo).
+ * Cumulative, resumable, cross-device.
+ */
+export function vectorRepoStats(store) {
+    const stateDir = store.stateDir;
+    return repoStatsFromStore(stateDir);
+}
+/** Data-safety invariant: regions retained vs bytes permanently deleted. */
+export function vectorDataInvariant(store) {
+    const stateDir = store.stateDir;
+    return dataInvariantStats(stateDir);
+}

package/dist/src/vector-search.js

@@ -0,0 +1,191 @@
+/**
+ * vector-search.ts — free functions for VectorStore search operations.
+ *
+ * Extracted from vectorStore.ts (PR0 split) to bring it under 500 lines.
+ * All functions take `store: VectorStore` as first param and access store
+ * fields/props directly via type-cast to access private fields (acceptable
+ * since these fields were effectively public via the original methods).
+ *
+ * Call sites in src/ and extensions/ are rewritten from `store.search(...)`
+ * → `vectorSearch(store, ...)` and `store.searchAsync(...)` →
+ * `vectorSearchAsync(store, ...)`.
+ */
+import { cosineSimilarity } from "./embedder.js";
+import { normalizeSessionId } from "./store.js";
+import { mmrRerank } from "./dedup/mmr.js";
+import { topK } from "./dedup/topk.js";
+import { listCheckpoints, getCheckpoint, maxCheckpointTimestamp, } from "./store/sqlite.js";
+import { initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
+import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
+import { stagedExpansion } from "./dedup/raptor/retrieval.js";
+// ---------------------------------------------------------------------------
+// raptorSearchHits — internal helper (NOT exported)
+// ---------------------------------------------------------------------------
+/**
+ * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
+ * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
+ * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
+ */
+function raptorSearchHits(store, sid, query, k) {
+    const t0 = Date.now();
+    try {
+        const stateDir = store.stateDir;
+        const cfg = store.cfg;
+        const embedder = store.embedder;
+        const record = store.record;
+        // S25 gate (a): honor the shadow contract at SERVE time. The tree is still
+        // built + persisted (logging-only) but NOT merged into recall while
+        // RAPTOR_SHADOW_MODE is anything other than "false".
+        if (isShadowMode())
+            return [];
+        const tree = rehydrateRaptorTree(sid, stateDir);
+        if (!tree || !tree.rootId)
+            return [];
+        // S25 gate (b): freshness + fallback guards. Skip a tree built before the
+        // newest checkpoint (stale → may reference trimmed/deduped leaves) or one
+        // whose root is a budget-exhausted extractive fallback (level 99).
+        if (tree.timedOut)
+            return [];
+        const maxTs = maxCheckpointTimestamp(sid, stateDir);
+        if (tree.builtAt && tree.builtAt < maxTs)
+            return [];
+        const leafIds = stagedExpansion(query, tree, {
+            embedder,
+            k,
+            topM: cfg.RAPTOR_CLUSTERS_PER_LEVEL,
+            mmrLambda: cfg.MMR_LAMBDA,
+        });
+        if (leafIds.length === 0)
+            return [];
+        const all = listCheckpoints(sid, stateDir).filter((cp) => cp.dedupStatus !== "removed");
+        const qv = embedder.embed(query);
+        const hits = [];
+        for (const id of leafIds) {
+            const cp = all.find((c) => c.checkpointId === id);
+            if (cp)
+                hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
+        }
+        // S25 monitoring: emit a raptor_serve decision so canary.ts can track
+        // p95 latency + the tier's live traffic (non-fatal, best-effort).
+        record("RAPTOR", hits.length > 0 ? "new" : "mark_only", `leaves=${leafIds.length}`, Date.now() - t0);
+        return hits;
+    }
+    catch {
+        return [];
+    }
+}
+// ---------------------------------------------------------------------------
+// vectorSearch
+// ---------------------------------------------------------------------------
+/**
+ * Semantic search within a session's checkpoints. Returns top-K by cosine
+ * similarity, diversified via MMR (QA #10) so a cluster of near-identical
+ * hits yields at most a few distinct-relevance results.
+ *
+ * Heap-based top-K (QA #4, O(N log k)) replaces the old full sort; MMR then
+ * reranks the candidate window for diversity.
+ */
+export function vectorSearch(store, sessionId, query, k = 3) {
+    const stateDir = store.stateDir;
+    const cfg = store.cfg;
+    const embedder = store.embedder;
+    const sid = normalizeSessionId(sessionId);
+    const checkpoints = listCheckpoints(sid, stateDir).filter((cp) => cp.dedupStatus !== "removed");
+    if (checkpoints.length === 0)
+        return [];
+    const qv = embedder.embed(query);
+    const scored = checkpoints.map((cp) => ({
+        checkpoint: cp,
+        score: cosineSimilarity(qv, cp.embedding),
+    }));
+    // Heap top-K over a widened window (2k) so MMR has diverse candidates.
+    const window = topK(scored.map((h) => ({ item: h, score: h.score })), Math.max(k * 2, k)).map((s) => s.item);
+    // MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
+    // (Sprint 14 flag), returning the plain relevance-ranked window instead.
+    if (!cfg.L2_ENABLED)
+        return window.slice(0, k);
+    // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
+    // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
+    // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
+    // O(n) flat leaves, tightening the block at read time.
+    if (cfg.RAPTOR_ENABLED) {
+        const rh = raptorSearchHits(store, sid, query, k);
+        if (rh.length > 0) {
+            const merged = [...window];
+            for (const h of rh) {
+                if (!merged.some((m) => m.checkpoint.checkpointId === h.checkpoint.checkpointId)) {
+                    merged.push(h);
+                }
+            }
+            const mmrItems = merged.map((h) => ({
+                item: h,
+                vector: h.checkpoint.embedding,
+                relevance: h.score,
+            }));
+            return mmrRerank(mmrItems, k, cfg.MMR_LAMBDA);
+        }
+    }
+    const mmrItems = window.map((h) => ({
+        item: h,
+        vector: h.checkpoint.embedding,
+        relevance: h.score,
+    }));
+    const ranked = mmrRerank(mmrItems, k, cfg.MMR_LAMBDA);
+    return ranked;
+}
+/**
+ * Slice 2: async cross-repo (or single-repo) recall via the PGlite/HNSW index.
+ *
+ * This is the ONLY async recall surface and is a BONUS path — the synchronous
+ * `vectorSearch` above remains the default. `opts.repoId` scopes to one repo;
+ * omit it for cross-repo nearest-neighbor recall (the headline capability the
+ * sync per-session scan cannot provide).
+ *
+ * Best-effort: if the index is disabled/empty/failing, we fall back to the
+ * synchronous per-session `vectorSearch` for THIS repo so callers always get
+ * a sensible result. Hydrates each hit's StoredCheckpoint from the authoritative
+ * node:sqlite store (the hit's repoId doubles as that repo's stateDir), then
+ * MMR-dedupes the merged set.
+ */
+export async function vectorSearchAsync(store, sessionId, query, k = 3, opts = {}) {
+    const cfg = store.cfg;
+    const embedder = store.embedder;
+    const sid = normalizeSessionId(sessionId);
+    const qv = embedder.embed(query);
+    // repoId filter: explicit opts.repoId wins; else this repo unless crossRepo.
+    const selfRepo = store.repoId;
+    const repoId = opts.repoId ?? (opts.crossRepo ? undefined : selfRepo);
+    let indexHits = [];
+    try {
+        await initVectorIndex();
+        indexHits = await vectorIndexSearch(qv, { k: Math.max(k * 2, k), repoId });
+    }
+    catch {
+        indexHits = [];
+    }
+    if (indexHits.length === 0) {
+        // Index empty/unavailable → synchronous per-session fallback (this repo).
+        return vectorSearch(store, sid, query, k);
+    }
+    // Hydrate each index hit from the authoritative node:sqlite store. repoId is
+    // that repo's stateDir, so cross-repo hits resolve against their own store.
+    // Tag cross-repo hits with their source repoId so the recall block can label
+    // them ("from repo <name>"); same-repo hits stay unlabeled.
+    const hydrated = [];
+    for (const h of indexHits) {
+        const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
+        if (cp && cp.dedupStatus !== "removed") {
+            const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
+            hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
+        }
+    }
+    if (hydrated.length === 0)
+        return vectorSearch(store, sid, query, k);
+    // MMR-dedupe the merged candidate set for diversity (mirrors sync search).
+    const mmrItems = hydrated.map((h) => ({
+        item: h,
+        vector: h.checkpoint.embedding,
+        relevance: h.score,
+    }));
+    return mmrRerank(mmrItems, k, cfg.MMR_LAMBDA);
+}