pi-mega-compact 0.7.9 → 0.8.0

package/dist/src/vectorStore/queries.js

@@ -1,83 +0,0 @@
-/**
- * vectorStore/queries.ts — Read-only queries: list, topSimilar, stats, repoStats,
- * dataInvariant, similarity.
- */
-import { cosineSimilarity } from "../embedder.js";
-import { normalizeSessionId } from "../store.js";
-import { listCheckpoints, loadSessionState, getDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "../store/sqlite.js";
-/** All checkpoints for a session (sorted by checkpointId). */
-export function listSession(ctx, sessionId) {
-    return listCheckpoints(normalizeSessionId(sessionId), ctx.stateDir);
-}
-/** Convenience for a raw vector cosine (exposed for tests). */
-export function similarityScore(a, b) {
-    return cosineSimilarity(a, b);
-}
-/**
- * Return the n most similar checkpoints to the current (most recent) checkpoint
- * by cosine similarity. Returns fewer than n if the session has fewer checkpoints.
- * The current checkpoint itself is excluded from results.
- */
-export function topSimilarSession(ctx, sessionId, n) {
-    const sid = normalizeSessionId(sessionId);
-    const checkpoints = listCheckpoints(sid, ctx.stateDir);
-    if (checkpoints.length <= 1)
-        return [];
-    // Find the most recent checkpoint (by checkpointId, which is sequential)
-    const ordered = [...checkpoints].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
-    const current = ordered[ordered.length - 1];
-    // Score all other checkpoints by similarity to current
-    const scored = ordered
-        .filter((cp) => cp.checkpointId !== current.checkpointId)
-        .map((cp) => ({
-        checkpoint: cp,
-        score: cosineSimilarity(current.embedding, cp.embedding),
-    }))
-        .sort((a, b) => b.score - a.score);
-    return scored.slice(0, n);
-}
-/**
- * Store statistics for status reporting / logging. Returns counts + the last
- * (highest-numbered) checkpoint, or nulls when the session is empty.
- */
-export function statsSession(ctx, sessionId) {
-    const sid = normalizeSessionId(sessionId);
-    const cps = listCheckpoints(sid, ctx.stateDir);
-    const state = loadSessionState(sid, ctx.stateDir);
-    const ordered = [...cps].sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
-    const last = ordered[ordered.length - 1];
-    const injected = state.injectedCheckpointIds.length;
-    const ds = getDedupStats(ctx.stateDir);
-    const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
-    const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
-    // Per-session "tokens saved" = Σ(original − stored) over this session's
-    // stored checkpoints. Deduped adds (whole region discarded, nothing stored)
-    // are counted in the repo-wide meta counter via repoStats(); the per-session
-    // DB sum here covers the rows that exist.
-    const sessionSaved = cps.reduce((s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)), 0);
-    return {
-        checkpointCount: cps.length,
-        totalTokenEstimate: sessionTok,
-        lastCheckpointId: last?.checkpointId,
-        lastSummary: last?.summary,
-        injectedCount: injected,
-        dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
-        storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
-        tokensSaved: sessionSaved,
-        originalTokens: sessionOrig,
-        dedupAttempts: ds.attempts,
-        dedupCollapsed: ds.deduped,
-    };
-}
-/**
- * Repo-wide stats — aggregates every session in this store (one per repo).
- * Cumulative, resumable, cross-device. Surfaces the dashboard's "Repo …"
- * figures; distinct from {@link statsSession} (per-session).
- */
-export function repoStatsStore(ctx) {
-    return repoStatsFromStore(ctx.stateDir);
-}
-/** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
-export function dataInvariantStore(ctx) {
-    return dataInvariantStats(ctx.stateDir);
-}

package/dist/src/vectorStore/search.js

@@ -1,95 +0,0 @@
-/**
- * vectorStore/search.ts — Semantic search and RAPTOR integration.
- */
-import { cosineSimilarity } from "../embedder.js";
-import { normalizeSessionId } from "../store.js";
-import { mmrRerank } from "../dedup/mmr.js";
-import { topK } from "../dedup/topk.js";
-import { listCheckpoints } from "../store/sqlite.js";
-import { rehydrateRaptorTree } from "../dedup/raptor/index.js";
-import { stagedExpansion } from "../dedup/raptor/retrieval.js";
-/**
- * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
- * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
- * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
- */
-export function raptorSearchHits(ctx, sid, query, k) {
-    try {
-        const tree = rehydrateRaptorTree(sid, ctx.stateDir);
-        if (!tree || !tree.rootId)
-            return [];
-        const leafIds = stagedExpansion(query, tree, {
-            embedder: ctx.embedder,
-            k,
-            topM: ctx.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
-            mmrLambda: ctx.cfg.MMR_LAMBDA,
-        });
-        if (leafIds.length === 0)
-            return [];
-        const all = listCheckpoints(sid, ctx.stateDir).filter((cp) => cp.dedupStatus !== "removed");
-        const qv = ctx.embedder.embed(query);
-        const hits = [];
-        for (const id of leafIds) {
-            const cp = all.find((c) => c.checkpointId === id);
-            if (cp)
-                hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
-        }
-        return hits;
-    }
-    catch {
-        return [];
-    }
-}
-/**
- * Semantic search within a session's checkpoints. Returns top-K by cosine
- * similarity, diversified via MMR (QA #10) so a cluster of near-identical
- * hits yields at most a few distinct-relevance results.
- *
- * Heap-based top-K (QA #4, O(N log k)) replaces the old full sort; MMR then
- * reranks the candidate window for diversity.
- */
-export function searchStore(ctx, sessionId, query, k = 3) {
-    const sid = normalizeSessionId(sessionId);
-    const checkpoints = listCheckpoints(sid, ctx.stateDir).filter((cp) => cp.dedupStatus !== "removed");
-    if (checkpoints.length === 0)
-        return [];
-    const qv = ctx.embedder.embed(query);
-    const scored = checkpoints.map((cp) => ({
-        checkpoint: cp,
-        score: cosineSimilarity(qv, cp.embedding),
-    }));
-    // Heap top-K over a widened window (2k) so MMR has diverse candidates.
-    const window = topK(scored.map((h) => ({ item: h, score: h.score })), Math.max(k * 2, k)).map((s) => s.item);
-    // MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
-    // (Sprint 14 flag), returning the plain relevance-ranked window instead.
-    if (!ctx.cfg.L2_ENABLED)
-        return window.slice(0, k);
-    // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
-    // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
-    // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
-    // O(n) flat leaves, tightening the block at read time.
-    if (ctx.cfg.RAPTOR_ENABLED) {
-        const raptorHits = raptorSearchHits(ctx, sid, query, k);
-        if (raptorHits.length > 0) {
-            const merged = [...window];
-            for (const rh of raptorHits) {
-                if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
-                    merged.push(rh);
-                }
-            }
-            const mmrItems = merged.map((h) => ({
-                item: h,
-                vector: h.checkpoint.embedding,
-                relevance: h.score,
-            }));
-            return mmrRerank(mmrItems, k, ctx.cfg.MMR_LAMBDA);
-        }
-    }
-    const mmrItems = window.map((h) => ({
-        item: h,
-        vector: h.checkpoint.embedding,
-        relevance: h.score,
-    }));
-    const ranked = mmrRerank(mmrItems, k, ctx.cfg.MMR_LAMBDA);
-    return ranked;
-}

package/dist/src/vectorStore/session.js

@@ -1,19 +0,0 @@
-/**
- * vectorStore/session.ts — Session-state mutations (injection tracking).
- */
-import { normalizeSessionId } from "../store.js";
-import { loadSessionState, saveSessionState } from "../store/sqlite.js";
-/** Mark a checkpoint as injected into the window (recall dedup). */
-export function markInjectedStore(ctx, sessionId, checkpointId) {
-    const sid = normalizeSessionId(sessionId);
-    const state = loadSessionState(sid, ctx.stateDir);
-    if (!state.injectedCheckpointIds.includes(checkpointId)) {
-        state.injectedCheckpointIds.push(checkpointId);
-        saveSessionState(sid, state, ctx.stateDir);
-    }
-}
-/** True if this checkpoint was already injected this session. */
-export function wasInjectedStore(ctx, sessionId, checkpointId) {
-    const state = loadSessionState(normalizeSessionId(sessionId), ctx.stateDir);
-    return state.injectedCheckpointIds.includes(checkpointId);
-}

package/dist/src/vectorStore/store.js

@@ -1,105 +0,0 @@
-/**
- * vectorStore/store.ts — VectorStore class (thin wrapper delegating to
- * sub-modules). Public API is identical to the pre-split monolith.
- */
-import { defaultEmbedder } from "../embedder.js";
-import { loadDedupConfig } from "../config/dedup.js";
-import { getStateDir } from "../store.js";
-import { openBloom } from "../store/bloom.js";
-import { migrateJsonToSqlite } from "../store/migrate.js";
-import { addToStore } from "./add.js";
-import { searchStore } from "./search.js";
-import { semDedupStore, dedupeCheck } from "./dedup.js";
-import { markInjectedStore, wasInjectedStore } from "./session.js";
-import { listSession, topSimilarSession, statsSession, repoStatsStore, dataInvariantStore, similarityScore, } from "./queries.js";
-export class VectorStore {
-    ctx;
-    constructor(opts = {}) {
-        const embedder = opts.embedder ?? defaultEmbedder();
-        const stateDir = opts.stateDir ?? getStateDir();
-        // Sprint 14: all tier flags/thresholds flow from the single config source
-        // (DedupConfig). The legacy opts.dedupSim / opts.l2Enabled remain accepted
-        // for backward-compat callers but flags are authoritative via `cfg`.
-        void opts.dedupSim;
-        void opts.l2Enabled;
-        // Sprint 12 L2 semantic tier. Threshold 0.85 is the default trigram
-        // embedder's honest firing point; a direct override is allowed for tests.
-        const cfg = opts.config ?? loadDedupConfig();
-        const l2Threshold = opts.l2Threshold ?? cfg.L2_COSINE;
-        this.ctx = {
-            embedder,
-            stateDir,
-            l2Threshold,
-            cfg,
-            eventsPath: opts.eventsPath,
-        };
-        // Sprint 8: bring any v0.1.0 JSON checkpoint files into SQLite (idempotent).
-        migrateJsonToSqlite(stateDir);
-        // Sprint 10: warm the bloom accelerator (accelerator only — SQLite stays
-        // source of truth; a bloom hit is always confirmed by a query below).
-        openBloom(stateDir);
-    }
-    /** @internal Shared context for sub-module operations. */
-    get context() {
-        return this.ctx;
-    }
-    /** @internal Kept for test compatibility — delegates to ctx.stateDir. */
-    get stateDir() {
-        return this.ctx.stateDir;
-    }
-    add(input) {
-        return addToStore(this.ctx, input);
-    }
-    search(sessionId, query, k = 3) {
-        return searchStore(this.ctx, sessionId, query, k);
-    }
-    semDedup(sessionId, threshold = this.ctx.cfg.SEMDEDUP_COSINE) {
-        return semDedupStore(this.ctx, sessionId, threshold);
-    }
-    dedupe(sessionId, regionHashOrText, isText = false) {
-        return dedupeCheck(this.ctx, sessionId, regionHashOrText, isText);
-    }
-    /** Mark a checkpoint as injected into the window (recall dedup). */
-    markInjected(sessionId, checkpointId) {
-        markInjectedStore(this.ctx, sessionId, checkpointId);
-    }
-    /** True if this checkpoint was already injected this session. */
-    wasInjected(sessionId, checkpointId) {
-        return wasInjectedStore(this.ctx, sessionId, checkpointId);
-    }
-    /** Convenience for a raw vector cosine (exposed for tests). */
-    similarity(a, b) {
-        return similarityScore(a, b);
-    }
-    /** All checkpoints for a session (sorted by checkpointId). */
-    list(sessionId) {
-        return listSession(this.ctx, sessionId);
-    }
-    /**
-     * Return the n most similar checkpoints to the current (most recent) checkpoint
-     * by cosine similarity. Returns fewer than n if the session has fewer checkpoints.
-     * The current checkpoint itself is excluded from results.
-     */
-    topSimilar(sessionId, n) {
-        return topSimilarSession(this.ctx, sessionId, n);
-    }
-    /**
-     * Store statistics for status reporting / logging. Returns counts + the last
-     * (highest-numbered) checkpoint, or nulls when the session is empty.
-     */
-    stats(sessionId) {
-        return statsSession(this.ctx, sessionId);
-    }
-    /**
-     * Repo-wide stats — aggregates every session in this store (one per repo).
-     * Cumulative, resumable, cross-device. Surfaces the dashboard's "Repo …"
-     * figures; distinct from {@link stats} (per-session).
-     */
-    repoStats() {
-        return repoStatsStore(this.ctx);
-    }
-    /** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
-    dataInvariant() {
-        return dataInvariantStore(this.ctx);
-    }
-}

package/dist/src/vectorStore/types.js

@@ -1,6 +0,0 @@
-/**
- * vectorStore/types.ts — Shared types, constants, and context interface for the
- * vector store sub-modules.
- */
-/** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
-export const L2_ENABLED = true;

package/dist/src/vectorStore/utils.js

@@ -1,23 +0,0 @@
-/**
- * vectorStore/utils.ts — Hashing and monitoring utilities.
- */
-import { createHash } from "node:crypto";
-import { logDecision } from "../monitoring.js";
-/** Stable hash of a compacted region, the dedup sentinel key. */
-export function computeRegionHash(regionText) {
-    // Normalize whitespace before hashing so "foo  bar" and "foo bar" dedup.
-    const normalized = regionText.replace(/\s+/g, " ").trim();
-    return createHash("sha256").update(normalized).digest("hex").slice(0, 16);
-}
-/** Emit a structured dedup-decision event (best-effort, never throws). */
-export function recordDecision(ctx, tier, result, reason, latencyMs) {
-    if (!ctx.eventsPath)
-        return;
-    logDecision(ctx.eventsPath, {
-        ts: Date.now(),
-        tier,
-        result,
-        reason,
-        latencyMs: Math.round(latencyMs * 100) / 100,
-    });
-}