pi-mega-compact 0.11.4 → 0.11.6

package/dist/src/queryReformulation/cache.js

@@ -0,0 +1,123 @@
+/**
+ * cache.ts — in-memory LRU cache with TTL for query reformulation results (S43A-8).
+ *
+ * Kept in a separate file to keep queryReformulation.ts under file limits.
+ * Also holds the env-constant defaults and the uncalibrated-constant bookkeeping.
+ */
+// ---------------------------------------------------------------------------
+// Env-Overridable Defaults (sourced from the main file's constants)
+// ---------------------------------------------------------------------------
+function envNum(name, def) {
+    const v = process.env[name];
+    if (v === undefined)
+        return def;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : def;
+}
+export const RRF_K = envNum("MEGACOMPACT_RRF_K", 60);
+export const EXPANSION_NEIGHBOR_COUNT = envNum("MEGACOMPACT_EXPANSION_NEIGHBOR_COUNT", 5);
+export const EXPANSION_TOP_TERMS = envNum("MEGACOMPACT_EXPANSION_TOP_TERMS", 5);
+export const VAGUE_MIN_WORDS = envNum("MEGACOMPACT_VAGUE_MIN_WORDS", 3);
+export const VAGUE_VERY_SHORT_WORDS = envNum("MEGACOMPACT_VAGUE_VERY_SHORT_WORDS", 2);
+export const QUERY_REFORM_CACHE_TTL_SECONDS = envNum("MEGACOMPACT_QUERY_REFORM_CACHE_TTL_SECONDS", 300);
+export const CACHE_MAX_SIZE = envNum("MEGACOMPACT_QUERY_REFORM_CACHE_SIZE", 100);
+const CACHE_STATS_INTERVAL = envNum("MEGACOMPACT_CACHE_STATS_INTERVAL", 100);
+/** Which constant names are uncalibrated (not backed by a paper/benchmark). */
+const UNCALIBRATED_NAMES = new Set([
+    "EXPANSION_NEIGHBOR_COUNT",
+    "EXPANSION_TOP_TERMS",
+    "VAGUE_MIN_WORDS",
+    "VAGUE_VERY_SHORT_WORDS",
+    "QUERY_REFORM_CACHE_TTL_SECONDS",
+]);
+export function computeUncalibrated(opts) {
+    const out = [];
+    if (opts.neighborCount === EXPANSION_NEIGHBOR_COUNT && UNCALIBRATED_NAMES.has("EXPANSION_NEIGHBOR_COUNT")) {
+        out.push("EXPANSION_NEIGHBOR_COUNT");
+    }
+    if (opts.topTerms === EXPANSION_TOP_TERMS && UNCALIBRATED_NAMES.has("EXPANSION_TOP_TERMS")) {
+        out.push("EXPANSION_TOP_TERMS");
+    }
+    if (opts.vagueMinWords === VAGUE_MIN_WORDS && UNCALIBRATED_NAMES.has("VAGUE_MIN_WORDS")) {
+        out.push("VAGUE_MIN_WORDS");
+    }
+    if (opts.vagueVeryShortWords === VAGUE_VERY_SHORT_WORDS && UNCALIBRATED_NAMES.has("VAGUE_VERY_SHORT_WORDS")) {
+        out.push("VAGUE_VERY_SHORT_WORDS");
+    }
+    if (opts.cacheTtlSeconds === QUERY_REFORM_CACHE_TTL_SECONDS && UNCALIBRATED_NAMES.has("QUERY_REFORM_CACHE_TTL_SECONDS")) {
+        out.push("QUERY_REFORM_CACHE_TTL_SECONDS");
+    }
+    return out;
+}
+const QUERY_CACHE = new Map();
+let cacheHits = 0;
+let cacheMisses = 0;
+let cacheEvictions = 0;
+let cacheCounterAtLastLog = 0;
+/** Read (and reset) cache stats. Used by the logger for periodic reporting. */
+export function readCacheStats() {
+    const out = { hits: cacheHits, misses: cacheMisses, evictions: cacheEvictions };
+    // Prevent unbounded growth of the delta-tracking state
+    cacheHits = 0;
+    cacheMisses = 0;
+    cacheEvictions = 0;
+    return out;
+}
+export function cacheGet(query, ttlMs) {
+    const entry = QUERY_CACHE.get(query);
+    if (!entry) {
+        cacheMisses++;
+        return undefined;
+    }
+    if (Date.now() - entry.cachedAt > ttlMs) {
+        QUERY_CACHE.delete(query);
+        cacheEvictions++;
+        cacheMisses++;
+        return undefined;
+    }
+    cacheHits++;
+    return { ...entry.result, fromCache: true };
+}
+export function cacheSet(query, result) {
+    // Evict oldest if over capacity
+    if (QUERY_CACHE.size >= CACHE_MAX_SIZE) {
+        let oldestKey;
+        let oldestTime = Infinity;
+        for (const [key, entry] of QUERY_CACHE) {
+            if (entry.cachedAt < oldestTime) {
+                oldestTime = entry.cachedAt;
+                oldestKey = key;
+            }
+        }
+        if (oldestKey) {
+            QUERY_CACHE.delete(oldestKey);
+            cacheEvictions++;
+        }
+    }
+    QUERY_CACHE.set(query, {
+        result: { ...result, fromCache: false },
+        cachedAt: Date.now(),
+    });
+}
+function logCacheStats(log) {
+    // Use the stable readCacheStats + reset
+    const stats = readCacheStats();
+    const total = stats.hits + stats.misses;
+    if (total === 0)
+        return;
+    log.info("query_reformulation_cache_stats", {
+        hits: stats.hits,
+        misses: stats.misses,
+        evictions: stats.evictions,
+        hitRate: total > 0 ? stats.hits / total : 0,
+    });
+}
+export function maybeLogCacheStats(log) {
+    if (!log)
+        return;
+    const total = cacheHits + cacheMisses;
+    if (total - cacheCounterAtLastLog < CACHE_STATS_INTERVAL)
+        return;
+    cacheCounterAtLastLog = total;
+    logCacheStats(log);
+}

package/dist/src/queryReformulation/rrf.js

@@ -0,0 +1,29 @@
+/**
+ * rrf.ts — Reciprocal Rank Fusion (RRF) for merging ranked lists (S43A-7).
+ *
+ * Pure function: takes multiple ranked lists and returns a single fused ranking
+ * using the standard RRF formula.
+ *
+ * Reference: Cormack, G.V., Clarke, C.L.A., Buettcher, S. "Reciprocal Rank
+ * Fusion Outperforms Condorcet and Individual Rank Learning Methods."
+ * SIGIR 2009.
+ */
+/**
+ * Reciprocal Rank Fusion — merge multiple ranked lists into a single score.
+ * Uses the standard RRF formula: score(id) = sum(1 / (k + rank_i(id))).
+ *
+ * k defaults to 60 — the canonical value from Cormack et al. 2009.
+ * Higher k reduces the impact of high rankings; lower k gives more weight to
+ * top-ranked items.
+ */
+export function reciprocalRankFusion(lists, k = 60) {
+    const scores = new Map();
+    for (const list of lists) {
+        for (const item of list) {
+            scores.set(item.id, (scores.get(item.id) ?? 0) + 1 / (k + item.rank));
+        }
+    }
+    return [...scores.entries()]
+        .map(([id, score]) => ({ id, score }))
+        .sort((a, b) => b.score - a.score);
+}

package/dist/src/queryReformulation/tfidf.js

@@ -0,0 +1,61 @@
+/**
+ * tfidf.ts — TF-IDF term extraction from neighbor documents (S43A-6).
+ *
+ * Extracts the highest-IDF-weighted terms from a set of neighbor documents
+ * for query expansion.
+ *
+ * Term frequency = raw count of the term in the neighbor document (normalized
+ * by document length for cross-doc comparability).
+ * Inverse document frequency = log(N / df) where N = total docs in corpus.
+ *
+ * This is NOT a stopword list: frequent terms like "the" naturally get a low
+ * IDF and rank below rare terms like "jwt" even when both have similar TF.
+ */
+/**
+ * Extract the top-N terms by TF-IDF weight from the given neighbor documents.
+ *
+ * Returns an empty array when:
+ *   - neighbors list is empty
+ *   - totalDocs <= 0
+ *   - no terms survive filtering (all numeric, all single-char, etc.)
+ */
+export function extractExpansionTerms(neighbors, corpus, topTerms) {
+    if (neighbors.length === 0)
+        return [];
+    const { totalDocs, docFreq } = corpus;
+    if (totalDocs <= 0)
+        return [];
+    // Aggregate term frequency across all neighbor documents
+    const termTf = new Map();
+    for (const neighbor of neighbors) {
+        const words = neighbor.text.toLowerCase().split(/\W+/).filter(Boolean);
+        if (words.length === 0)
+            continue;
+        // Normalize TF by document length so longer docs don't dominate
+        const lenNorm = words.length;
+        for (const word of words) {
+            // Skip pure number tokens (short numbers)
+            if (/^\d+$/.test(word) && word.length <= 4)
+                continue;
+            // Skip single characters
+            if (word.length < 2)
+                continue;
+            // Accumulate across neighbors (multiple docs amplify a term's importance)
+            termTf.set(word, (termTf.get(word) ?? 0) + (1 / lenNorm));
+        }
+    }
+    // Compute TF-IDF for each term
+    const scored = [];
+    for (const [term, tf] of termTf) {
+        const df = docFreq(term);
+        // IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) — smooth version
+        const idf = Math.log(1 + (totalDocs - df + 0.5) / (df + 0.5));
+        const tfIdf = tf * idf;
+        if (tfIdf > 0) {
+            scored.push({ term, tfIdf, df });
+        }
+    }
+    // Sort by TF-IDF descending, take top N
+    scored.sort((a, b) => b.tfIdf - a.tfIdf);
+    return scored.slice(0, topTerms);
+}

package/dist/src/queryReformulation/vagueness.js

@@ -0,0 +1,33 @@
+/**
+ * vagueness.ts — heuristic vagueness detection for S43 query reformulation.
+ *
+ * Detects whether a query is "vague" enough to benefit from reformulation.
+ */
+/**
+ * Heuristic: is this query "vague" enough to benefit from reformulation?
+ * Returns true if the query is short, question-like, or lacks specific terms.
+ *
+ * ANY match -> vague:
+ *   a) Total words <= `vagueMinWords` (e.g. "what about auth")
+ *   b) Contains a question word AND total words <= `vagueVeryShortWords`
+ *
+ * NOTE: Misclassification is low-cost: a specific query that's wrongly expanded
+ * just gets a slightly broader search; a vague query that's wrongly skipped
+ * just gets the raw-query search (today's behavior).
+ */
+export function isVagueQuery(query, opts) {
+    const cleaned = query.replace(/[^\w\s]/g, "").trim();
+    if (!cleaned)
+        return true; // empty or punctuation-only query is vague
+    const words = cleaned.split(/\s+/).filter(Boolean);
+    const wordCount = words.length;
+    // Rule a: very short query
+    if (wordCount <= opts.vagueMinWords)
+        return true;
+    // Rule b: contains a question word AND is short-ish
+    const questionWords = /^(what|why|how|when|where|who|which|does|did|is|are|was|were|can|could|would|should)\b/i;
+    const hasQuestionWord = words.some((w) => questionWords.test(w));
+    if (hasQuestionWord && wordCount <= opts.vagueVeryShortWords)
+        return true;
+    return false;
+}

package/dist/src/queryReformulation.js

@@ -0,0 +1,225 @@
+/**
+ * queryReformulation.ts — fully local, zero-LLM query reformulation for vague
+ * recall queries (S43).
+ *
+ * Shell: delegates TF-IDF extraction to queryReformulation/tfidf.ts and
+ * vagueness detection to queryReformulation/vagueness.ts. Orchestrates the
+ * pipeline here.
+ *
+ * Pipeline:
+ *   1. Embed the raw query via the project's default TrigramEmbedder.
+ *   2. Linear-scan the real `context_chunks` store for top-N cosine neighbors.
+ *   3. Extract high-IDF terms from those neighbors (real TF-IDF, no stopword
+ *      list — high-frequency terms naturally get low IDF).
+ *   4. Expand the query with the top-IDF terms.
+ *   5. Re-embed the expanded query and fuse both ranked lists via RRF.
+ *
+ * Pi-agnostic: no pi runtime types. Pure src/ module.
+ */
+import { isVagueQuery } from "./queryReformulation/vagueness.js";
+import { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+import { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+import { cacheGet, cacheSet, maybeLogCacheStats, computeUncalibrated, QUERY_REFORM_CACHE_TTL_SECONDS, EXPANSION_NEIGHBOR_COUNT, EXPANSION_TOP_TERMS, VAGUE_MIN_WORDS, VAGUE_VERY_SHORT_WORDS, RRF_K, } from "./queryReformulation/cache.js";
+export { isVagueQuery } from "./queryReformulation/vagueness.js";
+export { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+export { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+export { readCacheStats } from "./queryReformulation/cache.js";
+// ---------------------------------------------------------------------------
+// Embedding Neighbor Finder (S43A-5)
+// ---------------------------------------------------------------------------
+/**
+ * Find the top-N nearest neighbor chunks by real cosine similarity over the
+ * real `context_chunks` store. Uses the provided embedder + scan function.
+ */
+export function findExpansionNeighbors(query, embedder, scan, neighborCount) {
+    const embedding = embedder.embed(query);
+    if (embedding.length === 0)
+        return [];
+    return scan(embedding, neighborCount);
+}
+// ---------------------------------------------------------------------------
+// Build Expanded Query
+// ---------------------------------------------------------------------------
+/**
+ * Build an expanded query by appending the top IDF-weighted terms from the
+ * expansion neighbors to the original query. Simple concatenation: the
+ * expanded string is `"originalQuery topTerm1 topTerm2 ..."`.
+ */
+export function buildExpandedQuery(original, terms, maxTerms = EXPANSION_TOP_TERMS) {
+    if (terms.length === 0)
+        return original;
+    const extra = terms.slice(0, maxTerms).map((t) => t.term).join(" ");
+    return `${original} ${extra}`;
+}
+// ---------------------------------------------------------------------------
+// Public API: reformulateQuery (S43A-9)
+// ---------------------------------------------------------------------------
+/**
+ * Reformulate a query using the fully local pipeline:
+ *   1. Check cache (TTL-gated)
+ *   2. Check vagueness — skip if not vague
+ *   3. Find embedding neighbors
+ *   4. Extract TF-IDF terms from neighbor texts
+ *   5. Build expanded query
+ *   6. Optionally re-embed and search (caller handles RRF fusion)
+ *
+ * Error contract:
+ *   - If the embedder throws OR the scan fails: the error propagates to the
+ *     caller (NOT caught here). The caller is responsible for the
+ *     `query_reformulation_failed` event + fallback to raw query.
+ *   - If the corpus is empty: returns raw query with rrfApplied: false and
+ *     skipReason: "empty_corpus".
+ *   - If the corpus is too thin (< 2 neighbors): returns raw query with
+ *     rrfApplied: false and skipReason: "thin_corpus".
+ *   - If the query is not vague: returns raw query with rrfApplied: false and
+ *     skipReason: "not_vague".
+ */
+export function reformulateQuery(query, embedder, scan, corpus, neighborTexts, opts = {}, log) {
+    const config = {
+        neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+        topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+        vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+        vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+        cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+        rrfK: opts.rrfK ?? RRF_K,
+        searchLimit: opts.searchLimit ?? 5,
+    };
+    const ttlMs = config.cacheTtlSeconds * 1000;
+    // 0. Check cache
+    const cached = cacheGet(query, ttlMs);
+    if (cached) {
+        maybeLogCacheStats(log);
+        return {
+            result: cached,
+            rawNeighbors: cached.neighbors,
+            rawTerms: cached.neighbors.map((n) => ({ id: n.id, text: "" })),
+        };
+    }
+    // 1. Vagueness check
+    if (!isVagueQuery(query, { vagueMinWords: config.vagueMinWords, vagueVeryShortWords: config.vagueVeryShortWords })) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors: [],
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "not_vague",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "not_vague" });
+        return { result, rawNeighbors: [], rawTerms: [] };
+    }
+    // 2. Corpus check
+    if (corpus.totalDocs <= 0) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors: [],
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "empty_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "empty_corpus" });
+        return { result, rawNeighbors: [], rawTerms: [] };
+    }
+    // 3. Find embedding neighbors
+    const neighbors = findExpansionNeighbors(query, embedder, scan, config.neighborCount);
+    if (neighbors.length < 2) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors,
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "thin_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+        return { result, rawNeighbors: neighbors, rawTerms: [] };
+    }
+    // 4. Fetch neighbor texts for TF-IDF
+    const neighborIds = neighbors.map((n) => n.id);
+    const rawTerms = neighborTexts(neighborIds);
+    // 5. Extract expansion terms via TF-IDF
+    const expansionTerms = extractExpansionTerms(rawTerms, corpus, config.topTerms);
+    if (expansionTerms.length === 0) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors,
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "thin_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+        return { result, rawNeighbors: neighbors, rawTerms };
+    }
+    // 6. Build expanded query
+    const expanded = buildExpandedQuery(query, expansionTerms, config.topTerms);
+    const uncalibrated = computeUncalibrated(config);
+    const result = {
+        expanded,
+        original: query,
+        neighbors: neighbors.map((n) => ({ id: n.id, score: n.score })),
+        terms: expansionTerms,
+        rrfApplied: true,
+        fromCache: false,
+        uncalibrated,
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation", {
+        query: query.slice(0, 80),
+        expanded: expanded.slice(0, 120),
+        neighborCount: neighbors.length,
+        topTerms: expansionTerms.slice(0, 5).map((t) => t.term),
+        rrfApplied: true,
+        fromCache: false,
+        uncalibrated,
+    });
+    return { result, rawNeighbors: neighbors, rawTerms };
+}
+/**
+ * Full reformulation search pipeline:
+ *   1. Reformulate the query (embedding neighbors -> TF-IDF -> expansion).
+ *   2. If RRF was applied: run BOTH raw + expanded queries and fuse via RRF.
+ *   3. If no RRF: return raw-query results only.
+ */
+export function reformulationSearch(query, embedder, scan, corpus, neighborTexts, search, opts = {}, log) {
+    const config = {
+        neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+        topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+        vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+        vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+        cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+        rrfK: opts.rrfK ?? RRF_K,
+        searchLimit: opts.searchLimit ?? 5,
+    };
+    const { result: reformulation } = reformulateQuery(query, embedder, scan, corpus, neighborTexts, opts, log);
+    // Always run the raw query search (needed for the S41 quality gate and as
+    // a fallback when RRF is not applied).
+    const rawSearchResults = search(query, config.searchLimit);
+    if (!reformulation.rrfApplied) {
+        return { fused: rawSearchResults, reformulation, rawSearchResults };
+    }
+    // RRF path: run expanded query search and fuse
+    const expandedResults = search(reformulation.expanded, config.searchLimit);
+    const rawRanked = rawSearchResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+    const expandedRanked = expandedResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+    const fused = reciprocalRankFusion([rawRanked, expandedRanked], config.rrfK);
+    return { fused, reformulation, rawSearchResults };
+}