pi-mega-compact 0.11.5 → 0.11.7

package/dist/src/queryReformulation/tfidf.js

@@ -0,0 +1,61 @@
+/**
+ * tfidf.ts — TF-IDF term extraction from neighbor documents (S43A-6).
+ *
+ * Extracts the highest-IDF-weighted terms from a set of neighbor documents
+ * for query expansion.
+ *
+ * Term frequency = raw count of the term in the neighbor document (normalized
+ * by document length for cross-doc comparability).
+ * Inverse document frequency = log(N / df) where N = total docs in corpus.
+ *
+ * This is NOT a stopword list: frequent terms like "the" naturally get a low
+ * IDF and rank below rare terms like "jwt" even when both have similar TF.
+ */
+/**
+ * Extract the top-N terms by TF-IDF weight from the given neighbor documents.
+ *
+ * Returns an empty array when:
+ *   - neighbors list is empty
+ *   - totalDocs <= 0
+ *   - no terms survive filtering (all numeric, all single-char, etc.)
+ */
+export function extractExpansionTerms(neighbors, corpus, topTerms) {
+    if (neighbors.length === 0)
+        return [];
+    const { totalDocs, docFreq } = corpus;
+    if (totalDocs <= 0)
+        return [];
+    // Aggregate term frequency across all neighbor documents
+    const termTf = new Map();
+    for (const neighbor of neighbors) {
+        const words = neighbor.text.toLowerCase().split(/\W+/).filter(Boolean);
+        if (words.length === 0)
+            continue;
+        // Normalize TF by document length so longer docs don't dominate
+        const lenNorm = words.length;
+        for (const word of words) {
+            // Skip pure number tokens (short numbers)
+            if (/^\d+$/.test(word) && word.length <= 4)
+                continue;
+            // Skip single characters
+            if (word.length < 2)
+                continue;
+            // Accumulate across neighbors (multiple docs amplify a term's importance)
+            termTf.set(word, (termTf.get(word) ?? 0) + (1 / lenNorm));
+        }
+    }
+    // Compute TF-IDF for each term
+    const scored = [];
+    for (const [term, tf] of termTf) {
+        const df = docFreq(term);
+        // IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) — smooth version
+        const idf = Math.log(1 + (totalDocs - df + 0.5) / (df + 0.5));
+        const tfIdf = tf * idf;
+        if (tfIdf > 0) {
+            scored.push({ term, tfIdf, df });
+        }
+    }
+    // Sort by TF-IDF descending, take top N
+    scored.sort((a, b) => b.tfIdf - a.tfIdf);
+    return scored.slice(0, topTerms);
+}

package/dist/src/queryReformulation/vagueness.js

@@ -0,0 +1,33 @@
+/**
+ * vagueness.ts — heuristic vagueness detection for S43 query reformulation.
+ *
+ * Detects whether a query is "vague" enough to benefit from reformulation.
+ */
+/**
+ * Heuristic: is this query "vague" enough to benefit from reformulation?
+ * Returns true if the query is short, question-like, or lacks specific terms.
+ *
+ * ANY match -> vague:
+ *   a) Total words <= `vagueMinWords` (e.g. "what about auth")
+ *   b) Contains a question word AND total words <= `vagueVeryShortWords`
+ *
+ * NOTE: Misclassification is low-cost: a specific query that's wrongly expanded
+ * just gets a slightly broader search; a vague query that's wrongly skipped
+ * just gets the raw-query search (today's behavior).
+ */
+export function isVagueQuery(query, opts) {
+    const cleaned = query.replace(/[^\w\s]/g, "").trim();
+    if (!cleaned)
+        return true; // empty or punctuation-only query is vague
+    const words = cleaned.split(/\s+/).filter(Boolean);
+    const wordCount = words.length;
+    // Rule a: very short query
+    if (wordCount <= opts.vagueMinWords)
+        return true;
+    // Rule b: contains a question word AND is short-ish
+    const questionWords = /^(what|why|how|when|where|who|which|does|did|is|are|was|were|can|could|would|should)\b/i;
+    const hasQuestionWord = words.some((w) => questionWords.test(w));
+    if (hasQuestionWord && wordCount <= opts.vagueVeryShortWords)
+        return true;
+    return false;
+}

package/dist/src/queryReformulation.js

@@ -0,0 +1,225 @@
+/**
+ * queryReformulation.ts — fully local, zero-LLM query reformulation for vague
+ * recall queries (S43).
+ *
+ * Shell: delegates TF-IDF extraction to queryReformulation/tfidf.ts and
+ * vagueness detection to queryReformulation/vagueness.ts. Orchestrates the
+ * pipeline here.
+ *
+ * Pipeline:
+ *   1. Embed the raw query via the project's default TrigramEmbedder.
+ *   2. Linear-scan the real `context_chunks` store for top-N cosine neighbors.
+ *   3. Extract high-IDF terms from those neighbors (real TF-IDF, no stopword
+ *      list — high-frequency terms naturally get low IDF).
+ *   4. Expand the query with the top-IDF terms.
+ *   5. Re-embed the expanded query and fuse both ranked lists via RRF.
+ *
+ * Pi-agnostic: no pi runtime types. Pure src/ module.
+ */
+import { isVagueQuery } from "./queryReformulation/vagueness.js";
+import { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+import { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+import { cacheGet, cacheSet, maybeLogCacheStats, computeUncalibrated, QUERY_REFORM_CACHE_TTL_SECONDS, EXPANSION_NEIGHBOR_COUNT, EXPANSION_TOP_TERMS, VAGUE_MIN_WORDS, VAGUE_VERY_SHORT_WORDS, RRF_K, } from "./queryReformulation/cache.js";
+export { isVagueQuery } from "./queryReformulation/vagueness.js";
+export { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+export { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+export { readCacheStats } from "./queryReformulation/cache.js";
+// ---------------------------------------------------------------------------
+// Embedding Neighbor Finder (S43A-5)
+// ---------------------------------------------------------------------------
+/**
+ * Find the top-N nearest neighbor chunks by real cosine similarity over the
+ * real `context_chunks` store. Uses the provided embedder + scan function.
+ */
+export function findExpansionNeighbors(query, embedder, scan, neighborCount) {
+    const embedding = embedder.embed(query);
+    if (embedding.length === 0)
+        return [];
+    return scan(embedding, neighborCount);
+}
+// ---------------------------------------------------------------------------
+// Build Expanded Query
+// ---------------------------------------------------------------------------
+/**
+ * Build an expanded query by appending the top IDF-weighted terms from the
+ * expansion neighbors to the original query. Simple concatenation: the
+ * expanded string is `"originalQuery topTerm1 topTerm2 ..."`.
+ */
+export function buildExpandedQuery(original, terms, maxTerms = EXPANSION_TOP_TERMS) {
+    if (terms.length === 0)
+        return original;
+    const extra = terms.slice(0, maxTerms).map((t) => t.term).join(" ");
+    return `${original} ${extra}`;
+}
+// ---------------------------------------------------------------------------
+// Public API: reformulateQuery (S43A-9)
+// ---------------------------------------------------------------------------
+/**
+ * Reformulate a query using the fully local pipeline:
+ *   1. Check cache (TTL-gated)
+ *   2. Check vagueness — skip if not vague
+ *   3. Find embedding neighbors
+ *   4. Extract TF-IDF terms from neighbor texts
+ *   5. Build expanded query
+ *   6. Optionally re-embed and search (caller handles RRF fusion)
+ *
+ * Error contract:
+ *   - If the embedder throws OR the scan fails: the error propagates to the
+ *     caller (NOT caught here). The caller is responsible for the
+ *     `query_reformulation_failed` event + fallback to raw query.
+ *   - If the corpus is empty: returns raw query with rrfApplied: false and
+ *     skipReason: "empty_corpus".
+ *   - If the corpus is too thin (< 2 neighbors): returns raw query with
+ *     rrfApplied: false and skipReason: "thin_corpus".
+ *   - If the query is not vague: returns raw query with rrfApplied: false and
+ *     skipReason: "not_vague".
+ */
+export function reformulateQuery(query, embedder, scan, corpus, neighborTexts, opts = {}, log) {
+    const config = {
+        neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+        topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+        vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+        vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+        cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+        rrfK: opts.rrfK ?? RRF_K,
+        searchLimit: opts.searchLimit ?? 5,
+    };
+    const ttlMs = config.cacheTtlSeconds * 1000;
+    // 0. Check cache
+    const cached = cacheGet(query, ttlMs);
+    if (cached) {
+        maybeLogCacheStats(log);
+        return {
+            result: cached,
+            rawNeighbors: cached.neighbors,
+            rawTerms: cached.neighbors.map((n) => ({ id: n.id, text: "" })),
+        };
+    }
+    // 1. Vagueness check
+    if (!isVagueQuery(query, { vagueMinWords: config.vagueMinWords, vagueVeryShortWords: config.vagueVeryShortWords })) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors: [],
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "not_vague",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "not_vague" });
+        return { result, rawNeighbors: [], rawTerms: [] };
+    }
+    // 2. Corpus check
+    if (corpus.totalDocs <= 0) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors: [],
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "empty_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "empty_corpus" });
+        return { result, rawNeighbors: [], rawTerms: [] };
+    }
+    // 3. Find embedding neighbors
+    const neighbors = findExpansionNeighbors(query, embedder, scan, config.neighborCount);
+    if (neighbors.length < 2) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors,
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "thin_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+        return { result, rawNeighbors: neighbors, rawTerms: [] };
+    }
+    // 4. Fetch neighbor texts for TF-IDF
+    const neighborIds = neighbors.map((n) => n.id);
+    const rawTerms = neighborTexts(neighborIds);
+    // 5. Extract expansion terms via TF-IDF
+    const expansionTerms = extractExpansionTerms(rawTerms, corpus, config.topTerms);
+    if (expansionTerms.length === 0) {
+        const result = {
+            expanded: query,
+            original: query,
+            neighbors,
+            terms: [],
+            rrfApplied: false,
+            fromCache: false,
+            uncalibrated: computeUncalibrated(config),
+            skipReason: "thin_corpus",
+        };
+        cacheSet(query, result);
+        maybeLogCacheStats(log);
+        log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+        return { result, rawNeighbors: neighbors, rawTerms };
+    }
+    // 6. Build expanded query
+    const expanded = buildExpandedQuery(query, expansionTerms, config.topTerms);
+    const uncalibrated = computeUncalibrated(config);
+    const result = {
+        expanded,
+        original: query,
+        neighbors: neighbors.map((n) => ({ id: n.id, score: n.score })),
+        terms: expansionTerms,
+        rrfApplied: true,
+        fromCache: false,
+        uncalibrated,
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation", {
+        query: query.slice(0, 80),
+        expanded: expanded.slice(0, 120),
+        neighborCount: neighbors.length,
+        topTerms: expansionTerms.slice(0, 5).map((t) => t.term),
+        rrfApplied: true,
+        fromCache: false,
+        uncalibrated,
+    });
+    return { result, rawNeighbors: neighbors, rawTerms };
+}
+/**
+ * Full reformulation search pipeline:
+ *   1. Reformulate the query (embedding neighbors -> TF-IDF -> expansion).
+ *   2. If RRF was applied: run BOTH raw + expanded queries and fuse via RRF.
+ *   3. If no RRF: return raw-query results only.
+ */
+export function reformulationSearch(query, embedder, scan, corpus, neighborTexts, search, opts = {}, log) {
+    const config = {
+        neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+        topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+        vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+        vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+        cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+        rrfK: opts.rrfK ?? RRF_K,
+        searchLimit: opts.searchLimit ?? 5,
+    };
+    const { result: reformulation } = reformulateQuery(query, embedder, scan, corpus, neighborTexts, opts, log);
+    // Always run the raw query search (needed for the S41 quality gate and as
+    // a fallback when RRF is not applied).
+    const rawSearchResults = search(query, config.searchLimit);
+    if (!reformulation.rrfApplied) {
+        return { fused: rawSearchResults, reformulation, rawSearchResults };
+    }
+    // RRF path: run expanded query search and fuse
+    const expandedResults = search(reformulation.expanded, config.searchLimit);
+    const rawRanked = rawSearchResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+    const expandedRanked = expandedResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+    const fused = reciprocalRankFusion([rawRanked, expandedRanked], config.rrfK);
+    return { fused, reformulation, rawSearchResults };
+}

package/dist/src/recall.js

@@ -20,6 +20,19 @@ import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
 import { rehydrateRaptorTree, isShadowMode } from "./dedup/raptor/index.js";
 import { maxCheckpointTimestamp } from "./store/sqlite.js";
 import { normalizeSessionId } from "./store.js";
+// ---------------------------------------------------------------------------
+// S57 RAG suite feature flags + module imports. Each flag defaults OFF — the
+// flag-OFF path below is byte-identical to the pre-S57 recall path (recallQuery
+// === opts.query, the single searchRecall call), so the helpers are only reached
+// when their flag is ON. Wiring is additive; the proven v0.11.5 behavior is the
+// default.
+// ---------------------------------------------------------------------------
+import { RAG_QUERY_REFORMULATION, RAG_TIERED_ROUTER, RAG_RECALL_METRICS } from "./config.js";
+import { reformulateQuery, isVagueQuery } from "./queryReformulation.js";
+import { VAGUE_MIN_WORDS, VAGUE_VERY_SHORT_WORDS } from "./queryReformulation/cache.js";
+import { getTieredRouter } from "./tieredRouter.js";
+import { computeRecallMetrics } from "./recallMetrics.js";
+import { Logger } from "./log.js";
 /** Wrap a recall block so the model reads it as restored compacted context. */
 export function formatRecallBlock(hits) {
     if (hits.length === 0)
@@ -72,6 +85,107 @@ export function formatRaptorBlock(nodes) {
         "session so far. Use it as a map of what has been covered:\n\n" +
         parts.join("\n"));
 }
+// ---------------------------------------------------------------------------
+// S57 RAG suite helpers (B1/B2/B3). Each is only called when its feature flag
+// is ON; flag-OFF never reaches them. All degrade non-fatally to the standard
+// path on any error.
+// ---------------------------------------------------------------------------
+/**
+ * B1: If the query is vague, reformulate it via embedding-neighbor TF-IDF
+ * expansion. Returns the original query when reformulation is not supported
+ * or the query is already specific. Non-fatal: any error → original query.
+ */
+function reformulateRecallQuery(query, store, sessionId) {
+    try {
+        if (!isVagueQuery(query, {
+            vagueMinWords: VAGUE_MIN_WORDS,
+            vagueVeryShortWords: VAGUE_VERY_SHORT_WORDS,
+        }))
+            return query;
+        // Pre-compute neighbor candidates once. reformulateQuery embeds the query
+        // and calls `scan` internally; we return these pre-computed neighbors (same
+        // query) so the store needs no by-embedding lookup.
+        const { hits } = searchRecall({ sessionId, query, limit: 10, skipInjected: false }, store);
+        if (hits.length < 2)
+            return query; // too few neighbors for TF-IDF
+        const neighbors = hits.map((h) => ({
+            id: h.checkpoint.checkpointId,
+            score: h.score,
+        }));
+        const textById = new Map();
+        for (const h of hits)
+            textById.set(h.checkpoint.checkpointId, h.checkpoint.summary);
+        const scan = () => neighbors;
+        const corpus = {
+            totalDocs: neighbors.length,
+            docFreq: (term) => {
+                let df = 0;
+                for (const text of textById.values()) {
+                    if (text.toLowerCase().includes(term.toLowerCase()))
+                        df++;
+                }
+                return Math.max(df, 1);
+            },
+        };
+        const neighborTexts = (ids) => ids.map((id) => ({ id, text: textById.get(id) ?? "" }));
+        const { result } = reformulateQuery(query, defaultEmbedder(), scan, corpus, neighborTexts);
+        return result.expanded.length > query.length ? result.expanded : query;
+    }
+    catch {
+        return query; // non-fatal: fall back to original query
+    }
+}
+/**
+ * B2: Synchronous tiered recall. The TieredRouter.route() is async, so the
+ * sync path can only peek the L0 in-memory cache; on miss it falls through to
+ * the standard sync vector scan. The router's L1/L2 value is realized on the
+ * async path (recallAndInlineAsync). Non-fatal.
+ */
+function runTieredRecall(query, sessionId, limit, skip, store) {
+    try {
+        const router = getTieredRouter();
+        if (!router) {
+            const result = searchRecall({ sessionId, query, limit, skipInjected: skip }, store);
+            return { newHits: result.newHits, tier: "off" };
+        }
+        // Peek the L0 in-memory cache synchronously; fall through on miss.
+        const cached = router.peekCache(sessionId, query, limit);
+        if (cached && cached.length > 0)
+            return { newHits: cached, tier: "L0-cache" };
+        const result = searchRecall({ sessionId, query, limit, skipInjected: skip }, store);
+        return { newHits: result.newHits, tier: "sync-fallback" };
+    }
+    catch {
+        const result = searchRecall({ sessionId, query, limit, skipInjected: skip }, store);
+        return { newHits: result.newHits, tier: "fallback-error" };
+    }
+}
+/**
+ * B3: Compute recall-quality metrics on the injected hits and log them.
+ * Only called when RAG_RECALL_METRICS() is ON. Non-fatal.
+ */
+function scoreAndLogRecallMetrics(query, toInject) {
+    try {
+        const logger = new Logger();
+        const metrics = computeRecallMetrics(query, toInject);
+        logger.info("recall_metrics", {
+            hitCount: toInject.length,
+            score: metrics.score,
+            pass: metrics.pass,
+        });
+        if (!metrics.pass && toInject.length > 0) {
+            logger.info("recall_metrics_low_quality", {
+                score: metrics.score,
+                relevance: metrics.breakdown.relevance,
+                coverage: metrics.breakdown.coverage,
+                diversity: metrics.breakdown.diversity,
+            });
+        }
+    }
+    catch {
+        /* non-fatal: metrics never break recall */
+    }
+}
 /**
  * Run the unified recall+dudupe+prepare-inject pipeline. Does NOT touch pi;
  * it records injections via `markInjected` so the next call dedupes. The
@@ -123,7 +237,22 @@ export function recallAndInline(opts, store) {
     // F4: thread skipInjected through to searchRecall instead of hardcoding false
     // and re-implementing the filter here. newHits is already deduped when skip
     // is true (default); equals hits when skip is false (openclaw command path).
-    const { newHits } = searchRecall({ sessionId: opts.sessionId, query: opts.query, limit, skipInjected: skip }, store);
+    //
+    // S57 B1: optionally expand a vague query via TF-IDF neighbor terms.
+    // S57 B2: optionally route via the TieredRouter's sync L0 cache peek.
+    // Both flags default OFF — recallQuery === opts.query and the single
+    // searchRecall call reproduce the pre-S57 byte-identical path.
+    const recallQuery = RAG_QUERY_REFORMULATION()
+        ? reformulateRecallQuery(opts.query, store, opts.sessionId)
+        : opts.query;
+    let newHits;
+    if (RAG_TIERED_ROUTER()) {
+        newHits = runTieredRecall(recallQuery, opts.sessionId, limit, skip, store).newHits;
+    }
+    else {
+        const result = searchRecall({ sessionId: opts.sessionId, query: recallQuery, limit, skipInjected: skip }, store);
+        newHits = result.newHits;
+    }
     // F1: hoist one embedder instance for inline dedupe (matches the async path).
     // defaultEmbedder() is deterministic but creating it per hit wastes allocations.
     const embedder = defaultEmbedder();
@@ -154,6 +283,12 @@ export function recallAndInline(opts, store) {
         blockTokens += partTokens;
         vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
     }
+    // S57 B3: optionally score + log recall quality metrics (flag-OFF: skipped,
+    // byte-identical). Scores the injected hits against the ORIGINAL query so the
+    // metric measures whether the (possibly expanded) search results stay
+    // relevant to what the user actually asked.
+    if (RAG_RECALL_METRICS())
+        scoreAndLogRecallMetrics(opts.query, toInject);
     // F3: format once — one preamble, correct [1..n] numbering.
     const recallBlock = toInject.length > 0 ? formatRecallBlock(toInject) : "";
     const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);