pi-mega-compact 0.11.4 → 0.11.6

package/src/queryExpansion.ts

@@ -0,0 +1,206 @@
+/**
+ * queryExpansion.ts — S45 CRAG query expansion via embedding neighbors.
+ *
+ * Expands a query with synonym/related-term variants by finding embedding-
+ * similar checkpoints and extracting their discriminative terms. Fully local:
+ * uses the TrigramEmbedder (shipped default, zero network) and in-process
+ * cosine scan — no LLM call, no external API. PREVENT-PI-004 compliant.
+ *
+ * Pi-agnostic: no pi runtime types or imports from `src/`.
+ */
+
+import type { Embedder } from "./embedder.js";
+import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
+import type { VectorStore } from "./vectorStore.js";
+import { listCheckpoints } from "./store/sqlite.js";
+
+/**
+ * Options for query expansion.
+ */
+export interface QueryExpansionOptions {
+  /** Maximum number of expansion terms to return. Default 5. */
+  maxTerms?: number;
+  /** Minimum cosine similarity for a checkpoint to be considered. Default 0.3. */
+  minSimilarity?: number;
+  /** Embedder override (defaults to TrigramEmbedder, zero network). */
+  embedder?: Embedder;
+}
+
+const DEFAULT_MAX_TERMS = 5;
+const DEFAULT_MIN_SIM = 0.3;
+
+/**
+ * Tokenize text into lowercased alphanumeric terms, filtering stop words.
+ * Internal helper — no export needed outside this module.
+ */
+function tokenizeTerms(text: string): string[] {
+  const STOP_WORDS = new Set([
+    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "from", "as", "is", "was", "are", "were", "be",
+    "been", "has", "have", "had", "do", "does", "did", "will", "would",
+    "could", "should", "may", "might", "shall", "can", "not", "no", "nor",
+    "so", "if", "then", "than", "that", "this", "these", "those", "it",
+    "its", "i", "me", "my", "we", "our", "you", "your", "he", "she",
+    "they", "them", "their", "what", "which", "who", "whom", "when",
+    "where", "why", "how", "all", "each", "every", "both", "few", "more",
+    "most", "other", "some", "such", "only", "own", "same", "so", "than",
+    "too", "very", "just", "about", "above", "after", "again", "against",
+    "because", "before", "between", "down", "during", "out", "over",
+    "through", "under", "up", "also", "into", "off", "onto", "upon",
+  ]);
+
+  // Split on non-alphanumeric, lowercase, filter empties and stop words.
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((t) => t.length >= 3 && !STOP_WORDS.has(t));
+}
+
+/**
+ * Compute TF-IDF-like term importance scores for a set of documents.
+ * Returns a Map from term to score (log-frequency within doc * IDF).
+ */
+function termScores(
+  docs: string[],
+): Map<string, number> {
+  const docCount = docs.length;
+  if (docCount === 0) return new Map();
+
+  // Document frequency: how many docs each term appears in.
+  const df = new Map<string, number>();
+  const docTerms = docs.map((d) => {
+    const terms = tokenizeTerms(d);
+    const unique = new Set(terms);
+    for (const t of unique) {
+      df.set(t, (df.get(t) ?? 0) + 1);
+    }
+    return terms;
+  });
+
+  // Compute TF-IDF score for each term across all docs:
+  //   score = sum over docs of (tf * idf)
+  //   tf    = term frequency within doc (log-smooth)
+  //   idf   = log(1 + docCount / (1 + docFreq))
+  const scores = new Map<string, number>();
+  for (let i = 0; i < docTerms.length; i++) {
+    const terms = docTerms[i];
+    const tf = new Map<string, number>();
+    for (const t of terms) tf.set(t, (tf.get(t) ?? 0) + 1);
+    for (const [t, count] of tf) {
+      const tfVal = 1 + Math.log10(count);
+      const docFreq = df.get(t) ?? 1;
+      const idf = Math.log10(1 + docCount / (1 + docFreq));
+      scores.set(t, (scores.get(t) ?? 0) + tfVal * idf);
+    }
+  }
+
+  return scores;
+}
+
+/**
+ * Expand a query with related terms discovered from embedding-neighbor
+ * checkpoints. Uses TF-IDF term extraction from the most similar checkpoints
+ * to find novel, discriminative terms not already in the query.
+ *
+ * @param query   The original search query.
+ * @param store   VectorStore instance (for checkpoint access + embedder).
+ * @returns       Array of expansion terms (lowercased, max `maxTerms`).
+ */
+export function expandQuery(
+  query: string,
+  store: VectorStore,
+  opts: QueryExpansionOptions = {},
+): string[] {
+  const maxTerms = opts.maxTerms ?? DEFAULT_MAX_TERMS;
+  const minSim = opts.minSimilarity ?? DEFAULT_MIN_SIM;
+  const embedder = opts.embedder ?? store.embedder ?? defaultEmbedder();
+
+  // Embed the query and scan all checkpoints for nearest neighbors.
+  const qv = embedder.embed(query);
+
+  // We need a valid session scope. Since VectorStore checkpoint access flows
+  // through the SQLite store, list checkpoints from the store's stateDir.
+  // We scan all sessions' checkpoints in the local store for broad coverage.
+  // Aggregate candidate documents from the top-N similar checkpoints.
+  interface Candidate {
+    summary: string;
+    score: number;
+  }
+  const candidates: Candidate[] = [];
+  const allSessions = collectSessionIds(store.stateDir);
+  const seen = new Set<string>();
+
+  for (const sid of allSessions) {
+    const cps = listCheckpoints(sid, store.stateDir);
+    for (const cp of cps) {
+      if (!cp.summary || seen.has(cp.checkpointId)) continue;
+      seen.add(cp.checkpointId);
+      const sim = cosineSimilarity(qv, cp.embedding);
+      if (sim >= minSim) {
+        candidates.push({ summary: cp.summary, score: sim });
+      }
+    }
+  }
+
+  if (candidates.length === 0) return [];
+
+  // Sort by similarity descending, take the top 20 for term extraction.
+  const topCandidates = candidates
+    .sort((a, b) => b.score - a.score)
+    .slice(0, 20);
+
+  // Compute TF-IDF scores across the candidate summaries.
+  const docTexts = topCandidates.map((c) => c.summary);
+  const scores = termScores(docTexts);
+
+  // Filter out query terms: we want NOVEL terms, not the ones already in the query.
+  const queryTerms = new Set(tokenizeTerms(query));
+
+  // Sort terms by score descending, filter query terms and short/noise terms.
+  const ranked = [...scores.entries()]
+    .filter(([term]) => !queryTerms.has(term))
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, maxTerms)
+    .map(([term]) => term);
+
+  return ranked;
+}
+
+/**
+ * Collect unique session IDs that have at least one checkpoint in the store.
+ * Scans the SQLite checkpoint table. Returns an empty array on any error
+ * (non-fatal — query expansion degrades gracefully).
+ */
+function collectSessionIds(stateDir: string): string[] {
+  try {
+    // We import listCheckpoints per session dynamically. A simpler approach:
+    // scan via a direct SQL query rather than reading every session.
+    // Reuse store/sqlite.ts functions — we can read the sessions table.
+    // Fallback: try listing checkpoints for all known session IDs via
+    // the sessions table.
+    const { db } = loadDb(stateDir);
+    const rows = db
+      .prepare("SELECT DISTINCT session_id FROM checkpoint_epochs ORDER BY session_id")
+      .all() as Array<{ session_id: string }>;
+    return rows.map((r) => r.session_id);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Lazy-load the SQLite database handle for a given stateDir.
+ * Cached per stateDir to avoid re-opening the file on every call.
+ */
+const _dbCache = new Map<string, { db: import("node:sqlite").DatabaseSync }>();
+
+function loadDb(stateDir: string): { db: import("node:sqlite").DatabaseSync } {
+  const cached = _dbCache.get(stateDir);
+  if (cached) return cached;
+  const { DatabaseSync } = require("node:sqlite") as typeof import("node:sqlite");
+  const { join } = require("node:path") as typeof import("node:path");
+  const db = new DatabaseSync(join(stateDir, "mega-compact.db"));
+  db.exec("PRAGMA journal_mode=WAL");
+  _dbCache.set(stateDir, { db });
+  return { db };
+}

package/src/queryReformulation/cache.ts

@@ -0,0 +1,169 @@
+/**
+ * cache.ts — in-memory LRU cache with TTL for query reformulation results (S43A-8).
+ *
+ * Kept in a separate file to keep queryReformulation.ts under file limits.
+ * Also holds the env-constant defaults and the uncalibrated-constant bookkeeping.
+ */
+
+// ---------------------------------------------------------------------------
+// Env-Overridable Defaults (sourced from the main file's constants)
+// ---------------------------------------------------------------------------
+
+function envNum(name: string, def: number): number {
+  const v = process.env[name];
+  if (v === undefined) return def;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : def;
+}
+
+export const RRF_K = envNum("MEGACOMPACT_RRF_K", 60);
+export const EXPANSION_NEIGHBOR_COUNT = envNum("MEGACOMPACT_EXPANSION_NEIGHBOR_COUNT", 5);
+export const EXPANSION_TOP_TERMS = envNum("MEGACOMPACT_EXPANSION_TOP_TERMS", 5);
+export const VAGUE_MIN_WORDS = envNum("MEGACOMPACT_VAGUE_MIN_WORDS", 3);
+export const VAGUE_VERY_SHORT_WORDS = envNum("MEGACOMPACT_VAGUE_VERY_SHORT_WORDS", 2);
+export const QUERY_REFORM_CACHE_TTL_SECONDS = envNum("MEGACOMPACT_QUERY_REFORM_CACHE_TTL_SECONDS", 300);
+export const CACHE_MAX_SIZE = envNum("MEGACOMPACT_QUERY_REFORM_CACHE_SIZE", 100);
+const CACHE_STATS_INTERVAL = envNum("MEGACOMPACT_CACHE_STATS_INTERVAL", 100);
+
+/** Which constant names are uncalibrated (not backed by a paper/benchmark). */
+const UNCALIBRATED_NAMES: ReadonlySet<string> = new Set([
+  "EXPANSION_NEIGHBOR_COUNT",
+  "EXPANSION_TOP_TERMS",
+  "VAGUE_MIN_WORDS",
+  "VAGUE_VERY_SHORT_WORDS",
+  "QUERY_REFORM_CACHE_TTL_SECONDS",
+]);
+
+// ---------------------------------------------------------------------------
+// Which calibrated defaults are at their fallback (uncalibrated set)
+// ---------------------------------------------------------------------------
+
+export interface ReformulationConfig {
+  neighborCount: number;
+  topTerms: number;
+  vagueMinWords: number;
+  vagueVeryShortWords: number;
+  cacheTtlSeconds: number;
+  rrfK: number;
+  searchLimit: number;
+}
+
+export function computeUncalibrated(opts: ReformulationConfig): string[] {
+  const out: string[] = [];
+
+  if (opts.neighborCount === EXPANSION_NEIGHBOR_COUNT && UNCALIBRATED_NAMES.has("EXPANSION_NEIGHBOR_COUNT")) {
+    out.push("EXPANSION_NEIGHBOR_COUNT");
+  }
+  if (opts.topTerms === EXPANSION_TOP_TERMS && UNCALIBRATED_NAMES.has("EXPANSION_TOP_TERMS")) {
+    out.push("EXPANSION_TOP_TERMS");
+  }
+  if (opts.vagueMinWords === VAGUE_MIN_WORDS && UNCALIBRATED_NAMES.has("VAGUE_MIN_WORDS")) {
+    out.push("VAGUE_MIN_WORDS");
+  }
+  if (opts.vagueVeryShortWords === VAGUE_VERY_SHORT_WORDS && UNCALIBRATED_NAMES.has("VAGUE_VERY_SHORT_WORDS")) {
+    out.push("VAGUE_VERY_SHORT_WORDS");
+  }
+  if (opts.cacheTtlSeconds === QUERY_REFORM_CACHE_TTL_SECONDS && UNCALIBRATED_NAMES.has("QUERY_REFORM_CACHE_TTL_SECONDS")) {
+    out.push("QUERY_REFORM_CACHE_TTL_SECONDS");
+  }
+
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Cache (in-memory LRU + TTL)
+// ---------------------------------------------------------------------------
+
+interface ReformulationResult {
+  expanded: string;
+  original: string;
+  neighbors: Array<{ id: string; score: number }>;
+  terms: Array<{ term: string; tfIdf: number; df: number }>;
+  rrfApplied: boolean;
+  fromCache: boolean;
+  uncalibrated: string[];
+  skipReason?: string;
+}
+
+interface CacheEntry {
+  result: ReformulationResult;
+  cachedAt: number;
+}
+
+const QUERY_CACHE = new Map<string, CacheEntry>();
+let cacheHits = 0;
+let cacheMisses = 0;
+let cacheEvictions = 0;
+let cacheCounterAtLastLog = 0;
+
+/** Read (and reset) cache stats. Used by the logger for periodic reporting. */
+export function readCacheStats(): { hits: number; misses: number; evictions: number } {
+  const out = { hits: cacheHits, misses: cacheMisses, evictions: cacheEvictions };
+  // Prevent unbounded growth of the delta-tracking state
+  cacheHits = 0;
+  cacheMisses = 0;
+  cacheEvictions = 0;
+  return out;
+}
+
+export function cacheGet(query: string, ttlMs: number): ReformulationResult | undefined {
+  const entry = QUERY_CACHE.get(query);
+  if (!entry) {
+    cacheMisses++;
+    return undefined;
+  }
+  if (Date.now() - entry.cachedAt > ttlMs) {
+    QUERY_CACHE.delete(query);
+    cacheEvictions++;
+    cacheMisses++;
+    return undefined;
+  }
+  cacheHits++;
+  return { ...entry.result, fromCache: true };
+}
+
+export function cacheSet(query: string, result: ReformulationResult): void {
+  // Evict oldest if over capacity
+  if (QUERY_CACHE.size >= CACHE_MAX_SIZE) {
+    let oldestKey: string | undefined;
+    let oldestTime = Infinity;
+    for (const [key, entry] of QUERY_CACHE) {
+      if (entry.cachedAt < oldestTime) {
+        oldestTime = entry.cachedAt;
+        oldestKey = key;
+      }
+    }
+    if (oldestKey) {
+      QUERY_CACHE.delete(oldestKey);
+      cacheEvictions++;
+    }
+  }
+
+  QUERY_CACHE.set(query, {
+    result: { ...result, fromCache: false },
+    cachedAt: Date.now(),
+  });
+}
+
+function logCacheStats(log: { info: (event: string, fields?: Record<string, unknown>) => void }): void {
+  // Use the stable readCacheStats + reset
+  const stats = readCacheStats();
+  const total = stats.hits + stats.misses;
+  if (total === 0) return;
+  log.info("query_reformulation_cache_stats", {
+    hits: stats.hits,
+    misses: stats.misses,
+    evictions: stats.evictions,
+    hitRate: total > 0 ? stats.hits / total : 0,
+  });
+}
+
+export function maybeLogCacheStats(
+  log?: { info: (event: string, fields?: Record<string, unknown>) => void },
+): void {
+  if (!log) return;
+  const total = cacheHits + cacheMisses;
+  if (total - cacheCounterAtLastLog < CACHE_STATS_INTERVAL) return;
+  cacheCounterAtLastLog = total;
+  logCacheStats(log);
+}

package/src/queryReformulation/rrf.ts

@@ -0,0 +1,51 @@
+/**
+ * rrf.ts — Reciprocal Rank Fusion (RRF) for merging ranked lists (S43A-7).
+ *
+ * Pure function: takes multiple ranked lists and returns a single fused ranking
+ * using the standard RRF formula.
+ *
+ * Reference: Cormack, G.V., Clarke, C.L.A., Buettcher, S. "Reciprocal Rank
+ * Fusion Outperforms Condorcet and Individual Rank Learning Methods."
+ * SIGIR 2009.
+ */
+
+/**
+ * Items with their original rank position (1-based).
+ */
+export interface RankedItem {
+  id: string;
+  rank: number;
+}
+
+/**
+ * Items with fused RRF score.
+ */
+export interface FusedItem {
+  id: string;
+  score: number;
+}
+
+/**
+ * Reciprocal Rank Fusion — merge multiple ranked lists into a single score.
+ * Uses the standard RRF formula: score(id) = sum(1 / (k + rank_i(id))).
+ *
+ * k defaults to 60 — the canonical value from Cormack et al. 2009.
+ * Higher k reduces the impact of high rankings; lower k gives more weight to
+ * top-ranked items.
+ */
+export function reciprocalRankFusion(
+  lists: RankedItem[][],
+  k: number = 60,
+): FusedItem[] {
+  const scores = new Map<string, number>();
+
+  for (const list of lists) {
+    for (const item of list) {
+      scores.set(item.id, (scores.get(item.id) ?? 0) + 1 / (k + item.rank));
+    }
+  }
+
+  return [...scores.entries()]
+    .map(([id, score]) => ({ id, score }))
+    .sort((a, b) => b.score - a.score);
+}

package/src/queryReformulation/tfidf.ts

@@ -0,0 +1,79 @@
+/**
+ * tfidf.ts — TF-IDF term extraction from neighbor documents (S43A-6).
+ *
+ * Extracts the highest-IDF-weighted terms from a set of neighbor documents
+ * for query expansion.
+ *
+ * Term frequency = raw count of the term in the neighbor document (normalized
+ * by document length for cross-doc comparability).
+ * Inverse document frequency = log(N / df) where N = total docs in corpus.
+ *
+ * This is NOT a stopword list: frequent terms like "the" naturally get a low
+ * IDF and rank below rare terms like "jwt" even when both have similar TF.
+ */
+
+export interface CorpusStats {
+  totalDocs: number;
+  docFreq: (term: string) => number;
+}
+
+interface NeighborText {
+  id: string;
+  text: string;
+}
+
+/**
+ * Extract the top-N terms by TF-IDF weight from the given neighbor documents.
+ *
+ * Returns an empty array when:
+ *   - neighbors list is empty
+ *   - totalDocs <= 0
+ *   - no terms survive filtering (all numeric, all single-char, etc.)
+ */
+export function extractExpansionTerms(
+  neighbors: NeighborText[],
+  corpus: CorpusStats,
+  topTerms: number,
+): Array<{ term: string; tfIdf: number; df: number }> {
+  if (neighbors.length === 0) return [];
+  const { totalDocs, docFreq } = corpus;
+  if (totalDocs <= 0) return [];
+
+  // Aggregate term frequency across all neighbor documents
+  const termTf = new Map<string, number>();
+
+  for (const neighbor of neighbors) {
+    const words = neighbor.text.toLowerCase().split(/\W+/).filter(Boolean);
+    if (words.length === 0) continue;
+
+    // Normalize TF by document length so longer docs don't dominate
+    const lenNorm = words.length;
+
+    for (const word of words) {
+      // Skip pure number tokens (short numbers)
+      if (/^\d+$/.test(word) && word.length <= 4) continue;
+      // Skip single characters
+      if (word.length < 2) continue;
+
+      // Accumulate across neighbors (multiple docs amplify a term's importance)
+      termTf.set(word, (termTf.get(word) ?? 0) + (1 / lenNorm));
+    }
+  }
+
+  // Compute TF-IDF for each term
+  const scored: Array<{ term: string; tfIdf: number; df: number }> = [];
+
+  for (const [term, tf] of termTf) {
+    const df = docFreq(term);
+    // IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) — smooth version
+    const idf = Math.log(1 + (totalDocs - df + 0.5) / (df + 0.5));
+    const tfIdf = tf * idf;
+    if (tfIdf > 0) {
+      scored.push({ term, tfIdf, df });
+    }
+  }
+
+  // Sort by TF-IDF descending, take top N
+  scored.sort((a, b) => b.tfIdf - a.tfIdf);
+  return scored.slice(0, topTerms);
+}

package/src/queryReformulation/vagueness.ts

@@ -0,0 +1,38 @@
+/**
+ * vagueness.ts — heuristic vagueness detection for S43 query reformulation.
+ *
+ * Detects whether a query is "vague" enough to benefit from reformulation.
+ */
+
+/**
+ * Heuristic: is this query "vague" enough to benefit from reformulation?
+ * Returns true if the query is short, question-like, or lacks specific terms.
+ *
+ * ANY match -> vague:
+ *   a) Total words <= `vagueMinWords` (e.g. "what about auth")
+ *   b) Contains a question word AND total words <= `vagueVeryShortWords`
+ *
+ * NOTE: Misclassification is low-cost: a specific query that's wrongly expanded
+ * just gets a slightly broader search; a vague query that's wrongly skipped
+ * just gets the raw-query search (today's behavior).
+ */
+export function isVagueQuery(
+  query: string,
+  opts: { vagueMinWords: number; vagueVeryShortWords: number },
+): boolean {
+  const cleaned = query.replace(/[^\w\s]/g, "").trim();
+  if (!cleaned) return true; // empty or punctuation-only query is vague
+
+  const words = cleaned.split(/\s+/).filter(Boolean);
+  const wordCount = words.length;
+
+  // Rule a: very short query
+  if (wordCount <= opts.vagueMinWords) return true;
+
+  // Rule b: contains a question word AND is short-ish
+  const questionWords = /^(what|why|how|when|where|who|which|does|did|is|are|was|were|can|could|would|should)\b/i;
+  const hasQuestionWord = words.some((w) => questionWords.test(w));
+  if (hasQuestionWord && wordCount <= opts.vagueVeryShortWords) return true;
+
+  return false;
+}