pi-mega-compact 0.11.5 → 0.11.6

package/src/queryReformulation/cache.ts

@@ -0,0 +1,169 @@
+/**
+ * cache.ts — in-memory LRU cache with TTL for query reformulation results (S43A-8).
+ *
+ * Kept in a separate file to keep queryReformulation.ts under file limits.
+ * Also holds the env-constant defaults and the uncalibrated-constant bookkeeping.
+ */
+
+// ---------------------------------------------------------------------------
+// Env-Overridable Defaults (sourced from the main file's constants)
+// ---------------------------------------------------------------------------
+
+function envNum(name: string, def: number): number {
+  const v = process.env[name];
+  if (v === undefined) return def;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : def;
+}
+
+export const RRF_K = envNum("MEGACOMPACT_RRF_K", 60);
+export const EXPANSION_NEIGHBOR_COUNT = envNum("MEGACOMPACT_EXPANSION_NEIGHBOR_COUNT", 5);
+export const EXPANSION_TOP_TERMS = envNum("MEGACOMPACT_EXPANSION_TOP_TERMS", 5);
+export const VAGUE_MIN_WORDS = envNum("MEGACOMPACT_VAGUE_MIN_WORDS", 3);
+export const VAGUE_VERY_SHORT_WORDS = envNum("MEGACOMPACT_VAGUE_VERY_SHORT_WORDS", 2);
+export const QUERY_REFORM_CACHE_TTL_SECONDS = envNum("MEGACOMPACT_QUERY_REFORM_CACHE_TTL_SECONDS", 300);
+export const CACHE_MAX_SIZE = envNum("MEGACOMPACT_QUERY_REFORM_CACHE_SIZE", 100);
+const CACHE_STATS_INTERVAL = envNum("MEGACOMPACT_CACHE_STATS_INTERVAL", 100);
+
+/** Which constant names are uncalibrated (not backed by a paper/benchmark). */
+const UNCALIBRATED_NAMES: ReadonlySet<string> = new Set([
+  "EXPANSION_NEIGHBOR_COUNT",
+  "EXPANSION_TOP_TERMS",
+  "VAGUE_MIN_WORDS",
+  "VAGUE_VERY_SHORT_WORDS",
+  "QUERY_REFORM_CACHE_TTL_SECONDS",
+]);
+
+// ---------------------------------------------------------------------------
+// Which calibrated defaults are at their fallback (uncalibrated set)
+// ---------------------------------------------------------------------------
+
+export interface ReformulationConfig {
+  neighborCount: number;
+  topTerms: number;
+  vagueMinWords: number;
+  vagueVeryShortWords: number;
+  cacheTtlSeconds: number;
+  rrfK: number;
+  searchLimit: number;
+}
+
+export function computeUncalibrated(opts: ReformulationConfig): string[] {
+  const out: string[] = [];
+
+  if (opts.neighborCount === EXPANSION_NEIGHBOR_COUNT && UNCALIBRATED_NAMES.has("EXPANSION_NEIGHBOR_COUNT")) {
+    out.push("EXPANSION_NEIGHBOR_COUNT");
+  }
+  if (opts.topTerms === EXPANSION_TOP_TERMS && UNCALIBRATED_NAMES.has("EXPANSION_TOP_TERMS")) {
+    out.push("EXPANSION_TOP_TERMS");
+  }
+  if (opts.vagueMinWords === VAGUE_MIN_WORDS && UNCALIBRATED_NAMES.has("VAGUE_MIN_WORDS")) {
+    out.push("VAGUE_MIN_WORDS");
+  }
+  if (opts.vagueVeryShortWords === VAGUE_VERY_SHORT_WORDS && UNCALIBRATED_NAMES.has("VAGUE_VERY_SHORT_WORDS")) {
+    out.push("VAGUE_VERY_SHORT_WORDS");
+  }
+  if (opts.cacheTtlSeconds === QUERY_REFORM_CACHE_TTL_SECONDS && UNCALIBRATED_NAMES.has("QUERY_REFORM_CACHE_TTL_SECONDS")) {
+    out.push("QUERY_REFORM_CACHE_TTL_SECONDS");
+  }
+
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Cache (in-memory LRU + TTL)
+// ---------------------------------------------------------------------------
+
+interface ReformulationResult {
+  expanded: string;
+  original: string;
+  neighbors: Array<{ id: string; score: number }>;
+  terms: Array<{ term: string; tfIdf: number; df: number }>;
+  rrfApplied: boolean;
+  fromCache: boolean;
+  uncalibrated: string[];
+  skipReason?: string;
+}
+
+interface CacheEntry {
+  result: ReformulationResult;
+  cachedAt: number;
+}
+
+const QUERY_CACHE = new Map<string, CacheEntry>();
+let cacheHits = 0;
+let cacheMisses = 0;
+let cacheEvictions = 0;
+let cacheCounterAtLastLog = 0;
+
+/** Read (and reset) cache stats. Used by the logger for periodic reporting. */
+export function readCacheStats(): { hits: number; misses: number; evictions: number } {
+  const out = { hits: cacheHits, misses: cacheMisses, evictions: cacheEvictions };
+  // Prevent unbounded growth of the delta-tracking state
+  cacheHits = 0;
+  cacheMisses = 0;
+  cacheEvictions = 0;
+  return out;
+}
+
+export function cacheGet(query: string, ttlMs: number): ReformulationResult | undefined {
+  const entry = QUERY_CACHE.get(query);
+  if (!entry) {
+    cacheMisses++;
+    return undefined;
+  }
+  if (Date.now() - entry.cachedAt > ttlMs) {
+    QUERY_CACHE.delete(query);
+    cacheEvictions++;
+    cacheMisses++;
+    return undefined;
+  }
+  cacheHits++;
+  return { ...entry.result, fromCache: true };
+}
+
+export function cacheSet(query: string, result: ReformulationResult): void {
+  // Evict oldest if over capacity
+  if (QUERY_CACHE.size >= CACHE_MAX_SIZE) {
+    let oldestKey: string | undefined;
+    let oldestTime = Infinity;
+    for (const [key, entry] of QUERY_CACHE) {
+      if (entry.cachedAt < oldestTime) {
+        oldestTime = entry.cachedAt;
+        oldestKey = key;
+      }
+    }
+    if (oldestKey) {
+      QUERY_CACHE.delete(oldestKey);
+      cacheEvictions++;
+    }
+  }
+
+  QUERY_CACHE.set(query, {
+    result: { ...result, fromCache: false },
+    cachedAt: Date.now(),
+  });
+}
+
+function logCacheStats(log: { info: (event: string, fields?: Record<string, unknown>) => void }): void {
+  // Use the stable readCacheStats + reset
+  const stats = readCacheStats();
+  const total = stats.hits + stats.misses;
+  if (total === 0) return;
+  log.info("query_reformulation_cache_stats", {
+    hits: stats.hits,
+    misses: stats.misses,
+    evictions: stats.evictions,
+    hitRate: total > 0 ? stats.hits / total : 0,
+  });
+}
+
+export function maybeLogCacheStats(
+  log?: { info: (event: string, fields?: Record<string, unknown>) => void },
+): void {
+  if (!log) return;
+  const total = cacheHits + cacheMisses;
+  if (total - cacheCounterAtLastLog < CACHE_STATS_INTERVAL) return;
+  cacheCounterAtLastLog = total;
+  logCacheStats(log);
+}

package/src/queryReformulation/rrf.ts

@@ -0,0 +1,51 @@
+/**
+ * rrf.ts — Reciprocal Rank Fusion (RRF) for merging ranked lists (S43A-7).
+ *
+ * Pure function: takes multiple ranked lists and returns a single fused ranking
+ * using the standard RRF formula.
+ *
+ * Reference: Cormack, G.V., Clarke, C.L.A., Buettcher, S. "Reciprocal Rank
+ * Fusion Outperforms Condorcet and Individual Rank Learning Methods."
+ * SIGIR 2009.
+ */
+
+/**
+ * Items with their original rank position (1-based).
+ */
+export interface RankedItem {
+  id: string;
+  rank: number;
+}
+
+/**
+ * Items with fused RRF score.
+ */
+export interface FusedItem {
+  id: string;
+  score: number;
+}
+
+/**
+ * Reciprocal Rank Fusion — merge multiple ranked lists into a single score.
+ * Uses the standard RRF formula: score(id) = sum(1 / (k + rank_i(id))).
+ *
+ * k defaults to 60 — the canonical value from Cormack et al. 2009.
+ * Higher k reduces the impact of high rankings; lower k gives more weight to
+ * top-ranked items.
+ */
+export function reciprocalRankFusion(
+  lists: RankedItem[][],
+  k: number = 60,
+): FusedItem[] {
+  const scores = new Map<string, number>();
+
+  for (const list of lists) {
+    for (const item of list) {
+      scores.set(item.id, (scores.get(item.id) ?? 0) + 1 / (k + item.rank));
+    }
+  }
+
+  return [...scores.entries()]
+    .map(([id, score]) => ({ id, score }))
+    .sort((a, b) => b.score - a.score);
+}

package/src/queryReformulation/tfidf.ts

@@ -0,0 +1,79 @@
+/**
+ * tfidf.ts — TF-IDF term extraction from neighbor documents (S43A-6).
+ *
+ * Extracts the highest-IDF-weighted terms from a set of neighbor documents
+ * for query expansion.
+ *
+ * Term frequency = raw count of the term in the neighbor document (normalized
+ * by document length for cross-doc comparability).
+ * Inverse document frequency = log(N / df) where N = total docs in corpus.
+ *
+ * This is NOT a stopword list: frequent terms like "the" naturally get a low
+ * IDF and rank below rare terms like "jwt" even when both have similar TF.
+ */
+
+export interface CorpusStats {
+  totalDocs: number;
+  docFreq: (term: string) => number;
+}
+
+interface NeighborText {
+  id: string;
+  text: string;
+}
+
+/**
+ * Extract the top-N terms by TF-IDF weight from the given neighbor documents.
+ *
+ * Returns an empty array when:
+ *   - neighbors list is empty
+ *   - totalDocs <= 0
+ *   - no terms survive filtering (all numeric, all single-char, etc.)
+ */
+export function extractExpansionTerms(
+  neighbors: NeighborText[],
+  corpus: CorpusStats,
+  topTerms: number,
+): Array<{ term: string; tfIdf: number; df: number }> {
+  if (neighbors.length === 0) return [];
+  const { totalDocs, docFreq } = corpus;
+  if (totalDocs <= 0) return [];
+
+  // Aggregate term frequency across all neighbor documents
+  const termTf = new Map<string, number>();
+
+  for (const neighbor of neighbors) {
+    const words = neighbor.text.toLowerCase().split(/\W+/).filter(Boolean);
+    if (words.length === 0) continue;
+
+    // Normalize TF by document length so longer docs don't dominate
+    const lenNorm = words.length;
+
+    for (const word of words) {
+      // Skip pure number tokens (short numbers)
+      if (/^\d+$/.test(word) && word.length <= 4) continue;
+      // Skip single characters
+      if (word.length < 2) continue;
+
+      // Accumulate across neighbors (multiple docs amplify a term's importance)
+      termTf.set(word, (termTf.get(word) ?? 0) + (1 / lenNorm));
+    }
+  }
+
+  // Compute TF-IDF for each term
+  const scored: Array<{ term: string; tfIdf: number; df: number }> = [];
+
+  for (const [term, tf] of termTf) {
+    const df = docFreq(term);
+    // IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) — smooth version
+    const idf = Math.log(1 + (totalDocs - df + 0.5) / (df + 0.5));
+    const tfIdf = tf * idf;
+    if (tfIdf > 0) {
+      scored.push({ term, tfIdf, df });
+    }
+  }
+
+  // Sort by TF-IDF descending, take top N
+  scored.sort((a, b) => b.tfIdf - a.tfIdf);
+  return scored.slice(0, topTerms);
+}

package/src/queryReformulation/vagueness.ts

@@ -0,0 +1,38 @@
+/**
+ * vagueness.ts — heuristic vagueness detection for S43 query reformulation.
+ *
+ * Detects whether a query is "vague" enough to benefit from reformulation.
+ */
+
+/**
+ * Heuristic: is this query "vague" enough to benefit from reformulation?
+ * Returns true if the query is short, question-like, or lacks specific terms.
+ *
+ * ANY match -> vague:
+ *   a) Total words <= `vagueMinWords` (e.g. "what about auth")
+ *   b) Contains a question word AND total words <= `vagueVeryShortWords`
+ *
+ * NOTE: Misclassification is low-cost: a specific query that's wrongly expanded
+ * just gets a slightly broader search; a vague query that's wrongly skipped
+ * just gets the raw-query search (today's behavior).
+ */
+export function isVagueQuery(
+  query: string,
+  opts: { vagueMinWords: number; vagueVeryShortWords: number },
+): boolean {
+  const cleaned = query.replace(/[^\w\s]/g, "").trim();
+  if (!cleaned) return true; // empty or punctuation-only query is vague
+
+  const words = cleaned.split(/\s+/).filter(Boolean);
+  const wordCount = words.length;
+
+  // Rule a: very short query
+  if (wordCount <= opts.vagueMinWords) return true;
+
+  // Rule b: contains a question word AND is short-ish
+  const questionWords = /^(what|why|how|when|where|who|which|does|did|is|are|was|were|can|could|would|should)\b/i;
+  const hasQuestionWord = words.some((w) => questionWords.test(w));
+  if (hasQuestionWord && wordCount <= opts.vagueVeryShortWords) return true;
+
+  return false;
+}

package/src/queryReformulation.ts

@@ -0,0 +1,355 @@
+/**
+ * queryReformulation.ts — fully local, zero-LLM query reformulation for vague
+ * recall queries (S43).
+ *
+ * Shell: delegates TF-IDF extraction to queryReformulation/tfidf.ts and
+ * vagueness detection to queryReformulation/vagueness.ts. Orchestrates the
+ * pipeline here.
+ *
+ * Pipeline:
+ *   1. Embed the raw query via the project's default TrigramEmbedder.
+ *   2. Linear-scan the real `context_chunks` store for top-N cosine neighbors.
+ *   3. Extract high-IDF terms from those neighbors (real TF-IDF, no stopword
+ *      list — high-frequency terms naturally get low IDF).
+ *   4. Expand the query with the top-IDF terms.
+ *   5. Re-embed the expanded query and fuse both ranked lists via RRF.
+ *
+ * Pi-agnostic: no pi runtime types. Pure src/ module.
+ */
+
+import { isVagueQuery } from "./queryReformulation/vagueness.js";
+import { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+import { reciprocalRankFusion } from "./queryReformulation/rrf.js";
+import {
+  cacheGet,
+  cacheSet,
+  maybeLogCacheStats,
+  computeUncalibrated,
+  QUERY_REFORM_CACHE_TTL_SECONDS,
+  EXPANSION_NEIGHBOR_COUNT,
+  EXPANSION_TOP_TERMS,
+  VAGUE_MIN_WORDS,
+  VAGUE_VERY_SHORT_WORDS,
+  RRF_K,
+} from "./queryReformulation/cache.js";
+import type { ReformulationConfig } from "./queryReformulation/cache.js";
+import type { Vector } from "./embedder.js";
+
+export { isVagueQuery } from "./queryReformulation/vagueness.js";
+export { extractExpansionTerms } from "./queryReformulation/tfidf.js";
+export { reciprocalRankFusion, type RankedItem } from "./queryReformulation/rrf.js";
+export { readCacheStats } from "./queryReformulation/cache.js";
+export type { FusedItem } from "./queryReformulation/rrf.js";
+export type { ReformulationConfig } from "./queryReformulation/cache.js";
+
+// ---------------------------------------------------------------------------
+// Public Types
+// ---------------------------------------------------------------------------
+
+export interface ReformulationResult {
+  /** The expanded query string (equals `original` when no expansion happened). */
+  expanded: string;
+  /** The original query (always preserved). */
+  original: string;
+  /** Neighbor chunks that contributed expansion terms (empty when skipped). */
+  neighbors: Array<{ id: string; score: number }>;
+  /** Top weighted terms from TF-IDF over neighbors (empty when skipped). */
+  terms: Array<{ term: string; tfIdf: number; df: number }>;
+  /** Whether RRF fusion was applied (false = raw-query-only path). */
+  rrfApplied: boolean;
+  /** Whether this result came from the in-memory cache. */
+  fromCache: boolean;
+  /**
+   * List of config-constant names that were at their default (uncalibrated)
+   * values when this reformulation ran. Never includes RRF_K.
+   */
+  uncalibrated: string[];
+  /** Reasoning label when skipped ("not_vague", "empty_corpus", "thin_corpus"). */
+  skipReason?: string;
+}
+
+export interface CorpusStats {
+  totalDocs: number;
+  docFreq: (term: string) => number;
+}
+
+export interface EmbedderLike {
+  readonly dim: number;
+  embed(text: string): Vector;
+}
+
+/** Scanner: given an embedding vector, return top-N nearest neighbor ids + scores. */
+export type NeighborScanner = (
+  embedding: Vector,
+  limit: number,
+) => Array<{ id: string; score: number }>;
+
+/** Simple search: run a query, return id+score pairs. */
+export type SimpleSearch = (
+  query: string,
+  limit: number,
+) => Array<{ id: string; score: number }>;
+
+// ---------------------------------------------------------------------------
+// Embedding Neighbor Finder (S43A-5)
+// ---------------------------------------------------------------------------
+
+/**
+ * Find the top-N nearest neighbor chunks by real cosine similarity over the
+ * real `context_chunks` store. Uses the provided embedder + scan function.
+ */
+export function findExpansionNeighbors(
+  query: string,
+  embedder: EmbedderLike,
+  scan: NeighborScanner,
+  neighborCount: number,
+): Array<{ id: string; score: number }> {
+  const embedding = embedder.embed(query);
+  if (embedding.length === 0) return [];
+  return scan(embedding, neighborCount);
+}
+
+// ---------------------------------------------------------------------------
+// Build Expanded Query
+// ---------------------------------------------------------------------------
+
+/**
+ * Build an expanded query by appending the top IDF-weighted terms from the
+ * expansion neighbors to the original query. Simple concatenation: the
+ * expanded string is `"originalQuery topTerm1 topTerm2 ..."`.
+ */
+export function buildExpandedQuery(
+  original: string,
+  terms: Array<{ term: string }>,
+  maxTerms: number = EXPANSION_TOP_TERMS,
+): string {
+  if (terms.length === 0) return original;
+  const extra = terms.slice(0, maxTerms).map((t) => t.term).join(" ");
+  return `${original} ${extra}`;
+}
+
+// ---------------------------------------------------------------------------
+// Public API: reformulateQuery (S43A-9)
+// ---------------------------------------------------------------------------
+
+/**
+ * Reformulate a query using the fully local pipeline:
+ *   1. Check cache (TTL-gated)
+ *   2. Check vagueness — skip if not vague
+ *   3. Find embedding neighbors
+ *   4. Extract TF-IDF terms from neighbor texts
+ *   5. Build expanded query
+ *   6. Optionally re-embed and search (caller handles RRF fusion)
+ *
+ * Error contract:
+ *   - If the embedder throws OR the scan fails: the error propagates to the
+ *     caller (NOT caught here). The caller is responsible for the
+ *     `query_reformulation_failed` event + fallback to raw query.
+ *   - If the corpus is empty: returns raw query with rrfApplied: false and
+ *     skipReason: "empty_corpus".
+ *   - If the corpus is too thin (< 2 neighbors): returns raw query with
+ *     rrfApplied: false and skipReason: "thin_corpus".
+ *   - If the query is not vague: returns raw query with rrfApplied: false and
+ *     skipReason: "not_vague".
+ */
+export function reformulateQuery(
+  query: string,
+  embedder: EmbedderLike,
+  scan: NeighborScanner,
+  corpus: CorpusStats,
+  neighborTexts: (ids: string[]) => Array<{ id: string; text: string }>,
+  opts: Partial<ReformulationConfig> = {},
+  log?: { info: (event: string, fields?: Record<string, unknown>) => void; warn: (event: string, fields?: Record<string, unknown>) => void; error: (event: string, fields?: Record<string, unknown>) => void },
+): { result: ReformulationResult; rawNeighbors: Array<{ id: string; score: number }>; rawTerms: Array<{ id: string; text: string }> } {
+  const config: ReformulationConfig = {
+    neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+    topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+    vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+    vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+    cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+    rrfK: opts.rrfK ?? RRF_K,
+    searchLimit: opts.searchLimit ?? 5,
+  };
+
+  const ttlMs = config.cacheTtlSeconds * 1000;
+
+  // 0. Check cache
+  const cached = cacheGet(query, ttlMs);
+  if (cached) {
+    maybeLogCacheStats(log);
+    return {
+      result: cached,
+      rawNeighbors: cached.neighbors,
+      rawTerms: cached.neighbors.map((n) => ({ id: n.id, text: "" })),
+    };
+  }
+
+  // 1. Vagueness check
+  if (!isVagueQuery(query, { vagueMinWords: config.vagueMinWords, vagueVeryShortWords: config.vagueVeryShortWords })) {
+    const result: ReformulationResult = {
+      expanded: query,
+      original: query,
+      neighbors: [],
+      terms: [],
+      rrfApplied: false,
+      fromCache: false,
+      uncalibrated: computeUncalibrated(config),
+      skipReason: "not_vague",
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "not_vague" });
+    return { result, rawNeighbors: [], rawTerms: [] };
+  }
+
+  // 2. Corpus check
+  if (corpus.totalDocs <= 0) {
+    const result: ReformulationResult = {
+      expanded: query,
+      original: query,
+      neighbors: [],
+      terms: [],
+      rrfApplied: false,
+      fromCache: false,
+      uncalibrated: computeUncalibrated(config),
+      skipReason: "empty_corpus",
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "empty_corpus" });
+    return { result, rawNeighbors: [], rawTerms: [] };
+  }
+
+  // 3. Find embedding neighbors
+  const neighbors = findExpansionNeighbors(query, embedder, scan, config.neighborCount);
+  if (neighbors.length < 2) {
+    const result: ReformulationResult = {
+      expanded: query,
+      original: query,
+      neighbors,
+      terms: [],
+      rrfApplied: false,
+      fromCache: false,
+      uncalibrated: computeUncalibrated(config),
+      skipReason: "thin_corpus",
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+    return { result, rawNeighbors: neighbors, rawTerms: [] };
+  }
+
+  // 4. Fetch neighbor texts for TF-IDF
+  const neighborIds = neighbors.map((n) => n.id);
+  const rawTerms = neighborTexts(neighborIds);
+
+  // 5. Extract expansion terms via TF-IDF
+  const expansionTerms = extractExpansionTerms(rawTerms, corpus, config.topTerms);
+  if (expansionTerms.length === 0) {
+    const result: ReformulationResult = {
+      expanded: query,
+      original: query,
+      neighbors,
+      terms: [],
+      rrfApplied: false,
+      fromCache: false,
+      uncalibrated: computeUncalibrated(config),
+      skipReason: "thin_corpus",
+    };
+    cacheSet(query, result);
+    maybeLogCacheStats(log);
+    log?.info("query_reformulation_skipped", { query: query.slice(0, 80), reason: "thin_corpus" });
+    return { result, rawNeighbors: neighbors, rawTerms };
+  }
+
+  // 6. Build expanded query
+  const expanded = buildExpandedQuery(query, expansionTerms, config.topTerms);
+  const uncalibrated = computeUncalibrated(config);
+
+  const result: ReformulationResult = {
+    expanded,
+    original: query,
+    neighbors: neighbors.map((n) => ({ id: n.id, score: n.score })),
+    terms: expansionTerms,
+    rrfApplied: true,
+    fromCache: false,
+    uncalibrated,
+  };
+
+  cacheSet(query, result);
+  maybeLogCacheStats(log);
+
+  log?.info("query_reformulation", {
+    query: query.slice(0, 80),
+    expanded: expanded.slice(0, 120),
+    neighborCount: neighbors.length,
+    topTerms: expansionTerms.slice(0, 5).map((t) => t.term),
+    rrfApplied: true,
+    fromCache: false,
+    uncalibrated,
+  });
+
+  return { result, rawNeighbors: neighbors, rawTerms };
+}
+
+// ---------------------------------------------------------------------------
+// Public API: reformulationSearch (S43A-9b)
+// ---------------------------------------------------------------------------
+
+export interface ReformulationSearchResult {
+  /** Fused search results (RRF-merged or raw-query-only). */
+  fused: Array<{ id: string; score: number }>;
+  /** Reformulation metadata. */
+  reformulation: ReformulationResult;
+  /** Raw-query-only search results (for the S41 quality gate). */
+  rawSearchResults: Array<{ id: string; score: number }>;
+}
+
+/**
+ * Full reformulation search pipeline:
+ *   1. Reformulate the query (embedding neighbors -> TF-IDF -> expansion).
+ *   2. If RRF was applied: run BOTH raw + expanded queries and fuse via RRF.
+ *   3. If no RRF: return raw-query results only.
+ */
+export function reformulationSearch(
+  query: string,
+  embedder: EmbedderLike,
+  scan: NeighborScanner,
+  corpus: CorpusStats,
+  neighborTexts: (ids: string[]) => Array<{ id: string; text: string }>,
+  search: SimpleSearch,
+  opts: Partial<ReformulationConfig> = {},
+  log?: { info: (event: string, fields?: Record<string, unknown>) => void; warn: (event: string, fields?: Record<string, unknown>) => void; error: (event: string, fields?: Record<string, unknown>) => void },
+): ReformulationSearchResult {
+  const config: ReformulationConfig = {
+    neighborCount: opts.neighborCount ?? EXPANSION_NEIGHBOR_COUNT,
+    topTerms: opts.topTerms ?? EXPANSION_TOP_TERMS,
+    vagueMinWords: opts.vagueMinWords ?? VAGUE_MIN_WORDS,
+    vagueVeryShortWords: opts.vagueVeryShortWords ?? VAGUE_VERY_SHORT_WORDS,
+    cacheTtlSeconds: opts.cacheTtlSeconds ?? QUERY_REFORM_CACHE_TTL_SECONDS,
+    rrfK: opts.rrfK ?? RRF_K,
+    searchLimit: opts.searchLimit ?? 5,
+  };
+
+  const { result: reformulation } = reformulateQuery(
+    query, embedder, scan, corpus, neighborTexts, opts, log,
+  );
+
+  // Always run the raw query search (needed for the S41 quality gate and as
+  // a fallback when RRF is not applied).
+  const rawSearchResults = search(query, config.searchLimit);
+
+  if (!reformulation.rrfApplied) {
+    return { fused: rawSearchResults, reformulation, rawSearchResults };
+  }
+
+  // RRF path: run expanded query search and fuse
+  const expandedResults = search(reformulation.expanded, config.searchLimit);
+
+  const rawRanked = rawSearchResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+  const expandedRanked = expandedResults.map((r, i) => ({ id: r.id, rank: i + 1 }));
+
+  const fused = reciprocalRankFusion([rawRanked, expandedRanked], config.rrfK);
+
+  return { fused, reformulation, rawSearchResults };
+}