thumbgate 1.29.2 → 1.31.0

package/scripts/lesson-retrieval.js

@@ -3,20 +3,216 @@
 
 /**
  * Per-action lesson retrieval.
- * v3: bi-encoder retrieval → cross-encoder reranking
+ * v3: first-stage retrieval → field-aware BM25F reranking
  *
  * Stage 1 (bi-encoder): score all memories independently using token overlap,
  *   bigram Jaccard, tool-name matching, and recency decay.  Retrieve top-50.
- * Stage 2 (cross-encoder): rerank the top-50 candidates by computing a
- *   field-weighted BM25 score that processes (query, lesson) jointly, then
- *   blend with the original bi-encoder score.  Return top-maxResults.
+ * Stage 2 (BM25F): rerank the top-50 candidates with field-weighted lexical
+ *   evidence, then blend with the original retrieval score. This stage is not
+ *   a neural cross-encoder; neural/late-interaction/LLM stages live in the
+ *   explicit reranking cascade.
  */
 
 const RECENCY_DECAY_DAYS = 30;
 const RERANK_CANDIDATE_POOL = 50; // bi-encoder retrieves this many; reranker picks topK
+const MAX_RETRIEVAL_MEMORY_CHARS = 20000;
+
+// Line cap for reading the memory log during retrieval.
+//
+// This was 200, which quietly made relevance irrelevant. Retrieval scores memories and keeps
+// anything over 0.1, but it only ever SAW the newest 200 entries — so once 200 newer lessons
+// existed, the single most relevant lesson in the corpus became unreachable no matter how well
+// it matched. Measured on a synthetic corpus where the best-scoring lesson (0.183, threshold
+// 0.1) is the oldest entry:
+//
+//     corpus   150 -> found
+//     corpus   201 -> NOT found        <- cliff, purely from recency
+//     corpus 2,000 -> NOT found
+//
+// A firewall that forgets its oldest lessons forgets the ones it learned the hard way.
+//
+// The cap exists for cost, so it is set from measurement rather than taste. Worst case (every
+// entry scoring above threshold, so nothing filters out early):
+//
+//     200 entries 2.6 ms/call | 5,000 entries 2.6 ms/call | 20,000 entries 4.2 ms/call
+//
+// 5,000 therefore costs nothing measurable against the old 200 while covering realistic
+// corpora with wide headroom. Override with THUMBGATE_RETRIEVAL_MAX_LINES if a machine ever
+// grows past it.
+const MAX_RETRIEVAL_MEMORY_LINES = Math.max(
+  1,
+  Number(process.env.THUMBGATE_RETRIEVAL_MAX_LINES) || 5000,
+);
+
+function isRetrievableMemory(memory, options = {}) {
+  if (!memory || typeof memory !== 'object') return false;
+  const { looksLikeTransportBlob } = require('./feedback-sanitizer');
+  const title = String(memory.title || '');
+  const content = String(memory.content || '');
+  const combined = `${title}\n${content}`.trim();
+  const maxChars = Number.isFinite(options.maxMemoryChars)
+    ? Math.max(1, options.maxMemoryChars)
+    : MAX_RETRIEVAL_MEMORY_CHARS;
+  if (!combined || combined.length > maxChars) return false;
+  return !looksLikeTransportBlob(title)
+    && !looksLikeTransportBlob(content)
+    && !looksLikeTransportBlob(combined);
+}
+
+function selectRetrievalMemories(memories = [], options = {}) {
+  let selected = memories.filter((memory) => isRetrievableMemory(memory, options));
+  if (options.requireScope && !options.scope) {
+    throw new Error('Scoped lesson retrieval requires scope');
+  }
+  if (options.scope) {
+    const { selectRecordsForScope } = require('./memory-scope-readiness');
+    selected = selectRecordsForScope(selected, options.scope, {
+      includeShared: options.includeShared !== false,
+    }).allowed;
+  }
+  return selected.filter((memory) => matchesMetadataFilters(
+    memory,
+    options.metadataFilters || options.filters,
+  ));
+}
+
+function normalizedValues(value) {
+  const values = Array.isArray(value) ? value : value == null ? [] : [value];
+  return values.map((item) => String(item).trim().toLowerCase()).filter(Boolean);
+}
+
+function memoryValues(memory, field) {
+  if (field === 'toolsUsed') {
+    return normalizedValues([
+      ...(memory.metadata?.toolsUsed || []),
+      ...(memory.structuredRule?.metadata?.toolsUsed || []),
+    ]);
+  }
+  if (field === 'tags') return normalizedValues(memory.tags);
+  if (field === 'signal') {
+    return normalizedValues([
+      memory.signal,
+      memory.feedback,
+      ...(memory.tags || []),
+    ]);
+  }
+  return normalizedValues(memory.metadata?.[field] ?? memory[field]);
+}
+
+function matchesMetadataFilters(memory, filters = {}) {
+  if (!filters || typeof filters !== 'object') return true;
+  for (const field of ['domain', 'signal', 'source', 'toolsUsed']) {
+    const required = normalizedValues(filters[field]);
+    if (required.length === 0) continue;
+    const actual = memoryValues(memory, field);
+    if (!required.some((value) => actual.includes(value))) return false;
+  }
+  const requiredTags = normalizedValues(filters.tags);
+  if (requiredTags.length > 0) {
+    const actualTags = memoryValues(memory, 'tags');
+    const matches = filters.requireAllTags === false
+      ? requiredTags.some((tag) => actualTags.includes(tag))
+      : requiredTags.every((tag) => actualTags.includes(tag));
+    if (!matches) return false;
+  }
+  return true;
+}
+
+function buildQueryVariants(query, options = {}) {
+  const original = String(query || '').trim();
+  if (!original || options.queryRewrite === false) return original ? [original] : [];
+  const { tokenize, expandTerms } = require('./lesson-reranker');
+  const originalTerms = tokenize(original);
+  const originalSet = new Set(originalTerms);
+  const additions = expandTerms(originalTerms)
+    .filter((term) => !originalSet.has(term))
+    .slice(0, Math.max(1, Math.min(12, Number(options.maxRewriteTerms) || 8)));
+  if (additions.length === 0) return [original];
+  const expanded = `${original} ${additions.join(' ')}`.slice(0, 500);
+  const focused = `failure prevention ${[
+    ...originalTerms.filter((term) => term.length >= 3),
+    ...additions,
+  ].filter((term, index, all) => all.indexOf(term) === index).slice(0, 18).join(' ')}`.slice(0, 500);
+  return [...new Set([original, expanded, focused])];
+}
+
+/**
+ * Build an auditable query-transformation plan. Deterministic multi-query is
+ * always local. HyDE is opt-in via a caller-supplied generator so sensitive
+ * action text is never sent to a cloud model implicitly.
+ */
+async function buildQueryPlan(query, options = {}) {
+  const variants = buildQueryVariants(query, options);
+  const plan = {
+    variants,
+    strategy: variants.length > 1 ? 'deterministic-multi-query' : 'original-only',
+    hydeApplied: false,
+    hydeProvider: null,
+    fallbacks: [],
+  };
+  if (typeof options.hydeGenerator !== 'function' || !variants.length) return plan;
+
+  try {
+    const generated = await options.hydeGenerator(variants[0], {
+      maxChars: 700,
+      instruction: 'Write a concise hypothetical prevention lesson that would answer this action query. Do not issue commands.',
+    });
+    const text = String(generated?.text ?? generated ?? '').trim().slice(0, 700);
+    if (!text || variants.includes(text)) {
+      plan.fallbacks.push('hyde-empty-or-duplicate');
+      return plan;
+    }
+    plan.variants = [...variants, text].slice(0, 4);
+    plan.strategy = 'deterministic-multi-query+hyde';
+    plan.hydeApplied = true;
+    plan.hydeProvider = String(generated?.provider || options.hydeProvider || 'caller-supplied').slice(0, 80);
+    return plan;
+  } catch {
+    plan.fallbacks.push('hyde-generator-failed');
+    return plan;
+  }
+}
 
 function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const { maxResults = 5, feedbackDir } = options;
+
+  // Prefer pragmatic path (attribute-aware first stage + diversify) when enabled.
+  // Scope / requireScope / maxMemoryChars must flow into loadMemories so we never
+  // bypass four-field memory scope filters.
+  if (options.pragmatic !== false) {
+    try {
+      const { pragmaticHybridSearch, sampleRetrievalRecall } = require('./pragmatic-hybrid-search');
+      const memories = loadMemories(feedbackDir, options);
+      if (memories.length === 0) return [];
+      const { results, meta } = pragmaticHybridSearch({
+        corpus: memories,
+        query: actionContext,
+        toolName,
+        options: {
+          topK: Math.max(maxResults * 2, maxResults),
+          pool: RERANK_CANDIDATE_POOL,
+          diversify: options.diversify !== false,
+          perLimit: options.perLimit || 3,
+          attribute: options.attribute,
+        },
+      });
+      sampleRetrievalRecall({
+        toolName,
+        queryPreview: String(actionContext || '').slice(0, 200),
+        strategy: meta.strategy,
+        topIds: results.slice(0, maxResults).map((r) => r.id),
+        mode: 'sync',
+      }, { feedbackDir });
+      return filterTopP(
+        dedupeSupersededLessons(results),
+        resolveTopP(options),
+        { minKeep: options.minKeep },
+      ).slice(0, maxResults).map(shapeLesson);
+    } catch {
+      // fall through to classic path
+    }
+  }
+
   const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
   const { rerankLessons } = require('./lesson-reranker');
   const pathMod = require('path');
@@ -24,12 +220,15 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
     ? { MEMORY_LOG_PATH: pathMod.join(feedbackDir, 'memory-log.jsonl') }
     : getFeedbackPaths();
 
-  const memories = readJSONL(paths.MEMORY_LOG_PATH, { maxLines: 200 });
+  const memories = selectRetrievalMemories(
+    readJSONL(paths.MEMORY_LOG_PATH, { maxLines: MAX_RETRIEVAL_MEMORY_LINES }),
+    options,
+  );
   if (memories.length === 0) return [];
 
   const actionSig = buildActionSignature(toolName, actionContext);
 
-  // Stage 1 — bi-encoder: score all memories independently, take top-50 candidates
+  // Stage 1 — local first-stage score, take top-50 candidates
   const candidates = memories
     .map((mem) => ({
       ...mem,
@@ -41,7 +240,7 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
 
   if (candidates.length === 0) return [];
 
-  // Stage 2 — cross-encoder reranker: rerank candidates by joint (query, lesson) score
+  // Stage 2 — field-aware BM25F reranker (not a neural cross-encoder)
   const reranked = rerankLessons(actionContext, candidates, {
     topK: maxResults,
     toolName,
@@ -52,7 +251,7 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const deduped = dedupeSupersededLessons(reranked);
   const selected = filterTopP(deduped, resolveTopP(options), { minKeep: options.minKeep });
 
-  return selected.map((m) => ({
+  const shaped = selected.map((m) => ({
     id: m.id,
     title: m.title,
     content: m.content,
@@ -61,6 +260,28 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
     relevanceScore: m.rerankedScore ?? m.relevanceScore,
     timestamp: m.timestamp,
   }));
+
+  // Attach retrieval quality tier once (non-enumerable-ish via property on array)
+  try {
+    const { probeEmbeddingQuality } = require('./retrieval-quality-tier');
+    const quality = probeEmbeddingQuality({
+      indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
+    });
+    Object.defineProperty(shaped, 'retrievalMeta', {
+      value: {
+        strategy: 'lexical+bm25',
+        qualityTier: quality.qualityTier,
+        semanticClaimsAllowed: quality.semanticClaimsAllowed,
+        degradedReasons: quality.degradedReasons,
+        count: shaped.length,
+      },
+      enumerable: false,
+      configurable: true,
+    });
+  } catch {
+    // optional
+  }
+  return shaped;
 }
 
 /**
@@ -78,13 +299,16 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
  */
 function reciprocalRankFusion(rankedLists = [], options = {}) {
   const k = Number.isFinite(options.k) ? options.k : 60;
+  const weights = Array.isArray(options.weights) ? options.weights : [];
   const scores = new Map();
-  for (const list of rankedLists) {
+  for (let listIndex = 0; listIndex < rankedLists.length; listIndex += 1) {
+    const list = rankedLists[listIndex];
     if (!Array.isArray(list)) continue;
+    const weight = Number(weights[listIndex]) || 1;
     list.forEach((id, index) => {
       if (id === undefined || id === null) return;
       const rank = index + 1;
-      scores.set(id, (scores.get(id) || 0) + 1 / (k + rank));
+      scores.set(id, (scores.get(id) || 0) + weight / (k + rank));
     });
   }
   return [...scores.entries()]
@@ -92,17 +316,20 @@ function reciprocalRankFusion(rankedLists = [], options = {}) {
     .sort((a, b) => b.score - a.score);
 }
 
-function loadMemories(feedbackDir) {
+function loadMemories(feedbackDir, options = {}) {
   const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
   const pathMod = require('path');
   const paths = feedbackDir
     ? { MEMORY_LOG_PATH: pathMod.join(feedbackDir, 'memory-log.jsonl') }
     : getFeedbackPaths();
-  return readJSONL(paths.MEMORY_LOG_PATH, { maxLines: 200 });
+  return selectRetrievalMemories(
+    readJSONL(paths.MEMORY_LOG_PATH, { maxLines: MAX_RETRIEVAL_MEMORY_LINES }),
+    options,
+  );
 }
 
-function shapeLesson(m) {
-  return {
+function shapeLesson(m, retrieval = null) {
+  const shaped = {
     id: m.id,
     title: m.title,
     content: m.content,
@@ -111,6 +338,33 @@ function shapeLesson(m) {
     relevanceScore: m.rerankedScore ?? m.relevanceScore,
     timestamp: m.timestamp,
   };
+  if (retrieval) shaped.retrieval = retrieval;
+  return shaped;
+}
+
+/** Attach non-enumerable retrieval quality meta onto a result array. */
+function attachArrayRetrievalMeta(rows, options = {}) {
+  if (!Array.isArray(rows)) return rows;
+  try {
+    const { probeEmbeddingQuality } = require('./retrieval-quality-tier');
+    const quality = probeEmbeddingQuality({
+      indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
+    });
+    Object.defineProperty(rows, 'retrievalMeta', {
+      value: {
+        strategy: options.strategy || 'hybrid',
+        qualityTier: quality.qualityTier,
+        semanticClaimsAllowed: quality.semanticClaimsAllowed,
+        degradedReasons: quality.degradedReasons,
+        count: rows.length,
+      },
+      enumerable: false,
+      configurable: true,
+    });
+  } catch {
+    // optional
+  }
+  return rows;
 }
 
 /**
@@ -118,7 +372,7 @@ function shapeLesson(m) {
  * retrieveRelevantLessons. Used by the async gate path (gates-engine runAsync).
  *
  * Pipeline: lexical ranking ⊕ dense (embedding) ranking → Reciprocal Rank Fusion
- * → cross-encoder rerank → top-K. Dense recall surfaces past mistakes that share
+ * → BM25F rerank → top-K. Dense recall surfaces past mistakes that share
  * no keywords with the action (paraphrase/synonym) — the value lexical alone misses.
  *
  * HONEST DEGRADATION: if no real embedder is available, or embedding errors, this
@@ -140,7 +394,7 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     return retrieveRelevantLessons(toolName, actionContext, options);
   }
 
-  const memories = loadMemories(feedbackDir);
+  const memories = loadMemories(feedbackDir, options);
   if (memories.length === 0) return [];
 
   const actionSig = buildActionSignature(toolName, actionContext);
@@ -151,7 +405,6 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     .filter((m) => m.relevanceScore > 0.1)
     .sort((a, b) => b.relevanceScore - a.relevanceScore);
   const lexicalRanked = lexicalScored.slice(0, RERANK_CANDIDATE_POOL).map((m) => m.id);
-
   // Check if any lexical match is conclusive (exact/regex match on structured rule or high relevance)
   let conclusive = false;
   for (const candidate of lexicalScored) {
@@ -185,6 +438,17 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     return filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
   }
 
+  const queryPlan = lexicalScored[0]?.relevanceScore >= (options.rewriteBelowScore ?? 0.6)
+    ? {
+      variants: [String(actionContext || '')],
+      strategy: 'original-only-conclusive-lexical',
+      hydeApplied: false,
+      hydeProvider: null,
+      fallbacks: [],
+    }
+    : await buildQueryPlan(actionContext, options);
+  const queryVariants = queryPlan.variants;
+
   // WHERE-clause pruning: filter memories before vector search to only include
   // memories relevant to the current toolName or context.
   const prunedMemories = memories.filter((mem) => {
@@ -192,14 +456,18 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     const score = scoreRelevance(mem, toolName, actionContext, actionSig);
     if (score > 0.1) return true;
 
-    // 2. Otherwise, check tool compatibility: if toolsUsed is specified, it must contain our tool
-    const memTools = mem.metadata?.toolsUsed || [];
-    if (memTools.length > 0 && !memTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
-      return false;
-    }
-    const ruleTools = mem.structuredRule?.metadata?.toolsUsed || [];
-    if (ruleTools.length > 0 && !ruleTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
-      return false;
+    // 2. Tool compatibility is opt-in. A lesson learned through Bash can still
+    // matter while a Read/agent tool diagnoses the same incident. Callers that
+    // need a hard WHERE clause can use strictToolFilter or metadataFilters.
+    if (options.strictToolFilter === true) {
+      const memTools = mem.metadata?.toolsUsed || [];
+      if (memTools.length > 0 && !memTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
+        return false;
+      }
+      const ruleTools = mem.structuredRule?.metadata?.toolsUsed || [];
+      if (ruleTools.length > 0 && !ruleTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
+        return false;
+      }
     }
     return true;
   });
@@ -208,17 +476,80 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
   let semanticRanked = [];
   if (prunedMemories.length > 0) {
     try {
-      const dense = await embeddingIndex.semanticRank(actionContext, prunedMemories, {
-        feedbackDir,
-        embedder: options.embedder,
-      });
-      semanticRanked = dense.slice(0, RERANK_CANDIDATE_POOL).map((d) => d.id);
+      const denseLists = [];
+      for (const variant of queryVariants) {
+        const dense = await embeddingIndex.semanticRank(variant, prunedMemories, {
+          feedbackDir,
+          embedder: options.embedder,
+          embedderId: options.embedderId,
+          strictToolFilter: options.strictToolFilter,
+        });
+        const topScore = dense[0]?.score ?? 0;
+        const minimum = Math.max(
+          Number(options.minSemanticScore) || 0.15,
+          topScore - (Number(options.semanticScoreWindow) || 0.2),
+        );
+        denseLists.push(
+          dense
+            .filter((entry) => entry.score >= minimum)
+            .slice(0, RERANK_CANDIDATE_POOL)
+            .map((entry) => entry.id),
+        );
+      }
+      semanticRanked = reciprocalRankFusion(denseLists)
+        .slice(0, RERANK_CANDIDATE_POOL)
+        .map((entry) => entry.id);
     } catch {
       // Embedding failed at runtime → fall back to pure lexical.
       return retrieveRelevantLessons(toolName, actionContext, options);
     }
   }
 
+  // Pragmatic multi-stage path (turbopuffer hybrid playbook, local-only):
+  // multi-query (lexical ⊕ dense) → RRF → attribute-aware BM25 rerank → diversify.
+  // Falls back to legacy fuse+rerank if pragmatic module is unavailable.
+  try {
+    const { pragmaticHybridSearch, sampleRetrievalRecall } = require('./pragmatic-hybrid-search');
+    const { results, meta } = pragmaticHybridSearch({
+      corpus: memories,
+      query: actionContext,
+      toolName,
+      options: {
+        topK: Math.max(maxResults * 2, maxResults),
+        pool: RERANK_CANDIDATE_POOL,
+        denseRankedIds: semanticRanked,
+        queryVariants,
+        denseWeight: options.denseWeight,
+        fusionWeight: options.fusionWeight,
+        diversify: options.diversify !== false,
+        perLimit: options.perLimit || 3,
+        attribute: options.attribute,
+      },
+    });
+    sampleRetrievalRecall({
+      toolName,
+      queryPreview: String(actionContext || '').slice(0, 200),
+      strategy: meta.strategy,
+      topIds: results.slice(0, maxResults).map((r) => r.id),
+      densePool: meta.densePool,
+      lexicalPool: meta.lexicalPool,
+    }, { feedbackDir });
+    const cut = filterTopP(
+      dedupeSupersededLessons(results),
+      resolveTopP(options),
+      { minKeep: options.minKeep },
+    ).slice(0, maxResults);
+    const retrieval = options.includeRetrievalMeta ? {
+      ...meta,
+      queryVariants,
+      queryTransformation: queryPlan,
+      semanticProvider: options.embedderId || 'configured',
+    } : null;
+    return cut.map((lesson) => shapeLesson(lesson, retrieval));
+  } catch {
+    // Legacy fuse path below
+  }
+
   // Fuse. Candidate pool is the union — dense can introduce lessons lexical missed.
   const fused = reciprocalRankFusion([lexicalRanked, semanticRanked]);
   if (fused.length === 0) return [];
@@ -232,7 +563,7 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     .map((entry) => {
       const mem = byId.get(entry.id);
       if (!mem) return null;
-      // Carry a relevanceScore the cross-encoder can blend against. Prefer the
+      // Carry a relevanceScore the BM25F reranker can blend against. Prefer the
       // lexical score when present; otherwise use the normalized fusion score so
       // dense-only candidates still rank sensibly.
       const relevanceScore = lexById.has(entry.id)
@@ -246,7 +577,11 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
 
   const { rerankLessons } = require('./lesson-reranker');
   const reranked = rerankLessons(actionContext, candidates, { topK: maxResults, toolName });
-  return filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
+  const rows = filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
+  return attachArrayRetrievalMeta(rows, {
+    strategy: 'hybrid-rrf+bm25',
+    indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
+  });
 }
 
 function buildActionSignature(toolName, actionContext) {
@@ -482,4 +817,12 @@ module.exports = {
   filterTopP,
   resolveTopP,
   dedupeSupersededLessons,
+  isRetrievableMemory,
+  selectRetrievalMemories,
+  matchesMetadataFilters,
+  buildQueryVariants,
+  buildQueryPlan,
+  attachArrayRetrievalMeta,
+  MAX_RETRIEVAL_MEMORY_CHARS,
+  MAX_RETRIEVAL_MEMORY_LINES,
 };