thumbgate 1.30.0 → 1.34.0

package/scripts/lesson-retrieval.js

@@ -3,13 +3,14 @@
 
 /**
  * Per-action lesson retrieval.
- * v3: bi-encoder retrieval → cross-encoder reranking
+ * v3: first-stage retrieval → field-aware BM25F reranking
  *
  * Stage 1 (bi-encoder): score all memories independently using token overlap,
  *   bigram Jaccard, tool-name matching, and recency decay.  Retrieve top-50.
- * Stage 2 (cross-encoder): rerank the top-50 candidates by computing a
- *   field-weighted BM25 score that processes (query, lesson) jointly, then
- *   blend with the original bi-encoder score.  Return top-maxResults.
+ * Stage 2 (BM25F): rerank the top-50 candidates with field-weighted lexical
+ *   evidence, then blend with the original retrieval score. This stage is not
+ *   a neural cross-encoder; neural/late-interaction/LLM stages live in the
+ *   explicit reranking cascade.
  */
 
 const RECENCY_DECAY_DAYS = 30;
@@ -69,11 +70,149 @@ function selectRetrievalMemories(memories = [], options = {}) {
       includeShared: options.includeShared !== false,
     }).allowed;
   }
-  return selected;
+  return selected.filter((memory) => matchesMetadataFilters(
+    memory,
+    options.metadataFilters || options.filters,
+  ));
+}
+
+function normalizedValues(value) {
+  const values = Array.isArray(value) ? value : value == null ? [] : [value];
+  return values.map((item) => String(item).trim().toLowerCase()).filter(Boolean);
+}
+
+function memoryValues(memory, field) {
+  if (field === 'toolsUsed') {
+    return normalizedValues([
+      ...(memory.metadata?.toolsUsed || []),
+      ...(memory.structuredRule?.metadata?.toolsUsed || []),
+    ]);
+  }
+  if (field === 'tags') return normalizedValues(memory.tags);
+  if (field === 'signal') {
+    return normalizedValues([
+      memory.signal,
+      memory.feedback,
+      ...(memory.tags || []),
+    ]);
+  }
+  return normalizedValues(memory.metadata?.[field] ?? memory[field]);
+}
+
+function matchesMetadataFilters(memory, filters = {}) {
+  if (!filters || typeof filters !== 'object') return true;
+  for (const field of ['domain', 'signal', 'source', 'toolsUsed']) {
+    const required = normalizedValues(filters[field]);
+    if (required.length === 0) continue;
+    const actual = memoryValues(memory, field);
+    if (!required.some((value) => actual.includes(value))) return false;
+  }
+  const requiredTags = normalizedValues(filters.tags);
+  if (requiredTags.length > 0) {
+    const actualTags = memoryValues(memory, 'tags');
+    const matches = filters.requireAllTags === false
+      ? requiredTags.some((tag) => actualTags.includes(tag))
+      : requiredTags.every((tag) => actualTags.includes(tag));
+    if (!matches) return false;
+  }
+  return true;
+}
+
+function buildQueryVariants(query, options = {}) {
+  const original = String(query || '').trim();
+  if (!original || options.queryRewrite === false) return original ? [original] : [];
+  const { tokenize, expandTerms } = require('./lesson-reranker');
+  const originalTerms = tokenize(original);
+  const originalSet = new Set(originalTerms);
+  const additions = expandTerms(originalTerms)
+    .filter((term) => !originalSet.has(term))
+    .slice(0, Math.max(1, Math.min(12, Number(options.maxRewriteTerms) || 8)));
+  if (additions.length === 0) return [original];
+  const expanded = `${original} ${additions.join(' ')}`.slice(0, 500);
+  const focused = `failure prevention ${[
+    ...originalTerms.filter((term) => term.length >= 3),
+    ...additions,
+  ].filter((term, index, all) => all.indexOf(term) === index).slice(0, 18).join(' ')}`.slice(0, 500);
+  return [...new Set([original, expanded, focused])];
+}
+
+/**
+ * Build an auditable query-transformation plan. Deterministic multi-query is
+ * always local. HyDE is opt-in via a caller-supplied generator so sensitive
+ * action text is never sent to a cloud model implicitly.
+ */
+async function buildQueryPlan(query, options = {}) {
+  const variants = buildQueryVariants(query, options);
+  const plan = {
+    variants,
+    strategy: variants.length > 1 ? 'deterministic-multi-query' : 'original-only',
+    hydeApplied: false,
+    hydeProvider: null,
+    fallbacks: [],
+  };
+  if (typeof options.hydeGenerator !== 'function' || !variants.length) return plan;
+
+  try {
+    const generated = await options.hydeGenerator(variants[0], {
+      maxChars: 700,
+      instruction: 'Write a concise hypothetical prevention lesson that would answer this action query. Do not issue commands.',
+    });
+    const text = String(generated?.text ?? generated ?? '').trim().slice(0, 700);
+    if (!text || variants.includes(text)) {
+      plan.fallbacks.push('hyde-empty-or-duplicate');
+      return plan;
+    }
+    plan.variants = [...variants, text].slice(0, 4);
+    plan.strategy = 'deterministic-multi-query+hyde';
+    plan.hydeApplied = true;
+    plan.hydeProvider = String(generated?.provider || options.hydeProvider || 'caller-supplied').slice(0, 80);
+    return plan;
+  } catch {
+    plan.fallbacks.push('hyde-generator-failed');
+    return plan;
+  }
 }
 
 function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const { maxResults = 5, feedbackDir } = options;
+
+  // Prefer pragmatic path (attribute-aware first stage + diversify) when enabled.
+  // Scope / requireScope / maxMemoryChars must flow into loadMemories so we never
+  // bypass four-field memory scope filters.
+  if (options.pragmatic !== false) {
+    try {
+      const { pragmaticHybridSearch, sampleRetrievalRecall } = require('./pragmatic-hybrid-search');
+      const memories = loadMemories(feedbackDir, options);
+      if (memories.length === 0) return [];
+      const { results, meta } = pragmaticHybridSearch({
+        corpus: memories,
+        query: actionContext,
+        toolName,
+        options: {
+          topK: Math.max(maxResults * 2, maxResults),
+          pool: RERANK_CANDIDATE_POOL,
+          diversify: options.diversify !== false,
+          perLimit: options.perLimit || 3,
+          attribute: options.attribute,
+        },
+      });
+      sampleRetrievalRecall({
+        toolName,
+        queryPreview: String(actionContext || '').slice(0, 200),
+        strategy: meta.strategy,
+        topIds: results.slice(0, maxResults).map((r) => r.id),
+        mode: 'sync',
+      }, { feedbackDir });
+      return filterTopP(
+        dedupeSupersededLessons(results),
+        resolveTopP(options),
+        { minKeep: options.minKeep },
+      ).slice(0, maxResults).map(shapeLesson);
+    } catch {
+      // fall through to classic path
+    }
+  }
+
   const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
   const { rerankLessons } = require('./lesson-reranker');
   const pathMod = require('path');
@@ -89,7 +228,7 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
 
   const actionSig = buildActionSignature(toolName, actionContext);
 
-  // Stage 1 — bi-encoder: score all memories independently, take top-50 candidates
+  // Stage 1 — local first-stage score, take top-50 candidates
   const candidates = memories
     .map((mem) => ({
       ...mem,
@@ -101,7 +240,7 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
 
   if (candidates.length === 0) return [];
 
-  // Stage 2 — cross-encoder reranker: rerank candidates by joint (query, lesson) score
+  // Stage 2 — field-aware BM25F reranker (not a neural cross-encoder)
   const reranked = rerankLessons(actionContext, candidates, {
     topK: maxResults,
     toolName,
@@ -112,7 +251,7 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const deduped = dedupeSupersededLessons(reranked);
   const selected = filterTopP(deduped, resolveTopP(options), { minKeep: options.minKeep });
 
-  return selected.map((m) => ({
+  const shaped = selected.map((m) => ({
     id: m.id,
     title: m.title,
     content: m.content,
@@ -121,6 +260,28 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
     relevanceScore: m.rerankedScore ?? m.relevanceScore,
     timestamp: m.timestamp,
   }));
+
+  // Attach retrieval quality tier once (non-enumerable-ish via property on array)
+  try {
+    const { probeEmbeddingQuality } = require('./retrieval-quality-tier');
+    const quality = probeEmbeddingQuality({
+      indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
+    });
+    Object.defineProperty(shaped, 'retrievalMeta', {
+      value: {
+        strategy: 'lexical+bm25',
+        qualityTier: quality.qualityTier,
+        semanticClaimsAllowed: quality.semanticClaimsAllowed,
+        degradedReasons: quality.degradedReasons,
+        count: shaped.length,
+      },
+      enumerable: false,
+      configurable: true,
+    });
+  } catch {
+    // optional
+  }
+  return shaped;
 }
 
 /**
@@ -138,13 +299,16 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
  */
 function reciprocalRankFusion(rankedLists = [], options = {}) {
   const k = Number.isFinite(options.k) ? options.k : 60;
+  const weights = Array.isArray(options.weights) ? options.weights : [];
   const scores = new Map();
-  for (const list of rankedLists) {
+  for (let listIndex = 0; listIndex < rankedLists.length; listIndex += 1) {
+    const list = rankedLists[listIndex];
     if (!Array.isArray(list)) continue;
+    const weight = Number(weights[listIndex]) || 1;
     list.forEach((id, index) => {
       if (id === undefined || id === null) return;
       const rank = index + 1;
-      scores.set(id, (scores.get(id) || 0) + 1 / (k + rank));
+      scores.set(id, (scores.get(id) || 0) + weight / (k + rank));
     });
   }
   return [...scores.entries()]
@@ -164,8 +328,8 @@ function loadMemories(feedbackDir, options = {}) {
   );
 }
 
-function shapeLesson(m) {
-  return {
+function shapeLesson(m, retrieval = null) {
+  const shaped = {
     id: m.id,
     title: m.title,
     content: m.content,
@@ -174,6 +338,33 @@ function shapeLesson(m) {
     relevanceScore: m.rerankedScore ?? m.relevanceScore,
     timestamp: m.timestamp,
   };
+  if (retrieval) shaped.retrieval = retrieval;
+  return shaped;
+}
+
+/** Attach non-enumerable retrieval quality meta onto a result array. */
+function attachArrayRetrievalMeta(rows, options = {}) {
+  if (!Array.isArray(rows)) return rows;
+  try {
+    const { probeEmbeddingQuality } = require('./retrieval-quality-tier');
+    const quality = probeEmbeddingQuality({
+      indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
+    });
+    Object.defineProperty(rows, 'retrievalMeta', {
+      value: {
+        strategy: options.strategy || 'hybrid',
+        qualityTier: quality.qualityTier,
+        semanticClaimsAllowed: quality.semanticClaimsAllowed,
+        degradedReasons: quality.degradedReasons,
+        count: rows.length,
+      },
+      enumerable: false,
+      configurable: true,
+    });
+  } catch {
+    // optional
+  }
+  return rows;
 }
 
 /**
@@ -181,7 +372,7 @@ function shapeLesson(m) {
  * retrieveRelevantLessons. Used by the async gate path (gates-engine runAsync).
  *
  * Pipeline: lexical ranking ⊕ dense (embedding) ranking → Reciprocal Rank Fusion
- * → cross-encoder rerank → top-K. Dense recall surfaces past mistakes that share
+ * → BM25F rerank → top-K. Dense recall surfaces past mistakes that share
  * no keywords with the action (paraphrase/synonym) — the value lexical alone misses.
  *
  * HONEST DEGRADATION: if no real embedder is available, or embedding errors, this
@@ -214,7 +405,6 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     .filter((m) => m.relevanceScore > 0.1)
     .sort((a, b) => b.relevanceScore - a.relevanceScore);
   const lexicalRanked = lexicalScored.slice(0, RERANK_CANDIDATE_POOL).map((m) => m.id);
-
   // Check if any lexical match is conclusive (exact/regex match on structured rule or high relevance)
   let conclusive = false;
   for (const candidate of lexicalScored) {
@@ -248,6 +438,17 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     return filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
   }
 
+  const queryPlan = lexicalScored[0]?.relevanceScore >= (options.rewriteBelowScore ?? 0.6)
+    ? {
+      variants: [String(actionContext || '')],
+      strategy: 'original-only-conclusive-lexical',
+      hydeApplied: false,
+      hydeProvider: null,
+      fallbacks: [],
+    }
+    : await buildQueryPlan(actionContext, options);
+  const queryVariants = queryPlan.variants;
+
   // WHERE-clause pruning: filter memories before vector search to only include
   // memories relevant to the current toolName or context.
   const prunedMemories = memories.filter((mem) => {
@@ -255,14 +456,18 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     const score = scoreRelevance(mem, toolName, actionContext, actionSig);
     if (score > 0.1) return true;
 
-    // 2. Otherwise, check tool compatibility: if toolsUsed is specified, it must contain our tool
-    const memTools = mem.metadata?.toolsUsed || [];
-    if (memTools.length > 0 && !memTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
-      return false;
-    }
-    const ruleTools = mem.structuredRule?.metadata?.toolsUsed || [];
-    if (ruleTools.length > 0 && !ruleTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
-      return false;
+    // 2. Tool compatibility is opt-in. A lesson learned through Bash can still
+    // matter while a Read/agent tool diagnoses the same incident. Callers that
+    // need a hard WHERE clause can use strictToolFilter or metadataFilters.
+    if (options.strictToolFilter === true) {
+      const memTools = mem.metadata?.toolsUsed || [];
+      if (memTools.length > 0 && !memTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
+        return false;
+      }
+      const ruleTools = mem.structuredRule?.metadata?.toolsUsed || [];
+      if (ruleTools.length > 0 && !ruleTools.some(t => t.toLowerCase() === toolName.toLowerCase())) {
+        return false;
+      }
     }
     return true;
   });
@@ -271,17 +476,80 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
   let semanticRanked = [];
   if (prunedMemories.length > 0) {
     try {
-      const dense = await embeddingIndex.semanticRank(actionContext, prunedMemories, {
-        feedbackDir,
-        embedder: options.embedder,
-      });
-      semanticRanked = dense.slice(0, RERANK_CANDIDATE_POOL).map((d) => d.id);
+      const denseLists = [];
+      for (const variant of queryVariants) {
+        const dense = await embeddingIndex.semanticRank(variant, prunedMemories, {
+          feedbackDir,
+          embedder: options.embedder,
+          embedderId: options.embedderId,
+          strictToolFilter: options.strictToolFilter,
+        });
+        const topScore = dense[0]?.score ?? 0;
+        const minimum = Math.max(
+          Number(options.minSemanticScore) || 0.15,
+          topScore - (Number(options.semanticScoreWindow) || 0.2),
+        );
+        denseLists.push(
+          dense
+            .filter((entry) => entry.score >= minimum)
+            .slice(0, RERANK_CANDIDATE_POOL)
+            .map((entry) => entry.id),
+        );
+      }
+      semanticRanked = reciprocalRankFusion(denseLists)
+        .slice(0, RERANK_CANDIDATE_POOL)
+        .map((entry) => entry.id);
     } catch {
       // Embedding failed at runtime → fall back to pure lexical.
       return retrieveRelevantLessons(toolName, actionContext, options);
     }
   }
 
+  // Pragmatic multi-stage path (turbopuffer hybrid playbook, local-only):
+  // multi-query (lexical ⊕ dense) → RRF → attribute-aware BM25 rerank → diversify.
+  // Falls back to legacy fuse+rerank if pragmatic module is unavailable.
+  try {
+    const { pragmaticHybridSearch, sampleRetrievalRecall } = require('./pragmatic-hybrid-search');
+    const { results, meta } = pragmaticHybridSearch({
+      corpus: memories,
+      query: actionContext,
+      toolName,
+      options: {
+        topK: Math.max(maxResults * 2, maxResults),
+        pool: RERANK_CANDIDATE_POOL,
+        denseRankedIds: semanticRanked,
+        queryVariants,
+        denseWeight: options.denseWeight,
+        fusionWeight: options.fusionWeight,
+        diversify: options.diversify !== false,
+        perLimit: options.perLimit || 3,
+        attribute: options.attribute,
+      },
+    });
+    sampleRetrievalRecall({
+      toolName,
+      queryPreview: String(actionContext || '').slice(0, 200),
+      strategy: meta.strategy,
+      topIds: results.slice(0, maxResults).map((r) => r.id),
+      densePool: meta.densePool,
+      lexicalPool: meta.lexicalPool,
+    }, { feedbackDir });
+    const cut = filterTopP(
+      dedupeSupersededLessons(results),
+      resolveTopP(options),
+      { minKeep: options.minKeep },
+    ).slice(0, maxResults);
+    const retrieval = options.includeRetrievalMeta ? {
+      ...meta,
+      queryVariants,
+      queryTransformation: queryPlan,
+      semanticProvider: options.embedderId || 'configured',
+    } : null;
+    return cut.map((lesson) => shapeLesson(lesson, retrieval));
+  } catch {
+    // Legacy fuse path below
+  }
+
   // Fuse. Candidate pool is the union — dense can introduce lessons lexical missed.
   const fused = reciprocalRankFusion([lexicalRanked, semanticRanked]);
   if (fused.length === 0) return [];
@@ -295,7 +563,7 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
     .map((entry) => {
       const mem = byId.get(entry.id);
       if (!mem) return null;
-      // Carry a relevanceScore the cross-encoder can blend against. Prefer the
+      // Carry a relevanceScore the BM25F reranker can blend against. Prefer the
       // lexical score when present; otherwise use the normalized fusion score so
       // dense-only candidates still rank sensibly.
       const relevanceScore = lexById.has(entry.id)
@@ -309,7 +577,11 @@ async function retrieveRelevantLessonsAsync(toolName, actionContext, options = {
 
   const { rerankLessons } = require('./lesson-reranker');
   const reranked = rerankLessons(actionContext, candidates, { topK: maxResults, toolName });
-  return filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
+  const rows = filterTopP(dedupeSupersededLessons(reranked), resolveTopP(options), { minKeep: options.minKeep }).map(shapeLesson);
+  return attachArrayRetrievalMeta(rows, {
+    strategy: 'hybrid-rrf+bm25',
+    indexUpdatedAtMs: options.indexUpdatedAtMs ?? null,
+  });
 }
 
 function buildActionSignature(toolName, actionContext) {
@@ -547,6 +819,10 @@ module.exports = {
   dedupeSupersededLessons,
   isRetrievableMemory,
   selectRetrievalMemories,
+  matchesMetadataFilters,
+  buildQueryVariants,
+  buildQueryPlan,
+  attachArrayRetrievalMeta,
   MAX_RETRIEVAL_MEMORY_CHARS,
   MAX_RETRIEVAL_MEMORY_LINES,
 };

package/scripts/lesson-search.js

@@ -403,6 +403,8 @@ function scoreLesson(queryText, memory, parsed, sourceFeedback) {
   if (!queryText) {
     return {
       score: recencyScore(memory.timestamp),
+      evidenceScore: 0,
+      priorScore: recencyScore(memory.timestamp),
       matchedTokens: [],
     };
   }
@@ -410,14 +412,16 @@ function scoreLesson(queryText, memory, parsed, sourceFeedback) {
   const lessonText = buildLessonQuery(memory, parsed, sourceFeedback);
   const queryTokens = tokenize(queryText);
   const lessonTokens = tokenize(lessonText);
-  const score = jaccardSimilarity(queryTokens, lessonTokens)
-    + substringBoost(queryText, lessonText)
-    + recencyScore(memory.timestamp)
+  const evidenceScore = jaccardSimilarity(queryTokens, lessonTokens)
+    + substringBoost(queryText, lessonText);
+  const priorScore = recencyScore(memory.timestamp)
     + (memory.category === 'error' ? 0.05 : 0)
     + Math.min(0.2, scoreHybridMemoryMatch(queryText, memory).score * 0.1);
 
   return {
-    score,
+    score: evidenceScore + priorScore,
+    evidenceScore,
+    priorScore,
     matchedTokens: unique(queryTokens.filter((token) => lessonTokens.includes(token))),
   };
 }
@@ -427,7 +431,7 @@ function buildLessonResult(memory, sourceFeedback, options = {}) {
   const lessonQuery = buildLessonQuery(memory, parsed, sourceFeedback);
   const ruleMatches = readPreventionRuleMatches(lessonQuery, Number(options.ruleLimit || 3), options);
   const gateMatches = buildGateMatches(memory, parsed, Number(options.gateLimit || 3), options);
-  const { score, matchedTokens } = scoreLesson(options.query || '', memory, parsed, sourceFeedback);
+  const { score, evidenceScore, priorScore, matchedTokens } = scoreLesson(options.query || '', memory, parsed, sourceFeedback);
   const harnessRecommendations = buildHarnessRecommendations(memory, parsed, sourceFeedback, ruleMatches, gateMatches);
   const lifecycle = buildLifecycle(memory, parsed, sourceFeedback, ruleMatches, gateMatches, harnessRecommendations);
   const memoryLifecycle = buildMemoryLifecycleView(memory, { query: options.query || '' });
@@ -441,6 +445,8 @@ function buildLessonResult(memory, sourceFeedback, options = {}) {
     timestamp: memory.timestamp || null,
     sourceFeedbackId: memory.sourceFeedbackId || null,
     score: Number(score.toFixed(4)),
+    evidenceScore: Number(evidenceScore.toFixed(4)),
+    priorScore: Number(priorScore.toFixed(4)),
     matchedTokens,
     lesson: {
       summary: parsed.summary,
@@ -510,7 +516,10 @@ function searchLessons(query = '', options = {}) {
     results = results.filter((entry) => requiredTags.every((tag) => entry.tags.includes(tag)));
   }
   if (query) {
-    results = results.filter((entry) => entry.score > 0);
+    // Recency, error severity, and lifecycle priors can reorder candidates,
+    // but they are never evidence that a lesson answers the query. Returning a
+    // recent unrelated incident is worse than an honest empty result.
+    results = results.filter((entry) => entry.evidenceScore > 0);
   }
 
   results.sort((a, b) => {
@@ -518,7 +527,7 @@ function searchLessons(query = '', options = {}) {
     return String(b.timestamp || '').localeCompare(String(a.timestamp || ''));
   });
 
-  // Cross-encoder reranking: when a query is present, rerank the top-50 bi-encoder
+  // Field-aware BM25F reranking: when a query is present, rerank the top-50 first-stage
   // candidates using field-weighted BM25 so the most relevant lessons surface first.
   if (query && results.length > 1) {
     const { rerankLessons } = loadOptionalModule('./lesson-reranker', () => ({
@@ -558,9 +567,10 @@ function tryFts5Search(query, options) {
   // silently searching across tenants or sessions.
   if (options.scope || options.requireScope) return null;
   if (!process.env.LESSON_DB_SEARCH && !options.useFts5) return null;
+  let db = null;
   try {
     const { initDB, searchLessons: fts5Search, getStats } = require('./lesson-db');
-    const db = initDB();
+    db = initDB();
     const stats = getStats(db);
 
     // If DB is empty, skip (not yet backfilled)
@@ -629,6 +639,10 @@ function tryFts5Search(query, options) {
     };
   } catch (_err) {
     return null; // SQLite unavailable — fall through to JSONL
+  } finally {
+    if (db) {
+      try { db.close(); } catch { /* best-effort close */ }
+    }
   }
 }