thumbgate 1.30.0 → 1.34.0

package/scripts/lesson-embedding-maintenance.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const {
+  semanticRank,
+  isEmbedderAvailable,
+  lessonText,
+  hashText,
+  readCache,
+} = require('./lesson-embedding-index');
+const {
+  selectRetrievalMemories,
+  MAX_RETRIEVAL_MEMORY_LINES,
+} = require('./lesson-retrieval');
+
+function loadLessonCorpus(feedbackDir) {
+  const { readJSONL, getFeedbackPaths } = require('./feedback-loop');
+  const memoryPath = feedbackDir
+    ? path.join(feedbackDir, 'memory-log.jsonl')
+    : getFeedbackPaths().MEMORY_LOG_PATH;
+  const lessons = selectRetrievalMemories(
+    readJSONL(memoryPath, { maxLines: MAX_RETRIEVAL_MEMORY_LINES }),
+  );
+  return { lessons, memoryPath };
+}
+
+function evaluateEmbeddingIndexDrift(options = {}) {
+  const feedbackDir = options.feedbackDir || process.env.THUMBGATE_FEEDBACK_DIR;
+  const { lessons, memoryPath } = loadLessonCorpus(feedbackDir);
+  const cachePath = path.join(
+    feedbackDir || path.dirname(memoryPath),
+    options.cacheFile || 'lesson-embeddings.json',
+  );
+  const cache = readCache(cachePath);
+  let indexedCount = 0;
+  const missingIds = [];
+  const staleIds = [];
+  const providers = {};
+  const dimensions = {};
+  const corpusIds = new Set(lessons.map((lesson) => lesson.id));
+
+  for (const lesson of lessons) {
+    const entry = cache[lesson.id];
+    const valid = entry
+      && entry.hash === hashText(lessonText(lesson))
+      && Array.isArray(entry.vector)
+      && entry.vector.length > 0
+      && entry.dimension === entry.vector.length
+      && typeof entry.provider === 'string';
+    if (valid) {
+      indexedCount += 1;
+      providers[entry.provider] = (providers[entry.provider] || 0) + 1;
+      dimensions[entry.dimension] = (dimensions[entry.dimension] || 0) + 1;
+    } else if (entry) {
+      staleIds.push(lesson.id);
+    } else {
+      missingIds.push(lesson.id);
+    }
+  }
+
+  const orphanedIds = Object.keys(cache)
+    .filter((id) => !corpusIds.has(id))
+    .sort((left, right) => left.localeCompare(right));
+  missingIds.sort((left, right) => left.localeCompare(right));
+  staleIds.sort((left, right) => left.localeCompare(right));
+
+  const corpusCount = lessons.length;
+  const coverage = corpusCount ? indexedCount / corpusCount : 1;
+  const semanticProviderAvailable = options.embedder
+    ? true
+    : isEmbedderAvailable();
+  const minCoverage = Math.max(0, Math.min(1, Number(options.minCoverage) || 0.95));
+  const requireExact = options.requireExact !== false;
+  const enabled = semanticProviderAvailable || Object.keys(cache).length > 0;
+  const exactIdSet = missingIds.length === 0
+    && staleIds.length === 0
+    && orphanedIds.length === 0;
+  const status = !enabled
+    ? 'not_configured'
+    : coverage >= minCoverage
+      && staleIds.length === 0
+      && orphanedIds.length === 0
+      && (!requireExact || exactIdSet)
+      ? 'healthy'
+      : 'unhealthy';
+  return {
+    ok: status !== 'unhealthy',
+    status,
+    semanticProviderAvailable,
+    corpusCount,
+    indexedCount,
+    missingCount: missingIds.length,
+    missingIds,
+    staleCount: staleIds.length,
+    staleIds,
+    orphanedCount: orphanedIds.length,
+    orphanedIds,
+    exactIdSet,
+    requireExact,
+    coverage: Number(coverage.toFixed(4)),
+    minCoverage,
+    providers,
+    dimensions,
+    memoryPath,
+    cachePath,
+  };
+}
+
+async function backfillLessonEmbeddings(options = {}) {
+  const feedbackDir = options.feedbackDir || process.env.THUMBGATE_FEEDBACK_DIR;
+  if (!options.embedder && !isEmbedderAvailable()) {
+    const error = new Error(
+      'No semantic embedding provider configured. Set THUMBGATE_OLLAMA_EMBED_MODEL or configure another real provider.',
+    );
+    error.code = 'THUMBGATE_SEMANTIC_PROVIDER_UNAVAILABLE';
+    throw error;
+  }
+  const { lessons } = loadLessonCorpus(feedbackDir);
+  if (lessons.length > 0) {
+    await semanticRank('ThumbGate lesson embedding backfill health query', lessons, {
+      feedbackDir,
+      embedder: options.embedder,
+      embedderId: options.embedderId,
+      cacheFile: options.cacheFile,
+    });
+  }
+  return evaluateEmbeddingIndexDrift({
+    ...options,
+    feedbackDir,
+    embedder: options.embedder,
+  });
+}
+
+function parseArgs(argv) {
+  const args = { json: false, backfill: false, requireSemantic: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--json') args.json = true;
+    else if (arg === '--backfill') args.backfill = true;
+    else if (arg === '--require-semantic') args.requireSemantic = true;
+    else if (arg === '--feedback-dir') args.feedbackDir = argv[++index];
+    else if (arg === '--min-coverage') args.minCoverage = Number(argv[++index]);
+  }
+  return args;
+}
+
+async function runCli() {
+  const options = parseArgs(process.argv.slice(2));
+  let report;
+  try {
+    report = options.backfill
+      ? await backfillLessonEmbeddings(options)
+      : evaluateEmbeddingIndexDrift(options);
+  } catch (error) {
+    report = { ok: false, status: 'error', error: error.message, code: error.code || 'ERROR' };
+  }
+  const output = options.json
+    ? JSON.stringify(report, null, 2)
+    : `Lesson embeddings: ${report.status}; ${report.indexedCount || 0}/${report.corpusCount || 0} indexed; coverage=${report.coverage || 0}`;
+  process.stdout.write(`${output}\n`);
+  if (!report.ok || (options.requireSemantic && !report.semanticProviderAvailable)) {
+    process.exitCode = 1;
+  }
+}
+
+if (!module.parent) {
+  runCli();
+}
+
+module.exports = {
+  loadLessonCorpus,
+  evaluateEmbeddingIndexDrift,
+  backfillLessonEmbeddings,
+  parseArgs,
+};

package/scripts/lesson-reranker.js

@@ -1,11 +1,11 @@
 'use strict';
 
 /**
- * Cross-encoder reranker for lesson retrieval.
+ * Field-aware BM25F reranker for lesson retrieval.
  *
- * Unlike the bi-encoders already in use (Jaccard + bigram Jaccard), a
- * cross-encoder processes the (query, lesson) pair jointly — so it can
- * catch relevance signals that independent scoring misses:
+ * This is a deterministic lexical second stage, not a neural cross-encoder.
+ * It scores query terms against lesson fields jointly and catches relevance
+ * signals that the first-stage overlap score can miss:
  *
  *   - Field-weighted BM25: a query term in `whatWentWrong` is worth more
  *     than the same term in `tags`
@@ -55,6 +55,11 @@ const SYNONYM_GROUPS = [
   ['auth', 'authentication', 'authorization', 'token', 'api key', 'credential'],
   ['delete', 'remove', 'rm', 'drop', 'destroy', 'wipe'],
   ['merge', 'pull request', 'pr', 'rebase', 'squash'],
+  ['tablet', 'ipad', 'ios device'],
+  ['overlay', 'tailscale', 'mesh network', 'vpn'],
+  ['laptop', 'macbook', 'mac'],
+  ['phone', 'mobile', 'android', 'device'],
+  ['conversation', 'session', 'thread', 'chat'],
 ];
 
 // Regex patterns that indicate the query is about a failure/mistake.
@@ -83,7 +88,13 @@ function expandTerms(terms) {
   const expanded = new Set(terms);
   for (const term of terms) {
     for (const group of SYNONYM_GROUPS) {
-      if (group.some((syn) => syn.split(/\s+/).some((w) => w === term || term.includes(w)))) {
+      if (group.some((syn) => tokenize(syn).some((word) => (
+        word === term
+        || (word.length >= 4 && term.length >= 4 && (
+          word.startsWith(term)
+          || term.startsWith(word)
+        ))
+      )))) {
         group.forEach((syn) => tokenize(syn).forEach((t) => expanded.add(t)));
       }
     }
@@ -103,6 +114,28 @@ function getField(candidate, field) {
   return String(val);
 }
 
+/**
+ * Score exact entity-channel overlap separately from prose relevance.
+ *
+ * Tags, categories, and tool identities are low-cardinality metadata. Keeping
+ * this score visible prevents a tag-only hit from being swallowed by BM25
+ * field-length normalization and makes reranking traces explainable.
+ */
+function scoreEntityOverlap(queryTerms, candidate) {
+  const entityText = [
+    getField(candidate, 'tags'),
+    getField(candidate, 'category'),
+    getField(candidate, 'toolUsed'),
+    getField(candidate, 'toolName'),
+    ...(candidate.metadata?.toolsUsed || []),
+  ].filter(Boolean).join(' ');
+  const entityTokens = new Set(expandTerms(tokenize(entityText)));
+  const uniqueQueryTerms = [...new Set(queryTerms)];
+  if (uniqueQueryTerms.length === 0 || entityTokens.size === 0) return 0;
+  const hits = uniqueQueryTerms.filter((term) => entityTokens.has(term)).length;
+  return Number((hits / uniqueQueryTerms.length).toFixed(6));
+}
+
 /**
  * Compute field-weighted BM25 scores for a list of candidates (BM25F variant).
  *
@@ -189,7 +222,7 @@ function fieldWeightedBM25(queryTerms, candidates) {
 }
 
 /**
- * Rerank a list of lesson candidates using a cross-encoder approach.
+ * Rerank a list of lesson candidates using field-aware BM25F.
  *
  * @param {string} query          - The original retrieval query / action context
  * @param {Array}  candidates     - Lesson objects from the bi-encoder stage
@@ -224,12 +257,15 @@ function rerankLessons(query, candidates, options = {}) {
 
   const reranked = bm25Results.map(({ candidate, bm25Score }) => {
     const normBm25 = bm25Score / maxBm25;
+    const entityScore = scoreEntityOverlap(queryTerms, candidate);
 
     // Original bi-encoder score (field name differs between retrieval paths)
     const origScore = candidate.relevanceScore ?? candidate.score ?? 0;
 
     // Blend BM25 with original score
-    let finalScore = blendWeight * normBm25 + (1 - blendWeight) * origScore;
+    let finalScore = 0.85 * (
+      blendWeight * normBm25 + (1 - blendWeight) * origScore
+    ) + 0.15 * entityScore;
 
     // Signal coherence bonus: failure queries → negative lessons rank higher
     const candidateSignal =
@@ -252,7 +288,11 @@ function rerankLessons(query, candidates, options = {}) {
       }
     }
 
-    return { ...candidate, rerankedScore: Number(finalScore.toFixed(6)) };
+    return {
+      ...candidate,
+      entityScore,
+      rerankedScore: Number(finalScore.toFixed(6)),
+    };
   });
 
   return reranked
@@ -260,4 +300,10 @@ function rerankLessons(query, candidates, options = {}) {
     .slice(0, topK);
 }
 
-module.exports = { rerankLessons, fieldWeightedBM25, tokenize, expandTerms };
+module.exports = {
+  rerankLessons,
+  fieldWeightedBM25,
+  scoreEntityOverlap,
+  tokenize,
+  expandTerms,
+};