thumbgate 1.29.2 → 1.31.0

package/scripts/lesson-embedding-index.js

@@ -12,7 +12,8 @@
  *
  * Design constraints:
  *   - Embedding the whole corpus on every tool call is too expensive. We cache
- *     document vectors keyed by `id + sha1(text)` in <feedbackDir>/lesson-embeddings.json.
+ *     document vectors keyed by `id + sha256(text) + provider + dimension` in
+ *     <feedbackDir>/lesson-embeddings.json.
  *     Only the query is embedded per call; only new/changed lessons re-embed.
  *   - The embedder is reused from vector-store.embed (Gemini -> local transformers
  *     -> stub). No new embedding dependency.
@@ -92,15 +93,12 @@ function cosineSimilarity(a, b) {
  * WITHOUT loading a model. Returns true for the deterministic stub (test/CI safe).
  */
 function isEmbedderAvailable() {
-  if (process.env.THUMBGATE_VECTOR_STUB_EMBED === 'true') return true;
-  // Managed Gemini path
   try {
-    const { resolveGeminiEmbeddingConfig } = require('./gemini-embedding-policy');
-    const cfg = resolveGeminiEmbeddingConfig();
-    if (cfg && cfg.enabled && cfg.apiKey) return true;
-  } catch { /* policy module unavailable */ }
-  // The zero-dependency feature-hash provider is always available locally.
-  return true;
+    const { hasSemanticEmbeddingProvider } = require('./vector-store');
+    return hasSemanticEmbeddingProvider();
+  } catch {
+    return false;
+  }
 }
 
 function defaultEmbedder() {
@@ -122,12 +120,34 @@ function readCache(cachePath) {
 function writeCache(cachePath, cache) {
   try {
     fs.mkdirSync(path.dirname(cachePath), { recursive: true });
-    fs.writeFileSync(cachePath, JSON.stringify(cache));
+    const tmpPath = `${cachePath}.${process.pid}.tmp`;
+    fs.writeFileSync(tmpPath, JSON.stringify(cache), { mode: 0o600 });
+    fs.renameSync(tmpPath, cachePath);
+    fs.chmodSync(cachePath, 0o600);
   } catch {
     /* cache is best-effort; never throw into the hot path */
   }
 }
 
+function resolveProviderFingerprint(options, vector) {
+  if (options.embedder) {
+    return String(options.embedderId || options.embedder.providerId || 'injected-test');
+  }
+  try {
+    const { getLastEmbeddingProfile } = require('./vector-store');
+    const profile = getLastEmbeddingProfile();
+    const active = profile && profile.activeProfile;
+    return [
+      profile && profile.source,
+      active && active.id,
+      active && active.model,
+      Array.isArray(vector) ? vector.length : 0,
+    ].filter(Boolean).join(':') || 'unknown';
+  } catch {
+    return 'unknown';
+  }
+}
+
 /**
  * Rank lessons by dense (embedding) similarity to the query.
  * Returns [{ id, score }] sorted descending. Embeds the query once; reuses cached
@@ -136,10 +156,17 @@ function writeCache(cachePath, cache) {
  * @returns {Promise<Array<{id:string, score:number}>>}
  */
 async function semanticRank(queryText, lessons = [], options = {}) {
-  const { feedbackDir, embedder = defaultEmbedder(), persist = true, truncateDimension = null } = options;
+  const {
+    feedbackDir,
+    embedder = defaultEmbedder(),
+    persist = true,
+    pruneCache = true,
+    truncateDimension = null,
+    cacheFile = CACHE_FILE,
+  } = options;
   if (!queryText || !Array.isArray(lessons) || lessons.length === 0) return [];
 
-  const cachePath = getCachePath(feedbackDir);
+  const cachePath = path.join(resolveFeedbackDir(feedbackDir), path.basename(cacheFile));
   const cache = readCache(cachePath);
   let cacheDirty = false;
 
@@ -149,6 +176,11 @@ async function semanticRank(queryText, lessons = [], options = {}) {
   if (truncateDimension) {
     queryVector = truncateVector(queryVector, truncateDimension);
   }
+  const provider = resolveProviderFingerprint(options, queryVector);
+  const dimension = queryVector.length;
+  if (!options.embedder && /(?:built-in|feature-hash)/i.test(provider)) {
+    throw new Error('Semantic embedding provider degraded to feature hashing');
+  }
 
   const scored = [];
   for (const lesson of lessons) {
@@ -158,14 +190,22 @@ async function semanticRank(queryText, lessons = [], options = {}) {
     const hash = hashText(text);
 
     let entry = cache[lesson.id];
-    if (!entry || entry.hash !== hash || !Array.isArray(entry.vector)) {
+    if (
+      !entry
+      || entry.hash !== hash
+      || entry.provider !== provider
+      || entry.dimension !== dimension
+      || !Array.isArray(entry.vector)
+      || entry.vector.length !== dimension
+    ) {
       const vector = await embedder(text, {
         kind: 'document',
         task: 'code retrieval',
         title: lesson.title || undefined,
       });
       if (!Array.isArray(vector) || vector.length === 0) continue;
-      entry = { hash, vector };
+      if (vector.length !== dimension) continue;
+      entry = { hash, provider, dimension, vector };
       cache[lesson.id] = entry;
       cacheDirty = true;
     }
@@ -174,12 +214,16 @@ async function semanticRank(queryText, lessons = [], options = {}) {
     scored.push({ id: lesson.id, score: cosineSimilarity(queryVector, docVector) });
   }
 
-  // Prune cache entries for lessons no longer present (bounded growth).
-  const liveIds = new Set(lessons.map((l) => l && l.id).filter(Boolean));
-  for (const id of Object.keys(cache)) {
-    if (!liveIds.has(id)) {
-      delete cache[id];
-      cacheDirty = true;
+  // Prune only when the caller supplied the complete corpus. Metadata-filtered
+  // searches pass pruneCache=false so alternating filters cannot evict and
+  // regenerate one another's vectors.
+  if (pruneCache) {
+    const liveIds = new Set(lessons.map((l) => l && l.id).filter(Boolean));
+    for (const id of Object.keys(cache)) {
+      if (!liveIds.has(id)) {
+        delete cache[id];
+        cacheDirty = true;
+      }
     }
   }
 
@@ -196,4 +240,7 @@ module.exports = {
   lessonText,
   hashText,
   getCachePath,
+  readCache,
+  writeCache,
+  resolveProviderFingerprint,
 };

package/scripts/lesson-embedding-maintenance.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const {
+  semanticRank,
+  isEmbedderAvailable,
+  lessonText,
+  hashText,
+  readCache,
+} = require('./lesson-embedding-index');
+const {
+  selectRetrievalMemories,
+  MAX_RETRIEVAL_MEMORY_LINES,
+} = require('./lesson-retrieval');
+
+function loadLessonCorpus(feedbackDir) {
+  const { readJSONL, getFeedbackPaths } = require('./feedback-loop');
+  const memoryPath = feedbackDir
+    ? path.join(feedbackDir, 'memory-log.jsonl')
+    : getFeedbackPaths().MEMORY_LOG_PATH;
+  const lessons = selectRetrievalMemories(
+    readJSONL(memoryPath, { maxLines: MAX_RETRIEVAL_MEMORY_LINES }),
+  );
+  return { lessons, memoryPath };
+}
+
+function evaluateEmbeddingIndexDrift(options = {}) {
+  const feedbackDir = options.feedbackDir || process.env.THUMBGATE_FEEDBACK_DIR;
+  const { lessons, memoryPath } = loadLessonCorpus(feedbackDir);
+  const cachePath = path.join(
+    feedbackDir || path.dirname(memoryPath),
+    options.cacheFile || 'lesson-embeddings.json',
+  );
+  const cache = readCache(cachePath);
+  let indexedCount = 0;
+  const missingIds = [];
+  const staleIds = [];
+  const providers = {};
+  const dimensions = {};
+  const corpusIds = new Set(lessons.map((lesson) => lesson.id));
+
+  for (const lesson of lessons) {
+    const entry = cache[lesson.id];
+    const valid = entry
+      && entry.hash === hashText(lessonText(lesson))
+      && Array.isArray(entry.vector)
+      && entry.vector.length > 0
+      && entry.dimension === entry.vector.length
+      && typeof entry.provider === 'string';
+    if (valid) {
+      indexedCount += 1;
+      providers[entry.provider] = (providers[entry.provider] || 0) + 1;
+      dimensions[entry.dimension] = (dimensions[entry.dimension] || 0) + 1;
+    } else if (entry) {
+      staleIds.push(lesson.id);
+    } else {
+      missingIds.push(lesson.id);
+    }
+  }
+
+  const orphanedIds = Object.keys(cache)
+    .filter((id) => !corpusIds.has(id))
+    .sort((left, right) => left.localeCompare(right));
+  missingIds.sort((left, right) => left.localeCompare(right));
+  staleIds.sort((left, right) => left.localeCompare(right));
+
+  const corpusCount = lessons.length;
+  const coverage = corpusCount ? indexedCount / corpusCount : 1;
+  const semanticProviderAvailable = options.embedder
+    ? true
+    : isEmbedderAvailable();
+  const minCoverage = Math.max(0, Math.min(1, Number(options.minCoverage) || 0.95));
+  const requireExact = options.requireExact !== false;
+  const enabled = semanticProviderAvailable || Object.keys(cache).length > 0;
+  const exactIdSet = missingIds.length === 0
+    && staleIds.length === 0
+    && orphanedIds.length === 0;
+  const status = !enabled
+    ? 'not_configured'
+    : coverage >= minCoverage
+      && staleIds.length === 0
+      && orphanedIds.length === 0
+      && (!requireExact || exactIdSet)
+      ? 'healthy'
+      : 'unhealthy';
+  return {
+    ok: status !== 'unhealthy',
+    status,
+    semanticProviderAvailable,
+    corpusCount,
+    indexedCount,
+    missingCount: missingIds.length,
+    missingIds,
+    staleCount: staleIds.length,
+    staleIds,
+    orphanedCount: orphanedIds.length,
+    orphanedIds,
+    exactIdSet,
+    requireExact,
+    coverage: Number(coverage.toFixed(4)),
+    minCoverage,
+    providers,
+    dimensions,
+    memoryPath,
+    cachePath,
+  };
+}
+
+async function backfillLessonEmbeddings(options = {}) {
+  const feedbackDir = options.feedbackDir || process.env.THUMBGATE_FEEDBACK_DIR;
+  if (!options.embedder && !isEmbedderAvailable()) {
+    const error = new Error(
+      'No semantic embedding provider configured. Set THUMBGATE_OLLAMA_EMBED_MODEL or configure another real provider.',
+    );
+    error.code = 'THUMBGATE_SEMANTIC_PROVIDER_UNAVAILABLE';
+    throw error;
+  }
+  const { lessons } = loadLessonCorpus(feedbackDir);
+  if (lessons.length > 0) {
+    await semanticRank('ThumbGate lesson embedding backfill health query', lessons, {
+      feedbackDir,
+      embedder: options.embedder,
+      embedderId: options.embedderId,
+      cacheFile: options.cacheFile,
+    });
+  }
+  return evaluateEmbeddingIndexDrift({
+    ...options,
+    feedbackDir,
+    embedder: options.embedder,
+  });
+}
+
+function parseArgs(argv) {
+  const args = { json: false, backfill: false, requireSemantic: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--json') args.json = true;
+    else if (arg === '--backfill') args.backfill = true;
+    else if (arg === '--require-semantic') args.requireSemantic = true;
+    else if (arg === '--feedback-dir') args.feedbackDir = argv[++index];
+    else if (arg === '--min-coverage') args.minCoverage = Number(argv[++index]);
+  }
+  return args;
+}
+
+async function runCli() {
+  const options = parseArgs(process.argv.slice(2));
+  let report;
+  try {
+    report = options.backfill
+      ? await backfillLessonEmbeddings(options)
+      : evaluateEmbeddingIndexDrift(options);
+  } catch (error) {
+    report = { ok: false, status: 'error', error: error.message, code: error.code || 'ERROR' };
+  }
+  const output = options.json
+    ? JSON.stringify(report, null, 2)
+    : `Lesson embeddings: ${report.status}; ${report.indexedCount || 0}/${report.corpusCount || 0} indexed; coverage=${report.coverage || 0}`;
+  process.stdout.write(`${output}\n`);
+  if (!report.ok || (options.requireSemantic && !report.semanticProviderAvailable)) {
+    process.exitCode = 1;
+  }
+}
+
+if (!module.parent) {
+  runCli();
+}
+
+module.exports = {
+  loadLessonCorpus,
+  evaluateEmbeddingIndexDrift,
+  backfillLessonEmbeddings,
+  parseArgs,
+};

package/scripts/lesson-inference.js

@@ -170,6 +170,20 @@ function isPositiveSignal(signal) {
   return signal === 'positive' || signal === 'up';
 }
 
+function isHumanReviewedLesson(lesson = {}) {
+  return (lesson.metadata?.reviewOrigin || lesson.reviewOrigin) === 'human';
+}
+
+function dedupeHumanReviewedLessons(lessons = []) {
+  const byFeedback = new Map();
+  for (const lesson of lessons) {
+    if (isHumanReviewedLesson(lesson) && lesson.feedbackId) {
+      byFeedback.set(lesson.feedbackId, lesson);
+    }
+  }
+  return [...byFeedback.values()];
+}
+
 function selectStatusbarLesson() {
   const lessons = readJsonl(getLessonsPath())
     .slice()
@@ -242,10 +256,14 @@ function searchLessons({ query = '', limit = 10, signal } = {}) {
  */
 function getLessonStats() {
   const lessons = readJsonl(getLessonsPath());
-  const positive = lessons.filter((l) => l.signal === 'positive' || l.signal === 'up').length;
-  const negative = lessons.filter((l) => l.signal === 'negative' || l.signal === 'down').length;
-  const avgConfidence = lessons.length > 0 ? Math.round(lessons.reduce((s, l) => s + (l.confidence || 0), 0) / lessons.length) : 0;
-  return { total: lessons.length, positive, negative, avgConfidence };
+  const humanReviewed = dedupeHumanReviewedLessons(lessons);
+  const positive = humanReviewed.filter((lesson) => isPositiveSignal(lesson.signal)).length;
+  const negative = humanReviewed.filter((lesson) => isNegativeSignal(lesson.signal)).length;
+  const avgConfidence = humanReviewed.length > 0
+    ? Math.round(humanReviewed.reduce((sum, lesson) => sum + (lesson.confidence || 0), 0) / humanReviewed.length)
+    : 0;
+  return { total: positive + negative, positive, negative, avgConfidence,
+    rawTotal: lessons.length, excludedTotal: lessons.length - humanReviewed.length };
 }
 
 // ---------------------------------------------------------------------------
@@ -650,6 +668,7 @@ async function inferStructuredLessonLLM(conversationWindow, signal, context) {
 module.exports = {
   inferFromSurroundingMessages, createLesson, getRecentLesson,
   searchLessons, getLessonStats, getStatusbarLessonData, getAllLessonsForContext,
+  isHumanReviewedLesson,
   getLessonsPath, getRecentLessonPath,
   selectStatusbarLesson, getLessonKind, stripLessonPrefix,
   formatLessonTimestamp, buildStatusbarLessonLabel,

package/scripts/lesson-reranker.js

@@ -1,11 +1,11 @@
 'use strict';
 
 /**
- * Cross-encoder reranker for lesson retrieval.
+ * Field-aware BM25F reranker for lesson retrieval.
  *
- * Unlike the bi-encoders already in use (Jaccard + bigram Jaccard), a
- * cross-encoder processes the (query, lesson) pair jointly — so it can
- * catch relevance signals that independent scoring misses:
+ * This is a deterministic lexical second stage, not a neural cross-encoder.
+ * It scores query terms against lesson fields jointly and catches relevance
+ * signals that the first-stage overlap score can miss:
  *
  *   - Field-weighted BM25: a query term in `whatWentWrong` is worth more
  *     than the same term in `tags`
@@ -55,6 +55,11 @@ const SYNONYM_GROUPS = [
   ['auth', 'authentication', 'authorization', 'token', 'api key', 'credential'],
   ['delete', 'remove', 'rm', 'drop', 'destroy', 'wipe'],
   ['merge', 'pull request', 'pr', 'rebase', 'squash'],
+  ['tablet', 'ipad', 'ios device'],
+  ['overlay', 'tailscale', 'mesh network', 'vpn'],
+  ['laptop', 'macbook', 'mac'],
+  ['phone', 'mobile', 'android', 'device'],
+  ['conversation', 'session', 'thread', 'chat'],
 ];
 
 // Regex patterns that indicate the query is about a failure/mistake.
@@ -83,7 +88,13 @@ function expandTerms(terms) {
   const expanded = new Set(terms);
   for (const term of terms) {
     for (const group of SYNONYM_GROUPS) {
-      if (group.some((syn) => syn.split(/\s+/).some((w) => w === term || term.includes(w)))) {
+      if (group.some((syn) => tokenize(syn).some((word) => (
+        word === term
+        || (word.length >= 4 && term.length >= 4 && (
+          word.startsWith(term)
+          || term.startsWith(word)
+        ))
+      )))) {
         group.forEach((syn) => tokenize(syn).forEach((t) => expanded.add(t)));
       }
     }
@@ -103,6 +114,28 @@ function getField(candidate, field) {
   return String(val);
 }
 
+/**
+ * Score exact entity-channel overlap separately from prose relevance.
+ *
+ * Tags, categories, and tool identities are low-cardinality metadata. Keeping
+ * this score visible prevents a tag-only hit from being swallowed by BM25
+ * field-length normalization and makes reranking traces explainable.
+ */
+function scoreEntityOverlap(queryTerms, candidate) {
+  const entityText = [
+    getField(candidate, 'tags'),
+    getField(candidate, 'category'),
+    getField(candidate, 'toolUsed'),
+    getField(candidate, 'toolName'),
+    ...(candidate.metadata?.toolsUsed || []),
+  ].filter(Boolean).join(' ');
+  const entityTokens = new Set(expandTerms(tokenize(entityText)));
+  const uniqueQueryTerms = [...new Set(queryTerms)];
+  if (uniqueQueryTerms.length === 0 || entityTokens.size === 0) return 0;
+  const hits = uniqueQueryTerms.filter((term) => entityTokens.has(term)).length;
+  return Number((hits / uniqueQueryTerms.length).toFixed(6));
+}
+
 /**
  * Compute field-weighted BM25 scores for a list of candidates (BM25F variant).
  *
@@ -189,7 +222,7 @@ function fieldWeightedBM25(queryTerms, candidates) {
 }
 
 /**
- * Rerank a list of lesson candidates using a cross-encoder approach.
+ * Rerank a list of lesson candidates using field-aware BM25F.
  *
  * @param {string} query          - The original retrieval query / action context
  * @param {Array}  candidates     - Lesson objects from the bi-encoder stage
@@ -224,12 +257,15 @@ function rerankLessons(query, candidates, options = {}) {
 
   const reranked = bm25Results.map(({ candidate, bm25Score }) => {
     const normBm25 = bm25Score / maxBm25;
+    const entityScore = scoreEntityOverlap(queryTerms, candidate);
 
     // Original bi-encoder score (field name differs between retrieval paths)
     const origScore = candidate.relevanceScore ?? candidate.score ?? 0;
 
     // Blend BM25 with original score
-    let finalScore = blendWeight * normBm25 + (1 - blendWeight) * origScore;
+    let finalScore = 0.85 * (
+      blendWeight * normBm25 + (1 - blendWeight) * origScore
+    ) + 0.15 * entityScore;
 
     // Signal coherence bonus: failure queries → negative lessons rank higher
     const candidateSignal =
@@ -252,7 +288,11 @@ function rerankLessons(query, candidates, options = {}) {
       }
     }
 
-    return { ...candidate, rerankedScore: Number(finalScore.toFixed(6)) };
+    return {
+      ...candidate,
+      entityScore,
+      rerankedScore: Number(finalScore.toFixed(6)),
+    };
   });
 
   return reranked
@@ -260,4 +300,10 @@ function rerankLessons(query, candidates, options = {}) {
     .slice(0, topK);
 }
 
-module.exports = { rerankLessons, fieldWeightedBM25, tokenize, expandTerms };
+module.exports = {
+  rerankLessons,
+  fieldWeightedBM25,
+  scoreEntityOverlap,
+  tokenize,
+  expandTerms,
+};