thumbgate 1.4.1 → 1.4.3

package/scripts/lesson-reranker.js

@@ -0,0 +1,263 @@
+'use strict';
+
+/**
+ * Cross-encoder reranker for lesson retrieval.
+ *
+ * Unlike the bi-encoders already in use (Jaccard + bigram Jaccard), a
+ * cross-encoder processes the (query, lesson) pair jointly — so it can
+ * catch relevance signals that independent scoring misses:
+ *
+ *   - Field-weighted BM25: a query term in `whatWentWrong` is worth more
+ *     than the same term in `tags`
+ *   - Synonym/alias expansion: "force-push" ↔ "push --force", "deploy" ↔
+ *     "deployment", etc.
+ *   - Signal coherence: failure-sounding queries boost negative-signal lessons
+ *   - Tool name joint scoring: query toolName × lesson toolsUsed
+ *   - Score blending: reranked score is blended with the original retrieval
+ *     score so we never fully discard the bi-encoder's signal
+ *
+ * Usage:
+ *   const { rerankLessons } = require('./lesson-reranker');
+ *   const reranked = rerankLessons(query, candidates, { topK: 5, toolName });
+ */
+
+// BM25 hyper-parameters
+const BM25_K1 = 1.5;   // term saturation
+const BM25_B  = 0.75;  // length normalisation
+
+// Weight given to each lesson field when scoring a (query, lesson) pair.
+// Higher weight = query terms appearing in that field contribute more to score.
+const FIELD_WEIGHTS = {
+  whatWentWrong:  3.0,
+  whatToChange:   2.5,
+  howToAvoid:     2.0,
+  whatWorked:     2.0,
+  summary:        1.5,
+  content:        1.5,
+  context:        1.2,
+  title:          1.0,
+  rootCause:      1.0,
+  reasoning:      0.8,
+  tags:           0.5,
+  category:       0.4,
+};
+
+// Synonym clusters: any term in a group matches all others.
+const SYNONYM_GROUPS = [
+  ['force-push', 'force push', 'push --force', 'git push --force', 'force_push'],
+  ['main', 'main branch', 'master', 'trunk', 'protected branch'],
+  ['env', '.env', 'environment variable', 'env var', 'dotenv', 'secret'],
+  ['deploy', 'deployment', 'ship', 'release', 'publish', 'rollout'],
+  ['db', 'database', 'sqlite', 'postgres', 'postgresql', 'migration', 'migrate'],
+  ['test', 'tests', 'test suite', 'spec', 'failing test', 'test failure'],
+  ['ci', 'ci/cd', 'pipeline', 'github actions', 'workflow', 'build'],
+  ['lint', 'linter', 'eslint', 'prettier', 'format'],
+  ['auth', 'authentication', 'authorization', 'token', 'api key', 'credential'],
+  ['delete', 'remove', 'rm', 'drop', 'destroy', 'wipe'],
+  ['merge', 'pull request', 'pr', 'rebase', 'squash'],
+];
+
+// Regex patterns that indicate the query is about a failure/mistake.
+const FAILURE_PATTERN = /fail|error|wrong|broken|mistake|bad|incorrect|problem|issue|bug|crash|broke|exception/i;
+
+/**
+ * Tokenise text into lowercase word-like tokens of length >= 2.
+ * Hyphens and underscores are treated as delimiters so "force-push"
+ * becomes ["force", "push"].
+ * Exported so tests can verify expansion behaviour.
+ */
+function tokenize(text) {
+  if (!text) return [];
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')   // replace all non-word, non-space chars (incl. hyphens, dots) with space
+    .split(/[\s_]+/)             // split on whitespace and underscores
+    .filter((t) => t.length >= 2);
+}
+
+/**
+ * Expand a set of query tokens with synonyms from SYNONYM_GROUPS.
+ * Returns a deduplicated array of all terms (originals + expansions).
+ */
+function expandTerms(terms) {
+  const expanded = new Set(terms);
+  for (const term of terms) {
+    for (const group of SYNONYM_GROUPS) {
+      if (group.some((syn) => syn.split(/\s+/).some((w) => w === term || term.includes(w)))) {
+        group.forEach((syn) => tokenize(syn).forEach((t) => expanded.add(t)));
+      }
+    }
+  }
+  return [...expanded];
+}
+
+/**
+ * Extract the text value of a named field from a lesson candidate.
+ * Handles both the flat structure from lesson-retrieval.js and the nested
+ * { lesson: { whatWentWrong, ... } } structure from lesson-search.js.
+ */
+function getField(candidate, field) {
+  const nested = candidate.lesson;
+  const val = (nested && nested[field]) || candidate[field] || '';
+  if (Array.isArray(val)) return val.join(' ');
+  return String(val);
+}
+
+/**
+ * Compute field-weighted BM25 scores for a list of candidates (BM25F variant).
+ *
+ * BM25F processes the (query, lesson) pair jointly: query terms are weighted
+ * differently depending on which lesson field they appear in (via FIELD_WEIGHTS).
+ * IDF is computed at document level (how many docs contain the term across any
+ * field) so it stays positive regardless of field weights.
+ *
+ * Returns an array of { candidate, bm25Score } objects in the same order
+ * as the input.
+ */
+function fieldWeightedBM25(queryTerms, candidates) {
+  const N = candidates.length;
+  if (N === 0) return [];
+
+  const fieldEntries = Object.entries(FIELD_WEIGHTS);
+  const fieldKeys = Object.keys(FIELD_WEIGHTS);
+
+  // Precompute per-document, per-field token arrays (avoid re-tokenising)
+  const docFieldTokens = candidates.map((candidate) => {
+    const fieldMap = {};
+    for (const field of fieldKeys) {
+      fieldMap[field] = tokenize(getField(candidate, field));
+    }
+    return fieldMap;
+  });
+
+  // Per-field average token lengths across all documents
+  const avgFieldLen = {};
+  for (const field of fieldKeys) {
+    const total = docFieldTokens.reduce((sum, d) => sum + d[field].length, 0);
+    avgFieldLen[field] = total / N || 1;   // fallback to 1 to avoid /0
+  }
+
+  // Document-level df: count of documents that contain each term (any field).
+  // Keeping df as a plain count (not field-weighted) ensures IDF is always positive.
+  const df = new Map();
+  for (let i = 0; i < N; i++) {
+    const seenInDoc = new Set();
+    for (const field of fieldKeys) {
+      for (const tok of docFieldTokens[i][field]) {
+        if (!seenInDoc.has(tok)) {
+          df.set(tok, (df.get(tok) || 0) + 1);
+          seenInDoc.add(tok);
+        }
+      }
+    }
+  }
+
+  return candidates.map((candidate, i) => {
+    let score = 0;
+
+    for (const qTerm of queryTerms) {
+      const termDf = df.get(qTerm) || 0;
+      if (termDf === 0) continue;
+
+      // IDF is always positive because df ≤ N
+      const idf = Math.log((N - termDf + 0.5) / (termDf + 0.5) + 1);
+      if (idf <= 0) continue;
+
+      // BM25F: compute weighted sum of per-field normalised TF, then scale by IDF
+      let weightedTF = 0;
+      for (const [field, fieldWeight] of fieldEntries) {
+        const tokens = docFieldTokens[i][field];
+        const fieldLen = tokens.length;
+        if (fieldLen === 0) continue;
+
+        let tf = 0;
+        for (const t of tokens) {
+          if (t === qTerm) tf++;
+        }
+        if (tf === 0) continue;
+
+        const avgLen = avgFieldLen[field];
+        const normTF = tf / (tf + BM25_K1 * (1 - BM25_B + BM25_B * fieldLen / avgLen));
+        weightedTF += fieldWeight * normTF;
+      }
+
+      score += idf * weightedTF;
+    }
+
+    return { candidate, bm25Score: score };
+  });
+}
+
+/**
+ * Rerank a list of lesson candidates using a cross-encoder approach.
+ *
+ * @param {string} query          - The original retrieval query / action context
+ * @param {Array}  candidates     - Lesson objects from the bi-encoder stage
+ * @param {object} options
+ * @param {number} [options.topK=5]         - How many results to return
+ * @param {string} [options.toolName]       - Tool name from the triggering hook call
+ * @param {number} [options.blendWeight=0.7] - Weight given to BM25 score vs original
+ *                                            retrieval score (0 = all original, 1 = all BM25)
+ * @returns {Array} Reranked candidates with `rerankedScore` field added
+ */
+function rerankLessons(query, candidates, options = {}) {
+  const {
+    topK        = 5,
+    toolName    = '',
+    blendWeight = 0.7,
+  } = options;
+
+  if (!candidates || candidates.length === 0) return [];
+  if (candidates.length === 1) return candidates.slice(0, topK);
+
+  // Build expanded query term set
+  const rawTerms = tokenize((query || '') + (toolName ? ' ' + toolName : ''));
+  const queryTerms = expandTerms(rawTerms);
+
+  const isFailureQuery = FAILURE_PATTERN.test(query || '');
+
+  // Compute BM25 scores for all candidates
+  const bm25Results = fieldWeightedBM25(queryTerms, candidates);
+
+  // Normalise BM25 scores to [0, 1]
+  const maxBm25 = Math.max(...bm25Results.map((r) => r.bm25Score), 1e-9);
+
+  const reranked = bm25Results.map(({ candidate, bm25Score }) => {
+    const normBm25 = bm25Score / maxBm25;
+
+    // Original bi-encoder score (field name differs between retrieval paths)
+    const origScore = candidate.relevanceScore ?? candidate.score ?? 0;
+
+    // Blend BM25 with original score
+    let finalScore = blendWeight * normBm25 + (1 - blendWeight) * origScore;
+
+    // Signal coherence bonus: failure queries → negative lessons rank higher
+    const candidateSignal =
+      candidate.signal ||
+      (candidate.tags && candidate.tags.includes('negative') ? 'negative' : null);
+    if (isFailureQuery && candidateSignal === 'negative') {
+      finalScore *= 1.2;
+    }
+
+    // Tool name joint bonus: exact tool match between query context and lesson
+    if (toolName) {
+      const lessonTools = [
+        ...(candidate.metadata?.toolsUsed || []),
+        getField(candidate, 'toolUsed'),
+        getField(candidate, 'toolName'),
+      ].map((t) => (t || '').toLowerCase());
+
+      if (lessonTools.some((t) => t && t.includes(toolName.toLowerCase()))) {
+        finalScore *= 1.3;
+      }
+    }
+
+    return { ...candidate, rerankedScore: Number(finalScore.toFixed(6)) };
+  });
+
+  return reranked
+    .sort((a, b) => b.rerankedScore - a.rerankedScore)
+    .slice(0, topK);
+}
+
+module.exports = { rerankLessons, fieldWeightedBM25, tokenize, expandTerms };

package/scripts/lesson-retrieval.js

@@ -3,14 +3,22 @@
 
 /**
  * Per-action lesson retrieval.
- * v2: backward retrieval + bigram Jaccard fuzzy matching
+ * v3: bi-encoder retrieval → cross-encoder reranking
+ *
+ * Stage 1 (bi-encoder): score all memories independently using token overlap,
+ *   bigram Jaccard, tool-name matching, and recency decay.  Retrieve top-50.
+ * Stage 2 (cross-encoder): rerank the top-50 candidates by computing a
+ *   field-weighted BM25 score that processes (query, lesson) jointly, then
+ *   blend with the original bi-encoder score.  Return top-maxResults.
  */
 
 const RECENCY_DECAY_DAYS = 30;
+const RERANK_CANDIDATE_POOL = 50; // bi-encoder retrieves this many; reranker picks topK
 
 function retrieveRelevantLessons(toolName, actionContext, options = {}) {
   const { maxResults = 5, feedbackDir } = options;
   const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
+  const { rerankLessons } = require('./lesson-reranker');
   const pathMod = require('path');
   const paths = feedbackDir
     ? { MEMORY_LOG_PATH: pathMod.join(feedbackDir, 'memory-log.jsonl') }
@@ -21,24 +29,33 @@ function retrieveRelevantLessons(toolName, actionContext, options = {}) {
 
   const actionSig = buildActionSignature(toolName, actionContext);
 
-  const scored = memories.map((mem) => ({
-    ...mem,
-    relevanceScore: scoreRelevance(mem, toolName, actionContext, actionSig),
-  }));
-
-  return scored
+  // Stage 1 — bi-encoder: score all memories independently, take top-50 candidates
+  const candidates = memories
+    .map((mem) => ({
+      ...mem,
+      relevanceScore: scoreRelevance(mem, toolName, actionContext, actionSig),
+    }))
     .filter((m) => m.relevanceScore > 0.1)
     .sort((a, b) => b.relevanceScore - a.relevanceScore)
-    .slice(0, maxResults)
-    .map((m) => ({
-      id: m.id,
-      title: m.title,
-      content: m.content,
-      signal: m.tags?.includes('negative') ? 'negative' : 'positive',
-      rule: m.structuredRule || null,
-      relevanceScore: m.relevanceScore,
-      timestamp: m.timestamp,
-    }));
+    .slice(0, RERANK_CANDIDATE_POOL);
+
+  if (candidates.length === 0) return [];
+
+  // Stage 2 — cross-encoder reranker: rerank candidates by joint (query, lesson) score
+  const reranked = rerankLessons(actionContext, candidates, {
+    topK: maxResults,
+    toolName,
+  });
+
+  return reranked.map((m) => ({
+    id: m.id,
+    title: m.title,
+    content: m.content,
+    signal: m.tags?.includes('negative') ? 'negative' : 'positive',
+    rule: m.structuredRule || null,
+    relevanceScore: m.rerankedScore ?? m.relevanceScore,
+    timestamp: m.timestamp,
+  }));
 }
 
 function buildActionSignature(toolName, actionContext) {

package/scripts/lesson-search.js

@@ -511,6 +511,16 @@ function searchLessons(query = '', options = {}) {
     return String(b.timestamp || '').localeCompare(String(a.timestamp || ''));
   });
 
+  // Cross-encoder reranking: when a query is present, rerank the top-50 bi-encoder
+  // candidates using field-weighted BM25 so the most relevant lessons surface first.
+  if (query && results.length > 1) {
+    const { rerankLessons } = require('./lesson-reranker');
+    const pool = results.slice(0, 50);
+    const tail  = results.slice(50);
+    const reranked = rerankLessons(query, pool, { topK: pool.length });
+    results = [...reranked, ...tail];
+  }
+
   return {
     query: String(query || ''),
     limit,
@@ -630,10 +640,69 @@ function formatLessonSearchResults(payload) {
   return `${lines.join('\n')}\n`;
 }
 
+/**
+ * Enrich lesson search results with Perplexity web context.
+ * Queries Perplexity for each top result's root cause to find external
+ * documentation, known issues, or community solutions.
+ *
+ * Requires PERPLEXITY_API_KEY. Returns original results unmodified if
+ * the API key is missing or calls fail.
+ *
+ * @param {object} searchPayload - Output from searchLessons()
+ * @param {object} [options]
+ * @param {number} [options.enrichLimit=3] - Max results to enrich
+ * @returns {Promise<object>} searchPayload with enriched results
+ */
+async function enrichWithPerplexity(searchPayload, options = {}) {
+  const apiKey = process.env.PERPLEXITY_API_KEY;
+  if (!apiKey || !searchPayload.results || searchPayload.results.length === 0) {
+    return searchPayload;
+  }
+
+  const enrichLimit = Math.min(options.enrichLimit || 3, searchPayload.results.length);
+  const { PerplexityClient, normalizeSearchResults } = require('./perplexity-client');
+  const client = new PerplexityClient({ apiKey });
+
+  const enriched = await Promise.allSettled(
+    searchPayload.results.slice(0, enrichLimit).map(async (result) => {
+      const query = result.lesson.whatWentWrong || result.lesson.summary || result.title;
+      if (!query || query.length < 10) return result;
+
+      try {
+        const searchResults = await client.search({ query, maxResults: 3 });
+        const normalized = normalizeSearchResults(searchResults);
+        if (normalized.length > 0) {
+          result.perplexityContext = {
+            sources: normalized.slice(0, 3).map((s) => ({
+              title: s.title,
+              url: s.url,
+              snippet: s.snippet,
+            })),
+            enrichedAt: new Date().toISOString(),
+          };
+        }
+      } catch {
+        // Perplexity enrichment is best-effort; never block local results
+      }
+      return result;
+    })
+  );
+
+  for (let i = 0; i < enriched.length; i++) {
+    if (enriched[i].status === 'fulfilled') {
+      searchPayload.results[i] = enriched[i].value;
+    }
+  }
+
+  searchPayload.backend = searchPayload.backend + '+perplexity';
+  return searchPayload;
+}
+
 module.exports = {
   parseLessonContent,
   resolveLessonPaths,
   searchLessons,
+  enrichWithPerplexity,
   formatLessonSearchResults,
 };
 

package/scripts/perplexity-client.js

@@ -0,0 +1,210 @@
+'use strict';
+
+const DEFAULT_BASE_URL = 'https://api.perplexity.ai';
+const DEFAULT_TIMEOUT_MS = 120000;
+
+class PerplexityApiError extends Error {
+  constructor(message, details = {}) {
+    super(message);
+    this.name = 'PerplexityApiError';
+    this.status = details.status || null;
+    this.path = details.path || null;
+    this.body = details.body || null;
+  }
+}
+
+function redactSecrets(value) {
+  return String(value || '')
+    .replaceAll(/pplx-[A-Za-z0-9_-]+/g, 'pplx-***')
+    .replaceAll(/Bearer\s+[A-Za-z0-9._-]+/g, 'Bearer ***');
+}
+
+function trimTrailingSlash(value) {
+  let text = String(value || '');
+  while (text.endsWith('/')) {
+    text = text.slice(0, -1);
+  }
+  return text;
+}
+
+function ensureLeadingSlash(value) {
+  return String(value || '').startsWith('/') ? String(value) : `/${value}`;
+}
+
+function buildUrl(baseUrl, path) {
+  return `${trimTrailingSlash(baseUrl || DEFAULT_BASE_URL)}${ensureLeadingSlash(path)}`;
+}
+
+function timeoutSignal(timeoutMs) {
+  if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') {
+    return AbortSignal.timeout(timeoutMs);
+  }
+  return undefined;
+}
+
+function parseJsonSafe(text) {
+  if (!text) return {};
+  try {
+    return JSON.parse(text);
+  } catch {
+    return { raw: text };
+  }
+}
+
+function normalizeSearchResults(response) {
+  const candidates =
+    response?.results ||
+    response?.data ||
+    response?.output?.flatMap((item) => item.results || []) ||
+    [];
+
+  return candidates
+    .filter((item) => item && typeof item === 'object')
+    .map((item, index) => ({
+      rank: Number(item.rank || item.id || index + 1),
+      title: String(item.title || item.name || 'Untitled result'),
+      url: String(item.url || item.link || ''),
+      snippet: String(item.snippet || item.summary || item.text || ''),
+      source: String(item.source || ''),
+      date: item.date || item.published_date || item.last_updated || null,
+    }))
+    .filter((item) => item.url);
+}
+
+function extractChatText(response) {
+  return String(response?.choices?.[0]?.message?.content || '');
+}
+
+function extractAgentText(response) {
+  if (typeof response?.output_text === 'string') return response.output_text;
+  if (typeof response?.text === 'string') return response.text;
+  const output = Array.isArray(response?.output) ? response.output : [];
+  return output.flatMap(extractOutputItemText).join('\n').trim();
+}
+
+function extractOutputItemText(item) {
+  const textParts = [];
+  appendString(textParts, item?.content);
+  if (Array.isArray(item?.content)) {
+    textParts.push(...item.content.flatMap(extractContentPartText));
+  }
+  appendString(textParts, item?.text);
+  return textParts;
+}
+
+function extractContentPartText(part) {
+  const textParts = [];
+  appendString(textParts, part?.text);
+  appendString(textParts, part?.content);
+  return textParts;
+}
+
+function appendString(target, value) {
+  if (typeof value === 'string') target.push(value);
+}
+
+function extractCitations(response) {
+  const citations = response?.citations || response?.search_results || [];
+  return Array.isArray(citations) ? citations : [];
+}
+
+class PerplexityClient {
+  constructor(options = {}) {
+    this.apiKey = options.apiKey || process.env.PERPLEXITY_API_KEY || '';
+    this.baseUrl = options.baseUrl || process.env.PERPLEXITY_BASE_URL || DEFAULT_BASE_URL;
+    this.fetchFn = options.fetchFn || globalThis.fetch;
+    this.timeoutMs = Number(options.timeoutMs || process.env.PERPLEXITY_TIMEOUT_MS || DEFAULT_TIMEOUT_MS);
+
+    if (typeof this.fetchFn !== 'function') {
+      throw new PerplexityApiError('fetch is not available in this Node runtime');
+    }
+  }
+
+  hasApiKey() {
+    return Boolean(this.apiKey);
+  }
+
+  async requestJson(path, body, options = {}) {
+    if (!this.apiKey) {
+      throw new PerplexityApiError('PERPLEXITY_API_KEY is required for live Perplexity calls', { path });
+    }
+
+    const headers = {
+      Authorization: `Bearer ${this.apiKey}`,
+      'Content-Type': 'application/json',
+    };
+    if (options.headers) {
+      Object.assign(headers, options.headers);
+    }
+
+    const response = await this.fetchFn(buildUrl(this.baseUrl, path), {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(body),
+      signal: timeoutSignal(options.timeoutMs || this.timeoutMs),
+    });
+
+    const text = await response.text();
+    const json = parseJsonSafe(text);
+    if (!response.ok) {
+      throw new PerplexityApiError(`Perplexity API ${response.status} on ${path}: ${redactSecrets(text)}`, {
+        status: response.status,
+        path,
+        body: json,
+      });
+    }
+    return json;
+  }
+
+  chatCompletion({ model = 'sonar-pro', messages, options = {} }) {
+    if (!Array.isArray(messages) || messages.length === 0) {
+      throw new PerplexityApiError('chatCompletion requires at least one message');
+    }
+    return this.requestJson('/chat/completions', {
+      model,
+      messages,
+      ...options,
+    });
+  }
+
+  search({ query, maxResults = 5, maxTokensPerPage = 1024, options = {} }) {
+    if (!query) throw new PerplexityApiError('search requires a query');
+    return this.requestJson('/search', {
+      query,
+      max_results: maxResults,
+      max_tokens_per_page: maxTokensPerPage,
+      ...options,
+    });
+  }
+
+  agentResponse({
+    model = 'openai/gpt-5.4',
+    input,
+    instructions,
+    tools,
+    maxOutputTokens,
+    options = {},
+  }) {
+    if (!input) throw new PerplexityApiError('agentResponse requires input');
+    return this.requestJson('/v1/agent', {
+      model,
+      input,
+      ...(instructions ? { instructions } : {}),
+      ...(tools ? { tools } : {}),
+      ...(maxOutputTokens ? { max_output_tokens: maxOutputTokens } : {}),
+      ...options,
+    });
+  }
+}
+
+module.exports = {
+  DEFAULT_BASE_URL,
+  PerplexityApiError,
+  PerplexityClient,
+  buildUrl,
+  extractAgentText,
+  extractChatText,
+  extractCitations,
+  normalizeSearchResults,
+  redactSecrets,
+};