thumbgate 1.28.1 → 1.28.3

package/scripts/cross-encoder-reranker.js

@@ -0,0 +1,235 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Cross-Encoder Reranker for ThumbGate lesson retrieval.
+ *
+ * Two-stage retrieval:
+ *   Stage 1: Fast candidate retrieval (existing bigram Jaccard + keyword matching)
+ *   Stage 2: Cross-encoder reranking scores query-document pairs jointly
+ *
+ * The cross-encoder evaluates the query AND each lesson together (not independently),
+ * catching false positives that keyword/vector search misses.
+ *
+ * Architecture reference: "Advanced RAG Retrieval: Cross-Encoders & Reranking"
+ * (Towards Data Science, April 2026)
+ *
+ * When LLM is available (ANTHROPIC_API_KEY), uses Claude as the cross-encoder.
+ * Falls back to enhanced heuristic scoring when LLM is unavailable.
+ */
+
+const { retrieveRelevantLessons, scoreRelevance, buildActionSignature } = require('./lesson-retrieval');
+
+/**
+ * Heuristic cross-encoder: scores a (query, document) pair jointly.
+ * Unlike bi-encoder (independent embeddings), this examines the pair together
+ * to find semantic relationships that keyword matching misses.
+ */
+function heuristicCrossEncode(query, document) {
+  const queryLower = (query || '').toLowerCase();
+  const docLower = (document || '').toLowerCase();
+
+  let score = 0;
+
+  // 1. Exact substring containment (strongest signal)
+  if (queryLower.length > 3 && docLower.length > 3 &&
+      (docLower.includes(queryLower) || queryLower.includes(docLower))) {
+    score += 0.9;
+    return Math.min(score, 1);
+  }
+
+  // 2. Shared noun phrases (not just tokens — consecutive word pairs)
+  const queryPhrases = extractPhrases(queryLower);
+  const docPhrases = extractPhrases(docLower);
+  const phraseOverlap = queryPhrases.filter((p) => docPhrases.includes(p));
+  score += Math.min(phraseOverlap.length * 0.15, 0.5);
+
+  // 3. Semantic category matching
+  const categories = {
+    destructive: ['delete', 'remove', 'drop', 'destroy', 'wipe', 'truncate', 'rm -rf', 'force-push', 'reset --hard'],
+    git: ['git', 'push', 'pull', 'merge', 'rebase', 'branch', 'commit', 'checkout', 'stash'],
+    database: ['sql', 'query', 'table', 'migration', 'schema', 'database', 'insert', 'update', 'select'],
+    deploy: ['deploy', 'release', 'publish', 'railway', 'vercel', 'heroku', 'npm publish'],
+    security: ['secret', 'token', 'api key', 'password', 'credential', 'env', '.env', 'pem'],
+    file: ['edit', 'write', 'create', 'modify', 'config', 'package.json', 'readme'],
+  };
+
+  for (const [, terms] of Object.entries(categories)) {
+    const queryHit = terms.some((t) => queryLower.includes(t));
+    const docHit = terms.some((t) => docLower.includes(t));
+    if (queryHit && docHit) {
+      score += 0.25;
+      break; // Only count strongest category match
+    }
+  }
+
+  // 4. Action-target alignment (e.g., "git push" in query matches "push to main" in doc)
+  const queryVerbs = extractVerbs(queryLower);
+  const docVerbs = extractVerbs(docLower);
+  const verbOverlap = queryVerbs.filter((v) => docVerbs.includes(v));
+  score += Math.min(verbOverlap.length * 0.1, 0.3);
+
+  // 5. Negation alignment (both about what NOT to do)
+  const queryNegated = /\b(don'?t|never|avoid|block|prevent|stop)\b/.test(queryLower);
+  const docNegated = /\b(don'?t|never|avoid|block|prevent|stop)\b/.test(docLower);
+  if (queryNegated && docNegated) score += 0.1;
+
+  return Math.min(score, 1);
+}
+
+/**
+ * LLM cross-encoder: uses Claude to score relevance of query-document pairs.
+ * More accurate but requires API key and costs tokens.
+ */
+async function llmCrossEncode(query, documents) {
+  const { isAvailable, callClaudeJson, MODELS } = require('./llm-client');
+  if (!isAvailable()) return null;
+
+  const docList = documents
+    .map((d, i) => `[${i}] ${(d.title || '').slice(0, 100)} | ${(d.content || '').slice(0, 200)}`)
+    .join('\n');
+
+  const prompt = `You are a relevance scoring engine. Given a query and a list of documents, score each document's relevance to the query from 0.0 (irrelevant) to 1.0 (highly relevant).
+
+Query: "${query.slice(0, 300)}"
+
+Documents:
+${docList}
+
+Return ONLY a JSON array of scores, one per document. Example: [0.9, 0.2, 0.7, 0.1, 0.5]
+No other text.`;
+
+  try {
+    const scores = await callClaudeJson({
+      systemPrompt: 'You are a relevance scoring engine. Return only JSON arrays of numbers.',
+      userPrompt: prompt,
+      model: MODELS.FAST,
+      maxTokens: 256,
+      cache: true,
+    });
+    if (Array.isArray(scores) && scores.length === documents.length) {
+      return scores.map((s) => Math.max(0, Math.min(1, Number(s) || 0)));
+    }
+  } catch { /* fall back to heuristic */ }
+  return null;
+}
+
+/**
+ * Two-stage retrieval with cross-encoder reranking.
+ *
+ * Stage 1: Retrieve top N candidates using existing keyword + bigram matching
+ * Stage 2: Rerank candidates using cross-encoder (LLM or heuristic)
+ * Return top K results by cross-encoder score
+ */
+async function retrieveWithReranking(toolName, actionContext, options = {}) {
+  const {
+    candidateCount = 20,
+    maxResults = 5,
+    useLLM = false,
+    feedbackDir,
+  } = options;
+
+  // Stage 1: Fast candidate retrieval (existing system)
+  const candidates = retrieveRelevantLessons(toolName, actionContext, {
+    maxResults: candidateCount,
+    feedbackDir,
+  });
+
+  if (candidates.length === 0) return [];
+  if (candidates.length <= maxResults) return candidates;
+
+  const query = `${toolName || ''} ${actionContext || ''}`.trim();
+
+  // Stage 2: Cross-encoder reranking
+  let rerankedScores;
+
+  if (useLLM) {
+    rerankedScores = await llmCrossEncode(query, candidates);
+  }
+
+  // Fall back to heuristic cross-encoder if LLM unavailable or failed
+  if (!rerankedScores) {
+    rerankedScores = candidates.map((c) => {
+      const docText = `${c.title || ''} ${c.content || ''}`;
+      return heuristicCrossEncode(query, docText);
+    });
+  }
+
+  // Combine original relevance score with cross-encoder score
+  // Weight: 40% original, 60% cross-encoder (cross-encoder is more precise)
+  const reranked = candidates.map((c, i) => ({
+    ...c,
+    crossEncoderScore: rerankedScores[i],
+    combinedScore: c.relevanceScore * 0.4 + rerankedScores[i] * 0.6,
+  }));
+
+  return reranked
+    .sort((a, b) => b.combinedScore - a.combinedScore)
+    .slice(0, maxResults);
+}
+
+/**
+ * Synchronous version for use in PreToolUse hooks (cannot be async).
+ */
+function retrieveWithRerankingSync(toolName, actionContext, options = {}) {
+  const {
+    candidateCount = 20,
+    maxResults = 5,
+    feedbackDir,
+  } = options;
+
+  const candidates = retrieveRelevantLessons(toolName, actionContext, {
+    maxResults: candidateCount,
+    feedbackDir,
+  });
+
+  if (candidates.length === 0) return [];
+  if (candidates.length <= maxResults) return candidates;
+
+  const query = `${toolName || ''} ${actionContext || ''}`.trim();
+
+  const rerankedScores = candidates.map((c) => {
+    const docText = `${c.title || ''} ${c.content || ''}`;
+    return heuristicCrossEncode(query, docText);
+  });
+
+  const reranked = candidates.map((c, i) => ({
+    ...c,
+    crossEncoderScore: rerankedScores[i],
+    combinedScore: c.relevanceScore * 0.4 + rerankedScores[i] * 0.6,
+  }));
+
+  return reranked
+    .sort((a, b) => b.combinedScore - a.combinedScore)
+    .slice(0, maxResults);
+}
+
+// --- Utility functions ---
+
+function extractPhrases(text) {
+  const words = text.split(/\s+/).filter((w) => w.length > 2);
+  const phrases = [];
+  for (let i = 0; i < words.length - 1; i++) {
+    phrases.push(`${words[i]} ${words[i + 1]}`);
+  }
+  return phrases;
+}
+
+function extractVerbs(text) {
+  const verbPatterns = [
+    'push', 'pull', 'merge', 'delete', 'create', 'edit', 'write', 'read',
+    'deploy', 'install', 'remove', 'run', 'execute', 'build', 'test',
+    'commit', 'rebase', 'reset', 'drop', 'truncate', 'migrate', 'publish',
+    'block', 'allow', 'approve', 'deny', 'warn', 'log',
+  ];
+  return verbPatterns.filter((v) => text.includes(v));
+}
+
+module.exports = {
+  heuristicCrossEncode,
+  llmCrossEncode,
+  retrieveWithReranking,
+  retrieveWithRerankingSync,
+  extractPhrases,
+  extractVerbs,
+};

package/scripts/feedback-history-distiller.js

@@ -0,0 +1,385 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { resolveFeedbackDir: resolveSharedFeedbackDir } = require('./feedback-paths');
+const { readJsonlTail } = require('./fs-utils');
+const { redactSecretsDeep } = require('./secret-redaction');
+
+const DEFAULT_HISTORY_LIMIT = 10;
+
+const CORRECTION_PATTERNS = [
+  /\bdon['’]?t\b/i,
+  /\bdo not\b/i,
+  /\bnever\b/i,
+  /\bavoid\b/i,
+  /\bstop\b/i,
+  /\bmust\b/i,
+  /\bshould\b/i,
+  /\bneed to\b/i,
+  /\buse\b/i,
+  /\brun tests?\b/i,
+  /\binclude\b/i,
+  /\bprove\b/i,
+];
+
+const FAILURE_PATTERNS = [
+  /\bfailed\b/i,
+  /\bbroke\b/i,
+  /\berror\b/i,
+  /\bwrong\b/i,
+  /\bignored\b/i,
+  /\bskipped\b/i,
+  /\bhallucinat/i,
+  /\bclaimed done\b/i,
+  /\bwithout proof\b/i,
+  /\bwithout evidence\b/i,
+];
+
+const SUCCESS_PATTERNS = [
+  /\bworked\b/i,
+  /\bpassed\b/i,
+  /\bverified\b/i,
+  /\bproof\b/i,
+  /\bevidence\b/i,
+  /\btests?\b/i,
+  /\bfixed\b/i,
+  /\bsuccess/i,
+];
+
+function normalizeText(value) {
+  return String(value || '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function truncate(value, max = 180) {
+  const text = normalizeText(value);
+  if (text.length <= max) return text;
+  return `${text.slice(0, max - 1).trim()}…`;
+}
+
+function appendJsonl(filePath, record) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  // Redact secrets before any conversation turn touches disk — conversation-window.jsonl was the
+  // 2026-06-10 sk_live_ leak vector. See scripts/secret-redaction.js.
+  fs.appendFileSync(filePath, `${JSON.stringify(redactSecretsDeep(record))}\n`);
+}
+
+function resolveFeedbackDir(feedbackDir) {
+  if (feedbackDir) {
+    return resolveSharedFeedbackDir({ feedbackDir });
+  }
+  const env = { ...process.env };
+  delete env.INIT_CWD;
+  delete env.PWD;
+  return resolveSharedFeedbackDir({ env });
+}
+
+function getConversationPaths(feedbackDir) {
+  const resolved = resolveFeedbackDir(feedbackDir);
+  return {
+    feedbackDir: resolved,
+    conversationLogPath: path.join(resolved, 'conversation-window.jsonl'),
+    feedbackLogPath: path.join(resolved, 'feedback-log.jsonl'),
+  };
+}
+
+function normalizeChatHistory(entries = []) {
+  if (!Array.isArray(entries)) return [];
+  return entries
+    .map((entry) => {
+      if (typeof entry === 'string') {
+        const text = normalizeText(entry);
+        return text ? { author: null, text, timestamp: null, source: 'chat_history' } : null;
+      }
+      if (!entry || typeof entry !== 'object') return null;
+      const text = normalizeText(entry.text || entry.body || entry.message || entry.content);
+      if (!text) return null;
+      return {
+        author: normalizeText(entry.author || entry.role || entry.user || entry.name) || null,
+        text,
+        timestamp: normalizeText(entry.timestamp || entry.createdAt || entry.updatedAt) || null,
+        source: normalizeText(entry.source) || 'chat_history',
+      };
+    })
+    .filter(Boolean);
+}
+
+function recordConversationEntry(entry, options = {}) {
+  const { conversationLogPath } = getConversationPaths(options.feedbackDir);
+  const normalized = normalizeChatHistory([entry])[0];
+  if (!normalized) {
+    return { recorded: false, reason: 'empty_text', conversationLogPath };
+  }
+  const record = {
+    ...normalized,
+    timestamp: normalized.timestamp || new Date().toISOString(),
+  };
+  appendJsonl(conversationLogPath, record);
+  return { recorded: true, record, conversationLogPath };
+}
+
+function readRecentConversationWindow(options = {}) {
+  const limit = Number(options.limit || DEFAULT_HISTORY_LIMIT);
+  const { conversationLogPath } = getConversationPaths(options.feedbackDir);
+  return readJsonlTail(conversationLogPath, limit)
+    .map((entry) => normalizeChatHistory([entry])[0])
+    .filter(Boolean);
+}
+
+function findFeedbackEventById(feedbackId, options = {}) {
+  if (!feedbackId) return null;
+  const { feedbackLogPath } = getConversationPaths(options.feedbackDir);
+  const matches = readJsonlTail(feedbackLogPath, Number(options.searchLimit || 200));
+  for (let index = matches.length - 1; index >= 0; index -= 1) {
+    if (matches[index] && matches[index].id === feedbackId) {
+      return matches[index];
+    }
+  }
+  return null;
+}
+
+function buildLastActionMessage(lastAction) {
+  if (!lastAction || typeof lastAction !== 'object') return null;
+  const tool = normalizeText(lastAction.tool || lastAction.tool_name || 'unknown tool');
+  const file = normalizeText(lastAction.file || lastAction.path || '');
+  const detail = file ? `${tool} on ${file}` : tool;
+  return {
+    author: 'tool',
+    text: `Last action: ${detail}`,
+    timestamp: normalizeText(lastAction.timestamp) || null,
+    source: 'last_action',
+  };
+}
+
+function buildRelatedFeedbackMessages(feedbackEvent) {
+  if (!feedbackEvent || typeof feedbackEvent !== 'object') return [];
+  const messages = [];
+
+  if (feedbackEvent.conversationWindow && Array.isArray(feedbackEvent.conversationWindow)) {
+    for (const entry of normalizeChatHistory(feedbackEvent.conversationWindow)) {
+      messages.push({ ...entry, source: 'related_feedback_window' });
+    }
+  }
+
+  const snippets = [
+    feedbackEvent.submittedContext || null,
+    feedbackEvent.context || null,
+    feedbackEvent.whatWentWrong || null,
+    feedbackEvent.whatWorked || null,
+    feedbackEvent.whatToChange || null,
+  ].filter(Boolean);
+
+  for (const text of snippets) {
+    messages.push({
+      author: 'related-feedback',
+      text: normalizeText(text),
+      timestamp: normalizeText(feedbackEvent.timestamp) || null,
+      source: 'related_feedback',
+    });
+  }
+
+  const lastActionMessage = buildLastActionMessage(feedbackEvent.lastAction);
+  if (lastActionMessage) messages.push(lastActionMessage);
+
+  return messages;
+}
+
+function matchesAny(text, patterns) {
+  return patterns.some((pattern) => pattern.test(text));
+}
+
+function chooseMessage(messages, predicate) {
+  for (let index = messages.length - 1; index >= 0; index -= 1) {
+    if (predicate(messages[index])) return messages[index];
+  }
+  return null;
+}
+
+function buildRuleSuggestion(correctionMessage, signal) {
+  if (!correctionMessage) return null;
+  const text = truncate(correctionMessage.text, 120);
+  if (signal === 'negative' && /\bnever\b/i.test(text)) return text;
+  if (signal === 'negative' && /\bdo not\b/i.test(text)) {
+    return text.replace(/\bdo not\b/i, 'Never');
+  }
+  if (signal === 'negative' && /\bdon['’]?t\b/i.test(text)) {
+    return text
+      .replace(/\bdon['’]?t\b/i, 'Never')
+      .replace(/\bdo not\b/i, 'Never');
+  }
+  if (signal === 'negative' && /\bavoid\b/i.test(text)) {
+    return text.replace(/\bavoid\b/i, 'Avoid');
+  }
+  if (signal === 'negative') return `Follow the earlier instruction: ${text}`;
+  return `Repeat the successful pattern: ${text}`;
+}
+
+function inferNegativeDistillation(messages, params) {
+  const correctionMessage = chooseMessage(messages, (entry) => {
+    const text = normalizeText(entry.text);
+    return Boolean(text) && matchesAny(text, CORRECTION_PATTERNS);
+  });
+
+  const failureMessage = chooseMessage(messages, (entry) => {
+    const text = normalizeText(entry.text);
+    return Boolean(text) && matchesAny(text, FAILURE_PATTERNS);
+  }) || buildLastActionMessage(params.lastAction);
+
+  if (!correctionMessage && !failureMessage) {
+    return {
+      usedHistory: false,
+      inferredFields: {},
+      lessonProposal: null,
+      evidence: [],
+      source: 'none',
+    };
+  }
+
+  const evidence = [correctionMessage, failureMessage].filter(Boolean).map((entry) => truncate(entry.text));
+  const whatWentWrong = correctionMessage
+    ? `It ignored a prior instruction: ${truncate(correctionMessage.text, 140)}`
+    : `The failure centered on: ${truncate(failureMessage.text, 140)}`;
+  const whatToChange = correctionMessage
+    ? `Follow the earlier instruction instead of repeating the same pattern: ${truncate(correctionMessage.text, 140)}`
+    : failureMessage
+      ? `Inspect and correct the failing step before repeating it: ${truncate(failureMessage.text, 140)}`
+      : null;
+  const context = failureMessage
+    ? `History-aware distillation linked this failure to ${truncate(failureMessage.text, 140)}`
+    : `History-aware distillation linked this failure to ${truncate(correctionMessage.text, 140)}`;
+
+  return {
+    usedHistory: true,
+    source: correctionMessage ? correctionMessage.source : failureMessage.source,
+    inferredFields: {
+      context,
+      whatWentWrong,
+      whatToChange,
+    },
+    lessonProposal: {
+      summary: whatWentWrong,
+      proposedRule: buildRuleSuggestion(correctionMessage, 'negative'),
+      confidence: correctionMessage && failureMessage ? 0.92 : 0.78,
+    },
+    evidence,
+  };
+}
+
+function inferPositiveDistillation(messages) {
+  const successMessage = chooseMessage(messages, (entry) => {
+    const text = normalizeText(entry.text);
+    return Boolean(text) && matchesAny(text, SUCCESS_PATTERNS);
+  });
+
+  if (!successMessage) {
+    return {
+      usedHistory: false,
+      inferredFields: {},
+      lessonProposal: null,
+      evidence: [],
+      source: 'none',
+    };
+  }
+
+  const whatWorked = `The successful pattern was: ${truncate(successMessage.text, 140)}`;
+  return {
+    usedHistory: true,
+    source: successMessage.source,
+    inferredFields: {
+      context: `History-aware distillation found a successful pattern in recent conversation: ${truncate(successMessage.text, 140)}`,
+      whatWorked,
+    },
+    lessonProposal: {
+      summary: whatWorked,
+      proposedRule: buildRuleSuggestion(successMessage, 'positive'),
+      confidence: 0.81,
+    },
+    evidence: [truncate(successMessage.text)],
+  };
+}
+
+function distillFeedbackHistory(params = {}) {
+  const signal = String(params.signal || '').toLowerCase().trim();
+  const historyLimit = Number(params.historyLimit || DEFAULT_HISTORY_LIMIT);
+  const explicitHistory = normalizeChatHistory(params.chatHistory || params.messages || []);
+  const fallbackHistory = params.allowLocalConversationFallback
+    ? readRecentConversationWindow({ feedbackDir: params.feedbackDir, limit: historyLimit })
+    : [];
+  const relatedEvent = findFeedbackEventById(params.relatedFeedbackId, {
+    feedbackDir: params.feedbackDir,
+    searchLimit: params.searchLimit,
+  });
+
+  const conversationWindow = [
+    ...explicitHistory,
+    ...(explicitHistory.length === 0 ? fallbackHistory : []),
+    ...buildRelatedFeedbackMessages(relatedEvent),
+  ]
+    .filter((entry) => entry && normalizeText(entry.text))
+    .slice(-historyLimit);
+
+  const distillation = signal === 'negative'
+    ? inferNegativeDistillation(conversationWindow, params)
+    : inferPositiveDistillation(conversationWindow, params);
+
+  return {
+    usedHistory: distillation.usedHistory,
+    source: explicitHistory.length > 0
+      ? 'chat_history'
+      : fallbackHistory.length > 0
+        ? 'local_conversation_window'
+        : relatedEvent ? 'related_feedback' : 'none',
+    conversationWindow,
+    relatedFeedbackId: relatedEvent ? relatedEvent.id : null,
+    inferredFields: distillation.inferredFields,
+    lessonProposal: distillation.lessonProposal,
+    evidence: distillation.evidence,
+  };
+}
+
+function main(argv = process.argv.slice(2)) {
+  const [command, ...rest] = argv;
+  if (command !== 'record') {
+    console.error('Usage: node scripts/feedback-history-distiller.js record --author=user --text="..."');
+    process.exit(1);
+  }
+
+  const args = {};
+  for (const token of rest) {
+    if (!token.startsWith('--')) continue;
+    const [key, ...valueParts] = token.slice(2).split('=');
+    args[key] = valueParts.join('=');
+  }
+
+  const result = recordConversationEntry({
+    author: args.author || null,
+    text: args.text || '',
+    timestamp: args.timestamp || new Date().toISOString(),
+    source: args.source || 'cli_record',
+  }, {
+    feedbackDir: args.feedbackDir,
+  });
+
+  if (!result.recorded) {
+    console.error(result.reason || 'failed_to_record');
+    process.exit(1);
+  }
+
+  process.stdout.write(JSON.stringify(result, null, 2));
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  DEFAULT_HISTORY_LIMIT,
+  distillFeedbackHistory,
+  findFeedbackEventById,
+  getConversationPaths,
+  normalizeChatHistory,
+  readRecentConversationWindow,
+  recordConversationEntry,
+};