thumbgate 1.30.0 → 1.31.0

package/scripts/rerank-pipeline.js

@@ -0,0 +1,370 @@
+'use strict';
+
+/**
+ * Multi-stage rerank pipeline for ThumbGate's A+ target.
+ *
+ * Stages (always local-first; LLM optional and last):
+ *   1) Field-weighted BM25F pair scoring (lesson-reranker)
+ *   2) ColBERT-style MaxSim late interaction (colbert-style-maxsim)
+ *   3) Heuristic joint pair scorer (cross-encoder-reranker.heuristicCrossEncode)
+ *   4) Optional listwise LLM rerank on the final shortlist (useLLM / env)
+ *
+ * Honesty contract:
+ *   - Stage 3 is a *heuristic* cross-encoder, not a neural CE checkpoint
+ *   - Stage 2 is ColBERT-*style* MaxSim over hashed multi-vectors unless
+ *     a tokenEmbedder is supplied
+ *   - LLM stage is off by default; enable with useLLM:true or THUMBGATE_RERANK_LLM=1
+ *
+ * Pipeline version is exported so evals and statuslines can pin provenance.
+ */
+
+const PIPELINE_VERSION = '2026-07-31.a-plus.1';
+
+const { rerankLessons } = require('./lesson-reranker');
+const { rerankWithMaxSim, scoreLateInteraction } = require('./colbert-style-maxsim');
+const { heuristicCrossEncode, llmCrossEncode } = require('./cross-encoder-reranker');
+
+/**
+ * @typedef {object} RerankPipelineOptions
+ * @property {number} [topK=5]
+ * @property {string} [toolName]
+ * @property {boolean} [useLLM=false]
+ * @property {boolean} [useMaxSim=true]
+ * @property {boolean} [useHeuristicCe=true]
+ * @property {number} [bm25Pool=50] candidates to keep after BM25 before MaxSim
+ * @property {number} [llmShortlist=8] max docs sent to LLM listwise scorer
+ * @property {number} [wBm25=0.30]
+ * @property {number} [wMaxSim=0.35]
+ * @property {number} [wHeuristic=0.25]
+ * @property {number} [wOriginal=0.10]
+ * @property {(c: object) => string} [textOf]
+ */
+
+function defaultTextOf(c) {
+  if (!c || typeof c !== 'object') return String(c || '');
+  return [
+    c.title,
+    c.whatWentWrong,
+    c.whatToChange,
+    c.howToAvoid,
+    c.summary,
+    c.content,
+    c.context,
+    Array.isArray(c.tags) ? c.tags.join(' ') : '',
+  ].filter(Boolean).join(' ');
+}
+
+function envFlag(name) {
+  const v = String(process.env[name] || '').trim().toLowerCase();
+  return v === '1' || v === 'true' || v === 'yes' || v === 'on';
+}
+
+/**
+ * Fuse multi-stage scores into a final ranking.
+ * @param {string} query
+ * @param {Array<object>} candidates
+ * @param {RerankPipelineOptions} [options]
+ * @returns {Promise<{ results: Array<object>, meta: object }>}
+ */
+async function rerankPipeline(query, candidates, options = {}) {
+  const topK = options.topK ?? 5;
+  const toolName = options.toolName || '';
+  const useMaxSim = options.useMaxSim !== false;
+  const useHeuristicCe = options.useHeuristicCe !== false;
+  const useLLM = options.useLLM === true || envFlag('THUMBGATE_RERANK_LLM');
+  const bm25Pool = Math.max(topK, options.bm25Pool ?? 50);
+  const llmShortlist = Math.max(topK, options.llmShortlist ?? 8);
+  const textOf = options.textOf || defaultTextOf;
+  const wBm25 = options.wBm25 ?? 0.30;
+  const wMaxSim = options.wMaxSim ?? 0.35;
+  const wHeuristic = options.wHeuristic ?? 0.25;
+  const wOriginal = options.wOriginal ?? 0.10;
+
+  const meta = {
+    pipelineVersion: PIPELINE_VERSION,
+    stages: [],
+    useLLM,
+    useMaxSim,
+    useHeuristicCe,
+    inputCount: candidates?.length || 0,
+  };
+
+  if (!candidates || candidates.length === 0) {
+    return { results: [], meta: { ...meta, stages: ['empty'] } };
+  }
+
+  // --- Stage 1: BM25F ---
+  let pool = rerankLessons(query, candidates, {
+    topK: Math.min(bm25Pool, candidates.length),
+    toolName,
+    blendWeight: 0.7,
+  });
+  meta.stages.push('bm25f');
+
+  // Capture BM25 scores before MaxSim overwrites rerankedScore
+  pool = pool.map((c) => ({
+    ...c,
+    bm25Score: Number(c.rerankedScore ?? 0),
+    originalScore: Number(c.relevanceScore ?? c.score ?? 0),
+  }));
+
+  // --- Stage 2: ColBERT-style MaxSim ---
+  if (useMaxSim && pool.length > 1) {
+    pool = rerankWithMaxSim(query, pool, {
+      topK: pool.length,
+      textOf,
+      blendWeight: 1, // pure MaxSim into maxSimScore; we fuse ourselves
+      dim: options.dim,
+      ngram: options.ngram,
+      maxTokens: options.maxTokens,
+      tokenEmbedder: options.tokenEmbedder,
+    }).map((c) => ({
+      ...c,
+      // restore bm25 from previous map (rerankWithMaxSim spreads candidate)
+      bm25Score: c.bm25Score,
+      maxSimScore: Number(c.maxSimScore ?? 0),
+    }));
+    meta.stages.push('colbert-style-maxsim');
+  } else {
+    pool = pool.map((c) => ({ ...c, maxSimScore: 0 }));
+  }
+
+  // --- Stage 3: Heuristic joint pair scorer ---
+  if (useHeuristicCe) {
+    pool = pool.map((c) => {
+      const he = heuristicCrossEncode(
+        `${toolName} ${query}`.trim(),
+        textOf(c),
+      );
+      return { ...c, heuristicCeScore: he };
+    });
+    meta.stages.push('heuristic-pair-ce');
+  } else {
+    pool = pool.map((c) => ({ ...c, heuristicCeScore: 0 }));
+  }
+
+  // --- Fuse ---
+  // Normalize bm25 within pool for fair blend
+  const maxBm25 = Math.max(...pool.map((c) => c.bm25Score || 0), 1e-9);
+  pool = pool.map((c) => {
+    const nBm25 = (c.bm25Score || 0) / maxBm25;
+    const nOrig = Math.max(0, Math.min(1, c.originalScore || 0));
+    const nMs = Math.max(0, Math.min(1, c.maxSimScore || 0));
+    const nHe = Math.max(0, Math.min(1, c.heuristicCeScore || 0));
+    const fused =
+      wBm25 * nBm25 +
+      wMaxSim * nMs +
+      wHeuristic * nHe +
+      wOriginal * nOrig;
+    return {
+      ...c,
+      fusedScore: Number(fused.toFixed(6)),
+      rerankedScore: Number(fused.toFixed(6)),
+      pairwiseHeuristicScore: nHe,
+      lateInteractionScore: nMs,
+      crossEncoderScore: null,
+      combinedScore: Number(fused.toFixed(6)),
+      reranker: {
+        stages: [
+          'first-stage',
+          'bm25f',
+          'colbert-style-maxsim-hashed',
+          'pairwise-heuristic',
+          'score-fusion',
+        ],
+        fallbacks: ['neural-cross-encoder-not-configured'],
+      },
+    };
+  }).sort((a, b) => b.fusedScore - a.fusedScore);
+
+  meta.stages.push('score-fusion');
+
+  // --- Stage 4: optional LLM listwise on shortlist ---
+  let shortlist = pool.slice(0, Math.min(llmShortlist, pool.length));
+  if (useLLM && shortlist.length > 1) {
+    const llmScores = await llmCrossEncode(
+      `${toolName} ${query}`.trim(),
+      shortlist.map((c) => ({
+        title: c.title || '',
+        content: textOf(c).slice(0, 400),
+      })),
+    );
+    if (llmScores) {
+      shortlist = shortlist.map((c, i) => {
+        const llm = Math.max(0, Math.min(1, Number(llmScores[i]) || 0));
+        // Blend LLM lightly so a bad model cannot erase local signal
+        const final = 0.55 * llm + 0.45 * (c.fusedScore || 0);
+        return {
+          ...c,
+          llmRerankScore: llm,
+          fusedScore: Number(final.toFixed(6)),
+          rerankedScore: Number(final.toFixed(6)),
+          combinedScore: Number(final.toFixed(6)),
+        };
+      }).sort((a, b) => b.fusedScore - a.fusedScore);
+      meta.stages.push('llm-listwise');
+      meta.llmApplied = true;
+    } else {
+      meta.llmApplied = false;
+      meta.stages.push('llm-fallback');
+    }
+  } else {
+    meta.llmApplied = false;
+  }
+
+  const results = shortlist.slice(0, topK).map((c) => ({
+    ...c,
+    rerankPipelineVersion: PIPELINE_VERSION,
+  }));
+
+  meta.outputCount = results.length;
+  meta.rankDelta = computeRankDelta(candidates, results);
+
+  return { results, meta };
+}
+
+/**
+ * Sync path for PreToolUse hooks (no LLM).
+ */
+function rerankPipelineSync(query, candidates, options = {}) {
+  return rerankPipelineSyncImpl(query, candidates, options);
+}
+
+function rerankPipelineSyncImpl(query, candidates, options = {}) {
+  const topK = options.topK ?? 5;
+  const toolName = options.toolName || '';
+  const useMaxSim = options.useMaxSim !== false;
+  const useHeuristicCe = options.useHeuristicCe !== false;
+  const bm25Pool = Math.max(topK, options.bm25Pool ?? 50);
+  const textOf = options.textOf || defaultTextOf;
+  const wBm25 = options.wBm25 ?? 0.30;
+  const wMaxSim = options.wMaxSim ?? 0.35;
+  const wHeuristic = options.wHeuristic ?? 0.25;
+  const wOriginal = options.wOriginal ?? 0.10;
+
+  const meta = {
+    pipelineVersion: PIPELINE_VERSION,
+    stages: [],
+    useLLM: false,
+    useMaxSim,
+    useHeuristicCe,
+    inputCount: candidates?.length || 0,
+    llmApplied: false,
+  };
+
+  if (!candidates || candidates.length === 0) {
+    return { results: [], meta: { ...meta, stages: ['empty'] } };
+  }
+
+  let pool = rerankLessons(query, candidates, {
+    topK: Math.min(bm25Pool, candidates.length),
+    toolName,
+    blendWeight: 0.7,
+  }).map((c) => ({
+    ...c,
+    bm25Score: Number(c.rerankedScore ?? 0),
+    originalScore: Number(c.relevanceScore ?? c.score ?? 0),
+  }));
+  meta.stages.push('bm25f');
+
+  if (useMaxSim && pool.length > 1) {
+    pool = rerankWithMaxSim(query, pool, {
+      topK: pool.length,
+      textOf,
+      blendWeight: 1,
+    }).map((c) => ({
+      ...c,
+      bm25Score: c.bm25Score,
+      maxSimScore: Number(c.maxSimScore ?? 0),
+    }));
+    meta.stages.push('colbert-style-maxsim');
+  } else {
+    pool = pool.map((c) => ({ ...c, maxSimScore: 0 }));
+  }
+
+  if (useHeuristicCe) {
+    pool = pool.map((c) => ({
+      ...c,
+      heuristicCeScore: heuristicCrossEncode(`${toolName} ${query}`.trim(), textOf(c)),
+    }));
+    meta.stages.push('heuristic-pair-ce');
+  } else {
+    pool = pool.map((c) => ({ ...c, heuristicCeScore: 0 }));
+  }
+
+  const maxBm25 = Math.max(...pool.map((c) => c.bm25Score || 0), 1e-9);
+  pool = pool.map((c) => {
+    const nBm25 = (c.bm25Score || 0) / maxBm25;
+    const nOrig = Math.max(0, Math.min(1, c.originalScore || 0));
+    const nMs = Math.max(0, Math.min(1, c.maxSimScore || 0));
+    const nHe = Math.max(0, Math.min(1, c.heuristicCeScore || 0));
+    const fused =
+      wBm25 * nBm25 + wMaxSim * nMs + wHeuristic * nHe + wOriginal * nOrig;
+    return {
+      ...c,
+      fusedScore: Number(fused.toFixed(6)),
+      rerankedScore: Number(fused.toFixed(6)),
+      pairwiseHeuristicScore: nHe,
+      lateInteractionScore: nMs,
+      crossEncoderScore: null,
+      combinedScore: Number(fused.toFixed(6)),
+      rerankPipelineVersion: PIPELINE_VERSION,
+      reranker: {
+        stages: [
+          'first-stage',
+          'bm25f',
+          'colbert-style-maxsim-hashed',
+          'pairwise-heuristic',
+          'score-fusion',
+        ],
+        fallbacks: ['neural-cross-encoder-not-configured'],
+      },
+    };
+  }).sort((a, b) => b.fusedScore - a.fusedScore);
+
+  meta.stages.push('score-fusion');
+  const results = pool.slice(0, topK);
+  meta.outputCount = results.length;
+  meta.rankDelta = computeRankDelta(candidates, results);
+  return { results, meta };
+}
+
+/**
+ * Whether top-1 id changed vs original order (rank-delta signal).
+ * @param {Array<object>} original
+ * @param {Array<object>} reranked
+ * @returns {{ flipped: boolean, originalTopId: string|null, rerankedTopId: string|null }}
+ */
+function computeRankDelta(original, reranked) {
+  const idOf = (c) => c?.id || c?.lessonId || c?.title || null;
+  const originalTopId = original?.[0] ? idOf(original[0]) : null;
+  const rerankedTopId = reranked?.[0] ? idOf(reranked[0]) : null;
+  return {
+    flipped: Boolean(originalTopId && rerankedTopId && originalTopId !== rerankedTopId),
+    originalTopId,
+    rerankedTopId,
+  };
+}
+
+/**
+ * Pair-level scores for diagnostics / evals.
+ */
+function scorePair(query, document, opts = {}) {
+  const late = scoreLateInteraction(query, document, opts);
+  const he = heuristicCrossEncode(query, document);
+  return {
+    maxSim: late.score,
+    heuristicCe: he,
+    mode: late.mode,
+    pipelineVersion: PIPELINE_VERSION,
+  };
+}
+
+module.exports = {
+  PIPELINE_VERSION,
+  rerankPipeline,
+  rerankPipelineSync,
+  computeRankDelta,
+  scorePair,
+  defaultTextOf,
+};

package/scripts/rerank-quality-eval.js

@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+'use strict';
+
+const path = require('node:path');
+
+/**
+ * Golden-set regression eval for the rerank pipeline.
+ *
+ * Measures:
+ *   - Precision@1 / MRR on planted force-push / secret / deploy cases
+ *   - Rank-delta rate (how often #1 flips vs first-stage order)
+ *   - Stage presence (BM25, MaxSim, heuristic CE)
+ *
+ * Exit 0 when bounded deterministic floors are met. Provider holdouts and live
+ * traces are separate A+ requirements.
+ */
+
+const { rerankPipelineSync, PIPELINE_VERSION } = require('./rerank-pipeline');
+
+const GOLDEN = [
+  {
+    id: 'force-push',
+    query: 'git push --force to main',
+    toolName: 'Bash',
+    // first-stage order intentionally wrong (decoy first)
+    candidates: [
+      { id: 'decoy-deploy', title: 'Friday deploys', content: 'Ship every Friday', relevanceScore: 0.99 },
+      {
+        id: 'gold-force',
+        title: 'Force push blocked',
+        whatWentWrong: 'force push to main wiped history',
+        tags: ['git', 'force-push', 'negative'],
+        signal: 'negative',
+        relevanceScore: 0.2,
+        metadata: { toolsUsed: ['Bash'] },
+      },
+      { id: 'noise-weather', title: 'Weather', content: 'Paris is rainy', relevanceScore: 0.5 },
+    ],
+    relevantIds: ['gold-force'],
+  },
+  {
+    id: 'secrets-env',
+    query: 'commit .env with API keys',
+    toolName: 'Bash',
+    candidates: [
+      { id: 'decoy-readme', title: 'Update README', content: 'docs only', relevanceScore: 0.9 },
+      {
+        id: 'gold-secret',
+        title: 'Never commit secrets',
+        whatWentWrong: 'committed .env with api key token credential',
+        tags: ['secret', 'env', 'security'],
+        signal: 'negative',
+        relevanceScore: 0.25,
+      },
+    ],
+    relevantIds: ['gold-secret'],
+  },
+  {
+    id: 'rm-rf',
+    query: 'rm -rf production data',
+    toolName: 'Bash',
+    candidates: [
+      { id: 'decoy-test', title: 'Run unit tests', content: 'npm test passes', relevanceScore: 0.85 },
+      {
+        id: 'gold-rm',
+        title: 'Destructive delete blocked',
+        whatWentWrong: 'rm -rf wiped production database directory',
+        tags: ['delete', 'destructive'],
+        signal: 'negative',
+        relevanceScore: 0.3,
+      },
+    ],
+    relevantIds: ['gold-rm'],
+  },
+];
+
+function evaluate() {
+  let hitsAt1 = 0;
+  let mrr = 0;
+  let flips = 0;
+  const rows = [];
+
+  for (const caseRow of GOLDEN) {
+    const firstStageTop = caseRow.candidates[0]?.id;
+    const { results, meta } = rerankPipelineSync(caseRow.query, caseRow.candidates, {
+      topK: 3,
+      toolName: caseRow.toolName,
+    });
+    const topId = results[0]?.id;
+    const relevant = new Set(caseRow.relevantIds);
+    const hit1 = relevant.has(topId);
+    if (hit1) hitsAt1 += 1;
+
+    let rr = 0;
+    for (let i = 0; i < results.length; i += 1) {
+      if (relevant.has(results[i].id)) {
+        rr = 1 / (i + 1);
+        break;
+      }
+    }
+    mrr += rr;
+    if (meta.rankDelta?.flipped || (firstStageTop && topId && firstStageTop !== topId)) flips += 1;
+
+    rows.push({
+      id: caseRow.id,
+      topId,
+      hitAt1: hit1,
+      mrr: rr,
+      stages: meta.stages,
+      flipped: Boolean(meta.rankDelta?.flipped || (firstStageTop !== topId)),
+    });
+  }
+
+  const n = GOLDEN.length;
+  const report = {
+    pipelineVersion: PIPELINE_VERSION,
+    cases: n,
+    precisionAt1: hitsAt1 / n,
+    mrr: mrr / n,
+    rankDeltaRate: flips / n,
+    floors: {
+      precisionAt1: 1.0,
+      mrr: 1.0,
+      rankDeltaRateMin: 0.5,
+    },
+    rows,
+  };
+
+  report.pass =
+    report.precisionAt1 >= report.floors.precisionAt1 &&
+    report.mrr >= report.floors.mrr &&
+    report.rankDeltaRate >= report.floors.rankDeltaRateMin;
+
+  return report;
+}
+
+function main() {
+  const report = evaluate();
+  console.log(JSON.stringify(report, null, 2));
+  if (!report.pass) {
+    console.error('rerank-quality-eval: FAILED floors');
+    process.exit(1);
+  }
+  console.error('rerank-quality-eval: PASS (bounded golden floors met)');
+}
+
+function isCliEntrypoint(argv = process.argv) {
+  return Boolean(argv[1]) && path.resolve(argv[1]) === path.resolve(__filename);
+}
+
+if (isCliEntrypoint()) {
+  main();
+}
+
+module.exports = { evaluate, GOLDEN, isCliEntrypoint };

package/scripts/retrieval-hybrid-ablation.js

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const {
+  retrieveRelevantLessons,
+  retrieveRelevantLessonsAsync,
+} = require('./lesson-retrieval');
+const { scoreRanking, aggregateRankingScores } = require('./ir-metrics');
+
+const DEFAULT_FIXTURE_PATH = path.join(
+  __dirname,
+  '..',
+  'config',
+  'evals',
+  'retrieval-hybrid-ablation.json',
+);
+
+function createFixtureEmbedder() {
+  const concepts = [
+    /(erase|directory|tree|rm|delete|folder|snapshot)/i,
+    /(overwrite|remote|history|force|push|clobber|trunk|repository)/i,
+    /(replay|purchase|twice|duplicate|payment|idempotency|charge)/i,
+    /(phone|expired|conversation|session|identifier|mobile|resume)/i,
+    /(dashboard|color|design|palette|visual|ui)/i,
+  ];
+  return async (text) => {
+    const vector = concepts.map((pattern) => (pattern.test(String(text || '')) ? 1 : 0));
+    if (vector.every((value) => value === 0)) vector.push(0.001);
+    else vector.push(0);
+    return vector;
+  };
+}
+
+async function evaluateHybridAblation(options = {}) {
+  const fixturePath = options.fixturePath || DEFAULT_FIXTURE_PATH;
+  const fixture = options.fixture || JSON.parse(fs.readFileSync(fixturePath, 'utf8'));
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'thumbgate-hybrid-ablation-'));
+  try {
+    fs.writeFileSync(
+      path.join(tempDir, 'memory-log.jsonl'),
+      `${fixture.corpus.map((row) => JSON.stringify({
+        ...row,
+        timestamp: row.timestamp || new Date().toISOString(),
+      })).join('\n')}\n`,
+    );
+    const kValues = fixture.kValues || [1, 3];
+    const embedder = options.embedder || createFixtureEmbedder();
+    const lexicalRows = [];
+    const hybridRows = [];
+    for (const queryCase of fixture.queries) {
+      const lexical = retrieveRelevantLessons(queryCase.toolName, queryCase.query, {
+        feedbackDir: tempDir,
+        maxResults: Math.max(...kValues),
+        pragmatic: true,
+      });
+      const hybrid = await retrieveRelevantLessonsAsync(queryCase.toolName, queryCase.query, {
+        feedbackDir: tempDir,
+        maxResults: Math.max(...kValues),
+        embedder,
+        embedderId: options.embedderId || 'deterministic-semantic-fixture',
+        includeRetrievalMeta: true,
+      });
+      lexicalRows.push({
+        id: queryCase.id,
+        rankedIds: lexical.map((row) => row.id),
+        metrics: scoreRanking(lexical, queryCase.qrels, { kValues }),
+      });
+      hybridRows.push({
+        id: queryCase.id,
+        rankedIds: hybrid.map((row) => row.id),
+        retrieval: hybrid[0]?.retrieval || null,
+        metrics: scoreRanking(hybrid, queryCase.qrels, { kValues }),
+      });
+    }
+    const lexical = aggregateRankingScores(lexicalRows.map((row) => row.metrics), { kValues });
+    const hybrid = aggregateRankingScores(hybridRows.map((row) => row.metrics), { kValues });
+    const summary = {
+      mode: options.embedderId || 'deterministic-semantic-fixture',
+      queries: fixture.queries.length,
+      lexical,
+      hybrid,
+      lift: {
+        mrr: hybrid.mrr - lexical.mrr,
+        recallAt3: hybrid['recall@3'] - lexical['recall@3'],
+        ndcgAt3: hybrid['ndcg@3'] - lexical['ndcg@3'],
+      },
+    };
+    return {
+      passed: hybrid['recall@3'] >= 0.9
+        && hybrid.mrr >= lexical.mrr
+        && hybrid['ndcg@3'] >= lexical['ndcg@3']
+        && hybridRows.every((row) => row.retrieval?.densePool > 0),
+      fixturePath,
+      summary,
+      lexicalRows,
+      hybridRows,
+    };
+  } finally {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+}
+
+if (!module.parent) {
+  evaluateHybridAblation().then((result) => {
+    console.log(JSON.stringify(result, null, 2));
+    if (!result.passed) process.exitCode = 1;
+  }).catch((error) => {
+    console.error(error.message);
+    process.exitCode = 1;
+  });
+}
+
+module.exports = {
+  DEFAULT_FIXTURE_PATH,
+  createFixtureEmbedder,
+  evaluateHybridAblation,
+};