thumbgate 1.29.2 → 1.31.0

package/scripts/request-envelope.js

@@ -0,0 +1,178 @@
+'use strict';
+
+/**
+ * Production request envelope — one schema for LLM + retrieval observability.
+ *
+ * Every dashboard chat / routed generation path should create an envelope at
+ * start and finalize it before return so latency, cost, retrieval, and
+ * structured-output status are greppable from one object.
+ *
+ * Privacy: never store full prompts/tool payloads here; use redacted previews only.
+ */
+
+const crypto = require('node:crypto');
+const {
+  buildAgentAuditSpan,
+  evaluateAgentAuditTrace,
+} = require('./agent-audit-trace');
+
+const ENVELOPE_VERSION = '2026-07-31.p0.1';
+
+function newTraceId() {
+  if (typeof crypto.randomUUID === 'function') return crypto.randomUUID();
+  return `tr_${Date.now().toString(36)}_${crypto.randomBytes(6).toString('hex')}`;
+}
+
+function hashSensitiveText(text) {
+  return crypto.createHash('sha256').update(String(text || '')).digest('hex');
+}
+
+/**
+ * @param {object} [seed]
+ * @returns {object}
+ */
+function createRequestEnvelope(seed = {}) {
+  const startedAt = Number.isFinite(seed.startedAt) ? seed.startedAt : Date.now();
+  const traceId = seed.traceId || newTraceId();
+  const auditTrace = seed.promptHash
+    ? {
+      runId: traceId,
+      spans: [buildAgentAuditSpan({
+        runId: traceId,
+        spanId: `${traceId}:input`,
+        stage: 'input',
+        promptHash: seed.promptHash,
+        model: seed.model || null,
+      })],
+    }
+    : null;
+  return {
+    envelopeVersion: ENVELOPE_VERSION,
+    traceId,
+    startedAt,
+    endedAt: null,
+    latencyMs: null,
+    surface: seed.surface || 'unknown',
+    model: seed.model || null,
+    tier: seed.tier || null,
+    provider: seed.provider || null,
+    inputTokens: seed.inputTokens ?? null,
+    outputTokens: seed.outputTokens ?? null,
+    estimatedCostCents: seed.estimatedCostCents ?? null,
+    budget: seed.budget || null,
+    retrieval: seed.retrieval || null,
+    structured: seed.structured || null,
+    qualityTier: seed.qualityTier || null,
+    outcome: seed.outcome || 'pending',
+    error: seed.error || null,
+    auditTrace,
+  };
+}
+
+/**
+ * Finalize timing + optional fields. Pure-ish: returns a new object.
+ * @param {object} envelope
+ * @param {object} [patch]
+ */
+function finalizeRequestEnvelope(envelope, patch = {}) {
+  const endedAt = Number.isFinite(patch.endedAt) ? patch.endedAt : Date.now();
+  const startedAt = Number(envelope?.startedAt) || endedAt;
+  const finalized = {
+    ...envelope,
+    ...patch,
+    endedAt,
+    latencyMs: Math.max(0, endedAt - startedAt),
+    outcome: patch.outcome || envelope?.outcome || 'ok',
+  };
+
+  if (envelope?.auditTrace?.runId && Array.isArray(envelope.auditTrace.spans)) {
+    const evidenceIds = (patch.retrieval?.top || [])
+      .map((row) => row?.id)
+      .filter(Boolean);
+    const priorSpans = envelope.auditTrace.spans
+      .filter((span) => span?.stage !== 'decision');
+    const decisionSpan = buildAgentAuditSpan({
+      runId: envelope.auditTrace.runId,
+      spanId: `${envelope.auditTrace.runId}:decision`,
+      parentSpanId: priorSpans[0]?.spanId || null,
+      stage: 'decision',
+      model: finalized.model,
+      decision: finalized.outcome,
+      dataAccessed: evidenceIds.length ? ['retrieved_lessons'] : [],
+      evidenceIds,
+      safetyEvents: finalized.error ? [finalized.error] : [],
+      inputTokens: finalized.inputTokens,
+      outputTokens: finalized.outputTokens,
+      latencyMs: finalized.latencyMs,
+    });
+    const auditTrace = {
+      runId: envelope.auditTrace.runId,
+      spans: [...priorSpans, decisionSpan],
+    };
+    finalized.auditTrace = {
+      ...auditTrace,
+      evaluation: evaluateAgentAuditTrace(auditTrace),
+    };
+  }
+
+  return finalized;
+}
+
+/**
+ * Compact retrieval summary for the envelope (no lesson bodies).
+ * @param {Array<object>} rows
+ * @param {object} [meta]
+ */
+function summarizeRetrieval(rows = [], meta = {}) {
+  const top = (rows || []).slice(0, 8).map((r, i) => {
+    const rawScore = r.rerankedScore ?? r.relevanceScore ?? r.score;
+    const numericScore = Number(rawScore);
+    return {
+      rank: i + 1,
+      id: r.id || r.memoryId || null,
+      score: rawScore == null || !Number.isFinite(numericScore) ? null : numericScore,
+      signal: r.signal || null,
+    };
+  });
+  return {
+    strategy: meta.strategy || meta.retrievalStrategy || null,
+    count: Array.isArray(rows) ? rows.length : 0,
+    qualityTier: meta.qualityTier || null,
+    degradedReasons: meta.degradedReasons || [],
+    top,
+  };
+}
+
+/**
+ * Estimate tokens from text length (rough, offline-safe).
+ * @param {string} text
+ */
+function estimateTokensFromText(text) {
+  const s = String(text || '');
+  if (!s) return 0;
+  return Math.max(1, Math.ceil(s.length / 4));
+}
+
+/**
+ * Rough USD cents from token counts using Sonnet-ish defaults (conservative).
+ * @param {{ inputTokens?: number, outputTokens?: number, inputPerM?: number, outputPerM?: number }} opts
+ */
+function estimateCostCents(opts = {}) {
+  const input = Number(opts.inputTokens) || 0;
+  const output = Number(opts.outputTokens) || 0;
+  const inputPerM = Number(opts.inputPerM) || 3;
+  const outputPerM = Number(opts.outputPerM) || 15;
+  const usd = (input / 1e6) * inputPerM + (output / 1e6) * outputPerM;
+  return Number((usd * 100).toFixed(4));
+}
+
+module.exports = {
+  ENVELOPE_VERSION,
+  newTraceId,
+  hashSensitiveText,
+  createRequestEnvelope,
+  finalizeRequestEnvelope,
+  summarizeRetrieval,
+  estimateTokensFromText,
+  estimateCostCents,
+};

package/scripts/rerank-pipeline.js

@@ -0,0 +1,370 @@
+'use strict';
+
+/**
+ * Multi-stage rerank pipeline for ThumbGate's A+ target.
+ *
+ * Stages (always local-first; LLM optional and last):
+ *   1) Field-weighted BM25F pair scoring (lesson-reranker)
+ *   2) ColBERT-style MaxSim late interaction (colbert-style-maxsim)
+ *   3) Heuristic joint pair scorer (cross-encoder-reranker.heuristicCrossEncode)
+ *   4) Optional listwise LLM rerank on the final shortlist (useLLM / env)
+ *
+ * Honesty contract:
+ *   - Stage 3 is a *heuristic* cross-encoder, not a neural CE checkpoint
+ *   - Stage 2 is ColBERT-*style* MaxSim over hashed multi-vectors unless
+ *     a tokenEmbedder is supplied
+ *   - LLM stage is off by default; enable with useLLM:true or THUMBGATE_RERANK_LLM=1
+ *
+ * Pipeline version is exported so evals and statuslines can pin provenance.
+ */
+
+const PIPELINE_VERSION = '2026-07-31.a-plus.1';
+
+const { rerankLessons } = require('./lesson-reranker');
+const { rerankWithMaxSim, scoreLateInteraction } = require('./colbert-style-maxsim');
+const { heuristicCrossEncode, llmCrossEncode } = require('./cross-encoder-reranker');
+
+/**
+ * @typedef {object} RerankPipelineOptions
+ * @property {number} [topK=5]
+ * @property {string} [toolName]
+ * @property {boolean} [useLLM=false]
+ * @property {boolean} [useMaxSim=true]
+ * @property {boolean} [useHeuristicCe=true]
+ * @property {number} [bm25Pool=50] candidates to keep after BM25 before MaxSim
+ * @property {number} [llmShortlist=8] max docs sent to LLM listwise scorer
+ * @property {number} [wBm25=0.30]
+ * @property {number} [wMaxSim=0.35]
+ * @property {number} [wHeuristic=0.25]
+ * @property {number} [wOriginal=0.10]
+ * @property {(c: object) => string} [textOf]
+ */
+
+function defaultTextOf(c) {
+  if (!c || typeof c !== 'object') return String(c || '');
+  return [
+    c.title,
+    c.whatWentWrong,
+    c.whatToChange,
+    c.howToAvoid,
+    c.summary,
+    c.content,
+    c.context,
+    Array.isArray(c.tags) ? c.tags.join(' ') : '',
+  ].filter(Boolean).join(' ');
+}
+
+function envFlag(name) {
+  const v = String(process.env[name] || '').trim().toLowerCase();
+  return v === '1' || v === 'true' || v === 'yes' || v === 'on';
+}
+
+/**
+ * Fuse multi-stage scores into a final ranking.
+ * @param {string} query
+ * @param {Array<object>} candidates
+ * @param {RerankPipelineOptions} [options]
+ * @returns {Promise<{ results: Array<object>, meta: object }>}
+ */
+async function rerankPipeline(query, candidates, options = {}) {
+  const topK = options.topK ?? 5;
+  const toolName = options.toolName || '';
+  const useMaxSim = options.useMaxSim !== false;
+  const useHeuristicCe = options.useHeuristicCe !== false;
+  const useLLM = options.useLLM === true || envFlag('THUMBGATE_RERANK_LLM');
+  const bm25Pool = Math.max(topK, options.bm25Pool ?? 50);
+  const llmShortlist = Math.max(topK, options.llmShortlist ?? 8);
+  const textOf = options.textOf || defaultTextOf;
+  const wBm25 = options.wBm25 ?? 0.30;
+  const wMaxSim = options.wMaxSim ?? 0.35;
+  const wHeuristic = options.wHeuristic ?? 0.25;
+  const wOriginal = options.wOriginal ?? 0.10;
+
+  const meta = {
+    pipelineVersion: PIPELINE_VERSION,
+    stages: [],
+    useLLM,
+    useMaxSim,
+    useHeuristicCe,
+    inputCount: candidates?.length || 0,
+  };
+
+  if (!candidates || candidates.length === 0) {
+    return { results: [], meta: { ...meta, stages: ['empty'] } };
+  }
+
+  // --- Stage 1: BM25F ---
+  let pool = rerankLessons(query, candidates, {
+    topK: Math.min(bm25Pool, candidates.length),
+    toolName,
+    blendWeight: 0.7,
+  });
+  meta.stages.push('bm25f');
+
+  // Capture BM25 scores before MaxSim overwrites rerankedScore
+  pool = pool.map((c) => ({
+    ...c,
+    bm25Score: Number(c.rerankedScore ?? 0),
+    originalScore: Number(c.relevanceScore ?? c.score ?? 0),
+  }));
+
+  // --- Stage 2: ColBERT-style MaxSim ---
+  if (useMaxSim && pool.length > 1) {
+    pool = rerankWithMaxSim(query, pool, {
+      topK: pool.length,
+      textOf,
+      blendWeight: 1, // pure MaxSim into maxSimScore; we fuse ourselves
+      dim: options.dim,
+      ngram: options.ngram,
+      maxTokens: options.maxTokens,
+      tokenEmbedder: options.tokenEmbedder,
+    }).map((c) => ({
+      ...c,
+      // restore bm25 from previous map (rerankWithMaxSim spreads candidate)
+      bm25Score: c.bm25Score,
+      maxSimScore: Number(c.maxSimScore ?? 0),
+    }));
+    meta.stages.push('colbert-style-maxsim');
+  } else {
+    pool = pool.map((c) => ({ ...c, maxSimScore: 0 }));
+  }
+
+  // --- Stage 3: Heuristic joint pair scorer ---
+  if (useHeuristicCe) {
+    pool = pool.map((c) => {
+      const he = heuristicCrossEncode(
+        `${toolName} ${query}`.trim(),
+        textOf(c),
+      );
+      return { ...c, heuristicCeScore: he };
+    });
+    meta.stages.push('heuristic-pair-ce');
+  } else {
+    pool = pool.map((c) => ({ ...c, heuristicCeScore: 0 }));
+  }
+
+  // --- Fuse ---
+  // Normalize bm25 within pool for fair blend
+  const maxBm25 = Math.max(...pool.map((c) => c.bm25Score || 0), 1e-9);
+  pool = pool.map((c) => {
+    const nBm25 = (c.bm25Score || 0) / maxBm25;
+    const nOrig = Math.max(0, Math.min(1, c.originalScore || 0));
+    const nMs = Math.max(0, Math.min(1, c.maxSimScore || 0));
+    const nHe = Math.max(0, Math.min(1, c.heuristicCeScore || 0));
+    const fused =
+      wBm25 * nBm25 +
+      wMaxSim * nMs +
+      wHeuristic * nHe +
+      wOriginal * nOrig;
+    return {
+      ...c,
+      fusedScore: Number(fused.toFixed(6)),
+      rerankedScore: Number(fused.toFixed(6)),
+      pairwiseHeuristicScore: nHe,
+      lateInteractionScore: nMs,
+      crossEncoderScore: null,
+      combinedScore: Number(fused.toFixed(6)),
+      reranker: {
+        stages: [
+          'first-stage',
+          'bm25f',
+          'colbert-style-maxsim-hashed',
+          'pairwise-heuristic',
+          'score-fusion',
+        ],
+        fallbacks: ['neural-cross-encoder-not-configured'],
+      },
+    };
+  }).sort((a, b) => b.fusedScore - a.fusedScore);
+
+  meta.stages.push('score-fusion');
+
+  // --- Stage 4: optional LLM listwise on shortlist ---
+  let shortlist = pool.slice(0, Math.min(llmShortlist, pool.length));
+  if (useLLM && shortlist.length > 1) {
+    const llmScores = await llmCrossEncode(
+      `${toolName} ${query}`.trim(),
+      shortlist.map((c) => ({
+        title: c.title || '',
+        content: textOf(c).slice(0, 400),
+      })),
+    );
+    if (llmScores) {
+      shortlist = shortlist.map((c, i) => {
+        const llm = Math.max(0, Math.min(1, Number(llmScores[i]) || 0));
+        // Blend LLM lightly so a bad model cannot erase local signal
+        const final = 0.55 * llm + 0.45 * (c.fusedScore || 0);
+        return {
+          ...c,
+          llmRerankScore: llm,
+          fusedScore: Number(final.toFixed(6)),
+          rerankedScore: Number(final.toFixed(6)),
+          combinedScore: Number(final.toFixed(6)),
+        };
+      }).sort((a, b) => b.fusedScore - a.fusedScore);
+      meta.stages.push('llm-listwise');
+      meta.llmApplied = true;
+    } else {
+      meta.llmApplied = false;
+      meta.stages.push('llm-fallback');
+    }
+  } else {
+    meta.llmApplied = false;
+  }
+
+  const results = shortlist.slice(0, topK).map((c) => ({
+    ...c,
+    rerankPipelineVersion: PIPELINE_VERSION,
+  }));
+
+  meta.outputCount = results.length;
+  meta.rankDelta = computeRankDelta(candidates, results);
+
+  return { results, meta };
+}
+
+/**
+ * Sync path for PreToolUse hooks (no LLM).
+ */
+function rerankPipelineSync(query, candidates, options = {}) {
+  return rerankPipelineSyncImpl(query, candidates, options);
+}
+
+function rerankPipelineSyncImpl(query, candidates, options = {}) {
+  const topK = options.topK ?? 5;
+  const toolName = options.toolName || '';
+  const useMaxSim = options.useMaxSim !== false;
+  const useHeuristicCe = options.useHeuristicCe !== false;
+  const bm25Pool = Math.max(topK, options.bm25Pool ?? 50);
+  const textOf = options.textOf || defaultTextOf;
+  const wBm25 = options.wBm25 ?? 0.30;
+  const wMaxSim = options.wMaxSim ?? 0.35;
+  const wHeuristic = options.wHeuristic ?? 0.25;
+  const wOriginal = options.wOriginal ?? 0.10;
+
+  const meta = {
+    pipelineVersion: PIPELINE_VERSION,
+    stages: [],
+    useLLM: false,
+    useMaxSim,
+    useHeuristicCe,
+    inputCount: candidates?.length || 0,
+    llmApplied: false,
+  };
+
+  if (!candidates || candidates.length === 0) {
+    return { results: [], meta: { ...meta, stages: ['empty'] } };
+  }
+
+  let pool = rerankLessons(query, candidates, {
+    topK: Math.min(bm25Pool, candidates.length),
+    toolName,
+    blendWeight: 0.7,
+  }).map((c) => ({
+    ...c,
+    bm25Score: Number(c.rerankedScore ?? 0),
+    originalScore: Number(c.relevanceScore ?? c.score ?? 0),
+  }));
+  meta.stages.push('bm25f');
+
+  if (useMaxSim && pool.length > 1) {
+    pool = rerankWithMaxSim(query, pool, {
+      topK: pool.length,
+      textOf,
+      blendWeight: 1,
+    }).map((c) => ({
+      ...c,
+      bm25Score: c.bm25Score,
+      maxSimScore: Number(c.maxSimScore ?? 0),
+    }));
+    meta.stages.push('colbert-style-maxsim');
+  } else {
+    pool = pool.map((c) => ({ ...c, maxSimScore: 0 }));
+  }
+
+  if (useHeuristicCe) {
+    pool = pool.map((c) => ({
+      ...c,
+      heuristicCeScore: heuristicCrossEncode(`${toolName} ${query}`.trim(), textOf(c)),
+    }));
+    meta.stages.push('heuristic-pair-ce');
+  } else {
+    pool = pool.map((c) => ({ ...c, heuristicCeScore: 0 }));
+  }
+
+  const maxBm25 = Math.max(...pool.map((c) => c.bm25Score || 0), 1e-9);
+  pool = pool.map((c) => {
+    const nBm25 = (c.bm25Score || 0) / maxBm25;
+    const nOrig = Math.max(0, Math.min(1, c.originalScore || 0));
+    const nMs = Math.max(0, Math.min(1, c.maxSimScore || 0));
+    const nHe = Math.max(0, Math.min(1, c.heuristicCeScore || 0));
+    const fused =
+      wBm25 * nBm25 + wMaxSim * nMs + wHeuristic * nHe + wOriginal * nOrig;
+    return {
+      ...c,
+      fusedScore: Number(fused.toFixed(6)),
+      rerankedScore: Number(fused.toFixed(6)),
+      pairwiseHeuristicScore: nHe,
+      lateInteractionScore: nMs,
+      crossEncoderScore: null,
+      combinedScore: Number(fused.toFixed(6)),
+      rerankPipelineVersion: PIPELINE_VERSION,
+      reranker: {
+        stages: [
+          'first-stage',
+          'bm25f',
+          'colbert-style-maxsim-hashed',
+          'pairwise-heuristic',
+          'score-fusion',
+        ],
+        fallbacks: ['neural-cross-encoder-not-configured'],
+      },
+    };
+  }).sort((a, b) => b.fusedScore - a.fusedScore);
+
+  meta.stages.push('score-fusion');
+  const results = pool.slice(0, topK);
+  meta.outputCount = results.length;
+  meta.rankDelta = computeRankDelta(candidates, results);
+  return { results, meta };
+}
+
+/**
+ * Whether top-1 id changed vs original order (rank-delta signal).
+ * @param {Array<object>} original
+ * @param {Array<object>} reranked
+ * @returns {{ flipped: boolean, originalTopId: string|null, rerankedTopId: string|null }}
+ */
+function computeRankDelta(original, reranked) {
+  const idOf = (c) => c?.id || c?.lessonId || c?.title || null;
+  const originalTopId = original?.[0] ? idOf(original[0]) : null;
+  const rerankedTopId = reranked?.[0] ? idOf(reranked[0]) : null;
+  return {
+    flipped: Boolean(originalTopId && rerankedTopId && originalTopId !== rerankedTopId),
+    originalTopId,
+    rerankedTopId,
+  };
+}
+
+/**
+ * Pair-level scores for diagnostics / evals.
+ */
+function scorePair(query, document, opts = {}) {
+  const late = scoreLateInteraction(query, document, opts);
+  const he = heuristicCrossEncode(query, document);
+  return {
+    maxSim: late.score,
+    heuristicCe: he,
+    mode: late.mode,
+    pipelineVersion: PIPELINE_VERSION,
+  };
+}
+
+module.exports = {
+  PIPELINE_VERSION,
+  rerankPipeline,
+  rerankPipelineSync,
+  computeRankDelta,
+  scorePair,
+  defaultTextOf,
+};

package/scripts/rerank-quality-eval.js

@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+'use strict';
+
+const path = require('node:path');
+
+/**
+ * Golden-set regression eval for the rerank pipeline.
+ *
+ * Measures:
+ *   - Precision@1 / MRR on planted force-push / secret / deploy cases
+ *   - Rank-delta rate (how often #1 flips vs first-stage order)
+ *   - Stage presence (BM25, MaxSim, heuristic CE)
+ *
+ * Exit 0 when bounded deterministic floors are met. Provider holdouts and live
+ * traces are separate A+ requirements.
+ */
+
+const { rerankPipelineSync, PIPELINE_VERSION } = require('./rerank-pipeline');
+
+const GOLDEN = [
+  {
+    id: 'force-push',
+    query: 'git push --force to main',
+    toolName: 'Bash',
+    // first-stage order intentionally wrong (decoy first)
+    candidates: [
+      { id: 'decoy-deploy', title: 'Friday deploys', content: 'Ship every Friday', relevanceScore: 0.99 },
+      {
+        id: 'gold-force',
+        title: 'Force push blocked',
+        whatWentWrong: 'force push to main wiped history',
+        tags: ['git', 'force-push', 'negative'],
+        signal: 'negative',
+        relevanceScore: 0.2,
+        metadata: { toolsUsed: ['Bash'] },
+      },
+      { id: 'noise-weather', title: 'Weather', content: 'Paris is rainy', relevanceScore: 0.5 },
+    ],
+    relevantIds: ['gold-force'],
+  },
+  {
+    id: 'secrets-env',
+    query: 'commit .env with API keys',
+    toolName: 'Bash',
+    candidates: [
+      { id: 'decoy-readme', title: 'Update README', content: 'docs only', relevanceScore: 0.9 },
+      {
+        id: 'gold-secret',
+        title: 'Never commit secrets',
+        whatWentWrong: 'committed .env with api key token credential',
+        tags: ['secret', 'env', 'security'],
+        signal: 'negative',
+        relevanceScore: 0.25,
+      },
+    ],
+    relevantIds: ['gold-secret'],
+  },
+  {
+    id: 'rm-rf',
+    query: 'rm -rf production data',
+    toolName: 'Bash',
+    candidates: [
+      { id: 'decoy-test', title: 'Run unit tests', content: 'npm test passes', relevanceScore: 0.85 },
+      {
+        id: 'gold-rm',
+        title: 'Destructive delete blocked',
+        whatWentWrong: 'rm -rf wiped production database directory',
+        tags: ['delete', 'destructive'],
+        signal: 'negative',
+        relevanceScore: 0.3,
+      },
+    ],
+    relevantIds: ['gold-rm'],
+  },
+];
+
+function evaluate() {
+  let hitsAt1 = 0;
+  let mrr = 0;
+  let flips = 0;
+  const rows = [];
+
+  for (const caseRow of GOLDEN) {
+    const firstStageTop = caseRow.candidates[0]?.id;
+    const { results, meta } = rerankPipelineSync(caseRow.query, caseRow.candidates, {
+      topK: 3,
+      toolName: caseRow.toolName,
+    });
+    const topId = results[0]?.id;
+    const relevant = new Set(caseRow.relevantIds);
+    const hit1 = relevant.has(topId);
+    if (hit1) hitsAt1 += 1;
+
+    let rr = 0;
+    for (let i = 0; i < results.length; i += 1) {
+      if (relevant.has(results[i].id)) {
+        rr = 1 / (i + 1);
+        break;
+      }
+    }
+    mrr += rr;
+    if (meta.rankDelta?.flipped || (firstStageTop && topId && firstStageTop !== topId)) flips += 1;
+
+    rows.push({
+      id: caseRow.id,
+      topId,
+      hitAt1: hit1,
+      mrr: rr,
+      stages: meta.stages,
+      flipped: Boolean(meta.rankDelta?.flipped || (firstStageTop !== topId)),
+    });
+  }
+
+  const n = GOLDEN.length;
+  const report = {
+    pipelineVersion: PIPELINE_VERSION,
+    cases: n,
+    precisionAt1: hitsAt1 / n,
+    mrr: mrr / n,
+    rankDeltaRate: flips / n,
+    floors: {
+      precisionAt1: 1.0,
+      mrr: 1.0,
+      rankDeltaRateMin: 0.5,
+    },
+    rows,
+  };
+
+  report.pass =
+    report.precisionAt1 >= report.floors.precisionAt1 &&
+    report.mrr >= report.floors.mrr &&
+    report.rankDeltaRate >= report.floors.rankDeltaRateMin;
+
+  return report;
+}
+
+function main() {
+  const report = evaluate();
+  console.log(JSON.stringify(report, null, 2));
+  if (!report.pass) {
+    console.error('rerank-quality-eval: FAILED floors');
+    process.exit(1);
+  }
+  console.error('rerank-quality-eval: PASS (bounded golden floors met)');
+}
+
+function isCliEntrypoint(argv = process.argv) {
+  return Boolean(argv[1]) && path.resolve(argv[1]) === path.resolve(__filename);
+}
+
+if (isCliEntrypoint()) {
+  main();
+}
+
+module.exports = { evaluate, GOLDEN, isCliEntrypoint };