thumbgate 1.29.2 → 1.31.0

package/scripts/a-plus-evidence-scorecard.js

@@ -0,0 +1,303 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Fail-closed A+ readiness scorecard.
+ *
+ * A passing unit test or a checked-in module is repository evidence, not live
+ * production or commercial proof. This scorecard keeps those surfaces separate
+ * and awards A+/10 only when every check in every area is verified.
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const ROOT = path.join(__dirname, '..');
+const SCORECARD_VERSION = '2026-08-01.1';
+
+function read(root, relativePath) {
+  try {
+    return fs.readFileSync(path.join(root, relativePath), 'utf8');
+  } catch {
+    return '';
+  }
+}
+
+function hasAll(text, needles) {
+  return needles.every((needle) => text.includes(needle));
+}
+
+function safeEval(fn) {
+  try {
+    return fn() === true;
+  } catch {
+    return false;
+  }
+}
+
+function collectRepositoryEvidence(root = ROOT) {
+  const landing = read(root, 'public/index.html');
+  const feedback = read(root, 'scripts/feedback-loop.js');
+  const promotion = read(root, 'scripts/auto-promote-gates.js');
+  const buyerPaths = read(root, 'scripts/buyer-paths.js');
+  const hook = read(root, 'scripts/hook-pre-tool-use.js');
+  const gatesEngine = read(root, 'scripts/gates-engine.js');
+  const gateEvasionMatrix = read(root, 'tests/gate-evasion-matrix.test.js');
+  const retrieval = read(root, 'scripts/lesson-retrieval.js');
+  const crossEncoder = read(root, 'scripts/cross-encoder-reranker.js');
+  const productionArchitecture = read(root, 'docs/RAG_PRODUCTION_ARCHITECTURE.md');
+  const packageJson = read(root, 'package.json');
+
+  let qualitySuitePassed = false;
+  let rerankGoldenPassed = false;
+  if (root === ROOT) {
+    qualitySuitePassed = safeEval(() => require('./eval-quality-suite').runSuite().report.passed);
+    rerankGoldenPassed = safeEval(() => require('./rerank-quality-eval').evaluate().pass);
+  }
+
+  return {
+    landingVisualLoop: hasAll(landing, [
+      'hero-thumbs',
+      'before-after.svg',
+      'self-improving-thumbs-loop.svg',
+      'Is it really self-improving?',
+    ]),
+    landingBuyerRoutes: hasAll(landing, ['/checkout/pro', '/go/diagnostic-pay'])
+      && hasAll(buyerPaths, ['/go/pro', '/go/sprint', '/diagnostic']),
+    feedbackRewardReachable: feedback.includes('scoreFeedbackReward('),
+    feedbackPromotionReachable: promotion.includes('promote') && hook.includes('retrieveWithRerankingSync'),
+    preventionChangePromoted: feedback.includes('whatToChange')
+      && feedback.includes('promotion'),
+    architectureNamesHonest: hasAll(productionArchitecture, [
+      'LLM-as-a-judge output is diagnostic',
+      'A heuristic score is never',
+      'does not get to override a hard gate',
+    ]),
+    deterministicMultiQuery: retrieval.includes('buildQueryVariants'),
+    hydeExplicitAndBounded: hasAll(retrieval, ['hydeGenerator', 'hydeApplied', 'hydeProvider']),
+    rerankProductionWired: crossEncoder.includes("require('./rerank-pipeline')")
+      && crossEncoder.includes('rerankPipelineSync(query, candidates'),
+    rerankProvenance: hasAll(crossEncoder, [
+      'pairwiseHeuristicScore',
+      'crossEncoderScore',
+      'reranker',
+    ]),
+    rerankGoldenPassed,
+    qualitySuitePassed,
+    requestEnvelope: fs.existsSync(path.join(root, 'scripts/request-envelope.js')),
+    hardBudgets: fs.existsSync(path.join(root, 'scripts/tier-budget-guard.js')),
+    degradedRetrievalFlags: fs.existsSync(path.join(root, 'scripts/retrieval-quality-tier.js')),
+    structuredOutputValidation: fs.existsSync(path.join(root, 'scripts/rag-structured-output.js')),
+    tenantAclBeforeRetrieval: hasAll(productionArchitecture, [
+      'Filtering must happen before',
+      'Missing/mismatched tenant or principal',
+    ]),
+    commandPositionHardening: hasAll(gatesEngine, [
+      'LITERAL_COMMAND_SUBSTITUTION_HEADS',
+      'canonicalizeLiteralCommandSubstitutionHead',
+    ]) && gateEvasionMatrix.includes('literal command substitution'),
+    rawFrameworkDecisionDefended: hasAll(productionArchitecture, [
+      '## Framework decision',
+      'LangChain',
+      'LangGraph',
+      'LlamaIndex',
+      '## One complete RAG request',
+    ]),
+    scorecardInMainTest: packageJson.includes('test:a-plus-evidence'),
+  };
+}
+
+function check(id, label, passed, evidenceClass, remediation) {
+  return {
+    id,
+    label,
+    passed: passed === true,
+    evidenceClass,
+    remediation: passed === true ? null : remediation,
+  };
+}
+
+function shaMatches(live = {}) {
+  const candidate = String(live.candidateBuildSha || '').trim();
+  const deployed = String(live.deployedBuildSha || '').trim();
+  return candidate.length >= 7 && deployed.length >= 7 && candidate === deployed;
+}
+
+function evaluateReadiness({ repo = {}, live = {} } = {}) {
+  const production = live.production || {};
+  const retrieval = live.retrieval || {};
+  const security = live.security || {};
+  const commercial = live.commercial || {};
+
+  const areas = [
+    {
+      id: 'landing_conversion',
+      label: 'Landing page and conversion',
+      checks: [
+        check('visual_loop', 'Thumb visuals and simple learning diagrams ship', repo.landingVisualLoop, 'repository', 'Ship the visual thumbs-to-gate loop.'),
+        check('buyer_routes', 'First-party buyer routes are present', repo.landingBuyerRoutes, 'repository', 'Restore diagnostic, Pro, and sprint buyer routes.'),
+        check('live_landing', 'Candidate landing page is verified live', production.landingVerified === true && shaMatches(production), 'production', 'Verify the exact candidate SHA on the live landing page.'),
+      ],
+    },
+    {
+      id: 'self_improvement',
+      label: 'Self-improving control loop',
+      checks: [
+        check('reward_reachable', 'Feedback reward scoring is invoked by capture', repo.feedbackRewardReachable, 'repository', 'Wire reward scoring into the capture path.'),
+        check('promotion_reachable', 'Reviewed failures reach promotion and pre-action retrieval', repo.feedbackPromotionReachable, 'repository', 'Connect feedback promotion to the pre-action hook.'),
+        check('specific_change', 'Specific what-to-change guidance reaches prevention rules', repo.preventionChangePromoted, 'repository', 'Promote specific corrective instructions, not vague signals.'),
+        check('live_feedback', 'Fresh production feedback closes the loop', production.feedbackLoopVerified === true, 'production', 'Capture one real reviewed outcome and prove its next-action effect.'),
+      ],
+    },
+    {
+      id: 'architecture_honesty',
+      label: 'Judge, routing, and architecture honesty',
+      checks: [
+        check('honest_names', 'Judge, heuristic, neural, and enforcement stages are distinct', repo.architectureNamesHonest, 'repository', 'Document stage placement and prevent misleading model-level MoE claims.'),
+        check('route_trace', 'Live traces identify the provider and routed model', production.providerTraceVerified === true, 'production', 'Attach a secret-safe live route trace.'),
+      ],
+    },
+    {
+      id: 'query_transformation',
+      label: 'Query transformation, multi-query, and HyDE',
+      checks: [
+        check('multi_query', 'Bounded deterministic multi-query is implemented', repo.deterministicMultiQuery, 'repository', 'Implement bounded, inspectable query variants.'),
+        check('hyde_contract', 'HyDE is explicit, bounded, and provenance-bearing', repo.hydeExplicitAndBounded, 'repository', 'Add an explicit caller-supplied HyDE contract and fallback.'),
+        check('hyde_holdout', 'HyDE or multi-query improves a provider holdout', retrieval.queryTransformationHoldoutPassed === true, 'provider-holdout', 'Measure lift on a non-fixture provider holdout.'),
+      ],
+    },
+    {
+      id: 'reranking',
+      label: 'Reranking cascade',
+      checks: [
+        check('production_wiring', 'BM25F, local MaxSim, and pairwise fusion run in PreToolUse', repo.rerankProductionWired, 'repository', 'Wire the documented cascade into the production caller.'),
+        check('provenance', 'Heuristic and neural scores cannot masquerade as each other', repo.rerankProvenance, 'repository', 'Emit per-stage provenance and explicit fallbacks.'),
+        check('golden', 'Deterministic rerank golden floors pass', repo.rerankGoldenPassed, 'deterministic-eval', 'Fix rerank golden regressions.'),
+        check('neural_holdout', 'True neural pair/late-interaction holdout passes', retrieval.neuralRerankHoldoutPassed === true, 'provider-holdout', 'Run a pretrained pair scorer or token embedder on an external holdout.'),
+        check('llm_failures', 'LLM rerank failure modes pass live-provider tests', retrieval.llmRerankFailureModesPassed === true, 'provider-holdout', 'Test malformed, partial, injected, timed-out, and unavailable LLM reranks.'),
+      ],
+    },
+    {
+      id: 'evaluation',
+      label: 'Retrieval and answer evaluation',
+      checks: [
+        check('offline_suite', 'Recall, precision, MRR, nDCG, and answer proxy floors pass', repo.qualitySuitePassed, 'deterministic-eval', 'Fix the unified deterministic quality suite.'),
+        check('external_cases', 'External labeled holdout has at least 100 cases', Number(retrieval.externalHoldoutCases) >= 100, 'provider-holdout', 'Label and freeze at least 100 non-fixture cases.'),
+        check('judge_calibration', 'LLM judge is calibrated against human labels', retrieval.judgeCalibrationPassed === true, 'provider-holdout', 'Measure judge agreement and calibration against human-reviewed labels.'),
+      ],
+    },
+    {
+      id: 'production_controls',
+      label: 'Latency, cost, and observability',
+      checks: [
+        check('request_envelope', 'Request trace, token, cost, and retrieval envelope exists', repo.requestEnvelope, 'repository', 'Add a request envelope.'),
+        check('hard_budgets', 'Per-request and daily tier budgets fail closed', repo.hardBudgets, 'repository', 'Add hard cost and tier budgets.'),
+        check('degraded_flags', 'Stale or stub retrieval is labeled degraded', repo.degradedRetrievalFlags, 'repository', 'Expose retrieval quality tiers.'),
+        check('live_slo', 'Production p95 and cost SLOs pass under load', production.loadTestPassed === true && Number(production.p95LatencyMs) > 0, 'production', 'Run a production-like load test and attach p95/cost evidence.'),
+        check('cache_batch', 'Live cache and batching savings are measured', production.cacheAndBatchingMeasured === true, 'production', 'Measure cache hit rate and batching cost/latency lift.'),
+      ],
+    },
+    {
+      id: 'failure_security',
+      label: 'Failure modes, validation, ACL, and tenancy',
+      checks: [
+        check('structured', 'Structured answers and citations are validated', repo.structuredOutputValidation, 'repository', 'Validate output shape and citation relationships.'),
+        check('acl_order', 'Tenant/document ACL runs before retrieval and hydration', repo.tenantAclBeforeRetrieval, 'repository', 'Enforce authorization before ranking.'),
+        check('command_evasion', 'Literal command-substitution evasions are canonicalized and tested', repo.commandPositionHardening, 'repository', 'Ratchet deterministic command-position substitutions in the evasion matrix.'),
+        check('penetration_test', 'Tenant isolation has external penetration evidence', security.tenantPenTestPassed === true, 'security-review', 'Run a professional tenant-isolation penetration test.'),
+        check('incident_drill', 'Hallucination, stale-index, miss, and leak drills pass', security.failureDrillPassed === true, 'production', 'Run and retain production-like failure drills.'),
+      ],
+    },
+    {
+      id: 'framework_pipeline',
+      label: 'Framework decision and end-to-end RAG defense',
+      checks: [
+        check('decision', 'Raw versus LangChain/LangGraph/LlamaIndex tradeoffs are defended', repo.rawFrameworkDecisionDefended, 'repository', 'Document the complete pipeline and framework decision.'),
+        check('ratchet', 'The evidence scorecard runs in the main test chain', repo.scorecardInMainTest, 'repository', 'Wire this scorecard into the test chain.'),
+      ],
+    },
+    {
+      id: 'commercial_validation',
+      label: 'Value, willingness to pay, and captured money',
+      checks: [
+        check('buyer_conversations', 'At least 10 target-buyer value conversations are evidenced', Number(commercial.buyerConversations) >= 10, 'commercial', 'Complete and retain 10 target-buyer value conversations.'),
+        check('payment_asks', 'At least 3 exact-price payment asks are evidenced', Number(commercial.paymentAsks) >= 3, 'commercial', 'Make three exact-price payment asks to qualified buyers.'),
+        check('external_payment', 'At least one non-owner external payment is reconciled', Number(commercial.externalPayments) >= 1, 'provider', 'Capture and reconcile one real external payment.'),
+        check('provider_truth', 'Provider catalog and product attribution are verified', commercial.providerRevenueVerified === true, 'provider', 'Attach exact provider catalog and product-attributed revenue evidence.'),
+      ],
+    },
+  ];
+
+  for (const area of areas) {
+    const passed = area.checks.filter((row) => row.passed).length;
+    area.score = Number((10 * passed / area.checks.length).toFixed(1));
+    area.grade = area.score === 10 ? 'A+' : area.score >= 9 ? 'A' : area.score >= 8 ? 'B' : area.score >= 7 ? 'C' : area.score >= 6 ? 'D' : 'F';
+    area.status = area.score === 10 ? 'verified' : passed === 0 ? 'blocked' : 'partial';
+  }
+
+  const score = Number((areas.reduce((sum, area) => sum + area.score, 0) / areas.length).toFixed(1));
+  const atTarget = areas.every((area) => area.score === 10);
+  return {
+    scorecardVersion: SCORECARD_VERSION,
+    generatedAt: new Date().toISOString(),
+    target: { score: 10, grade: 'A+' },
+    atTarget,
+    score,
+    grade: atTarget ? 'A+' : score >= 9 ? 'A' : score >= 8 ? 'B' : score >= 7 ? 'C' : score >= 6 ? 'D' : 'F',
+    areas,
+    blockers: areas.flatMap((area) => area.checks
+      .filter((row) => !row.passed)
+      .map((row) => ({ area: area.id, check: row.id, evidenceClass: row.evidenceClass, remediation: row.remediation }))),
+  };
+}
+
+function formatMarkdown(report) {
+  const lines = [
+    '# ThumbGate A+ evidence scorecard',
+    '',
+    `Overall: **${report.score}/10 (${report.grade})**`,
+    `Target verified: **${report.atTarget ? 'YES' : 'NO'}**`,
+    '',
+    '| Area | Score | Grade | Status |',
+    '|---|---:|:---:|---|',
+    ...report.areas.map((area) => `| ${area.label} | ${area.score}/10 | ${area.grade} | ${area.status} |`),
+    '',
+    '## Remaining evidence blockers',
+    '',
+    ...(report.blockers.length
+      ? report.blockers.map((row) => `- **${row.area}/${row.check}** (${row.evidenceClass}): ${row.remediation}`)
+      : ['- None. Every repository, production, provider, security, and commercial check is verified.']),
+    '',
+  ];
+  return lines.join('\n');
+}
+
+function loadLiveEvidence(argv) {
+  const index = argv.indexOf('--evidence');
+  if (index === -1 || !argv[index + 1]) return {};
+  return JSON.parse(fs.readFileSync(path.resolve(argv[index + 1]), 'utf8'));
+}
+
+function main() {
+  const live = loadLiveEvidence(process.argv.slice(2));
+  const repo = collectRepositoryEvidence();
+  const report = evaluateReadiness({ repo, live });
+  process.stdout.write(process.argv.includes('--json')
+    ? `${JSON.stringify(report, null, 2)}\n`
+    : `${formatMarkdown(report)}\n`);
+  if (process.argv.includes('--require-a-plus') && !report.atTarget) process.exitCode = 1;
+}
+
+function isCliEntrypoint(argv = process.argv) {
+  return Boolean(argv[1]) && path.resolve(argv[1]) === path.resolve(__filename);
+}
+
+if (isCliEntrypoint()) main();
+
+module.exports = {
+  SCORECARD_VERSION,
+  collectRepositoryEvidence,
+  evaluateReadiness,
+  formatMarkdown,
+  isCliEntrypoint,
+};

package/scripts/activation-quickstart.js

@@ -93,6 +93,7 @@ async function runActivationFlow({ ask, out, isTTY, deps = {} }) {
         whatToChange: `Block this action: ${mistake}`,
         tags: 'quickstart,activation,first-rule',
         gateAction: 'block',
+        reviewOrigin: 'human',
       });
     } catch {
       // Capture failure should not abort the activation aha.

package/scripts/agent-outcome-monitor.js

@@ -5,11 +5,13 @@ const fs = require('node:fs');
 const os = require('node:os');
 const path = require('node:path');
 const { calculateTaskOutcomeMetrics, readTaskOutcomes } = require('./task-outcomes');
+const { computeToolKpis } = require('./tool-kpi-tracker');
 
 const DEFAULT_THRESHOLDS = path.join(__dirname, '..', 'config', 'agent-outcome-monitor-thresholds.json');
 const DEFAULT_HOSTED_ORIGIN = 'https://thumbgate-production.up.railway.app';
 const DEFAULT_MONITOR_PATH = '/v1/task-outcomes/monitor';
 const DEFAULT_SCHEDULE_ID = 'thumbgate-agent-outcome-monitor';
+const DEFAULT_MINIMUM_TOOL_CALLS = 20;
 
 function monitorTaskOutcomes(outcomes = [], options = {}) {
   const metrics = calculateTaskOutcomeMetrics(outcomes);
@@ -72,6 +74,71 @@ function monitorTaskOutcomes(outcomes = [], options = {}) {
   };
 }
 
+function monitorProductionSignals(outcomes = [], toolKpis = {}, options = {}) {
+  const taskReport = monitorTaskOutcomes(outcomes, options);
+  const requestedMinimumToolCalls = Number(options.minimumToolCalls);
+  const minimumToolCalls = Number.isFinite(requestedMinimumToolCalls) && requestedMinimumToolCalls >= 1
+    ? Math.floor(requestedMinimumToolCalls)
+    : DEFAULT_MINIMUM_TOOL_CALLS;
+  const measuredToolCalls = Number(toolKpis.totalCalls);
+  const totalToolCalls = Number.isFinite(measuredToolCalls) && measuredToolCalls >= 0
+    ? Math.floor(measuredToolCalls)
+    : 0;
+  const toolAlerts = [];
+
+  if (totalToolCalls < minimumToolCalls) {
+    toolAlerts.push({
+      id: 'minimum-tool-calls',
+      severity: 'block',
+      message: `Need ${minimumToolCalls} observed tool calls; observed ${totalToolCalls}.`,
+    });
+  }
+  for (const tool of (toolKpis.tools || [])) {
+    if (tool.requestCount < 3) continue;
+    if (tool.successRate < 90) {
+      toolAlerts.push({
+        id: `tool-success-${tool.toolName}`,
+        severity: 'block',
+        actual: tool.successRate,
+        expected: 'gte 90',
+        message: `${tool.toolName} success rate ${tool.successRate}% is below 90%.`,
+      });
+    }
+    if (tool.p95 > 500) {
+      toolAlerts.push({
+        id: `tool-latency-${tool.toolName}`,
+        severity: 'warn',
+        actual: tool.p95,
+        expected: 'lte 500',
+        message: `${tool.toolName} p95 latency ${tool.p95}ms is above 500ms.`,
+      });
+    }
+  }
+
+  const alerts = [...taskReport.alerts, ...toolAlerts];
+  let verdict = taskReport.verdict;
+  if (taskReport.verdict !== 'blocked' && toolAlerts.some((alert) => alert.id === 'minimum-tool-calls')) {
+    verdict = 'insufficient_evidence';
+  } else if (toolAlerts.some((alert) => alert.severity === 'block')) {
+    verdict = 'blocked';
+  } else if (verdict === 'healthy' && toolAlerts.length > 0) {
+    verdict = 'watch';
+  }
+
+  return {
+    ...taskReport,
+    verdict,
+    alerts,
+    observability: {
+      minimumToolCalls,
+      totalToolCalls,
+      evidenceStatus: totalToolCalls >= minimumToolCalls ? 'measured' : 'insufficient_evidence',
+      tools: toolKpis.tools || [],
+      servers: toolKpis.servers || [],
+    },
+  };
+}
+
 function flattenMetricValues(metrics) {
   return {
     workingRate: metrics.task.workingRate,
@@ -228,8 +295,9 @@ async function main(argv = process.argv.slice(2)) {
 
   const report = options.hosted
     ? await fetchHostedMonitor(options)
-    : monitorTaskOutcomes(
+    : monitorProductionSignals(
       readTaskOutcomes({ inputPath: options.inputPath }),
+      computeToolKpis(),
       { thresholdsPath: options.thresholdsPath },
     );
   if (options.outputPath) {
@@ -255,7 +323,9 @@ module.exports = {
   flattenMetricValues,
   installAgentOutcomeMonitorSchedule,
   main,
+  monitorProductionSignals,
   monitorTaskOutcomes,
   parseArgs,
   passesRule,
+  DEFAULT_MINIMUM_TOOL_CALLS,
 };

package/scripts/async-eval-observability.js

@@ -3,11 +3,19 @@
 
 const fs = require('node:fs');
 const path = require('node:path');
+const {
+  claimSupportScore,
+  evaluateAnswerQuality,
+  queryCoverage,
+  splitAnswerClaims,
+} = require('./rag-structured-output');
 
 const DEFAULT_THRESHOLDS = {
   faithfulness: 0.72,
   answerRelevance: 0.45,
   contextPrecision: 0.5,
+  groundedness: 0.75,
+  citationPrecision: 1,
 };
 
 function tokenize(value) {
@@ -30,10 +38,7 @@ function overlapScore(left, right) {
 }
 
 function splitClaims(response) {
-  return String(response || '')
-    .split(/(?:[.!?]\s+|\n+)/)
-    .map((claim) => claim.trim())
-    .filter((claim) => claim.length > 0);
+  return splitAnswerClaims(response);
 }
 
 function normalizeContexts(contexts) {
@@ -44,11 +49,10 @@ function normalizeContexts(contexts) {
 
 function scoreFaithfulness(response, contexts) {
   const claims = splitClaims(response);
-  const contextText = normalizeContexts(contexts).join('\n');
+  const contextItems = normalizeContexts(contexts);
   if (claims.length === 0) return { score: 0, supportedClaims: 0, totalClaims: 0 };
   const supportedClaims = claims.filter((claim) => {
-    const normalized = claim.toLowerCase();
-    return contextText.toLowerCase().includes(normalized) || overlapScore(claim, contextText) >= 0.58;
+    return contextItems.some((context) => claimSupportScore(claim, context) >= 0.4);
   }).length;
   return {
     score: Number((supportedClaims / claims.length).toFixed(4)),
@@ -58,7 +62,7 @@ function scoreFaithfulness(response, contexts) {
 }
 
 function scoreAnswerRelevance(question, response) {
-  const score = overlapScore(question, response);
+  const score = queryCoverage(question, response);
   return {
     score: Number(score.toFixed(4)),
     matchedQuestionTerms: unique(tokenize(question).filter((token) => tokenize(response).includes(token))),
@@ -97,14 +101,29 @@ function evaluateGeneration(testCase, options = {}) {
     contexts,
     testCase.reference || testCase.groundTruth || ''
   );
+  const answerQuality = evaluateAnswerQuality({
+    query: testCase.question || testCase.user_input,
+    answer: testCase.response || testCase.answer,
+    referenceAnswer: testCase.reference || testCase.groundTruth || '',
+    citations: testCase.citations,
+    contexts: contexts.map((text, index) => ({ id: `context-${index + 1}`, text })),
+  }, {
+    minFaithfulness: thresholds.faithfulness,
+    minGroundedness: thresholds.groundedness,
+    minAnswerRelevance: thresholds.answerRelevance,
+  });
   const scores = {
     faithfulness: faithfulness.score,
     answerRelevance: answerRelevance.score,
     contextPrecision: contextPrecision.score,
+    groundedness: answerQuality.metrics.groundedness,
+    citationPrecision: answerQuality.metrics.citationPrecision,
   };
   const passed = scores.faithfulness >= thresholds.faithfulness
     && scores.answerRelevance >= thresholds.answerRelevance
-    && scores.contextPrecision >= thresholds.contextPrecision;
+    && scores.contextPrecision >= thresholds.contextPrecision
+    && scores.groundedness >= thresholds.groundedness
+    && scores.citationPrecision >= thresholds.citationPrecision;
 
   return {
     id: String(testCase.id || testCase.traceId || 'case'),
@@ -116,6 +135,7 @@ function evaluateGeneration(testCase, options = {}) {
       faithfulness,
       answerRelevance,
       contextPrecision,
+      answerQuality,
     },
   };
 }
@@ -155,6 +175,8 @@ function buildEvalReport(cases, options = {}) {
     faithfulness: average(results.map((result) => result.scores.faithfulness)),
     answerRelevance: average(results.map((result) => result.scores.answerRelevance)),
     contextPrecision: average(results.map((result) => result.scores.contextPrecision)),
+    groundedness: average(results.map((result) => result.scores.groundedness)),
+    citationPrecision: average(results.map((result) => result.scores.citationPrecision)),
   };
 
   return {
@@ -166,7 +188,7 @@ function buildEvalReport(cases, options = {}) {
     passRate: results.length === 0 ? 0 : Number(((passed / results.length) * 100).toFixed(2)),
     aggregate,
     passedThreshold: failed === 0,
-    metrics: ['faithfulness', 'answerRelevance', 'contextPrecision'],
+    metrics: ['faithfulness', 'answerRelevance', 'contextPrecision', 'groundedness', 'citationPrecision'],
     sinks: {
       ci: true,
       langsmithCompatible: true,
@@ -228,7 +250,10 @@ module.exports = {
   scoreFaithfulness,
 };
 
-if (require.main === module) {
+const invokedDirectly = Boolean(process.argv[1])
+  && path.resolve(process.argv[1]) === __filename;
+
+if (invokedDirectly) {
   main().catch((err) => {
     console.error(err.stack || err.message);
     process.exitCode = 1;

package/scripts/audit-trail.js

@@ -61,7 +61,18 @@ function recordAuditEvent(params = {}) {
     source: params.source || 'gates-engine',
   };
 
-  fs.appendFileSync(logPath, JSON.stringify(record) + '\n');
+  // Safe stringify: never let circular/toxic tool inputs crash the gate path
+  // (Antithesis-style invariant: evaluation + audit must not throw).
+  let line;
+  try {
+    line = JSON.stringify(record);
+  } catch (err) {
+    line = JSON.stringify({
+      ...record,
+      toolInput: { _unserializable: true, reason: String(err.message || err).slice(0, 120) },
+    });
+  }
+  fs.appendFileSync(logPath, `${line}\n`);
   try {
     const { trainAndPersistInterventionPolicy } = require('./intervention-policy');
     trainAndPersistInterventionPolicy(path.dirname(logPath));
@@ -73,10 +84,13 @@ function recordAuditEvent(params = {}) {
 
 /**
  * Strip secrets and large payloads from tool input before audit storage.
+ * Drop circular references so JSON.stringify never throws mid-gate evaluation.
  */
 function sanitizeToolInput(toolInput) {
+  if (!toolInput || typeof toolInput !== 'object') return {};
   const safe = {};
   const MAX_VALUE_LEN = 200;
+  const seen = new WeakSet();
 
   for (const [key, value] of Object.entries(toolInput)) {
     if (typeof value === 'string') {
@@ -88,6 +102,17 @@ function sanitizeToolInput(toolInput) {
           ? value.slice(0, MAX_VALUE_LEN) + '...'
           : value;
       }
+    } else if (value && typeof value === 'object') {
+      if (seen.has(value)) {
+        safe[key] = '[Circular]';
+        continue;
+      }
+      seen.add(value);
+      try {
+        safe[key] = JSON.parse(JSON.stringify(value, getCircularReplacer()));
+      } catch {
+        safe[key] = `[unserializable:${typeof value}]`;
+      }
     } else {
       safe[key] = value;
     }
@@ -95,6 +120,17 @@ function sanitizeToolInput(toolInput) {
   return safe;
 }
 
+function getCircularReplacer() {
+  const seen = new WeakSet();
+  return function circularReplacer(_key, value) {
+    if (value && typeof value === 'object') {
+      if (seen.has(value)) return '[Circular]';
+      seen.add(value);
+    }
+    return value;
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Auto-feedback from audit events
 // ---------------------------------------------------------------------------