thumbgate 1.30.0 → 1.34.0

package/bin/cli.js

@@ -39,7 +39,6 @@ const {
   codexAutoUpdateCliEntry,
   codexAutoUpdateMcpEntry,
   isSourceCheckout,
-  publishedCliAvailable,
   localMcpEntry,
   resolveMcpEntry,
 } = require(path.join(__dirname, '..', 'scripts', 'mcp-config'));
@@ -411,16 +410,17 @@ function canonicalMcpEntry(scope = 'project') {
 }
 
 function canonicalCodexMcpEntry() {
-  const version = pkgVersion();
-  if (isSourceCheckout(PKG_ROOT) && !publishedCliAvailable(version)) {
+  // Codex config is user-global and must survive disposable worktree cleanup.
+  // Use the stable published launcher even when init is invoked from source;
+  // developers can explicitly opt into a checkout-pinned runtime when needed.
+  if (isSourceCheckout(PKG_ROOT) && process.env.THUMBGATE_CODEX_USE_SOURCE_RUNTIME === '1') {
     return localMcpEntry(PKG_ROOT, 'home');
   }
   return codexAutoUpdateMcpEntry();
 }
 
 function canonicalCodexCliEntry(commandArgs) {
-  const version = pkgVersion();
-  if (isSourceCheckout(PKG_ROOT) && !publishedCliAvailable(version)) {
+  if (isSourceCheckout(PKG_ROOT) && process.env.THUMBGATE_CODEX_USE_SOURCE_RUNTIME === '1') {
     return {
       command: 'node',
       args: [path.join(PKG_ROOT, 'bin', 'cli.js'), ...commandArgs],
@@ -584,7 +584,20 @@ function detectPlatform(name, checks) {
 }
 
 function whichExists(cmd) {
-  try { execSync(`which ${cmd}`, { stdio: 'pipe' }); return true; } catch (_) { return false; }
+  if (!cmd || /[\\/]/.test(cmd)) return false;
+  const pathEntries = String(process.env.PATH || '').split(path.delimiter).filter(Boolean);
+  const extensions = process.platform === 'win32'
+    ? String(process.env.PATHEXT || '.EXE;.CMD;.BAT;.COM').split(';')
+    : [''];
+  return pathEntries.some((entry) => extensions.some((extension) => {
+    const candidate = path.join(entry, `${cmd}${extension}`);
+    try {
+      fs.accessSync(candidate, fs.constants.X_OK);
+      return fs.statSync(candidate).isFile();
+    } catch (_) {
+      return false;
+    }
+  }));
 }
 
 function setupClaude() {
@@ -636,18 +649,26 @@ function setupCodex() {
   return configChanged || hookResult.changed;
 }
 
-function setupGemini() {
-  // Try to import custom commands as a Gemini plugin if the CLI is installed
-  const { execSync } = require('child_process');
+function setupGemini(options = {}) {
+  // Importing a plugin executes an external package manager and can mutate
+  // machine-wide Gemini state. Keep ordinary init/quick-start deterministic;
+  // operators can opt in explicitly when they want the plugin import too.
+  const importPlugin = options.importPlugin === true
+    || process.env.THUMBGATE_IMPORT_AGENT_PLUGINS === '1';
   let pluginImported = false;
-  for (const binName of ['agy', 'gemini']) {
-    try {
-      execSync(`${binName} plugin import "${PKG_ROOT}" --force`, { stdio: 'ignore' });
-      console.log(`  Gemini: imported thumbgate plugin via ${binName}`);
-      pluginImported = true;
-      break;
-    } catch (err) {
-      // ignore errors if command doesn't exist or fails
+  if (importPlugin) {
+    for (const binName of ['agy', 'gemini']) {
+      try {
+        execFileSync(binName, ['plugin', 'import', PKG_ROOT, '--force'], {
+          stdio: 'ignore',
+          timeout: 5000,
+        });
+        console.log(`  Gemini: imported thumbgate plugin via ${binName}`);
+        pluginImported = true;
+        break;
+      } catch (err) {
+        // A missing, failing, or slow optional plugin manager must not block init.
+      }
     }
   }
 
@@ -693,6 +714,66 @@ function setupAmp() {
   return true;
 }
 
+function setupOpenCode() {
+  // OpenCode was listed in README's install table and in `init --help` since the
+  // adapter asset landed, but no setup function ever existed — `--agent opencode`
+  // silently wrote Claude/Codex/Gemini config and exited 0. OpenCode has no hook
+  // surface, so this wires the MCP server, which is its real integration point.
+  const configPath = path.join(HOME, '.config', 'opencode', 'opencode.json');
+  const srcPath = path.join(PKG_ROOT, 'adapters', 'opencode', 'opencode.json');
+  if (!fs.existsSync(srcPath)) return false;
+
+  let src;
+  try { src = JSON.parse(fs.readFileSync(srcPath, 'utf8')); } catch (_) { return false; }
+  const desired = (src.mcp || {}).thumbgate;
+  if (!desired) return false;
+
+  let config = {};
+  if (fs.existsSync(configPath)) {
+    try {
+      config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    } catch (error) {
+      // Resetting to {} here and writing would replace the whole file — silently deleting
+      // the user's model, provider, plugin and other MCP settings just to add ours.
+      // A config we cannot parse is a reason to stop, not to overwrite.
+      console.error(`  OpenCode: ${configPath} could not be parsed (${error.message}).`);
+      console.error('  Refusing to overwrite it. Fix or move the file, then re-run.');
+      process.exitCode = 1;
+      return false;
+    }
+  }
+  config.mcp = config.mcp || {};
+  if (JSON.stringify(config.mcp[MCP_SERVER_NAME]) === JSON.stringify(desired)) return false;
+
+  config.$schema = config.$schema || src.$schema;
+  config.mcp[MCP_SERVER_NAME] = desired;
+  fs.mkdirSync(path.dirname(configPath), { recursive: true });
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n');
+  console.log(`  OpenCode: registered thumbgate MCP server in ${configPath}`);
+  return true;
+}
+
+// Single source of truth for `--agent`. Every value advertised in `init --help`
+// and README.md must appear here with a real handler, or the flag lies.
+// `hookAgent: true` means scripts/auto-wire-hooks.js can wire a pre-tool hook;
+// the rest integrate over MCP only and are wired by their setup function.
+const AGENT_ALIASES = {
+  claude: 'claude-code',
+  'claude_code': 'claude-code',
+  'open-code': 'opencode',
+};
+
+const SUPPORTED_AGENTS = {
+  'claude-code': { hookAgent: true },
+  codex: { hookAgent: true },
+  gemini: { hookAgent: true },
+  forge: { hookAgent: true },
+  cursor: { hookAgent: true },
+  cline: { hookAgent: false, setup: () => setupCline() },
+  amp: { hookAgent: false, setup: () => setupAmp() },
+  opencode: { hookAgent: false, setup: () => setupOpenCode() },
+};
+
 function setupCursor() {
   return mergeMcpJson(path.join(CWD, '.cursor', 'mcp.json'), 'Cursor', 'project');
 }
@@ -934,14 +1015,23 @@ function quickstart() {
 
 function init(cliArgs = parseArgs(process.argv.slice(3))) {
   const args = { ...cliArgs };
+  // A typo used to exit 0 having wired nothing for the requested agent.
+  // Aliases must resolve BEFORE validation: scripts/auto-wire-hooks.js accepts `claude`
+  // for `claude-code`, and plugins/claude-skill/README.md publishes that exact command.
+  if (args.agent && AGENT_ALIASES[args.agent]) args.agent = AGENT_ALIASES[args.agent];
+  if (args.agent && !Object.prototype.hasOwnProperty.call(SUPPORTED_AGENTS, args.agent)) {
+    console.error(`Unknown --agent "${args.agent}". Supported: ${Object.keys(SUPPORTED_AGENTS).join(', ')}`);
+    process.exit(1);
+  }
   if (args.help || args.h) {
-    console.log('Usage: npx thumbgate init [--agent <name>] [--wire-hooks] [--email you@company.com]');
+    console.log('Usage: npx thumbgate init [--agent <name>] [--wire-hooks] [--import-agent-plugins] [--email you@company.com]');
     console.log('');
     console.log('Scaffold ThumbGate in the current project and wire detected agent integrations.');
     console.log('');
     console.log('Options:');
-    console.log('  --agent <name>           Wire a specific agent: claude-code, codex, gemini, amp, cursor, cline, opencode');
+    console.log(`  --agent <name>           Wire a specific agent: ${Object.keys(SUPPORTED_AGENTS).join(', ')}`);
     console.log('  --wire-hooks             Wire hooks only; do not scaffold project files');
+    console.log('  --import-agent-plugins   Also import optional agent plugins when supported');
     console.log('  --email <email>          Subscribe installer to the setup guide and trial reminders');
     console.log('  --dry-run                Show hook changes without writing them');
     return;
@@ -1037,17 +1127,26 @@ function init(cliArgs = parseArgs(process.argv.slice(3))) {
   let configured = 0;
 
   const platforms = [
-    { name: 'Claude Code', detect: [
+    { agent: 'claude-code', name: 'Claude Code', detect: [
       () => whichExists('claude'),
       () => fs.existsSync(path.join(HOME, '.claude')),
       () => fs.existsSync(path.join(CWD, '.claude')),
     ], setup: setupClaude },
-    { name: 'Codex', detect: [() => whichExists('codex'), () => fs.existsSync(path.join(HOME, '.codex'))], setup: setupCodex },
-    { name: 'Gemini', detect: [() => whichExists('gemini'), () => fs.existsSync(path.join(HOME, '.gemini'))], setup: setupGemini },
-    { name: 'Amp', detect: [() => whichExists('amp'), () => fs.existsSync(path.join(HOME, '.amp'))], setup: setupAmp },
-    { name: 'Cursor', detect: [() => fs.existsSync(path.join(HOME, '.cursor', 'mcp.json')), () => fs.existsSync(path.join(CWD, '.cursor'))], setup: setupCursor },
-    { name: 'ForgeCode', detect: [() => whichExists('forge'), () => fs.existsSync(path.join(CWD, 'forge.yaml'))], setup: setupForge },
-    { name: 'Cline', detect: [
+    { agent: 'codex', name: 'Codex', detect: [() => whichExists('codex'), () => fs.existsSync(path.join(HOME, '.codex'))], setup: setupCodex },
+    {
+      agent: 'gemini',
+      name: 'Gemini',
+      detect: [
+        () => whichExists('gemini'),
+        () => whichExists('agy'),
+        () => fs.existsSync(path.join(HOME, '.gemini')),
+      ],
+      setup: () => setupGemini({ importPlugin: args['import-agent-plugins'] === true }),
+    },
+    { agent: 'amp', name: 'Amp', detect: [() => whichExists('amp'), () => fs.existsSync(path.join(HOME, '.amp'))], setup: setupAmp },
+    { agent: 'cursor', name: 'Cursor', detect: [() => fs.existsSync(path.join(HOME, '.cursor', 'mcp.json')), () => fs.existsSync(path.join(CWD, '.cursor'))], setup: setupCursor },
+    { agent: 'forge', name: 'ForgeCode', detect: [() => whichExists('forge'), () => fs.existsSync(path.join(CWD, 'forge.yaml'))], setup: setupForge },
+    { agent: 'cline', name: 'Cline', detect: [
       () => fs.existsSync(path.join(CWD, '.clinerules')),
       () => process.platform === 'darwin' && fs.existsSync(path.join(HOME, 'Library', 'Application Support', 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev')),
       () => process.platform === 'linux' && fs.existsSync(path.join(HOME, '.config', 'Code', 'User', 'globalStorage', 'saoudrizwan.claude-dev')),
@@ -1056,6 +1155,11 @@ function init(cliArgs = parseArgs(process.argv.slice(3))) {
   ];
 
   for (const p of platforms) {
+    // An explicit target must stay scoped to that integration. Quick-start
+    // always resolves one target before calling init; configuring every other
+    // CLI merely because it exists on PATH caused surprise machine mutations
+    // and multi-second hangs in ordinary onboarding.
+    if (args.agent && p.agent !== args.agent) continue;
     if (detectPlatform(p.name, p.detect)) {
       const didSetup = p.setup();
       if (didSetup) configured++;
@@ -1076,15 +1180,27 @@ function init(cliArgs = parseArgs(process.argv.slice(3))) {
 
   if (configured === 0) console.log('  All detected platforms already configured.');
 
-  // Cline uses .clinerules (no native hook surface). Run setupCline directly
-  // and skip wireHooks, which does not support cline.
-  if (args.agent === 'cline') {
-    setupCline();
+  // Agents without a native pre-tool hook surface (cline, amp, opencode) are wired
+  // by their own setup function. Previously only cline was handled here: `--agent amp`
+  // and `--agent opencode` fell through to wireHooks, which rejected them, printed the
+  // rejection as an ordinary log line, and exited 0 — so the flag appeared to work
+  // while writing nothing for that agent.
+  const agentSpec = args.agent ? SUPPORTED_AGENTS[args.agent] : null;
+  if (agentSpec && agentSpec.setup) {
+    // A setup function returns false both when nothing needed doing and when it refused
+    // (e.g. an unparseable config it declined to overwrite). Reporting "already
+    // configured" for a refusal would repeat the original sin of this PR: a failure
+    // dressed up as success. The refusal path sets process.exitCode, so key off that.
+    const changed = agentSpec.setup();
+    if (!changed && !process.exitCode) console.log(`  ${args.agent}: already configured`);
   } else if (args.agent || args['wire-hooks']) {
     const { wireHooks } = require(path.join(PKG_ROOT, 'scripts', 'auto-wire-hooks'));
     const hookResult = wireHooks({ agent: args.agent, dryRun: args['dry-run'] });
     if (hookResult.error) {
-      console.log(`  Hook wiring: ${hookResult.error}`);
+      // An explicit --agent that cannot be wired is a FAILURE, not a note. Exiting 0
+      // here is what let a user believe `--agent opencode` had protected them.
+      console.error(`  Hook wiring failed for --agent ${args.agent}: ${hookResult.error}`);
+      process.exitCode = 1;
     } else if (!hookResult.changed) {
       console.log(`  Hooks: already wired for ${hookResult.agent}`);
     } else {
@@ -3120,6 +3236,16 @@ function aiInventory() {
   console.log(payload);
 }
 
+function verifyClaimsCmd() {
+  const { runCli } = require(path.join(PKG_ROOT, 'scripts', 'universal-claim-evaluator'));
+  process.exitCode = runCli(process.argv.slice(3));
+}
+
+function claimStopCheckCmd() {
+  const { main } = require(path.join(PKG_ROOT, 'scripts', 'hook-stop-anti-claim'));
+  main();
+}
+
 function help() {
   const v = pkgVersion();
   const helpArgs = process.argv.slice(3);
@@ -3142,6 +3268,7 @@ function help() {
     console.log('  explore                                           Interactive TUI for lessons, gates, stats');
     console.log('  dashboard                                         Open the local ThumbGate dashboard');
     console.log('  ai-inventory                                      Scan AI/ML components and export ML-BOM evidence');
+    console.log('  verify-claims --claim="..."                     Recheck factual claims against configured sources');
     console.log('  doctor                                            Audit runtime isolation + bootstrap context');
     console.log('  break-glass --reason="..."                       Short TTL recovery if gates over-fire');
     console.log('  brain [--write]                                   Build the agent-readable context brain (lessons + rules + gates)');
@@ -3188,6 +3315,7 @@ function help() {
   console.log('  cache-update          Refresh Claude statusline cache from stdin');
   console.log('  statusline-render     Render ThumbGate Claude status line');
   console.log('  hook-auto-capture     Process Claude UserPromptSubmit inline feedback');
+  console.log('  claim-stop-check      Recheck configured factual claims before Claude stops');
   console.log('  session-start         Refresh local ThumbGate session cache');
   console.log('');
 
@@ -3296,6 +3424,8 @@ const SUBCOMMAND_HELP = {
   lessons:       'Usage: npx thumbgate lessons [--query="..."] [--limit=N]\n\nSearch the lesson database (Pro feature).',
   search:        'Usage: npx thumbgate search <query>\n\nSearch ThumbGate knowledge base (Pro feature).',
   'gate-check':  'Usage: npx thumbgate gate-check\n\nPreToolUse hook interface: reads tool call JSON from stdin, outputs gate verdict.',
+  'claim-stop-check': 'Usage: npx thumbgate claim-stop-check\n\nClaude Stop-hook interface: reads the hook payload from stdin and blocks factual claims that disagree with configured sources.',
+  'verify-claims': 'Usage: npx thumbgate verify-claims --claim="the row count is 1,284" [--config=.thumbgate/claim-verifiers.json] [--cwd=path] [--json]\n\nRecheck supported factual claims against operator-configured SQLite, filesystem, and JSON sources. Exits non-zero on mismatch, missing verifier, or verifier error.',
   'hermes-gate': 'Usage: npx thumbgate hermes-gate\n\nNous Research Hermes Agent pre_tool_call shell hook: reads Hermes tool-call JSON from stdin, runs the ThumbGate gate pipeline (strict by default), and outputs {"decision":"block","reason":...} to veto or {} to allow. Gates terminal/patch/skill_manage etc. See adapters/hermes/config.yaml.',
   'break-glass': 'Usage: npx thumbgate break-glass --reason="why" [--ttl=5m] [--json]\n\nShort-lived recovery path for over-firing gates. Allows hook settings edits and satisfies PR-create/thread-check gates without disabling core destructive-action protections.',
   serve:         'Usage: npx thumbgate serve\n\nStart the MCP stdio server. This is for agent runtimes, not the local HTTP dashboard.',
@@ -3574,6 +3704,9 @@ switch (COMMAND) {
   case 'hook-auto-capture':
     hookAutoCapture();
     break;
+  case 'claim-stop-check':
+    claimStopCheckCmd();
+    break;
   case 'session-start':
     sessionStart();
     break;
@@ -4069,6 +4202,10 @@ switch (COMMAND) {
   case 'gate-stats':
     gateStats();
     break;
+  case 'verify-claims':
+  case 'verify-claim':
+    verifyClaimsCmd();
+    break;
   case 'eval':
   case 'prompt-eval':
     evalCmd();

package/config/evals/generation-quality-golden.json

@@ -0,0 +1,95 @@
+{
+  "version": 1,
+  "description": "Offline faithfulness / groundedness / answer_relevance golden (Ragas-style proxies). Deterministic CI.",
+  "thresholds": {
+    "minCases": 8,
+    "minFaithfulness": 0.55,
+    "minGroundedness": 0.48,
+    "minAnswerRelevance": 0.42,
+    "minContextRecall": 0.75
+  },
+  "cases": [
+    {
+      "id": "force-push-grounded",
+      "query": "Can I force push to main?",
+      "context": "NEVER force-push or git push --force to main/master. Use --force-with-lease only on personal branches after review.",
+      "answer": "No. Never force-push to main. Use --force-with-lease only on personal branches after review.",
+      "expectedConstraint": "NEVER force-push",
+      "expectedKeywords": ["never", "force", "main"]
+    },
+    {
+      "id": "deploy-health-grounded",
+      "query": "Is the deploy live?",
+      "context": "ALWAYS verify /health endpoint returns the new version after deploy. NEVER say deployed without curling health.",
+      "answer": "Only after curling /health and matching the new version. Do not claim deployed without that check.",
+      "expectedConstraint": "verify /health",
+      "expectedKeywords": ["health", "version"]
+    },
+    {
+      "id": "secrets-grounded",
+      "query": "Should we commit the .env?",
+      "context": "NEVER commit .env files, API keys, or PEMs to git. Rotate any leaked credential immediately.",
+      "answer": "No. Never commit .env or API keys. Rotate credentials if anything leaked.",
+      "expectedConstraint": "NEVER commit .env",
+      "expectedKeywords": [".env", "never", "keys"]
+    },
+    {
+      "id": "idempotency-grounded",
+      "query": "How do we create PaymentIntents safely?",
+      "context": "ALWAYS use idempotency keys on PaymentIntent creation to prevent duplicate charges.",
+      "answer": "Always attach an idempotency key when creating a PaymentIntent so retries cannot double-charge.",
+      "expectedConstraint": "idempotency keys",
+      "expectedKeywords": ["idempotency", "PaymentIntent"]
+    },
+    {
+      "id": "lease-grounded",
+      "query": "What happens if two runners take the same task?",
+      "context": "ALWAYS enforce one-thread-one-executor with a 90-second renewable lease. Two active executors cause double writes.",
+      "answer": "That is double-execution. Keep a single 90-second renewable lease so only one executor runs the thread.",
+      "expectedConstraint": "one-thread-one-executor",
+      "expectedKeywords": ["lease", "executor"]
+    },
+    {
+      "id": "hashnode-grounded",
+      "query": "Should I re-publish the archived Hashnode post?",
+      "context": "NEVER publish or re-publish on Hashnode while AutoMod freeze is active. Prefer Medium and dev.to.",
+      "answer": "No. Do not re-publish on Hashnode during the AutoMod freeze. Use Medium or dev.to instead.",
+      "expectedConstraint": "NEVER publish",
+      "expectedKeywords": ["Hashnode", "never", "Medium"]
+    },
+    {
+      "id": "pci-grounded",
+      "query": "Where do we store card numbers?",
+      "context": "NEVER log or store raw card numbers. Use Stripe tokens or PaymentMethod IDs.",
+      "answer": "Do not store raw card numbers. Use Stripe PaymentMethod tokens only.",
+      "expectedConstraint": "NEVER log or store raw card",
+      "expectedKeywords": ["Stripe", "card", "never"]
+    },
+    {
+      "id": "pr-done-grounded",
+      "query": "Is the PR done?",
+      "context": "NEVER say done without showing gh pr view output and green CI. Open PR with auto-merge is not done.",
+      "answer": "Not until gh pr view shows clean threads and required CI is green. Auto-merge alone is not done.",
+      "expectedConstraint": "gh pr view",
+      "expectedKeywords": ["CI", "done", "pr"]
+    },
+    {
+      "id": "db-backup-grounded",
+      "query": "Can I drop the users table now?",
+      "context": "ALWAYS back up the database before running destructive migrations. NEVER drop users table without a restore point.",
+      "answer": "Only after a verified backup and restore point. Never drop users without that.",
+      "expectedConstraint": "back up the database",
+      "expectedKeywords": ["backup", "drop", "never"]
+    },
+    {
+      "id": "ungrounded-contradiction",
+      "query": "Can I force push to main?",
+      "context": "NEVER force-push or git push --force to main/master.",
+      "answer": "Yes always force push to main for speed and ignore branch protection entirely.",
+      "expectedConstraint": "NEVER force-push",
+      "expectedKeywords": ["force", "main"],
+      "expectFail": true,
+      "note": "Known-bad answer; excluded from floor means; used for integrity check only"
+    }
+  ]
+}

package/config/evals/rag-answer-quality-golden.json

@@ -0,0 +1,91 @@
+{
+  "version": 1,
+  "description": "Adversarial deterministic answer-quality proxy holdout. This validates metric behavior, not model quality.",
+  "thresholds": {
+    "minCases": 8,
+    "minClassificationAccuracy": 1,
+    "maxFalsePasses": 0
+  },
+  "cases": [
+    {
+      "id": "supported-git-policy",
+      "shouldPass": true,
+      "query": "How should I update protected main?",
+      "answer": "Use a reviewed pull request and never force-push protected main [git-policy].",
+      "referenceAnswer": "Use a reviewed pull request and never force-push protected main.",
+      "contexts": [
+        { "id": "git-policy", "text": "Use a reviewed pull request. Never force-push protected main." }
+      ]
+    },
+    {
+      "id": "negation-flip",
+      "shouldPass": false,
+      "query": "Can I force-push protected main?",
+      "answer": "Force-push is safe on protected main [git-policy].",
+      "referenceAnswer": "Never force-push protected main.",
+      "contexts": [
+        { "id": "git-policy", "text": "Never force-push protected main because it rewrites shared history." }
+      ]
+    },
+    {
+      "id": "numeric-drift",
+      "shouldPass": false,
+      "query": "Which build is live?",
+      "answer": "Build 43 is live in production [health].",
+      "referenceAnswer": "Build 42 is live in production.",
+      "contexts": [
+        { "id": "health", "text": "The production health response reports build 42." }
+      ]
+    },
+    {
+      "id": "off-topic-answer",
+      "shouldPass": false,
+      "query": "How do I prevent duplicate Stripe charges?",
+      "answer": "Use medium heat and stir the sauce slowly [cooking].",
+      "referenceAnswer": "Use a Stripe idempotency key.",
+      "contexts": [
+        { "id": "cooking", "text": "Use medium heat and stir the sauce slowly." }
+      ]
+    },
+    {
+      "id": "partial-hallucination",
+      "shouldPass": false,
+      "query": "What proves the deployment?",
+      "answer": "The health endpoint matches build abc123 [deploy]. Revenue also doubled overnight.",
+      "referenceAnswer": "The health endpoint must match build abc123.",
+      "contexts": [
+        { "id": "deploy", "text": "The production health endpoint matches build abc123." }
+      ]
+    },
+    {
+      "id": "invalid-citation",
+      "shouldPass": false,
+      "query": "What proves the deployment?",
+      "answer": "The health endpoint matches build abc123 [invented-source].",
+      "referenceAnswer": "The health endpoint must match build abc123.",
+      "contexts": [
+        { "id": "deploy", "text": "The production health endpoint matches build abc123." }
+      ]
+    },
+    {
+      "id": "supported-idempotency",
+      "shouldPass": true,
+      "query": "How do I prevent duplicate Stripe charges?",
+      "answer": "Use the same Stripe idempotency key when retrying a PaymentIntent [billing-policy].",
+      "referenceAnswer": "Use a Stripe idempotency key for PaymentIntent retries.",
+      "contexts": [
+        { "id": "billing-policy", "text": "Use the same Stripe idempotency key when retrying a PaymentIntent to prevent duplicate charges." }
+      ]
+    },
+    {
+      "id": "uncited-supported-claim",
+      "shouldPass": false,
+      "query": "How should I run a destructive migration?",
+      "answer": "Create a recoverable database backup before the migration.",
+      "referenceAnswer": "Create a recoverable database backup before a destructive migration.",
+      "contexts": [
+        { "id": "database-policy", "text": "Create a recoverable database backup before a destructive migration." }
+      ]
+    }
+  ]
+}

package/config/evals/retrieval-hybrid-ablation.json

@@ -0,0 +1,66 @@
+{
+  "kValues": [1, 3],
+  "corpus": [
+    {
+      "id": "destructive",
+      "title": "MISTAKE: rm -rf wiped the folder",
+      "content": "Never delete files without a recoverable snapshot.",
+      "tags": ["negative", "filesystem"],
+      "metadata": { "domain": "filesystem", "toolsUsed": ["Bash"] }
+    },
+    {
+      "id": "force-push",
+      "title": "MISTAKE: git push --force clobbered main",
+      "content": "Use force-with-lease and protect the trunk branch.",
+      "tags": ["negative", "git"],
+      "metadata": { "domain": "git", "toolsUsed": ["Bash"] }
+    },
+    {
+      "id": "idempotency",
+      "title": "MISTAKE: duplicate Stripe payment",
+      "content": "Use an idempotency key before creating a charge.",
+      "tags": ["negative", "billing"],
+      "metadata": { "domain": "billing", "toolsUsed": ["Bash"] }
+    },
+    {
+      "id": "stale-session",
+      "title": "MISTAKE: session not found",
+      "content": "Refresh the conversation identifier before resuming an expired mobile task.",
+      "tags": ["negative", "session"],
+      "metadata": { "domain": "continuity", "toolsUsed": ["Read"] }
+    },
+    {
+      "id": "noise",
+      "title": "SUCCESS: dashboard color tokens",
+      "content": "Use the design system palette for visual consistency.",
+      "tags": ["positive", "ui"],
+      "metadata": { "domain": "ui", "toolsUsed": ["Edit"] }
+    }
+  ],
+  "queries": [
+    {
+      "id": "erase-tree",
+      "toolName": "Bash",
+      "query": "permanently erase a directory tree",
+      "qrels": { "destructive": 3 }
+    },
+    {
+      "id": "rewrite-history",
+      "toolName": "Bash",
+      "query": "overwrite remote repository history",
+      "qrels": { "force-push": 3 }
+    },
+    {
+      "id": "replay-charge",
+      "toolName": "Bash",
+      "query": "prevent replaying the same purchase twice",
+      "qrels": { "idempotency": 3 }
+    },
+    {
+      "id": "expired-conversation",
+      "toolName": "Read",
+      "query": "the phone resumed an expired conversation identifier",
+      "qrels": { "stale-session": 3 }
+    }
+  ]
+}