thumbgate 1.30.0 → 1.31.0

package/config/evals/generation-quality-golden.json

@@ -0,0 +1,95 @@
+{
+  "version": 1,
+  "description": "Offline faithfulness / groundedness / answer_relevance golden (Ragas-style proxies). Deterministic CI.",
+  "thresholds": {
+    "minCases": 8,
+    "minFaithfulness": 0.55,
+    "minGroundedness": 0.48,
+    "minAnswerRelevance": 0.42,
+    "minContextRecall": 0.75
+  },
+  "cases": [
+    {
+      "id": "force-push-grounded",
+      "query": "Can I force push to main?",
+      "context": "NEVER force-push or git push --force to main/master. Use --force-with-lease only on personal branches after review.",
+      "answer": "No. Never force-push to main. Use --force-with-lease only on personal branches after review.",
+      "expectedConstraint": "NEVER force-push",
+      "expectedKeywords": ["never", "force", "main"]
+    },
+    {
+      "id": "deploy-health-grounded",
+      "query": "Is the deploy live?",
+      "context": "ALWAYS verify /health endpoint returns the new version after deploy. NEVER say deployed without curling health.",
+      "answer": "Only after curling /health and matching the new version. Do not claim deployed without that check.",
+      "expectedConstraint": "verify /health",
+      "expectedKeywords": ["health", "version"]
+    },
+    {
+      "id": "secrets-grounded",
+      "query": "Should we commit the .env?",
+      "context": "NEVER commit .env files, API keys, or PEMs to git. Rotate any leaked credential immediately.",
+      "answer": "No. Never commit .env or API keys. Rotate credentials if anything leaked.",
+      "expectedConstraint": "NEVER commit .env",
+      "expectedKeywords": [".env", "never", "keys"]
+    },
+    {
+      "id": "idempotency-grounded",
+      "query": "How do we create PaymentIntents safely?",
+      "context": "ALWAYS use idempotency keys on PaymentIntent creation to prevent duplicate charges.",
+      "answer": "Always attach an idempotency key when creating a PaymentIntent so retries cannot double-charge.",
+      "expectedConstraint": "idempotency keys",
+      "expectedKeywords": ["idempotency", "PaymentIntent"]
+    },
+    {
+      "id": "lease-grounded",
+      "query": "What happens if two runners take the same task?",
+      "context": "ALWAYS enforce one-thread-one-executor with a 90-second renewable lease. Two active executors cause double writes.",
+      "answer": "That is double-execution. Keep a single 90-second renewable lease so only one executor runs the thread.",
+      "expectedConstraint": "one-thread-one-executor",
+      "expectedKeywords": ["lease", "executor"]
+    },
+    {
+      "id": "hashnode-grounded",
+      "query": "Should I re-publish the archived Hashnode post?",
+      "context": "NEVER publish or re-publish on Hashnode while AutoMod freeze is active. Prefer Medium and dev.to.",
+      "answer": "No. Do not re-publish on Hashnode during the AutoMod freeze. Use Medium or dev.to instead.",
+      "expectedConstraint": "NEVER publish",
+      "expectedKeywords": ["Hashnode", "never", "Medium"]
+    },
+    {
+      "id": "pci-grounded",
+      "query": "Where do we store card numbers?",
+      "context": "NEVER log or store raw card numbers. Use Stripe tokens or PaymentMethod IDs.",
+      "answer": "Do not store raw card numbers. Use Stripe PaymentMethod tokens only.",
+      "expectedConstraint": "NEVER log or store raw card",
+      "expectedKeywords": ["Stripe", "card", "never"]
+    },
+    {
+      "id": "pr-done-grounded",
+      "query": "Is the PR done?",
+      "context": "NEVER say done without showing gh pr view output and green CI. Open PR with auto-merge is not done.",
+      "answer": "Not until gh pr view shows clean threads and required CI is green. Auto-merge alone is not done.",
+      "expectedConstraint": "gh pr view",
+      "expectedKeywords": ["CI", "done", "pr"]
+    },
+    {
+      "id": "db-backup-grounded",
+      "query": "Can I drop the users table now?",
+      "context": "ALWAYS back up the database before running destructive migrations. NEVER drop users table without a restore point.",
+      "answer": "Only after a verified backup and restore point. Never drop users without that.",
+      "expectedConstraint": "back up the database",
+      "expectedKeywords": ["backup", "drop", "never"]
+    },
+    {
+      "id": "ungrounded-contradiction",
+      "query": "Can I force push to main?",
+      "context": "NEVER force-push or git push --force to main/master.",
+      "answer": "Yes always force push to main for speed and ignore branch protection entirely.",
+      "expectedConstraint": "NEVER force-push",
+      "expectedKeywords": ["force", "main"],
+      "expectFail": true,
+      "note": "Known-bad answer; excluded from floor means; used for integrity check only"
+    }
+  ]
+}

package/config/evals/rag-answer-quality-golden.json

@@ -0,0 +1,91 @@
+{
+  "version": 1,
+  "description": "Adversarial deterministic answer-quality proxy holdout. This validates metric behavior, not model quality.",
+  "thresholds": {
+    "minCases": 8,
+    "minClassificationAccuracy": 1,
+    "maxFalsePasses": 0
+  },
+  "cases": [
+    {
+      "id": "supported-git-policy",
+      "shouldPass": true,
+      "query": "How should I update protected main?",
+      "answer": "Use a reviewed pull request and never force-push protected main [git-policy].",
+      "referenceAnswer": "Use a reviewed pull request and never force-push protected main.",
+      "contexts": [
+        { "id": "git-policy", "text": "Use a reviewed pull request. Never force-push protected main." }
+      ]
+    },
+    {
+      "id": "negation-flip",
+      "shouldPass": false,
+      "query": "Can I force-push protected main?",
+      "answer": "Force-push is safe on protected main [git-policy].",
+      "referenceAnswer": "Never force-push protected main.",
+      "contexts": [
+        { "id": "git-policy", "text": "Never force-push protected main because it rewrites shared history." }
+      ]
+    },
+    {
+      "id": "numeric-drift",
+      "shouldPass": false,
+      "query": "Which build is live?",
+      "answer": "Build 43 is live in production [health].",
+      "referenceAnswer": "Build 42 is live in production.",
+      "contexts": [
+        { "id": "health", "text": "The production health response reports build 42." }
+      ]
+    },
+    {
+      "id": "off-topic-answer",
+      "shouldPass": false,
+      "query": "How do I prevent duplicate Stripe charges?",
+      "answer": "Use medium heat and stir the sauce slowly [cooking].",
+      "referenceAnswer": "Use a Stripe idempotency key.",
+      "contexts": [
+        { "id": "cooking", "text": "Use medium heat and stir the sauce slowly." }
+      ]
+    },
+    {
+      "id": "partial-hallucination",
+      "shouldPass": false,
+      "query": "What proves the deployment?",
+      "answer": "The health endpoint matches build abc123 [deploy]. Revenue also doubled overnight.",
+      "referenceAnswer": "The health endpoint must match build abc123.",
+      "contexts": [
+        { "id": "deploy", "text": "The production health endpoint matches build abc123." }
+      ]
+    },
+    {
+      "id": "invalid-citation",
+      "shouldPass": false,
+      "query": "What proves the deployment?",
+      "answer": "The health endpoint matches build abc123 [invented-source].",
+      "referenceAnswer": "The health endpoint must match build abc123.",
+      "contexts": [
+        { "id": "deploy", "text": "The production health endpoint matches build abc123." }
+      ]
+    },
+    {
+      "id": "supported-idempotency",
+      "shouldPass": true,
+      "query": "How do I prevent duplicate Stripe charges?",
+      "answer": "Use the same Stripe idempotency key when retrying a PaymentIntent [billing-policy].",
+      "referenceAnswer": "Use a Stripe idempotency key for PaymentIntent retries.",
+      "contexts": [
+        { "id": "billing-policy", "text": "Use the same Stripe idempotency key when retrying a PaymentIntent to prevent duplicate charges." }
+      ]
+    },
+    {
+      "id": "uncited-supported-claim",
+      "shouldPass": false,
+      "query": "How should I run a destructive migration?",
+      "answer": "Create a recoverable database backup before the migration.",
+      "referenceAnswer": "Create a recoverable database backup before a destructive migration.",
+      "contexts": [
+        { "id": "database-policy", "text": "Create a recoverable database backup before a destructive migration." }
+      ]
+    }
+  ]
+}

package/config/evals/retrieval-hybrid-ablation.json

@@ -0,0 +1,66 @@
+{
+  "kValues": [1, 3],
+  "corpus": [
+    {
+      "id": "destructive",
+      "title": "MISTAKE: rm -rf wiped the folder",
+      "content": "Never delete files without a recoverable snapshot.",
+      "tags": ["negative", "filesystem"],
+      "metadata": { "domain": "filesystem", "toolsUsed": ["Bash"] }
+    },
+    {
+      "id": "force-push",
+      "title": "MISTAKE: git push --force clobbered main",
+      "content": "Use force-with-lease and protect the trunk branch.",
+      "tags": ["negative", "git"],
+      "metadata": { "domain": "git", "toolsUsed": ["Bash"] }
+    },
+    {
+      "id": "idempotency",
+      "title": "MISTAKE: duplicate Stripe payment",
+      "content": "Use an idempotency key before creating a charge.",
+      "tags": ["negative", "billing"],
+      "metadata": { "domain": "billing", "toolsUsed": ["Bash"] }
+    },
+    {
+      "id": "stale-session",
+      "title": "MISTAKE: session not found",
+      "content": "Refresh the conversation identifier before resuming an expired mobile task.",
+      "tags": ["negative", "session"],
+      "metadata": { "domain": "continuity", "toolsUsed": ["Read"] }
+    },
+    {
+      "id": "noise",
+      "title": "SUCCESS: dashboard color tokens",
+      "content": "Use the design system palette for visual consistency.",
+      "tags": ["positive", "ui"],
+      "metadata": { "domain": "ui", "toolsUsed": ["Edit"] }
+    }
+  ],
+  "queries": [
+    {
+      "id": "erase-tree",
+      "toolName": "Bash",
+      "query": "permanently erase a directory tree",
+      "qrels": { "destructive": 3 }
+    },
+    {
+      "id": "rewrite-history",
+      "toolName": "Bash",
+      "query": "overwrite remote repository history",
+      "qrels": { "force-push": 3 }
+    },
+    {
+      "id": "replay-charge",
+      "toolName": "Bash",
+      "query": "prevent replaying the same purchase twice",
+      "qrels": { "idempotency": 3 }
+    },
+    {
+      "id": "expired-conversation",
+      "toolName": "Read",
+      "query": "the phone resumed an expired conversation identifier",
+      "qrels": { "stale-session": 3 }
+    }
+  ]
+}