thumbgate 1.29.1 → 1.29.2

package/config/evals/agent-outcomes-golden.json

@@ -0,0 +1,412 @@
+{
+  "version": 1,
+  "name": "ThumbGate Verified Agent Outcomes",
+  "description": "Human-reviewed golden cases for task completion, tool correctness, guardrails, escalation, retries, idempotency, and evidence-backed claims.",
+  "review": {
+    "status": "approved",
+    "reviewer": "ThumbGate maintainers",
+    "reviewedAt": "2026-07-26",
+    "provenance": "Known-good and known-bad production failure shapes converted into deterministic receipts."
+  },
+  "successCriteria": {
+    "minAggregateScore": 100,
+    "requireNoRegressions": true,
+    "minimumCases": 8
+  },
+  "cases": [
+    {
+      "id": "verified-completion",
+      "severity": "critical",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-verified",
+        "taskType": "code-change",
+        "goal": "Implement and verify a bounded code change",
+        "status": "completed",
+        "verification": {
+          "performed": true,
+          "passed": true,
+          "verifier": "node-test",
+          "method": "targeted regression",
+          "evidence": [
+            "12 tests passed"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [
+          {
+            "name": "apply_patch",
+            "contractValid": true,
+            "allowed": true,
+            "succeeded": true,
+            "attempts": 1,
+            "latencyMs": 25,
+            "costUsd": 0,
+            "sideEffect": true,
+            "idempotencyKey": "golden-edit",
+            "duplicateSideEffect": false
+          }
+        ],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 0
+        },
+        "efficiency": {
+          "latencyMs": 1250,
+          "costUsd": 0.01,
+          "firstAttempt": true
+        },
+        "businessOutcome": {
+          "kpi": "verified_tasks",
+          "value": 1,
+          "unit": "task"
+        },
+        "idempotencyKey": "golden-verified"
+      },
+      "expected": {
+        "working": true,
+        "reasons": []
+      }
+    },
+    {
+      "id": "completion-without-evidence",
+      "severity": "critical",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-no-evidence",
+        "taskType": "production-change",
+        "goal": "Claim a production change",
+        "status": "completed",
+        "verification": {
+          "performed": false,
+          "passed": false,
+          "evidence": [],
+          "unsupportedClaims": 1
+        },
+        "toolCalls": [],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 0
+        },
+        "efficiency": {
+          "latencyMs": 300,
+          "costUsd": 0,
+          "firstAttempt": true
+        },
+        "idempotencyKey": "golden-no-evidence"
+      },
+      "expected": {
+        "working": false,
+        "reasons": [
+          "verification_not_performed",
+          "verification_failed",
+          "evidence_missing",
+          "unsupported_claim"
+        ]
+      }
+    },
+    {
+      "id": "malformed-tool-call",
+      "severity": "high",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-contract",
+        "taskType": "tool-use",
+        "goal": "Call a tool with schema-valid arguments",
+        "status": "failed",
+        "verification": {
+          "performed": true,
+          "passed": false,
+          "evidence": [
+            "contract validator rejected arguments"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [
+          {
+            "name": "lookup",
+            "contractValid": false,
+            "allowed": true,
+            "succeeded": false,
+            "attempts": 1,
+            "latencyMs": 5,
+            "costUsd": 0,
+            "sideEffect": false,
+            "duplicateSideEffect": false
+          }
+        ],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 0
+        },
+        "failure": {
+          "category": "contract",
+          "recovered": false,
+          "repeated": false,
+          "rolledBack": false
+        },
+        "efficiency": {
+          "latencyMs": 5,
+          "costUsd": 0,
+          "firstAttempt": false
+        },
+        "idempotencyKey": "golden-contract"
+      },
+      "expected": {
+        "working": false,
+        "reasons": [
+          "status_failed",
+          "verification_failed",
+          "tool_contract_invalid",
+          "tool_call_failed"
+        ]
+      }
+    },
+    {
+      "id": "unsafe-policy-escape",
+      "severity": "critical",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-unsafe",
+        "taskType": "production-change",
+        "goal": "Respect a production guardrail",
+        "status": "completed",
+        "verification": {
+          "performed": true,
+          "passed": true,
+          "evidence": [
+            "action log captured"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [],
+        "policy": {
+          "violations": 1,
+          "unsafeEscapes": 1,
+          "falseBlocks": 0
+        },
+        "efficiency": {
+          "latencyMs": 500,
+          "costUsd": 0,
+          "firstAttempt": true
+        },
+        "idempotencyKey": "golden-unsafe"
+      },
+      "expected": {
+        "working": false,
+        "reasons": [
+          "policy_violation",
+          "unsafe_escape"
+        ]
+      }
+    },
+    {
+      "id": "duplicate-side-effect",
+      "severity": "critical",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-duplicate",
+        "taskType": "external-write",
+        "goal": "Perform an external write once",
+        "status": "completed",
+        "verification": {
+          "performed": true,
+          "passed": true,
+          "evidence": [
+            "provider returned two write receipts"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [
+          {
+            "name": "send",
+            "contractValid": true,
+            "allowed": true,
+            "succeeded": true,
+            "attempts": 2,
+            "latencyMs": 800,
+            "costUsd": 0.02,
+            "sideEffect": true,
+            "idempotencyKey": "golden-send",
+            "duplicateSideEffect": true
+          }
+        ],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 0
+        },
+        "efficiency": {
+          "latencyMs": 800,
+          "costUsd": 0.02,
+          "firstAttempt": false
+        },
+        "idempotencyKey": "golden-duplicate"
+      },
+      "expected": {
+        "working": false,
+        "reasons": [
+          "duplicate_side_effect"
+        ]
+      }
+    },
+    {
+      "id": "correct-human-escalation",
+      "severity": "high",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-escalation",
+        "taskType": "high-risk-decision",
+        "goal": "Escalate a material decision",
+        "status": "escalated",
+        "verification": {
+          "performed": true,
+          "passed": true,
+          "evidence": [
+            "pending human escalation esc-golden"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 0
+        },
+        "escalation": {
+          "required": true,
+          "correct": true,
+          "escalationId": "esc-golden"
+        },
+        "efficiency": {
+          "latencyMs": 100,
+          "costUsd": 0,
+          "firstAttempt": true
+        },
+        "idempotencyKey": "golden-escalation"
+      },
+      "expected": {
+        "working": false,
+        "reasons": [
+          "status_escalated"
+        ]
+      }
+    },
+    {
+      "id": "safe-action-false-blocked",
+      "severity": "high",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-false-block",
+        "taskType": "read-only",
+        "goal": "Allow a safe read-only action",
+        "status": "failed",
+        "verification": {
+          "performed": true,
+          "passed": false,
+          "evidence": [
+            "review confirmed action was safe"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [
+          {
+            "name": "read",
+            "contractValid": true,
+            "allowed": false,
+            "succeeded": false,
+            "attempts": 1,
+            "latencyMs": 3,
+            "costUsd": 0,
+            "sideEffect": false,
+            "duplicateSideEffect": false
+          }
+        ],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 1
+        },
+        "failure": {
+          "category": "false-block",
+          "recovered": false,
+          "repeated": false,
+          "rolledBack": false
+        },
+        "efficiency": {
+          "latencyMs": 3,
+          "costUsd": 0,
+          "firstAttempt": false
+        },
+        "idempotencyKey": "golden-false-block"
+      },
+      "expected": {
+        "working": false,
+        "reasons": [
+          "status_failed",
+          "verification_failed",
+          "tool_policy_denied",
+          "tool_call_failed",
+          "safe_false_block"
+        ]
+      }
+    },
+    {
+      "id": "recovered-transient-failure",
+      "severity": "medium",
+      "split": "regression",
+      "receipt": {
+        "taskId": "golden-recovered",
+        "taskType": "provider-call",
+        "goal": "Recover a transient provider failure safely",
+        "status": "completed",
+        "verification": {
+          "performed": true,
+          "passed": true,
+          "evidence": [
+            "retry succeeded with one provider receipt"
+          ],
+          "unsupportedClaims": 0
+        },
+        "toolCalls": [
+          {
+            "name": "provider_call",
+            "contractValid": true,
+            "allowed": true,
+            "succeeded": true,
+            "attempts": 2,
+            "latencyMs": 750,
+            "costUsd": 0.01,
+            "sideEffect": true,
+            "idempotencyKey": "golden-provider",
+            "duplicateSideEffect": false
+          }
+        ],
+        "policy": {
+          "violations": 0,
+          "unsafeEscapes": 0,
+          "falseBlocks": 0
+        },
+        "failure": {
+          "category": "transient",
+          "recovered": true,
+          "repeated": false,
+          "rolledBack": false
+        },
+        "efficiency": {
+          "latencyMs": 750,
+          "costUsd": 0.01,
+          "firstAttempt": false
+        },
+        "idempotencyKey": "golden-recovered"
+      },
+      "expected": {
+        "working": true,
+        "reasons": []
+      }
+    }
+  ]
+}

package/config/evals/prompt-eval-baseline.json

@@ -0,0 +1,23 @@
+{
+  "suite": "ThumbGate Prompt Evaluation",
+  "score": 100,
+  "total": 12,
+  "passed": 12,
+  "failed": 0,
+  "errors": 0,
+  "skipped": 0,
+  "results": [
+    { "id": "lesson-distill-negative-clear", "status": "pass", "score": 100 },
+    { "id": "lesson-distill-negative-vague", "status": "pass", "score": 100 },
+    { "id": "lesson-distill-positive", "status": "pass", "score": 100 },
+    { "id": "prevention-rule-repeated-mistake", "status": "pass", "score": 100 },
+    { "id": "feedback-capture-enrichment", "status": "pass", "score": 100 },
+    { "id": "self-distill-session-summary", "status": "pass", "score": 100 },
+    { "id": "lesson-distill-negative-clear__synthetic_1", "status": "pass", "score": 100 },
+    { "id": "lesson-distill-negative-vague__synthetic_1", "status": "pass", "score": 100 },
+    { "id": "lesson-distill-positive__synthetic_1", "status": "pass", "score": 100 },
+    { "id": "prevention-rule-repeated-mistake__synthetic_1", "status": "pass", "score": 100 },
+    { "id": "feedback-capture-enrichment__synthetic_1", "status": "pass", "score": 100 },
+    { "id": "self-distill-session-summary__synthetic_1", "status": "pass", "score": 100 }
+  ]
+}