npm - thumbgate - Versions diffs - 1.29.1 → 1.30.0 - Mend

thumbgate 1.29.1 → 1.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (77) hide show

package/.claude/commands/dashboard.md +11 -1
package/.claude/commands/thumbgate-dashboard.md +23 -8
package/.claude-plugin/plugin.json +1 -1
package/.well-known/mcp/server-card.json +1 -1
package/README.md +61 -1
package/adapters/claude/.mcp.json +2 -2
package/adapters/forge/forge.yaml +3 -3
package/adapters/mcp/server-stdio.js +164 -7
package/adapters/opencode/opencode.json +1 -1
package/bin/cli.js +7 -5
package/commands/dashboard.md +11 -1
package/commands/thumbgate-dashboard.md +23 -8
package/config/agent-outcome-monitor-thresholds.json +63 -0
package/config/evals/agent-outcomes-baseline.json +17 -0
package/config/evals/agent-outcomes-golden.json +412 -0
package/config/evals/prompt-eval-baseline.json +23 -0
package/config/mcp-allowlists.json +26 -2
package/config/post-deploy-marketing-pages.json +26 -1
package/config/schemas/task-outcome-receipt.schema.json +296 -0
package/openapi/openapi.yaml +235 -0
package/package.json +55 -11
package/public/architecture.html +130 -0
package/public/assets/diagrams/agent-integration.png +0 -0
package/public/assets/diagrams/before-after.svg +21 -0
package/public/assets/diagrams/decision.svg +36 -0
package/public/assets/diagrams/feedback-pipeline.png +0 -0
package/public/assets/diagrams/loop.svg +34 -0
package/public/assets/diagrams/plugin-topology.png +0 -0
package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
package/public/assets/diagrams/stack.svg +18 -0
package/public/assets/diagrams/thumbgate-architecture.png +0 -0
package/public/case-studies.html +151 -0
package/public/eval-scorecard.html +195 -0
package/public/eval-scorecard.json +18 -0
package/public/evaluations.html +168 -0
package/public/index.html +6 -3
package/public/numbers.html +2 -2
package/public/whitepaper.html +189 -0
package/scripts/activation-quickstart.js +1 -0
package/scripts/agent-outcome-eval.js +130 -0
package/scripts/agent-outcome-monitor.js +331 -0
package/scripts/agent-reasoning-traces.js +8 -9
package/scripts/async-job-runner.js +107 -13
package/scripts/billing.js +3 -1
package/scripts/claude-feedback-sync.js +3 -2
package/scripts/cli-feedback.js +13 -7
package/scripts/cross-encoder-reranker.js +3 -0
package/scripts/durability/step.js +121 -12
package/scripts/feedback-aggregate.js +5 -2
package/scripts/feedback-loop.js +244 -182
package/scripts/gates-engine.js +512 -22
package/scripts/generate-case-study-outreach.js +253 -0
package/scripts/generate-eval-scorecard.js +276 -0
package/scripts/growth-campaigns.js +183 -0
package/scripts/human-escalation.js +265 -0
package/scripts/hybrid-feedback-context.js +93 -50
package/scripts/jsonl-watcher.js +1 -0
package/scripts/judge-reward-function.js +30 -18
package/scripts/lesson-inference.js +23 -4
package/scripts/lesson-retrieval.js +71 -4
package/scripts/lesson-search.js +26 -3
package/scripts/mcp-config.js +26 -5
package/scripts/mcp-oauth.js +37 -2
package/scripts/model-eval.js +308 -0
package/scripts/parallel-workflow-orchestrator.js +86 -22
package/scripts/prompt-eval.js +81 -4
package/scripts/published-cli.js +11 -1
package/scripts/refresh-proof-pack.js +261 -0
package/scripts/risk-scorer.js +144 -15
package/scripts/schedule-manager.js +249 -0
package/scripts/statusline-local-stats.js +1 -1
package/scripts/task-outcomes.js +425 -0
package/scripts/thumbgate-bench.js +13 -0
package/scripts/tool-contract-validator.js +287 -59
package/scripts/tool-kpi-tracker.js +124 -0
package/scripts/tool-registry.js +192 -1
package/src/api/server.js +355 -89

package/public/whitepaper.html ADDED Viewed

@@ -0,0 +1,189 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<meta name="generator" content="ThumbGate">
+<meta name="author" content="Igor Ganapolsky">
+<title>How We Know ThumbGate Works — Evaluation White Paper</title>
+<meta name="description" content="Evaluation white paper for ThumbGate: golden datasets, offline regression, tool-call correctness, latency, cost, human review, and production monitoring — with diagrams and re-run commands.">
+<meta property="og:title" content="How We Know ThumbGate Works">
+<meta property="og:description" content="Seven proof dimensions for a pre-action gate: goldens, offline tests, tool-call metrics, latency, cost, human review, production monitoring.">
+<meta property="og:type" content="article">
+<meta property="og:url" content="https://thumbgate.ai/whitepaper">
+<link rel="canonical" href="https://thumbgate.ai/whitepaper">
+<link rel="icon" type="image/png" href="/thumbgate-icon.png">
+<script defer data-domain="thumbgate.ai" src="https://plausible.io/js/script.js"></script>
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "TechArticle",
+  "headline": "How We Know ThumbGate Works",
+  "description": "Buyer-facing evaluation white paper covering golden datasets, offline regression, tool-call correctness, latency, cost, human review, and production monitoring.",
+  "url": "https://thumbgate.ai/whitepaper",
+  "dateModified": "2026-07-29",
+  "author": { "@type": "Person", "name": "Igor Ganapolsky", "url": "https://github.com/IgorGanapolsky" }
+}
+</script>
+<style>
+  :root {
+    --bg:#0b0f14; --panel:#111823; --border:#1e2a3a; --text:#e6edf3;
+    --muted:#8b98a5; --cyan:#39c5cf; --green:#3fb950; --red:#f85149;
+  }
+  * { margin:0; padding:0; box-sizing:border-box; }
+  body { background:var(--bg); color:var(--text); font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif; line-height:1.65; }
+  nav { padding:1rem 2rem; border-bottom:1px solid var(--border); display:flex; gap:1.25rem; flex-wrap:wrap; align-items:center; }
+  nav a { color:var(--muted); text-decoration:none; font-size:0.9rem; }
+  nav a:hover { color:var(--cyan); }
+  nav .brand { color:var(--text); font-weight:700; }
+  .container { max-width:900px; margin:0 auto; padding:2.5rem 1.5rem 4rem; }
+  h1 { font-size:2rem; margin-bottom:0.4rem; }
+  h2 { font-size:1.3rem; margin:2.3rem 0 0.7rem; color:var(--cyan); }
+  h3 { font-size:1.05rem; margin:1.3rem 0 0.4rem; }
+  .subtitle { color:var(--muted); font-size:1.05rem; margin-bottom:1.25rem; }
+  p, li { margin:0.65rem 0; }
+  ul, ol { padding-left:1.3rem; }
+  table { width:100%; border-collapse:collapse; margin:1rem 0; font-size:0.92rem; }
+  th, td { text-align:left; padding:0.5rem 0.65rem; border-bottom:1px solid var(--border); vertical-align:top; }
+  th { color:var(--muted); }
+  code { background:#0d1420; border:1px solid var(--border); border-radius:4px; padding:0.1rem 0.35rem; font-size:0.86em; color:var(--cyan); }
+  pre { background:#0d1420; border:1px solid var(--border); border-radius:8px; padding:1rem; overflow-x:auto; font-size:0.85rem; margin:1rem 0; }
+  .panel { background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:1.1rem 1.25rem; margin:1rem 0; }
+  .rule { border-left:3px solid var(--cyan); padding:0.65rem 1rem; background:var(--panel); margin:1.25rem 0; font-weight:600; }
+  figure { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:1rem; margin:1rem 0; }
+  figure img { width:100%; height:auto; display:block; border-radius:8px; }
+  figcaption { color:var(--muted); font-size:0.88rem; margin-top:0.65rem; }
+  a.cta { display:inline-block; background:var(--cyan); color:#001018; font-weight:700; padding:0.7rem 1rem; border-radius:8px; text-decoration:none; margin:0.4rem 0.6rem 0.4rem 0; }
+  a.inline { color:var(--cyan); }
+  .footer-note { color:var(--muted); font-size:0.85rem; margin-top:2.5rem; text-align:center; }
+  .footer-note a { color:var(--cyan); text-decoration:none; }
+  .good { color:var(--green); } .bad { color:var(--red); }
+</style>
+</head>
+<body>
+<nav>
+  <a class="brand" href="/">ThumbGate</a>
+  <a href="/whitepaper">White paper</a>
+  <a href="/architecture">Architecture</a>
+  <a href="/eval-scorecard">Scorecard</a>
+  <a href="/evaluations">Evaluations</a>
+  <a href="/case-studies">Case studies</a>
+  <a href="/numbers">Numbers</a>
+  <a href="/pricing">Pricing</a>
+</nav>
+<main class="container">
+  <h1>How we know ThumbGate works</h1>
+  <p class="subtitle">Evaluation white paper · Updated 2026-07-29 · Source: <code>docs/HOW-WE-KNOW-IT-WORKS.md</code></p>
+  <div class="rule">Offline goldens + live monitors, not vibes. <code>unsafeActionRate</code> must stay <span class="good">0</span>.</div>
+  <p>ThumbGate is a local-first PreToolUse enforcement layer: feedback becomes lessons, repeated failures become gates, and risky tool calls are checked before execution. This page is the buyer-readable map from that claim to evidence.</p>
+  <figure>
+    <img src="/assets/diagrams/thumbgate-architecture.png" alt="ThumbGate system architecture diagram" width="1200" height="800" loading="eager">
+    <figcaption>System map: agents, hooks, gate engine, governance, security, feedback promotion. Full gallery on <a class="inline" href="/architecture">/architecture</a>.</figcaption>
+  </figure>
+  <h2>1. Golden evaluation dataset</h2>
+  <p>Committed suites encode expected allow / deny / warn decisions and reference answers — not aspirational prose.</p>
+  <table>
+    <tr><th>Pack</th><th>Path</th></tr>
+    <tr><td>ThumbGate Bench</td><td><code>bench/thumbgate-bench.json</code></td></tr>
+    <tr><td>Agent safety eval</td><td><code>config/evals/agent-safety-eval.json</code></td></tr>
+    <tr><td>Prompt eval</td><td><code>bench/prompt-eval-suite.json</code></td></tr>
+    <tr><td>Observability / grounding</td><td><code>bench/observability-eval-suite.json</code></td></tr>
+    <tr><td>ProgramBench smoke</td><td><code>bench/programbench-smoke.json</code></td></tr>
+    <tr><td>Shell golden tests</td><td><code>verification/golden_tests/**</code></td></tr>
+  </table>
+  <p>Live failures feed offline goldens through <code>scripts/llm-behavior-monitor.js</code> promotion candidates.</p>
+  <h2>2. Offline regression tests</h2>
+  <pre>npm test
+npm run test:coverage
+npm run prove:adapters
+npm run prove:automation
+npm run self-heal:check</pre>
+  <p>Observed results land in <a class="inline" href="https://github.com/IgorGanapolsky/ThumbGate/blob/main/docs/VERIFICATION_EVIDENCE.md">docs/VERIFICATION_EVIDENCE.md</a>. Releases also require Changesets, SemVer, and version sync.</p>
+  <h2>3. Tool-call correctness</h2>
+  <p>ThumbGate Bench measures whether the gate decides correctly on the actual tool call.</p>
+  <table>
+    <tr><th>Metric</th><th>Intent</th></tr>
+    <tr><td><code>taskSuccessRate</code></td><td>Decision matches expected</td></tr>
+    <tr><td><code>unsafeActionRate</code></td><td class="good">Must stay 0</td></tr>
+    <tr><td><code>capabilityRate</code></td><td>Safe work still allowed</td></tr>
+    <tr><td><code>falseBlockRate</code></td><td>No “block everything” cheat</td></tr>
+    <tr><td><code>replayStability</code></td><td>Same suite → same decisions</td></tr>
+  </table>
+  <p><a class="cta" href="/eval-scorecard">Open the live scorecard</a> <a class="inline" href="https://github.com/IgorGanapolsky/ThumbGate/blob/main/docs/THUMBGATE_BENCH.md">Bench methodology →</a></p>
+  <h2>4. Latency</h2>
+  <figure>
+    <img src="/assets/diagrams/pre-action-gate-loop.svg" alt="Pre-action gate loop" width="900" height="360" loading="lazy">
+    <figcaption>Enforcement is local: deterministic match first, optional local embeddings, no cloud model on the critical path.</figcaption>
+  </figure>
+  <ul>
+    <li>No LLM on block/allow for the PreToolUse decision</li>
+    <li>Routing budgets: <code>lowLatencyBudgetMs: 300</code> for the fast classical lane</li>
+    <li>High-risk ambiguity can stop for human review instead of spending tokens</li>
+  </ul>
+  <h2>5. Cost</h2>
+  <ul>
+    <li>A blocked tool call does not spend the model round-trip</li>
+    <li><code>npx thumbgate cost</code> prints conservative $ saved from recorded blocks</li>
+    <li>Budget ledger + tokenomics guardrails prevent runaway spend</li>
+    <li>Methodology: <a class="inline" href="/agents-cost-savings">/agents-cost-savings</a> · inventory: <a class="inline" href="/numbers">/numbers</a></li>
+  </ul>
+  <h2>6. Human review</h2>
+  <ul>
+    <li>Classifier route <code>human_review</code> for credentials, customer data, regulated workflows, payments</li>
+    <li>Rubrics require verification evidence before “done”</li>
+    <li>Protected actions need scoped approval; break-glass is short-lived and narrow</li>
+    <li>Outbound social replies stay draft-only until a human publishes</li>
+  </ul>
+  <h2>7. Production monitoring</h2>
+  <figure>
+    <img src="/assets/diagrams/feedback-pipeline.png" alt="Feedback pipeline sequence diagram" width="1200" height="700" loading="lazy">
+    <figcaption>Capture → promote → block the repeat. Production monitors exist to catch silence (fail-open hooks) as well as loud breakage.</figcaption>
+  </figure>
+  <ul>
+    <li>Deploy health: Railway <code>/health</code> version + dashboard smoke</li>
+    <li><code>self-heal:check</code> — budget, tests, prove lanes must be HEALTHY</li>
+    <li>Gate stats + LLM behavior rates (wrong-tool, malformed, drift)</li>
+    <li>Published-artifact evasion jobs and silent-gate canaries</li>
+  </ul>
+  <h2>20-minute buyer audit</h2>
+  <pre>git clone https://github.com/IgorGanapolsky/ThumbGate
+cd ThumbGate &amp;&amp; npm ci
+npm test
+npm run prove:adapters &amp;&amp; npm run prove:automation
+npm run self-heal:check
+npm run thumbgate:bench -- --json
+curl -s https://thumbgate-production.up.railway.app/health</pre>
+  <h2>What this paper is not</h2>
+  <div class="panel">
+    <ul>
+      <li>Not external customer revenue proof — see commercial truth in the repo</li>
+      <li>Not a claim that free installs hard-block every risky command by default</li>
+      <li>Not a substitute for human review on payments, credentials, or regulated actions</li>
+    </ul>
+  </div>
+  <p>
+    <a class="cta" href="/case-studies">Case studies</a>
+    <a class="cta" href="/evaluations">ML evaluations</a>
+    <a class="cta" href="/architecture">Diagram gallery</a>
+  </p>
+  <p class="footer-note">
+    Longform source: <a href="https://github.com/IgorGanapolsky/ThumbGate/blob/main/docs/HOW-WE-KNOW-IT-WORKS.md">docs/HOW-WE-KNOW-IT-WORKS.md</a>.
+    Built by <a href="https://github.com/IgorGanapolsky">Igor Ganapolsky</a>.
+  </p>
+</main>
+</body>
+</html>

package/scripts/activation-quickstart.js CHANGED Viewed

@@ -93,6 +93,7 @@ async function runActivationFlow({ ask, out, isTTY, deps = {} }) {
         whatToChange: `Block this action: ${mistake}`,
         tags: 'quickstart,activation,first-rule',
         gateAction: 'block',
+        reviewOrigin: 'human',
       });
     } catch {
       // Capture failure should not abort the activation aha.

package/scripts/agent-outcome-eval.js ADDED Viewed

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const {
+  TASK_OUTCOME_SCHEMA,
+  calculateTaskOutcomeMetrics,
+  normalizeTaskOutcome,
+} = require('./task-outcomes');
+const { validateToolContract } = require('./tool-contract-validator');
+const ROOT = path.join(__dirname, '..');
+const DEFAULT_SUITE = path.join(ROOT, 'config', 'evals', 'agent-outcomes-golden.json');
+const DEFAULT_BASELINE = path.join(ROOT, 'config', 'evals', 'agent-outcomes-baseline.json');
+function runCase(testCase = {}) {
+  const receipt = normalizeTaskOutcome(testCase.receipt, new Date('2026-07-26T00:00:00.000Z'));
+  const validation = validateToolContract(TASK_OUTCOME_SCHEMA, receipt);
+  const expectedReasons = [...(testCase.expected?.reasons || [])]
+    .sort((left, right) => left.localeCompare(right));
+  const actualReasons = [...(receipt.workingReasons || [])]
+    .sort((left, right) => left.localeCompare(right));
+  const checks = [
+    {
+      id: 'schema_valid',
+      pass: validation.valid,
+      detail: validation.valid ? 'receipt schema valid' : validation.errors.join('; '),
+    },
+    {
+      id: 'working_verdict',
+      pass: receipt.working === testCase.expected?.working,
+      detail: `expected ${testCase.expected?.working}, got ${receipt.working}`,
+    },
+    {
+      id: 'reason_codes',
+      pass: JSON.stringify(actualReasons) === JSON.stringify(expectedReasons),
+      detail: `expected [${expectedReasons.join(', ')}], got [${actualReasons.join(', ')}]`,
+    },
+  ];
+  const passed = checks.every((check) => check.pass);
+  return {
+    id: testCase.id,
+    status: passed ? 'pass' : 'fail',
+    score: Math.round((checks.filter((check) => check.pass).length / checks.length) * 100),
+    severity: testCase.severity,
+    split: testCase.split,
+    checks,
+    receipt,
+  };
+}
+function runAgentOutcomeEval(options = {}) {
+  const suitePath = path.resolve(options.suitePath || DEFAULT_SUITE);
+  const suite = JSON.parse(fs.readFileSync(suitePath, 'utf8'));
+  if (!Array.isArray(suite.cases) || suite.cases.length === 0) {
+    throw new Error('Agent outcome suite must contain non-empty cases');
+  }
+  const results = suite.cases.map(runCase);
+  const passed = results.filter((result) => result.status === 'pass').length;
+  const score = Math.round((results.reduce((sum, result) => sum + result.score, 0) / results.length));
+  const minimumCases = Number(suite.successCriteria?.minimumCases || 1);
+  const minimumScore = Number(suite.successCriteria?.minAggregateScore || 100);
+  const baselinePath = path.resolve(options.baselinePath || DEFAULT_BASELINE);
+  const baseline = fs.existsSync(baselinePath)
+    ? JSON.parse(fs.readFileSync(baselinePath, 'utf8'))
+    : null;
+  const regressions = compareBaseline(results, baseline);
+  const report = {
+    suite: suite.name,
+    generatedAt: new Date().toISOString(),
+    evidenceStatus: results.length >= minimumCases ? 'measured' : 'insufficient_evidence',
+    total: results.length,
+    passed,
+    failed: results.length - passed,
+    score,
+    minimumScore,
+    minimumCases,
+    regressions,
+    pass: results.length >= minimumCases
+      && score >= minimumScore
+      && (!suite.successCriteria?.requireNoRegressions || regressions.length === 0),
+    metrics: calculateTaskOutcomeMetrics(results.map((result) => result.receipt)),
+    results,
+  };
+  return report;
+}
+function compareBaseline(results, baseline) {
+  if (!baseline) return [];
+  const current = new Map(results.map((result) => [result.id, result]));
+  const regressions = [];
+  for (const previous of baseline.results || []) {
+    const result = current.get(previous.id);
+    if (!result) {
+      regressions.push({ id: previous.id, reason: 'missing_case' });
+    } else if (result.score < previous.score || (previous.status === 'pass' && result.status !== 'pass')) {
+      regressions.push({
+        id: previous.id,
+        reason: 'score_or_status_regression',
+        baselineScore: previous.score,
+        currentScore: result.score,
+      });
+    }
+  }
+  return regressions;
+}
+function isCliInvocation() {
+  return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
+}
+if (isCliInvocation()) {
+  const suitePath = process.argv.find((arg) => arg.startsWith('--suite='))?.slice(8);
+  const baselinePath = process.argv.find((arg) => arg.startsWith('--baseline='))?.slice(11);
+  const outputPath = process.argv.find((arg) => arg.startsWith('--output='))?.slice(9);
+  const report = runAgentOutcomeEval({ suitePath, baselinePath });
+  if (outputPath) {
+    fs.mkdirSync(path.dirname(path.resolve(outputPath)), { recursive: true });
+    fs.writeFileSync(path.resolve(outputPath), `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+  }
+  console.log(JSON.stringify(report, null, 2));
+  process.exitCode = report.pass ? 0 : 1;
+}
+module.exports = {
+  compareBaseline,
+  runAgentOutcomeEval,
+  runCase,
+};

package/scripts/agent-outcome-monitor.js ADDED Viewed

@@ -0,0 +1,331 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { calculateTaskOutcomeMetrics, readTaskOutcomes } = require('./task-outcomes');
+const { computeToolKpis } = require('./tool-kpi-tracker');
+const DEFAULT_THRESHOLDS = path.join(__dirname, '..', 'config', 'agent-outcome-monitor-thresholds.json');
+const DEFAULT_HOSTED_ORIGIN = 'https://thumbgate-production.up.railway.app';
+const DEFAULT_MONITOR_PATH = '/v1/task-outcomes/monitor';
+const DEFAULT_SCHEDULE_ID = 'thumbgate-agent-outcome-monitor';
+const DEFAULT_MINIMUM_TOOL_CALLS = 20;
+function monitorTaskOutcomes(outcomes = [], options = {}) {
+  const metrics = calculateTaskOutcomeMetrics(outcomes);
+  const thresholds = options.thresholds || JSON.parse(fs.readFileSync(
+    path.resolve(options.thresholdsPath || DEFAULT_THRESHOLDS),
+    'utf8',
+  ));
+  const minimumSamples = Number(thresholds.minimumSamples || 1);
+  if (metrics.sampleSize < minimumSamples) {
+    return {
+      generatedAt: new Date().toISOString(),
+      verdict: 'insufficient_evidence',
+      sampleSize: metrics.sampleSize,
+      minimumSamples,
+      alerts: [{
+        id: 'minimum-samples',
+        severity: 'block',
+        message: `Need ${minimumSamples} task outcomes; observed ${metrics.sampleSize}.`,
+      }],
+      metrics,
+    };
+  }
+  const values = flattenMetricValues(metrics);
+  const alerts = [];
+  for (const [id, rule] of Object.entries(thresholds)) {
+    if (id === 'minimumSamples') continue;
+    const actual = values[id];
+    if (actual === null || actual === undefined) {
+      alerts.push({
+        id: `${id}-missing`,
+        severity: rule.severity || 'warn',
+        message: `${id} has no measured denominator.`,
+      });
+      continue;
+    }
+    if (!passesRule(actual, rule)) {
+      alerts.push({
+        id: `${id}-threshold`,
+        severity: rule.severity || 'warn',
+        actual,
+        expected: `${rule.operator} ${rule.value}`,
+        message: `${id}=${actual} violates ${rule.operator} ${rule.value}.`,
+      });
+    }
+  }
+  let verdict = 'healthy';
+  if (alerts.some((alert) => alert.severity === 'block')) {
+    verdict = 'blocked';
+  } else if (alerts.length > 0) {
+    verdict = 'watch';
+  }
+  return {
+    generatedAt: new Date().toISOString(),
+    verdict,
+    sampleSize: metrics.sampleSize,
+    minimumSamples,
+    alerts,
+    metrics,
+  };
+}
+function monitorProductionSignals(outcomes = [], toolKpis = {}, options = {}) {
+  const taskReport = monitorTaskOutcomes(outcomes, options);
+  const requestedMinimumToolCalls = Number(options.minimumToolCalls);
+  const minimumToolCalls = Number.isFinite(requestedMinimumToolCalls) && requestedMinimumToolCalls >= 1
+    ? Math.floor(requestedMinimumToolCalls)
+    : DEFAULT_MINIMUM_TOOL_CALLS;
+  const measuredToolCalls = Number(toolKpis.totalCalls);
+  const totalToolCalls = Number.isFinite(measuredToolCalls) && measuredToolCalls >= 0
+    ? Math.floor(measuredToolCalls)
+    : 0;
+  const toolAlerts = [];
+  if (totalToolCalls < minimumToolCalls) {
+    toolAlerts.push({
+      id: 'minimum-tool-calls',
+      severity: 'block',
+      message: `Need ${minimumToolCalls} observed tool calls; observed ${totalToolCalls}.`,
+    });
+  }
+  for (const tool of (toolKpis.tools || [])) {
+    if (tool.requestCount < 3) continue;
+    if (tool.successRate < 90) {
+      toolAlerts.push({
+        id: `tool-success-${tool.toolName}`,
+        severity: 'block',
+        actual: tool.successRate,
+        expected: 'gte 90',
+        message: `${tool.toolName} success rate ${tool.successRate}% is below 90%.`,
+      });
+    }
+    if (tool.p95 > 500) {
+      toolAlerts.push({
+        id: `tool-latency-${tool.toolName}`,
+        severity: 'warn',
+        actual: tool.p95,
+        expected: 'lte 500',
+        message: `${tool.toolName} p95 latency ${tool.p95}ms is above 500ms.`,
+      });
+    }
+  }
+  const alerts = [...taskReport.alerts, ...toolAlerts];
+  let verdict = taskReport.verdict;
+  if (taskReport.verdict !== 'blocked' && toolAlerts.some((alert) => alert.id === 'minimum-tool-calls')) {
+    verdict = 'insufficient_evidence';
+  } else if (toolAlerts.some((alert) => alert.severity === 'block')) {
+    verdict = 'blocked';
+  } else if (verdict === 'healthy' && toolAlerts.length > 0) {
+    verdict = 'watch';
+  }
+  return {
+    ...taskReport,
+    verdict,
+    alerts,
+    observability: {
+      minimumToolCalls,
+      totalToolCalls,
+      evidenceStatus: totalToolCalls >= minimumToolCalls ? 'measured' : 'insufficient_evidence',
+      tools: toolKpis.tools || [],
+      servers: toolKpis.servers || [],
+    },
+  };
+}
+function flattenMetricValues(metrics) {
+  return {
+    workingRate: metrics.task.workingRate,
+    verifiedCompletionRate: metrics.task.verifiedCompletionRate,
+    evidenceBackedCompletionRate: metrics.task.evidenceBackedCompletionRate,
+    unsupportedClaimRate: metrics.task.unsupportedClaimRate,
+    toolContractAccuracy: metrics.tools.contractAccuracy,
+    executionSuccessRate: metrics.tools.executionSuccessRate,
+    duplicateSideEffectRate: metrics.tools.duplicateSideEffectRate,
+    unsafeEscapeRate: metrics.safety.unsafeEscapeRate,
+    policyViolationRate: metrics.safety.policyViolationRate,
+    safeFalseBlockRate: metrics.safety.safeFalseBlockRate,
+    correctEscalationRate: metrics.escalation.correctEscalationRate,
+    latencyP95Ms: metrics.efficiency.latencyP95Ms,
+  };
+}
+function passesRule(actual, rule) {
+  if (rule.operator === 'gte') return actual >= Number(rule.value);
+  if (rule.operator === 'lte') return actual <= Number(rule.value);
+  if (rule.operator === 'gt') return actual > Number(rule.value);
+  if (rule.operator === 'lt') return actual < Number(rule.value);
+  if (rule.operator === 'eq') return actual === Number(rule.value);
+  throw new Error(`Unsupported threshold operator '${rule.operator}'`);
+}
+function isCliInvocation() {
+  return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
+}
+function parseArgs(argv = []) {
+  return {
+    hosted: argv.includes('--hosted'),
+    installSchedule: argv.includes('--install-schedule'),
+    inputPath: valueForArg(argv, '--input='),
+    thresholdsPath: valueForArg(argv, '--thresholds='),
+    outputPath: valueForArg(argv, '--output='),
+    baseUrl: valueForArg(argv, '--base-url='),
+    workingDirectory: valueForArg(argv, '--working-directory='),
+  };
+}
+function valueForArg(argv, prefix) {
+  const value = argv.find((arg) => arg.startsWith(prefix));
+  return value ? value.slice(prefix.length) : undefined;
+}
+async function fetchHostedMonitor(options = {}) {
+  const env = options.env || process.env;
+  const { loadObservabilityEnv } = require('./observability-env');
+  loadObservabilityEnv({
+    env,
+    operatorPath: options.operatorPath,
+    observabilityPath: options.observabilityPath,
+    applyStripeManagedFiles: false,
+  });
+  const apiKey = String(env.THUMBGATE_OPERATOR_KEY || env.THUMBGATE_API_KEY || '').trim();
+  if (!apiKey) {
+    return {
+      generatedAt: new Date().toISOString(),
+      verdict: 'not_configured',
+      source: 'hosted',
+      reason: 'operator_authentication_unavailable',
+    };
+  }
+  const baseUrl = options.baseUrl
+    || env.THUMBGATE_BILLING_API_BASE_URL
+    || DEFAULT_HOSTED_ORIGIN;
+  const url = new URL(DEFAULT_MONITOR_PATH, baseUrl);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), Number(options.timeoutMs || 15000));
+  try {
+    const response = await (options.fetchImpl || globalThis.fetch)(url, {
+      method: 'GET',
+      headers: {
+        authorization: `Bearer ${apiKey}`,
+        accept: 'application/json',
+      },
+      signal: controller.signal,
+    });
+    if (!response.ok) {
+      return {
+        generatedAt: new Date().toISOString(),
+        verdict: 'unavailable',
+        source: 'hosted',
+        httpStatus: response.status,
+        reason: 'hosted_monitor_http_error',
+      };
+    }
+    const report = await response.json();
+    return {
+      ...report,
+      source: 'hosted',
+    };
+  } catch (error) {
+    return {
+      generatedAt: new Date().toISOString(),
+      verdict: 'unavailable',
+      source: 'hosted',
+      reason: error?.name === 'AbortError' ? 'hosted_monitor_timeout' : 'hosted_monitor_request_failed',
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+function buildAgentOutcomeMonitorSchedule(options = {}) {
+  const workingDirectory = path.resolve(options.workingDirectory || process.cwd());
+  const outputPath = path.resolve(
+    options.outputPath
+      || path.join(os.homedir(), '.thumbgate', 'reports', 'agent-outcome-monitor.json'),
+  );
+  const args = [
+    __filename,
+    '--hosted',
+    `--output=${outputPath}`,
+  ];
+  if (options.baseUrl) args.push(`--base-url=${options.baseUrl}`);
+  const command = [
+    'const { spawnSync } = require(\'node:child_process\');',
+    `const result = spawnSync(process.execPath, ${JSON.stringify(args)}, {`,
+    `  cwd: ${JSON.stringify(workingDirectory)},`,
+    '  env: process.env,',
+    '  stdio: \'inherit\',',
+    '});',
+    'if (result.error) throw result.error;',
+    'process.exit(typeof result.status === \'number\' ? result.status : 1);',
+  ].join(' ');
+  return {
+    id: DEFAULT_SCHEDULE_ID,
+    name: 'ThumbGate Agent Outcome Monitor',
+    description: 'Checks hosted task outcomes daily against fail-closed production thresholds.',
+    schedule: options.schedule || 'daily 10:17',
+    command,
+    workingDirectory,
+  };
+}
+function installAgentOutcomeMonitorSchedule(options, manager) {
+  const scheduleManager = manager || require('./schedule-manager');
+  return scheduleManager.createSchedule(buildAgentOutcomeMonitorSchedule(options || {}));
+}
+async function main(argv = process.argv.slice(2)) {
+  const options = parseArgs(argv);
+  if (options.installSchedule) {
+    const result = installAgentOutcomeMonitorSchedule(options);
+    console.log(JSON.stringify(result, null, 2));
+    process.exitCode = result.success ? 0 : 1;
+    return result;
+  }
+  const report = options.hosted
+    ? await fetchHostedMonitor(options)
+    : monitorProductionSignals(
+      readTaskOutcomes({ inputPath: options.inputPath }),
+      computeToolKpis(),
+      { thresholdsPath: options.thresholdsPath },
+    );
+  if (options.outputPath) {
+    const outputPath = path.resolve(options.outputPath);
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+  }
+  console.log(JSON.stringify(report, null, 2));
+  process.exitCode = report.verdict === 'healthy' ? 0 : 1;
+  return report;
+}
+if (isCliInvocation()) {
+  main().catch((error) => {
+    console.error(`Agent outcome monitor failed: ${error.message}`);
+    process.exitCode = 1;
+  });
+}
+module.exports = {
+  buildAgentOutcomeMonitorSchedule,
+  fetchHostedMonitor,
+  flattenMetricValues,
+  installAgentOutcomeMonitorSchedule,
+  main,
+  monitorProductionSignals,
+  monitorTaskOutcomes,
+  parseArgs,
+  passesRule,
+  DEFAULT_MINIMUM_TOOL_CALLS,
+};