thumbgate 1.28.4 → 1.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/dashboard.md +11 -1
- package/.claude/commands/thumbgate-dashboard.md +23 -8
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/llms.txt +18 -10
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +66 -3
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +88 -2
- package/adapters/opencode/opencode.json +1 -1
- package/bin/cli.js +8 -8
- package/bin/postinstall.js +4 -13
- package/commands/dashboard.md +11 -1
- package/commands/thumbgate-dashboard.md +23 -8
- package/config/agent-outcome-monitor-thresholds.json +63 -0
- package/config/evals/agent-outcomes-baseline.json +17 -0
- package/config/evals/agent-outcomes-golden.json +412 -0
- package/config/evals/prompt-eval-baseline.json +23 -0
- package/config/github-about.json +5 -4
- package/config/post-deploy-marketing-pages.json +6 -6
- package/config/schemas/task-outcome-receipt.schema.json +296 -0
- package/docs/integrations/grafana/README.md +109 -0
- package/docs/integrations/grafana/thumbgate-revenue-evidence-dashboard.json +1930 -0
- package/openapi/openapi.yaml +475 -5
- package/package.json +75 -22
- package/public/agent-manager.html +10 -11
- package/public/agents-cost-savings.html +2 -2
- package/public/assets/brand/thumbgate-logo-transparent.svg +6 -11
- package/public/assets/brand/thumbgate-mark-inline-v3.svg +11 -10
- package/public/assets/brand/thumbgate-mark.svg +10 -11
- package/public/blog/inside-your-boundary.html +114 -0
- package/public/blog/process-over-outcome-gates.html +119 -0
- package/public/blog.html +296 -402
- package/public/brand/thumbgate-mark.svg +5 -9
- package/public/codex-enterprise.html +2 -2
- package/public/compare.html +12 -3
- package/public/diagnostic.html +79 -29
- package/public/guide.html +4 -4
- package/public/index.html +1090 -2098
- package/public/install.html +3 -3
- package/public/js/buyer-intent.js +33 -18
- package/public/numbers.html +2 -2
- package/public/pricing.html +268 -408
- package/public/pro.html +4 -4
- package/scripts/agent-outcome-eval.js +130 -0
- package/scripts/agent-outcome-monitor.js +261 -0
- package/scripts/agent-reasoning-traces.js +8 -9
- package/scripts/async-job-runner.js +107 -13
- package/scripts/billing.js +456 -126
- package/scripts/buyer-paths.js +102 -0
- package/scripts/cli-feedback.js +2 -2
- package/scripts/commercial-offer.js +18 -10
- package/scripts/durability/step.js +121 -12
- package/scripts/external-customer-audit.js +881 -0
- package/scripts/feedback-loop.js +26 -0
- package/scripts/gates-engine.js +554 -19
- package/scripts/grafana-revenue-evidence.js +856 -0
- package/scripts/human-escalation.js +265 -0
- package/scripts/hybrid-feedback-context.js +93 -50
- package/scripts/jsonl-window.js +89 -0
- package/scripts/judge-reward-function.js +30 -18
- package/scripts/lesson-embedding-index.js +3 -7
- package/scripts/meta-agent-loop.js +20 -2
- package/scripts/observability-env.js +139 -0
- package/scripts/observability-setup.js +55 -0
- package/scripts/plausible-domain-config.js +4 -0
- package/scripts/prompt-eval.js +81 -4
- package/scripts/provider-live-evidence.js +1290 -0
- package/scripts/provider-payment-reconciler.js +442 -0
- package/scripts/provider-revenue-evidence.js +249 -0
- package/scripts/rate-limiter.js +1 -5
- package/scripts/revenue-action-eligibility.js +414 -0
- package/scripts/revenue-evidence-remediation.js +694 -0
- package/scripts/revenue-offer-system.js +709 -0
- package/scripts/sales-pipeline.js +1117 -0
- package/scripts/schedule-manager.js +249 -0
- package/scripts/seo-gsd.js +8 -4
- package/scripts/stripe-credentials.js +37 -0
- package/scripts/stripe-revenue-catalog-audit.js +363 -0
- package/scripts/stripe-revenue-catalog.js +164 -0
- package/scripts/task-outcomes.js +425 -0
- package/scripts/telemetry-analytics.js +23 -3
- package/scripts/tool-contract-validator.js +287 -59
- package/scripts/tool-registry.js +143 -0
- package/scripts/vector-store.js +83 -7
- package/scripts/workflow-intake-queue.js +483 -0
- package/src/api/server.js +647 -118
package/public/pro.html
CHANGED
|
@@ -37,7 +37,7 @@ __GA_BOOTSTRAP__
|
|
|
37
37
|
<script type="application/ld+json">
|
|
38
38
|
{
|
|
39
39
|
"@context": "https://schema.org",
|
|
40
|
-
"@type": "FAQPage", "mainEntity": [{ "@type": "Question", "name": "How is Pro different from the free install?", "acceptedAnswer": { "@type": "Answer", "text": "Free keeps local recall, checks, and MCP. Pro adds the personal dashboard, DPO export, auto-connect, and founder support." } }, { "@type": "Question", "name": "Does Pro require a cloud account?", "acceptedAnswer": { "@type": "Answer", "text": "No. Pro stays local-first
|
|
40
|
+
"@type": "FAQPage", "mainEntity": [{ "@type": "Question", "name": "How is Pro different from the free install?", "acceptedAnswer": { "@type": "Answer", "text": "Free keeps local recall, checks, and MCP. Pro adds the personal dashboard, DPO export, auto-connect, and founder support." } }, { "@type": "Question", "name": "Does Pro require a cloud account?", "acceptedAnswer": { "@type": "Answer", "text": "No. Pro stays local-first. Enterprise service work is qualified and signed after intake; hosted team sync and a hosted org dashboard are not generally available." } }, { "@type": "Question", "name": "What happens after checkout?", "acceptedAnswer": { "@type": "Answer", "text": "You activate Pro, connect the local dashboard, and inspect blocked actions, lessons, and exports." } }, { "@type": "Question", "name": "When should I choose Enterprise instead of Pro?", "acceptedAnswer": { "@type": "Answer", "text": "Choose Enterprise service work after proof when two or three consequential local workflows need explicit approval boundaries, rollback paths, and evidence ownership. It does not imply generally available hosted team features." } }]
|
|
41
41
|
}
|
|
42
42
|
</script>
|
|
43
43
|
|
|
@@ -926,7 +926,7 @@ __GA_BOOTSTRAP__
|
|
|
926
926
|
<div class="team-card">
|
|
927
927
|
<div class="section-label" style="text-align:left;margin-bottom:8px;">When Enterprise is better</div>
|
|
928
928
|
<h3>Need shared enforcement?</h3>
|
|
929
|
-
<p>Choose Enterprise
|
|
929
|
+
<p>Choose Enterprise service work after proof when two or three consequential local workflows need explicit approval boundaries, rollback paths, and evidence ownership. The proposal-only paths are a $15,000 30-day pilot and $10,000/month after a completed pilot.</p>
|
|
930
930
|
<div class="hero-actions" style="margin-top:18px;">
|
|
931
931
|
<a class="btn-secondary" href="/#workflow-sprint-intake">Book an Enterprise Pilot Call</a>
|
|
932
932
|
</div>
|
|
@@ -951,7 +951,7 @@ __GA_BOOTSTRAP__
|
|
|
951
951
|
</div>
|
|
952
952
|
<div class="faq-item">
|
|
953
953
|
<button class="faq-q" type="button" onclick="toggleFaq(this)" onkeydown="handleFaqKeydown(event)" aria-expanded="false">Does Pro require a cloud account?</button>
|
|
954
|
-
<div class="faq-a">No. Pro is still local-first for the individual operator lane. Enterprise is
|
|
954
|
+
<div class="faq-a">No. Pro is still local-first for the individual operator lane. Enterprise service work is qualified and signed after intake; hosted team sync, a hosted org dashboard, and hosted review views are not generally available.</div>
|
|
955
955
|
</div>
|
|
956
956
|
<div class="faq-item">
|
|
957
957
|
<button class="faq-q" type="button" onclick="toggleFaq(this)" onkeydown="handleFaqKeydown(event)" aria-expanded="false">What happens after checkout?</button>
|
|
@@ -959,7 +959,7 @@ __GA_BOOTSTRAP__
|
|
|
959
959
|
</div>
|
|
960
960
|
<div class="faq-item">
|
|
961
961
|
<button class="faq-q" type="button" onclick="toggleFaq(this)" onkeydown="handleFaqKeydown(event)" aria-expanded="false">When should I choose Enterprise instead of Pro?</button>
|
|
962
|
-
<div class="faq-a">Choose Enterprise when
|
|
962
|
+
<div class="faq-a">Choose Enterprise service work after proof when two or three consequential local workflows need explicit approval boundaries, rollback paths, and evidence ownership. The current pilot uses the local runtime and does not promise hosted team sync, a hosted org dashboard, or hosted review views.</div>
|
|
963
963
|
</div>
|
|
964
964
|
</div>
|
|
965
965
|
</div>
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const fs = require('node:fs');
|
|
5
|
+
const path = require('node:path');
|
|
6
|
+
const {
|
|
7
|
+
TASK_OUTCOME_SCHEMA,
|
|
8
|
+
calculateTaskOutcomeMetrics,
|
|
9
|
+
normalizeTaskOutcome,
|
|
10
|
+
} = require('./task-outcomes');
|
|
11
|
+
const { validateToolContract } = require('./tool-contract-validator');
|
|
12
|
+
|
|
13
|
+
const ROOT = path.join(__dirname, '..');
|
|
14
|
+
const DEFAULT_SUITE = path.join(ROOT, 'config', 'evals', 'agent-outcomes-golden.json');
|
|
15
|
+
const DEFAULT_BASELINE = path.join(ROOT, 'config', 'evals', 'agent-outcomes-baseline.json');
|
|
16
|
+
|
|
17
|
+
function runCase(testCase = {}) {
|
|
18
|
+
const receipt = normalizeTaskOutcome(testCase.receipt, new Date('2026-07-26T00:00:00.000Z'));
|
|
19
|
+
const validation = validateToolContract(TASK_OUTCOME_SCHEMA, receipt);
|
|
20
|
+
const expectedReasons = [...(testCase.expected?.reasons || [])]
|
|
21
|
+
.sort((left, right) => left.localeCompare(right));
|
|
22
|
+
const actualReasons = [...(receipt.workingReasons || [])]
|
|
23
|
+
.sort((left, right) => left.localeCompare(right));
|
|
24
|
+
const checks = [
|
|
25
|
+
{
|
|
26
|
+
id: 'schema_valid',
|
|
27
|
+
pass: validation.valid,
|
|
28
|
+
detail: validation.valid ? 'receipt schema valid' : validation.errors.join('; '),
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
id: 'working_verdict',
|
|
32
|
+
pass: receipt.working === testCase.expected?.working,
|
|
33
|
+
detail: `expected ${testCase.expected?.working}, got ${receipt.working}`,
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
id: 'reason_codes',
|
|
37
|
+
pass: JSON.stringify(actualReasons) === JSON.stringify(expectedReasons),
|
|
38
|
+
detail: `expected [${expectedReasons.join(', ')}], got [${actualReasons.join(', ')}]`,
|
|
39
|
+
},
|
|
40
|
+
];
|
|
41
|
+
const passed = checks.every((check) => check.pass);
|
|
42
|
+
return {
|
|
43
|
+
id: testCase.id,
|
|
44
|
+
status: passed ? 'pass' : 'fail',
|
|
45
|
+
score: Math.round((checks.filter((check) => check.pass).length / checks.length) * 100),
|
|
46
|
+
severity: testCase.severity,
|
|
47
|
+
split: testCase.split,
|
|
48
|
+
checks,
|
|
49
|
+
receipt,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function runAgentOutcomeEval(options = {}) {
|
|
54
|
+
const suitePath = path.resolve(options.suitePath || DEFAULT_SUITE);
|
|
55
|
+
const suite = JSON.parse(fs.readFileSync(suitePath, 'utf8'));
|
|
56
|
+
if (!Array.isArray(suite.cases) || suite.cases.length === 0) {
|
|
57
|
+
throw new Error('Agent outcome suite must contain non-empty cases');
|
|
58
|
+
}
|
|
59
|
+
const results = suite.cases.map(runCase);
|
|
60
|
+
const passed = results.filter((result) => result.status === 'pass').length;
|
|
61
|
+
const score = Math.round((results.reduce((sum, result) => sum + result.score, 0) / results.length));
|
|
62
|
+
const minimumCases = Number(suite.successCriteria?.minimumCases || 1);
|
|
63
|
+
const minimumScore = Number(suite.successCriteria?.minAggregateScore || 100);
|
|
64
|
+
const baselinePath = path.resolve(options.baselinePath || DEFAULT_BASELINE);
|
|
65
|
+
const baseline = fs.existsSync(baselinePath)
|
|
66
|
+
? JSON.parse(fs.readFileSync(baselinePath, 'utf8'))
|
|
67
|
+
: null;
|
|
68
|
+
const regressions = compareBaseline(results, baseline);
|
|
69
|
+
const report = {
|
|
70
|
+
suite: suite.name,
|
|
71
|
+
generatedAt: new Date().toISOString(),
|
|
72
|
+
evidenceStatus: results.length >= minimumCases ? 'measured' : 'insufficient_evidence',
|
|
73
|
+
total: results.length,
|
|
74
|
+
passed,
|
|
75
|
+
failed: results.length - passed,
|
|
76
|
+
score,
|
|
77
|
+
minimumScore,
|
|
78
|
+
minimumCases,
|
|
79
|
+
regressions,
|
|
80
|
+
pass: results.length >= minimumCases
|
|
81
|
+
&& score >= minimumScore
|
|
82
|
+
&& (!suite.successCriteria?.requireNoRegressions || regressions.length === 0),
|
|
83
|
+
metrics: calculateTaskOutcomeMetrics(results.map((result) => result.receipt)),
|
|
84
|
+
results,
|
|
85
|
+
};
|
|
86
|
+
return report;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function compareBaseline(results, baseline) {
|
|
90
|
+
if (!baseline) return [];
|
|
91
|
+
const current = new Map(results.map((result) => [result.id, result]));
|
|
92
|
+
const regressions = [];
|
|
93
|
+
for (const previous of baseline.results || []) {
|
|
94
|
+
const result = current.get(previous.id);
|
|
95
|
+
if (!result) {
|
|
96
|
+
regressions.push({ id: previous.id, reason: 'missing_case' });
|
|
97
|
+
} else if (result.score < previous.score || (previous.status === 'pass' && result.status !== 'pass')) {
|
|
98
|
+
regressions.push({
|
|
99
|
+
id: previous.id,
|
|
100
|
+
reason: 'score_or_status_regression',
|
|
101
|
+
baselineScore: previous.score,
|
|
102
|
+
currentScore: result.score,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return regressions;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function isCliInvocation() {
|
|
110
|
+
return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (isCliInvocation()) {
|
|
114
|
+
const suitePath = process.argv.find((arg) => arg.startsWith('--suite='))?.slice(8);
|
|
115
|
+
const baselinePath = process.argv.find((arg) => arg.startsWith('--baseline='))?.slice(11);
|
|
116
|
+
const outputPath = process.argv.find((arg) => arg.startsWith('--output='))?.slice(9);
|
|
117
|
+
const report = runAgentOutcomeEval({ suitePath, baselinePath });
|
|
118
|
+
if (outputPath) {
|
|
119
|
+
fs.mkdirSync(path.dirname(path.resolve(outputPath)), { recursive: true });
|
|
120
|
+
fs.writeFileSync(path.resolve(outputPath), `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
121
|
+
}
|
|
122
|
+
console.log(JSON.stringify(report, null, 2));
|
|
123
|
+
process.exitCode = report.pass ? 0 : 1;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
module.exports = {
|
|
127
|
+
compareBaseline,
|
|
128
|
+
runAgentOutcomeEval,
|
|
129
|
+
runCase,
|
|
130
|
+
};
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const fs = require('node:fs');
|
|
5
|
+
const os = require('node:os');
|
|
6
|
+
const path = require('node:path');
|
|
7
|
+
const { calculateTaskOutcomeMetrics, readTaskOutcomes } = require('./task-outcomes');
|
|
8
|
+
|
|
9
|
+
const DEFAULT_THRESHOLDS = path.join(__dirname, '..', 'config', 'agent-outcome-monitor-thresholds.json');
|
|
10
|
+
const DEFAULT_HOSTED_ORIGIN = 'https://thumbgate-production.up.railway.app';
|
|
11
|
+
const DEFAULT_MONITOR_PATH = '/v1/task-outcomes/monitor';
|
|
12
|
+
const DEFAULT_SCHEDULE_ID = 'thumbgate-agent-outcome-monitor';
|
|
13
|
+
|
|
14
|
+
function monitorTaskOutcomes(outcomes = [], options = {}) {
|
|
15
|
+
const metrics = calculateTaskOutcomeMetrics(outcomes);
|
|
16
|
+
const thresholds = options.thresholds || JSON.parse(fs.readFileSync(
|
|
17
|
+
path.resolve(options.thresholdsPath || DEFAULT_THRESHOLDS),
|
|
18
|
+
'utf8',
|
|
19
|
+
));
|
|
20
|
+
const minimumSamples = Number(thresholds.minimumSamples || 1);
|
|
21
|
+
if (metrics.sampleSize < minimumSamples) {
|
|
22
|
+
return {
|
|
23
|
+
generatedAt: new Date().toISOString(),
|
|
24
|
+
verdict: 'insufficient_evidence',
|
|
25
|
+
sampleSize: metrics.sampleSize,
|
|
26
|
+
minimumSamples,
|
|
27
|
+
alerts: [{
|
|
28
|
+
id: 'minimum-samples',
|
|
29
|
+
severity: 'block',
|
|
30
|
+
message: `Need ${minimumSamples} task outcomes; observed ${metrics.sampleSize}.`,
|
|
31
|
+
}],
|
|
32
|
+
metrics,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const values = flattenMetricValues(metrics);
|
|
37
|
+
const alerts = [];
|
|
38
|
+
for (const [id, rule] of Object.entries(thresholds)) {
|
|
39
|
+
if (id === 'minimumSamples') continue;
|
|
40
|
+
const actual = values[id];
|
|
41
|
+
if (actual === null || actual === undefined) {
|
|
42
|
+
alerts.push({
|
|
43
|
+
id: `${id}-missing`,
|
|
44
|
+
severity: rule.severity || 'warn',
|
|
45
|
+
message: `${id} has no measured denominator.`,
|
|
46
|
+
});
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if (!passesRule(actual, rule)) {
|
|
50
|
+
alerts.push({
|
|
51
|
+
id: `${id}-threshold`,
|
|
52
|
+
severity: rule.severity || 'warn',
|
|
53
|
+
actual,
|
|
54
|
+
expected: `${rule.operator} ${rule.value}`,
|
|
55
|
+
message: `${id}=${actual} violates ${rule.operator} ${rule.value}.`,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
let verdict = 'healthy';
|
|
60
|
+
if (alerts.some((alert) => alert.severity === 'block')) {
|
|
61
|
+
verdict = 'blocked';
|
|
62
|
+
} else if (alerts.length > 0) {
|
|
63
|
+
verdict = 'watch';
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
generatedAt: new Date().toISOString(),
|
|
67
|
+
verdict,
|
|
68
|
+
sampleSize: metrics.sampleSize,
|
|
69
|
+
minimumSamples,
|
|
70
|
+
alerts,
|
|
71
|
+
metrics,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function flattenMetricValues(metrics) {
|
|
76
|
+
return {
|
|
77
|
+
workingRate: metrics.task.workingRate,
|
|
78
|
+
verifiedCompletionRate: metrics.task.verifiedCompletionRate,
|
|
79
|
+
evidenceBackedCompletionRate: metrics.task.evidenceBackedCompletionRate,
|
|
80
|
+
unsupportedClaimRate: metrics.task.unsupportedClaimRate,
|
|
81
|
+
toolContractAccuracy: metrics.tools.contractAccuracy,
|
|
82
|
+
executionSuccessRate: metrics.tools.executionSuccessRate,
|
|
83
|
+
duplicateSideEffectRate: metrics.tools.duplicateSideEffectRate,
|
|
84
|
+
unsafeEscapeRate: metrics.safety.unsafeEscapeRate,
|
|
85
|
+
policyViolationRate: metrics.safety.policyViolationRate,
|
|
86
|
+
safeFalseBlockRate: metrics.safety.safeFalseBlockRate,
|
|
87
|
+
correctEscalationRate: metrics.escalation.correctEscalationRate,
|
|
88
|
+
latencyP95Ms: metrics.efficiency.latencyP95Ms,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function passesRule(actual, rule) {
|
|
93
|
+
if (rule.operator === 'gte') return actual >= Number(rule.value);
|
|
94
|
+
if (rule.operator === 'lte') return actual <= Number(rule.value);
|
|
95
|
+
if (rule.operator === 'gt') return actual > Number(rule.value);
|
|
96
|
+
if (rule.operator === 'lt') return actual < Number(rule.value);
|
|
97
|
+
if (rule.operator === 'eq') return actual === Number(rule.value);
|
|
98
|
+
throw new Error(`Unsupported threshold operator '${rule.operator}'`);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function isCliInvocation() {
|
|
102
|
+
return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function parseArgs(argv = []) {
|
|
106
|
+
return {
|
|
107
|
+
hosted: argv.includes('--hosted'),
|
|
108
|
+
installSchedule: argv.includes('--install-schedule'),
|
|
109
|
+
inputPath: valueForArg(argv, '--input='),
|
|
110
|
+
thresholdsPath: valueForArg(argv, '--thresholds='),
|
|
111
|
+
outputPath: valueForArg(argv, '--output='),
|
|
112
|
+
baseUrl: valueForArg(argv, '--base-url='),
|
|
113
|
+
workingDirectory: valueForArg(argv, '--working-directory='),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function valueForArg(argv, prefix) {
|
|
118
|
+
const value = argv.find((arg) => arg.startsWith(prefix));
|
|
119
|
+
return value ? value.slice(prefix.length) : undefined;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function fetchHostedMonitor(options = {}) {
|
|
123
|
+
const env = options.env || process.env;
|
|
124
|
+
const { loadObservabilityEnv } = require('./observability-env');
|
|
125
|
+
loadObservabilityEnv({
|
|
126
|
+
env,
|
|
127
|
+
operatorPath: options.operatorPath,
|
|
128
|
+
observabilityPath: options.observabilityPath,
|
|
129
|
+
applyStripeManagedFiles: false,
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
const apiKey = String(env.THUMBGATE_OPERATOR_KEY || env.THUMBGATE_API_KEY || '').trim();
|
|
133
|
+
if (!apiKey) {
|
|
134
|
+
return {
|
|
135
|
+
generatedAt: new Date().toISOString(),
|
|
136
|
+
verdict: 'not_configured',
|
|
137
|
+
source: 'hosted',
|
|
138
|
+
reason: 'operator_authentication_unavailable',
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const baseUrl = options.baseUrl
|
|
143
|
+
|| env.THUMBGATE_BILLING_API_BASE_URL
|
|
144
|
+
|| DEFAULT_HOSTED_ORIGIN;
|
|
145
|
+
const url = new URL(DEFAULT_MONITOR_PATH, baseUrl);
|
|
146
|
+
const controller = new AbortController();
|
|
147
|
+
const timeout = setTimeout(() => controller.abort(), Number(options.timeoutMs || 15000));
|
|
148
|
+
try {
|
|
149
|
+
const response = await (options.fetchImpl || globalThis.fetch)(url, {
|
|
150
|
+
method: 'GET',
|
|
151
|
+
headers: {
|
|
152
|
+
authorization: `Bearer ${apiKey}`,
|
|
153
|
+
accept: 'application/json',
|
|
154
|
+
},
|
|
155
|
+
signal: controller.signal,
|
|
156
|
+
});
|
|
157
|
+
if (!response.ok) {
|
|
158
|
+
return {
|
|
159
|
+
generatedAt: new Date().toISOString(),
|
|
160
|
+
verdict: 'unavailable',
|
|
161
|
+
source: 'hosted',
|
|
162
|
+
httpStatus: response.status,
|
|
163
|
+
reason: 'hosted_monitor_http_error',
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
const report = await response.json();
|
|
167
|
+
return {
|
|
168
|
+
...report,
|
|
169
|
+
source: 'hosted',
|
|
170
|
+
};
|
|
171
|
+
} catch (error) {
|
|
172
|
+
return {
|
|
173
|
+
generatedAt: new Date().toISOString(),
|
|
174
|
+
verdict: 'unavailable',
|
|
175
|
+
source: 'hosted',
|
|
176
|
+
reason: error?.name === 'AbortError' ? 'hosted_monitor_timeout' : 'hosted_monitor_request_failed',
|
|
177
|
+
};
|
|
178
|
+
} finally {
|
|
179
|
+
clearTimeout(timeout);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function buildAgentOutcomeMonitorSchedule(options = {}) {
|
|
184
|
+
const workingDirectory = path.resolve(options.workingDirectory || process.cwd());
|
|
185
|
+
const outputPath = path.resolve(
|
|
186
|
+
options.outputPath
|
|
187
|
+
|| path.join(os.homedir(), '.thumbgate', 'reports', 'agent-outcome-monitor.json'),
|
|
188
|
+
);
|
|
189
|
+
const args = [
|
|
190
|
+
__filename,
|
|
191
|
+
'--hosted',
|
|
192
|
+
`--output=${outputPath}`,
|
|
193
|
+
];
|
|
194
|
+
if (options.baseUrl) args.push(`--base-url=${options.baseUrl}`);
|
|
195
|
+
const command = [
|
|
196
|
+
'const { spawnSync } = require(\'node:child_process\');',
|
|
197
|
+
`const result = spawnSync(process.execPath, ${JSON.stringify(args)}, {`,
|
|
198
|
+
` cwd: ${JSON.stringify(workingDirectory)},`,
|
|
199
|
+
' env: process.env,',
|
|
200
|
+
' stdio: \'inherit\',',
|
|
201
|
+
'});',
|
|
202
|
+
'if (result.error) throw result.error;',
|
|
203
|
+
'process.exit(typeof result.status === \'number\' ? result.status : 1);',
|
|
204
|
+
].join(' ');
|
|
205
|
+
return {
|
|
206
|
+
id: DEFAULT_SCHEDULE_ID,
|
|
207
|
+
name: 'ThumbGate Agent Outcome Monitor',
|
|
208
|
+
description: 'Checks hosted task outcomes daily against fail-closed production thresholds.',
|
|
209
|
+
schedule: options.schedule || 'daily 10:17',
|
|
210
|
+
command,
|
|
211
|
+
workingDirectory,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function installAgentOutcomeMonitorSchedule(options, manager) {
|
|
216
|
+
const scheduleManager = manager || require('./schedule-manager');
|
|
217
|
+
return scheduleManager.createSchedule(buildAgentOutcomeMonitorSchedule(options || {}));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function main(argv = process.argv.slice(2)) {
|
|
221
|
+
const options = parseArgs(argv);
|
|
222
|
+
if (options.installSchedule) {
|
|
223
|
+
const result = installAgentOutcomeMonitorSchedule(options);
|
|
224
|
+
console.log(JSON.stringify(result, null, 2));
|
|
225
|
+
process.exitCode = result.success ? 0 : 1;
|
|
226
|
+
return result;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const report = options.hosted
|
|
230
|
+
? await fetchHostedMonitor(options)
|
|
231
|
+
: monitorTaskOutcomes(
|
|
232
|
+
readTaskOutcomes({ inputPath: options.inputPath }),
|
|
233
|
+
{ thresholdsPath: options.thresholdsPath },
|
|
234
|
+
);
|
|
235
|
+
if (options.outputPath) {
|
|
236
|
+
const outputPath = path.resolve(options.outputPath);
|
|
237
|
+
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
238
|
+
fs.writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
239
|
+
}
|
|
240
|
+
console.log(JSON.stringify(report, null, 2));
|
|
241
|
+
process.exitCode = report.verdict === 'healthy' ? 0 : 1;
|
|
242
|
+
return report;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
if (isCliInvocation()) {
|
|
246
|
+
main().catch((error) => {
|
|
247
|
+
console.error(`Agent outcome monitor failed: ${error.message}`);
|
|
248
|
+
process.exitCode = 1;
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
module.exports = {
|
|
253
|
+
buildAgentOutcomeMonitorSchedule,
|
|
254
|
+
fetchHostedMonitor,
|
|
255
|
+
flattenMetricValues,
|
|
256
|
+
installAgentOutcomeMonitorSchedule,
|
|
257
|
+
main,
|
|
258
|
+
monitorTaskOutcomes,
|
|
259
|
+
parseArgs,
|
|
260
|
+
passesRule,
|
|
261
|
+
};
|
|
@@ -107,10 +107,6 @@ function redactTraceText(value, maxLength = MAX_TEXT) {
|
|
|
107
107
|
return text.length > maxLength ? `${text.slice(0, maxLength - 1)}…` : text;
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
-
function hashText(value) {
|
|
111
|
-
return crypto.createHash('sha256').update(String(value || '')).digest('hex').slice(0, 16);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
110
|
function extractMessages(record = {}) {
|
|
115
111
|
const candidates = [
|
|
116
112
|
record.steps,
|
|
@@ -135,7 +131,7 @@ function normalizeAgentTraceRecord(record = {}, options = {}) {
|
|
|
135
131
|
const messages = extractMessages(record);
|
|
136
132
|
const steps = messages.map((message, index) => normalizeStep(message, index)).filter(Boolean);
|
|
137
133
|
const taskType = options.taskType || record.taskType || inferTaskType(record, steps);
|
|
138
|
-
const traceId = record.traceId || record.id || record.uuid || `trace_${
|
|
134
|
+
const traceId = record.traceId || record.id || record.uuid || `trace_${crypto.randomUUID()}`;
|
|
139
135
|
const outcome = normalizeOutcome(record);
|
|
140
136
|
|
|
141
137
|
return {
|
|
@@ -171,11 +167,11 @@ function normalizeStep(message = {}, index = 0) {
|
|
|
171
167
|
role,
|
|
172
168
|
eventType,
|
|
173
169
|
text: eventType === 'reasoning' ? '[REDACTED_REASONING_TRACE]' : redacted,
|
|
174
|
-
textHash:
|
|
170
|
+
textHash: null,
|
|
175
171
|
reasoning: reasoningRaw ? {
|
|
176
172
|
present: true,
|
|
177
173
|
charCount: String(reasoningRaw).length,
|
|
178
|
-
hash:
|
|
174
|
+
hash: null,
|
|
179
175
|
} : null,
|
|
180
176
|
toolCalls,
|
|
181
177
|
error: detectError(redacted, message),
|
|
@@ -191,7 +187,10 @@ function extractToolCalls(message = {}, content = '') {
|
|
|
191
187
|
const fn = call.function || call;
|
|
192
188
|
calls.push({
|
|
193
189
|
name: String(fn.name || call.name || call.tool || 'unknown'),
|
|
194
|
-
|
|
190
|
+
// Deliberately do not persist a deterministic argument fingerprint.
|
|
191
|
+
// Low-entropy values (flags, small IDs, booleans) can be recovered by
|
|
192
|
+
// enumerating candidate inputs even when only a truncated hash is stored.
|
|
193
|
+
argumentsHash: null,
|
|
195
194
|
});
|
|
196
195
|
}
|
|
197
196
|
|
|
@@ -584,7 +583,7 @@ function formatTraceAnalyticsReport(report = {}) {
|
|
|
584
583
|
lines.push(`- ${candidate.gateId}: ${candidate.recommendation}`);
|
|
585
584
|
}
|
|
586
585
|
if (!report.gateCandidates?.length) lines.push('- None: trace shapes are currently healthy.');
|
|
587
|
-
lines.push('', 'Privacy: raw hidden reasoning
|
|
586
|
+
lines.push('', 'Privacy: raw hidden reasoning and deterministic content fingerprints are not stored; only event labels and redacted observable text are retained.', '');
|
|
588
587
|
return `${lines.join('\n')}\n`;
|
|
589
588
|
}
|
|
590
589
|
|
|
@@ -9,6 +9,7 @@ const { runVerificationLoop } = require('./verification-loop');
|
|
|
9
9
|
const { createExperiment } = require('./experiment-tracker');
|
|
10
10
|
const { recommendEvolutionTarget } = require('./workspace-evolver');
|
|
11
11
|
const { ensureDir } = require('./fs-utils');
|
|
12
|
+
const { recordTaskOutcome } = require('./task-outcomes');
|
|
12
13
|
|
|
13
14
|
const JOB_LOG_FILENAME = 'job-log.jsonl';
|
|
14
15
|
const JOB_CONTROL_FILENAME = 'job-control.json';
|
|
@@ -306,6 +307,107 @@ function appendJobLog(result) {
|
|
|
306
307
|
appendJSONL(getJobRuntimePaths(result.jobId).logPath, result);
|
|
307
308
|
}
|
|
308
309
|
|
|
310
|
+
function taskOutcomeStatus(result, verificationPassed) {
|
|
311
|
+
if (verificationPassed) return 'completed';
|
|
312
|
+
if (result.status === 'failed' || result.status === 'cancelled') return 'failed';
|
|
313
|
+
return 'partial';
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function attachTaskOutcome(result, state, options = {}) {
|
|
317
|
+
const verification = result.phases?.verification;
|
|
318
|
+
const stageHistory = Array.isArray(state.stageHistory) ? state.stageHistory : [];
|
|
319
|
+
const failed = result.status === 'failed' || result.status === 'cancelled';
|
|
320
|
+
const verificationPassed = verification?.accepted === true;
|
|
321
|
+
const verificationPerformed = Boolean(verification);
|
|
322
|
+
const status = taskOutcomeStatus(result, verificationPassed);
|
|
323
|
+
const evidence = [];
|
|
324
|
+
if (verificationPerformed) {
|
|
325
|
+
evidence.push(`verification score ${verification.score}; attempts ${verification.attempts}`);
|
|
326
|
+
}
|
|
327
|
+
if (state.lastError?.message) evidence.push(`execution error: ${state.lastError.message}`);
|
|
328
|
+
|
|
329
|
+
const outcome = recordTaskOutcome({
|
|
330
|
+
taskId: result.jobId,
|
|
331
|
+
taskType: 'async-job',
|
|
332
|
+
goal: state.jobSpec?.context || `Execute managed job ${result.jobId}`,
|
|
333
|
+
expectedOutcome: 'Complete all stages and pass post-run verification',
|
|
334
|
+
status,
|
|
335
|
+
verification: {
|
|
336
|
+
performed: verificationPerformed,
|
|
337
|
+
passed: verificationPassed,
|
|
338
|
+
verifier: verificationPerformed ? 'verification-loop' : undefined,
|
|
339
|
+
method: verificationPerformed ? 'prevention-rule verification' : undefined,
|
|
340
|
+
evidence,
|
|
341
|
+
unsupportedClaims: 0,
|
|
342
|
+
},
|
|
343
|
+
toolCalls: stageHistory.map((stage) => ({
|
|
344
|
+
name: stage.name,
|
|
345
|
+
contractValid: true,
|
|
346
|
+
allowed: true,
|
|
347
|
+
succeeded: true,
|
|
348
|
+
attempts: 1,
|
|
349
|
+
latencyMs: 0,
|
|
350
|
+
costUsd: 0,
|
|
351
|
+
sideEffect: false,
|
|
352
|
+
duplicateSideEffect: false,
|
|
353
|
+
})),
|
|
354
|
+
policy: {
|
|
355
|
+
violations: 0,
|
|
356
|
+
unsafeEscapes: 0,
|
|
357
|
+
falseBlocks: 0,
|
|
358
|
+
},
|
|
359
|
+
failure: failed ? {
|
|
360
|
+
category: state.lastError?.code || result.status,
|
|
361
|
+
recovered: false,
|
|
362
|
+
repeated: false,
|
|
363
|
+
rolledBack: false,
|
|
364
|
+
} : undefined,
|
|
365
|
+
efficiency: {
|
|
366
|
+
latencyMs: result.durationMs,
|
|
367
|
+
costUsd: 0,
|
|
368
|
+
firstAttempt: verification?.attempts === 1,
|
|
369
|
+
},
|
|
370
|
+
traceId: result.jobId,
|
|
371
|
+
idempotencyKey: `${result.jobId}:${result.status}:${state.endedAt || state.updatedAt}`,
|
|
372
|
+
metadata: {
|
|
373
|
+
source: 'async-job-runner',
|
|
374
|
+
completedStages: stageHistory.length,
|
|
375
|
+
},
|
|
376
|
+
}, options);
|
|
377
|
+
result.taskOutcome = outcome.receipt;
|
|
378
|
+
return result;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function verificationFeedbackContext(job, verification) {
|
|
382
|
+
if (!verification) {
|
|
383
|
+
return `Job ${job.id} completed without post-run verification`;
|
|
384
|
+
}
|
|
385
|
+
if (verification.accepted) {
|
|
386
|
+
return `Job ${job.id} passed verification after ${verification.attempts} attempt(s)`;
|
|
387
|
+
}
|
|
388
|
+
const violations = verification.finalVerification?.violations || [];
|
|
389
|
+
const patterns = violations.map((violation) => violation.pattern).join('; ');
|
|
390
|
+
return `Job ${job.id} failed verification after ${verification.attempts} attempt(s): ${patterns}`;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
function verificationFeedbackFields(verification) {
|
|
394
|
+
if (!verification) {
|
|
395
|
+
return {
|
|
396
|
+
whatWentWrong: 'Post-run verification was skipped, so task success is unverified',
|
|
397
|
+
whatToChange: 'Run standard verification before recording a completed task',
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
if (verification.accepted) {
|
|
401
|
+
return {
|
|
402
|
+
whatWorked: 'Verification loop accepted output',
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
return {
|
|
406
|
+
whatWentWrong: `Failed ${verification.attempts} verification attempts`,
|
|
407
|
+
whatToChange: 'Improve output to avoid known mistake patterns',
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
309
411
|
function readJobLog(limit) {
|
|
310
412
|
const entries = readJSONL(getJobRuntimePaths().logPath);
|
|
311
413
|
return limit ? entries.slice(-limit) : entries;
|
|
@@ -794,6 +896,7 @@ function executeJob(job, options = {}) {
|
|
|
794
896
|
feedback,
|
|
795
897
|
improvementExperiment,
|
|
796
898
|
});
|
|
899
|
+
attachTaskOutcome(result, failedState);
|
|
797
900
|
appendJobLog(result);
|
|
798
901
|
return result;
|
|
799
902
|
}
|
|
@@ -820,19 +923,9 @@ function executeJob(job, options = {}) {
|
|
|
820
923
|
const feedback = normalizedJob.recordFeedback === false
|
|
821
924
|
? null
|
|
822
925
|
: captureFeedback({
|
|
823
|
-
signal:
|
|
824
|
-
context:
|
|
825
|
-
|
|
826
|
-
: verification.accepted
|
|
827
|
-
? `Job ${normalizedJob.id} passed verification after ${verification.attempts} attempt(s)`
|
|
828
|
-
: `Job ${normalizedJob.id} failed verification after ${verification.attempts} attempt(s): ${(verification.finalVerification.violations || []).map((violation) => violation.pattern).join('; ')}`,
|
|
829
|
-
whatWorked: !verification
|
|
830
|
-
? 'Operational job completed successfully'
|
|
831
|
-
: verification.accepted
|
|
832
|
-
? 'Verification loop accepted output'
|
|
833
|
-
: undefined,
|
|
834
|
-
whatWentWrong: verification && !verification.accepted ? `Failed ${verification.attempts} verification attempts` : undefined,
|
|
835
|
-
whatToChange: verification && !verification.accepted ? 'Improve output to avoid known mistake patterns' : undefined,
|
|
926
|
+
signal: verification?.accepted ? 'up' : 'down',
|
|
927
|
+
context: verificationFeedbackContext(normalizedJob, verification),
|
|
928
|
+
...verificationFeedbackFields(verification),
|
|
836
929
|
tags: !verification
|
|
837
930
|
? [...normalizedJob.tags, 'async-job-runner', 'verification-skipped']
|
|
838
931
|
: [...normalizedJob.tags, 'verification-loop'],
|
|
@@ -863,6 +956,7 @@ function executeJob(job, options = {}) {
|
|
|
863
956
|
feedback,
|
|
864
957
|
improvementExperiment,
|
|
865
958
|
});
|
|
959
|
+
attachTaskOutcome(result, terminalState);
|
|
866
960
|
appendJobLog(result);
|
|
867
961
|
return result;
|
|
868
962
|
}
|