thumbgate 1.28.4 → 1.29.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/.claude/commands/dashboard.md +11 -1
  2. package/.claude/commands/thumbgate-dashboard.md +23 -8
  3. package/.claude-plugin/plugin.json +1 -1
  4. package/.well-known/llms.txt +18 -10
  5. package/.well-known/mcp/server-card.json +1 -1
  6. package/README.md +66 -3
  7. package/adapters/claude/.mcp.json +2 -2
  8. package/adapters/forge/forge.yaml +3 -3
  9. package/adapters/mcp/server-stdio.js +88 -2
  10. package/adapters/opencode/opencode.json +1 -1
  11. package/bin/cli.js +8 -8
  12. package/bin/postinstall.js +4 -13
  13. package/commands/dashboard.md +11 -1
  14. package/commands/thumbgate-dashboard.md +23 -8
  15. package/config/agent-outcome-monitor-thresholds.json +63 -0
  16. package/config/evals/agent-outcomes-baseline.json +17 -0
  17. package/config/evals/agent-outcomes-golden.json +412 -0
  18. package/config/evals/prompt-eval-baseline.json +23 -0
  19. package/config/github-about.json +5 -4
  20. package/config/post-deploy-marketing-pages.json +6 -6
  21. package/config/schemas/task-outcome-receipt.schema.json +296 -0
  22. package/docs/integrations/grafana/README.md +109 -0
  23. package/docs/integrations/grafana/thumbgate-revenue-evidence-dashboard.json +1930 -0
  24. package/openapi/openapi.yaml +475 -5
  25. package/package.json +75 -22
  26. package/public/agent-manager.html +10 -11
  27. package/public/agents-cost-savings.html +2 -2
  28. package/public/assets/brand/thumbgate-logo-transparent.svg +6 -11
  29. package/public/assets/brand/thumbgate-mark-inline-v3.svg +11 -10
  30. package/public/assets/brand/thumbgate-mark.svg +10 -11
  31. package/public/blog/inside-your-boundary.html +114 -0
  32. package/public/blog/process-over-outcome-gates.html +119 -0
  33. package/public/blog.html +296 -402
  34. package/public/brand/thumbgate-mark.svg +5 -9
  35. package/public/codex-enterprise.html +2 -2
  36. package/public/compare.html +12 -3
  37. package/public/diagnostic.html +79 -29
  38. package/public/guide.html +4 -4
  39. package/public/index.html +1090 -2098
  40. package/public/install.html +3 -3
  41. package/public/js/buyer-intent.js +33 -18
  42. package/public/numbers.html +2 -2
  43. package/public/pricing.html +268 -408
  44. package/public/pro.html +4 -4
  45. package/scripts/agent-outcome-eval.js +130 -0
  46. package/scripts/agent-outcome-monitor.js +261 -0
  47. package/scripts/agent-reasoning-traces.js +8 -9
  48. package/scripts/async-job-runner.js +107 -13
  49. package/scripts/billing.js +456 -126
  50. package/scripts/buyer-paths.js +102 -0
  51. package/scripts/cli-feedback.js +2 -2
  52. package/scripts/commercial-offer.js +18 -10
  53. package/scripts/durability/step.js +121 -12
  54. package/scripts/external-customer-audit.js +881 -0
  55. package/scripts/feedback-loop.js +26 -0
  56. package/scripts/gates-engine.js +554 -19
  57. package/scripts/grafana-revenue-evidence.js +856 -0
  58. package/scripts/human-escalation.js +265 -0
  59. package/scripts/hybrid-feedback-context.js +93 -50
  60. package/scripts/jsonl-window.js +89 -0
  61. package/scripts/judge-reward-function.js +30 -18
  62. package/scripts/lesson-embedding-index.js +3 -7
  63. package/scripts/meta-agent-loop.js +20 -2
  64. package/scripts/observability-env.js +139 -0
  65. package/scripts/observability-setup.js +55 -0
  66. package/scripts/plausible-domain-config.js +4 -0
  67. package/scripts/prompt-eval.js +81 -4
  68. package/scripts/provider-live-evidence.js +1290 -0
  69. package/scripts/provider-payment-reconciler.js +442 -0
  70. package/scripts/provider-revenue-evidence.js +249 -0
  71. package/scripts/rate-limiter.js +1 -5
  72. package/scripts/revenue-action-eligibility.js +414 -0
  73. package/scripts/revenue-evidence-remediation.js +694 -0
  74. package/scripts/revenue-offer-system.js +709 -0
  75. package/scripts/sales-pipeline.js +1117 -0
  76. package/scripts/schedule-manager.js +249 -0
  77. package/scripts/seo-gsd.js +8 -4
  78. package/scripts/stripe-credentials.js +37 -0
  79. package/scripts/stripe-revenue-catalog-audit.js +363 -0
  80. package/scripts/stripe-revenue-catalog.js +164 -0
  81. package/scripts/task-outcomes.js +425 -0
  82. package/scripts/telemetry-analytics.js +23 -3
  83. package/scripts/tool-contract-validator.js +287 -59
  84. package/scripts/tool-registry.js +143 -0
  85. package/scripts/vector-store.js +83 -7
  86. package/scripts/workflow-intake-queue.js +483 -0
  87. package/src/api/server.js +647 -118
@@ -0,0 +1,164 @@
1
+ 'use strict';
2
+
3
+ const STRIPE_REVENUE_CATALOG_VERSION = 'thumbgate-stripe-revenue-catalog-v1';
4
+
5
+ // These immutable Stripe price/product pairs are the reviewed ThumbGate
6
+ // buyer rails. Product names are intentionally not evidence: the live $499
7
+ // diagnostic uses a generic product name and that same product also has a
8
+ // separate $999 price. Exact price + product + commercial terms prevent both
9
+ // false negatives and name/product lookalike attribution.
10
+ const DEFAULT_STRIPE_REVENUE_CATALOG = Object.freeze([
11
+ Object.freeze({
12
+ offerId: 'pro_monthly',
13
+ priceId: 'price_1THQY7GGBpd520QYHoS7RG0J',
14
+ productId: 'prod_UE7SR5NFBkumEp',
15
+ unitAmountCents: 1900,
16
+ currency: 'usd',
17
+ cadence: 'month',
18
+ intervalCount: 1,
19
+ status: 'current',
20
+ expectedPriceActive: true,
21
+ expectedProductActive: true,
22
+ }),
23
+ Object.freeze({
24
+ offerId: 'pro_annual',
25
+ priceId: 'price_1THQZ7GGBpd520QYxzDRnxhB',
26
+ productId: 'prod_UE7SR5NFBkumEp',
27
+ unitAmountCents: 14900,
28
+ currency: 'usd',
29
+ cadence: 'year',
30
+ intervalCount: 1,
31
+ status: 'current',
32
+ expectedPriceActive: true,
33
+ expectedProductActive: true,
34
+ }),
35
+ Object.freeze({
36
+ offerId: 'workflow_hardening_diagnostic',
37
+ priceId: 'price_1TsO6kGGBpd520QYbbEgThb3',
38
+ productId: 'prod_Us8Nf20z0bOb1g',
39
+ unitAmountCents: 49900,
40
+ currency: 'usd',
41
+ cadence: 'one_time',
42
+ intervalCount: null,
43
+ status: 'current',
44
+ expectedPriceActive: true,
45
+ expectedProductActive: true,
46
+ }),
47
+ Object.freeze({
48
+ offerId: 'team_monthly_legacy',
49
+ priceId: 'price_1TMIagGGBpd520QY1fUOawZt',
50
+ productId: 'prod_UIxPkrYI2OvDmW',
51
+ unitAmountCents: 4900,
52
+ currency: 'usd',
53
+ cadence: 'month',
54
+ intervalCount: 1,
55
+ status: 'retired_existing_contracts_only',
56
+ expectedPriceActive: true,
57
+ expectedProductActive: false,
58
+ }),
59
+ ]);
60
+
61
+ function normalizeId(value) {
62
+ return String(value || '').trim();
63
+ }
64
+
65
+ function normalizeCurrency(value) {
66
+ return String(value || '').trim().toLowerCase();
67
+ }
68
+
69
+ function normalizePositiveInteger(value) {
70
+ const number = Number(value);
71
+ return Number.isSafeInteger(number) && number > 0 ? number : null;
72
+ }
73
+
74
+ function inspectStripePrice(price = {}) {
75
+ const product = price?.product;
76
+ const productId = typeof product === 'string'
77
+ ? normalizeId(product)
78
+ : normalizeId(product?.id);
79
+ const recurring = price?.recurring || null;
80
+ return {
81
+ priceId: normalizeId(price?.id),
82
+ productId,
83
+ unitAmountCents: normalizePositiveInteger(price?.unit_amount),
84
+ currency: normalizeCurrency(price?.currency),
85
+ cadence: recurring ? normalizeId(recurring.interval).toLowerCase() : 'one_time',
86
+ intervalCount: recurring ? normalizePositiveInteger(recurring.interval_count || 1) : null,
87
+ };
88
+ }
89
+
90
+ function validateStripeRevenueCatalog(catalog = DEFAULT_STRIPE_REVENUE_CATALOG) {
91
+ if (!Array.isArray(catalog) || catalog.length === 0) {
92
+ return { ok: false, gap: 'Stripe revenue catalog must contain at least one exact offer identity.' };
93
+ }
94
+ const seenPrices = new Set();
95
+ for (const entry of catalog) {
96
+ const priceId = normalizeId(entry?.priceId);
97
+ const productId = normalizeId(entry?.productId);
98
+ const amount = normalizePositiveInteger(entry?.unitAmountCents);
99
+ const currency = normalizeCurrency(entry?.currency);
100
+ const cadence = normalizeId(entry?.cadence).toLowerCase();
101
+ const intervalCount = cadence === 'one_time'
102
+ ? null
103
+ : normalizePositiveInteger(entry?.intervalCount || 1);
104
+ if (!normalizeId(entry?.offerId) || !/^price_[A-Za-z0-9_]+$/.test(priceId)
105
+ || !/^prod_[A-Za-z0-9_]+$/.test(productId) || !amount
106
+ || !/^[a-z]{3}$/.test(currency)
107
+ || !['one_time', 'day', 'week', 'month', 'year'].includes(cadence)
108
+ || (cadence !== 'one_time' && !intervalCount)
109
+ || typeof entry?.expectedPriceActive !== 'boolean'
110
+ || typeof entry?.expectedProductActive !== 'boolean') {
111
+ return { ok: false, gap: `Invalid Stripe revenue catalog entry for ${normalizeId(entry?.offerId) || 'unknown offer'}.` };
112
+ }
113
+ if (seenPrices.has(priceId)) {
114
+ return { ok: false, gap: `Duplicate Stripe revenue catalog price: ${priceId}.` };
115
+ }
116
+ seenPrices.add(priceId);
117
+ }
118
+ return { ok: true, gap: null };
119
+ }
120
+
121
+ function matchStripeRevenueCatalogPrice(price = {}, catalog = DEFAULT_STRIPE_REVENUE_CATALOG) {
122
+ const catalogValidation = validateStripeRevenueCatalog(catalog);
123
+ if (!catalogValidation.ok) {
124
+ return { matched: false, complete: false, reason: 'catalog_invalid', gap: catalogValidation.gap };
125
+ }
126
+ const observed = inspectStripePrice(price);
127
+ if (!observed.priceId || !observed.productId || !observed.unitAmountCents
128
+ || !observed.currency || !observed.cadence
129
+ || (observed.cadence !== 'one_time' && !observed.intervalCount)) {
130
+ return { matched: false, complete: false, reason: 'price_identity_incomplete', observed };
131
+ }
132
+ const entry = catalog.find((candidate) => candidate.priceId === observed.priceId);
133
+ if (!entry) {
134
+ return { matched: false, complete: true, reason: 'price_not_in_catalog', observed };
135
+ }
136
+ const expected = {
137
+ priceId: entry.priceId,
138
+ productId: entry.productId,
139
+ unitAmountCents: entry.unitAmountCents,
140
+ currency: normalizeCurrency(entry.currency),
141
+ cadence: normalizeId(entry.cadence).toLowerCase(),
142
+ intervalCount: entry.cadence === 'one_time' ? null : Number(entry.intervalCount || 1),
143
+ };
144
+ const matched = Object.keys(expected).every((key) => observed[key] === expected[key]);
145
+ return {
146
+ matched,
147
+ complete: true,
148
+ reason: matched ? 'exact_catalog_match' : 'catalog_terms_mismatch',
149
+ offerId: matched ? entry.offerId : null,
150
+ status: matched ? entry.status : null,
151
+ observed,
152
+ };
153
+ }
154
+
155
+ const defaultCatalogValidation = validateStripeRevenueCatalog(DEFAULT_STRIPE_REVENUE_CATALOG);
156
+ if (!defaultCatalogValidation.ok) throw new Error(defaultCatalogValidation.gap);
157
+
158
+ module.exports = {
159
+ STRIPE_REVENUE_CATALOG_VERSION,
160
+ DEFAULT_STRIPE_REVENUE_CATALOG,
161
+ inspectStripePrice,
162
+ validateStripeRevenueCatalog,
163
+ matchStripeRevenueCatalogPrice,
164
+ };
@@ -0,0 +1,425 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ /**
5
+ * Task Outcomes — evidence-backed proof that an agent is working.
6
+ *
7
+ * A response, tool call, or demo is not a success event. This ledger stores
8
+ * task-level outcomes and computes transparent component metrics without
9
+ * hiding weak behavior behind a single composite score.
10
+ */
11
+
12
+ const crypto = require('node:crypto');
13
+ const fs = require('node:fs');
14
+ const path = require('node:path');
15
+ const { getFeedbackPaths } = require('./feedback-paths');
16
+ const { validateToolContract } = require('./tool-contract-validator');
17
+
18
+ const OUTCOMES_FILE = 'task-outcome-receipts.jsonl';
19
+ const SCHEMA_PATH = path.join(__dirname, '..', 'config', 'schemas', 'task-outcome-receipt.schema.json');
20
+ const TASK_OUTCOME_SCHEMA = JSON.parse(fs.readFileSync(SCHEMA_PATH, 'utf8'));
21
+
22
+ function getTaskOutcomesPath(options = {}) {
23
+ return path.join(getFeedbackPaths(options).FEEDBACK_DIR, OUTCOMES_FILE);
24
+ }
25
+
26
+ function normalizeTaskOutcome(input = {}, now = new Date()) {
27
+ const verification = input.verification || {};
28
+ const policy = input.policy || {};
29
+ const efficiency = input.efficiency || {};
30
+ const toolCalls = Array.isArray(input.toolCalls) ? input.toolCalls : [];
31
+ const receipt = {
32
+ taskId: cleanString(input.taskId),
33
+ taskType: cleanString(input.taskType || 'general'),
34
+ goal: cleanString(input.goal),
35
+ expectedOutcome: optionalString(input.expectedOutcome),
36
+ status: input.status || 'failed',
37
+ verification: {
38
+ performed: verification.performed === true,
39
+ passed: verification.passed === true,
40
+ verifier: optionalString(verification.verifier),
41
+ method: optionalString(verification.method),
42
+ evidence: cleanStringArray(verification.evidence),
43
+ unsupportedClaims: nonNegativeInteger(verification.unsupportedClaims),
44
+ },
45
+ toolCalls: toolCalls.map(normalizeToolCall),
46
+ policy: {
47
+ violations: nonNegativeInteger(policy.violations),
48
+ unsafeEscapes: nonNegativeInteger(policy.unsafeEscapes),
49
+ falseBlocks: nonNegativeInteger(policy.falseBlocks),
50
+ },
51
+ failure: normalizeFailure(input.failure),
52
+ escalation: normalizeEscalation(input.escalation),
53
+ efficiency: {
54
+ latencyMs: nonNegativeNumber(efficiency.latencyMs),
55
+ costUsd: nonNegativeNumber(efficiency.costUsd),
56
+ firstAttempt: efficiency.firstAttempt === true,
57
+ },
58
+ businessOutcome: normalizeBusinessOutcome(input.businessOutcome),
59
+ traceId: optionalString(input.traceId),
60
+ idempotencyKey: optionalString(input.idempotencyKey || input.taskId),
61
+ versions: normalizeVersions(input.versions),
62
+ metadata: isPlainObject(input.metadata) ? input.metadata : {},
63
+ recordedAt: input.recordedAt || now.toISOString(),
64
+ };
65
+
66
+ removeUndefined(receipt);
67
+ const verdict = evaluateWorkingVerdict(receipt);
68
+ receipt.working = verdict.working;
69
+ receipt.workingReasons = verdict.reasons;
70
+ receipt.receiptHash = hashReceipt(receipt);
71
+ return receipt;
72
+ }
73
+
74
+ function evaluateWorkingVerdict(receipt = {}) {
75
+ const reasons = [];
76
+ if (receipt.status !== 'completed') reasons.push(`status_${receipt.status || 'unknown'}`);
77
+ if (!receipt.verification?.performed) reasons.push('verification_not_performed');
78
+ if (!receipt.verification?.passed) reasons.push('verification_failed');
79
+ if (!receipt.verification?.evidence?.length) reasons.push('evidence_missing');
80
+ if (Number(receipt.verification?.unsupportedClaims || 0) > 0) reasons.push('unsupported_claim');
81
+ if ((receipt.toolCalls || []).some((call) => !call.contractValid)) reasons.push('tool_contract_invalid');
82
+ if ((receipt.toolCalls || []).some((call) => !call.allowed)) reasons.push('tool_policy_denied');
83
+ if ((receipt.toolCalls || []).some((call) => !call.succeeded)) reasons.push('tool_call_failed');
84
+ if ((receipt.toolCalls || []).some((call) => call.duplicateSideEffect)) reasons.push('duplicate_side_effect');
85
+ if (Number(receipt.policy?.violations || 0) > 0) reasons.push('policy_violation');
86
+ if (Number(receipt.policy?.unsafeEscapes || 0) > 0) reasons.push('unsafe_escape');
87
+ if (Number(receipt.policy?.falseBlocks || 0) > 0) reasons.push('safe_false_block');
88
+ if (receipt.escalation?.required && receipt.escalation?.correct !== true) reasons.push('incorrect_escalation');
89
+ return { working: reasons.length === 0, reasons };
90
+ }
91
+
92
+ function recordTaskOutcome(input = {}, options = {}) {
93
+ const receipt = normalizeTaskOutcome(input, options.now || new Date());
94
+ const validation = validateToolContract(TASK_OUTCOME_SCHEMA, receipt);
95
+ if (!validation.valid) {
96
+ const error = new Error(`Invalid task outcome receipt: ${validation.errors.join('; ')}`);
97
+ error.code = 'THUMBGATE_TASK_OUTCOME_INVALID';
98
+ error.validationErrors = validation.errors;
99
+ throw error;
100
+ }
101
+
102
+ const outcomesPath = getTaskOutcomesPath(options);
103
+ const existing = readTaskOutcomes(options);
104
+ const duplicate = existing.find((entry) => entry.idempotencyKey === receipt.idempotencyKey);
105
+ if (duplicate) {
106
+ if (duplicate.receiptHash !== receipt.receiptHash) {
107
+ const error = new Error(`Conflicting task outcome for idempotency key '${receipt.idempotencyKey}'`);
108
+ error.code = 'THUMBGATE_IDEMPOTENCY_CONFLICT';
109
+ throw error;
110
+ }
111
+ return { recorded: false, duplicate: true, receipt: duplicate };
112
+ }
113
+
114
+ fs.mkdirSync(path.dirname(outcomesPath), { recursive: true });
115
+ fs.appendFileSync(outcomesPath, `${JSON.stringify(receipt)}\n`, 'utf8');
116
+ recordOutcomeTrace(receipt, options);
117
+ return { recorded: true, duplicate: false, receipt };
118
+ }
119
+
120
+ function readTaskOutcomes(options = {}) {
121
+ const outcomesPath = options.inputPath
122
+ ? path.resolve(options.inputPath)
123
+ : getTaskOutcomesPath(options);
124
+ let raw = '';
125
+ try {
126
+ raw = fs.readFileSync(outcomesPath, 'utf8');
127
+ } catch {
128
+ return [];
129
+ }
130
+ return raw.split('\n')
131
+ .map((line) => line.trim())
132
+ .filter(Boolean)
133
+ .flatMap((line) => {
134
+ try {
135
+ return [JSON.parse(line)];
136
+ } catch {
137
+ return [];
138
+ }
139
+ });
140
+ }
141
+
142
+ function getTaskOutcome(taskId, options = {}) {
143
+ const outcomes = readTaskOutcomes(options);
144
+ for (let index = outcomes.length - 1; index >= 0; index -= 1) {
145
+ if (outcomes[index].taskId === taskId) return outcomes[index];
146
+ }
147
+ return null;
148
+ }
149
+
150
+ function calculateTaskOutcomeMetrics(outcomes = []) {
151
+ const valid = outcomes.filter((entry) => entry && typeof entry === 'object');
152
+ const toolCalls = valid.flatMap((entry) => entry.toolCalls || []);
153
+ const verifiedSuccesses = valid.filter((entry) => entry.status === 'completed' && entry.verification?.passed);
154
+ const escalationEligible = valid.filter((entry) => entry.escalation?.required);
155
+ const failures = valid.filter((entry) => entry.status === 'failed');
156
+ const latencies = valid.map((entry) => Number(entry.efficiency?.latencyMs)).filter(Number.isFinite);
157
+ const totalCostUsd = valid.reduce((sum, entry) => sum + nonNegativeNumber(entry.efficiency?.costUsd), 0);
158
+ const businessOutcomes = aggregateBusinessOutcomes(valid);
159
+
160
+ return {
161
+ generatedAt: new Date().toISOString(),
162
+ sampleSize: valid.length,
163
+ evidenceStatus: valid.length > 0 ? 'measured' : 'insufficient_evidence',
164
+ task: {
165
+ workingRate: rate(valid.filter((entry) => entry.working).length, valid.length),
166
+ verifiedCompletionRate: rate(verifiedSuccesses.length, valid.length),
167
+ evidenceBackedCompletionRate: rate(
168
+ verifiedSuccesses.filter((entry) => entry.verification?.evidence?.length > 0).length,
169
+ valid.length,
170
+ ),
171
+ unsupportedClaimRate: rate(
172
+ valid.filter((entry) => Number(entry.verification?.unsupportedClaims || 0) > 0).length,
173
+ valid.length,
174
+ ),
175
+ firstAttemptSuccessRate: rate(
176
+ verifiedSuccesses.filter((entry) => entry.efficiency?.firstAttempt).length,
177
+ valid.length,
178
+ ),
179
+ repeatedFailureRate: rate(
180
+ failures.filter((entry) => entry.failure?.repeated).length,
181
+ failures.length,
182
+ ),
183
+ recoveryRate: rate(
184
+ failures.filter((entry) => entry.failure?.recovered).length,
185
+ failures.length,
186
+ ),
187
+ rollbackRate: rate(
188
+ failures.filter((entry) => entry.failure?.rolledBack).length,
189
+ failures.length,
190
+ ),
191
+ },
192
+ tools: {
193
+ calls: toolCalls.length,
194
+ contractAccuracy: rate(toolCalls.filter((call) => call.contractValid).length, toolCalls.length),
195
+ executionSuccessRate: rate(toolCalls.filter((call) => call.succeeded).length, toolCalls.length),
196
+ duplicateSideEffectRate: rate(toolCalls.filter((call) => call.duplicateSideEffect).length, toolCalls.length),
197
+ retryRate: rate(toolCalls.filter((call) => call.attempts > 1).length, toolCalls.length),
198
+ },
199
+ safety: {
200
+ unsafeEscapeRate: rate(valid.filter((entry) => Number(entry.policy?.unsafeEscapes || 0) > 0).length, valid.length),
201
+ policyViolationRate: rate(valid.filter((entry) => Number(entry.policy?.violations || 0) > 0).length, valid.length),
202
+ safeFalseBlockRate: rate(valid.filter((entry) => Number(entry.policy?.falseBlocks || 0) > 0).length, valid.length),
203
+ },
204
+ escalation: {
205
+ eligible: escalationEligible.length,
206
+ correctEscalationRate: rate(
207
+ escalationEligible.filter((entry) => entry.escalation?.correct).length,
208
+ escalationEligible.length,
209
+ ),
210
+ },
211
+ efficiency: {
212
+ latencyP50Ms: percentile(latencies, 0.5),
213
+ latencyP95Ms: percentile(latencies, 0.95),
214
+ totalCostUsd: roundMoney(totalCostUsd),
215
+ costPerVerifiedSuccessUsd: verifiedSuccesses.length
216
+ ? roundMoney(totalCostUsd / verifiedSuccesses.length)
217
+ : null,
218
+ },
219
+ businessOutcomes,
220
+ };
221
+ }
222
+
223
+ function recordOutcomeTrace(receipt, options) {
224
+ if (options.recordTrace === false) return;
225
+ try {
226
+ const { recordReasoningTrace } = require('./agent-reasoning-traces');
227
+ const messages = [
228
+ { role: 'user', content: `intent: ${receipt.goal}` },
229
+ ...receipt.toolCalls.map((call) => ({
230
+ role: 'assistant',
231
+ content: `tool: ${call.name}`,
232
+ tool_calls: [{ function: { name: call.name, arguments: {} } }],
233
+ })),
234
+ {
235
+ role: 'tool',
236
+ content: `tool response: ${receipt.toolCalls.every((call) => call.succeeded) ? 'success' : 'failure'}`,
237
+ },
238
+ {
239
+ role: 'assistant',
240
+ content: receipt.verification.evidence.length
241
+ ? `verification evidence: ${receipt.verification.evidence.join('; ')}`
242
+ : 'verification missing',
243
+ },
244
+ ];
245
+ recordReasoningTrace({
246
+ trace_id: receipt.traceId || receipt.taskId,
247
+ task_type: receipt.taskType,
248
+ messages,
249
+ success: receipt.working,
250
+ outcome: {
251
+ success: receipt.working,
252
+ terminalState: receipt.status,
253
+ },
254
+ source: 'task-outcome-receipt',
255
+ }, options);
256
+ } catch {
257
+ // Outcome recording is authoritative; best-effort trace analytics cannot
258
+ // make a verified receipt disappear.
259
+ }
260
+ }
261
+
262
+ function normalizeToolCall(call = {}) {
263
+ const sideEffect = call.sideEffect === true;
264
+ return {
265
+ name: cleanString(call.name),
266
+ contractValid: call.contractValid === true,
267
+ allowed: call.allowed === true,
268
+ succeeded: call.succeeded === true,
269
+ attempts: Math.max(1, nonNegativeInteger(call.attempts || 1)),
270
+ latencyMs: nonNegativeNumber(call.latencyMs),
271
+ costUsd: nonNegativeNumber(call.costUsd),
272
+ sideEffect,
273
+ idempotencyKey: optionalString(call.idempotencyKey),
274
+ duplicateSideEffect: call.duplicateSideEffect === true,
275
+ };
276
+ }
277
+
278
+ function normalizeFailure(value) {
279
+ if (!value) return undefined;
280
+ return {
281
+ category: optionalString(value.category),
282
+ recovered: value.recovered === true,
283
+ repeated: value.repeated === true,
284
+ rolledBack: value.rolledBack === true,
285
+ };
286
+ }
287
+
288
+ function normalizeEscalation(value) {
289
+ if (!value) return undefined;
290
+ return {
291
+ required: value.required === true,
292
+ correct: value.correct === true,
293
+ escalationId: optionalString(value.escalationId),
294
+ };
295
+ }
296
+
297
+ function normalizeBusinessOutcome(value) {
298
+ if (!value) return undefined;
299
+ return {
300
+ kpi: cleanString(value.kpi),
301
+ value: Number(value.value),
302
+ unit: cleanString(value.unit),
303
+ };
304
+ }
305
+
306
+ function normalizeVersions(value) {
307
+ if (!value) return undefined;
308
+ return {
309
+ model: optionalString(value.model),
310
+ prompt: optionalString(value.prompt),
311
+ tools: optionalString(value.tools),
312
+ policy: optionalString(value.policy),
313
+ release: optionalString(value.release),
314
+ };
315
+ }
316
+
317
+ function aggregateBusinessOutcomes(outcomes) {
318
+ const groups = new Map();
319
+ for (const outcome of outcomes) {
320
+ const item = outcome.businessOutcome;
321
+ if (!item) continue;
322
+ const key = `${item.kpi}\0${item.unit}`;
323
+ const current = groups.get(key) || { kpi: item.kpi, unit: item.unit, value: 0, tasks: 0 };
324
+ current.value += Number(item.value || 0);
325
+ current.tasks += 1;
326
+ groups.set(key, current);
327
+ }
328
+ return Array.from(groups.values()).map((entry) => ({ ...entry, value: roundMetric(entry.value) }));
329
+ }
330
+
331
+ function hashReceipt(receipt) {
332
+ const copy = { ...receipt };
333
+ delete copy.receiptHash;
334
+ delete copy.recordedAt;
335
+ return crypto.createHash('sha256').update(stableStringify(copy)).digest('hex');
336
+ }
337
+
338
+ function stableStringify(value) {
339
+ if (!value || typeof value !== 'object') return JSON.stringify(value);
340
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
341
+ const keys = Object.keys(value).sort((left, right) => left.localeCompare(right));
342
+ const properties = keys.map((key) => [
343
+ JSON.stringify(key),
344
+ stableStringify(value[key]),
345
+ ].join(':'));
346
+ return ['{', properties.join(','), '}'].join('');
347
+ }
348
+
349
+ function percentile(values, quantile) {
350
+ if (!values.length) return null;
351
+ const sorted = [...values].sort((a, b) => a - b);
352
+ const index = Math.ceil(quantile * sorted.length) - 1;
353
+ return roundMetric(sorted[Math.max(0, index)]);
354
+ }
355
+
356
+ function rate(numerator, denominator) {
357
+ return denominator > 0 ? roundMetric(numerator / denominator) : null;
358
+ }
359
+
360
+ function cleanString(value) {
361
+ return String(value ?? '').trim();
362
+ }
363
+
364
+ function optionalString(value) {
365
+ const result = cleanString(value);
366
+ return result || undefined;
367
+ }
368
+
369
+ function cleanStringArray(value) {
370
+ return Array.isArray(value) ? value.map(cleanString).filter(Boolean) : [];
371
+ }
372
+
373
+ function nonNegativeNumber(value) {
374
+ const number = Number(value);
375
+ return Number.isFinite(number) ? Math.max(0, number) : 0;
376
+ }
377
+
378
+ function nonNegativeInteger(value) {
379
+ return Math.floor(nonNegativeNumber(value));
380
+ }
381
+
382
+ function roundMetric(value) {
383
+ return Number.isFinite(value) ? Math.round(value * 10000) / 10000 : null;
384
+ }
385
+
386
+ function roundMoney(value) {
387
+ return Number.isFinite(value) ? Math.round(value * 1000000) / 1000000 : null;
388
+ }
389
+
390
+ function isPlainObject(value) {
391
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
392
+ }
393
+
394
+ function removeUndefined(value) {
395
+ if (!value || typeof value !== 'object') return;
396
+ for (const key of Object.keys(value)) {
397
+ if (value[key] === undefined) delete value[key];
398
+ else removeUndefined(value[key]);
399
+ }
400
+ }
401
+
402
+ function isCliInvocation() {
403
+ return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
404
+ }
405
+
406
+ if (isCliInvocation()) {
407
+ const command = process.argv[2] || 'metrics';
408
+ if (command === 'metrics') {
409
+ console.log(JSON.stringify(calculateTaskOutcomeMetrics(readTaskOutcomes()), null, 2));
410
+ } else {
411
+ console.error('Usage: task-outcomes.js metrics');
412
+ process.exitCode = 1;
413
+ }
414
+ }
415
+
416
+ module.exports = {
417
+ TASK_OUTCOME_SCHEMA,
418
+ calculateTaskOutcomeMetrics,
419
+ evaluateWorkingVerdict,
420
+ getTaskOutcome,
421
+ getTaskOutcomesPath,
422
+ normalizeTaskOutcome,
423
+ readTaskOutcomes,
424
+ recordTaskOutcome,
425
+ };
@@ -19,6 +19,7 @@ const MARKETING_CLICK_EVENT_TYPES = new Set([
19
19
  'cta_click',
20
20
  'checkout_start',
21
21
  'checkout_bootstrap',
22
+ 'diagnostic_checkout_confirmed',
22
23
  'checkout_interstitial_cta_clicked',
23
24
  'chatgpt_gpt_open',
24
25
  'chatgpt_gpt_click',
@@ -31,6 +32,14 @@ const MARKETING_CLICK_EVENT_TYPES = new Set([
31
32
  'github_repo_click',
32
33
  'community_landing_redirect',
33
34
  ]);
35
+ const CHECKOUT_START_EVENT_TYPES = new Set([
36
+ 'checkout_start',
37
+ 'checkout_bootstrap',
38
+ // A diagnostic buyer has submitted a valid receipt email and the server is
39
+ // issuing the external Payment Link redirect. This is stronger than a CTA
40
+ // click and is the service equivalent of the Pro checkout bootstrap.
41
+ 'diagnostic_checkout_confirmed',
42
+ ]);
34
43
 
35
44
  function shouldIncludeLegacyTelemetry() {
36
45
  if (
@@ -91,6 +100,10 @@ function isMarketingClickEvent(eventType) {
91
100
  return MARKETING_CLICK_EVENT_TYPES.has(String(eventType || '').toLowerCase());
92
101
  }
93
102
 
103
+ function isCheckoutStartEvent(eventType) {
104
+ return CHECKOUT_START_EVENT_TYPES.has(String(eventType || '').toLowerCase());
105
+ }
106
+
94
107
  function getTelemetryPath(feedbackDir) {
95
108
  return path.join(feedbackDir, TELEMETRY_FILE_NAME);
96
109
  }
@@ -666,7 +679,7 @@ function summarizeExternalTrafficQuality(events) {
666
679
  if ((entry.page || entry.landingPath) && !pathRow.pages.includes(entry.page || entry.landingPath)) {
667
680
  pathRow.pages.push(entry.page || entry.landingPath);
668
681
  }
669
- if (eventType === 'checkout_start' || eventType === 'checkout_bootstrap') {
682
+ if (isCheckoutStartEvent(eventType)) {
670
683
  pathRow.checkoutStarts += 1;
671
684
  }
672
685
  visitorPaths.set(visitorKey, pathRow);
@@ -683,7 +696,7 @@ function summarizeExternalTrafficQuality(events) {
683
696
  externalCtaClicks += 1;
684
697
  }
685
698
 
686
- if (eventType === 'checkout_start' || eventType === 'checkout_bootstrap') {
699
+ if (isCheckoutStartEvent(eventType)) {
687
700
  externalCheckoutStarts += 1;
688
701
  incrementCounter(checkoutStartsBySource, entry.source);
689
702
  incrementCounter(checkoutStartsByTrafficChannel, entry.trafficChannel);
@@ -814,6 +827,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
814
827
  let ctaClicks = 0;
815
828
  let ctaImpressions = 0;
816
829
  let checkoutStarts = 0;
830
+ let diagnosticCheckoutStarts = 0;
817
831
  let checkoutInterstitialViews = 0;
818
832
  let checkoutBotDeflections = 0;
819
833
  let checkoutInterstitialClicks = 0;
@@ -922,8 +936,11 @@ function getTelemetrySummary(feedbackDir, options = {}) {
922
936
  }
923
937
  }
924
938
 
925
- if ((entry.eventType || entry.event) === 'checkout_start' || (entry.eventType || entry.event) === 'checkout_bootstrap') {
939
+ if (isCheckoutStartEvent(entry.eventType || entry.event)) {
926
940
  checkoutStarts += 1;
941
+ if ((entry.eventType || entry.event) === 'diagnostic_checkout_confirmed') {
942
+ diagnosticCheckoutStarts += 1;
943
+ }
927
944
  incrementCounter(checkoutStartsBySource, entry.source);
928
945
  incrementCounter(checkoutStartsByCampaign, entry.utmCampaign);
929
946
  incrementCounter(checkoutStartsByTrafficChannel, entry.trafficChannel);
@@ -1112,6 +1129,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
1112
1129
  gptOpens,
1113
1130
  gptActionCalls: chatgptActionCalls,
1114
1131
  checkoutStarts,
1132
+ diagnosticCheckoutStarts,
1115
1133
  checkoutInterstitialViews,
1116
1134
  checkoutInterstitialClicks,
1117
1135
  trialEmails,
@@ -1133,6 +1151,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
1133
1151
  pageViews,
1134
1152
  ctaClicks,
1135
1153
  checkoutStarts,
1154
+ diagnosticCheckoutStarts,
1136
1155
  checkoutInterstitialViews,
1137
1156
  checkoutBotDeflections,
1138
1157
  checkoutInterstitialClicks,
@@ -1317,6 +1336,7 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
1317
1336
  ctas: {
1318
1337
  totalClicks: summary.web.ctaClicks,
1319
1338
  checkoutStarts: summary.web.checkoutStarts,
1339
+ diagnosticCheckoutStarts: summary.web.diagnosticCheckoutStarts,
1320
1340
  checkoutInterstitialViews: summary.web.checkoutInterstitialViews,
1321
1341
  checkoutBotDeflections: summary.web.checkoutBotDeflections,
1322
1342
  checkoutInterstitialClicks: summary.web.checkoutInterstitialClicks,