thumbgate 1.28.4 → 1.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/dashboard.md +11 -1
- package/.claude/commands/thumbgate-dashboard.md +23 -8
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/llms.txt +18 -10
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +66 -3
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +88 -2
- package/adapters/opencode/opencode.json +1 -1
- package/bin/cli.js +8 -8
- package/bin/postinstall.js +4 -13
- package/commands/dashboard.md +11 -1
- package/commands/thumbgate-dashboard.md +23 -8
- package/config/agent-outcome-monitor-thresholds.json +63 -0
- package/config/evals/agent-outcomes-baseline.json +17 -0
- package/config/evals/agent-outcomes-golden.json +412 -0
- package/config/evals/prompt-eval-baseline.json +23 -0
- package/config/github-about.json +5 -4
- package/config/post-deploy-marketing-pages.json +6 -6
- package/config/schemas/task-outcome-receipt.schema.json +296 -0
- package/docs/integrations/grafana/README.md +109 -0
- package/docs/integrations/grafana/thumbgate-revenue-evidence-dashboard.json +1930 -0
- package/openapi/openapi.yaml +475 -5
- package/package.json +75 -22
- package/public/agent-manager.html +10 -11
- package/public/agents-cost-savings.html +2 -2
- package/public/assets/brand/thumbgate-logo-transparent.svg +6 -11
- package/public/assets/brand/thumbgate-mark-inline-v3.svg +11 -10
- package/public/assets/brand/thumbgate-mark.svg +10 -11
- package/public/blog/inside-your-boundary.html +114 -0
- package/public/blog/process-over-outcome-gates.html +119 -0
- package/public/blog.html +296 -402
- package/public/brand/thumbgate-mark.svg +5 -9
- package/public/codex-enterprise.html +2 -2
- package/public/compare.html +12 -3
- package/public/diagnostic.html +79 -29
- package/public/guide.html +4 -4
- package/public/index.html +1090 -2098
- package/public/install.html +3 -3
- package/public/js/buyer-intent.js +33 -18
- package/public/numbers.html +2 -2
- package/public/pricing.html +268 -408
- package/public/pro.html +4 -4
- package/scripts/agent-outcome-eval.js +130 -0
- package/scripts/agent-outcome-monitor.js +261 -0
- package/scripts/agent-reasoning-traces.js +8 -9
- package/scripts/async-job-runner.js +107 -13
- package/scripts/billing.js +456 -126
- package/scripts/buyer-paths.js +102 -0
- package/scripts/cli-feedback.js +2 -2
- package/scripts/commercial-offer.js +18 -10
- package/scripts/durability/step.js +121 -12
- package/scripts/external-customer-audit.js +881 -0
- package/scripts/feedback-loop.js +26 -0
- package/scripts/gates-engine.js +554 -19
- package/scripts/grafana-revenue-evidence.js +856 -0
- package/scripts/human-escalation.js +265 -0
- package/scripts/hybrid-feedback-context.js +93 -50
- package/scripts/jsonl-window.js +89 -0
- package/scripts/judge-reward-function.js +30 -18
- package/scripts/lesson-embedding-index.js +3 -7
- package/scripts/meta-agent-loop.js +20 -2
- package/scripts/observability-env.js +139 -0
- package/scripts/observability-setup.js +55 -0
- package/scripts/plausible-domain-config.js +4 -0
- package/scripts/prompt-eval.js +81 -4
- package/scripts/provider-live-evidence.js +1290 -0
- package/scripts/provider-payment-reconciler.js +442 -0
- package/scripts/provider-revenue-evidence.js +249 -0
- package/scripts/rate-limiter.js +1 -5
- package/scripts/revenue-action-eligibility.js +414 -0
- package/scripts/revenue-evidence-remediation.js +694 -0
- package/scripts/revenue-offer-system.js +709 -0
- package/scripts/sales-pipeline.js +1117 -0
- package/scripts/schedule-manager.js +249 -0
- package/scripts/seo-gsd.js +8 -4
- package/scripts/stripe-credentials.js +37 -0
- package/scripts/stripe-revenue-catalog-audit.js +363 -0
- package/scripts/stripe-revenue-catalog.js +164 -0
- package/scripts/task-outcomes.js +425 -0
- package/scripts/telemetry-analytics.js +23 -3
- package/scripts/tool-contract-validator.js +287 -59
- package/scripts/tool-registry.js +143 -0
- package/scripts/vector-store.js +83 -7
- package/scripts/workflow-intake-queue.js +483 -0
- package/src/api/server.js +647 -118
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const STRIPE_REVENUE_CATALOG_VERSION = 'thumbgate-stripe-revenue-catalog-v1';
|
|
4
|
+
|
|
5
|
+
// These immutable Stripe price/product pairs are the reviewed ThumbGate
|
|
6
|
+
// buyer rails. Product names are intentionally not evidence: the live $499
|
|
7
|
+
// diagnostic uses a generic product name and that same product also has a
|
|
8
|
+
// separate $999 price. Exact price + product + commercial terms prevent both
|
|
9
|
+
// false negatives and name/product lookalike attribution.
|
|
10
|
+
const DEFAULT_STRIPE_REVENUE_CATALOG = Object.freeze([
|
|
11
|
+
Object.freeze({
|
|
12
|
+
offerId: 'pro_monthly',
|
|
13
|
+
priceId: 'price_1THQY7GGBpd520QYHoS7RG0J',
|
|
14
|
+
productId: 'prod_UE7SR5NFBkumEp',
|
|
15
|
+
unitAmountCents: 1900,
|
|
16
|
+
currency: 'usd',
|
|
17
|
+
cadence: 'month',
|
|
18
|
+
intervalCount: 1,
|
|
19
|
+
status: 'current',
|
|
20
|
+
expectedPriceActive: true,
|
|
21
|
+
expectedProductActive: true,
|
|
22
|
+
}),
|
|
23
|
+
Object.freeze({
|
|
24
|
+
offerId: 'pro_annual',
|
|
25
|
+
priceId: 'price_1THQZ7GGBpd520QYxzDRnxhB',
|
|
26
|
+
productId: 'prod_UE7SR5NFBkumEp',
|
|
27
|
+
unitAmountCents: 14900,
|
|
28
|
+
currency: 'usd',
|
|
29
|
+
cadence: 'year',
|
|
30
|
+
intervalCount: 1,
|
|
31
|
+
status: 'current',
|
|
32
|
+
expectedPriceActive: true,
|
|
33
|
+
expectedProductActive: true,
|
|
34
|
+
}),
|
|
35
|
+
Object.freeze({
|
|
36
|
+
offerId: 'workflow_hardening_diagnostic',
|
|
37
|
+
priceId: 'price_1TsO6kGGBpd520QYbbEgThb3',
|
|
38
|
+
productId: 'prod_Us8Nf20z0bOb1g',
|
|
39
|
+
unitAmountCents: 49900,
|
|
40
|
+
currency: 'usd',
|
|
41
|
+
cadence: 'one_time',
|
|
42
|
+
intervalCount: null,
|
|
43
|
+
status: 'current',
|
|
44
|
+
expectedPriceActive: true,
|
|
45
|
+
expectedProductActive: true,
|
|
46
|
+
}),
|
|
47
|
+
Object.freeze({
|
|
48
|
+
offerId: 'team_monthly_legacy',
|
|
49
|
+
priceId: 'price_1TMIagGGBpd520QY1fUOawZt',
|
|
50
|
+
productId: 'prod_UIxPkrYI2OvDmW',
|
|
51
|
+
unitAmountCents: 4900,
|
|
52
|
+
currency: 'usd',
|
|
53
|
+
cadence: 'month',
|
|
54
|
+
intervalCount: 1,
|
|
55
|
+
status: 'retired_existing_contracts_only',
|
|
56
|
+
expectedPriceActive: true,
|
|
57
|
+
expectedProductActive: false,
|
|
58
|
+
}),
|
|
59
|
+
]);
|
|
60
|
+
|
|
61
|
+
function normalizeId(value) {
|
|
62
|
+
return String(value || '').trim();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function normalizeCurrency(value) {
|
|
66
|
+
return String(value || '').trim().toLowerCase();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function normalizePositiveInteger(value) {
|
|
70
|
+
const number = Number(value);
|
|
71
|
+
return Number.isSafeInteger(number) && number > 0 ? number : null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function inspectStripePrice(price = {}) {
|
|
75
|
+
const product = price?.product;
|
|
76
|
+
const productId = typeof product === 'string'
|
|
77
|
+
? normalizeId(product)
|
|
78
|
+
: normalizeId(product?.id);
|
|
79
|
+
const recurring = price?.recurring || null;
|
|
80
|
+
return {
|
|
81
|
+
priceId: normalizeId(price?.id),
|
|
82
|
+
productId,
|
|
83
|
+
unitAmountCents: normalizePositiveInteger(price?.unit_amount),
|
|
84
|
+
currency: normalizeCurrency(price?.currency),
|
|
85
|
+
cadence: recurring ? normalizeId(recurring.interval).toLowerCase() : 'one_time',
|
|
86
|
+
intervalCount: recurring ? normalizePositiveInteger(recurring.interval_count || 1) : null,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function validateStripeRevenueCatalog(catalog = DEFAULT_STRIPE_REVENUE_CATALOG) {
|
|
91
|
+
if (!Array.isArray(catalog) || catalog.length === 0) {
|
|
92
|
+
return { ok: false, gap: 'Stripe revenue catalog must contain at least one exact offer identity.' };
|
|
93
|
+
}
|
|
94
|
+
const seenPrices = new Set();
|
|
95
|
+
for (const entry of catalog) {
|
|
96
|
+
const priceId = normalizeId(entry?.priceId);
|
|
97
|
+
const productId = normalizeId(entry?.productId);
|
|
98
|
+
const amount = normalizePositiveInteger(entry?.unitAmountCents);
|
|
99
|
+
const currency = normalizeCurrency(entry?.currency);
|
|
100
|
+
const cadence = normalizeId(entry?.cadence).toLowerCase();
|
|
101
|
+
const intervalCount = cadence === 'one_time'
|
|
102
|
+
? null
|
|
103
|
+
: normalizePositiveInteger(entry?.intervalCount || 1);
|
|
104
|
+
if (!normalizeId(entry?.offerId) || !/^price_[A-Za-z0-9_]+$/.test(priceId)
|
|
105
|
+
|| !/^prod_[A-Za-z0-9_]+$/.test(productId) || !amount
|
|
106
|
+
|| !/^[a-z]{3}$/.test(currency)
|
|
107
|
+
|| !['one_time', 'day', 'week', 'month', 'year'].includes(cadence)
|
|
108
|
+
|| (cadence !== 'one_time' && !intervalCount)
|
|
109
|
+
|| typeof entry?.expectedPriceActive !== 'boolean'
|
|
110
|
+
|| typeof entry?.expectedProductActive !== 'boolean') {
|
|
111
|
+
return { ok: false, gap: `Invalid Stripe revenue catalog entry for ${normalizeId(entry?.offerId) || 'unknown offer'}.` };
|
|
112
|
+
}
|
|
113
|
+
if (seenPrices.has(priceId)) {
|
|
114
|
+
return { ok: false, gap: `Duplicate Stripe revenue catalog price: ${priceId}.` };
|
|
115
|
+
}
|
|
116
|
+
seenPrices.add(priceId);
|
|
117
|
+
}
|
|
118
|
+
return { ok: true, gap: null };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function matchStripeRevenueCatalogPrice(price = {}, catalog = DEFAULT_STRIPE_REVENUE_CATALOG) {
|
|
122
|
+
const catalogValidation = validateStripeRevenueCatalog(catalog);
|
|
123
|
+
if (!catalogValidation.ok) {
|
|
124
|
+
return { matched: false, complete: false, reason: 'catalog_invalid', gap: catalogValidation.gap };
|
|
125
|
+
}
|
|
126
|
+
const observed = inspectStripePrice(price);
|
|
127
|
+
if (!observed.priceId || !observed.productId || !observed.unitAmountCents
|
|
128
|
+
|| !observed.currency || !observed.cadence
|
|
129
|
+
|| (observed.cadence !== 'one_time' && !observed.intervalCount)) {
|
|
130
|
+
return { matched: false, complete: false, reason: 'price_identity_incomplete', observed };
|
|
131
|
+
}
|
|
132
|
+
const entry = catalog.find((candidate) => candidate.priceId === observed.priceId);
|
|
133
|
+
if (!entry) {
|
|
134
|
+
return { matched: false, complete: true, reason: 'price_not_in_catalog', observed };
|
|
135
|
+
}
|
|
136
|
+
const expected = {
|
|
137
|
+
priceId: entry.priceId,
|
|
138
|
+
productId: entry.productId,
|
|
139
|
+
unitAmountCents: entry.unitAmountCents,
|
|
140
|
+
currency: normalizeCurrency(entry.currency),
|
|
141
|
+
cadence: normalizeId(entry.cadence).toLowerCase(),
|
|
142
|
+
intervalCount: entry.cadence === 'one_time' ? null : Number(entry.intervalCount || 1),
|
|
143
|
+
};
|
|
144
|
+
const matched = Object.keys(expected).every((key) => observed[key] === expected[key]);
|
|
145
|
+
return {
|
|
146
|
+
matched,
|
|
147
|
+
complete: true,
|
|
148
|
+
reason: matched ? 'exact_catalog_match' : 'catalog_terms_mismatch',
|
|
149
|
+
offerId: matched ? entry.offerId : null,
|
|
150
|
+
status: matched ? entry.status : null,
|
|
151
|
+
observed,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const defaultCatalogValidation = validateStripeRevenueCatalog(DEFAULT_STRIPE_REVENUE_CATALOG);
|
|
156
|
+
if (!defaultCatalogValidation.ok) throw new Error(defaultCatalogValidation.gap);
|
|
157
|
+
|
|
158
|
+
module.exports = {
|
|
159
|
+
STRIPE_REVENUE_CATALOG_VERSION,
|
|
160
|
+
DEFAULT_STRIPE_REVENUE_CATALOG,
|
|
161
|
+
inspectStripePrice,
|
|
162
|
+
validateStripeRevenueCatalog,
|
|
163
|
+
matchStripeRevenueCatalogPrice,
|
|
164
|
+
};
|
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Task Outcomes — evidence-backed proof that an agent is working.
|
|
6
|
+
*
|
|
7
|
+
* A response, tool call, or demo is not a success event. This ledger stores
|
|
8
|
+
* task-level outcomes and computes transparent component metrics without
|
|
9
|
+
* hiding weak behavior behind a single composite score.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const crypto = require('node:crypto');
|
|
13
|
+
const fs = require('node:fs');
|
|
14
|
+
const path = require('node:path');
|
|
15
|
+
const { getFeedbackPaths } = require('./feedback-paths');
|
|
16
|
+
const { validateToolContract } = require('./tool-contract-validator');
|
|
17
|
+
|
|
18
|
+
const OUTCOMES_FILE = 'task-outcome-receipts.jsonl';
|
|
19
|
+
const SCHEMA_PATH = path.join(__dirname, '..', 'config', 'schemas', 'task-outcome-receipt.schema.json');
|
|
20
|
+
const TASK_OUTCOME_SCHEMA = JSON.parse(fs.readFileSync(SCHEMA_PATH, 'utf8'));
|
|
21
|
+
|
|
22
|
+
function getTaskOutcomesPath(options = {}) {
|
|
23
|
+
return path.join(getFeedbackPaths(options).FEEDBACK_DIR, OUTCOMES_FILE);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function normalizeTaskOutcome(input = {}, now = new Date()) {
|
|
27
|
+
const verification = input.verification || {};
|
|
28
|
+
const policy = input.policy || {};
|
|
29
|
+
const efficiency = input.efficiency || {};
|
|
30
|
+
const toolCalls = Array.isArray(input.toolCalls) ? input.toolCalls : [];
|
|
31
|
+
const receipt = {
|
|
32
|
+
taskId: cleanString(input.taskId),
|
|
33
|
+
taskType: cleanString(input.taskType || 'general'),
|
|
34
|
+
goal: cleanString(input.goal),
|
|
35
|
+
expectedOutcome: optionalString(input.expectedOutcome),
|
|
36
|
+
status: input.status || 'failed',
|
|
37
|
+
verification: {
|
|
38
|
+
performed: verification.performed === true,
|
|
39
|
+
passed: verification.passed === true,
|
|
40
|
+
verifier: optionalString(verification.verifier),
|
|
41
|
+
method: optionalString(verification.method),
|
|
42
|
+
evidence: cleanStringArray(verification.evidence),
|
|
43
|
+
unsupportedClaims: nonNegativeInteger(verification.unsupportedClaims),
|
|
44
|
+
},
|
|
45
|
+
toolCalls: toolCalls.map(normalizeToolCall),
|
|
46
|
+
policy: {
|
|
47
|
+
violations: nonNegativeInteger(policy.violations),
|
|
48
|
+
unsafeEscapes: nonNegativeInteger(policy.unsafeEscapes),
|
|
49
|
+
falseBlocks: nonNegativeInteger(policy.falseBlocks),
|
|
50
|
+
},
|
|
51
|
+
failure: normalizeFailure(input.failure),
|
|
52
|
+
escalation: normalizeEscalation(input.escalation),
|
|
53
|
+
efficiency: {
|
|
54
|
+
latencyMs: nonNegativeNumber(efficiency.latencyMs),
|
|
55
|
+
costUsd: nonNegativeNumber(efficiency.costUsd),
|
|
56
|
+
firstAttempt: efficiency.firstAttempt === true,
|
|
57
|
+
},
|
|
58
|
+
businessOutcome: normalizeBusinessOutcome(input.businessOutcome),
|
|
59
|
+
traceId: optionalString(input.traceId),
|
|
60
|
+
idempotencyKey: optionalString(input.idempotencyKey || input.taskId),
|
|
61
|
+
versions: normalizeVersions(input.versions),
|
|
62
|
+
metadata: isPlainObject(input.metadata) ? input.metadata : {},
|
|
63
|
+
recordedAt: input.recordedAt || now.toISOString(),
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
removeUndefined(receipt);
|
|
67
|
+
const verdict = evaluateWorkingVerdict(receipt);
|
|
68
|
+
receipt.working = verdict.working;
|
|
69
|
+
receipt.workingReasons = verdict.reasons;
|
|
70
|
+
receipt.receiptHash = hashReceipt(receipt);
|
|
71
|
+
return receipt;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function evaluateWorkingVerdict(receipt = {}) {
|
|
75
|
+
const reasons = [];
|
|
76
|
+
if (receipt.status !== 'completed') reasons.push(`status_${receipt.status || 'unknown'}`);
|
|
77
|
+
if (!receipt.verification?.performed) reasons.push('verification_not_performed');
|
|
78
|
+
if (!receipt.verification?.passed) reasons.push('verification_failed');
|
|
79
|
+
if (!receipt.verification?.evidence?.length) reasons.push('evidence_missing');
|
|
80
|
+
if (Number(receipt.verification?.unsupportedClaims || 0) > 0) reasons.push('unsupported_claim');
|
|
81
|
+
if ((receipt.toolCalls || []).some((call) => !call.contractValid)) reasons.push('tool_contract_invalid');
|
|
82
|
+
if ((receipt.toolCalls || []).some((call) => !call.allowed)) reasons.push('tool_policy_denied');
|
|
83
|
+
if ((receipt.toolCalls || []).some((call) => !call.succeeded)) reasons.push('tool_call_failed');
|
|
84
|
+
if ((receipt.toolCalls || []).some((call) => call.duplicateSideEffect)) reasons.push('duplicate_side_effect');
|
|
85
|
+
if (Number(receipt.policy?.violations || 0) > 0) reasons.push('policy_violation');
|
|
86
|
+
if (Number(receipt.policy?.unsafeEscapes || 0) > 0) reasons.push('unsafe_escape');
|
|
87
|
+
if (Number(receipt.policy?.falseBlocks || 0) > 0) reasons.push('safe_false_block');
|
|
88
|
+
if (receipt.escalation?.required && receipt.escalation?.correct !== true) reasons.push('incorrect_escalation');
|
|
89
|
+
return { working: reasons.length === 0, reasons };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function recordTaskOutcome(input = {}, options = {}) {
|
|
93
|
+
const receipt = normalizeTaskOutcome(input, options.now || new Date());
|
|
94
|
+
const validation = validateToolContract(TASK_OUTCOME_SCHEMA, receipt);
|
|
95
|
+
if (!validation.valid) {
|
|
96
|
+
const error = new Error(`Invalid task outcome receipt: ${validation.errors.join('; ')}`);
|
|
97
|
+
error.code = 'THUMBGATE_TASK_OUTCOME_INVALID';
|
|
98
|
+
error.validationErrors = validation.errors;
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const outcomesPath = getTaskOutcomesPath(options);
|
|
103
|
+
const existing = readTaskOutcomes(options);
|
|
104
|
+
const duplicate = existing.find((entry) => entry.idempotencyKey === receipt.idempotencyKey);
|
|
105
|
+
if (duplicate) {
|
|
106
|
+
if (duplicate.receiptHash !== receipt.receiptHash) {
|
|
107
|
+
const error = new Error(`Conflicting task outcome for idempotency key '${receipt.idempotencyKey}'`);
|
|
108
|
+
error.code = 'THUMBGATE_IDEMPOTENCY_CONFLICT';
|
|
109
|
+
throw error;
|
|
110
|
+
}
|
|
111
|
+
return { recorded: false, duplicate: true, receipt: duplicate };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
fs.mkdirSync(path.dirname(outcomesPath), { recursive: true });
|
|
115
|
+
fs.appendFileSync(outcomesPath, `${JSON.stringify(receipt)}\n`, 'utf8');
|
|
116
|
+
recordOutcomeTrace(receipt, options);
|
|
117
|
+
return { recorded: true, duplicate: false, receipt };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function readTaskOutcomes(options = {}) {
|
|
121
|
+
const outcomesPath = options.inputPath
|
|
122
|
+
? path.resolve(options.inputPath)
|
|
123
|
+
: getTaskOutcomesPath(options);
|
|
124
|
+
let raw = '';
|
|
125
|
+
try {
|
|
126
|
+
raw = fs.readFileSync(outcomesPath, 'utf8');
|
|
127
|
+
} catch {
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
return raw.split('\n')
|
|
131
|
+
.map((line) => line.trim())
|
|
132
|
+
.filter(Boolean)
|
|
133
|
+
.flatMap((line) => {
|
|
134
|
+
try {
|
|
135
|
+
return [JSON.parse(line)];
|
|
136
|
+
} catch {
|
|
137
|
+
return [];
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function getTaskOutcome(taskId, options = {}) {
|
|
143
|
+
const outcomes = readTaskOutcomes(options);
|
|
144
|
+
for (let index = outcomes.length - 1; index >= 0; index -= 1) {
|
|
145
|
+
if (outcomes[index].taskId === taskId) return outcomes[index];
|
|
146
|
+
}
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function calculateTaskOutcomeMetrics(outcomes = []) {
|
|
151
|
+
const valid = outcomes.filter((entry) => entry && typeof entry === 'object');
|
|
152
|
+
const toolCalls = valid.flatMap((entry) => entry.toolCalls || []);
|
|
153
|
+
const verifiedSuccesses = valid.filter((entry) => entry.status === 'completed' && entry.verification?.passed);
|
|
154
|
+
const escalationEligible = valid.filter((entry) => entry.escalation?.required);
|
|
155
|
+
const failures = valid.filter((entry) => entry.status === 'failed');
|
|
156
|
+
const latencies = valid.map((entry) => Number(entry.efficiency?.latencyMs)).filter(Number.isFinite);
|
|
157
|
+
const totalCostUsd = valid.reduce((sum, entry) => sum + nonNegativeNumber(entry.efficiency?.costUsd), 0);
|
|
158
|
+
const businessOutcomes = aggregateBusinessOutcomes(valid);
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
generatedAt: new Date().toISOString(),
|
|
162
|
+
sampleSize: valid.length,
|
|
163
|
+
evidenceStatus: valid.length > 0 ? 'measured' : 'insufficient_evidence',
|
|
164
|
+
task: {
|
|
165
|
+
workingRate: rate(valid.filter((entry) => entry.working).length, valid.length),
|
|
166
|
+
verifiedCompletionRate: rate(verifiedSuccesses.length, valid.length),
|
|
167
|
+
evidenceBackedCompletionRate: rate(
|
|
168
|
+
verifiedSuccesses.filter((entry) => entry.verification?.evidence?.length > 0).length,
|
|
169
|
+
valid.length,
|
|
170
|
+
),
|
|
171
|
+
unsupportedClaimRate: rate(
|
|
172
|
+
valid.filter((entry) => Number(entry.verification?.unsupportedClaims || 0) > 0).length,
|
|
173
|
+
valid.length,
|
|
174
|
+
),
|
|
175
|
+
firstAttemptSuccessRate: rate(
|
|
176
|
+
verifiedSuccesses.filter((entry) => entry.efficiency?.firstAttempt).length,
|
|
177
|
+
valid.length,
|
|
178
|
+
),
|
|
179
|
+
repeatedFailureRate: rate(
|
|
180
|
+
failures.filter((entry) => entry.failure?.repeated).length,
|
|
181
|
+
failures.length,
|
|
182
|
+
),
|
|
183
|
+
recoveryRate: rate(
|
|
184
|
+
failures.filter((entry) => entry.failure?.recovered).length,
|
|
185
|
+
failures.length,
|
|
186
|
+
),
|
|
187
|
+
rollbackRate: rate(
|
|
188
|
+
failures.filter((entry) => entry.failure?.rolledBack).length,
|
|
189
|
+
failures.length,
|
|
190
|
+
),
|
|
191
|
+
},
|
|
192
|
+
tools: {
|
|
193
|
+
calls: toolCalls.length,
|
|
194
|
+
contractAccuracy: rate(toolCalls.filter((call) => call.contractValid).length, toolCalls.length),
|
|
195
|
+
executionSuccessRate: rate(toolCalls.filter((call) => call.succeeded).length, toolCalls.length),
|
|
196
|
+
duplicateSideEffectRate: rate(toolCalls.filter((call) => call.duplicateSideEffect).length, toolCalls.length),
|
|
197
|
+
retryRate: rate(toolCalls.filter((call) => call.attempts > 1).length, toolCalls.length),
|
|
198
|
+
},
|
|
199
|
+
safety: {
|
|
200
|
+
unsafeEscapeRate: rate(valid.filter((entry) => Number(entry.policy?.unsafeEscapes || 0) > 0).length, valid.length),
|
|
201
|
+
policyViolationRate: rate(valid.filter((entry) => Number(entry.policy?.violations || 0) > 0).length, valid.length),
|
|
202
|
+
safeFalseBlockRate: rate(valid.filter((entry) => Number(entry.policy?.falseBlocks || 0) > 0).length, valid.length),
|
|
203
|
+
},
|
|
204
|
+
escalation: {
|
|
205
|
+
eligible: escalationEligible.length,
|
|
206
|
+
correctEscalationRate: rate(
|
|
207
|
+
escalationEligible.filter((entry) => entry.escalation?.correct).length,
|
|
208
|
+
escalationEligible.length,
|
|
209
|
+
),
|
|
210
|
+
},
|
|
211
|
+
efficiency: {
|
|
212
|
+
latencyP50Ms: percentile(latencies, 0.5),
|
|
213
|
+
latencyP95Ms: percentile(latencies, 0.95),
|
|
214
|
+
totalCostUsd: roundMoney(totalCostUsd),
|
|
215
|
+
costPerVerifiedSuccessUsd: verifiedSuccesses.length
|
|
216
|
+
? roundMoney(totalCostUsd / verifiedSuccesses.length)
|
|
217
|
+
: null,
|
|
218
|
+
},
|
|
219
|
+
businessOutcomes,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function recordOutcomeTrace(receipt, options) {
|
|
224
|
+
if (options.recordTrace === false) return;
|
|
225
|
+
try {
|
|
226
|
+
const { recordReasoningTrace } = require('./agent-reasoning-traces');
|
|
227
|
+
const messages = [
|
|
228
|
+
{ role: 'user', content: `intent: ${receipt.goal}` },
|
|
229
|
+
...receipt.toolCalls.map((call) => ({
|
|
230
|
+
role: 'assistant',
|
|
231
|
+
content: `tool: ${call.name}`,
|
|
232
|
+
tool_calls: [{ function: { name: call.name, arguments: {} } }],
|
|
233
|
+
})),
|
|
234
|
+
{
|
|
235
|
+
role: 'tool',
|
|
236
|
+
content: `tool response: ${receipt.toolCalls.every((call) => call.succeeded) ? 'success' : 'failure'}`,
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
role: 'assistant',
|
|
240
|
+
content: receipt.verification.evidence.length
|
|
241
|
+
? `verification evidence: ${receipt.verification.evidence.join('; ')}`
|
|
242
|
+
: 'verification missing',
|
|
243
|
+
},
|
|
244
|
+
];
|
|
245
|
+
recordReasoningTrace({
|
|
246
|
+
trace_id: receipt.traceId || receipt.taskId,
|
|
247
|
+
task_type: receipt.taskType,
|
|
248
|
+
messages,
|
|
249
|
+
success: receipt.working,
|
|
250
|
+
outcome: {
|
|
251
|
+
success: receipt.working,
|
|
252
|
+
terminalState: receipt.status,
|
|
253
|
+
},
|
|
254
|
+
source: 'task-outcome-receipt',
|
|
255
|
+
}, options);
|
|
256
|
+
} catch {
|
|
257
|
+
// Outcome recording is authoritative; best-effort trace analytics cannot
|
|
258
|
+
// make a verified receipt disappear.
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function normalizeToolCall(call = {}) {
|
|
263
|
+
const sideEffect = call.sideEffect === true;
|
|
264
|
+
return {
|
|
265
|
+
name: cleanString(call.name),
|
|
266
|
+
contractValid: call.contractValid === true,
|
|
267
|
+
allowed: call.allowed === true,
|
|
268
|
+
succeeded: call.succeeded === true,
|
|
269
|
+
attempts: Math.max(1, nonNegativeInteger(call.attempts || 1)),
|
|
270
|
+
latencyMs: nonNegativeNumber(call.latencyMs),
|
|
271
|
+
costUsd: nonNegativeNumber(call.costUsd),
|
|
272
|
+
sideEffect,
|
|
273
|
+
idempotencyKey: optionalString(call.idempotencyKey),
|
|
274
|
+
duplicateSideEffect: call.duplicateSideEffect === true,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function normalizeFailure(value) {
|
|
279
|
+
if (!value) return undefined;
|
|
280
|
+
return {
|
|
281
|
+
category: optionalString(value.category),
|
|
282
|
+
recovered: value.recovered === true,
|
|
283
|
+
repeated: value.repeated === true,
|
|
284
|
+
rolledBack: value.rolledBack === true,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function normalizeEscalation(value) {
|
|
289
|
+
if (!value) return undefined;
|
|
290
|
+
return {
|
|
291
|
+
required: value.required === true,
|
|
292
|
+
correct: value.correct === true,
|
|
293
|
+
escalationId: optionalString(value.escalationId),
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function normalizeBusinessOutcome(value) {
|
|
298
|
+
if (!value) return undefined;
|
|
299
|
+
return {
|
|
300
|
+
kpi: cleanString(value.kpi),
|
|
301
|
+
value: Number(value.value),
|
|
302
|
+
unit: cleanString(value.unit),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function normalizeVersions(value) {
|
|
307
|
+
if (!value) return undefined;
|
|
308
|
+
return {
|
|
309
|
+
model: optionalString(value.model),
|
|
310
|
+
prompt: optionalString(value.prompt),
|
|
311
|
+
tools: optionalString(value.tools),
|
|
312
|
+
policy: optionalString(value.policy),
|
|
313
|
+
release: optionalString(value.release),
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function aggregateBusinessOutcomes(outcomes) {
|
|
318
|
+
const groups = new Map();
|
|
319
|
+
for (const outcome of outcomes) {
|
|
320
|
+
const item = outcome.businessOutcome;
|
|
321
|
+
if (!item) continue;
|
|
322
|
+
const key = `${item.kpi}\0${item.unit}`;
|
|
323
|
+
const current = groups.get(key) || { kpi: item.kpi, unit: item.unit, value: 0, tasks: 0 };
|
|
324
|
+
current.value += Number(item.value || 0);
|
|
325
|
+
current.tasks += 1;
|
|
326
|
+
groups.set(key, current);
|
|
327
|
+
}
|
|
328
|
+
return Array.from(groups.values()).map((entry) => ({ ...entry, value: roundMetric(entry.value) }));
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function hashReceipt(receipt) {
|
|
332
|
+
const copy = { ...receipt };
|
|
333
|
+
delete copy.receiptHash;
|
|
334
|
+
delete copy.recordedAt;
|
|
335
|
+
return crypto.createHash('sha256').update(stableStringify(copy)).digest('hex');
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function stableStringify(value) {
|
|
339
|
+
if (!value || typeof value !== 'object') return JSON.stringify(value);
|
|
340
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
|
|
341
|
+
const keys = Object.keys(value).sort((left, right) => left.localeCompare(right));
|
|
342
|
+
const properties = keys.map((key) => [
|
|
343
|
+
JSON.stringify(key),
|
|
344
|
+
stableStringify(value[key]),
|
|
345
|
+
].join(':'));
|
|
346
|
+
return ['{', properties.join(','), '}'].join('');
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function percentile(values, quantile) {
|
|
350
|
+
if (!values.length) return null;
|
|
351
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
352
|
+
const index = Math.ceil(quantile * sorted.length) - 1;
|
|
353
|
+
return roundMetric(sorted[Math.max(0, index)]);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function rate(numerator, denominator) {
|
|
357
|
+
return denominator > 0 ? roundMetric(numerator / denominator) : null;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function cleanString(value) {
|
|
361
|
+
return String(value ?? '').trim();
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
function optionalString(value) {
|
|
365
|
+
const result = cleanString(value);
|
|
366
|
+
return result || undefined;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function cleanStringArray(value) {
|
|
370
|
+
return Array.isArray(value) ? value.map(cleanString).filter(Boolean) : [];
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function nonNegativeNumber(value) {
|
|
374
|
+
const number = Number(value);
|
|
375
|
+
return Number.isFinite(number) ? Math.max(0, number) : 0;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function nonNegativeInteger(value) {
|
|
379
|
+
return Math.floor(nonNegativeNumber(value));
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function roundMetric(value) {
|
|
383
|
+
return Number.isFinite(value) ? Math.round(value * 10000) / 10000 : null;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
function roundMoney(value) {
|
|
387
|
+
return Number.isFinite(value) ? Math.round(value * 1000000) / 1000000 : null;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function isPlainObject(value) {
|
|
391
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function removeUndefined(value) {
|
|
395
|
+
if (!value || typeof value !== 'object') return;
|
|
396
|
+
for (const key of Object.keys(value)) {
|
|
397
|
+
if (value[key] === undefined) delete value[key];
|
|
398
|
+
else removeUndefined(value[key]);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function isCliInvocation() {
|
|
403
|
+
return Boolean(process.argv[1]) && path.resolve(process.argv[1]) === __filename;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
if (isCliInvocation()) {
|
|
407
|
+
const command = process.argv[2] || 'metrics';
|
|
408
|
+
if (command === 'metrics') {
|
|
409
|
+
console.log(JSON.stringify(calculateTaskOutcomeMetrics(readTaskOutcomes()), null, 2));
|
|
410
|
+
} else {
|
|
411
|
+
console.error('Usage: task-outcomes.js metrics');
|
|
412
|
+
process.exitCode = 1;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
module.exports = {
|
|
417
|
+
TASK_OUTCOME_SCHEMA,
|
|
418
|
+
calculateTaskOutcomeMetrics,
|
|
419
|
+
evaluateWorkingVerdict,
|
|
420
|
+
getTaskOutcome,
|
|
421
|
+
getTaskOutcomesPath,
|
|
422
|
+
normalizeTaskOutcome,
|
|
423
|
+
readTaskOutcomes,
|
|
424
|
+
recordTaskOutcome,
|
|
425
|
+
};
|
|
@@ -19,6 +19,7 @@ const MARKETING_CLICK_EVENT_TYPES = new Set([
|
|
|
19
19
|
'cta_click',
|
|
20
20
|
'checkout_start',
|
|
21
21
|
'checkout_bootstrap',
|
|
22
|
+
'diagnostic_checkout_confirmed',
|
|
22
23
|
'checkout_interstitial_cta_clicked',
|
|
23
24
|
'chatgpt_gpt_open',
|
|
24
25
|
'chatgpt_gpt_click',
|
|
@@ -31,6 +32,14 @@ const MARKETING_CLICK_EVENT_TYPES = new Set([
|
|
|
31
32
|
'github_repo_click',
|
|
32
33
|
'community_landing_redirect',
|
|
33
34
|
]);
|
|
35
|
+
const CHECKOUT_START_EVENT_TYPES = new Set([
|
|
36
|
+
'checkout_start',
|
|
37
|
+
'checkout_bootstrap',
|
|
38
|
+
// A diagnostic buyer has submitted a valid receipt email and the server is
|
|
39
|
+
// issuing the external Payment Link redirect. This is stronger than a CTA
|
|
40
|
+
// click and is the service equivalent of the Pro checkout bootstrap.
|
|
41
|
+
'diagnostic_checkout_confirmed',
|
|
42
|
+
]);
|
|
34
43
|
|
|
35
44
|
function shouldIncludeLegacyTelemetry() {
|
|
36
45
|
if (
|
|
@@ -91,6 +100,10 @@ function isMarketingClickEvent(eventType) {
|
|
|
91
100
|
return MARKETING_CLICK_EVENT_TYPES.has(String(eventType || '').toLowerCase());
|
|
92
101
|
}
|
|
93
102
|
|
|
103
|
+
function isCheckoutStartEvent(eventType) {
|
|
104
|
+
return CHECKOUT_START_EVENT_TYPES.has(String(eventType || '').toLowerCase());
|
|
105
|
+
}
|
|
106
|
+
|
|
94
107
|
function getTelemetryPath(feedbackDir) {
|
|
95
108
|
return path.join(feedbackDir, TELEMETRY_FILE_NAME);
|
|
96
109
|
}
|
|
@@ -666,7 +679,7 @@ function summarizeExternalTrafficQuality(events) {
|
|
|
666
679
|
if ((entry.page || entry.landingPath) && !pathRow.pages.includes(entry.page || entry.landingPath)) {
|
|
667
680
|
pathRow.pages.push(entry.page || entry.landingPath);
|
|
668
681
|
}
|
|
669
|
-
if (eventType
|
|
682
|
+
if (isCheckoutStartEvent(eventType)) {
|
|
670
683
|
pathRow.checkoutStarts += 1;
|
|
671
684
|
}
|
|
672
685
|
visitorPaths.set(visitorKey, pathRow);
|
|
@@ -683,7 +696,7 @@ function summarizeExternalTrafficQuality(events) {
|
|
|
683
696
|
externalCtaClicks += 1;
|
|
684
697
|
}
|
|
685
698
|
|
|
686
|
-
if (eventType
|
|
699
|
+
if (isCheckoutStartEvent(eventType)) {
|
|
687
700
|
externalCheckoutStarts += 1;
|
|
688
701
|
incrementCounter(checkoutStartsBySource, entry.source);
|
|
689
702
|
incrementCounter(checkoutStartsByTrafficChannel, entry.trafficChannel);
|
|
@@ -814,6 +827,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
814
827
|
let ctaClicks = 0;
|
|
815
828
|
let ctaImpressions = 0;
|
|
816
829
|
let checkoutStarts = 0;
|
|
830
|
+
let diagnosticCheckoutStarts = 0;
|
|
817
831
|
let checkoutInterstitialViews = 0;
|
|
818
832
|
let checkoutBotDeflections = 0;
|
|
819
833
|
let checkoutInterstitialClicks = 0;
|
|
@@ -922,8 +936,11 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
922
936
|
}
|
|
923
937
|
}
|
|
924
938
|
|
|
925
|
-
if ((entry.eventType || entry.event)
|
|
939
|
+
if (isCheckoutStartEvent(entry.eventType || entry.event)) {
|
|
926
940
|
checkoutStarts += 1;
|
|
941
|
+
if ((entry.eventType || entry.event) === 'diagnostic_checkout_confirmed') {
|
|
942
|
+
diagnosticCheckoutStarts += 1;
|
|
943
|
+
}
|
|
927
944
|
incrementCounter(checkoutStartsBySource, entry.source);
|
|
928
945
|
incrementCounter(checkoutStartsByCampaign, entry.utmCampaign);
|
|
929
946
|
incrementCounter(checkoutStartsByTrafficChannel, entry.trafficChannel);
|
|
@@ -1112,6 +1129,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
1112
1129
|
gptOpens,
|
|
1113
1130
|
gptActionCalls: chatgptActionCalls,
|
|
1114
1131
|
checkoutStarts,
|
|
1132
|
+
diagnosticCheckoutStarts,
|
|
1115
1133
|
checkoutInterstitialViews,
|
|
1116
1134
|
checkoutInterstitialClicks,
|
|
1117
1135
|
trialEmails,
|
|
@@ -1133,6 +1151,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
1133
1151
|
pageViews,
|
|
1134
1152
|
ctaClicks,
|
|
1135
1153
|
checkoutStarts,
|
|
1154
|
+
diagnosticCheckoutStarts,
|
|
1136
1155
|
checkoutInterstitialViews,
|
|
1137
1156
|
checkoutBotDeflections,
|
|
1138
1157
|
checkoutInterstitialClicks,
|
|
@@ -1317,6 +1336,7 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
|
|
|
1317
1336
|
ctas: {
|
|
1318
1337
|
totalClicks: summary.web.ctaClicks,
|
|
1319
1338
|
checkoutStarts: summary.web.checkoutStarts,
|
|
1339
|
+
diagnosticCheckoutStarts: summary.web.diagnosticCheckoutStarts,
|
|
1320
1340
|
checkoutInterstitialViews: summary.web.checkoutInterstitialViews,
|
|
1321
1341
|
checkoutBotDeflections: summary.web.checkoutBotDeflections,
|
|
1322
1342
|
checkoutInterstitialClicks: summary.web.checkoutInterstitialClicks,
|