thumbgate 1.15.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +6 -6
- package/.claude-plugin/plugin.json +3 -3
- package/.well-known/llms.txt +5 -5
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +59 -35
- package/adapters/chatgpt/openapi.yaml +118 -2
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/mcp/server-stdio.js +210 -84
- package/adapters/opencode/opencode.json +1 -1
- package/bench/prompt-eval-suite.json +5 -1
- package/bin/cli.js +157 -8
- package/config/evals/agent-safety-eval.json +338 -22
- package/config/gates/routine.json +43 -0
- package/config/github-about.json +3 -3
- package/config/model-candidates.json +131 -0
- package/openapi/openapi.yaml +118 -2
- package/package.json +55 -48
- package/public/blog.html +7 -7
- package/public/codex-plugin.html +6 -6
- package/public/compare.html +29 -23
- package/public/dashboard.html +82 -10
- package/public/guide.html +28 -28
- package/public/index.html +216 -98
- package/public/learn.html +50 -22
- package/public/lessons.html +1 -1
- package/public/numbers.html +17 -17
- package/public/pro.html +82 -18
- package/scripts/agent-audit-trace.js +55 -0
- package/scripts/agent-memory-lifecycle.js +96 -0
- package/scripts/agent-readiness-plan.js +118 -0
- package/scripts/agentic-data-pipeline.js +21 -1
- package/scripts/agents-sdk-sandbox-plan.js +57 -0
- package/scripts/ai-org-governance.js +98 -0
- package/scripts/ai-search-distribution.js +43 -0
- package/scripts/artifact-agent-plan.js +81 -0
- package/scripts/billing.js +27 -8
- package/scripts/cli-schema.js +18 -2
- package/scripts/code-mode-mcp-plan.js +71 -0
- package/scripts/context-engine.js +1 -2
- package/scripts/context-manager.js +4 -1
- package/scripts/dashboard-render-spec.js +1 -1
- package/scripts/dashboard.js +275 -9
- package/scripts/decision-journal.js +13 -3
- package/scripts/document-workflow-governance.js +62 -0
- package/scripts/enterprise-agent-rollout.js +34 -0
- package/scripts/experience-replay-governance.js +69 -0
- package/scripts/export-hf-dataset.js +1 -1
- package/scripts/feedback-loop.js +92 -4
- package/scripts/feedback-to-rules.js +17 -23
- package/scripts/gates-engine.js +4 -6
- package/scripts/growth-campaigns.js +49 -0
- package/scripts/harness-selector.js +16 -4
- package/scripts/hybrid-supervisor-agent.js +64 -0
- package/scripts/inference-cache-policy.js +72 -0
- package/scripts/inference-economics.js +53 -0
- package/scripts/internal-agent-bootstrap.js +12 -2
- package/scripts/knowledge-layer-plan.js +108 -0
- package/scripts/lesson-inference.js +183 -44
- package/scripts/lesson-search.js +4 -1
- package/scripts/llm-client.js +157 -26
- package/scripts/mailer/resend-mailer.js +112 -1
- package/scripts/mcp-transport-strategy.js +66 -0
- package/scripts/memory-store-governance.js +60 -0
- package/scripts/meta-agent-loop.js +7 -13
- package/scripts/model-access-eligibility.js +38 -0
- package/scripts/model-migration-readiness.js +55 -0
- package/scripts/operational-integrity.js +96 -3
- package/scripts/otel-declarative-config.js +56 -0
- package/scripts/perplexity-client.js +1 -1
- package/scripts/post-training-governance.js +34 -0
- package/scripts/private-core-boundary.js +72 -0
- package/scripts/production-agent-readiness.js +40 -0
- package/scripts/prompt-eval.js +564 -32
- package/scripts/prompt-programs.js +93 -0
- package/scripts/provider-action-normalizer.js +585 -0
- package/scripts/scaling-law-claims.js +60 -0
- package/scripts/security-scanner.js +1 -1
- package/scripts/self-distill-agent.js +7 -32
- package/scripts/seo-gsd.js +232 -55
- package/scripts/skill-rag-router.js +53 -0
- package/scripts/spec-gate.js +1 -1
- package/scripts/student-consistent-training.js +73 -0
- package/scripts/synthetic-data-provenance.js +98 -0
- package/scripts/task-context-result.js +81 -0
- package/scripts/telemetry-analytics.js +149 -0
- package/scripts/thompson-sampling.js +2 -2
- package/scripts/token-savings.js +7 -6
- package/scripts/token-tco.js +46 -0
- package/scripts/tool-registry.js +63 -3
- package/scripts/verification-loop.js +10 -1
- package/scripts/verifier-scoring.js +71 -0
- package/scripts/workflow-sentinel.js +284 -28
- package/scripts/workspace-agent-routines.js +118 -0
- package/src/api/server.js +381 -120
- package/scripts/analytics-report.js +0 -328
- package/scripts/autonomous-workflow.js +0 -377
- package/scripts/billing-setup.js +0 -109
- package/scripts/creator-campaigns.js +0 -239
- package/scripts/cross-encoder-reranker.js +0 -235
- package/scripts/daemon-manager.js +0 -108
- package/scripts/decision-trace.js +0 -354
- package/scripts/delegation-runtime.js +0 -896
- package/scripts/dispatch-brief.js +0 -159
- package/scripts/distribution-surfaces.js +0 -110
- package/scripts/feedback-history-distiller.js +0 -382
- package/scripts/funnel-analytics.js +0 -35
- package/scripts/history-distiller.js +0 -200
- package/scripts/hosted-job-launcher.js +0 -256
- package/scripts/intent-router.js +0 -392
- package/scripts/lesson-reranker.js +0 -263
- package/scripts/lesson-retrieval.js +0 -148
- package/scripts/managed-lesson-agent.js +0 -183
- package/scripts/operational-dashboard.js +0 -103
- package/scripts/operational-summary.js +0 -129
- package/scripts/operator-artifacts.js +0 -608
- package/scripts/optimize-context.js +0 -17
- package/scripts/org-dashboard.js +0 -206
- package/scripts/partner-orchestration.js +0 -146
- package/scripts/predictive-insights.js +0 -356
- package/scripts/pulse.js +0 -80
- package/scripts/reflector-agent.js +0 -221
- package/scripts/sales-pipeline.js +0 -681
- package/scripts/session-episode-store.js +0 -329
- package/scripts/session-health-sensor.js +0 -242
- package/scripts/session-report.js +0 -120
- package/scripts/swarm-coordinator.js +0 -81
- package/scripts/tool-kpi-tracker.js +0 -12
- package/scripts/webhook-delivery.js +0 -62
- package/scripts/workflow-sprint-intake.js +0 -475
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
function normalizeText(value) {
|
|
5
|
+
if (value === undefined || value === null) return '';
|
|
6
|
+
return String(value).trim();
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function classifyRetrievalFailure(input = {}) {
|
|
10
|
+
const query = normalizeText(input.query);
|
|
11
|
+
const evidence = normalizeText(input.evidence);
|
|
12
|
+
const confidence = Number(input.confidence ?? 0.5);
|
|
13
|
+
const failed = input.failed === true || confidence < 0.35 || /unknown|not enough|cannot determine/i.test(evidence);
|
|
14
|
+
if (!failed) return 'none';
|
|
15
|
+
if (!query) return 'irreducible';
|
|
16
|
+
if (/\band\b|\bor\b|,|;|\bcompare\b|\bmultiple\b/i.test(query)) return 'question_decomposition';
|
|
17
|
+
if (evidence && !new RegExp(query.split(/\s+/).slice(0, 3).join('|'), 'i').test(evidence)) return 'query_rewrite';
|
|
18
|
+
if (evidence.length > 1200 || /irrelevant|too broad|many results/i.test(evidence)) return 'evidence_focus';
|
|
19
|
+
return 'query_rewrite';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function routeRetrievalSkill(input = {}) {
|
|
23
|
+
const failure = classifyRetrievalFailure(input);
|
|
24
|
+
const skill = {
|
|
25
|
+
none: 'skip_retrieval',
|
|
26
|
+
query_rewrite: 'rewrite_query',
|
|
27
|
+
question_decomposition: 'decompose_question',
|
|
28
|
+
evidence_focus: 'focus_evidence',
|
|
29
|
+
irreducible: 'exit_unknown',
|
|
30
|
+
}[failure];
|
|
31
|
+
return {
|
|
32
|
+
failure,
|
|
33
|
+
skill,
|
|
34
|
+
retrieve: failure !== 'none' && failure !== 'irreducible',
|
|
35
|
+
reason: reasonForFailure(failure),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function reasonForFailure(failure) {
|
|
40
|
+
if (failure === 'none') {
|
|
41
|
+
return 'Model answer is sufficiently confident; retrieval would waste budget.';
|
|
42
|
+
}
|
|
43
|
+
if (failure === 'irreducible') {
|
|
44
|
+
return 'Query lacks enough structure for retrieval; ask for clarification or return unknown.';
|
|
45
|
+
}
|
|
46
|
+
return 'Failure state detected; route to a typed retrieval repair skill before retrying generation.';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
module.exports = {
|
|
50
|
+
classifyRetrievalFailure,
|
|
51
|
+
reasonForFailure,
|
|
52
|
+
routeRetrievalSkill,
|
|
53
|
+
};
|
package/scripts/spec-gate.js
CHANGED
|
@@ -341,7 +341,7 @@ if (isCliInvocation()) {
|
|
|
341
341
|
const entries = loadSpecAudit();
|
|
342
342
|
console.log(JSON.stringify(summarizeSpecAudit(entries), null, 2));
|
|
343
343
|
} else {
|
|
344
|
-
console.error(`Unknown command: ${command}. Use: check,
|
|
344
|
+
console.error(`Unknown command: ${command}. Use: check, checks, audit`);
|
|
345
345
|
process.exit(1);
|
|
346
346
|
}
|
|
347
347
|
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
function buildStudentConsistentTrainingPlan(options = {}) {
|
|
4
|
+
const student = options.student || 'target-agent-policy';
|
|
5
|
+
const teacher = options.teacher || 'frontier-reviewer';
|
|
6
|
+
const dataset = options.dataset || 'thumbgate-feedback-lessons';
|
|
7
|
+
const holdout = options.holdout || 'feedback-gate-holdout';
|
|
8
|
+
|
|
9
|
+
return {
|
|
10
|
+
method: 'student_consistent_synthetic_sft',
|
|
11
|
+
dataset,
|
|
12
|
+
teacher,
|
|
13
|
+
student,
|
|
14
|
+
generationContract: {
|
|
15
|
+
teacherRole: 'adds capability tokens: corrected decision, missing evidence, safer action',
|
|
16
|
+
studentRole: 'preserves target agent style: terse format, tool discipline, gate vocabulary',
|
|
17
|
+
rejectIf: [
|
|
18
|
+
'teacher rewrites the answer into unsupported style',
|
|
19
|
+
'lesson cannot be traced to source feedback',
|
|
20
|
+
'sample contains secrets or private customer context',
|
|
21
|
+
'sample teaches a shortcut that bypasses evidence gates',
|
|
22
|
+
],
|
|
23
|
+
},
|
|
24
|
+
requiredArtifacts: [
|
|
25
|
+
'source feedback id',
|
|
26
|
+
'student baseline response',
|
|
27
|
+
'teacher correction',
|
|
28
|
+
'student-consistent final sample',
|
|
29
|
+
'redaction report',
|
|
30
|
+
'holdout eval result',
|
|
31
|
+
],
|
|
32
|
+
evals: {
|
|
33
|
+
holdout,
|
|
34
|
+
compareAgainst: ['raw_teacher_sft', 'self_distill_only', 'no_training_baseline'],
|
|
35
|
+
metrics: ['gate_precision', 'gate_recall', 'unsupported_claim_rate', 'style_drift_rate'],
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function evaluateStudentConsistentTrainingSample(sample = {}) {
|
|
41
|
+
const issues = [];
|
|
42
|
+
|
|
43
|
+
if (!sample.sourceFeedbackId) issues.push('missing_source_feedback_id');
|
|
44
|
+
if (!sample.studentBaseline) issues.push('missing_student_baseline');
|
|
45
|
+
if (!sample.teacherCorrection) issues.push('missing_teacher_correction');
|
|
46
|
+
if (!sample.finalSample) issues.push('missing_final_sample');
|
|
47
|
+
if (!sample.redacted) issues.push('redaction_required');
|
|
48
|
+
if (!sample.holdoutEval) issues.push('holdout_eval_required');
|
|
49
|
+
|
|
50
|
+
const text = [
|
|
51
|
+
sample.studentBaseline,
|
|
52
|
+
sample.teacherCorrection,
|
|
53
|
+
sample.finalSample,
|
|
54
|
+
].filter(Boolean).join('\n');
|
|
55
|
+
if (/(api[_-]?key|secret|token|password)\s*[:=]/i.test(text)) {
|
|
56
|
+
issues.push('secret_like_content');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (sample.styleDriftRate !== undefined && Number(sample.styleDriftRate) > 0.15) {
|
|
60
|
+
issues.push('style_drift_too_high');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return {
|
|
64
|
+
decision: issues.length ? 'warn' : 'allow',
|
|
65
|
+
issues,
|
|
66
|
+
onPolicy: !issues.includes('style_drift_too_high') && Boolean(sample.studentBaseline),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = {
|
|
71
|
+
buildStudentConsistentTrainingPlan,
|
|
72
|
+
evaluateStudentConsistentTrainingSample,
|
|
73
|
+
};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
function buildSyntheticDataProvenanceRecord(input = {}) {
|
|
4
|
+
return {
|
|
5
|
+
sampleId: input.sampleId || null,
|
|
6
|
+
sourceFeedbackId: input.sourceFeedbackId || null,
|
|
7
|
+
teacher: {
|
|
8
|
+
model: input.teacherModel || null,
|
|
9
|
+
baseModelFamily: input.teacherBaseModelFamily || null,
|
|
10
|
+
promptHash: input.teacherPromptHash || null,
|
|
11
|
+
riskLabel: input.teacherRiskLabel || 'unknown',
|
|
12
|
+
},
|
|
13
|
+
student: {
|
|
14
|
+
model: input.studentModel || null,
|
|
15
|
+
baseModelFamily: input.studentBaseModelFamily || null,
|
|
16
|
+
},
|
|
17
|
+
generation: {
|
|
18
|
+
generatedAt: input.generatedAt || null,
|
|
19
|
+
filterReportId: input.filterReportId || null,
|
|
20
|
+
redactionReportId: input.redactionReportId || null,
|
|
21
|
+
datasetVersion: input.datasetVersion || null,
|
|
22
|
+
},
|
|
23
|
+
evals: {
|
|
24
|
+
semanticFilterPassed: Boolean(input.semanticFilterPassed),
|
|
25
|
+
behavioralHoldoutPassed: Boolean(input.behavioralHoldoutPassed),
|
|
26
|
+
styleDriftScore: Number.isFinite(input.styleDriftScore) ? input.styleDriftScore : null,
|
|
27
|
+
hiddenTraitProbePassed: Boolean(input.hiddenTraitProbePassed),
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function evaluateSyntheticDataPromotion(record = {}) {
|
|
33
|
+
const teacher = record.teacher || {};
|
|
34
|
+
const student = record.student || {};
|
|
35
|
+
const generation = record.generation || {};
|
|
36
|
+
const evals = record.evals || {};
|
|
37
|
+
const issues = [
|
|
38
|
+
...missingIdentityIssues(record),
|
|
39
|
+
...missingModelIssues(teacher, student),
|
|
40
|
+
...missingGenerationIssues(generation),
|
|
41
|
+
...failedEvalIssues(evals),
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
const sameBaseFamily = Boolean(
|
|
45
|
+
teacher.baseModelFamily
|
|
46
|
+
&& student.baseModelFamily
|
|
47
|
+
&& teacher.baseModelFamily === student.baseModelFamily,
|
|
48
|
+
);
|
|
49
|
+
if (sameBaseFamily && teacher.riskLabel !== 'trusted') {
|
|
50
|
+
issues.push('same_base_teacher_requires_trusted_risk_label');
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return {
|
|
54
|
+
decision: issues.length ? 'deny' : 'allow',
|
|
55
|
+
issues,
|
|
56
|
+
sameBaseFamily,
|
|
57
|
+
riskClass: sameBaseFamily ? 'subliminal_learning_sensitive' : 'standard_distillation',
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function missingIdentityIssues(record) {
|
|
62
|
+
const issues = [];
|
|
63
|
+
if (!record.sampleId) issues.push('missing_sample_id');
|
|
64
|
+
if (!record.sourceFeedbackId) issues.push('missing_source_feedback_id');
|
|
65
|
+
return issues;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function missingModelIssues(teacher, student) {
|
|
69
|
+
const issues = [];
|
|
70
|
+
if (!teacher.model) issues.push('missing_teacher_model');
|
|
71
|
+
if (!teacher.baseModelFamily) issues.push('missing_teacher_base_model_family');
|
|
72
|
+
if (!student.model) issues.push('missing_student_model');
|
|
73
|
+
if (!student.baseModelFamily) issues.push('missing_student_base_model_family');
|
|
74
|
+
return issues;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function missingGenerationIssues(generation) {
|
|
78
|
+
const issues = [];
|
|
79
|
+
if (!generation.filterReportId) issues.push('missing_filter_report');
|
|
80
|
+
if (!generation.redactionReportId) issues.push('missing_redaction_report');
|
|
81
|
+
if (!generation.datasetVersion) issues.push('missing_dataset_version');
|
|
82
|
+
return issues;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function failedEvalIssues(evals) {
|
|
86
|
+
const issues = [];
|
|
87
|
+
if (!evals.semanticFilterPassed) issues.push('semantic_filter_failed_or_missing');
|
|
88
|
+
if (!evals.behavioralHoldoutPassed) issues.push('behavioral_holdout_required');
|
|
89
|
+
if (!evals.hiddenTraitProbePassed) issues.push('hidden_trait_probe_required');
|
|
90
|
+
if (evals.styleDriftScore === null) issues.push('missing_style_drift_score');
|
|
91
|
+
if (evals.styleDriftScore !== null && evals.styleDriftScore > 0.15) issues.push('style_drift_too_high');
|
|
92
|
+
return issues;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
module.exports = {
|
|
96
|
+
buildSyntheticDataProvenanceRecord,
|
|
97
|
+
evaluateSyntheticDataPromotion,
|
|
98
|
+
};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
function normalizeText(value) {
|
|
5
|
+
if (value === undefined || value === null) return '';
|
|
6
|
+
return String(value).trim();
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function normalizeList(value) {
|
|
10
|
+
return Array.isArray(value) ? value.map(normalizeText).filter(Boolean) : [];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function buildTaskContextResultQuery(input = {}) {
|
|
14
|
+
const task = normalizeText(input.task);
|
|
15
|
+
const context = normalizeList(input.context);
|
|
16
|
+
const result = normalizeText(input.result);
|
|
17
|
+
const tools = normalizeList(input.tools);
|
|
18
|
+
const files = normalizeList(input.files);
|
|
19
|
+
const sequence = normalizeList(input.sequence);
|
|
20
|
+
const audience = normalizeText(input.audience);
|
|
21
|
+
const creditBudget = normalizeText(input.creditBudget) || 'standard';
|
|
22
|
+
const missing = [];
|
|
23
|
+
if (!task) missing.push('task');
|
|
24
|
+
if (context.length === 0 && files.length === 0 && tools.length === 0) missing.push('context');
|
|
25
|
+
if (!result) missing.push('result');
|
|
26
|
+
const contextParts = [
|
|
27
|
+
...context,
|
|
28
|
+
...files.map((file) => `file:${file}`),
|
|
29
|
+
...tools.map((tool) => `tool:${tool}`),
|
|
30
|
+
];
|
|
31
|
+
const sequenceText = sequence
|
|
32
|
+
.map((step, index) => `${index + 1}. ${step}`)
|
|
33
|
+
.join(' ');
|
|
34
|
+
|
|
35
|
+
return {
|
|
36
|
+
pattern: 'TaskContextResult',
|
|
37
|
+
status: missing.length === 0 ? 'ready' : 'needs_context',
|
|
38
|
+
missing,
|
|
39
|
+
query: [
|
|
40
|
+
`Task: ${task || '[required]'}`,
|
|
41
|
+
`Context: ${contextParts.join('; ') || '[required]'}`,
|
|
42
|
+
`Result: ${result || '[required]'}`,
|
|
43
|
+
audience ? `Audience: ${audience}` : null,
|
|
44
|
+
sequence.length > 0 ? `Sequence: ${sequenceText}` : null,
|
|
45
|
+
`Credit budget: ${creditBudget}`,
|
|
46
|
+
].filter(Boolean).join('\n'),
|
|
47
|
+
governance: {
|
|
48
|
+
explicitTools: tools,
|
|
49
|
+
explicitFiles: files,
|
|
50
|
+
multiStep: sequence.length > 1,
|
|
51
|
+
highCreditRisk: /web|browser|scrape|dashboard|presentation|multi[-\s]?step|full/i.test(`${task} ${result} ${sequence.join(' ')}`),
|
|
52
|
+
recommendation: creditBudget === 'low'
|
|
53
|
+
? 'Use focused read-only work unless the operator approves a larger run.'
|
|
54
|
+
: 'Run through ThumbGate gates before write/tool side effects.',
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function reviewTaskContextResultQuery(input = {}) {
|
|
60
|
+
const plan = input.pattern === 'TaskContextResult' ? input : buildTaskContextResultQuery(input);
|
|
61
|
+
const issues = [];
|
|
62
|
+
for (const field of plan.missing || []) issues.push({ field, issue: 'missing_tcr_component' });
|
|
63
|
+
if (plan.governance.highCreditRisk && !/high|approved|enterprise/i.test(plan.query)) {
|
|
64
|
+
issues.push({ field: 'creditBudget', issue: 'expensive_workflow_without_budget_ack' });
|
|
65
|
+
}
|
|
66
|
+
if (plan.governance.multiStep && !/Sequence:/i.test(plan.query)) {
|
|
67
|
+
issues.push({ field: 'sequence', issue: 'missing_ordered_steps' });
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
status: issues.length === 0 ? 'pass' : 'warn',
|
|
71
|
+
issues,
|
|
72
|
+
recommendation: issues.length === 0
|
|
73
|
+
? 'Query is concrete enough for a workspace agent.'
|
|
74
|
+
: 'Clarify task, context, result, tools/files, or credit budget before dispatch.',
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
module.exports = {
|
|
79
|
+
buildTaskContextResultQuery,
|
|
80
|
+
reviewTaskContextResultQuery,
|
|
81
|
+
};
|
|
@@ -315,6 +315,15 @@ function sanitizeTelemetryPayload(payload = {}, headers = {}) {
|
|
|
315
315
|
pricingInterest: pickFirstText(raw.pricingInterest, raw.interestLevel),
|
|
316
316
|
seoQuery: pickFirstText(raw.seoQuery, raw.query),
|
|
317
317
|
seoSurface: pickFirstText(raw.seoSurface, raw.searchSurface, raw.surface),
|
|
318
|
+
sectionId: pickFirstText(raw.sectionId, raw.section),
|
|
319
|
+
sectionLabel: pickFirstText(raw.sectionLabel),
|
|
320
|
+
lastVisibleSection: pickFirstText(raw.lastVisibleSection, raw.visibleSection),
|
|
321
|
+
dwellBucket: pickFirstText(raw.dwellBucket, raw.engagementBucket),
|
|
322
|
+
scrollBucket: pickFirstText(raw.scrollBucket),
|
|
323
|
+
engagementMs: normalizeInteger(raw.engagementMs),
|
|
324
|
+
maxScrollPercent: normalizeInteger(raw.maxScrollPercent ?? raw.scrollPercent),
|
|
325
|
+
buyerEmailFocused: Boolean(raw.buyerEmailFocused),
|
|
326
|
+
buyerEmailCaptured: Boolean(raw.buyerEmailCaptured),
|
|
318
327
|
trafficChannel: inferTrafficChannel(raw, referrerHost),
|
|
319
328
|
failureCode: pickFirstText(raw.failureCode),
|
|
320
329
|
httpStatus: normalizeInteger(raw.httpStatus),
|
|
@@ -442,10 +451,19 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
442
451
|
const pricingInterestByLevel = {};
|
|
443
452
|
const seoLandingViewsBySurface = {};
|
|
444
453
|
const seoLandingViewsByQuery = {};
|
|
454
|
+
const trackedLinkHitsBySlug = {};
|
|
455
|
+
const trackedLinkCheckoutStartsBySlug = {};
|
|
456
|
+
const sectionViewsById = {};
|
|
457
|
+
const pageExitsByLastVisibleSection = {};
|
|
458
|
+
const pageExitsByDwellBucket = {};
|
|
459
|
+
const pageExitsByScrollBucket = {};
|
|
460
|
+
const ctaImpressionsById = {};
|
|
461
|
+
const ctaImpressionsByPlacement = {};
|
|
445
462
|
const cliByPlatform = {};
|
|
446
463
|
const cliByVersion = {};
|
|
447
464
|
let pageViews = 0;
|
|
448
465
|
let ctaClicks = 0;
|
|
466
|
+
let ctaImpressions = 0;
|
|
449
467
|
let checkoutStarts = 0;
|
|
450
468
|
let checkoutFailures = 0;
|
|
451
469
|
let checkoutCancelled = 0;
|
|
@@ -458,6 +476,9 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
458
476
|
let buyerLossSignals = 0;
|
|
459
477
|
let pricingInterestEvents = 0;
|
|
460
478
|
let seoLandingViews = 0;
|
|
479
|
+
let pageExitEvents = 0;
|
|
480
|
+
let emailFocusEvents = 0;
|
|
481
|
+
let emailAbandonEvents = 0;
|
|
461
482
|
let webEvents = 0;
|
|
462
483
|
let webEventsWithVisitorId = 0;
|
|
463
484
|
let webEventsWithSessionId = 0;
|
|
@@ -465,6 +486,10 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
465
486
|
let attributedPageViews = 0;
|
|
466
487
|
let attributedCheckoutStarts = 0;
|
|
467
488
|
let latestSeenAt = null;
|
|
489
|
+
let exitEngagementMsTotal = 0;
|
|
490
|
+
let exitEngagementMsCount = 0;
|
|
491
|
+
let exitScrollPercentTotal = 0;
|
|
492
|
+
let exitScrollPercentCount = 0;
|
|
468
493
|
|
|
469
494
|
for (const entry of events) {
|
|
470
495
|
incrementCounter(byClientType, entry.clientType || entry.client || 'unknown');
|
|
@@ -506,6 +531,9 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
506
531
|
incrementCounter(ctaClicksByOfferCode, entry.offerCode);
|
|
507
532
|
incrementCounter(ctaClicksByCampaignVariant, entry.campaignVariant);
|
|
508
533
|
incrementCounter(byCtaId, entry.ctaId);
|
|
534
|
+
if (entry.linkSlug && (entry.eventType || entry.event) === 'cta_click') {
|
|
535
|
+
incrementCounter(trackedLinkHitsBySlug, entry.linkSlug);
|
|
536
|
+
}
|
|
509
537
|
}
|
|
510
538
|
|
|
511
539
|
if ((entry.eventType || entry.event) === 'checkout_start' || (entry.eventType || entry.event) === 'checkout_bootstrap') {
|
|
@@ -517,6 +545,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
517
545
|
incrementCounter(checkoutStartsByCommunity, entry.community);
|
|
518
546
|
incrementCounter(checkoutStartsByOfferCode, entry.offerCode);
|
|
519
547
|
incrementCounter(checkoutStartsByCampaignVariant, entry.campaignVariant);
|
|
548
|
+
if (entry.linkSlug) incrementCounter(trackedLinkCheckoutStartsBySlug, entry.linkSlug);
|
|
520
549
|
const starterKey = pickFirstText(
|
|
521
550
|
entry.acquisitionId,
|
|
522
551
|
entry.visitorId,
|
|
@@ -581,6 +610,39 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
581
610
|
buyerLossSignals += 1;
|
|
582
611
|
}
|
|
583
612
|
|
|
613
|
+
if ((entry.eventType || entry.event) === 'section_view') {
|
|
614
|
+
incrementCounter(sectionViewsById, entry.sectionId);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
if ((entry.eventType || entry.event) === 'cta_impression') {
|
|
618
|
+
ctaImpressions += 1;
|
|
619
|
+
incrementCounter(ctaImpressionsById, entry.ctaId);
|
|
620
|
+
incrementCounter(ctaImpressionsByPlacement, entry.ctaPlacement);
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
if ((entry.eventType || entry.event) === 'page_exit') {
|
|
624
|
+
pageExitEvents += 1;
|
|
625
|
+
incrementCounter(pageExitsByLastVisibleSection, entry.lastVisibleSection);
|
|
626
|
+
incrementCounter(pageExitsByDwellBucket, entry.dwellBucket);
|
|
627
|
+
incrementCounter(pageExitsByScrollBucket, entry.scrollBucket);
|
|
628
|
+
if (entry.engagementMs !== null) {
|
|
629
|
+
exitEngagementMsTotal += entry.engagementMs;
|
|
630
|
+
exitEngagementMsCount += 1;
|
|
631
|
+
}
|
|
632
|
+
if (entry.maxScrollPercent !== null) {
|
|
633
|
+
exitScrollPercentTotal += entry.maxScrollPercent;
|
|
634
|
+
exitScrollPercentCount += 1;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
if ((entry.eventType || entry.event) === 'buyer_email_focus') {
|
|
639
|
+
emailFocusEvents += 1;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
if ((entry.eventType || entry.event) === 'buyer_email_abandon') {
|
|
643
|
+
emailAbandonEvents += 1;
|
|
644
|
+
}
|
|
645
|
+
|
|
584
646
|
if ((entry.eventType || entry.event) === 'pricing_interest') {
|
|
585
647
|
pricingInterestEvents += 1;
|
|
586
648
|
incrementCounter(pricingInterestByLevel, entry.pricingInterest);
|
|
@@ -623,6 +685,17 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
623
685
|
pageViewsByTrafficChannel[channelKey] || 0
|
|
624
686
|
);
|
|
625
687
|
}
|
|
688
|
+
|
|
689
|
+
const trackedLinkConversionBySlug = {};
|
|
690
|
+
for (const slugKey of new Set([
|
|
691
|
+
...Object.keys(trackedLinkHitsBySlug),
|
|
692
|
+
...Object.keys(trackedLinkCheckoutStartsBySlug),
|
|
693
|
+
])) {
|
|
694
|
+
trackedLinkConversionBySlug[slugKey] = safeRate(
|
|
695
|
+
trackedLinkCheckoutStartsBySlug[slugKey] || 0,
|
|
696
|
+
trackedLinkHitsBySlug[slugKey] || 0
|
|
697
|
+
);
|
|
698
|
+
}
|
|
626
699
|
const installCopies = byEventType.install_copy || 0;
|
|
627
700
|
const gptOpens = (byEventType.chatgpt_gpt_open || 0) + (byEventType.chatgpt_gpt_click || 0);
|
|
628
701
|
const trialEmails = byEventType.trial_email_captured || 0;
|
|
@@ -666,6 +739,10 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
666
739
|
buyerLossSignals,
|
|
667
740
|
pricingInterestEvents,
|
|
668
741
|
seoLandingViews,
|
|
742
|
+
ctaImpressions,
|
|
743
|
+
pageExitEvents,
|
|
744
|
+
emailFocusEvents,
|
|
745
|
+
emailAbandonEvents,
|
|
669
746
|
pageViewToCheckoutRate: safeRate(checkoutStarts, pageViews),
|
|
670
747
|
visitorToCheckoutRate: safeRate(checkoutStarts, webVisitors.size),
|
|
671
748
|
visitorIdCoverageRate: safeRate(webEventsWithVisitorId, webEvents),
|
|
@@ -674,6 +751,12 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
674
751
|
attributedPageViews,
|
|
675
752
|
attributedCheckoutStarts,
|
|
676
753
|
attributionCoverageRate: safeRate(attributedPageViews, pageViews),
|
|
754
|
+
averageExitEngagementMs: exitEngagementMsCount > 0
|
|
755
|
+
? Math.round(exitEngagementMsTotal / exitEngagementMsCount)
|
|
756
|
+
: 0,
|
|
757
|
+
averageExitScrollPercent: exitScrollPercentCount > 0
|
|
758
|
+
? Math.round(exitScrollPercentTotal / exitScrollPercentCount)
|
|
759
|
+
: 0,
|
|
677
760
|
},
|
|
678
761
|
cli: {
|
|
679
762
|
uniqueInstalls: cliInstalls.size,
|
|
@@ -716,9 +799,18 @@ function getTelemetrySummary(feedbackDir, options = {}) {
|
|
|
716
799
|
pricingInterestByLevel,
|
|
717
800
|
seoLandingViewsBySurface,
|
|
718
801
|
seoLandingViewsByQuery,
|
|
802
|
+
sectionViewsById,
|
|
803
|
+
pageExitsByLastVisibleSection,
|
|
804
|
+
pageExitsByDwellBucket,
|
|
805
|
+
pageExitsByScrollBucket,
|
|
806
|
+
ctaImpressionsById,
|
|
807
|
+
ctaImpressionsByPlacement,
|
|
719
808
|
checkoutConversionBySource,
|
|
720
809
|
checkoutConversionByCampaign,
|
|
721
810
|
checkoutConversionByTrafficChannel,
|
|
811
|
+
trackedLinkHitsBySlug,
|
|
812
|
+
trackedLinkCheckoutStartsBySlug,
|
|
813
|
+
trackedLinkConversionBySlug,
|
|
722
814
|
},
|
|
723
815
|
recent: summarizeRecentEvents(events),
|
|
724
816
|
};
|
|
@@ -744,6 +836,20 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
|
|
|
744
836
|
const topBuyerLossReason = getTopCounterEntry(summary.marketing.buyerLossReasons);
|
|
745
837
|
const topSeoSurface = getTopCounterEntry(summary.marketing.seoLandingViewsBySurface);
|
|
746
838
|
const topSeoQuery = getTopCounterEntry(summary.marketing.seoLandingViewsByQuery);
|
|
839
|
+
const topViewedSection = getTopCounterEntry(summary.marketing.sectionViewsById);
|
|
840
|
+
const topExitSection = getTopCounterEntry(summary.marketing.pageExitsByLastVisibleSection);
|
|
841
|
+
const topExitDwellBucket = getTopCounterEntry(summary.marketing.pageExitsByDwellBucket);
|
|
842
|
+
const topImpressionCta = getTopCounterEntry(summary.marketing.ctaImpressionsById);
|
|
843
|
+
const impressionToClickRateById = {};
|
|
844
|
+
for (const ctaId of new Set([
|
|
845
|
+
...Object.keys(summary.marketing.ctaImpressionsById || {}),
|
|
846
|
+
...Object.keys(summary.marketing.byCtaId || {}),
|
|
847
|
+
])) {
|
|
848
|
+
impressionToClickRateById[ctaId] = safeRate(
|
|
849
|
+
(summary.marketing.byCtaId || {})[ctaId] || 0,
|
|
850
|
+
(summary.marketing.ctaImpressionsById || {})[ctaId] || 0
|
|
851
|
+
);
|
|
852
|
+
}
|
|
747
853
|
|
|
748
854
|
return {
|
|
749
855
|
window: summary.window,
|
|
@@ -788,6 +894,7 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
|
|
|
788
894
|
checkoutFailures: summary.web.checkoutFailures,
|
|
789
895
|
checkoutCancelled: summary.web.checkoutCancelled,
|
|
790
896
|
checkoutAbandoned: summary.web.checkoutAbandoned,
|
|
897
|
+
ctaImpressions: summary.web.ctaImpressions,
|
|
791
898
|
successPageViews: summary.web.checkoutSuccessPageViews,
|
|
792
899
|
cancelPageViews: summary.web.checkoutCancelPageViews,
|
|
793
900
|
paidConfirmations: summary.web.checkoutPaidConfirmations,
|
|
@@ -824,6 +931,25 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
|
|
|
824
931
|
successPageViewRate: safeRate(summary.web.checkoutSuccessPageViews, summary.web.checkoutStarts),
|
|
825
932
|
conversionByTrafficChannel: summary.marketing.checkoutConversionByTrafficChannel,
|
|
826
933
|
},
|
|
934
|
+
behavior: {
|
|
935
|
+
sectionViewsById: summary.marketing.sectionViewsById,
|
|
936
|
+
ctaImpressionsById: summary.marketing.ctaImpressionsById,
|
|
937
|
+
ctaImpressionsByPlacement: summary.marketing.ctaImpressionsByPlacement,
|
|
938
|
+
pageExits: summary.web.pageExitEvents,
|
|
939
|
+
exitsByLastVisibleSection: summary.marketing.pageExitsByLastVisibleSection,
|
|
940
|
+
exitsByDwellBucket: summary.marketing.pageExitsByDwellBucket,
|
|
941
|
+
exitsByScrollBucket: summary.marketing.pageExitsByScrollBucket,
|
|
942
|
+
emailFocusEvents: summary.web.emailFocusEvents,
|
|
943
|
+
emailAbandonEvents: summary.web.emailAbandonEvents,
|
|
944
|
+
emailAbandonRate: safeRate(summary.web.emailAbandonEvents, summary.web.emailFocusEvents),
|
|
945
|
+
averageExitEngagementMs: summary.web.averageExitEngagementMs,
|
|
946
|
+
averageExitScrollPercent: summary.web.averageExitScrollPercent,
|
|
947
|
+
impressionToClickRateById,
|
|
948
|
+
topViewedSection: topViewedSection ? { key: topViewedSection[0], count: topViewedSection[1] } : null,
|
|
949
|
+
topExitSection: topExitSection ? { key: topExitSection[0], count: topExitSection[1] } : null,
|
|
950
|
+
topExitDwellBucket: topExitDwellBucket ? { key: topExitDwellBucket[0], count: topExitDwellBucket[1] } : null,
|
|
951
|
+
topImpressionCta: topImpressionCta ? { key: topImpressionCta[0], count: topImpressionCta[1] } : null,
|
|
952
|
+
},
|
|
827
953
|
buyerLoss: {
|
|
828
954
|
totalSignals: summary.web.buyerLossSignals,
|
|
829
955
|
reasonsByCode: summary.marketing.buyerLossReasons,
|
|
@@ -842,6 +968,29 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
|
|
|
842
968
|
topSurface: topSeoSurface ? { key: topSeoSurface[0], count: topSeoSurface[1] } : null,
|
|
843
969
|
topQuery: topSeoQuery ? { key: topSeoQuery[0], count: topSeoQuery[1] } : null,
|
|
844
970
|
},
|
|
971
|
+
trackedLinks: (() => {
|
|
972
|
+
const hits = summary.marketing.trackedLinkHitsBySlug || {};
|
|
973
|
+
const conversions = summary.marketing.trackedLinkCheckoutStartsBySlug || {};
|
|
974
|
+
const conversionRate = summary.marketing.trackedLinkConversionBySlug || {};
|
|
975
|
+
const totalHits = Object.values(hits).reduce((acc, n) => acc + n, 0);
|
|
976
|
+
const totalCheckoutStarts = Object.values(conversions).reduce((acc, n) => acc + n, 0);
|
|
977
|
+
const top = getTopCounterEntry(hits);
|
|
978
|
+
const bySlug = {};
|
|
979
|
+
for (const slug of new Set([...Object.keys(hits), ...Object.keys(conversions)])) {
|
|
980
|
+
bySlug[slug] = {
|
|
981
|
+
hits: hits[slug] || 0,
|
|
982
|
+
checkoutStarts: conversions[slug] || 0,
|
|
983
|
+
conversionRate: conversionRate[slug] || 0,
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
return {
|
|
987
|
+
totalHits,
|
|
988
|
+
totalCheckoutStarts,
|
|
989
|
+
overallConversionRate: safeRate(totalCheckoutStarts, totalHits),
|
|
990
|
+
bySlug,
|
|
991
|
+
topSlug: top ? { key: top[0], count: top[1] } : null,
|
|
992
|
+
};
|
|
993
|
+
})(),
|
|
845
994
|
cli: summary.cli,
|
|
846
995
|
recent: summary.recent,
|
|
847
996
|
};
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Implements per-category reliability estimates (ML-01) and exponential
|
|
6
6
|
* time-decay weighting with half-life of 7 days (ML-02).
|
|
7
7
|
*
|
|
8
|
-
* Source:
|
|
8
|
+
* Source: Adapted from earlier local feedback-modeling experiments.
|
|
9
9
|
* Algorithm: Beta-Bernoulli update with Marsaglia-Tsang gamma sampling for
|
|
10
10
|
* posterior draws. Zero external npm dependencies.
|
|
11
11
|
*
|
|
@@ -45,7 +45,7 @@ const DECAY_FLOOR = 0.01;
|
|
|
45
45
|
const MIN_SAMPLES_THRESHOLD = 5;
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
|
-
* Default category taxonomy — mirrors
|
|
48
|
+
* Default category taxonomy — mirrors the legacy 8-keyword categories plus
|
|
49
49
|
* 'uncategorized' as the catch-all. Used when initializing a new model.
|
|
50
50
|
*/
|
|
51
51
|
const DEFAULT_CATEGORIES = [
|
package/scripts/token-savings.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* Why this exists:
|
|
8
8
|
* The mission of ThumbGate is "stop paying for the same AI mistake
|
|
9
|
-
* twice." Every time a Pre-Action
|
|
9
|
+
* twice." Every time a Pre-Action Check blocks a known-bad tool call,
|
|
10
10
|
* the agent does NOT make a round-trip to the model. That's:
|
|
11
11
|
*
|
|
12
12
|
* - input tokens you didn't spend (system prompt + tool defs +
|
|
@@ -22,10 +22,11 @@
|
|
|
22
22
|
* Defaults are intentionally conservative — the goal is "you almost
|
|
23
23
|
* certainly saved at least this much," not "let's flatter ourselves."
|
|
24
24
|
*
|
|
25
|
-
* Pricing snapshot (USD per 1M tokens, retrieved 2026-04-
|
|
25
|
+
* Pricing snapshot (USD per 1M tokens, retrieved 2026-04-22 from Anthropic's
|
|
26
|
+
* official prompt-caching pricing table):
|
|
26
27
|
* Sonnet 4.5: $3 input, $15 output
|
|
27
|
-
* Opus 4.6: $
|
|
28
|
-
* Haiku 4.5: $
|
|
28
|
+
* Opus 4.6: $5 input, $25 output
|
|
29
|
+
* Haiku 4.5: $1 input, $5 output
|
|
29
30
|
* GPT-4o: $2.50 input, $10 output
|
|
30
31
|
*
|
|
31
32
|
* If the caller doesn't pass a modelMix, we assume a Sonnet-heavy
|
|
@@ -36,8 +37,8 @@
|
|
|
36
37
|
const DEFAULT_MODEL_PRICES = Object.freeze({
|
|
37
38
|
// USD per 1M tokens
|
|
38
39
|
'claude-sonnet-4-5': { input: 3.0, output: 15.0 },
|
|
39
|
-
'claude-opus-4-6': { input:
|
|
40
|
-
'claude-haiku-4-5': { input: 0
|
|
40
|
+
'claude-opus-4-6': { input: 5.0, output: 25.0 },
|
|
41
|
+
'claude-haiku-4-5': { input: 1.0, output: 5.0 },
|
|
41
42
|
'gpt-4o': { input: 2.50, output: 10.0 },
|
|
42
43
|
});
|
|
43
44
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
function computeCostPerMillionTokens(input = {}) {
|
|
4
|
+
const gpuDollarsPerHour = Number(input.gpuDollarsPerHour || 0);
|
|
5
|
+
const tokensPerSecond = Number(input.tokensPerSecond || 0);
|
|
6
|
+
if (gpuDollarsPerHour <= 0 || tokensPerSecond <= 0) return Infinity;
|
|
7
|
+
const tokensPerHour = tokensPerSecond * 60 * 60;
|
|
8
|
+
return Number(((gpuDollarsPerHour / tokensPerHour) * 1000000).toFixed(6));
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function evaluateInferenceTco(input = {}) {
|
|
12
|
+
let costPerMillionTokens = computeCostPerMillionTokens(input);
|
|
13
|
+
if (input.costPerMillionTokens !== undefined) {
|
|
14
|
+
costPerMillionTokens = Number(input.costPerMillionTokens);
|
|
15
|
+
}
|
|
16
|
+
const tokensPerRun = Number(input.tokensPerRun || 0);
|
|
17
|
+
const runsPerDay = Number(input.runsPerDay || 0);
|
|
18
|
+
const usefulBlocksPerDay = Number(input.usefulBlocksPerDay || 0);
|
|
19
|
+
const minutesSavedPerBlock = Number(input.minutesSavedPerBlock || 16);
|
|
20
|
+
const laborDollarsPerHour = Number(input.laborDollarsPerHour || 100);
|
|
21
|
+
|
|
22
|
+
const dailyTokenCost = Number(((costPerMillionTokens / 1000000) * tokensPerRun * runsPerDay).toFixed(4));
|
|
23
|
+
const dailyValue = Number(((usefulBlocksPerDay * minutesSavedPerBlock / 60) * laborDollarsPerHour).toFixed(4));
|
|
24
|
+
const roi = dailyTokenCost > 0 ? Number((dailyValue / dailyTokenCost).toFixed(2)) : Infinity;
|
|
25
|
+
const issues = [];
|
|
26
|
+
|
|
27
|
+
if (!Number.isFinite(costPerMillionTokens)) issues.push('missing_token_tco_inputs');
|
|
28
|
+
if (!tokensPerRun) issues.push('missing_tokens_per_run');
|
|
29
|
+
if (!runsPerDay) issues.push('missing_runs_per_day');
|
|
30
|
+
if (!usefulBlocksPerDay) issues.push('missing_useful_blocks_per_day');
|
|
31
|
+
|
|
32
|
+
return {
|
|
33
|
+
decision: issues.length ? 'warn' : 'allow',
|
|
34
|
+
issues,
|
|
35
|
+
costPerMillionTokens,
|
|
36
|
+
dailyTokenCost,
|
|
37
|
+
dailyValue,
|
|
38
|
+
roi,
|
|
39
|
+
metric: 'cost_per_useful_blocked_failure',
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
module.exports = {
|
|
44
|
+
computeCostPerMillionTokens,
|
|
45
|
+
evaluateInferenceTco,
|
|
46
|
+
};
|