voyageai-cli 1.29.0 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -8
- package/package.json +1 -1
- package/src/commands/benchmark.js +22 -8
- package/src/commands/chat.js +18 -0
- package/src/commands/chunk.js +10 -0
- package/src/commands/demo.js +4 -0
- package/src/commands/embed.js +13 -0
- package/src/commands/estimate.js +3 -0
- package/src/commands/eval.js +6 -0
- package/src/commands/explain.js +2 -0
- package/src/commands/generate.js +2 -0
- package/src/commands/ingest.js +4 -0
- package/src/commands/init.js +2 -0
- package/src/commands/mcp-server.js +2 -0
- package/src/commands/models.js +2 -0
- package/src/commands/ping.js +7 -0
- package/src/commands/pipeline.js +15 -0
- package/src/commands/playground.js +52 -6
- package/src/commands/query.js +16 -0
- package/src/commands/rerank.js +12 -0
- package/src/commands/scaffold.js +2 -0
- package/src/commands/search.js +11 -0
- package/src/commands/similarity.js +9 -0
- package/src/commands/store.js +4 -0
- package/src/commands/workflow.js +286 -0
- package/src/lib/capability-report.js +134 -0
- package/src/lib/chat.js +32 -1
- package/src/lib/config.js +2 -0
- package/src/lib/cost-display.js +107 -0
- package/src/lib/explanations.js +6 -0
- package/src/lib/llm.js +125 -18
- package/src/lib/quality-audit.js +71 -0
- package/src/lib/security/blocked-domains.json +17 -0
- package/src/lib/security-audit.js +198 -0
- package/src/lib/telemetry.js +23 -1
- package/src/lib/workflow-scaffold.js +61 -0
- package/src/lib/workflow-test-runner.js +208 -0
- package/src/lib/workflow.js +128 -2
- package/src/playground/announcements.md +9 -0
- package/src/playground/assets/announcements/appstore.jpg +0 -0
- package/src/playground/assets/announcements/circuits.jpg +0 -0
- package/src/playground/assets/announcements/csvingest.jpg +0 -0
- package/src/playground/assets/announcements/green-wave.jpg +0 -0
- package/src/playground/help/workflow-nodes.js +472 -0
- package/src/playground/index.html +1482 -184
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const CAP_ICONS = {
|
|
4
|
+
NETWORK: '🌐',
|
|
5
|
+
WRITE_DB: '💾',
|
|
6
|
+
LLM: '🤖',
|
|
7
|
+
LOOP: '🔄',
|
|
8
|
+
READ_DB: '📊',
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const SEVERITY_ICONS = {
|
|
12
|
+
critical: '🔴',
|
|
13
|
+
high: '🟠',
|
|
14
|
+
medium: '🟡',
|
|
15
|
+
low: '🔵',
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Generate a markdown-formatted capability report for a workflow package.
|
|
20
|
+
*
|
|
21
|
+
* @param {object} definition - Parsed workflow JSON
|
|
22
|
+
* @param {Array<{severity: string, message: string, stepId?: string}>} securityFindings
|
|
23
|
+
* @param {Array<{level: string, message: string}>} qualityIssues
|
|
24
|
+
* @param {{ total: number, passed: number, failed: number, results?: Array }} [testResults]
|
|
25
|
+
* @returns {string} Markdown-formatted report
|
|
26
|
+
*/
|
|
27
|
+
function generateCapabilityReport(definition, securityFindings, qualityIssues, testResults) {
|
|
28
|
+
const lines = [];
|
|
29
|
+
|
|
30
|
+
const name = definition?.name || 'Unknown Workflow';
|
|
31
|
+
lines.push(`## 📋 Workflow Validation Report: ${name}`);
|
|
32
|
+
lines.push('');
|
|
33
|
+
|
|
34
|
+
// ── Capabilities ──
|
|
35
|
+
const { extractCapabilities } = require('./security-audit');
|
|
36
|
+
const caps = definition ? [...extractCapabilities(definition)] : [];
|
|
37
|
+
|
|
38
|
+
lines.push('### Capabilities');
|
|
39
|
+
if (caps.length === 0) {
|
|
40
|
+
lines.push('No special capabilities detected.');
|
|
41
|
+
} else {
|
|
42
|
+
for (const cap of caps) {
|
|
43
|
+
lines.push(`- ${CAP_ICONS[cap] || '•'} **${cap}**`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
lines.push('');
|
|
47
|
+
|
|
48
|
+
// ── Security Findings ──
|
|
49
|
+
lines.push('### Security Audit');
|
|
50
|
+
if (!securityFindings || securityFindings.length === 0) {
|
|
51
|
+
lines.push('✅ No security issues found.');
|
|
52
|
+
} else {
|
|
53
|
+
const counts = { critical: 0, high: 0, medium: 0, low: 0 };
|
|
54
|
+
for (const f of securityFindings) {
|
|
55
|
+
counts[f.severity] = (counts[f.severity] || 0) + 1;
|
|
56
|
+
}
|
|
57
|
+
const summary = Object.entries(counts)
|
|
58
|
+
.filter(([, v]) => v > 0)
|
|
59
|
+
.map(([k, v]) => `${SEVERITY_ICONS[k]} ${v} ${k.toUpperCase()}`)
|
|
60
|
+
.join(' | ');
|
|
61
|
+
lines.push(summary);
|
|
62
|
+
lines.push('');
|
|
63
|
+
lines.push('| Severity | Finding | Step |');
|
|
64
|
+
lines.push('|----------|---------|------|');
|
|
65
|
+
for (const f of securityFindings) {
|
|
66
|
+
lines.push(`| ${SEVERITY_ICONS[f.severity]} ${f.severity.toUpperCase()} | ${f.message} | ${f.stepId || '—'} |`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
lines.push('');
|
|
70
|
+
|
|
71
|
+
// ── Quality ──
|
|
72
|
+
lines.push('### Quality Audit');
|
|
73
|
+
if (!qualityIssues || qualityIssues.length === 0) {
|
|
74
|
+
lines.push('✅ No quality issues found.');
|
|
75
|
+
} else {
|
|
76
|
+
const errorCount = qualityIssues.filter(i => i.level === 'error').length;
|
|
77
|
+
const warningCount = qualityIssues.filter(i => i.level === 'warning').length;
|
|
78
|
+
const suggestionCount = qualityIssues.filter(i => i.level === 'suggestion').length;
|
|
79
|
+
|
|
80
|
+
const parts = [];
|
|
81
|
+
if (errorCount) parts.push(`❌ ${errorCount} error(s)`);
|
|
82
|
+
if (warningCount) parts.push(`⚠️ ${warningCount} warning(s)`);
|
|
83
|
+
if (suggestionCount) parts.push(`💡 ${suggestionCount} suggestion(s)`);
|
|
84
|
+
lines.push(parts.join(' | '));
|
|
85
|
+
lines.push('');
|
|
86
|
+
|
|
87
|
+
for (const issue of qualityIssues) {
|
|
88
|
+
const icon = issue.level === 'error' ? '❌' : issue.level === 'warning' ? '⚠️' : '💡';
|
|
89
|
+
lines.push(`- ${icon} **[${issue.level.toUpperCase()}]** ${issue.message}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
lines.push('');
|
|
93
|
+
|
|
94
|
+
// ── Test Results ──
|
|
95
|
+
lines.push('### Test Results');
|
|
96
|
+
if (!testResults) {
|
|
97
|
+
lines.push('⏭️ No test results available.');
|
|
98
|
+
} else if (testResults.total === 0) {
|
|
99
|
+
lines.push('⏭️ No test cases found.');
|
|
100
|
+
} else {
|
|
101
|
+
const status = testResults.failed === 0 ? '✅' : '❌';
|
|
102
|
+
lines.push(`${status} **${testResults.passed}/${testResults.total}** tests passed`);
|
|
103
|
+
if (testResults.results && testResults.results.length > 0) {
|
|
104
|
+
lines.push('');
|
|
105
|
+
for (const r of testResults.results) {
|
|
106
|
+
const icon = r.passed ? '✅' : '❌';
|
|
107
|
+
lines.push(`- ${icon} ${r.name || r.file}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
lines.push('');
|
|
112
|
+
|
|
113
|
+
// ── Overall Summary ──
|
|
114
|
+
const criticalCount = (securityFindings || []).filter(f => f.severity === 'critical').length;
|
|
115
|
+
const highCount = (securityFindings || []).filter(f => f.severity === 'high').length;
|
|
116
|
+
const qualityErrors = (qualityIssues || []).filter(i => i.level === 'error').length;
|
|
117
|
+
const testsFailed = testResults ? testResults.failed : 0;
|
|
118
|
+
|
|
119
|
+
lines.push('### Summary');
|
|
120
|
+
if (criticalCount === 0 && highCount === 0 && qualityErrors === 0 && testsFailed === 0) {
|
|
121
|
+
lines.push('✅ **All checks passed.** This workflow is ready for review.');
|
|
122
|
+
} else {
|
|
123
|
+
const issues = [];
|
|
124
|
+
if (criticalCount) issues.push(`${criticalCount} critical security finding(s)`);
|
|
125
|
+
if (highCount) issues.push(`${highCount} high security finding(s)`);
|
|
126
|
+
if (qualityErrors) issues.push(`${qualityErrors} quality error(s)`);
|
|
127
|
+
if (testsFailed) issues.push(`${testsFailed} test failure(s)`);
|
|
128
|
+
lines.push(`⚠️ **Issues found:** ${issues.join(', ')}`);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return lines.join('\n');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
module.exports = { generateCapabilityReport };
|
package/src/lib/chat.js
CHANGED
|
@@ -202,9 +202,15 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
|
|
|
202
202
|
// 3. Generate response (streaming)
|
|
203
203
|
let fullResponse = '';
|
|
204
204
|
const stream = opts.stream !== false;
|
|
205
|
+
let llmUsage = { inputTokens: 0, outputTokens: 0 };
|
|
205
206
|
|
|
206
207
|
try {
|
|
207
208
|
for await (const chunk of llm.chat(messages, { stream })) {
|
|
209
|
+
// Check for __usage sentinel (yielded as final item from LLM providers)
|
|
210
|
+
if (typeof chunk === 'object' && chunk !== null && chunk.__usage) {
|
|
211
|
+
llmUsage = chunk.__usage;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
208
214
|
fullResponse += chunk;
|
|
209
215
|
yield { type: 'chunk', data: chunk };
|
|
210
216
|
}
|
|
@@ -240,7 +246,13 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
|
|
|
240
246
|
metadata: {
|
|
241
247
|
retrievalTimeMs,
|
|
242
248
|
generationTimeMs,
|
|
243
|
-
tokens
|
|
249
|
+
tokens: {
|
|
250
|
+
...tokens,
|
|
251
|
+
llmInput: llmUsage.inputTokens,
|
|
252
|
+
llmOutput: llmUsage.outputTokens,
|
|
253
|
+
},
|
|
254
|
+
llmModel: llm.model,
|
|
255
|
+
llmProvider: llm.name,
|
|
244
256
|
contextDocsUsed: docs.length,
|
|
245
257
|
},
|
|
246
258
|
},
|
|
@@ -284,11 +296,18 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
|
|
|
284
296
|
// Track messages for the tool-calling loop (mutable copy)
|
|
285
297
|
const messages = [...initialMessages];
|
|
286
298
|
const toolCallLog = [];
|
|
299
|
+
const totalLlmUsage = { inputTokens: 0, outputTokens: 0 };
|
|
287
300
|
|
|
288
301
|
// 3. Agent loop
|
|
289
302
|
for (let iteration = 0; iteration < maxIterations; iteration++) {
|
|
290
303
|
const response = await llm.chatWithTools(messages, tools);
|
|
291
304
|
|
|
305
|
+
// Accumulate LLM usage from each chatWithTools call
|
|
306
|
+
if (response.usage) {
|
|
307
|
+
totalLlmUsage.inputTokens += response.usage.inputTokens || 0;
|
|
308
|
+
totalLlmUsage.outputTokens += response.usage.outputTokens || 0;
|
|
309
|
+
}
|
|
310
|
+
|
|
292
311
|
// Text response: done
|
|
293
312
|
if (response.type === 'text') {
|
|
294
313
|
const fullResponse = response.content;
|
|
@@ -321,6 +340,12 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
|
|
|
321
340
|
iterationCount: iteration + 1,
|
|
322
341
|
toolCallCount: toolCallLog.length,
|
|
323
342
|
totalTimeMs,
|
|
343
|
+
tokens: {
|
|
344
|
+
llmInput: totalLlmUsage.inputTokens,
|
|
345
|
+
llmOutput: totalLlmUsage.outputTokens,
|
|
346
|
+
},
|
|
347
|
+
llmModel: llm.model,
|
|
348
|
+
llmProvider: llm.name,
|
|
324
349
|
},
|
|
325
350
|
},
|
|
326
351
|
};
|
|
@@ -406,6 +431,12 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
|
|
|
406
431
|
toolCallCount: toolCallLog.length,
|
|
407
432
|
totalTimeMs: Date.now() - start,
|
|
408
433
|
maxIterationsReached: true,
|
|
434
|
+
tokens: {
|
|
435
|
+
llmInput: totalLlmUsage.inputTokens,
|
|
436
|
+
llmOutput: totalLlmUsage.outputTokens,
|
|
437
|
+
},
|
|
438
|
+
llmModel: llm.model,
|
|
439
|
+
llmProvider: llm.name,
|
|
409
440
|
},
|
|
410
441
|
},
|
|
411
442
|
};
|
package/src/lib/config.js
CHANGED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { MODEL_CATALOG } = require('./catalog');
|
|
4
|
+
const { getConfigValue } = require('./config');
|
|
5
|
+
const ui = require('./ui');
|
|
6
|
+
|
|
7
|
+
const COMPETITOR_PRICE = 0.13; // OpenAI text-embedding-3-large per 1M tokens
|
|
8
|
+
const LARGE_PRICE = 0.12; // voyage-4-large per 1M tokens
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Show a one-line cost summary after a CLI operation.
|
|
12
|
+
* Only displays when `show-cost` config is enabled.
|
|
13
|
+
* Respects --json and --quiet flags.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} model - Model name used
|
|
16
|
+
* @param {number} tokens - Total tokens consumed
|
|
17
|
+
* @param {object} [opts] - Command options
|
|
18
|
+
* @param {boolean} [opts.json] - JSON output mode (suppress cost)
|
|
19
|
+
* @param {boolean} [opts.quiet] - Quiet mode (suppress cost)
|
|
20
|
+
*/
|
|
21
|
+
function showCostSummary(model, tokens, opts = {}) {
|
|
22
|
+
if (opts.json || opts.quiet) return;
|
|
23
|
+
if (!isEnabled()) return;
|
|
24
|
+
if (!tokens || tokens <= 0) return;
|
|
25
|
+
|
|
26
|
+
const entry = MODEL_CATALOG.find(m => m.name === model);
|
|
27
|
+
const price = entry?.pricePerMToken ?? LARGE_PRICE;
|
|
28
|
+
const cost = (tokens / 1_000_000) * price;
|
|
29
|
+
const largeCost = (tokens / 1_000_000) * LARGE_PRICE;
|
|
30
|
+
|
|
31
|
+
const costStr = formatCost(cost);
|
|
32
|
+
const tokStr = tokens.toLocaleString();
|
|
33
|
+
|
|
34
|
+
console.log();
|
|
35
|
+
console.log(ui.dim(` 💰 ${costStr} (${tokStr} tokens, ${model})`));
|
|
36
|
+
|
|
37
|
+
if (price < LARGE_PRICE) {
|
|
38
|
+
const savingsPercent = Math.round((1 - price / LARGE_PRICE) * 100);
|
|
39
|
+
const largeStr = formatCost(largeCost);
|
|
40
|
+
console.log(ui.dim(` Symmetric (voyage-4-large): ${largeStr} — ${savingsPercent}% savings`));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Show a combined cost summary for operations with multiple API calls
|
|
46
|
+
* (e.g., query with embed + rerank).
|
|
47
|
+
*
|
|
48
|
+
* @param {Array<{model: string, tokens: number, label?: string}>} operations
|
|
49
|
+
* @param {object} [opts]
|
|
50
|
+
*/
|
|
51
|
+
function showCombinedCostSummary(operations, opts = {}) {
|
|
52
|
+
if (opts.json || opts.quiet) return;
|
|
53
|
+
if (!isEnabled()) return;
|
|
54
|
+
|
|
55
|
+
let totalCost = 0;
|
|
56
|
+
let totalLargeCost = 0;
|
|
57
|
+
let totalTokens = 0;
|
|
58
|
+
|
|
59
|
+
for (const op of operations) {
|
|
60
|
+
if (!op.tokens || op.tokens <= 0) continue;
|
|
61
|
+
const entry = MODEL_CATALOG.find(m => m.name === op.model);
|
|
62
|
+
const price = entry?.pricePerMToken ?? LARGE_PRICE;
|
|
63
|
+
totalCost += (op.tokens / 1_000_000) * price;
|
|
64
|
+
totalLargeCost += (op.tokens / 1_000_000) * LARGE_PRICE;
|
|
65
|
+
totalTokens += op.tokens;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (totalTokens <= 0) return;
|
|
69
|
+
|
|
70
|
+
console.log();
|
|
71
|
+
console.log(ui.dim(` 💰 ${formatCost(totalCost)} total (${totalTokens.toLocaleString()} tokens)`));
|
|
72
|
+
for (const op of operations) {
|
|
73
|
+
if (!op.tokens || op.tokens <= 0) continue;
|
|
74
|
+
const entry = MODEL_CATALOG.find(m => m.name === op.model);
|
|
75
|
+
const price = entry?.pricePerMToken ?? LARGE_PRICE;
|
|
76
|
+
const cost = (op.tokens / 1_000_000) * price;
|
|
77
|
+
const label = op.label || op.model;
|
|
78
|
+
console.log(ui.dim(` ${label}: ${formatCost(cost)} (${op.tokens.toLocaleString()} tokens)`));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (totalCost < totalLargeCost) {
|
|
82
|
+
const savingsPercent = Math.round((1 - totalCost / totalLargeCost) * 100);
|
|
83
|
+
console.log(ui.dim(` Symmetric (all voyage-4-large): ${formatCost(totalLargeCost)} — ${savingsPercent}% savings`));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Check if cost display is enabled via config.
|
|
89
|
+
* @returns {boolean}
|
|
90
|
+
*/
|
|
91
|
+
function isEnabled() {
|
|
92
|
+
const val = getConfigValue('show-cost');
|
|
93
|
+
return val === true || val === 'true';
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Format a cost value for display.
|
|
98
|
+
* @param {number} cost
|
|
99
|
+
* @returns {string}
|
|
100
|
+
*/
|
|
101
|
+
function formatCost(cost) {
|
|
102
|
+
if (cost < 0.000001) return '$0.000000';
|
|
103
|
+
if (cost < 0.01) return `$${cost.toFixed(6)}`;
|
|
104
|
+
return `$${cost.toFixed(4)}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = { showCostSummary, showCombinedCostSummary, isEnabled, formatCost };
|
package/src/lib/explanations.js
CHANGED
|
@@ -589,9 +589,15 @@ const concepts = {
|
|
|
589
589
|
``,
|
|
590
590
|
`${pc.bold('Validate it yourself:')} Use ${pc.cyan('vai benchmark space')} to embed identical text`,
|
|
591
591
|
`with all Voyage 4 models and see the cross-model cosine similarities.`,
|
|
592
|
+
``,
|
|
593
|
+
`${pc.bold('Interactive proof:')} Try the ${pc.cyan('Shared Space Explorer')} at`,
|
|
594
|
+
`${pc.cyan('vaicli.com/shared-space')} — embed text with all three models simultaneously`,
|
|
595
|
+
`and see 0.95+ cross-model similarity in a live 3×3 matrix, scatter plot, and`,
|
|
596
|
+
`cost comparison. Share your results directly to LinkedIn.`,
|
|
592
597
|
].join('\n'),
|
|
593
598
|
links: [
|
|
594
599
|
'https://blog.voyageai.com/2026/01/15/voyage-4-model-family/',
|
|
600
|
+
'https://vaicli.com/shared-space',
|
|
595
601
|
],
|
|
596
602
|
tryIt: [
|
|
597
603
|
'vai benchmark space',
|
package/src/lib/llm.js
CHANGED
|
@@ -147,15 +147,24 @@ class AnthropicProvider {
|
|
|
147
147
|
const json = await res.json();
|
|
148
148
|
const text = json.content?.[0]?.text || '';
|
|
149
149
|
yield text;
|
|
150
|
+
// Yield usage sentinel
|
|
151
|
+
const usage = json.usage || {};
|
|
152
|
+
yield { __usage: { inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0 } };
|
|
150
153
|
return;
|
|
151
154
|
}
|
|
152
155
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
+
// Manual SSE loop to capture usage from streaming events
|
|
157
|
+
const usage = { inputTokens: 0, outputTokens: 0 };
|
|
158
|
+
for await (const chunk of parseSSEWithMeta(res.body)) {
|
|
159
|
+
if (chunk.__event === 'message_start' && chunk.__data?.message?.usage) {
|
|
160
|
+
usage.inputTokens = chunk.__data.message.usage.input_tokens || 0;
|
|
161
|
+
} else if (chunk.__event === 'message_delta' && chunk.__data?.usage) {
|
|
162
|
+
usage.outputTokens = chunk.__data.usage.output_tokens || 0;
|
|
163
|
+
} else if (chunk.__event === 'content_block_delta' && chunk.__data?.delta?.text) {
|
|
164
|
+
yield chunk.__data.delta.text;
|
|
156
165
|
}
|
|
157
|
-
|
|
158
|
-
}
|
|
166
|
+
}
|
|
167
|
+
yield { __usage: usage };
|
|
159
168
|
}
|
|
160
169
|
|
|
161
170
|
/**
|
|
@@ -163,7 +172,7 @@ class AnthropicProvider {
|
|
|
163
172
|
* @param {Array} messages - Conversation messages
|
|
164
173
|
* @param {Array} tools - Tool definitions in Anthropic format
|
|
165
174
|
* @param {object} [options]
|
|
166
|
-
* @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
|
|
175
|
+
* @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
|
|
167
176
|
*/
|
|
168
177
|
async chatWithTools(messages, tools, options = {}) {
|
|
169
178
|
const model = options.model || this.model;
|
|
@@ -200,6 +209,8 @@ class AnthropicProvider {
|
|
|
200
209
|
|
|
201
210
|
const json = await res.json();
|
|
202
211
|
const stopReason = json.stop_reason || 'end_turn';
|
|
212
|
+
const apiUsage = json.usage || {};
|
|
213
|
+
const usage = { inputTokens: apiUsage.input_tokens || 0, outputTokens: apiUsage.output_tokens || 0 };
|
|
203
214
|
|
|
204
215
|
// Check for tool_use blocks
|
|
205
216
|
const toolBlocks = (json.content || []).filter(b => b.type === 'tool_use');
|
|
@@ -212,6 +223,7 @@ class AnthropicProvider {
|
|
|
212
223
|
arguments: b.input,
|
|
213
224
|
})),
|
|
214
225
|
stopReason,
|
|
226
|
+
usage,
|
|
215
227
|
_raw: json.content,
|
|
216
228
|
};
|
|
217
229
|
}
|
|
@@ -222,6 +234,7 @@ class AnthropicProvider {
|
|
|
222
234
|
type: 'text',
|
|
223
235
|
content: textBlocks.map(b => b.text).join(''),
|
|
224
236
|
stopReason,
|
|
237
|
+
usage,
|
|
225
238
|
};
|
|
226
239
|
}
|
|
227
240
|
|
|
@@ -324,6 +337,11 @@ class OpenAIProvider {
|
|
|
324
337
|
messages,
|
|
325
338
|
};
|
|
326
339
|
|
|
340
|
+
// Request usage data in streaming mode
|
|
341
|
+
if (stream) {
|
|
342
|
+
body.stream_options = { include_usage: true };
|
|
343
|
+
}
|
|
344
|
+
|
|
327
345
|
const res = await fetch(`${this.baseUrl}/v1/chat/completions`, {
|
|
328
346
|
method: 'POST',
|
|
329
347
|
headers: {
|
|
@@ -342,14 +360,25 @@ class OpenAIProvider {
|
|
|
342
360
|
const json = await res.json();
|
|
343
361
|
const text = json.choices?.[0]?.message?.content || '';
|
|
344
362
|
yield text;
|
|
363
|
+
const apiUsage = json.usage || {};
|
|
364
|
+
yield { __usage: { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 } };
|
|
345
365
|
return;
|
|
346
366
|
}
|
|
347
367
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
368
|
+
// Manual SSE loop to capture usage from final streaming chunk
|
|
369
|
+
const usage = { inputTokens: 0, outputTokens: 0 };
|
|
370
|
+
for await (const chunk of parseSSEWithMeta(res.body)) {
|
|
371
|
+
const data = chunk.__data;
|
|
372
|
+
if (data === '[DONE]') continue;
|
|
373
|
+
// Final chunk with usage stats (stream_options: include_usage)
|
|
374
|
+
if (data?.usage) {
|
|
375
|
+
usage.inputTokens = data.usage.prompt_tokens || 0;
|
|
376
|
+
usage.outputTokens = data.usage.completion_tokens || 0;
|
|
377
|
+
}
|
|
378
|
+
const content = data?.choices?.[0]?.delta?.content;
|
|
379
|
+
if (content) yield content;
|
|
380
|
+
}
|
|
381
|
+
yield { __usage: usage };
|
|
353
382
|
}
|
|
354
383
|
|
|
355
384
|
/**
|
|
@@ -357,7 +386,7 @@ class OpenAIProvider {
|
|
|
357
386
|
* @param {Array} messages - Conversation messages
|
|
358
387
|
* @param {Array} tools - Tool definitions in OpenAI format
|
|
359
388
|
* @param {object} [options]
|
|
360
|
-
* @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
|
|
389
|
+
* @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
|
|
361
390
|
*/
|
|
362
391
|
async chatWithTools(messages, tools, options = {}) {
|
|
363
392
|
const model = options.model || this.model;
|
|
@@ -389,6 +418,8 @@ class OpenAIProvider {
|
|
|
389
418
|
const choice = json.choices?.[0] || {};
|
|
390
419
|
const msg = choice.message || {};
|
|
391
420
|
const stopReason = choice.finish_reason || 'stop';
|
|
421
|
+
const apiUsage = json.usage || {};
|
|
422
|
+
const usage = { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 };
|
|
392
423
|
|
|
393
424
|
if (msg.tool_calls && msg.tool_calls.length > 0) {
|
|
394
425
|
return {
|
|
@@ -401,6 +432,7 @@ class OpenAIProvider {
|
|
|
401
432
|
: tc.function.arguments,
|
|
402
433
|
})),
|
|
403
434
|
stopReason,
|
|
435
|
+
usage,
|
|
404
436
|
_raw: msg,
|
|
405
437
|
};
|
|
406
438
|
}
|
|
@@ -409,6 +441,7 @@ class OpenAIProvider {
|
|
|
409
441
|
type: 'text',
|
|
410
442
|
content: msg.content || '',
|
|
411
443
|
stopReason,
|
|
444
|
+
usage,
|
|
412
445
|
};
|
|
413
446
|
}
|
|
414
447
|
|
|
@@ -502,14 +535,25 @@ class OllamaProvider {
|
|
|
502
535
|
const json = await res.json();
|
|
503
536
|
const text = json.choices?.[0]?.message?.content || '';
|
|
504
537
|
yield text;
|
|
538
|
+
// Ollama may not return usage, default to 0
|
|
539
|
+
const apiUsage = json.usage || {};
|
|
540
|
+
yield { __usage: { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 } };
|
|
505
541
|
return;
|
|
506
542
|
}
|
|
507
543
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
544
|
+
// Manual SSE loop (Ollama may not support stream_options)
|
|
545
|
+
const usage = { inputTokens: 0, outputTokens: 0 };
|
|
546
|
+
for await (const chunk of parseSSEWithMeta(res.body)) {
|
|
547
|
+
const data = chunk.__data;
|
|
548
|
+
if (data === '[DONE]') continue;
|
|
549
|
+
if (data?.usage) {
|
|
550
|
+
usage.inputTokens = data.usage.prompt_tokens || 0;
|
|
551
|
+
usage.outputTokens = data.usage.completion_tokens || 0;
|
|
552
|
+
}
|
|
553
|
+
const content = data?.choices?.[0]?.delta?.content;
|
|
554
|
+
if (content) yield content;
|
|
555
|
+
}
|
|
556
|
+
yield { __usage: usage };
|
|
513
557
|
}
|
|
514
558
|
|
|
515
559
|
/**
|
|
@@ -517,7 +561,7 @@ class OllamaProvider {
|
|
|
517
561
|
* @param {Array} messages - Conversation messages
|
|
518
562
|
* @param {Array} tools - Tool definitions in OpenAI format
|
|
519
563
|
* @param {object} [options]
|
|
520
|
-
* @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
|
|
564
|
+
* @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
|
|
521
565
|
*/
|
|
522
566
|
async chatWithTools(messages, tools, options = {}) {
|
|
523
567
|
const model = options.model || this.model;
|
|
@@ -544,6 +588,9 @@ class OllamaProvider {
|
|
|
544
588
|
const choice = json.choices?.[0] || {};
|
|
545
589
|
const msg = choice.message || {};
|
|
546
590
|
const stopReason = choice.finish_reason || 'stop';
|
|
591
|
+
// Ollama may not return usage, default to 0
|
|
592
|
+
const apiUsage = json.usage || {};
|
|
593
|
+
const usage = { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 };
|
|
547
594
|
|
|
548
595
|
if (msg.tool_calls && msg.tool_calls.length > 0) {
|
|
549
596
|
return {
|
|
@@ -556,6 +603,7 @@ class OllamaProvider {
|
|
|
556
603
|
: tc.function.arguments,
|
|
557
604
|
})),
|
|
558
605
|
stopReason,
|
|
606
|
+
usage,
|
|
559
607
|
_raw: msg,
|
|
560
608
|
};
|
|
561
609
|
}
|
|
@@ -564,6 +612,7 @@ class OllamaProvider {
|
|
|
564
612
|
type: 'text',
|
|
565
613
|
content: msg.content || '',
|
|
566
614
|
stopReason,
|
|
615
|
+
usage,
|
|
567
616
|
};
|
|
568
617
|
}
|
|
569
618
|
|
|
@@ -622,6 +671,64 @@ class OllamaProvider {
|
|
|
622
671
|
// SSE Stream Parser
|
|
623
672
|
// ============================================
|
|
624
673
|
|
|
674
|
+
/**
|
|
675
|
+
* Parse a Server-Sent Events stream, yielding raw event+data pairs.
|
|
676
|
+
* Unlike parseSSE, this preserves event types and full data objects
|
|
677
|
+
* so callers can extract both content and metadata (e.g. usage stats).
|
|
678
|
+
*
|
|
679
|
+
* @param {ReadableStream} body - Response body stream
|
|
680
|
+
* @yields {{ __event: string|null, __data: object|string }} Parsed SSE events
|
|
681
|
+
*/
|
|
682
|
+
async function* parseSSEWithMeta(body) {
|
|
683
|
+
const decoder = new TextDecoder();
|
|
684
|
+
let buffer = '';
|
|
685
|
+
let currentEvent = null;
|
|
686
|
+
|
|
687
|
+
for await (const chunk of body) {
|
|
688
|
+
buffer += decoder.decode(chunk, { stream: true });
|
|
689
|
+
|
|
690
|
+
const lines = buffer.split('\n');
|
|
691
|
+
buffer = lines.pop() || '';
|
|
692
|
+
|
|
693
|
+
for (const line of lines) {
|
|
694
|
+
if (line.startsWith('event: ')) {
|
|
695
|
+
currentEvent = line.slice(7).trim();
|
|
696
|
+
continue;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
if (line.startsWith('data: ')) {
|
|
700
|
+
const rawData = line.slice(6);
|
|
701
|
+
|
|
702
|
+
if (rawData === '[DONE]') {
|
|
703
|
+
yield { __event: currentEvent, __data: '[DONE]' };
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
let parsed;
|
|
708
|
+
try {
|
|
709
|
+
parsed = JSON.parse(rawData);
|
|
710
|
+
} catch {
|
|
711
|
+
continue;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
yield { __event: currentEvent, __data: parsed };
|
|
715
|
+
currentEvent = null;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// Process remaining buffer
|
|
721
|
+
if (buffer.trim() && buffer.startsWith('data: ')) {
|
|
722
|
+
const rawData = buffer.slice(6);
|
|
723
|
+
if (rawData !== '[DONE]') {
|
|
724
|
+
try {
|
|
725
|
+
const parsed = JSON.parse(rawData);
|
|
726
|
+
yield { __event: currentEvent, __data: parsed };
|
|
727
|
+
} catch { /* skip */ }
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
625
732
|
/**
|
|
626
733
|
* Parse a Server-Sent Events stream.
|
|
627
734
|
* @param {ReadableStream} body - Response body stream
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const CATEGORIES = ['retrieval', 'analysis', 'ingestion', 'domain-specific', 'utility', 'integration'];
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Run quality audit on a workflow definition and its package.
|
|
10
|
+
* @param {object} definition - Parsed workflow JSON
|
|
11
|
+
* @param {object} pkg - Parsed package.json
|
|
12
|
+
* @param {string} [packagePath] - Path to the package directory
|
|
13
|
+
* @returns {Array<{level: string, message: string}>}
|
|
14
|
+
*/
|
|
15
|
+
function qualityAudit(definition, pkg, packagePath) {
|
|
16
|
+
const issues = [];
|
|
17
|
+
|
|
18
|
+
// Package metadata checks
|
|
19
|
+
if (!pkg.description || pkg.description.length < 20) {
|
|
20
|
+
issues.push({ level: 'error', message: 'Package description too short (min 20 chars)' });
|
|
21
|
+
}
|
|
22
|
+
if (!pkg.author) {
|
|
23
|
+
issues.push({ level: 'error', message: 'Package must have an author' });
|
|
24
|
+
}
|
|
25
|
+
if (!pkg.license) {
|
|
26
|
+
issues.push({ level: 'warning', message: 'No license specified' });
|
|
27
|
+
}
|
|
28
|
+
if (!pkg.vai?.category || !CATEGORIES.includes(pkg.vai.category)) {
|
|
29
|
+
issues.push({ level: 'error', message: 'Invalid or missing vai.category' });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// README checks
|
|
33
|
+
if (packagePath) {
|
|
34
|
+
const readmePath = path.join(packagePath, 'README.md');
|
|
35
|
+
if (!fs.existsSync(readmePath)) {
|
|
36
|
+
issues.push({ level: 'error', message: 'Missing README.md' });
|
|
37
|
+
} else {
|
|
38
|
+
const readme = fs.readFileSync(readmePath, 'utf8');
|
|
39
|
+
if (readme.length < 200) {
|
|
40
|
+
issues.push({ level: 'warning', message: 'README is very short (< 200 chars)' });
|
|
41
|
+
}
|
|
42
|
+
if (!readme.includes('## Usage') && !readme.includes('## Install')) {
|
|
43
|
+
issues.push({ level: 'warning', message: 'README should include Usage or Install section' });
|
|
44
|
+
}
|
|
45
|
+
if (readme.includes('TODO')) {
|
|
46
|
+
issues.push({ level: 'warning', message: 'README contains TODO placeholders' });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Workflow definition quality
|
|
52
|
+
if (definition && Array.isArray(definition.steps)) {
|
|
53
|
+
if (definition.steps.length === 1) {
|
|
54
|
+
issues.push({ level: 'suggestion', message: 'Single-step workflows may not warrant a package — consider documenting as a CLI example instead' });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Branding
|
|
59
|
+
if (!definition?.branding?.icon) {
|
|
60
|
+
issues.push({ level: 'suggestion', message: 'Consider adding branding.icon for store display' });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Naming — should be descriptive, not generic
|
|
64
|
+
if (definition?.name && /^(test|my|workflow|demo|example)/i.test(definition.name)) {
|
|
65
|
+
issues.push({ level: 'warning', message: `Workflow name "${definition.name}" is too generic` });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return issues;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
module.exports = { qualityAudit, CATEGORIES };
|