voyageai-cli 1.29.0 → 1.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +82 -8
  2. package/package.json +1 -1
  3. package/src/commands/benchmark.js +22 -8
  4. package/src/commands/chat.js +18 -0
  5. package/src/commands/chunk.js +10 -0
  6. package/src/commands/demo.js +4 -0
  7. package/src/commands/embed.js +13 -0
  8. package/src/commands/estimate.js +3 -0
  9. package/src/commands/eval.js +6 -0
  10. package/src/commands/explain.js +2 -0
  11. package/src/commands/generate.js +2 -0
  12. package/src/commands/ingest.js +4 -0
  13. package/src/commands/init.js +2 -0
  14. package/src/commands/mcp-server.js +2 -0
  15. package/src/commands/models.js +2 -0
  16. package/src/commands/ping.js +7 -0
  17. package/src/commands/pipeline.js +15 -0
  18. package/src/commands/playground.js +52 -6
  19. package/src/commands/query.js +16 -0
  20. package/src/commands/rerank.js +12 -0
  21. package/src/commands/scaffold.js +2 -0
  22. package/src/commands/search.js +11 -0
  23. package/src/commands/similarity.js +9 -0
  24. package/src/commands/store.js +4 -0
  25. package/src/commands/workflow.js +286 -0
  26. package/src/lib/capability-report.js +134 -0
  27. package/src/lib/chat.js +32 -1
  28. package/src/lib/config.js +2 -0
  29. package/src/lib/cost-display.js +107 -0
  30. package/src/lib/explanations.js +6 -0
  31. package/src/lib/llm.js +125 -18
  32. package/src/lib/quality-audit.js +71 -0
  33. package/src/lib/security/blocked-domains.json +17 -0
  34. package/src/lib/security-audit.js +198 -0
  35. package/src/lib/telemetry.js +23 -1
  36. package/src/lib/workflow-scaffold.js +61 -0
  37. package/src/lib/workflow-test-runner.js +208 -0
  38. package/src/lib/workflow.js +128 -2
  39. package/src/playground/announcements.md +9 -0
  40. package/src/playground/assets/announcements/appstore.jpg +0 -0
  41. package/src/playground/assets/announcements/circuits.jpg +0 -0
  42. package/src/playground/assets/announcements/csvingest.jpg +0 -0
  43. package/src/playground/assets/announcements/green-wave.jpg +0 -0
  44. package/src/playground/help/workflow-nodes.js +472 -0
  45. package/src/playground/index.html +1482 -184
@@ -0,0 +1,134 @@
1
+ 'use strict';
2
+
3
+ const CAP_ICONS = {
4
+ NETWORK: '🌐',
5
+ WRITE_DB: '💾',
6
+ LLM: '🤖',
7
+ LOOP: '🔄',
8
+ READ_DB: '📊',
9
+ };
10
+
11
+ const SEVERITY_ICONS = {
12
+ critical: '🔴',
13
+ high: '🟠',
14
+ medium: '🟡',
15
+ low: '🔵',
16
+ };
17
+
18
+ /**
19
+ * Generate a markdown-formatted capability report for a workflow package.
20
+ *
21
+ * @param {object} definition - Parsed workflow JSON
22
+ * @param {Array<{severity: string, message: string, stepId?: string}>} securityFindings
23
+ * @param {Array<{level: string, message: string}>} qualityIssues
24
+ * @param {{ total: number, passed: number, failed: number, results?: Array }} [testResults]
25
+ * @returns {string} Markdown-formatted report
26
+ */
27
+ function generateCapabilityReport(definition, securityFindings, qualityIssues, testResults) {
28
+ const lines = [];
29
+
30
+ const name = definition?.name || 'Unknown Workflow';
31
+ lines.push(`## 📋 Workflow Validation Report: ${name}`);
32
+ lines.push('');
33
+
34
+ // ── Capabilities ──
35
+ const { extractCapabilities } = require('./security-audit');
36
+ const caps = definition ? [...extractCapabilities(definition)] : [];
37
+
38
+ lines.push('### Capabilities');
39
+ if (caps.length === 0) {
40
+ lines.push('No special capabilities detected.');
41
+ } else {
42
+ for (const cap of caps) {
43
+ lines.push(`- ${CAP_ICONS[cap] || '•'} **${cap}**`);
44
+ }
45
+ }
46
+ lines.push('');
47
+
48
+ // ── Security Findings ──
49
+ lines.push('### Security Audit');
50
+ if (!securityFindings || securityFindings.length === 0) {
51
+ lines.push('✅ No security issues found.');
52
+ } else {
53
+ const counts = { critical: 0, high: 0, medium: 0, low: 0 };
54
+ for (const f of securityFindings) {
55
+ counts[f.severity] = (counts[f.severity] || 0) + 1;
56
+ }
57
+ const summary = Object.entries(counts)
58
+ .filter(([, v]) => v > 0)
59
+ .map(([k, v]) => `${SEVERITY_ICONS[k]} ${v} ${k.toUpperCase()}`)
60
+ .join(' | ');
61
+ lines.push(summary);
62
+ lines.push('');
63
+ lines.push('| Severity | Finding | Step |');
64
+ lines.push('|----------|---------|------|');
65
+ for (const f of securityFindings) {
66
+ lines.push(`| ${SEVERITY_ICONS[f.severity]} ${f.severity.toUpperCase()} | ${f.message} | ${f.stepId || '—'} |`);
67
+ }
68
+ }
69
+ lines.push('');
70
+
71
+ // ── Quality ──
72
+ lines.push('### Quality Audit');
73
+ if (!qualityIssues || qualityIssues.length === 0) {
74
+ lines.push('✅ No quality issues found.');
75
+ } else {
76
+ const errorCount = qualityIssues.filter(i => i.level === 'error').length;
77
+ const warningCount = qualityIssues.filter(i => i.level === 'warning').length;
78
+ const suggestionCount = qualityIssues.filter(i => i.level === 'suggestion').length;
79
+
80
+ const parts = [];
81
+ if (errorCount) parts.push(`❌ ${errorCount} error(s)`);
82
+ if (warningCount) parts.push(`⚠️ ${warningCount} warning(s)`);
83
+ if (suggestionCount) parts.push(`💡 ${suggestionCount} suggestion(s)`);
84
+ lines.push(parts.join(' | '));
85
+ lines.push('');
86
+
87
+ for (const issue of qualityIssues) {
88
+ const icon = issue.level === 'error' ? '❌' : issue.level === 'warning' ? '⚠️' : '💡';
89
+ lines.push(`- ${icon} **[${issue.level.toUpperCase()}]** ${issue.message}`);
90
+ }
91
+ }
92
+ lines.push('');
93
+
94
+ // ── Test Results ──
95
+ lines.push('### Test Results');
96
+ if (!testResults) {
97
+ lines.push('⏭️ No test results available.');
98
+ } else if (testResults.total === 0) {
99
+ lines.push('⏭️ No test cases found.');
100
+ } else {
101
+ const status = testResults.failed === 0 ? '✅' : '❌';
102
+ lines.push(`${status} **${testResults.passed}/${testResults.total}** tests passed`);
103
+ if (testResults.results && testResults.results.length > 0) {
104
+ lines.push('');
105
+ for (const r of testResults.results) {
106
+ const icon = r.passed ? '✅' : '❌';
107
+ lines.push(`- ${icon} ${r.name || r.file}`);
108
+ }
109
+ }
110
+ }
111
+ lines.push('');
112
+
113
+ // ── Overall Summary ──
114
+ const criticalCount = (securityFindings || []).filter(f => f.severity === 'critical').length;
115
+ const highCount = (securityFindings || []).filter(f => f.severity === 'high').length;
116
+ const qualityErrors = (qualityIssues || []).filter(i => i.level === 'error').length;
117
+ const testsFailed = testResults ? testResults.failed : 0;
118
+
119
+ lines.push('### Summary');
120
+ if (criticalCount === 0 && highCount === 0 && qualityErrors === 0 && testsFailed === 0) {
121
+ lines.push('✅ **All checks passed.** This workflow is ready for review.');
122
+ } else {
123
+ const issues = [];
124
+ if (criticalCount) issues.push(`${criticalCount} critical security finding(s)`);
125
+ if (highCount) issues.push(`${highCount} high security finding(s)`);
126
+ if (qualityErrors) issues.push(`${qualityErrors} quality error(s)`);
127
+ if (testsFailed) issues.push(`${testsFailed} test failure(s)`);
128
+ lines.push(`⚠️ **Issues found:** ${issues.join(', ')}`);
129
+ }
130
+
131
+ return lines.join('\n');
132
+ }
133
+
134
+ module.exports = { generateCapabilityReport };
package/src/lib/chat.js CHANGED
@@ -202,9 +202,15 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
202
202
  // 3. Generate response (streaming)
203
203
  let fullResponse = '';
204
204
  const stream = opts.stream !== false;
205
+ let llmUsage = { inputTokens: 0, outputTokens: 0 };
205
206
 
206
207
  try {
207
208
  for await (const chunk of llm.chat(messages, { stream })) {
209
+ // Check for __usage sentinel (yielded as final item from LLM providers)
210
+ if (typeof chunk === 'object' && chunk !== null && chunk.__usage) {
211
+ llmUsage = chunk.__usage;
212
+ continue;
213
+ }
208
214
  fullResponse += chunk;
209
215
  yield { type: 'chunk', data: chunk };
210
216
  }
@@ -240,7 +246,13 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
240
246
  metadata: {
241
247
  retrievalTimeMs,
242
248
  generationTimeMs,
243
- tokens,
249
+ tokens: {
250
+ ...tokens,
251
+ llmInput: llmUsage.inputTokens,
252
+ llmOutput: llmUsage.outputTokens,
253
+ },
254
+ llmModel: llm.model,
255
+ llmProvider: llm.name,
244
256
  contextDocsUsed: docs.length,
245
257
  },
246
258
  },
@@ -284,11 +296,18 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
284
296
  // Track messages for the tool-calling loop (mutable copy)
285
297
  const messages = [...initialMessages];
286
298
  const toolCallLog = [];
299
+ const totalLlmUsage = { inputTokens: 0, outputTokens: 0 };
287
300
 
288
301
  // 3. Agent loop
289
302
  for (let iteration = 0; iteration < maxIterations; iteration++) {
290
303
  const response = await llm.chatWithTools(messages, tools);
291
304
 
305
+ // Accumulate LLM usage from each chatWithTools call
306
+ if (response.usage) {
307
+ totalLlmUsage.inputTokens += response.usage.inputTokens || 0;
308
+ totalLlmUsage.outputTokens += response.usage.outputTokens || 0;
309
+ }
310
+
292
311
  // Text response: done
293
312
  if (response.type === 'text') {
294
313
  const fullResponse = response.content;
@@ -321,6 +340,12 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
321
340
  iterationCount: iteration + 1,
322
341
  toolCallCount: toolCallLog.length,
323
342
  totalTimeMs,
343
+ tokens: {
344
+ llmInput: totalLlmUsage.inputTokens,
345
+ llmOutput: totalLlmUsage.outputTokens,
346
+ },
347
+ llmModel: llm.model,
348
+ llmProvider: llm.name,
324
349
  },
325
350
  },
326
351
  };
@@ -406,6 +431,12 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
406
431
  toolCallCount: toolCallLog.length,
407
432
  totalTimeMs: Date.now() - start,
408
433
  maxIterationsReached: true,
434
+ tokens: {
435
+ llmInput: totalLlmUsage.inputTokens,
436
+ llmOutput: totalLlmUsage.outputTokens,
437
+ },
438
+ llmModel: llm.model,
439
+ llmProvider: llm.name,
409
440
  },
410
441
  },
411
442
  };
package/src/lib/config.js CHANGED
@@ -18,6 +18,8 @@ const KEY_MAP = {
18
18
  'llm-api-key': 'llmApiKey',
19
19
  'llm-model': 'llmModel',
20
20
  'llm-base-url': 'llmBaseUrl',
21
+ 'show-cost': 'show-cost',
22
+ 'telemetry': 'telemetry',
21
23
  };
22
24
 
23
25
  // Keys whose values should be masked in output
@@ -0,0 +1,107 @@
1
+ 'use strict';
2
+
3
+ const { MODEL_CATALOG } = require('./catalog');
4
+ const { getConfigValue } = require('./config');
5
+ const ui = require('./ui');
6
+
7
+ const COMPETITOR_PRICE = 0.13; // OpenAI text-embedding-3-large per 1M tokens
8
+ const LARGE_PRICE = 0.12; // voyage-4-large per 1M tokens
9
+
10
+ /**
11
+ * Show a one-line cost summary after a CLI operation.
12
+ * Only displays when `show-cost` config is enabled.
13
+ * Respects --json and --quiet flags.
14
+ *
15
+ * @param {string} model - Model name used
16
+ * @param {number} tokens - Total tokens consumed
17
+ * @param {object} [opts] - Command options
18
+ * @param {boolean} [opts.json] - JSON output mode (suppress cost)
19
+ * @param {boolean} [opts.quiet] - Quiet mode (suppress cost)
20
+ */
21
+ function showCostSummary(model, tokens, opts = {}) {
22
+ if (opts.json || opts.quiet) return;
23
+ if (!isEnabled()) return;
24
+ if (!tokens || tokens <= 0) return;
25
+
26
+ const entry = MODEL_CATALOG.find(m => m.name === model);
27
+ const price = entry?.pricePerMToken ?? LARGE_PRICE;
28
+ const cost = (tokens / 1_000_000) * price;
29
+ const largeCost = (tokens / 1_000_000) * LARGE_PRICE;
30
+
31
+ const costStr = formatCost(cost);
32
+ const tokStr = tokens.toLocaleString();
33
+
34
+ console.log();
35
+ console.log(ui.dim(` 💰 ${costStr} (${tokStr} tokens, ${model})`));
36
+
37
+ if (price < LARGE_PRICE) {
38
+ const savingsPercent = Math.round((1 - price / LARGE_PRICE) * 100);
39
+ const largeStr = formatCost(largeCost);
40
+ console.log(ui.dim(` Symmetric (voyage-4-large): ${largeStr} — ${savingsPercent}% savings`));
41
+ }
42
+ }
43
+
44
+ /**
45
+ * Show a combined cost summary for operations with multiple API calls
46
+ * (e.g., query with embed + rerank).
47
+ *
48
+ * @param {Array<{model: string, tokens: number, label?: string}>} operations
49
+ * @param {object} [opts]
50
+ */
51
+ function showCombinedCostSummary(operations, opts = {}) {
52
+ if (opts.json || opts.quiet) return;
53
+ if (!isEnabled()) return;
54
+
55
+ let totalCost = 0;
56
+ let totalLargeCost = 0;
57
+ let totalTokens = 0;
58
+
59
+ for (const op of operations) {
60
+ if (!op.tokens || op.tokens <= 0) continue;
61
+ const entry = MODEL_CATALOG.find(m => m.name === op.model);
62
+ const price = entry?.pricePerMToken ?? LARGE_PRICE;
63
+ totalCost += (op.tokens / 1_000_000) * price;
64
+ totalLargeCost += (op.tokens / 1_000_000) * LARGE_PRICE;
65
+ totalTokens += op.tokens;
66
+ }
67
+
68
+ if (totalTokens <= 0) return;
69
+
70
+ console.log();
71
+ console.log(ui.dim(` 💰 ${formatCost(totalCost)} total (${totalTokens.toLocaleString()} tokens)`));
72
+ for (const op of operations) {
73
+ if (!op.tokens || op.tokens <= 0) continue;
74
+ const entry = MODEL_CATALOG.find(m => m.name === op.model);
75
+ const price = entry?.pricePerMToken ?? LARGE_PRICE;
76
+ const cost = (op.tokens / 1_000_000) * price;
77
+ const label = op.label || op.model;
78
+ console.log(ui.dim(` ${label}: ${formatCost(cost)} (${op.tokens.toLocaleString()} tokens)`));
79
+ }
80
+
81
+ if (totalCost < totalLargeCost) {
82
+ const savingsPercent = Math.round((1 - totalCost / totalLargeCost) * 100);
83
+ console.log(ui.dim(` Symmetric (all voyage-4-large): ${formatCost(totalLargeCost)} — ${savingsPercent}% savings`));
84
+ }
85
+ }
86
+
87
+ /**
88
+ * Check if cost display is enabled via config.
89
+ * @returns {boolean}
90
+ */
91
+ function isEnabled() {
92
+ const val = getConfigValue('show-cost');
93
+ return val === true || val === 'true';
94
+ }
95
+
96
+ /**
97
+ * Format a cost value for display.
98
+ * @param {number} cost
99
+ * @returns {string}
100
+ */
101
+ function formatCost(cost) {
102
+ if (cost < 0.000001) return '$0.000000';
103
+ if (cost < 0.01) return `$${cost.toFixed(6)}`;
104
+ return `$${cost.toFixed(4)}`;
105
+ }
106
+
107
+ module.exports = { showCostSummary, showCombinedCostSummary, isEnabled, formatCost };
@@ -589,9 +589,15 @@ const concepts = {
589
589
  ``,
590
590
  `${pc.bold('Validate it yourself:')} Use ${pc.cyan('vai benchmark space')} to embed identical text`,
591
591
  `with all Voyage 4 models and see the cross-model cosine similarities.`,
592
+ ``,
593
+ `${pc.bold('Interactive proof:')} Try the ${pc.cyan('Shared Space Explorer')} at`,
594
+ `${pc.cyan('vaicli.com/shared-space')} — embed text with all three models simultaneously`,
595
+ `and see 0.95+ cross-model similarity in a live 3×3 matrix, scatter plot, and`,
596
+ `cost comparison. Share your results directly to LinkedIn.`,
592
597
  ].join('\n'),
593
598
  links: [
594
599
  'https://blog.voyageai.com/2026/01/15/voyage-4-model-family/',
600
+ 'https://vaicli.com/shared-space',
595
601
  ],
596
602
  tryIt: [
597
603
  'vai benchmark space',
package/src/lib/llm.js CHANGED
@@ -147,15 +147,24 @@ class AnthropicProvider {
147
147
  const json = await res.json();
148
148
  const text = json.content?.[0]?.text || '';
149
149
  yield text;
150
+ // Yield usage sentinel
151
+ const usage = json.usage || {};
152
+ yield { __usage: { inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0 } };
150
153
  return;
151
154
  }
152
155
 
153
- yield* parseSSE(res.body, (event, data) => {
154
- if (event === 'content_block_delta' && data.delta?.text) {
155
- return data.delta.text;
156
+ // Manual SSE loop to capture usage from streaming events
157
+ const usage = { inputTokens: 0, outputTokens: 0 };
158
+ for await (const chunk of parseSSEWithMeta(res.body)) {
159
+ if (chunk.__event === 'message_start' && chunk.__data?.message?.usage) {
160
+ usage.inputTokens = chunk.__data.message.usage.input_tokens || 0;
161
+ } else if (chunk.__event === 'message_delta' && chunk.__data?.usage) {
162
+ usage.outputTokens = chunk.__data.usage.output_tokens || 0;
163
+ } else if (chunk.__event === 'content_block_delta' && chunk.__data?.delta?.text) {
164
+ yield chunk.__data.delta.text;
156
165
  }
157
- return null;
158
- });
166
+ }
167
+ yield { __usage: usage };
159
168
  }
160
169
 
161
170
  /**
@@ -163,7 +172,7 @@ class AnthropicProvider {
163
172
  * @param {Array} messages - Conversation messages
164
173
  * @param {Array} tools - Tool definitions in Anthropic format
165
174
  * @param {object} [options]
166
- * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
175
+ * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
167
176
  */
168
177
  async chatWithTools(messages, tools, options = {}) {
169
178
  const model = options.model || this.model;
@@ -200,6 +209,8 @@ class AnthropicProvider {
200
209
 
201
210
  const json = await res.json();
202
211
  const stopReason = json.stop_reason || 'end_turn';
212
+ const apiUsage = json.usage || {};
213
+ const usage = { inputTokens: apiUsage.input_tokens || 0, outputTokens: apiUsage.output_tokens || 0 };
203
214
 
204
215
  // Check for tool_use blocks
205
216
  const toolBlocks = (json.content || []).filter(b => b.type === 'tool_use');
@@ -212,6 +223,7 @@ class AnthropicProvider {
212
223
  arguments: b.input,
213
224
  })),
214
225
  stopReason,
226
+ usage,
215
227
  _raw: json.content,
216
228
  };
217
229
  }
@@ -222,6 +234,7 @@ class AnthropicProvider {
222
234
  type: 'text',
223
235
  content: textBlocks.map(b => b.text).join(''),
224
236
  stopReason,
237
+ usage,
225
238
  };
226
239
  }
227
240
 
@@ -324,6 +337,11 @@ class OpenAIProvider {
324
337
  messages,
325
338
  };
326
339
 
340
+ // Request usage data in streaming mode
341
+ if (stream) {
342
+ body.stream_options = { include_usage: true };
343
+ }
344
+
327
345
  const res = await fetch(`${this.baseUrl}/v1/chat/completions`, {
328
346
  method: 'POST',
329
347
  headers: {
@@ -342,14 +360,25 @@ class OpenAIProvider {
342
360
  const json = await res.json();
343
361
  const text = json.choices?.[0]?.message?.content || '';
344
362
  yield text;
363
+ const apiUsage = json.usage || {};
364
+ yield { __usage: { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 } };
345
365
  return;
346
366
  }
347
367
 
348
- yield* parseSSE(res.body, (_event, data) => {
349
- if (data === '[DONE]') return null;
350
- const content = data.choices?.[0]?.delta?.content;
351
- return content || null;
352
- });
368
+ // Manual SSE loop to capture usage from final streaming chunk
369
+ const usage = { inputTokens: 0, outputTokens: 0 };
370
+ for await (const chunk of parseSSEWithMeta(res.body)) {
371
+ const data = chunk.__data;
372
+ if (data === '[DONE]') continue;
373
+ // Final chunk with usage stats (stream_options: include_usage)
374
+ if (data?.usage) {
375
+ usage.inputTokens = data.usage.prompt_tokens || 0;
376
+ usage.outputTokens = data.usage.completion_tokens || 0;
377
+ }
378
+ const content = data?.choices?.[0]?.delta?.content;
379
+ if (content) yield content;
380
+ }
381
+ yield { __usage: usage };
353
382
  }
354
383
 
355
384
  /**
@@ -357,7 +386,7 @@ class OpenAIProvider {
357
386
  * @param {Array} messages - Conversation messages
358
387
  * @param {Array} tools - Tool definitions in OpenAI format
359
388
  * @param {object} [options]
360
- * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
389
+ * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
361
390
  */
362
391
  async chatWithTools(messages, tools, options = {}) {
363
392
  const model = options.model || this.model;
@@ -389,6 +418,8 @@ class OpenAIProvider {
389
418
  const choice = json.choices?.[0] || {};
390
419
  const msg = choice.message || {};
391
420
  const stopReason = choice.finish_reason || 'stop';
421
+ const apiUsage = json.usage || {};
422
+ const usage = { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 };
392
423
 
393
424
  if (msg.tool_calls && msg.tool_calls.length > 0) {
394
425
  return {
@@ -401,6 +432,7 @@ class OpenAIProvider {
401
432
  : tc.function.arguments,
402
433
  })),
403
434
  stopReason,
435
+ usage,
404
436
  _raw: msg,
405
437
  };
406
438
  }
@@ -409,6 +441,7 @@ class OpenAIProvider {
409
441
  type: 'text',
410
442
  content: msg.content || '',
411
443
  stopReason,
444
+ usage,
412
445
  };
413
446
  }
414
447
 
@@ -502,14 +535,25 @@ class OllamaProvider {
502
535
  const json = await res.json();
503
536
  const text = json.choices?.[0]?.message?.content || '';
504
537
  yield text;
538
+ // Ollama may not return usage, default to 0
539
+ const apiUsage = json.usage || {};
540
+ yield { __usage: { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 } };
505
541
  return;
506
542
  }
507
543
 
508
- yield* parseSSE(res.body, (_event, data) => {
509
- if (data === '[DONE]') return null;
510
- const content = data.choices?.[0]?.delta?.content;
511
- return content || null;
512
- });
544
+ // Manual SSE loop (Ollama may not support stream_options)
545
+ const usage = { inputTokens: 0, outputTokens: 0 };
546
+ for await (const chunk of parseSSEWithMeta(res.body)) {
547
+ const data = chunk.__data;
548
+ if (data === '[DONE]') continue;
549
+ if (data?.usage) {
550
+ usage.inputTokens = data.usage.prompt_tokens || 0;
551
+ usage.outputTokens = data.usage.completion_tokens || 0;
552
+ }
553
+ const content = data?.choices?.[0]?.delta?.content;
554
+ if (content) yield content;
555
+ }
556
+ yield { __usage: usage };
513
557
  }
514
558
 
515
559
  /**
@@ -517,7 +561,7 @@ class OllamaProvider {
517
561
  * @param {Array} messages - Conversation messages
518
562
  * @param {Array} tools - Tool definitions in OpenAI format
519
563
  * @param {object} [options]
520
- * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
564
+ * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
521
565
  */
522
566
  async chatWithTools(messages, tools, options = {}) {
523
567
  const model = options.model || this.model;
@@ -544,6 +588,9 @@ class OllamaProvider {
544
588
  const choice = json.choices?.[0] || {};
545
589
  const msg = choice.message || {};
546
590
  const stopReason = choice.finish_reason || 'stop';
591
+ // Ollama may not return usage, default to 0
592
+ const apiUsage = json.usage || {};
593
+ const usage = { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 };
547
594
 
548
595
  if (msg.tool_calls && msg.tool_calls.length > 0) {
549
596
  return {
@@ -556,6 +603,7 @@ class OllamaProvider {
556
603
  : tc.function.arguments,
557
604
  })),
558
605
  stopReason,
606
+ usage,
559
607
  _raw: msg,
560
608
  };
561
609
  }
@@ -564,6 +612,7 @@ class OllamaProvider {
564
612
  type: 'text',
565
613
  content: msg.content || '',
566
614
  stopReason,
615
+ usage,
567
616
  };
568
617
  }
569
618
 
@@ -622,6 +671,64 @@ class OllamaProvider {
622
671
  // SSE Stream Parser
623
672
  // ============================================
624
673
 
674
+ /**
675
+ * Parse a Server-Sent Events stream, yielding raw event+data pairs.
676
+ * Unlike parseSSE, this preserves event types and full data objects
677
+ * so callers can extract both content and metadata (e.g. usage stats).
678
+ *
679
+ * @param {ReadableStream} body - Response body stream
680
+ * @yields {{ __event: string|null, __data: object|string }} Parsed SSE events
681
+ */
682
+ async function* parseSSEWithMeta(body) {
683
+ const decoder = new TextDecoder();
684
+ let buffer = '';
685
+ let currentEvent = null;
686
+
687
+ for await (const chunk of body) {
688
+ buffer += decoder.decode(chunk, { stream: true });
689
+
690
+ const lines = buffer.split('\n');
691
+ buffer = lines.pop() || '';
692
+
693
+ for (const line of lines) {
694
+ if (line.startsWith('event: ')) {
695
+ currentEvent = line.slice(7).trim();
696
+ continue;
697
+ }
698
+
699
+ if (line.startsWith('data: ')) {
700
+ const rawData = line.slice(6);
701
+
702
+ if (rawData === '[DONE]') {
703
+ yield { __event: currentEvent, __data: '[DONE]' };
704
+ return;
705
+ }
706
+
707
+ let parsed;
708
+ try {
709
+ parsed = JSON.parse(rawData);
710
+ } catch {
711
+ continue;
712
+ }
713
+
714
+ yield { __event: currentEvent, __data: parsed };
715
+ currentEvent = null;
716
+ }
717
+ }
718
+ }
719
+
720
+ // Process remaining buffer
721
+ if (buffer.trim() && buffer.startsWith('data: ')) {
722
+ const rawData = buffer.slice(6);
723
+ if (rawData !== '[DONE]') {
724
+ try {
725
+ const parsed = JSON.parse(rawData);
726
+ yield { __event: currentEvent, __data: parsed };
727
+ } catch { /* skip */ }
728
+ }
729
+ }
730
+ }
731
+
625
732
  /**
626
733
  * Parse a Server-Sent Events stream.
627
734
  * @param {ReadableStream} body - Response body stream
@@ -0,0 +1,71 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+
6
+ const CATEGORIES = ['retrieval', 'analysis', 'ingestion', 'domain-specific', 'utility', 'integration'];
7
+
8
+ /**
9
+ * Run quality audit on a workflow definition and its package.
10
+ * @param {object} definition - Parsed workflow JSON
11
+ * @param {object} pkg - Parsed package.json
12
+ * @param {string} [packagePath] - Path to the package directory
13
+ * @returns {Array<{level: string, message: string}>}
14
+ */
15
+ function qualityAudit(definition, pkg, packagePath) {
16
+ const issues = [];
17
+
18
+ // Package metadata checks
19
+ if (!pkg.description || pkg.description.length < 20) {
20
+ issues.push({ level: 'error', message: 'Package description too short (min 20 chars)' });
21
+ }
22
+ if (!pkg.author) {
23
+ issues.push({ level: 'error', message: 'Package must have an author' });
24
+ }
25
+ if (!pkg.license) {
26
+ issues.push({ level: 'warning', message: 'No license specified' });
27
+ }
28
+ if (!pkg.vai?.category || !CATEGORIES.includes(pkg.vai.category)) {
29
+ issues.push({ level: 'error', message: 'Invalid or missing vai.category' });
30
+ }
31
+
32
+ // README checks
33
+ if (packagePath) {
34
+ const readmePath = path.join(packagePath, 'README.md');
35
+ if (!fs.existsSync(readmePath)) {
36
+ issues.push({ level: 'error', message: 'Missing README.md' });
37
+ } else {
38
+ const readme = fs.readFileSync(readmePath, 'utf8');
39
+ if (readme.length < 200) {
40
+ issues.push({ level: 'warning', message: 'README is very short (< 200 chars)' });
41
+ }
42
+ if (!readme.includes('## Usage') && !readme.includes('## Install')) {
43
+ issues.push({ level: 'warning', message: 'README should include Usage or Install section' });
44
+ }
45
+ if (readme.includes('TODO')) {
46
+ issues.push({ level: 'warning', message: 'README contains TODO placeholders' });
47
+ }
48
+ }
49
+ }
50
+
51
+ // Workflow definition quality
52
+ if (definition && Array.isArray(definition.steps)) {
53
+ if (definition.steps.length === 1) {
54
+ issues.push({ level: 'suggestion', message: 'Single-step workflows may not warrant a package — consider documenting as a CLI example instead' });
55
+ }
56
+ }
57
+
58
+ // Branding
59
+ if (!definition?.branding?.icon) {
60
+ issues.push({ level: 'suggestion', message: 'Consider adding branding.icon for store display' });
61
+ }
62
+
63
+ // Naming — should be descriptive, not generic
64
+ if (definition?.name && /^(test|my|workflow|demo|example)/i.test(definition.name)) {
65
+ issues.push({ level: 'warning', message: `Workflow name "${definition.name}" is too generic` });
66
+ }
67
+
68
+ return issues;
69
+ }
70
+
71
+ module.exports = { qualityAudit, CATEGORIES };