voyageai-cli 1.29.0 → 1.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -8
- package/package.json +1 -1
- package/src/cli.js +6 -0
- package/src/commands/benchmark.js +22 -8
- package/src/commands/chat.js +50 -11
- package/src/commands/chunk.js +10 -0
- package/src/commands/demo.js +4 -0
- package/src/commands/embed.js +13 -0
- package/src/commands/estimate.js +3 -0
- package/src/commands/eval.js +6 -0
- package/src/commands/explain.js +2 -0
- package/src/commands/export.js +124 -0
- package/src/commands/generate.js +2 -0
- package/src/commands/import.js +195 -0
- package/src/commands/index-workspace.js +239 -0
- package/src/commands/ingest.js +4 -0
- package/src/commands/init.js +2 -0
- package/src/commands/mcp-server.js +115 -3
- package/src/commands/models.js +2 -0
- package/src/commands/ping.js +7 -0
- package/src/commands/pipeline.js +15 -0
- package/src/commands/playground.js +163 -9
- package/src/commands/query.js +16 -0
- package/src/commands/rerank.js +12 -0
- package/src/commands/scaffold.js +2 -0
- package/src/commands/search.js +11 -0
- package/src/commands/similarity.js +9 -0
- package/src/commands/store.js +4 -0
- package/src/commands/workflow.js +286 -0
- package/src/lib/capability-report.js +134 -0
- package/src/lib/chat.js +32 -1
- package/src/lib/config.js +2 -0
- package/src/lib/cost-display.js +107 -0
- package/src/lib/explanations.js +6 -0
- package/src/lib/export/contexts/benchmark-export.js +27 -0
- package/src/lib/export/contexts/chat-export.js +41 -0
- package/src/lib/export/contexts/explore-export.js +22 -0
- package/src/lib/export/contexts/search-export.js +54 -0
- package/src/lib/export/contexts/workflow-export.js +80 -0
- package/src/lib/export/formats/clipboard-export.js +29 -0
- package/src/lib/export/formats/csv-export.js +45 -0
- package/src/lib/export/formats/json-export.js +50 -0
- package/src/lib/export/formats/markdown-export.js +189 -0
- package/src/lib/export/formats/mermaid-export.js +274 -0
- package/src/lib/export/formats/pdf-export.js +117 -0
- package/src/lib/export/formats/png-export.js +96 -0
- package/src/lib/export/formats/svg-export.js +116 -0
- package/src/lib/export/index.js +175 -0
- package/src/lib/llm.js +125 -18
- package/src/lib/quality-audit.js +71 -0
- package/src/lib/security/blocked-domains.json +17 -0
- package/src/lib/security-audit.js +198 -0
- package/src/lib/telemetry.js +23 -1
- package/src/lib/workflow-scaffold.js +61 -0
- package/src/lib/workflow-test-runner.js +208 -0
- package/src/lib/workflow.js +333 -28
- package/src/mcp/install.js +280 -7
- package/src/mcp/schemas/index.js +40 -0
- package/src/mcp/server.js +2 -0
- package/src/mcp/tools/workspace.js +463 -0
- package/src/playground/announcements.md +56 -0
- package/src/playground/help/workflow-nodes.js +472 -0
- package/src/playground/index.html +13134 -8507
- package/src/playground/vendor/mermaid.min.js +2811 -0
- package/src/workflows/rag-chat.json +165 -0
- package/src/workflows/tests/consistency-check.happy-path.test.json +28 -0
- package/src/workflows/tests/consistency-check.missing-source.test.json +26 -0
- package/src/workflows/tests/cost-analysis.happy-path.test.json +28 -0
- package/src/workflows/tests/enrich-and-ingest.happy-path.test.json +38 -0
- package/src/workflows/tests/enrich-and-ingest.notify-fails.test.json +38 -0
- package/src/workflows/tests/intelligent-ingest.all-filtered.test.json +26 -0
- package/src/workflows/tests/intelligent-ingest.happy-path.test.json +28 -0
- package/src/workflows/tests/kb-health-report.custom-queries.test.json +24 -0
- package/src/workflows/tests/kb-health-report.happy-path.test.json +26 -0
- package/src/workflows/tests/multi-collection-search.happy-path.test.json +40 -0
- package/src/workflows/tests/multi-collection-search.one-empty.test.json +28 -0
- package/src/workflows/tests/rag-chat.happy-path.test.json +26 -0
- package/src/workflows/tests/rag-chat.no-relevant-results.test.json +25 -0
- package/src/workflows/tests/research-and-summarize.happy-path.test.json +33 -0
- package/src/workflows/tests/research-and-summarize.no-results.test.json +29 -0
- package/src/workflows/tests/search-with-fallback.empty-both.test.json +24 -0
- package/src/workflows/tests/search-with-fallback.fallback-branch.test.json +24 -0
- package/src/workflows/tests/search-with-fallback.happy-path.test.json +27 -0
- package/src/workflows/tests/smart-ingest.duplicate-detected.test.json +34 -0
- package/src/workflows/tests/smart-ingest.happy-path.test.json +31 -0
package/src/commands/workflow.js
CHANGED
|
@@ -188,6 +188,12 @@ function registerWorkflow(program) {
|
|
|
188
188
|
}
|
|
189
189
|
|
|
190
190
|
// Execute workflow
|
|
191
|
+
const telemetry = require('../lib/telemetry');
|
|
192
|
+
const wfDone = telemetry.timer('cli_workflow_run', {
|
|
193
|
+
workflowName,
|
|
194
|
+
stepCount: definition.steps?.length || 0,
|
|
195
|
+
isBuiltin: !!(definition._source === 'builtin'),
|
|
196
|
+
});
|
|
191
197
|
try {
|
|
192
198
|
const result = await executeWorkflow(definition, {
|
|
193
199
|
inputs: opts.input,
|
|
@@ -224,6 +230,8 @@ function registerWorkflow(program) {
|
|
|
224
230
|
console.error();
|
|
225
231
|
}
|
|
226
232
|
|
|
233
|
+
wfDone();
|
|
234
|
+
|
|
227
235
|
// Output
|
|
228
236
|
if (opts.json) {
|
|
229
237
|
console.log(JSON.stringify(result.output, null, 2));
|
|
@@ -261,6 +269,269 @@ function registerWorkflow(program) {
|
|
|
261
269
|
}
|
|
262
270
|
});
|
|
263
271
|
|
|
272
|
+
// ── workflow check <path> ──
|
|
273
|
+
wfCmd
|
|
274
|
+
.command('check <path>')
|
|
275
|
+
.description('Run validation, security, and quality checks on a workflow package')
|
|
276
|
+
.option('--security', 'Run security checks only', false)
|
|
277
|
+
.option('--quality', 'Run quality checks only', false)
|
|
278
|
+
.option('--all', 'Run all check tiers', false)
|
|
279
|
+
.option('--json', 'Output machine-readable JSON', false)
|
|
280
|
+
.option('--ci', 'Output CI-optimized JSON with summary metadata', false)
|
|
281
|
+
.action((pkgPath, opts) => {
|
|
282
|
+
const { validateSchemaEnhanced, loadWorkflow } = require('../lib/workflow');
|
|
283
|
+
const { securityAudit, extractCapabilities } = require('../lib/security-audit');
|
|
284
|
+
const { qualityAudit } = require('../lib/quality-audit');
|
|
285
|
+
const fs = require('fs');
|
|
286
|
+
|
|
287
|
+
const resolvedPath = path.resolve(pkgPath);
|
|
288
|
+
const runAll = opts.all || (!opts.security && !opts.quality);
|
|
289
|
+
|
|
290
|
+
// Load workflow definition
|
|
291
|
+
let definition;
|
|
292
|
+
let pkg = {};
|
|
293
|
+
let workflowFile;
|
|
294
|
+
|
|
295
|
+
try {
|
|
296
|
+
// Check if it's a directory (package) or a single file
|
|
297
|
+
const stat = fs.statSync(resolvedPath);
|
|
298
|
+
if (stat.isDirectory()) {
|
|
299
|
+
// Package directory
|
|
300
|
+
const pkgJsonPath = path.join(resolvedPath, 'package.json');
|
|
301
|
+
if (fs.existsSync(pkgJsonPath)) {
|
|
302
|
+
pkg = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8'));
|
|
303
|
+
}
|
|
304
|
+
// Find workflow file
|
|
305
|
+
workflowFile = pkg.main || 'workflow.json';
|
|
306
|
+
const wfPath = path.join(resolvedPath, workflowFile);
|
|
307
|
+
if (fs.existsSync(wfPath)) {
|
|
308
|
+
definition = JSON.parse(fs.readFileSync(wfPath, 'utf8'));
|
|
309
|
+
} else {
|
|
310
|
+
console.error(ui.error(`Workflow file not found: ${wfPath}`));
|
|
311
|
+
process.exit(1);
|
|
312
|
+
}
|
|
313
|
+
} else {
|
|
314
|
+
// Single file
|
|
315
|
+
definition = loadWorkflow(resolvedPath);
|
|
316
|
+
}
|
|
317
|
+
} catch (err) {
|
|
318
|
+
console.error(ui.error(err.message));
|
|
319
|
+
process.exit(1);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const results = { schema: [], security: [], quality: [], capabilities: [] };
|
|
323
|
+
|
|
324
|
+
// L1: Schema validation (always runs)
|
|
325
|
+
if (runAll || (!opts.security && !opts.quality)) {
|
|
326
|
+
results.schema = validateSchemaEnhanced(definition);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// L2: Security
|
|
330
|
+
if (runAll || opts.security) {
|
|
331
|
+
const packageDir = fs.statSync(resolvedPath).isDirectory() ? resolvedPath : null;
|
|
332
|
+
results.security = securityAudit(definition, packageDir);
|
|
333
|
+
results.capabilities = [...extractCapabilities(definition)];
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// L3: Quality
|
|
337
|
+
if (runAll || opts.quality) {
|
|
338
|
+
const packageDir = fs.statSync(resolvedPath).isDirectory() ? resolvedPath : null;
|
|
339
|
+
results.quality = qualityAudit(definition, pkg, packageDir);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// CI output (machine-readable JSON with summary metadata)
|
|
343
|
+
if (opts.ci) {
|
|
344
|
+
const criticalCount = results.security.filter(f => f.severity === 'critical').length;
|
|
345
|
+
const highCount = results.security.filter(f => f.severity === 'high').length;
|
|
346
|
+
const schemaPass = results.schema.length === 0;
|
|
347
|
+
const securityPass = criticalCount === 0 && highCount === 0;
|
|
348
|
+
const qualityPass = results.quality.filter(i => i.level === 'error').length === 0;
|
|
349
|
+
const ciOutput = {
|
|
350
|
+
schema: { pass: schemaPass, errors: results.schema },
|
|
351
|
+
security: { pass: securityPass, findings: results.security },
|
|
352
|
+
quality: { pass: qualityPass, issues: results.quality },
|
|
353
|
+
capabilities: results.capabilities,
|
|
354
|
+
summary: {
|
|
355
|
+
passAll: schemaPass && securityPass && qualityPass,
|
|
356
|
+
criticalCount,
|
|
357
|
+
highCount,
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
console.log(JSON.stringify(ciOutput, null, 2));
|
|
361
|
+
if (criticalCount > 0) process.exit(2);
|
|
362
|
+
if (highCount > 0) process.exit(1);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// JSON output
|
|
367
|
+
if (opts.json) {
|
|
368
|
+
console.log(JSON.stringify(results, null, 2));
|
|
369
|
+
// Exit code 1 if critical/high security findings
|
|
370
|
+
const hasCritical = results.security.some(f => f.severity === 'critical' || f.severity === 'high');
|
|
371
|
+
if (hasCritical) process.exit(1);
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Pretty output
|
|
376
|
+
console.log();
|
|
377
|
+
console.log(pc.bold(`Workflow Check: ${definition.name || pkgPath}`));
|
|
378
|
+
console.log(pc.dim('═'.repeat(50)));
|
|
379
|
+
|
|
380
|
+
// Schema
|
|
381
|
+
if (results.schema.length > 0) {
|
|
382
|
+
console.log();
|
|
383
|
+
console.log(pc.bold('Schema Validation:'));
|
|
384
|
+
for (const e of results.schema) {
|
|
385
|
+
console.log(` ${pc.red('✗')} ${e}`);
|
|
386
|
+
}
|
|
387
|
+
} else if (runAll || (!opts.security && !opts.quality)) {
|
|
388
|
+
console.log();
|
|
389
|
+
console.log(`${pc.bold('Schema Validation:')} ${pc.green('✔ passed')}`);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// Security
|
|
393
|
+
if (runAll || opts.security) {
|
|
394
|
+
console.log();
|
|
395
|
+
console.log(pc.bold('Security Audit:'));
|
|
396
|
+
if (results.security.length === 0) {
|
|
397
|
+
console.log(` ${pc.green('✔')} No security issues found`);
|
|
398
|
+
} else {
|
|
399
|
+
for (const f of results.security) {
|
|
400
|
+
const color = f.severity === 'critical' ? pc.red :
|
|
401
|
+
f.severity === 'high' ? pc.red :
|
|
402
|
+
f.severity === 'medium' ? pc.yellow : pc.dim;
|
|
403
|
+
const icon = f.severity === 'critical' || f.severity === 'high' ? '✗' : '⚠';
|
|
404
|
+
console.log(` ${color(icon)} [${f.severity.toUpperCase()}] ${f.message}`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Capability flags
|
|
409
|
+
if (results.capabilities.length > 0) {
|
|
410
|
+
console.log();
|
|
411
|
+
console.log(pc.bold('Capabilities:'));
|
|
412
|
+
const capIcons = { NETWORK: '🌐', WRITE_DB: '💾', LLM: '🤖', LOOP: '🔄', READ_DB: '📊' };
|
|
413
|
+
for (const cap of results.capabilities) {
|
|
414
|
+
console.log(` ${capIcons[cap] || '•'} ${cap}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// Quality
|
|
420
|
+
if (runAll || opts.quality) {
|
|
421
|
+
console.log();
|
|
422
|
+
console.log(pc.bold('Quality Audit:'));
|
|
423
|
+
if (results.quality.length === 0) {
|
|
424
|
+
console.log(` ${pc.green('✔')} No quality issues found`);
|
|
425
|
+
} else {
|
|
426
|
+
for (const issue of results.quality) {
|
|
427
|
+
const color = issue.level === 'error' ? pc.red :
|
|
428
|
+
issue.level === 'warning' ? pc.yellow : pc.dim;
|
|
429
|
+
const icon = issue.level === 'error' ? '✗' : issue.level === 'warning' ? '⚠' : 'ℹ';
|
|
430
|
+
console.log(` ${color(icon)} [${issue.level.toUpperCase()}] ${issue.message}`);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
console.log();
|
|
436
|
+
|
|
437
|
+
// Exit code 1 if critical/high security findings
|
|
438
|
+
const hasCritical = results.security.some(f => f.severity === 'critical' || f.severity === 'high');
|
|
439
|
+
if (hasCritical) process.exit(1);
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
// ── workflow test <path> ──
|
|
443
|
+
wfCmd
|
|
444
|
+
.command('test <path>')
|
|
445
|
+
.description('Run test fixtures for a workflow package')
|
|
446
|
+
.option('--test <name>', 'Run a specific test case by name')
|
|
447
|
+
.option('--json', 'Output machine-readable JSON', false)
|
|
448
|
+
.action(async (pkgPath, opts) => {
|
|
449
|
+
const { loadWorkflow } = require('../lib/workflow');
|
|
450
|
+
const { runAllTests, loadTestCases } = require('../lib/workflow-test-runner');
|
|
451
|
+
const fs = require('fs');
|
|
452
|
+
|
|
453
|
+
const resolvedPath = path.resolve(pkgPath);
|
|
454
|
+
|
|
455
|
+
// Load workflow definition
|
|
456
|
+
let definition;
|
|
457
|
+
try {
|
|
458
|
+
const stat = fs.statSync(resolvedPath);
|
|
459
|
+
if (stat.isDirectory()) {
|
|
460
|
+
const wfPath = path.join(resolvedPath, 'workflow.json');
|
|
461
|
+
if (fs.existsSync(wfPath)) {
|
|
462
|
+
definition = JSON.parse(fs.readFileSync(wfPath, 'utf8'));
|
|
463
|
+
} else {
|
|
464
|
+
console.error(ui.error(`Workflow file not found: ${wfPath}`));
|
|
465
|
+
process.exit(1);
|
|
466
|
+
}
|
|
467
|
+
} else {
|
|
468
|
+
console.error(ui.error('Path must be a workflow package directory'));
|
|
469
|
+
process.exit(1);
|
|
470
|
+
}
|
|
471
|
+
} catch (err) {
|
|
472
|
+
console.error(ui.error(err.message));
|
|
473
|
+
process.exit(1);
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// Check for tests directory
|
|
477
|
+
const testsDir = path.join(resolvedPath, 'tests');
|
|
478
|
+
if (!fs.existsSync(testsDir)) {
|
|
479
|
+
console.error(ui.error('No tests/ directory found in package'));
|
|
480
|
+
process.exit(1);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
const testCases = loadTestCases(resolvedPath);
|
|
484
|
+
if (testCases.length === 0) {
|
|
485
|
+
console.error(ui.error('No *.test.json files found in tests/'));
|
|
486
|
+
process.exit(1);
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
try {
|
|
490
|
+
const aggregate = await runAllTests(definition, resolvedPath, {
|
|
491
|
+
testName: opts.test,
|
|
492
|
+
});
|
|
493
|
+
|
|
494
|
+
if (opts.json) {
|
|
495
|
+
console.log(JSON.stringify(aggregate, null, 2));
|
|
496
|
+
if (aggregate.failed > 0) process.exit(1);
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// Pretty output
|
|
501
|
+
console.log();
|
|
502
|
+
console.log(pc.bold(`Workflow Tests: ${definition.name || pkgPath}`));
|
|
503
|
+
console.log(pc.dim('═'.repeat(50)));
|
|
504
|
+
console.log();
|
|
505
|
+
|
|
506
|
+
for (const result of aggregate.results) {
|
|
507
|
+
const icon = result.passed ? pc.green('✔') : pc.red('✗');
|
|
508
|
+
console.log(`${icon} ${result.name || result.file}`);
|
|
509
|
+
|
|
510
|
+
if (result.error) {
|
|
511
|
+
console.log(` ${pc.red(result.error)}`);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
for (const assertion of result.assertions) {
|
|
515
|
+
const aIcon = assertion.pass ? pc.green(' ✔') : pc.red(' ✗');
|
|
516
|
+
console.log(`${aIcon} ${assertion.message}`);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
for (const err of (result.errors || [])) {
|
|
520
|
+
console.log(` ${pc.red('Error:')} ${err}`);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
console.log();
|
|
525
|
+
console.log(`${pc.bold('Summary:')} ${pc.green(`${aggregate.passed} passed`)}, ${aggregate.failed > 0 ? pc.red(`${aggregate.failed} failed`) : `${aggregate.failed} failed`}`);
|
|
526
|
+
console.log();
|
|
527
|
+
|
|
528
|
+
if (aggregate.failed > 0) process.exit(1);
|
|
529
|
+
} catch (err) {
|
|
530
|
+
console.error(ui.error(err.message));
|
|
531
|
+
process.exit(1);
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
|
|
264
535
|
// ── workflow validate <file> ──
|
|
265
536
|
wfCmd
|
|
266
537
|
.command('validate <file>')
|
|
@@ -404,9 +675,11 @@ function registerWorkflow(program) {
|
|
|
404
675
|
packageName = WORKFLOW_PREFIX + packageName;
|
|
405
676
|
}
|
|
406
677
|
|
|
678
|
+
const telemetry = require('../lib/telemetry');
|
|
407
679
|
console.log(`Installing ${pc.cyan(packageName)}...`);
|
|
408
680
|
|
|
409
681
|
try {
|
|
682
|
+
telemetry.send('cli_workflow_install', { packageName });
|
|
410
683
|
const result = installPackage(packageName, { global: opts.global });
|
|
411
684
|
console.log(`${pc.green('✔')} Downloaded ${pc.cyan(packageName)}@${result.version}`);
|
|
412
685
|
|
|
@@ -417,6 +690,17 @@ function registerWorkflow(program) {
|
|
|
417
690
|
const steps = validation.definition?.steps?.length || 0;
|
|
418
691
|
const tools = (validation.pkg?.vai?.tools || []).join(', ');
|
|
419
692
|
console.log(`${pc.green('✔')} Validated workflow definition (${steps} steps${tools ? `, tools: ${tools}` : ''})`);
|
|
693
|
+
|
|
694
|
+
// Display capability flags
|
|
695
|
+
if (validation.definition) {
|
|
696
|
+
const { extractCapabilities } = require('../lib/security-audit');
|
|
697
|
+
const caps = extractCapabilities(validation.definition);
|
|
698
|
+
if (caps.size > 0) {
|
|
699
|
+
const capIcons = { NETWORK: '🌐', WRITE_DB: '💾', LLM: '🤖', LOOP: '🔄', READ_DB: '📊' };
|
|
700
|
+
const capList = [...caps].map(c => `${capIcons[c] || '•'} ${c}`).join(' ');
|
|
701
|
+
console.log(`${pc.dim('Capabilities:')} ${capList}`);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
420
704
|
} else {
|
|
421
705
|
console.log(`${pc.yellow('⚠')} Validation issues:`);
|
|
422
706
|
for (const e of validation.errors) {
|
|
@@ -479,11 +763,13 @@ function registerWorkflow(program) {
|
|
|
479
763
|
.action(async (query, opts) => {
|
|
480
764
|
const { searchNpm } = require('../lib/npm-utils');
|
|
481
765
|
|
|
766
|
+
const telemetry = require('../lib/telemetry');
|
|
482
767
|
console.log(`Searching npm for vai-workflow packages matching "${query}"...`);
|
|
483
768
|
console.log();
|
|
484
769
|
|
|
485
770
|
try {
|
|
486
771
|
const results = await searchNpm(query, { limit: parseInt(opts.limit, 10) });
|
|
772
|
+
telemetry.send('cli_workflow_search', { query: query.slice(0, 50), resultCount: results.length });
|
|
487
773
|
|
|
488
774
|
if (opts.json) {
|
|
489
775
|
console.log(JSON.stringify(results, null, 2));
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const CAP_ICONS = {
|
|
4
|
+
NETWORK: '🌐',
|
|
5
|
+
WRITE_DB: '💾',
|
|
6
|
+
LLM: '🤖',
|
|
7
|
+
LOOP: '🔄',
|
|
8
|
+
READ_DB: '📊',
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const SEVERITY_ICONS = {
|
|
12
|
+
critical: '🔴',
|
|
13
|
+
high: '🟠',
|
|
14
|
+
medium: '🟡',
|
|
15
|
+
low: '🔵',
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Generate a markdown-formatted capability report for a workflow package.
|
|
20
|
+
*
|
|
21
|
+
* @param {object} definition - Parsed workflow JSON
|
|
22
|
+
* @param {Array<{severity: string, message: string, stepId?: string}>} securityFindings
|
|
23
|
+
* @param {Array<{level: string, message: string}>} qualityIssues
|
|
24
|
+
* @param {{ total: number, passed: number, failed: number, results?: Array }} [testResults]
|
|
25
|
+
* @returns {string} Markdown-formatted report
|
|
26
|
+
*/
|
|
27
|
+
function generateCapabilityReport(definition, securityFindings, qualityIssues, testResults) {
|
|
28
|
+
const lines = [];
|
|
29
|
+
|
|
30
|
+
const name = definition?.name || 'Unknown Workflow';
|
|
31
|
+
lines.push(`## 📋 Workflow Validation Report: ${name}`);
|
|
32
|
+
lines.push('');
|
|
33
|
+
|
|
34
|
+
// ── Capabilities ──
|
|
35
|
+
const { extractCapabilities } = require('./security-audit');
|
|
36
|
+
const caps = definition ? [...extractCapabilities(definition)] : [];
|
|
37
|
+
|
|
38
|
+
lines.push('### Capabilities');
|
|
39
|
+
if (caps.length === 0) {
|
|
40
|
+
lines.push('No special capabilities detected.');
|
|
41
|
+
} else {
|
|
42
|
+
for (const cap of caps) {
|
|
43
|
+
lines.push(`- ${CAP_ICONS[cap] || '•'} **${cap}**`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
lines.push('');
|
|
47
|
+
|
|
48
|
+
// ── Security Findings ──
|
|
49
|
+
lines.push('### Security Audit');
|
|
50
|
+
if (!securityFindings || securityFindings.length === 0) {
|
|
51
|
+
lines.push('✅ No security issues found.');
|
|
52
|
+
} else {
|
|
53
|
+
const counts = { critical: 0, high: 0, medium: 0, low: 0 };
|
|
54
|
+
for (const f of securityFindings) {
|
|
55
|
+
counts[f.severity] = (counts[f.severity] || 0) + 1;
|
|
56
|
+
}
|
|
57
|
+
const summary = Object.entries(counts)
|
|
58
|
+
.filter(([, v]) => v > 0)
|
|
59
|
+
.map(([k, v]) => `${SEVERITY_ICONS[k]} ${v} ${k.toUpperCase()}`)
|
|
60
|
+
.join(' | ');
|
|
61
|
+
lines.push(summary);
|
|
62
|
+
lines.push('');
|
|
63
|
+
lines.push('| Severity | Finding | Step |');
|
|
64
|
+
lines.push('|----------|---------|------|');
|
|
65
|
+
for (const f of securityFindings) {
|
|
66
|
+
lines.push(`| ${SEVERITY_ICONS[f.severity]} ${f.severity.toUpperCase()} | ${f.message} | ${f.stepId || '—'} |`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
lines.push('');
|
|
70
|
+
|
|
71
|
+
// ── Quality ──
|
|
72
|
+
lines.push('### Quality Audit');
|
|
73
|
+
if (!qualityIssues || qualityIssues.length === 0) {
|
|
74
|
+
lines.push('✅ No quality issues found.');
|
|
75
|
+
} else {
|
|
76
|
+
const errorCount = qualityIssues.filter(i => i.level === 'error').length;
|
|
77
|
+
const warningCount = qualityIssues.filter(i => i.level === 'warning').length;
|
|
78
|
+
const suggestionCount = qualityIssues.filter(i => i.level === 'suggestion').length;
|
|
79
|
+
|
|
80
|
+
const parts = [];
|
|
81
|
+
if (errorCount) parts.push(`❌ ${errorCount} error(s)`);
|
|
82
|
+
if (warningCount) parts.push(`⚠️ ${warningCount} warning(s)`);
|
|
83
|
+
if (suggestionCount) parts.push(`💡 ${suggestionCount} suggestion(s)`);
|
|
84
|
+
lines.push(parts.join(' | '));
|
|
85
|
+
lines.push('');
|
|
86
|
+
|
|
87
|
+
for (const issue of qualityIssues) {
|
|
88
|
+
const icon = issue.level === 'error' ? '❌' : issue.level === 'warning' ? '⚠️' : '💡';
|
|
89
|
+
lines.push(`- ${icon} **[${issue.level.toUpperCase()}]** ${issue.message}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
lines.push('');
|
|
93
|
+
|
|
94
|
+
// ── Test Results ──
|
|
95
|
+
lines.push('### Test Results');
|
|
96
|
+
if (!testResults) {
|
|
97
|
+
lines.push('⏭️ No test results available.');
|
|
98
|
+
} else if (testResults.total === 0) {
|
|
99
|
+
lines.push('⏭️ No test cases found.');
|
|
100
|
+
} else {
|
|
101
|
+
const status = testResults.failed === 0 ? '✅' : '❌';
|
|
102
|
+
lines.push(`${status} **${testResults.passed}/${testResults.total}** tests passed`);
|
|
103
|
+
if (testResults.results && testResults.results.length > 0) {
|
|
104
|
+
lines.push('');
|
|
105
|
+
for (const r of testResults.results) {
|
|
106
|
+
const icon = r.passed ? '✅' : '❌';
|
|
107
|
+
lines.push(`- ${icon} ${r.name || r.file}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
lines.push('');
|
|
112
|
+
|
|
113
|
+
// ── Overall Summary ──
|
|
114
|
+
const criticalCount = (securityFindings || []).filter(f => f.severity === 'critical').length;
|
|
115
|
+
const highCount = (securityFindings || []).filter(f => f.severity === 'high').length;
|
|
116
|
+
const qualityErrors = (qualityIssues || []).filter(i => i.level === 'error').length;
|
|
117
|
+
const testsFailed = testResults ? testResults.failed : 0;
|
|
118
|
+
|
|
119
|
+
lines.push('### Summary');
|
|
120
|
+
if (criticalCount === 0 && highCount === 0 && qualityErrors === 0 && testsFailed === 0) {
|
|
121
|
+
lines.push('✅ **All checks passed.** This workflow is ready for review.');
|
|
122
|
+
} else {
|
|
123
|
+
const issues = [];
|
|
124
|
+
if (criticalCount) issues.push(`${criticalCount} critical security finding(s)`);
|
|
125
|
+
if (highCount) issues.push(`${highCount} high security finding(s)`);
|
|
126
|
+
if (qualityErrors) issues.push(`${qualityErrors} quality error(s)`);
|
|
127
|
+
if (testsFailed) issues.push(`${testsFailed} test failure(s)`);
|
|
128
|
+
lines.push(`⚠️ **Issues found:** ${issues.join(', ')}`);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return lines.join('\n');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
module.exports = { generateCapabilityReport };
|
package/src/lib/chat.js
CHANGED
|
@@ -202,9 +202,15 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
|
|
|
202
202
|
// 3. Generate response (streaming)
|
|
203
203
|
let fullResponse = '';
|
|
204
204
|
const stream = opts.stream !== false;
|
|
205
|
+
let llmUsage = { inputTokens: 0, outputTokens: 0 };
|
|
205
206
|
|
|
206
207
|
try {
|
|
207
208
|
for await (const chunk of llm.chat(messages, { stream })) {
|
|
209
|
+
// Check for __usage sentinel (yielded as final item from LLM providers)
|
|
210
|
+
if (typeof chunk === 'object' && chunk !== null && chunk.__usage) {
|
|
211
|
+
llmUsage = chunk.__usage;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
208
214
|
fullResponse += chunk;
|
|
209
215
|
yield { type: 'chunk', data: chunk };
|
|
210
216
|
}
|
|
@@ -240,7 +246,13 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
|
|
|
240
246
|
metadata: {
|
|
241
247
|
retrievalTimeMs,
|
|
242
248
|
generationTimeMs,
|
|
243
|
-
tokens
|
|
249
|
+
tokens: {
|
|
250
|
+
...tokens,
|
|
251
|
+
llmInput: llmUsage.inputTokens,
|
|
252
|
+
llmOutput: llmUsage.outputTokens,
|
|
253
|
+
},
|
|
254
|
+
llmModel: llm.model,
|
|
255
|
+
llmProvider: llm.name,
|
|
244
256
|
contextDocsUsed: docs.length,
|
|
245
257
|
},
|
|
246
258
|
},
|
|
@@ -284,11 +296,18 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
|
|
|
284
296
|
// Track messages for the tool-calling loop (mutable copy)
|
|
285
297
|
const messages = [...initialMessages];
|
|
286
298
|
const toolCallLog = [];
|
|
299
|
+
const totalLlmUsage = { inputTokens: 0, outputTokens: 0 };
|
|
287
300
|
|
|
288
301
|
// 3. Agent loop
|
|
289
302
|
for (let iteration = 0; iteration < maxIterations; iteration++) {
|
|
290
303
|
const response = await llm.chatWithTools(messages, tools);
|
|
291
304
|
|
|
305
|
+
// Accumulate LLM usage from each chatWithTools call
|
|
306
|
+
if (response.usage) {
|
|
307
|
+
totalLlmUsage.inputTokens += response.usage.inputTokens || 0;
|
|
308
|
+
totalLlmUsage.outputTokens += response.usage.outputTokens || 0;
|
|
309
|
+
}
|
|
310
|
+
|
|
292
311
|
// Text response: done
|
|
293
312
|
if (response.type === 'text') {
|
|
294
313
|
const fullResponse = response.content;
|
|
@@ -321,6 +340,12 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
|
|
|
321
340
|
iterationCount: iteration + 1,
|
|
322
341
|
toolCallCount: toolCallLog.length,
|
|
323
342
|
totalTimeMs,
|
|
343
|
+
tokens: {
|
|
344
|
+
llmInput: totalLlmUsage.inputTokens,
|
|
345
|
+
llmOutput: totalLlmUsage.outputTokens,
|
|
346
|
+
},
|
|
347
|
+
llmModel: llm.model,
|
|
348
|
+
llmProvider: llm.name,
|
|
324
349
|
},
|
|
325
350
|
},
|
|
326
351
|
};
|
|
@@ -406,6 +431,12 @@ async function* agentChatTurn({ query, llm, history, opts = {} }) {
|
|
|
406
431
|
toolCallCount: toolCallLog.length,
|
|
407
432
|
totalTimeMs: Date.now() - start,
|
|
408
433
|
maxIterationsReached: true,
|
|
434
|
+
tokens: {
|
|
435
|
+
llmInput: totalLlmUsage.inputTokens,
|
|
436
|
+
llmOutput: totalLlmUsage.outputTokens,
|
|
437
|
+
},
|
|
438
|
+
llmModel: llm.model,
|
|
439
|
+
llmProvider: llm.name,
|
|
409
440
|
},
|
|
410
441
|
},
|
|
411
442
|
};
|
package/src/lib/config.js
CHANGED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { MODEL_CATALOG } = require('./catalog');
|
|
4
|
+
const { getConfigValue } = require('./config');
|
|
5
|
+
const ui = require('./ui');
|
|
6
|
+
|
|
7
|
+
const COMPETITOR_PRICE = 0.13; // OpenAI text-embedding-3-large per 1M tokens
|
|
8
|
+
const LARGE_PRICE = 0.12; // voyage-4-large per 1M tokens
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Show a one-line cost summary after a CLI operation.
|
|
12
|
+
* Only displays when `show-cost` config is enabled.
|
|
13
|
+
* Respects --json and --quiet flags.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} model - Model name used
|
|
16
|
+
* @param {number} tokens - Total tokens consumed
|
|
17
|
+
* @param {object} [opts] - Command options
|
|
18
|
+
* @param {boolean} [opts.json] - JSON output mode (suppress cost)
|
|
19
|
+
* @param {boolean} [opts.quiet] - Quiet mode (suppress cost)
|
|
20
|
+
*/
|
|
21
|
+
function showCostSummary(model, tokens, opts = {}) {
|
|
22
|
+
if (opts.json || opts.quiet) return;
|
|
23
|
+
if (!isEnabled()) return;
|
|
24
|
+
if (!tokens || tokens <= 0) return;
|
|
25
|
+
|
|
26
|
+
const entry = MODEL_CATALOG.find(m => m.name === model);
|
|
27
|
+
const price = entry?.pricePerMToken ?? LARGE_PRICE;
|
|
28
|
+
const cost = (tokens / 1_000_000) * price;
|
|
29
|
+
const largeCost = (tokens / 1_000_000) * LARGE_PRICE;
|
|
30
|
+
|
|
31
|
+
const costStr = formatCost(cost);
|
|
32
|
+
const tokStr = tokens.toLocaleString();
|
|
33
|
+
|
|
34
|
+
console.log();
|
|
35
|
+
console.log(ui.dim(` 💰 ${costStr} (${tokStr} tokens, ${model})`));
|
|
36
|
+
|
|
37
|
+
if (price < LARGE_PRICE) {
|
|
38
|
+
const savingsPercent = Math.round((1 - price / LARGE_PRICE) * 100);
|
|
39
|
+
const largeStr = formatCost(largeCost);
|
|
40
|
+
console.log(ui.dim(` Symmetric (voyage-4-large): ${largeStr} — ${savingsPercent}% savings`));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Show a combined cost summary for operations with multiple API calls
|
|
46
|
+
* (e.g., query with embed + rerank).
|
|
47
|
+
*
|
|
48
|
+
* @param {Array<{model: string, tokens: number, label?: string}>} operations
|
|
49
|
+
* @param {object} [opts]
|
|
50
|
+
*/
|
|
51
|
+
function showCombinedCostSummary(operations, opts = {}) {
|
|
52
|
+
if (opts.json || opts.quiet) return;
|
|
53
|
+
if (!isEnabled()) return;
|
|
54
|
+
|
|
55
|
+
let totalCost = 0;
|
|
56
|
+
let totalLargeCost = 0;
|
|
57
|
+
let totalTokens = 0;
|
|
58
|
+
|
|
59
|
+
for (const op of operations) {
|
|
60
|
+
if (!op.tokens || op.tokens <= 0) continue;
|
|
61
|
+
const entry = MODEL_CATALOG.find(m => m.name === op.model);
|
|
62
|
+
const price = entry?.pricePerMToken ?? LARGE_PRICE;
|
|
63
|
+
totalCost += (op.tokens / 1_000_000) * price;
|
|
64
|
+
totalLargeCost += (op.tokens / 1_000_000) * LARGE_PRICE;
|
|
65
|
+
totalTokens += op.tokens;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (totalTokens <= 0) return;
|
|
69
|
+
|
|
70
|
+
console.log();
|
|
71
|
+
console.log(ui.dim(` 💰 ${formatCost(totalCost)} total (${totalTokens.toLocaleString()} tokens)`));
|
|
72
|
+
for (const op of operations) {
|
|
73
|
+
if (!op.tokens || op.tokens <= 0) continue;
|
|
74
|
+
const entry = MODEL_CATALOG.find(m => m.name === op.model);
|
|
75
|
+
const price = entry?.pricePerMToken ?? LARGE_PRICE;
|
|
76
|
+
const cost = (op.tokens / 1_000_000) * price;
|
|
77
|
+
const label = op.label || op.model;
|
|
78
|
+
console.log(ui.dim(` ${label}: ${formatCost(cost)} (${op.tokens.toLocaleString()} tokens)`));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (totalCost < totalLargeCost) {
|
|
82
|
+
const savingsPercent = Math.round((1 - totalCost / totalLargeCost) * 100);
|
|
83
|
+
console.log(ui.dim(` Symmetric (all voyage-4-large): ${formatCost(totalLargeCost)} — ${savingsPercent}% savings`));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Check if cost display is enabled via config.
|
|
89
|
+
* @returns {boolean}
|
|
90
|
+
*/
|
|
91
|
+
function isEnabled() {
|
|
92
|
+
const val = getConfigValue('show-cost');
|
|
93
|
+
return val === true || val === 'true';
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Format a cost value for display.
|
|
98
|
+
* @param {number} cost
|
|
99
|
+
* @returns {string}
|
|
100
|
+
*/
|
|
101
|
+
function formatCost(cost) {
|
|
102
|
+
if (cost < 0.000001) return '$0.000000';
|
|
103
|
+
if (cost < 0.01) return `$${cost.toFixed(6)}`;
|
|
104
|
+
return `$${cost.toFixed(4)}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
module.exports = { showCostSummary, showCombinedCostSummary, isEnabled, formatCost };
|
package/src/lib/explanations.js
CHANGED
|
@@ -589,9 +589,15 @@ const concepts = {
|
|
|
589
589
|
``,
|
|
590
590
|
`${pc.bold('Validate it yourself:')} Use ${pc.cyan('vai benchmark space')} to embed identical text`,
|
|
591
591
|
`with all Voyage 4 models and see the cross-model cosine similarities.`,
|
|
592
|
+
``,
|
|
593
|
+
`${pc.bold('Interactive proof:')} Try the ${pc.cyan('Shared Space Explorer')} at`,
|
|
594
|
+
`${pc.cyan('vaicli.com/shared-space')} — embed text with all three models simultaneously`,
|
|
595
|
+
`and see 0.95+ cross-model similarity in a live 3×3 matrix, scatter plot, and`,
|
|
596
|
+
`cost comparison. Share your results directly to LinkedIn.`,
|
|
592
597
|
].join('\n'),
|
|
593
598
|
links: [
|
|
594
599
|
'https://blog.voyageai.com/2026/01/15/voyage-4-model-family/',
|
|
600
|
+
'https://vaicli.com/shared-space',
|
|
595
601
|
],
|
|
596
602
|
tryIt: [
|
|
597
603
|
'vai benchmark space',
|