voyageai-cli 1.29.0 → 1.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -8
- package/package.json +1 -1
- package/src/commands/benchmark.js +22 -8
- package/src/commands/chat.js +18 -0
- package/src/commands/chunk.js +10 -0
- package/src/commands/demo.js +4 -0
- package/src/commands/embed.js +13 -0
- package/src/commands/estimate.js +3 -0
- package/src/commands/eval.js +6 -0
- package/src/commands/explain.js +2 -0
- package/src/commands/generate.js +2 -0
- package/src/commands/ingest.js +4 -0
- package/src/commands/init.js +2 -0
- package/src/commands/mcp-server.js +2 -0
- package/src/commands/models.js +2 -0
- package/src/commands/ping.js +7 -0
- package/src/commands/pipeline.js +15 -0
- package/src/commands/playground.js +52 -6
- package/src/commands/query.js +16 -0
- package/src/commands/rerank.js +12 -0
- package/src/commands/scaffold.js +2 -0
- package/src/commands/search.js +11 -0
- package/src/commands/similarity.js +9 -0
- package/src/commands/store.js +4 -0
- package/src/commands/workflow.js +286 -0
- package/src/lib/capability-report.js +134 -0
- package/src/lib/chat.js +32 -1
- package/src/lib/config.js +2 -0
- package/src/lib/cost-display.js +107 -0
- package/src/lib/explanations.js +6 -0
- package/src/lib/llm.js +125 -18
- package/src/lib/quality-audit.js +71 -0
- package/src/lib/security/blocked-domains.json +17 -0
- package/src/lib/security-audit.js +198 -0
- package/src/lib/telemetry.js +23 -1
- package/src/lib/workflow-scaffold.js +61 -0
- package/src/lib/workflow-test-runner.js +208 -0
- package/src/lib/workflow.js +128 -2
- package/src/playground/announcements.md +9 -0
- package/src/playground/assets/announcements/appstore.jpg +0 -0
- package/src/playground/assets/announcements/circuits.jpg +0 -0
- package/src/playground/assets/announcements/csvingest.jpg +0 -0
- package/src/playground/assets/announcements/green-wave.jpg +0 -0
- package/src/playground/help/workflow-nodes.js +472 -0
- package/src/playground/index.html +1482 -184
|
@@ -67,6 +67,9 @@ function loadAnnouncementsFromMarkdown() {
|
|
|
67
67
|
badge: metadata.badge || 'Info',
|
|
68
68
|
published: metadata.published || new Date().toISOString().split('T')[0],
|
|
69
69
|
expires: metadata.expires || '2099-12-31',
|
|
70
|
+
bg_image: metadata.bg_image || null,
|
|
71
|
+
bg_color: metadata.bg_color || null,
|
|
72
|
+
icon: metadata.icon || null,
|
|
70
73
|
cta: {
|
|
71
74
|
label: metadata.cta_label || 'Learn More',
|
|
72
75
|
action: metadata.cta_action || 'link',
|
|
@@ -159,7 +162,9 @@ function createPlaygroundServer() {
|
|
|
159
162
|
try {
|
|
160
163
|
// Serve HTML
|
|
161
164
|
if (req.method === 'GET' && (req.url === '/' || req.url === '/index.html')) {
|
|
162
|
-
const
|
|
165
|
+
const { getVersion } = require('../lib/banner');
|
|
166
|
+
let html = fs.readFileSync(htmlPath, 'utf8');
|
|
167
|
+
html = html.replace('</head>', `<script>window.__VAI_VERSION__="${getVersion()}";</script></head>`);
|
|
163
168
|
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
|
164
169
|
res.end(html);
|
|
165
170
|
return;
|
|
@@ -223,6 +228,25 @@ function createPlaygroundServer() {
|
|
|
223
228
|
return;
|
|
224
229
|
}
|
|
225
230
|
|
|
231
|
+
// Serve announcement assets: /assets/announcements/{filename}
|
|
232
|
+
const assetMatch = req.url.match(/^\/assets\/announcements\/([a-zA-Z0-9_.-]+\.(jpg|jpeg|png|webp|gif))$/);
|
|
233
|
+
if (req.method === 'GET' && assetMatch) {
|
|
234
|
+
const assetPath = path.join(__dirname, '..', 'playground', 'assets', 'announcements', assetMatch[1]);
|
|
235
|
+
if (fs.existsSync(assetPath)) {
|
|
236
|
+
const mimeTypes = { jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', webp: 'image/webp', gif: 'image/gif' };
|
|
237
|
+
const data = fs.readFileSync(assetPath);
|
|
238
|
+
res.writeHead(200, {
|
|
239
|
+
'Content-Type': mimeTypes[assetMatch[2]] || 'application/octet-stream',
|
|
240
|
+
'Cache-Control': 'public, max-age=86400',
|
|
241
|
+
});
|
|
242
|
+
res.end(data);
|
|
243
|
+
} else {
|
|
244
|
+
res.writeHead(404);
|
|
245
|
+
res.end('Asset not found');
|
|
246
|
+
}
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
|
|
226
250
|
// API: Models
|
|
227
251
|
if (req.method === 'GET' && req.url === '/api/models') {
|
|
228
252
|
const models = MODEL_CATALOG.filter(m => !m.legacy && !m.local && !m.unreleased);
|
|
@@ -360,6 +384,14 @@ function createPlaygroundServer() {
|
|
|
360
384
|
return;
|
|
361
385
|
}
|
|
362
386
|
|
|
387
|
+
// API: Workflow node help
|
|
388
|
+
if (req.method === 'GET' && req.url === '/api/workflows/node-help') {
|
|
389
|
+
const nodeHelp = require('../playground/help/workflow-nodes');
|
|
390
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
391
|
+
res.end(JSON.stringify({ nodeHelp }));
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
|
|
363
395
|
// API: Chat config (GET)
|
|
364
396
|
if (req.method === 'GET' && req.url === '/api/chat/config') {
|
|
365
397
|
const { resolveLLMConfig } = require('../lib/llm');
|
|
@@ -656,6 +688,17 @@ function createPlaygroundServer() {
|
|
|
656
688
|
description: def.description || '',
|
|
657
689
|
}));
|
|
658
690
|
|
|
691
|
+
// Derive capabilities from tools
|
|
692
|
+
const toolsList = vai.tools || [];
|
|
693
|
+
const capabilities = [];
|
|
694
|
+
if (toolsList.includes('http')) capabilities.push('NETWORK');
|
|
695
|
+
if (toolsList.includes('ingest') || toolsList.includes('aggregate')) capabilities.push('WRITE_DB');
|
|
696
|
+
if (toolsList.includes('generate')) capabilities.push('LLM');
|
|
697
|
+
if (toolsList.includes('loop') || toolsList.includes('forEach')) capabilities.push('LOOP');
|
|
698
|
+
if (toolsList.some(t => ['query','search','collections','aggregate'].includes(t))) capabilities.push('READ_DB');
|
|
699
|
+
|
|
700
|
+
const isOfficial = (r.name || '').startsWith('@vaicli/');
|
|
701
|
+
|
|
659
702
|
return {
|
|
660
703
|
name: shortName,
|
|
661
704
|
packageName: r.name,
|
|
@@ -663,11 +706,10 @@ function createPlaygroundServer() {
|
|
|
663
706
|
description: r.description || '',
|
|
664
707
|
category,
|
|
665
708
|
tags: vai.tags || [],
|
|
666
|
-
tools:
|
|
667
|
-
steps: vai.steps ||
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
tier: (r.name || '').startsWith('@vaicli/') ? 'official' : 'community',
|
|
709
|
+
tools: toolsList,
|
|
710
|
+
steps: vai.steps || toolsList.length || 0,
|
|
711
|
+
toolCount: toolsList.length,
|
|
712
|
+
tier: isOfficial ? 'official' : 'community',
|
|
671
713
|
downloads: 0,
|
|
672
714
|
featured: FEATURED.includes(shortName),
|
|
673
715
|
installed: installedNames.has(r.name),
|
|
@@ -676,6 +718,10 @@ function createPlaygroundServer() {
|
|
|
676
718
|
author,
|
|
677
719
|
assets,
|
|
678
720
|
inputs,
|
|
721
|
+
capabilities,
|
|
722
|
+
verified: isOfficial,
|
|
723
|
+
security: [],
|
|
724
|
+
rating: null,
|
|
679
725
|
};
|
|
680
726
|
});
|
|
681
727
|
|
package/src/commands/query.js
CHANGED
|
@@ -5,6 +5,7 @@ const { generateEmbeddings, apiRequest } = require('../lib/api');
|
|
|
5
5
|
const { getMongoCollection } = require('../lib/mongo');
|
|
6
6
|
const { loadProject } = require('../lib/project');
|
|
7
7
|
const ui = require('../lib/ui');
|
|
8
|
+
const { showCombinedCostSummary } = require('../lib/cost-display');
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Register the query command on a Commander program.
|
|
@@ -33,6 +34,7 @@ function registerQuery(program) {
|
|
|
33
34
|
.option('-q, --quiet', 'Suppress non-essential output')
|
|
34
35
|
.action(async (text, opts) => {
|
|
35
36
|
let client;
|
|
37
|
+
const telemetry = require('../lib/telemetry');
|
|
36
38
|
try {
|
|
37
39
|
// Merge project config
|
|
38
40
|
const { config: proj } = loadProject();
|
|
@@ -51,6 +53,14 @@ function registerQuery(program) {
|
|
|
51
53
|
process.exit(1);
|
|
52
54
|
}
|
|
53
55
|
|
|
56
|
+
const done = telemetry.timer('cli_query', {
|
|
57
|
+
model,
|
|
58
|
+
rerankModel: doRerank ? rerankModel : undefined,
|
|
59
|
+
rerank: doRerank,
|
|
60
|
+
limit: opts.limit,
|
|
61
|
+
topK: opts.topK,
|
|
62
|
+
});
|
|
63
|
+
|
|
54
64
|
const useColor = !opts.json;
|
|
55
65
|
const useSpinner = useColor && !opts.quiet;
|
|
56
66
|
|
|
@@ -242,8 +252,14 @@ function registerQuery(program) {
|
|
|
242
252
|
if (!opts.quiet) {
|
|
243
253
|
const totalTokens = embedTokens + rerankTokens;
|
|
244
254
|
console.log(ui.dim(` Tokens: ${totalTokens} (embed: ${embedTokens}${rerankTokens ? `, rerank: ${rerankTokens}` : ''})`));
|
|
255
|
+
const costOps = [{ model, tokens: embedTokens, label: `embed (${model})` }];
|
|
256
|
+
if (rerankTokens) costOps.push({ model: rerankModel, tokens: rerankTokens, label: `rerank (${rerankModel})` });
|
|
257
|
+
showCombinedCostSummary(costOps, opts);
|
|
245
258
|
}
|
|
259
|
+
|
|
260
|
+
done({ resultCount: finalResults.length });
|
|
246
261
|
} catch (err) {
|
|
262
|
+
telemetry.send('cli_error', { command: 'query', errorType: err.constructor.name });
|
|
247
263
|
console.error(ui.error(err.message));
|
|
248
264
|
process.exit(1);
|
|
249
265
|
} finally {
|
package/src/commands/rerank.js
CHANGED
|
@@ -4,6 +4,7 @@ const fs = require('fs');
|
|
|
4
4
|
const { DEFAULT_RERANK_MODEL } = require('../lib/catalog');
|
|
5
5
|
const { apiRequest } = require('../lib/api');
|
|
6
6
|
const ui = require('../lib/ui');
|
|
7
|
+
const { showCostSummary } = require('../lib/cost-display');
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Register the rerank command on a Commander program.
|
|
@@ -25,6 +26,7 @@ function registerRerank(program) {
|
|
|
25
26
|
.option('--json', 'Machine-readable JSON output')
|
|
26
27
|
.option('-q, --quiet', 'Suppress non-essential output')
|
|
27
28
|
.action(async (opts) => {
|
|
29
|
+
const telemetry = require('../lib/telemetry');
|
|
28
30
|
try {
|
|
29
31
|
let documents = opts.documents;
|
|
30
32
|
|
|
@@ -81,6 +83,12 @@ function registerRerank(program) {
|
|
|
81
83
|
opts.model = chosenModel;
|
|
82
84
|
}
|
|
83
85
|
|
|
86
|
+
const done = telemetry.timer('cli_rerank', {
|
|
87
|
+
model: opts.model,
|
|
88
|
+
docCount: documents.length,
|
|
89
|
+
topK: opts.topK,
|
|
90
|
+
});
|
|
91
|
+
|
|
84
92
|
const body = {
|
|
85
93
|
query: opts.query,
|
|
86
94
|
documents,
|
|
@@ -120,6 +128,7 @@ function registerRerank(program) {
|
|
|
120
128
|
if (result.usage) {
|
|
121
129
|
console.log(ui.label('Tokens', ui.dim(String(result.usage.total_tokens))));
|
|
122
130
|
}
|
|
131
|
+
showCostSummary(result.model || opts.model, result.usage?.total_tokens || 0, opts);
|
|
123
132
|
console.log('');
|
|
124
133
|
}
|
|
125
134
|
|
|
@@ -134,7 +143,10 @@ function registerRerank(program) {
|
|
|
134
143
|
|
|
135
144
|
console.log('');
|
|
136
145
|
console.log(ui.success('Reranking complete'));
|
|
146
|
+
|
|
147
|
+
done();
|
|
137
148
|
} catch (err) {
|
|
149
|
+
telemetry.send('cli_error', { command: 'rerank', errorType: err.constructor.name });
|
|
138
150
|
console.error(ui.error(err.message));
|
|
139
151
|
process.exit(1);
|
|
140
152
|
}
|
package/src/commands/scaffold.js
CHANGED
|
@@ -55,7 +55,9 @@ function registerScaffold(program) {
|
|
|
55
55
|
.option('--dry-run', 'Show what would be created without writing')
|
|
56
56
|
.option('-q, --quiet', 'Suppress non-essential output')
|
|
57
57
|
.action(async (name, opts) => {
|
|
58
|
+
const telemetry = require('../lib/telemetry');
|
|
58
59
|
try {
|
|
60
|
+
telemetry.send('cli_scaffold', { template: opts.target });
|
|
59
61
|
const target = opts.target;
|
|
60
62
|
const projectDir = path.resolve(process.cwd(), name);
|
|
61
63
|
|
package/src/commands/search.js
CHANGED
|
@@ -4,6 +4,7 @@ const { getDefaultModel } = require('../lib/catalog');
|
|
|
4
4
|
const { generateEmbeddings } = require('../lib/api');
|
|
5
5
|
const { getMongoCollection } = require('../lib/mongo');
|
|
6
6
|
const ui = require('../lib/ui');
|
|
7
|
+
const { showCostSummary } = require('../lib/cost-display');
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Register the search command on a Commander program.
|
|
@@ -29,7 +30,13 @@ function registerSearch(program) {
|
|
|
29
30
|
.option('-q, --quiet', 'Suppress non-essential output')
|
|
30
31
|
.action(async (opts) => {
|
|
31
32
|
let client;
|
|
33
|
+
const telemetry = require('../lib/telemetry');
|
|
32
34
|
try {
|
|
35
|
+
const done = telemetry.timer('cli_search', {
|
|
36
|
+
model: opts.model,
|
|
37
|
+
limit: opts.limit,
|
|
38
|
+
});
|
|
39
|
+
|
|
33
40
|
const useColor = !opts.json;
|
|
34
41
|
const useSpinner = useColor && !opts.quiet;
|
|
35
42
|
let spin;
|
|
@@ -93,9 +100,12 @@ function registerSearch(program) {
|
|
|
93
100
|
if (!opts.quiet) {
|
|
94
101
|
console.log(ui.label('Query', ui.cyan(`"${opts.query}"`)));
|
|
95
102
|
console.log(ui.label('Results', String(cleanResults.length)));
|
|
103
|
+
showCostSummary(opts.model, embedResult.usage?.total_tokens || 0, opts);
|
|
96
104
|
console.log('');
|
|
97
105
|
}
|
|
98
106
|
|
|
107
|
+
done({ resultCount: cleanResults.length });
|
|
108
|
+
|
|
99
109
|
if (cleanResults.length === 0) {
|
|
100
110
|
console.log(ui.yellow('No results found.'));
|
|
101
111
|
return;
|
|
@@ -113,6 +123,7 @@ function registerSearch(program) {
|
|
|
113
123
|
console.log('');
|
|
114
124
|
}
|
|
115
125
|
} catch (err) {
|
|
126
|
+
telemetry.send('cli_error', { command: 'search', errorType: err.constructor.name });
|
|
116
127
|
console.error(ui.error(err.message));
|
|
117
128
|
process.exit(1);
|
|
118
129
|
} finally {
|
|
@@ -5,6 +5,7 @@ const { generateEmbeddings } = require('../lib/api');
|
|
|
5
5
|
const { cosineSimilarity } = require('../lib/math');
|
|
6
6
|
const { getDefaultModel } = require('../lib/catalog');
|
|
7
7
|
const ui = require('../lib/ui');
|
|
8
|
+
const { showCostSummary } = require('../lib/cost-display');
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Register the similarity command on a Commander program.
|
|
@@ -23,6 +24,7 @@ function registerSimilarity(program) {
|
|
|
23
24
|
.option('--json', 'Machine-readable JSON output')
|
|
24
25
|
.option('-q, --quiet', 'Suppress non-essential output')
|
|
25
26
|
.action(async (texts, opts) => {
|
|
27
|
+
const telemetry = require('../lib/telemetry');
|
|
26
28
|
try {
|
|
27
29
|
let textA = null;
|
|
28
30
|
let compareTexts = [];
|
|
@@ -57,6 +59,8 @@ function registerSimilarity(program) {
|
|
|
57
59
|
process.exit(1);
|
|
58
60
|
}
|
|
59
61
|
|
|
62
|
+
const done = telemetry.timer('cli_similarity', { model: opts.model });
|
|
63
|
+
|
|
60
64
|
// Batch all texts into one API call
|
|
61
65
|
const allTexts = [textA, ...compareTexts];
|
|
62
66
|
|
|
@@ -113,6 +117,7 @@ function registerSimilarity(program) {
|
|
|
113
117
|
console.log(ui.label('Text B', `"${truncate(compareTexts[0], 70)}"`));
|
|
114
118
|
console.log(ui.label('Model', ui.cyan(model)));
|
|
115
119
|
console.log(ui.label('Tokens', ui.dim(String(tokens))));
|
|
120
|
+
showCostSummary(model, tokens, opts);
|
|
116
121
|
console.log('');
|
|
117
122
|
} else {
|
|
118
123
|
// One-vs-many comparison
|
|
@@ -152,9 +157,13 @@ function registerSimilarity(program) {
|
|
|
152
157
|
|
|
153
158
|
console.log('');
|
|
154
159
|
console.log(` ${ui.dim(`${results.length} comparisons, ${tokens} tokens`)}`);
|
|
160
|
+
showCostSummary(model, tokens, opts);
|
|
155
161
|
console.log('');
|
|
156
162
|
}
|
|
163
|
+
|
|
164
|
+
done();
|
|
157
165
|
} catch (err) {
|
|
166
|
+
telemetry.send('cli_error', { command: 'similarity', errorType: err.constructor.name });
|
|
158
167
|
console.error(ui.error(err.message));
|
|
159
168
|
process.exit(1);
|
|
160
169
|
}
|
package/src/commands/store.js
CHANGED
|
@@ -29,6 +29,8 @@ function registerStore(program) {
|
|
|
29
29
|
.option('-q, --quiet', 'Suppress non-essential output')
|
|
30
30
|
.action(async (opts) => {
|
|
31
31
|
let client;
|
|
32
|
+
const telemetry = require('../lib/telemetry');
|
|
33
|
+
const done = telemetry.timer('cli_store', { model: opts.model });
|
|
32
34
|
try {
|
|
33
35
|
// Batch mode: .jsonl file
|
|
34
36
|
if (opts.file && opts.file.endsWith('.jsonl')) {
|
|
@@ -102,7 +104,9 @@ function registerStore(program) {
|
|
|
102
104
|
console.log(ui.label('Tokens', String(embedResult.usage.total_tokens)));
|
|
103
105
|
}
|
|
104
106
|
}
|
|
107
|
+
done();
|
|
105
108
|
} catch (err) {
|
|
109
|
+
telemetry.send('cli_error', { command: 'store', errorType: err.constructor.name });
|
|
106
110
|
console.error(ui.error(err.message));
|
|
107
111
|
process.exit(1);
|
|
108
112
|
} finally {
|
package/src/commands/workflow.js
CHANGED
|
@@ -188,6 +188,12 @@ function registerWorkflow(program) {
|
|
|
188
188
|
}
|
|
189
189
|
|
|
190
190
|
// Execute workflow
|
|
191
|
+
const telemetry = require('../lib/telemetry');
|
|
192
|
+
const wfDone = telemetry.timer('cli_workflow_run', {
|
|
193
|
+
workflowName,
|
|
194
|
+
stepCount: definition.steps?.length || 0,
|
|
195
|
+
isBuiltin: !!(definition._source === 'builtin'),
|
|
196
|
+
});
|
|
191
197
|
try {
|
|
192
198
|
const result = await executeWorkflow(definition, {
|
|
193
199
|
inputs: opts.input,
|
|
@@ -224,6 +230,8 @@ function registerWorkflow(program) {
|
|
|
224
230
|
console.error();
|
|
225
231
|
}
|
|
226
232
|
|
|
233
|
+
wfDone();
|
|
234
|
+
|
|
227
235
|
// Output
|
|
228
236
|
if (opts.json) {
|
|
229
237
|
console.log(JSON.stringify(result.output, null, 2));
|
|
@@ -261,6 +269,269 @@ function registerWorkflow(program) {
|
|
|
261
269
|
}
|
|
262
270
|
});
|
|
263
271
|
|
|
272
|
+
// ── workflow check <path> ──
|
|
273
|
+
wfCmd
|
|
274
|
+
.command('check <path>')
|
|
275
|
+
.description('Run validation, security, and quality checks on a workflow package')
|
|
276
|
+
.option('--security', 'Run security checks only', false)
|
|
277
|
+
.option('--quality', 'Run quality checks only', false)
|
|
278
|
+
.option('--all', 'Run all check tiers', false)
|
|
279
|
+
.option('--json', 'Output machine-readable JSON', false)
|
|
280
|
+
.option('--ci', 'Output CI-optimized JSON with summary metadata', false)
|
|
281
|
+
.action((pkgPath, opts) => {
|
|
282
|
+
const { validateSchemaEnhanced, loadWorkflow } = require('../lib/workflow');
|
|
283
|
+
const { securityAudit, extractCapabilities } = require('../lib/security-audit');
|
|
284
|
+
const { qualityAudit } = require('../lib/quality-audit');
|
|
285
|
+
const fs = require('fs');
|
|
286
|
+
|
|
287
|
+
const resolvedPath = path.resolve(pkgPath);
|
|
288
|
+
const runAll = opts.all || (!opts.security && !opts.quality);
|
|
289
|
+
|
|
290
|
+
// Load workflow definition
|
|
291
|
+
let definition;
|
|
292
|
+
let pkg = {};
|
|
293
|
+
let workflowFile;
|
|
294
|
+
|
|
295
|
+
try {
|
|
296
|
+
// Check if it's a directory (package) or a single file
|
|
297
|
+
const stat = fs.statSync(resolvedPath);
|
|
298
|
+
if (stat.isDirectory()) {
|
|
299
|
+
// Package directory
|
|
300
|
+
const pkgJsonPath = path.join(resolvedPath, 'package.json');
|
|
301
|
+
if (fs.existsSync(pkgJsonPath)) {
|
|
302
|
+
pkg = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8'));
|
|
303
|
+
}
|
|
304
|
+
// Find workflow file
|
|
305
|
+
workflowFile = pkg.main || 'workflow.json';
|
|
306
|
+
const wfPath = path.join(resolvedPath, workflowFile);
|
|
307
|
+
if (fs.existsSync(wfPath)) {
|
|
308
|
+
definition = JSON.parse(fs.readFileSync(wfPath, 'utf8'));
|
|
309
|
+
} else {
|
|
310
|
+
console.error(ui.error(`Workflow file not found: ${wfPath}`));
|
|
311
|
+
process.exit(1);
|
|
312
|
+
}
|
|
313
|
+
} else {
|
|
314
|
+
// Single file
|
|
315
|
+
definition = loadWorkflow(resolvedPath);
|
|
316
|
+
}
|
|
317
|
+
} catch (err) {
|
|
318
|
+
console.error(ui.error(err.message));
|
|
319
|
+
process.exit(1);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const results = { schema: [], security: [], quality: [], capabilities: [] };
|
|
323
|
+
|
|
324
|
+
// L1: Schema validation (always runs)
|
|
325
|
+
if (runAll || (!opts.security && !opts.quality)) {
|
|
326
|
+
results.schema = validateSchemaEnhanced(definition);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// L2: Security
|
|
330
|
+
if (runAll || opts.security) {
|
|
331
|
+
const packageDir = fs.statSync(resolvedPath).isDirectory() ? resolvedPath : null;
|
|
332
|
+
results.security = securityAudit(definition, packageDir);
|
|
333
|
+
results.capabilities = [...extractCapabilities(definition)];
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// L3: Quality
|
|
337
|
+
if (runAll || opts.quality) {
|
|
338
|
+
const packageDir = fs.statSync(resolvedPath).isDirectory() ? resolvedPath : null;
|
|
339
|
+
results.quality = qualityAudit(definition, pkg, packageDir);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// CI output (machine-readable JSON with summary metadata)
|
|
343
|
+
if (opts.ci) {
|
|
344
|
+
const criticalCount = results.security.filter(f => f.severity === 'critical').length;
|
|
345
|
+
const highCount = results.security.filter(f => f.severity === 'high').length;
|
|
346
|
+
const schemaPass = results.schema.length === 0;
|
|
347
|
+
const securityPass = criticalCount === 0 && highCount === 0;
|
|
348
|
+
const qualityPass = results.quality.filter(i => i.level === 'error').length === 0;
|
|
349
|
+
const ciOutput = {
|
|
350
|
+
schema: { pass: schemaPass, errors: results.schema },
|
|
351
|
+
security: { pass: securityPass, findings: results.security },
|
|
352
|
+
quality: { pass: qualityPass, issues: results.quality },
|
|
353
|
+
capabilities: results.capabilities,
|
|
354
|
+
summary: {
|
|
355
|
+
passAll: schemaPass && securityPass && qualityPass,
|
|
356
|
+
criticalCount,
|
|
357
|
+
highCount,
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
console.log(JSON.stringify(ciOutput, null, 2));
|
|
361
|
+
if (criticalCount > 0) process.exit(2);
|
|
362
|
+
if (highCount > 0) process.exit(1);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// JSON output
|
|
367
|
+
if (opts.json) {
|
|
368
|
+
console.log(JSON.stringify(results, null, 2));
|
|
369
|
+
// Exit code 1 if critical/high security findings
|
|
370
|
+
const hasCritical = results.security.some(f => f.severity === 'critical' || f.severity === 'high');
|
|
371
|
+
if (hasCritical) process.exit(1);
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Pretty output
|
|
376
|
+
console.log();
|
|
377
|
+
console.log(pc.bold(`Workflow Check: ${definition.name || pkgPath}`));
|
|
378
|
+
console.log(pc.dim('═'.repeat(50)));
|
|
379
|
+
|
|
380
|
+
// Schema
|
|
381
|
+
if (results.schema.length > 0) {
|
|
382
|
+
console.log();
|
|
383
|
+
console.log(pc.bold('Schema Validation:'));
|
|
384
|
+
for (const e of results.schema) {
|
|
385
|
+
console.log(` ${pc.red('✗')} ${e}`);
|
|
386
|
+
}
|
|
387
|
+
} else if (runAll || (!opts.security && !opts.quality)) {
|
|
388
|
+
console.log();
|
|
389
|
+
console.log(`${pc.bold('Schema Validation:')} ${pc.green('✔ passed')}`);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// Security
|
|
393
|
+
if (runAll || opts.security) {
|
|
394
|
+
console.log();
|
|
395
|
+
console.log(pc.bold('Security Audit:'));
|
|
396
|
+
if (results.security.length === 0) {
|
|
397
|
+
console.log(` ${pc.green('✔')} No security issues found`);
|
|
398
|
+
} else {
|
|
399
|
+
for (const f of results.security) {
|
|
400
|
+
const color = f.severity === 'critical' ? pc.red :
|
|
401
|
+
f.severity === 'high' ? pc.red :
|
|
402
|
+
f.severity === 'medium' ? pc.yellow : pc.dim;
|
|
403
|
+
const icon = f.severity === 'critical' || f.severity === 'high' ? '✗' : '⚠';
|
|
404
|
+
console.log(` ${color(icon)} [${f.severity.toUpperCase()}] ${f.message}`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Capability flags
|
|
409
|
+
if (results.capabilities.length > 0) {
|
|
410
|
+
console.log();
|
|
411
|
+
console.log(pc.bold('Capabilities:'));
|
|
412
|
+
const capIcons = { NETWORK: '🌐', WRITE_DB: '💾', LLM: '🤖', LOOP: '🔄', READ_DB: '📊' };
|
|
413
|
+
for (const cap of results.capabilities) {
|
|
414
|
+
console.log(` ${capIcons[cap] || '•'} ${cap}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// Quality
|
|
420
|
+
if (runAll || opts.quality) {
|
|
421
|
+
console.log();
|
|
422
|
+
console.log(pc.bold('Quality Audit:'));
|
|
423
|
+
if (results.quality.length === 0) {
|
|
424
|
+
console.log(` ${pc.green('✔')} No quality issues found`);
|
|
425
|
+
} else {
|
|
426
|
+
for (const issue of results.quality) {
|
|
427
|
+
const color = issue.level === 'error' ? pc.red :
|
|
428
|
+
issue.level === 'warning' ? pc.yellow : pc.dim;
|
|
429
|
+
const icon = issue.level === 'error' ? '✗' : issue.level === 'warning' ? '⚠' : 'ℹ';
|
|
430
|
+
console.log(` ${color(icon)} [${issue.level.toUpperCase()}] ${issue.message}`);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
console.log();
|
|
436
|
+
|
|
437
|
+
// Exit code 1 if critical/high security findings
|
|
438
|
+
const hasCritical = results.security.some(f => f.severity === 'critical' || f.severity === 'high');
|
|
439
|
+
if (hasCritical) process.exit(1);
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
// ── workflow test <path> ──
|
|
443
|
+
wfCmd
|
|
444
|
+
.command('test <path>')
|
|
445
|
+
.description('Run test fixtures for a workflow package')
|
|
446
|
+
.option('--test <name>', 'Run a specific test case by name')
|
|
447
|
+
.option('--json', 'Output machine-readable JSON', false)
|
|
448
|
+
.action(async (pkgPath, opts) => {
|
|
449
|
+
const { loadWorkflow } = require('../lib/workflow');
|
|
450
|
+
const { runAllTests, loadTestCases } = require('../lib/workflow-test-runner');
|
|
451
|
+
const fs = require('fs');
|
|
452
|
+
|
|
453
|
+
const resolvedPath = path.resolve(pkgPath);
|
|
454
|
+
|
|
455
|
+
// Load workflow definition
|
|
456
|
+
let definition;
|
|
457
|
+
try {
|
|
458
|
+
const stat = fs.statSync(resolvedPath);
|
|
459
|
+
if (stat.isDirectory()) {
|
|
460
|
+
const wfPath = path.join(resolvedPath, 'workflow.json');
|
|
461
|
+
if (fs.existsSync(wfPath)) {
|
|
462
|
+
definition = JSON.parse(fs.readFileSync(wfPath, 'utf8'));
|
|
463
|
+
} else {
|
|
464
|
+
console.error(ui.error(`Workflow file not found: ${wfPath}`));
|
|
465
|
+
process.exit(1);
|
|
466
|
+
}
|
|
467
|
+
} else {
|
|
468
|
+
console.error(ui.error('Path must be a workflow package directory'));
|
|
469
|
+
process.exit(1);
|
|
470
|
+
}
|
|
471
|
+
} catch (err) {
|
|
472
|
+
console.error(ui.error(err.message));
|
|
473
|
+
process.exit(1);
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
// Check for tests directory
|
|
477
|
+
const testsDir = path.join(resolvedPath, 'tests');
|
|
478
|
+
if (!fs.existsSync(testsDir)) {
|
|
479
|
+
console.error(ui.error('No tests/ directory found in package'));
|
|
480
|
+
process.exit(1);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
const testCases = loadTestCases(resolvedPath);
|
|
484
|
+
if (testCases.length === 0) {
|
|
485
|
+
console.error(ui.error('No *.test.json files found in tests/'));
|
|
486
|
+
process.exit(1);
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
try {
|
|
490
|
+
const aggregate = await runAllTests(definition, resolvedPath, {
|
|
491
|
+
testName: opts.test,
|
|
492
|
+
});
|
|
493
|
+
|
|
494
|
+
if (opts.json) {
|
|
495
|
+
console.log(JSON.stringify(aggregate, null, 2));
|
|
496
|
+
if (aggregate.failed > 0) process.exit(1);
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// Pretty output
|
|
501
|
+
console.log();
|
|
502
|
+
console.log(pc.bold(`Workflow Tests: ${definition.name || pkgPath}`));
|
|
503
|
+
console.log(pc.dim('═'.repeat(50)));
|
|
504
|
+
console.log();
|
|
505
|
+
|
|
506
|
+
for (const result of aggregate.results) {
|
|
507
|
+
const icon = result.passed ? pc.green('✔') : pc.red('✗');
|
|
508
|
+
console.log(`${icon} ${result.name || result.file}`);
|
|
509
|
+
|
|
510
|
+
if (result.error) {
|
|
511
|
+
console.log(` ${pc.red(result.error)}`);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
for (const assertion of result.assertions) {
|
|
515
|
+
const aIcon = assertion.pass ? pc.green(' ✔') : pc.red(' ✗');
|
|
516
|
+
console.log(`${aIcon} ${assertion.message}`);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
for (const err of (result.errors || [])) {
|
|
520
|
+
console.log(` ${pc.red('Error:')} ${err}`);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
console.log();
|
|
525
|
+
console.log(`${pc.bold('Summary:')} ${pc.green(`${aggregate.passed} passed`)}, ${aggregate.failed > 0 ? pc.red(`${aggregate.failed} failed`) : `${aggregate.failed} failed`}`);
|
|
526
|
+
console.log();
|
|
527
|
+
|
|
528
|
+
if (aggregate.failed > 0) process.exit(1);
|
|
529
|
+
} catch (err) {
|
|
530
|
+
console.error(ui.error(err.message));
|
|
531
|
+
process.exit(1);
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
|
|
264
535
|
// ── workflow validate <file> ──
|
|
265
536
|
wfCmd
|
|
266
537
|
.command('validate <file>')
|
|
@@ -404,9 +675,11 @@ function registerWorkflow(program) {
|
|
|
404
675
|
packageName = WORKFLOW_PREFIX + packageName;
|
|
405
676
|
}
|
|
406
677
|
|
|
678
|
+
const telemetry = require('../lib/telemetry');
|
|
407
679
|
console.log(`Installing ${pc.cyan(packageName)}...`);
|
|
408
680
|
|
|
409
681
|
try {
|
|
682
|
+
telemetry.send('cli_workflow_install', { packageName });
|
|
410
683
|
const result = installPackage(packageName, { global: opts.global });
|
|
411
684
|
console.log(`${pc.green('✔')} Downloaded ${pc.cyan(packageName)}@${result.version}`);
|
|
412
685
|
|
|
@@ -417,6 +690,17 @@ function registerWorkflow(program) {
|
|
|
417
690
|
const steps = validation.definition?.steps?.length || 0;
|
|
418
691
|
const tools = (validation.pkg?.vai?.tools || []).join(', ');
|
|
419
692
|
console.log(`${pc.green('✔')} Validated workflow definition (${steps} steps${tools ? `, tools: ${tools}` : ''})`);
|
|
693
|
+
|
|
694
|
+
// Display capability flags
|
|
695
|
+
if (validation.definition) {
|
|
696
|
+
const { extractCapabilities } = require('../lib/security-audit');
|
|
697
|
+
const caps = extractCapabilities(validation.definition);
|
|
698
|
+
if (caps.size > 0) {
|
|
699
|
+
const capIcons = { NETWORK: '🌐', WRITE_DB: '💾', LLM: '🤖', LOOP: '🔄', READ_DB: '📊' };
|
|
700
|
+
const capList = [...caps].map(c => `${capIcons[c] || '•'} ${c}`).join(' ');
|
|
701
|
+
console.log(`${pc.dim('Capabilities:')} ${capList}`);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
420
704
|
} else {
|
|
421
705
|
console.log(`${pc.yellow('⚠')} Validation issues:`);
|
|
422
706
|
for (const e of validation.errors) {
|
|
@@ -479,11 +763,13 @@ function registerWorkflow(program) {
|
|
|
479
763
|
.action(async (query, opts) => {
|
|
480
764
|
const { searchNpm } = require('../lib/npm-utils');
|
|
481
765
|
|
|
766
|
+
const telemetry = require('../lib/telemetry');
|
|
482
767
|
console.log(`Searching npm for vai-workflow packages matching "${query}"...`);
|
|
483
768
|
console.log();
|
|
484
769
|
|
|
485
770
|
try {
|
|
486
771
|
const results = await searchNpm(query, { limit: parseInt(opts.limit, 10) });
|
|
772
|
+
telemetry.send('cli_workflow_search', { query: query.slice(0, 50), resultCount: results.length });
|
|
487
773
|
|
|
488
774
|
if (opts.json) {
|
|
489
775
|
console.log(JSON.stringify(results, null, 2));
|