voyageai-cli 1.29.0 → 1.30.1

package/src/lib/export/index.js

@@ -0,0 +1,175 @@
+'use strict';
+
+const { renderJson, renderJsonl } = require('./formats/json-export');
+const { renderCsv } = require('./formats/csv-export');
+const { renderMarkdown } = require('./formats/markdown-export');
+const { renderMermaid } = require('./formats/mermaid-export');
+const { copyToClipboard } = require('./formats/clipboard-export');
+const { renderSvgAsync } = require('./formats/svg-export');
+const { renderPng } = require('./formats/png-export');
+const { renderPdf } = require('./formats/pdf-export');
+
+const { normalizeWorkflow, WORKFLOW_FORMATS } = require('./contexts/workflow-export');
+const { normalizeSearch, SEARCH_FORMATS } = require('./contexts/search-export');
+const { normalizeChat, CHAT_FORMATS } = require('./contexts/chat-export');
+const { normalizeBenchmark, BENCHMARK_FORMATS } = require('./contexts/benchmark-export');
+const { normalizeExplore, EXPLORE_FORMATS } = require('./contexts/explore-export');
+
+// ════════════════════════════════════════════════════════════════════
+// Context → Formats mapping
+// ════════════════════════════════════════════════════════════════════
+
+const FORMAT_MAP = {
+  workflow: WORKFLOW_FORMATS,
+  search: SEARCH_FORMATS,
+  chat: CHAT_FORMATS,
+  benchmark: BENCHMARK_FORMATS,
+  explore: EXPLORE_FORMATS,
+};
+
+const TRANSFORMERS = {
+  workflow: normalizeWorkflow,
+  search: normalizeSearch,
+  chat: normalizeChat,
+  benchmark: normalizeBenchmark,
+  explore: normalizeExplore,
+};
+
+const RENDERERS = {
+  json: renderJson,
+  jsonl: renderJsonl,
+  csv: renderCsv,
+  markdown: renderMarkdown,
+  mermaid: renderMermaid,
+  svg: renderSvgAsync,
+  png: renderPng,
+  pdf: renderPdf,
+};
+
+// Formats that produce binary (Buffer) output
+const BINARY_FORMATS = new Set(['png', 'pdf']);
+
+const EXT_MAP = {
+  json: '.json',
+  jsonl: '.jsonl',
+  csv: '.csv',
+  markdown: '.md',
+  mermaid: '.mmd',
+  svg: '.svg',
+  png: '.png',
+  pdf: '.pdf',
+};
+
+// ════════════════════════════════════════════════════════════════════
+// Public API
+// ════════════════════════════════════════════════════════════════════
+
+/**
+ * Get supported formats for a given context.
+ * @param {string} context
+ * @returns {string[]}
+ */
+function getFormatsForContext(context) {
+  return FORMAT_MAP[context] || [];
+}
+
+/**
+ * Build a deterministic filename for the export.
+ * Pattern: {context}-{name}-{timestamp}.{ext}
+ * @param {string} context
+ * @param {object} data
+ * @param {string} format
+ * @returns {string}
+ */
+function buildFilename(context, data, format) {
+  const name = (data.name || data.sessionId || data.query || context)
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 40);
+  const ts = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15);
+  const ext = EXT_MAP[format] || '.txt';
+  return `${context}-${name}-${ts}${ext}`;
+}
+
+/**
+ * Main export entry point.
+ *
+ * @param {object} params
+ * @param {string} params.context - 'workflow' | 'search' | 'chat' | 'benchmark' | 'explore'
+ * @param {string} params.format - 'json' | 'jsonl' | 'csv' | 'markdown' | 'mermaid' | 'clipboard'
+ * @param {object} params.data - Raw source data
+ * @param {object} [params.options] - Format & context specific options
+ * @returns {{ content: string, mimeType: string, suggestedFilename: string, format: string }}
+ */
+async function exportArtifact({ context, format, data, options = {} }) {
+  // Handle clipboard as a meta-format: pick the best underlying format and copy
+  const isClipboard = format === 'clipboard';
+  const effectiveFormat = isClipboard ? pickClipboardFormat(context) : format;
+
+  // Validate
+  const supported = getFormatsForContext(context);
+  if (!supported.includes(format)) {
+    throw new ExportError(
+      `Format "${format}" not supported for ${context}. Supported: ${supported.join(', ')}`
+    );
+  }
+
+  // Transform
+  const transformer = TRANSFORMERS[context];
+  if (!transformer) {
+    throw new ExportError(`Unknown export context: "${context}"`);
+  }
+  const normalized = transformer(data, options);
+
+  // Render
+  const renderer = RENDERERS[effectiveFormat];
+  if (!renderer) {
+    throw new ExportError(`No renderer for format: "${effectiveFormat}"`);
+  }
+  const output = await renderer(normalized, options);
+  const isBinary = BINARY_FORMATS.has(effectiveFormat);
+
+  // Clipboard side-effect (text only — binary clipboard handled by Electron)
+  if (isClipboard) {
+    const ok = copyToClipboard(typeof output.content === 'string' ? output.content : output.content.toString());
+    if (!ok) {
+      throw new ExportError('Failed to copy to clipboard — unsupported platform or missing clipboard tool');
+    }
+  }
+
+  return {
+    content: output.content,
+    mimeType: output.mimeType,
+    suggestedFilename: buildFilename(context, data, effectiveFormat),
+    format: effectiveFormat,
+  };
+}
+
+/**
+ * Pick the best text format for clipboard based on context.
+ */
+function pickClipboardFormat(context) {
+  switch (context) {
+    case 'workflow': return 'mermaid';
+    case 'search': return 'json';
+    case 'chat': return 'markdown';
+    case 'benchmark': return 'json';
+    case 'explore': return 'json';
+    default: return 'json';
+  }
+}
+
+class ExportError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'ExportError';
+  }
+}
+
+module.exports = {
+  exportArtifact,
+  getFormatsForContext,
+  buildFilename,
+  ExportError,
+};

package/src/lib/llm.js

@@ -147,15 +147,24 @@ class AnthropicProvider {
       const json = await res.json();
       const text = json.content?.[0]?.text || '';
       yield text;
+      // Yield usage sentinel
+      const usage = json.usage || {};
+      yield { __usage: { inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0 } };
       return;
     }
 
-    yield* parseSSE(res.body, (event, data) => {
-      if (event === 'content_block_delta' && data.delta?.text) {
-        return data.delta.text;
+    // Manual SSE loop to capture usage from streaming events
+    const usage = { inputTokens: 0, outputTokens: 0 };
+    for await (const chunk of parseSSEWithMeta(res.body)) {
+      if (chunk.__event === 'message_start' && chunk.__data?.message?.usage) {
+        usage.inputTokens = chunk.__data.message.usage.input_tokens || 0;
+      } else if (chunk.__event === 'message_delta' && chunk.__data?.usage) {
+        usage.outputTokens = chunk.__data.usage.output_tokens || 0;
+      } else if (chunk.__event === 'content_block_delta' && chunk.__data?.delta?.text) {
+        yield chunk.__data.delta.text;
       }
-      return null;
-    });
+    }
+    yield { __usage: usage };
   }
 
   /**
@@ -163,7 +172,7 @@ class AnthropicProvider {
    * @param {Array} messages - Conversation messages
    * @param {Array} tools - Tool definitions in Anthropic format
    * @param {object} [options]
-   * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
+   * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
    */
   async chatWithTools(messages, tools, options = {}) {
     const model = options.model || this.model;
@@ -200,6 +209,8 @@ class AnthropicProvider {
 
     const json = await res.json();
     const stopReason = json.stop_reason || 'end_turn';
+    const apiUsage = json.usage || {};
+    const usage = { inputTokens: apiUsage.input_tokens || 0, outputTokens: apiUsage.output_tokens || 0 };
 
     // Check for tool_use blocks
     const toolBlocks = (json.content || []).filter(b => b.type === 'tool_use');
@@ -212,6 +223,7 @@ class AnthropicProvider {
           arguments: b.input,
         })),
         stopReason,
+        usage,
         _raw: json.content,
       };
     }
@@ -222,6 +234,7 @@ class AnthropicProvider {
       type: 'text',
       content: textBlocks.map(b => b.text).join(''),
       stopReason,
+      usage,
     };
   }
 
@@ -324,6 +337,11 @@ class OpenAIProvider {
       messages,
     };
 
+    // Request usage data in streaming mode
+    if (stream) {
+      body.stream_options = { include_usage: true };
+    }
+
     const res = await fetch(`${this.baseUrl}/v1/chat/completions`, {
       method: 'POST',
       headers: {
@@ -342,14 +360,25 @@ class OpenAIProvider {
       const json = await res.json();
       const text = json.choices?.[0]?.message?.content || '';
       yield text;
+      const apiUsage = json.usage || {};
+      yield { __usage: { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 } };
       return;
     }
 
-    yield* parseSSE(res.body, (_event, data) => {
-      if (data === '[DONE]') return null;
-      const content = data.choices?.[0]?.delta?.content;
-      return content || null;
-    });
+    // Manual SSE loop to capture usage from final streaming chunk
+    const usage = { inputTokens: 0, outputTokens: 0 };
+    for await (const chunk of parseSSEWithMeta(res.body)) {
+      const data = chunk.__data;
+      if (data === '[DONE]') continue;
+      // Final chunk with usage stats (stream_options: include_usage)
+      if (data?.usage) {
+        usage.inputTokens = data.usage.prompt_tokens || 0;
+        usage.outputTokens = data.usage.completion_tokens || 0;
+      }
+      const content = data?.choices?.[0]?.delta?.content;
+      if (content) yield content;
+    }
+    yield { __usage: usage };
   }
 
   /**
@@ -357,7 +386,7 @@ class OpenAIProvider {
    * @param {Array} messages - Conversation messages
    * @param {Array} tools - Tool definitions in OpenAI format
    * @param {object} [options]
-   * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
+   * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
    */
   async chatWithTools(messages, tools, options = {}) {
     const model = options.model || this.model;
@@ -389,6 +418,8 @@ class OpenAIProvider {
     const choice = json.choices?.[0] || {};
     const msg = choice.message || {};
     const stopReason = choice.finish_reason || 'stop';
+    const apiUsage = json.usage || {};
+    const usage = { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 };
 
     if (msg.tool_calls && msg.tool_calls.length > 0) {
       return {
@@ -401,6 +432,7 @@ class OpenAIProvider {
             : tc.function.arguments,
         })),
         stopReason,
+        usage,
         _raw: msg,
       };
     }
@@ -409,6 +441,7 @@ class OpenAIProvider {
       type: 'text',
       content: msg.content || '',
       stopReason,
+      usage,
     };
   }
 
@@ -502,14 +535,25 @@ class OllamaProvider {
       const json = await res.json();
       const text = json.choices?.[0]?.message?.content || '';
       yield text;
+      // Ollama may not return usage, default to 0
+      const apiUsage = json.usage || {};
+      yield { __usage: { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 } };
       return;
     }
 
-    yield* parseSSE(res.body, (_event, data) => {
-      if (data === '[DONE]') return null;
-      const content = data.choices?.[0]?.delta?.content;
-      return content || null;
-    });
+    // Manual SSE loop (Ollama may not support stream_options)
+    const usage = { inputTokens: 0, outputTokens: 0 };
+    for await (const chunk of parseSSEWithMeta(res.body)) {
+      const data = chunk.__data;
+      if (data === '[DONE]') continue;
+      if (data?.usage) {
+        usage.inputTokens = data.usage.prompt_tokens || 0;
+        usage.outputTokens = data.usage.completion_tokens || 0;
+      }
+      const content = data?.choices?.[0]?.delta?.content;
+      if (content) yield content;
+    }
+    yield { __usage: usage };
   }
 
   /**
@@ -517,7 +561,7 @@ class OllamaProvider {
    * @param {Array} messages - Conversation messages
    * @param {Array} tools - Tool definitions in OpenAI format
    * @param {object} [options]
-   * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string}>}
+   * @returns {Promise<{type: 'text'|'tool_calls', content?: string, calls?: Array, stopReason: string, usage: object}>}
    */
   async chatWithTools(messages, tools, options = {}) {
     const model = options.model || this.model;
@@ -544,6 +588,9 @@ class OllamaProvider {
     const choice = json.choices?.[0] || {};
     const msg = choice.message || {};
     const stopReason = choice.finish_reason || 'stop';
+    // Ollama may not return usage, default to 0
+    const apiUsage = json.usage || {};
+    const usage = { inputTokens: apiUsage.prompt_tokens || 0, outputTokens: apiUsage.completion_tokens || 0 };
 
     if (msg.tool_calls && msg.tool_calls.length > 0) {
       return {
@@ -556,6 +603,7 @@ class OllamaProvider {
             : tc.function.arguments,
         })),
         stopReason,
+        usage,
         _raw: msg,
       };
     }
@@ -564,6 +612,7 @@ class OllamaProvider {
       type: 'text',
       content: msg.content || '',
       stopReason,
+      usage,
     };
   }
 
@@ -622,6 +671,64 @@ class OllamaProvider {
 // SSE Stream Parser
 // ============================================
 
+/**
+ * Parse a Server-Sent Events stream, yielding raw event+data pairs.
+ * Unlike parseSSE, this preserves event types and full data objects
+ * so callers can extract both content and metadata (e.g. usage stats).
+ *
+ * @param {ReadableStream} body - Response body stream
+ * @yields {{ __event: string|null, __data: object|string }} Parsed SSE events
+ */
+async function* parseSSEWithMeta(body) {
+  const decoder = new TextDecoder();
+  let buffer = '';
+  let currentEvent = null;
+
+  for await (const chunk of body) {
+    buffer += decoder.decode(chunk, { stream: true });
+
+    const lines = buffer.split('\n');
+    buffer = lines.pop() || '';
+
+    for (const line of lines) {
+      if (line.startsWith('event: ')) {
+        currentEvent = line.slice(7).trim();
+        continue;
+      }
+
+      if (line.startsWith('data: ')) {
+        const rawData = line.slice(6);
+
+        if (rawData === '[DONE]') {
+          yield { __event: currentEvent, __data: '[DONE]' };
+          return;
+        }
+
+        let parsed;
+        try {
+          parsed = JSON.parse(rawData);
+        } catch {
+          continue;
+        }
+
+        yield { __event: currentEvent, __data: parsed };
+        currentEvent = null;
+      }
+    }
+  }
+
+  // Process remaining buffer
+  if (buffer.trim() && buffer.startsWith('data: ')) {
+    const rawData = buffer.slice(6);
+    if (rawData !== '[DONE]') {
+      try {
+        const parsed = JSON.parse(rawData);
+        yield { __event: currentEvent, __data: parsed };
+      } catch { /* skip */ }
+    }
+  }
+}
+
 /**
  * Parse a Server-Sent Events stream.
  * @param {ReadableStream} body - Response body stream

package/src/lib/quality-audit.js

@@ -0,0 +1,71 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const CATEGORIES = ['retrieval', 'analysis', 'ingestion', 'domain-specific', 'utility', 'integration'];
+
+/**
+ * Run quality audit on a workflow definition and its package.
+ * @param {object} definition - Parsed workflow JSON
+ * @param {object} pkg - Parsed package.json
+ * @param {string} [packagePath] - Path to the package directory
+ * @returns {Array<{level: string, message: string}>}
+ */
+function qualityAudit(definition, pkg, packagePath) {
+  const issues = [];
+
+  // Package metadata checks
+  if (!pkg.description || pkg.description.length < 20) {
+    issues.push({ level: 'error', message: 'Package description too short (min 20 chars)' });
+  }
+  if (!pkg.author) {
+    issues.push({ level: 'error', message: 'Package must have an author' });
+  }
+  if (!pkg.license) {
+    issues.push({ level: 'warning', message: 'No license specified' });
+  }
+  if (!pkg.vai?.category || !CATEGORIES.includes(pkg.vai.category)) {
+    issues.push({ level: 'error', message: 'Invalid or missing vai.category' });
+  }
+
+  // README checks
+  if (packagePath) {
+    const readmePath = path.join(packagePath, 'README.md');
+    if (!fs.existsSync(readmePath)) {
+      issues.push({ level: 'error', message: 'Missing README.md' });
+    } else {
+      const readme = fs.readFileSync(readmePath, 'utf8');
+      if (readme.length < 200) {
+        issues.push({ level: 'warning', message: 'README is very short (< 200 chars)' });
+      }
+      if (!readme.includes('## Usage') && !readme.includes('## Install')) {
+        issues.push({ level: 'warning', message: 'README should include Usage or Install section' });
+      }
+      if (readme.includes('TODO')) {
+        issues.push({ level: 'warning', message: 'README contains TODO placeholders' });
+      }
+    }
+  }
+
+  // Workflow definition quality
+  if (definition && Array.isArray(definition.steps)) {
+    if (definition.steps.length === 1) {
+      issues.push({ level: 'suggestion', message: 'Single-step workflows may not warrant a package — consider documenting as a CLI example instead' });
+    }
+  }
+
+  // Branding
+  if (!definition?.branding?.icon) {
+    issues.push({ level: 'suggestion', message: 'Consider adding branding.icon for store display' });
+  }
+
+  // Naming — should be descriptive, not generic
+  if (definition?.name && /^(test|my|workflow|demo|example)/i.test(definition.name)) {
+    issues.push({ level: 'warning', message: `Workflow name "${definition.name}" is too generic` });
+  }
+
+  return issues;
+}
+
+module.exports = { qualityAudit, CATEGORIES };

package/src/lib/security/blocked-domains.json

@@ -0,0 +1,17 @@
+[
+  "evil.com",
+  "malware.com",
+  "requestbin.com",
+  "webhook.site",
+  "pipedream.net",
+  "hookbin.com",
+  "requestcatcher.com",
+  "canarytokens.com",
+  "burpcollaborator.net",
+  "interact.sh",
+  "oastify.com",
+  "dnslog.cn",
+  "ceye.io",
+  "bxss.me",
+  "xss.ht"
+]