voyageai-cli 1.29.0 → 1.30.0

package/src/lib/security/blocked-domains.json

@@ -0,0 +1,17 @@
+[
+  "evil.com",
+  "malware.com",
+  "requestbin.com",
+  "webhook.site",
+  "pipedream.net",
+  "hookbin.com",
+  "requestcatcher.com",
+  "canarytokens.com",
+  "burpcollaborator.net",
+  "interact.sh",
+  "oastify.com",
+  "dnslog.cn",
+  "ceye.io",
+  "bxss.me",
+  "xss.ht"
+]

package/src/lib/security-audit.js

@@ -0,0 +1,198 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// ════════════════════════════════════════════════════════════════════
+// Blocked Domains
+// ════════════════════════════════════════════════════════════════════
+
+const BLOCKED_DOMAINS = new Set(
+  JSON.parse(fs.readFileSync(path.join(__dirname, 'security', 'blocked-domains.json'), 'utf8'))
+);
+
+/**
+ * Check if a URL targets a blocked domain.
+ * @param {string} url
+ * @returns {boolean}
+ */
+function isBlockedDomain(url) {
+  try {
+    const hostname = new URL(url).hostname;
+    return BLOCKED_DOMAINS.has(hostname);
+  } catch {
+    return false;
+  }
+}
+
+// ════════════════════════════════════════════════════════════════════
+// Capability Flags
+// ════════════════════════════════════════════════════════════════════
+
+/**
+ * Extract capability flags from a workflow definition.
+ * @param {object} definition
+ * @returns {Set<string>}
+ */
+function extractCapabilities(definition) {
+  const caps = new Set();
+  if (!definition || !Array.isArray(definition.steps)) return caps;
+
+  for (const step of definition.steps) {
+    switch (step.tool) {
+      case 'http':
+        caps.add('NETWORK');
+        break;
+      case 'ingest':
+        caps.add('WRITE_DB');
+        break;
+      case 'aggregate':
+        caps.add('READ_DB');
+        if (step.inputs?.allowWrites) caps.add('WRITE_DB');
+        if (Array.isArray(step.inputs?.pipeline)) {
+          for (const stage of step.inputs.pipeline) {
+            const key = Object.keys(stage)[0];
+            if (key === '$out' || key === '$merge') caps.add('WRITE_DB');
+          }
+        }
+        break;
+      case 'generate':
+        caps.add('LLM');
+        break;
+      case 'loop':
+        caps.add('LOOP');
+        break;
+      case 'query':
+      case 'search':
+      case 'collections':
+        caps.add('READ_DB');
+        break;
+    }
+
+    // forEach counts as LOOP
+    if (step.forEach) caps.add('LOOP');
+  }
+
+  return caps;
+}
+
+// ════════════════════════════════════════════════════════════════════
+// Security Audit
+// ════════════════════════════════════════════════════════════════════
+
+/**
+ * Run security audit on a workflow definition.
+ * @param {object} definition - Parsed workflow JSON
+ * @param {string} [packagePath] - Path to the package directory (for package-level checks)
+ * @returns {Array<{severity: string, message: string, stepId?: string}>}
+ */
+function securityAudit(definition, packagePath) {
+  const findings = [];
+
+  if (!definition || !Array.isArray(definition.steps)) return findings;
+
+  for (const step of definition.steps) {
+    // HTTP step checks
+    if (step.tool === 'http') {
+      const url = step.inputs?.url;
+      if (typeof url === 'string') {
+        if (!url.includes('{{')) {
+          // Static URL — check against blocklist
+          if (isBlockedDomain(url)) {
+            findings.push({ severity: 'critical', message: `HTTP step "${step.id}" targets blocked domain`, stepId: step.id });
+          }
+          // Flag non-HTTPS
+          if (url.startsWith('http://')) {
+            findings.push({ severity: 'medium', message: `HTTP step "${step.id}" uses insecure HTTP`, stepId: step.id });
+          }
+        } else {
+          // Dynamic URLs always flagged for review
+          findings.push({ severity: 'high', message: `HTTP step "${step.id}" has dynamic URL (needs review)`, stepId: step.id });
+        }
+      }
+    }
+
+    // Aggregate: Flag write stages
+    if (step.tool === 'aggregate') {
+      if (step.inputs?.allowWrites) {
+        findings.push({ severity: 'high', message: `Aggregate step "${step.id}" allows write operations ($out/$merge)`, stepId: step.id });
+      }
+      const pipeline = step.inputs?.pipeline;
+      if (Array.isArray(pipeline)) {
+        for (const stage of pipeline) {
+          const key = Object.keys(stage)[0];
+          if (key === '$out' || key === '$merge') {
+            findings.push({ severity: 'critical', message: `Aggregate step "${step.id}" contains ${key} stage`, stepId: step.id });
+          }
+        }
+      }
+    }
+
+    // Generate: Check for prompt injection patterns
+    if (step.tool === 'generate') {
+      const prompt = step.inputs?.prompt || '';
+      const systemPrompt = step.inputs?.systemPrompt || '';
+      for (const text of [prompt, systemPrompt]) {
+        if (typeof text === 'string') {
+          if (/ignore\s+(previous|all)\s+instructions/i.test(text)) {
+            findings.push({ severity: 'high', message: `Suspicious prompt pattern in "${step.id}"`, stepId: step.id });
+          }
+          if (/system\s*:\s*/i.test(text)) {
+            findings.push({ severity: 'medium', message: `Prompt contains "system:" prefix in "${step.id}"`, stepId: step.id });
+          }
+        }
+      }
+    }
+
+    // Ingest: Flag dynamic db/collection names
+    if (step.tool === 'ingest') {
+      const db = step.inputs?.db;
+      const coll = step.inputs?.collection;
+      if (typeof db === 'string' && db.includes('{{')) {
+        findings.push({ severity: 'medium', message: `Ingest step "${step.id}" uses dynamic database name`, stepId: step.id });
+      }
+      if (typeof coll === 'string' && coll.includes('{{')) {
+        findings.push({ severity: 'medium', message: `Ingest step "${step.id}" uses dynamic collection name`, stepId: step.id });
+      }
+    }
+
+    // Loop: Check for unbounded iterations
+    if (step.tool === 'loop') {
+      if (!step.inputs?.maxIterations || step.inputs.maxIterations > 1000) {
+        findings.push({ severity: 'medium', message: `Loop step "${step.id}" has high/unbounded maxIterations`, stepId: step.id });
+      }
+    }
+  }
+
+  // Package-level checks
+  if (packagePath) {
+    try {
+      const files = fs.readdirSync(packagePath);
+      const jsFiles = files.filter(f => /\.(js|ts|mjs|cjs)$/.test(f) && f !== 'node_modules');
+      if (jsFiles.length > 0) {
+        findings.push({ severity: 'critical', message: `Package contains executable code: ${jsFiles.join(', ')}` });
+      }
+    } catch {
+      // Ignore if can't read directory
+    }
+
+    try {
+      const pkgPath = path.join(packagePath, 'package.json');
+      if (fs.existsSync(pkgPath)) {
+        const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+        const dangerousScripts = ['preinstall', 'install', 'postinstall', 'preuninstall', 'postuninstall'];
+        for (const script of dangerousScripts) {
+          if (pkg.scripts?.[script]) {
+            findings.push({ severity: 'critical', message: `Package has "${script}" lifecycle script` });
+          }
+        }
+      }
+    } catch {
+      // Ignore if can't read package.json
+    }
+  }
+
+  return findings;
+}
+
+module.exports = { securityAudit, extractCapabilities, isBlockedDomain };

package/src/lib/telemetry.js

@@ -69,4 +69,26 @@ function send(event, extra = {}) {
   } catch { /* telemetry should never break the CLI */ }
 }
 
-module.exports = { send, isEnabled };
+/**
+ * Create a timer that auto-sends a telemetry event on completion.
+ * Usage:
+ *   const done = telemetry.timer('cli_query', { model: 'voyage-4-large' });
+ *   // ... do work ...
+ *   done({ resultCount: 5 });  // sends event with durationMs calculated
+ *
+ * @param {string} event - Event name
+ * @param {object} [baseFields] - Fields known at start time
+ * @returns {function} done(extraFields) - Call to send the event
+ */
+function timer(event, baseFields = {}) {
+  const start = Date.now();
+  return (extraFields = {}) => {
+    send(event, {
+      ...baseFields,
+      ...extraFields,
+      durationMs: Date.now() - start,
+    });
+  };
+}
+
+module.exports = { send, isEnabled, timer };

package/src/lib/workflow-scaffold.js

@@ -288,6 +288,67 @@ function scaffoldPackage(options) {
   fs.writeFileSync(path.join(outputDir, 'LICENSE'), license);
   files.push('LICENSE');
 
+  // Create tests/ directory with a sample test case
+  const testsDir = path.join(outputDir, 'tests');
+  fs.mkdirSync(testsDir, { recursive: true });
+
+  // Build sample mocks based on the tools used
+  const sampleMocks = {};
+  for (const tool of tools) {
+    if (tool === 'query' || tool === 'search') {
+      sampleMocks[tool] = { results: [{ text: 'Sample result', score: 0.95 }], resultCount: 1 };
+    } else if (tool === 'embed') {
+      sampleMocks[tool] = { embedding: [0.1, 0.2, 0.3], model: 'voyage-3-large', dimensions: 3 };
+    } else if (tool === 'rerank') {
+      sampleMocks[tool] = { results: [{ text: 'Reranked result', score: 0.98 }], resultCount: 1 };
+    } else if (tool === 'generate') {
+      sampleMocks[tool] = { text: 'Generated text response', model: 'mock-llm', provider: 'mock' };
+    }
+  }
+
+  // Build sample inputs from definition
+  const sampleInputs = {};
+  if (definition.inputs) {
+    for (const [key, schema] of Object.entries(definition.inputs)) {
+      if (schema.default !== undefined) {
+        sampleInputs[key] = schema.default;
+      } else if (schema.type === 'number') {
+        sampleInputs[key] = 10;
+      } else if (schema.type === 'boolean') {
+        sampleInputs[key] = true;
+      } else {
+        sampleInputs[key] = 'test value';
+      }
+    }
+  }
+
+  // Build expected steps
+  const expectedSteps = {};
+  if (definition.steps) {
+    for (const step of definition.steps) {
+      expectedSteps[step.id] = { status: 'completed' };
+    }
+  }
+
+  const sampleTestCase = {
+    name: 'basic workflow test',
+    inputs: sampleInputs,
+    mocks: sampleMocks,
+    expect: {
+      steps: expectedSteps,
+      noErrors: true,
+    },
+  };
+
+  fs.writeFileSync(path.join(testsDir, 'basic.test.json'), JSON.stringify(sampleTestCase, null, 2) + '\n');
+  files.push('tests/basic.test.json');
+
+  // Create fixtures directory with .gitkeep
+  const fixturesDir = path.join(testsDir, 'fixtures');
+  fs.mkdirSync(fixturesDir, { recursive: true });
+  fs.writeFileSync(path.join(fixturesDir, '.gitkeep'), '');
+  files.push('tests/fixtures/.gitkeep');
+
   return { dir: outputDir, files };
 }
 

package/src/lib/workflow-test-runner.js

@@ -0,0 +1,208 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const { executeWorkflow } = require('./workflow');
+
+/**
+ * Run a single workflow test case against a definition.
+ *
+ * @param {object} definition - Parsed workflow JSON
+ * @param {object} testCase - Test case object { name, inputs, mocks, expect }
+ * @returns {Promise<{ passed: boolean, assertions: Array<{pass: boolean, message: string}>, errors: string[] }>}
+ */
+async function runWorkflowTest(definition, testCase) {
+  const results = { passed: true, assertions: [], errors: [] };
+
+  // Build mock executors from test case mocks
+  const mockExecutors = {};
+  for (const [tool, mockOutput] of Object.entries(testCase.mocks || {})) {
+    mockExecutors[tool] = async () => mockOutput;
+  }
+
+  let result;
+  try {
+    result = await executeWorkflow(definition, {
+      inputs: testCase.inputs || {},
+      _mockExecutors: mockExecutors,
+    });
+  } catch (err) {
+    results.passed = false;
+    results.errors.push(err.message);
+    return results;
+  }
+
+  // Check step statuses
+  if (testCase.expect && testCase.expect.steps) {
+    for (const [stepId, expected] of Object.entries(testCase.expect.steps)) {
+      const stepResult = result.steps.find(s => s.id === stepId);
+      if (!stepResult) {
+        results.assertions.push({ pass: false, message: `Step "${stepId}" not found in results` });
+        results.passed = false;
+      } else if (expected.status === 'completed') {
+        if (stepResult.skipped) {
+          results.assertions.push({ pass: false, message: `Step "${stepId}" was skipped, expected completed` });
+          results.passed = false;
+        } else if (stepResult.error) {
+          results.assertions.push({ pass: false, message: `Step "${stepId}" errored: ${stepResult.error}` });
+          results.passed = false;
+        } else {
+          results.assertions.push({ pass: true, message: `Step "${stepId}" completed` });
+        }
+      } else if (expected.status === 'skipped') {
+        if (stepResult.skipped) {
+          results.assertions.push({ pass: true, message: `Step "${stepId}" skipped` });
+        } else {
+          results.assertions.push({ pass: false, message: `Step "${stepId}" should have been skipped` });
+          results.passed = false;
+        }
+      }
+    }
+  }
+
+  // Check output shape
+  if (testCase.expect && testCase.expect.output) {
+    for (const [key, constraint] of Object.entries(testCase.expect.output)) {
+      const value = result.output && result.output[key];
+      if (constraint.type === 'array') {
+        if (!Array.isArray(value)) {
+          results.assertions.push({ pass: false, message: `output.${key} should be array, got ${typeof value}` });
+          results.passed = false;
+          continue;
+        }
+      }
+      if (constraint.type === 'string') {
+        if (typeof value !== 'string') {
+          results.assertions.push({ pass: false, message: `output.${key} should be string, got ${typeof value}` });
+          results.passed = false;
+          continue;
+        }
+      }
+      if (constraint.type === 'number') {
+        if (typeof value !== 'number') {
+          results.assertions.push({ pass: false, message: `output.${key} should be number, got ${typeof value}` });
+          results.passed = false;
+          continue;
+        }
+      }
+      if (constraint.type === 'object') {
+        if (!value || typeof value !== 'object' || Array.isArray(value)) {
+          results.assertions.push({ pass: false, message: `output.${key} should be object` });
+          results.passed = false;
+          continue;
+        }
+      }
+      if (constraint.minLength != null) {
+        if (!value || value.length < constraint.minLength) {
+          results.assertions.push({ pass: false, message: `output.${key} length ${value ? value.length : 0} < ${constraint.minLength}` });
+          results.passed = false;
+          continue;
+        }
+      }
+      // If we got here with a type check, it passed
+      if (constraint.type || constraint.minLength != null) {
+        results.assertions.push({ pass: true, message: `output.${key} matches expected shape` });
+      }
+    }
+  }
+
+  // Check noErrors flag
+  if (testCase.expect && testCase.expect.noErrors) {
+    const hasErrors = result.steps.some(s => s.error);
+    if (hasErrors) {
+      const errorSteps = result.steps.filter(s => s.error).map(s => `${s.id}: ${s.error}`);
+      results.assertions.push({ pass: false, message: `Expected no errors but found: ${errorSteps.join('; ')}` });
+      results.passed = false;
+    } else {
+      results.assertions.push({ pass: true, message: 'No step errors' });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Load test cases from a workflow package's tests/ directory.
+ *
+ * @param {string} packagePath - Path to the workflow package directory
+ * @returns {Array<object>} Array of test case objects
+ */
+function loadTestCases(packagePath) {
+  const testsDir = path.join(packagePath, 'tests');
+  if (!fs.existsSync(testsDir)) {
+    return [];
+  }
+
+  const files = fs.readdirSync(testsDir).filter(f => f.endsWith('.test.json'));
+  const testCases = [];
+
+  for (const file of files) {
+    const filePath = path.join(testsDir, file);
+    try {
+      const content = fs.readFileSync(filePath, 'utf8');
+      const testCase = JSON.parse(content);
+      testCase._file = file;
+      testCases.push(testCase);
+    } catch (err) {
+      testCases.push({
+        name: file,
+        _file: file,
+        _error: `Failed to load: ${err.message}`,
+      });
+    }
+  }
+
+  return testCases;
+}
+
+/**
+ * Run all test cases for a workflow package.
+ *
+ * @param {object} definition - Parsed workflow JSON
+ * @param {string} packagePath - Path to the workflow package directory
+ * @param {object} [options]
+ * @param {string} [options.testName] - Run only a specific test by name
+ * @returns {Promise<{ total: number, passed: number, failed: number, results: Array }>}
+ */
+async function runAllTests(definition, packagePath, options = {}) {
+  let testCases = loadTestCases(packagePath);
+
+  if (options.testName) {
+    testCases = testCases.filter(t => t.name === options.testName);
+  }
+
+  const results = [];
+  let passed = 0;
+  let failed = 0;
+
+  for (const testCase of testCases) {
+    if (testCase._error) {
+      results.push({ name: testCase.name, file: testCase._file, passed: false, error: testCase._error, assertions: [] });
+      failed++;
+      continue;
+    }
+
+    const result = await runWorkflowTest(definition, testCase);
+    results.push({
+      name: testCase.name || testCase._file,
+      file: testCase._file,
+      passed: result.passed,
+      assertions: result.assertions,
+      errors: result.errors,
+    });
+
+    if (result.passed) {
+      passed++;
+    } else {
+      failed++;
+    }
+  }
+
+  return { total: testCases.length, passed, failed, results };
+}
+
+module.exports = {
+  runWorkflowTest,
+  loadTestCases,
+  runAllTests,
+};