voyageai-cli 1.24.0 → 1.26.1

package/src/commands/scaffold.js

@@ -129,6 +129,23 @@ function registerScaffold(program) {
             });
           }
         }
+
+        // Add binary files (copied as-is from templates dir)
+        if (structure.binaryFiles) {
+          for (const file of structure.binaryFiles) {
+            const srcPath = path.join(__dirname, '..', 'lib', 'templates', target, file.source);
+            const destPath = path.join(projectDir, file.output);
+            const binarySize = fs.existsSync(srcPath) ? fs.statSync(srcPath).size : 0;
+            manifest.push({
+              path: file.output,
+              fullPath: destPath,
+              source: `${target}/${file.source}`,
+              size: binarySize,
+              binary: true,
+              binarySrc: srcPath,
+            });
+          }
+        }
         
         // JSON output mode
         if (opts.json) {
@@ -171,7 +188,12 @@ function registerScaffold(program) {
         
         // Write all files
         for (const file of manifest) {
-          writeFile(file.fullPath, file.content);
+          if (file.binary) {
+            ensureDir(path.dirname(file.fullPath));
+            fs.copyFileSync(file.binarySrc, file.fullPath);
+          } else {
+            writeFile(file.fullPath, file.content);
+          }
           if (!opts.quiet) {
             console.log(`  ${ui.cyan('✓')} ${file.path}`);
           }

package/src/commands/workflow.js

@@ -0,0 +1,336 @@
+'use strict';
+
+const pc = require('picocolors');
+const ui = require('../lib/ui');
+
+/**
+ * Parse repeatable --input key=value options into an object.
+ * Used as Commander's option reducer.
+ *
+ * @param {string} pair - "key=value" string
+ * @param {object} prev - Accumulated object
+ * @returns {object}
+ */
+function collectInputs(pair, prev) {
+  const eq = pair.indexOf('=');
+  if (eq === -1) {
+    throw new Error(`Invalid input format: "${pair}". Expected key=value`);
+  }
+  const key = pair.slice(0, eq).trim();
+  const value = pair.slice(eq + 1).trim();
+  prev[key] = value;
+  return prev;
+}
+
+/**
+ * Register the workflow command on a Commander program.
+ * @param {import('commander').Command} program
+ */
+function registerWorkflow(program) {
+  const wfCmd = program
+    .command('workflow')
+    .alias('wf')
+    .description('Run, validate, and manage composable RAG workflows');
+
+  // ── workflow run <file> ──
+  wfCmd
+    .command('run <file>')
+    .description('Execute a workflow file or built-in template')
+    .option('--input <key=value>', 'Set a workflow input (repeatable)', collectInputs, {})
+    .option('--db <name>', 'Override default database')
+    .option('--collection <name>', 'Override default collection')
+    .option('--json', 'Output results as JSON', false)
+    .option('--quiet', 'Suppress progress output', false)
+    .option('--dry-run', 'Show execution plan without running', false)
+    .option('--verbose', 'Show step details', false)
+    .action(async (file, opts) => {
+      const { loadWorkflow, executeWorkflow, buildExecutionPlan, validateWorkflow } = require('../lib/workflow');
+
+      let definition;
+      try {
+        definition = loadWorkflow(file);
+      } catch (err) {
+        console.error(ui.error(err.message));
+        process.exit(1);
+      }
+
+      // Validation
+      const errors = validateWorkflow(definition);
+      if (errors.length > 0) {
+        console.error(ui.error('Workflow validation failed:'));
+        for (const e of errors) console.error(`  ${pc.red('-')} ${e}`);
+        process.exit(1);
+      }
+
+      const workflowName = definition.name || file;
+
+      if (opts.dryRun) {
+        // Dry run: show plan
+        const layers = buildExecutionPlan(definition.steps);
+        const stepMap = new Map(definition.steps.map(s => [s.id, s]));
+
+        if (opts.json) {
+          console.log(JSON.stringify({ name: workflowName, layers, steps: definition.steps, inputs: opts.input }, null, 2));
+          return;
+        }
+
+        console.log();
+        console.log(`${pc.bold('vai workflow:')} ${workflowName} ${pc.dim('(dry run)')}`);
+        console.log(pc.dim('═'.repeat(50)));
+        console.log();
+
+        // Show inputs
+        const inputKeys = Object.keys(opts.input);
+        if (inputKeys.length > 0) {
+          console.log(pc.bold('Inputs:'));
+          for (const [k, v] of Object.entries(opts.input)) {
+            console.log(`  ${pc.cyan(k)}: ${JSON.stringify(v)}`);
+          }
+          console.log();
+        }
+
+        // Show execution plan
+        console.log(pc.bold('Execution plan:'));
+        let stepNum = 1;
+        for (let i = 0; i < layers.length; i++) {
+          const layer = layers[i];
+          for (const stepId of layer) {
+            const step = stepMap.get(stepId);
+            const toolDesc = `${step.tool}(${summarizeInputs(step.inputs)})`;
+            console.log(`  ${pc.dim(`${stepNum}.`)} ${pc.cyan(stepId)} ${pc.dim('->')} ${toolDesc}`);
+            stepNum++;
+          }
+          if (layer.length > 1) {
+            console.log(`     ${pc.dim(`(${layer.join(' and ')} run in parallel)`)}`);
+          }
+        }
+        console.log();
+        return;
+      }
+
+      // Execute workflow
+      try {
+        const result = await executeWorkflow(definition, {
+          inputs: opts.input,
+          db: opts.db,
+          collection: opts.collection,
+          dryRun: false,
+          verbose: opts.verbose,
+          json: opts.json,
+          onStepStart: !opts.quiet ? (stepId, step) => {
+            process.stderr.write(`  ${pc.dim('...')} ${step.name || stepId}\r`);
+          } : undefined,
+          onStepComplete: !opts.quiet ? (stepId, output, durationMs) => {
+            const stepDef = definition.steps.find(s => s.id === stepId);
+            const summary = summarizeOutput(stepDef?.tool, output);
+            console.error(`  ${pc.green('✔')} ${stepDef?.name || stepId}  ${pc.dim(summary)}  ${pc.dim(`[${durationMs}ms]`)}`);
+          } : undefined,
+          onStepSkip: !opts.quiet ? (stepId, reason) => {
+            const stepDef = definition.steps.find(s => s.id === stepId);
+            console.error(`  ${pc.yellow('⊘')} ${stepDef?.name || stepId}  ${pc.dim(`skipped: ${reason}`)}`);
+          } : undefined,
+          onStepError: !opts.quiet ? (stepId, err) => {
+            const stepDef = definition.steps.find(s => s.id === stepId);
+            console.error(`  ${pc.red('✗')} ${stepDef?.name || stepId}  ${pc.red(err.message)}`);
+          } : undefined,
+        });
+
+        if (!opts.quiet) {
+          console.error();
+          console.error(`${pc.bold('vai workflow:')} ${workflowName}`);
+          console.error(pc.dim('═'.repeat(50)));
+          // Step summaries already printed via callbacks
+          console.error();
+          console.error(`${pc.dim('Complete.')} ${result.steps.length} steps, ${result.totalTimeMs}ms total.`);
+          console.error();
+        }
+
+        // Output
+        if (opts.json) {
+          console.log(JSON.stringify(result.output, null, 2));
+        } else if (result.output) {
+          // Pretty-print top results if they exist
+          const output = result.output;
+          if (output.results && Array.isArray(output.results)) {
+            const top = output.results.slice(0, 5);
+            console.log(pc.bold('Top results:'));
+            for (let i = 0; i < top.length; i++) {
+              const r = top[i];
+              const source = r.source || r.text?.slice(0, 50) || `result ${i + 1}`;
+              const score = r.score != null ? ` (${r.score.toFixed(2)})` : '';
+              console.log(`  ${pc.dim(`[${i + 1}]`)} ${source}${pc.dim(score)}`);
+            }
+          } else if (output.summary) {
+            console.log(output.summary);
+          } else if (output.comparison) {
+            console.log(pc.bold('Cost comparison:'));
+            for (const item of output.comparison) {
+              if (item && item.model) {
+                console.log(`  ${pc.cyan(item.model)}: $${item.totalCost} total (embed: $${item.embeddingCost}, queries: $${item.monthlyQueryCost}/mo)`);
+              }
+            }
+          } else {
+            console.log(JSON.stringify(output, null, 2));
+          }
+        }
+      } catch (err) {
+        console.error(ui.error(err.message));
+        if (opts.verbose) {
+          console.error(pc.dim(err.stack));
+        }
+        process.exit(1);
+      }
+    });
+
+  // ── workflow validate <file> ──
+  wfCmd
+    .command('validate <file>')
+    .description('Validate a workflow file without executing')
+    .action((file) => {
+      const { loadWorkflow, validateWorkflow, buildExecutionPlan } = require('../lib/workflow');
+
+      let definition;
+      try {
+        definition = loadWorkflow(file);
+      } catch (err) {
+        console.error(ui.error(err.message));
+        process.exit(1);
+      }
+
+      const errors = validateWorkflow(definition);
+      if (errors.length > 0) {
+        console.error(ui.error(`Workflow has ${errors.length} error(s):`));
+        for (const e of errors) console.error(`  ${pc.red('-')} ${e}`);
+        process.exit(1);
+      }
+
+      const layers = buildExecutionPlan(definition.steps);
+      console.log(ui.success(`${definition.name} is valid`));
+      console.log(`  ${pc.dim('Steps:')} ${definition.steps.length}`);
+      console.log(`  ${pc.dim('Layers:')} ${layers.length} (${layers.map(l => l.length).join(' + ')} steps)`);
+
+      if (definition.inputs) {
+        const required = Object.entries(definition.inputs).filter(([, s]) => s.required).map(([k]) => k);
+        if (required.length > 0) {
+          console.log(`  ${pc.dim('Required inputs:')} ${required.join(', ')}`);
+        }
+      }
+    });
+
+  // ── workflow list ──
+  wfCmd
+    .command('list')
+    .description('List built-in workflow templates')
+    .action(() => {
+      const { listBuiltinWorkflows } = require('../lib/workflow');
+
+      const workflows = listBuiltinWorkflows();
+      if (workflows.length === 0) {
+        console.log(pc.dim('No built-in workflows found.'));
+        return;
+      }
+
+      console.log();
+      console.log(pc.bold('Built-in workflow templates:'));
+      console.log();
+      for (const wf of workflows) {
+        console.log(`  ${pc.cyan(wf.name.padEnd(28))} ${wf.description}`);
+      }
+      console.log();
+      console.log(pc.dim('Run with: vai workflow run <name> --input key=value'));
+      console.log();
+    });
+
+  // ── workflow init ──
+  wfCmd
+    .command('init')
+    .description('Scaffold a new workflow file')
+    .option('--name <name>', 'Workflow name')
+    .option('--output <file>', 'Output file path')
+    .action((opts) => {
+      const fs = require('fs');
+      const name = opts.name || 'my-workflow';
+      const filename = opts.output || `./${name}.vai-workflow.json`;
+
+      const scaffold = {
+        $schema: 'https://vai.dev/schemas/workflow-v1.json',
+        name: name.replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase()),
+        description: 'TODO: describe what this workflow does',
+        version: '1.0.0',
+        inputs: {
+          query: {
+            type: 'string',
+            description: 'The search query',
+            required: true,
+          },
+        },
+        defaults: {},
+        steps: [
+          {
+            id: 'search',
+            name: 'Search knowledge base',
+            tool: 'query',
+            inputs: {
+              query: '{{ inputs.query }}',
+              limit: 10,
+            },
+          },
+        ],
+        output: {
+          results: '{{ search.output.results }}',
+          query: '{{ inputs.query }}',
+        },
+      };
+
+      fs.writeFileSync(filename, JSON.stringify(scaffold, null, 2) + '\n');
+      console.log(ui.success(`Created ${filename}`));
+      console.log(`  ${pc.dim('Run with:')} vai workflow run ${filename} --input query="your question"`);
+      console.log(`  ${pc.dim('Validate:')} vai workflow validate ${filename}`);
+    });
+}
+
+/**
+ * Summarize step inputs for dry-run display.
+ */
+function summarizeInputs(inputs) {
+  if (!inputs) return '';
+  const parts = [];
+  for (const [k, v] of Object.entries(inputs)) {
+    if (typeof v === 'string' && v.includes('{{')) {
+      parts.push(`${k}=<ref>`);
+    } else if (typeof v === 'string') {
+      parts.push(`${k}=${v.length > 20 ? v.slice(0, 20) + '...' : v}`);
+    } else if (typeof v === 'number' || typeof v === 'boolean') {
+      parts.push(`${k}=${v}`);
+    }
+  }
+  return parts.join(', ');
+}
+
+/**
+ * Summarize step output for display.
+ */
+function summarizeOutput(tool, output) {
+  if (!output) return '';
+  if (output.results && output.resultCount != null) {
+    return `${output.resultCount} results`;
+  }
+  if (output.insertedCount != null) {
+    return `${output.insertedCount} docs inserted`;
+  }
+  if (output.similarity != null) {
+    return `similarity: ${output.similarity.toFixed(4)}`;
+  }
+  if (output.text) {
+    return `${output.text.length} chars`;
+  }
+  if (output.embedding) {
+    return `${output.dimensions}d embedding`;
+  }
+  if (output.totalCost != null) {
+    return `$${output.totalCost}`;
+  }
+  return '';
+}
+
+module.exports = { registerWorkflow, collectInputs };

package/src/lib/chat.js

@@ -3,8 +3,9 @@
 /**
  * Chat Orchestrator
  *
- * Coordinates the retrieval pipeline (embed → search → rerank)
+ * Coordinates the retrieval pipeline (embed -> search -> rerank)
  * with LLM generation and history management.
+ * Supports both pipeline mode (fixed RAG) and agent mode (tool-calling).
  */
 
 const { generateEmbeddings, apiRequest } = require('./api');
@@ -32,12 +33,12 @@ function resolveSourceLabel(doc) {
   return doc.source || meta.source || doc._id?.toString() || 'unknown';
 }
 const { getMongoCollection } = require('./mongo');
-const { buildMessages } = require('./prompt');
+const { buildMessages, buildAgentMessages } = require('./prompt');
 const { getDefaultModel, DEFAULT_RERANK_MODEL } = require('./catalog');
 const { loadProject } = require('./project');
 
 /**
- * Perform retrieval: embed query → vector search → optional rerank.
+ * Perform retrieval: embed query -> vector search -> optional rerank.
  *
  * @param {object} params
  * @param {string} params.query - User's question
@@ -154,7 +155,7 @@ async function retrieve({ query, db, collection, opts = {} }) {
 }
 
 /**
- * Execute a single chat turn: retrieve context → build prompt → generate response.
+ * Execute a single chat turn: retrieve context -> build prompt -> generate response.
  *
  * @param {object} params
  * @param {string} params.query - User's question
@@ -246,7 +247,172 @@ async function* chatTurn({ query, db, collection, llm, history, opts = {} }) {
   };
 }
 
+/**
+ * Execute a single agent chat turn: LLM decides which tools to call.
+ *
+ * @param {object} params
+ * @param {string} params.query - User's question
+ * @param {object} params.llm - LLM provider instance (must have chatWithTools)
+ * @param {import('./history').ChatHistory} params.history - Chat history
+ * @param {object} [params.opts] - Additional options
+ * @param {string} [params.opts.systemPrompt] - Override agent system prompt
+ * @param {number} [params.opts.maxIterations] - Max tool-calling iterations (default 10)
+ * @param {string} [params.opts.db] - Default database for tool calls
+ * @param {string} [params.opts.collection] - Default collection for tool calls
+ * @returns {AsyncGenerator<{type: string, data: any}>}
+ *   Yields: { type: 'tool_call', data: { name, args, result, error, timeMs } }
+ *           { type: 'chunk', data: string }
+ *           { type: 'done', data: { fullResponse, toolCalls, metadata } }
+ */
+async function* agentChatTurn({ query, llm, history, opts = {} }) {
+  const { getToolDefinitions, executeTool } = require('./tool-registry');
+
+  const maxIterations = opts.maxIterations || 10;
+  const start = Date.now();
+
+  // 1. Build initial messages
+  const initialMessages = buildAgentMessages({
+    query,
+    history: history.getMessagesWithBudget(8000),
+    systemPrompt: opts.systemPrompt,
+  });
+
+  // 2. Get tool definitions for this provider
+  const format = llm.name === 'anthropic' ? 'anthropic' : 'openai';
+  const tools = getToolDefinitions(format);
+
+  // Track messages for the tool-calling loop (mutable copy)
+  const messages = [...initialMessages];
+  const toolCallLog = [];
+
+  // 3. Agent loop
+  for (let iteration = 0; iteration < maxIterations; iteration++) {
+    const response = await llm.chatWithTools(messages, tools);
+
+    // Text response: done
+    if (response.type === 'text') {
+      const fullResponse = response.content;
+      yield { type: 'chunk', data: fullResponse };
+
+      const totalTimeMs = Date.now() - start;
+
+      // Store turns in history
+      await history.addTurn({ role: 'user', content: query });
+      await history.addTurn({
+        role: 'assistant',
+        content: fullResponse,
+        metadata: {
+          mode: 'agent',
+          llmProvider: llm.name,
+          llmModel: llm.model,
+          toolCallCount: toolCallLog.length,
+          iterationCount: iteration + 1,
+          totalTimeMs,
+        },
+      });
+
+      yield {
+        type: 'done',
+        data: {
+          fullResponse,
+          toolCalls: toolCallLog,
+          metadata: {
+            mode: 'agent',
+            iterationCount: iteration + 1,
+            toolCallCount: toolCallLog.length,
+            totalTimeMs,
+          },
+        },
+      };
+      return;
+    }
+
+    // Tool calls: execute each and continue loop
+    if (response.type === 'tool_calls') {
+      // Append assistant tool-call message
+      messages.push(llm.formatAssistantToolCall(response));
+
+      for (const call of response.calls) {
+        const callStart = Date.now();
+        let result;
+        let error = null;
+
+        // Inject default db/collection if not provided
+        const args = { ...call.arguments };
+        if (opts.db && !args.db) args.db = opts.db;
+        if (opts.collection && !args.collection) args.collection = opts.collection;
+
+        try {
+          result = await executeTool(call.name, args);
+        } catch (err) {
+          error = err.message;
+          result = { content: [{ type: 'text', text: `Error: ${err.message}` }] };
+        }
+
+        const callTimeMs = Date.now() - callStart;
+
+        // Extract text content from result for the LLM
+        const resultText = result.content
+          ? result.content.map(c => c.text || JSON.stringify(c)).join('\n')
+          : JSON.stringify(result.structuredContent || {});
+
+        // Append tool result message
+        messages.push(llm.formatToolResult(call.id, resultText, !!error));
+
+        const logEntry = {
+          name: call.name,
+          args,
+          result: result.structuredContent || null,
+          error,
+          timeMs: callTimeMs,
+        };
+        toolCallLog.push(logEntry);
+
+        yield { type: 'tool_call', data: logEntry };
+      }
+
+      // Continue loop to let LLM see results and decide next action
+      continue;
+    }
+  }
+
+  // Max iterations reached: yield a fallback message
+  const fallback = 'I reached the maximum number of tool-calling iterations. Here is what I found so far based on the tool results above.';
+  yield { type: 'chunk', data: fallback };
+
+  await history.addTurn({ role: 'user', content: query });
+  await history.addTurn({
+    role: 'assistant',
+    content: fallback,
+    metadata: {
+      mode: 'agent',
+      llmProvider: llm.name,
+      llmModel: llm.model,
+      toolCallCount: toolCallLog.length,
+      iterationCount: maxIterations,
+      totalTimeMs: Date.now() - start,
+      maxIterationsReached: true,
+    },
+  });
+
+  yield {
+    type: 'done',
+    data: {
+      fullResponse: fallback,
+      toolCalls: toolCallLog,
+      metadata: {
+        mode: 'agent',
+        iterationCount: maxIterations,
+        toolCallCount: toolCallLog.length,
+        totalTimeMs: Date.now() - start,
+        maxIterationsReached: true,
+      },
+    },
+  };
+}
+
 module.exports = {
   retrieve,
   chatTurn,
+  agentChatTurn,
 };

package/src/lib/explanations.js

@@ -1434,6 +1434,50 @@ const concepts = {
       'vai chat --db myapp --collection knowledge',
     ],
   },
+
+  workflows: {
+    title: 'Agentic Workflows',
+    summary: 'Composable, multi-step RAG pipelines as JSON files',
+    content: [
+      `${pc.cyan('Workflows')} are composable, multi-step RAG pipelines defined as portable`,
+      `JSON files. Think Docker Compose or GitHub Actions, but for search and retrieval`,
+      `pipelines.`,
+      ``,
+      `${pc.bold('Why workflows?')} Instead of writing bash scripts to chain vai commands,`,
+      `a workflow file captures the intent declaratively. Workflows are reproducible,`,
+      `shareable (commit them to git), and inspectable.`,
+      ``,
+      `${pc.bold('How it works:')}`,
+      `Each workflow defines a DAG (directed acyclic graph) of steps. Each step maps`,
+      `to a vai operation (query, search, rerank, embed, etc.) or a control flow`,
+      `operation (merge, filter, transform, generate). Steps reference outputs from`,
+      `previous steps using ${pc.cyan('{{ template expressions }}')} for data flow.`,
+      ``,
+      `${pc.bold('Template expressions:')}`,
+      `  ${pc.cyan('{{ inputs.query }}')}              workflow input parameter`,
+      `  ${pc.cyan('{{ search.output.results }}')}      results from a previous step`,
+      `  ${pc.cyan('{{ merge.output.results[0] }}')}    array indexing`,
+      `  ${pc.cyan('{{ defaults.db }}')}                workflow default value`,
+      ``,
+      `${pc.bold('Parallel execution:')} The engine automatically detects independent steps`,
+      `and runs them in parallel. No configuration needed.`,
+      ``,
+      `${pc.bold('Step types:')}`,
+      `  ${pc.dim('VAI tools:')}     query, search, rerank, embed, similarity, ingest,`,
+      `                 collections, models, explain, estimate`,
+      `  ${pc.dim('Control flow:')} merge, filter, transform, generate`,
+      ``,
+      `${pc.bold('Built-in templates:')} Run ${pc.cyan('vai workflow list')} to see available`,
+      `templates like multi-collection-search, smart-ingest, research-and-summarize.`,
+    ].join('\n'),
+    links: ['https://github.com/mrlynn/voyageai-cli#workflows'],
+    tryIt: [
+      'vai workflow list',
+      'vai workflow init --name my-pipeline',
+      'vai workflow validate ./my-pipeline.vai-workflow.json',
+      'vai workflow run multi-collection-search --input query="test" --dry-run',
+    ],
+  },
 };
 
 /**
@@ -1598,6 +1642,15 @@ const aliases = {
   conversational: 'chat',
   'chat-history': 'chat',
   llm: 'chat',
+  // Workflow aliases
+  workflow: 'workflows',
+  workflows: 'workflows',
+  'rag-pipeline': 'workflows',
+  composable: 'workflows',
+  'workflow-engine': 'workflows',
+  'vai-workflow': 'workflows',
+  pipeline: 'workflows',
+  dag: 'workflows',
 };
 
 /**