voyageai-cli 1.22.0 → 1.23.0

package/src/commands/init.js

@@ -2,50 +2,12 @@
 
 const fs = require('fs');
 const path = require('path');
-const readline = require('readline');
-const { MODEL_CATALOG } = require('../lib/catalog');
-const { STRATEGIES } = require('../lib/chunker');
 const { defaultProjectConfig, saveProject, findProjectFile, PROJECT_FILE } = require('../lib/project');
+const { runWizard } = require('../lib/wizard');
+const { createCLIRenderer } = require('../lib/wizard-cli');
+const { initSteps } = require('../lib/wizard-steps-init');
 const ui = require('../lib/ui');
 
-/**
- * Prompt the user for input with a default value.
- * @param {readline.Interface} rl
- * @param {string} question
- * @param {string} [defaultVal]
- * @returns {Promise<string>}
- */
-function ask(rl, question, defaultVal) {
-  const suffix = defaultVal ? ` ${ui.dim(`(${defaultVal})`)}` : '';
-  return new Promise((resolve) => {
-    rl.question(`  ${question}${suffix}: `, (answer) => {
-      resolve(answer.trim() || defaultVal || '');
-    });
-  });
-}
-
-/**
- * Prompt for a choice from a list.
- * @param {readline.Interface} rl
- * @param {string} question
- * @param {string[]} choices
- * @param {string} defaultVal
- * @returns {Promise<string>}
- */
-async function askChoice(rl, question, choices, defaultVal) {
-  console.log('');
-  for (let i = 0; i < choices.length; i++) {
-    const marker = choices[i] === defaultVal ? ui.cyan('→') : ' ';
-    console.log(`  ${marker} ${i + 1}. ${choices[i]}`);
-  }
-  const answer = await ask(rl, question, defaultVal);
-  // Accept number or value
-  const num = parseInt(answer, 10);
-  if (num >= 1 && num <= choices.length) return choices[num - 1];
-  if (choices.includes(answer)) return answer;
-  return defaultVal;
-}
-
 /**
  * Register the init command on a Commander program.
  * @param {import('commander').Command} program
@@ -72,7 +34,7 @@ function registerInit(program) {
 
       // Non-interactive mode
       if (opts.yes || opts.json) {
-        const filePath = saveProject(defaults);
+        saveProject(defaults);
         if (opts.json) {
           console.log(JSON.stringify(defaults, null, 2));
         } else if (!opts.quiet) {
@@ -81,71 +43,46 @@ function registerInit(program) {
         return;
       }
 
-      // Interactive mode
-      console.log('');
-      console.log(ui.bold('  🚀 Initialize Voyage AI Project'));
-      console.log(ui.dim('  Creates .vai.json in the current directory.'));
-      console.log(ui.dim('  Press Enter to accept defaults.'));
-      console.log('');
-
-      const rl = readline.createInterface({
-        input: process.stdin,
-        output: process.stdout,
+      // Interactive mode — use wizard
+      const { answers, cancelled } = await runWizard({
+        steps: initSteps,
+        config: {},
+        renderer: createCLIRenderer({
+          title: '🚀 Initialize Voyage AI Project',
+          doneMessage: 'Project initialized!',
+        }),
       });
 
-      try {
-        // Embedding model
-        const embeddingModels = MODEL_CATALOG
-          .filter(m => m.type === 'embedding' && !m.legacy && !m.unreleased)
-          .map(m => m.name);
-        const model = await askChoice(rl, 'Embedding model', embeddingModels, defaults.model);
-
-        // MongoDB settings
-        console.log('');
-        console.log(ui.bold('  MongoDB Atlas'));
-        const db = await ask(rl, 'Database name', defaults.db || 'myapp');
-        const collection = await ask(rl, 'Collection name', defaults.collection || 'documents');
-        const field = await ask(rl, 'Embedding field', defaults.field);
-        const index = await ask(rl, 'Vector index name', defaults.index);
-
-        // Dimensions
-        const modelInfo = MODEL_CATALOG.find(m => m.name === model);
-        const defaultDims = modelInfo && modelInfo.dimensions.includes('1024') ? '1024' : '512';
-        const dimensions = parseInt(await ask(rl, 'Dimensions', defaultDims), 10) || parseInt(defaultDims, 10);
+      if (cancelled) {
+        process.exit(0);
+      }
 
-        // Chunking
-        console.log('');
-        console.log(ui.bold('  Chunking'));
-        const strategy = await askChoice(rl, 'Chunk strategy', STRATEGIES, defaults.chunk.strategy);
-        const chunkSize = parseInt(await ask(rl, 'Chunk size (chars)', String(defaults.chunk.size)), 10);
-        const chunkOverlap = parseInt(await ask(rl, 'Chunk overlap (chars)', String(defaults.chunk.overlap)), 10);
+      // Build config from answers
+      const config = {
+        model: answers.model || defaults.model,
+        db: answers.db || defaults.db,
+        collection: answers.collection || defaults.collection,
+        field: answers.field || defaults.field,
+        inputType: 'document',
+        dimensions: parseInt(answers.dimensions, 10) || defaults.dimensions,
+        index: answers.index || defaults.index,
+        chunk: {
+          strategy: answers.chunkStrategy || defaults.chunk.strategy,
+          size: parseInt(answers.chunkSize, 10) || defaults.chunk.size,
+          overlap: parseInt(answers.chunkOverlap, 10) || defaults.chunk.overlap,
+        },
+      };
 
-        const config = {
-          model,
-          db,
-          collection,
-          field,
-          inputType: 'document',
-          dimensions,
-          index,
-          chunk: {
-            strategy,
-            size: chunkSize,
-            overlap: chunkOverlap,
-          },
-        };
+      const filePath = saveProject(config);
+      const relPath = path.relative(process.cwd(), filePath);
 
-        const filePath = saveProject(config);
-        console.log('');
-        console.log(ui.success(`Created ${path.relative(process.cwd(), filePath)}`));
+      if (!opts.quiet) {
         console.log('');
         console.log(ui.dim('  Next steps:'));
         console.log(ui.dim('    vai chunk ./docs/           # Chunk your documents'));
-        console.log(ui.dim('    vai pipeline ./docs/        # Chunk → embed → store (coming soon)'));
+        console.log(ui.dim('    vai pipeline ./docs/        # Chunk → embed → store'));
         console.log(ui.dim('    vai search --query "..."    # Search your collection'));
         console.log('');
-      } finally {
-        rl.close();
       }
     });
 }

package/src/commands/mcp-server.js

@@ -0,0 +1,49 @@
+'use strict';
+
+/**
+ * Register the mcp-server command (aliased as mcp).
+ * @param {import('commander').Command} program
+ */
+function registerMcpServer(program) {
+  const cmd = program
+    .command('mcp-server')
+    .alias('mcp')
+    .description('Start the MCP (Model Context Protocol) server — expose vai tools to AI agents')
+    .option('--transport <mode>', 'Transport mode: stdio or http', 'stdio')
+    .option('--port <number>', 'HTTP port (http transport only)', (v) => parseInt(v, 10), 3100)
+    .option('--host <address>', 'Bind address (http transport only)', '127.0.0.1')
+    .option('--db <name>', 'Default MongoDB database for tools')
+    .option('--collection <name>', 'Default collection for tools')
+    .option('--verbose', 'Enable debug logging to stderr')
+    .action(async (opts) => {
+      if (opts.verbose) {
+        process.env.VAI_MCP_VERBOSE = '1';
+      }
+
+      // Set default db/collection if provided via CLI
+      if (opts.db) process.env.VAI_DEFAULT_DB = opts.db;
+      if (opts.collection) process.env.VAI_DEFAULT_COLLECTION = opts.collection;
+
+      const { runStdioServer, runHttpServer } = require('../mcp/server');
+
+      if (opts.transport === 'http') {
+        await runHttpServer({ port: opts.port, host: opts.host });
+      } else if (opts.transport === 'stdio') {
+        await runStdioServer();
+      } else {
+        console.error(`Unknown transport: ${opts.transport}. Use "stdio" or "http".`);
+        process.exit(1);
+      }
+    });
+
+  // Subcommand: generate-key
+  cmd
+    .command('generate-key')
+    .description('Generate a new API key for remote MCP server authentication')
+    .action(() => {
+      const { generateKey } = require('../mcp/server');
+      generateKey();
+    });
+}
+
+module.exports = { registerMcpServer };

package/src/commands/ping.js

@@ -190,6 +190,58 @@ function registerPing(program) {
         }
       }
 
+      // ── LLM provider ping (optional) ──
+      const { createLLMProvider, resolveLLMConfig } = require('../lib/llm');
+      const llmConfig = resolveLLMConfig();
+      if (llmConfig.provider) {
+        const llmStart = Date.now();
+        let llmSpin;
+        if (useSpinner) {
+          llmSpin = ui.spinner(`Testing LLM provider (${llmConfig.provider})...`);
+          llmSpin.start();
+        }
+
+        try {
+          const llm = createLLMProvider();
+          const pingResult = await llm.ping();
+          const llmElapsed = Date.now() - llmStart;
+
+          results.llm = { ok: pingResult.ok, elapsed: llmElapsed, provider: llmConfig.provider, model: pingResult.model };
+          if (pingResult.error) results.llm.error = pingResult.error;
+
+          if (llmSpin) llmSpin.stop();
+
+          if (!opts.json && !opts.quiet) {
+            console.log('');
+            if (pingResult.ok) {
+              console.log(ui.success(`LLM Provider connected ${ui.dim('(' + llmElapsed + 'ms)')}`));
+              console.log(ui.label('Provider', llmConfig.provider));
+              console.log(ui.label('Model', pingResult.model));
+            } else {
+              console.log(ui.error(`LLM Provider failed: ${pingResult.error}`));
+            }
+          }
+        } catch (err) {
+          if (llmSpin) llmSpin.stop();
+          const llmElapsed = Date.now() - llmStart;
+          results.llm = { ok: false, elapsed: llmElapsed, provider: llmConfig.provider, error: err.message };
+          if (!opts.json && !opts.quiet) {
+            console.log('');
+            console.log(ui.error(`LLM Provider error: ${err.message}`));
+          }
+        }
+      }
+
+      // ── Chat readiness summary ──
+      if (!opts.json && !opts.quiet && llmConfig.provider) {
+        console.log('');
+        if (results.voyage?.ok && results.llm?.ok) {
+          console.log(ui.success('Chat is ready. Run: vai chat'));
+        } else if (!results.llm?.ok) {
+          console.log(ui.warn('Chat requires a working LLM provider. Check your configuration.'));
+        }
+      }
+
       // Emit JSON at the end with all results
       if (opts.json) {
         console.log(JSON.stringify({ ok: true, ...results }, null, 2));

package/src/commands/pipeline.js

@@ -62,6 +62,7 @@ function registerPipeline(program) {
     .option('--ignore <dirs>', 'Directory names to skip', 'node_modules,.git,__pycache__')
     .option('--create-index', 'Auto-create vector search index if it doesn\'t exist')
     .option('--dry-run', 'Show what would happen without executing')
+    .option('--estimate', 'Show estimated tokens and cost without executing')
     .option('--json', 'Machine-readable JSON output')
     .option('-q, --quiet', 'Suppress non-essential output')
     .action(async (input, opts) => {
@@ -75,7 +76,7 @@ function registerPipeline(program) {
         const collection = opts.collection || proj.collection;
         const field = opts.field || proj.field || 'embedding';
         const index = opts.index || proj.index || 'vector_index';
-        const model = opts.model || proj.model || getDefaultModel();
+        let model = opts.model || proj.model || getDefaultModel();
         const dimensions = opts.dimensions || proj.dimensions;
         const strategy = opts.strategy || projChunk.strategy || 'recursive';
         const chunkSize = opts.chunkSize || projChunk.size || 512;
@@ -175,22 +176,35 @@ function registerPipeline(program) {
 
         // Dry run — stop here
         if (opts.dryRun) {
+          const { estimateCost, formatCostEstimate } = require('../lib/cost');
+          const est = estimateCost(totalTokens, model);
           if (opts.json) {
             console.log(JSON.stringify({
               dryRun: true,
               files: files.length,
               chunks: allChunks.length,
               estimatedTokens: totalTokens,
+              estimatedCost: est.cost,
+              pricePerMToken: est.pricePerMToken,
               strategy, chunkSize, overlap, model, db, collection, field,
             }, null, 2));
           } else {
             console.log(ui.success(`Dry run complete: ${fmtNum(allChunks.length)} chunks from ${files.length} files.`));
-            const cost = (totalTokens / 1e6) * 0.12;
-            console.log(ui.dim(`  Estimated embedding cost: ~$${cost.toFixed(4)} with ${model}`));
+            console.log('');
+            console.log(formatCostEstimate(est));
+            console.log('');
           }
           return;
         }
 
+        // Estimate — show comparison table, let user confirm or switch model, then continue
+        if (opts.estimate) {
+          const { confirmOrSwitchModel } = require('../lib/cost');
+          const chosenModel = await confirmOrSwitchModel(totalTokens, model, { json: opts.json });
+          if (!chosenModel) return; // cancelled
+          model = chosenModel;
+        }
+
         // Step 3: Embed in batches
         if (verbose) console.log(ui.bold('Step 2/3 — Embedding'));
 

package/src/commands/playground.js

@@ -61,6 +61,9 @@ function createPlaygroundServer() {
 
   const htmlPath = path.join(__dirname, '..', 'playground', 'index.html');
 
+  // Chat history — scoped to the server lifetime (in-memory, no persistence)
+  let _chatHistory = null;
+
   const server = http.createServer(async (req, res) => {
     // CORS headers for local dev
     res.setHeader('Access-Control-Allow-Origin', '*');
@@ -229,6 +232,40 @@ function createPlaygroundServer() {
         return;
       }
 
+      // API: Chat config (GET)
+      if (req.method === 'GET' && req.url === '/api/chat/config') {
+        const { resolveLLMConfig } = require('../lib/llm');
+        const { loadProject } = require('../lib/project');
+        const llmConfig = resolveLLMConfig();
+        const { config: proj } = loadProject();
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({
+          provider: llmConfig.provider || null,
+          model: llmConfig.model || null,
+          hasLLMKey: !!llmConfig.apiKey || llmConfig.provider === 'ollama',
+          db: proj.db || null,
+          collection: proj.collection || null,
+          chat: proj.chat || {},
+        }));
+        return;
+      }
+
+      // API: Chat models — list available models for a provider
+      if (req.method === 'GET' && req.url?.startsWith('/api/chat/models')) {
+        const url = new URL(req.url, 'http://localhost');
+        const provider = url.searchParams.get('provider');
+        if (!provider) {
+          res.writeHead(400, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ error: 'provider query param required' }));
+          return;
+        }
+        const { listModels } = require('../lib/llm');
+        const models = await listModels(provider);
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ provider, models }));
+        return;
+      }
+
       // API: Config
       if (req.method === 'GET' && req.url === '/api/config') {
         const key = process.env.VOYAGE_API_KEY || getConfigValue('apiKey');
@@ -240,6 +277,74 @@ function createPlaygroundServer() {
         return;
       }
 
+      // API: Settings origins — where each config value comes from
+      if (req.method === 'GET' && req.url === '/api/settings/origins') {
+        const { resolveLLMConfig } = require('../lib/llm');
+        const { loadProject } = require('../lib/project');
+        const { config: proj } = loadProject();
+        const chatConf = proj.chat || {};
+
+        function resolveOrigin(envVar, configKey, projectValue) {
+          if (envVar && process.env[envVar]) return 'env';
+          if (configKey && getConfigValue(configKey)) return 'config';
+          if (projectValue) return 'project';
+          return 'default';
+        }
+
+        const origins = {
+          apiKey: resolveOrigin('VOYAGE_API_KEY', 'apiKey'),
+          apiBase: resolveOrigin('VOYAGE_API_BASE', 'baseUrl'),
+          provider: resolveOrigin('VAI_LLM_PROVIDER', 'llmProvider', chatConf.provider),
+          model: resolveOrigin('VAI_LLM_MODEL', 'llmModel', chatConf.model),
+          llmApiKey: resolveOrigin('VAI_LLM_API_KEY', 'llmApiKey'),
+          db: proj.db ? 'project' : 'default',
+          collection: proj.collection ? 'project' : 'default',
+        };
+
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify(origins));
+        return;
+      }
+
+      // API: Save chat config (POST) — persists to .vai.json
+      // Placed before generic POST handler so it doesn't require Voyage API key
+      if (req.method === 'POST' && req.url === '/api/chat/config') {
+        const { loadProject, saveProject } = require('../lib/project');
+        const body = await readBody(req);
+        let parsed;
+        try {
+          parsed = JSON.parse(body);
+        } catch {
+          res.writeHead(400, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ error: 'Invalid JSON' }));
+          return;
+        }
+
+        const { config: proj, filePath } = loadProject();
+
+        // Update top-level project fields
+        if (parsed.db !== undefined) proj.db = parsed.db;
+        if (parsed.collection !== undefined) proj.collection = parsed.collection;
+
+        // Update chat-specific settings
+        proj.chat = proj.chat || {};
+        if (parsed.provider !== undefined) proj.chat.provider = parsed.provider;
+        if (parsed.model !== undefined) proj.chat.model = parsed.model;
+        if (parsed.maxDocs !== undefined) proj.chat.maxContextDocs = parsed.maxDocs;
+        if (parsed.rerank !== undefined) proj.chat.rerank = parsed.rerank;
+        if (parsed.systemPrompt !== undefined) proj.chat.systemPrompt = parsed.systemPrompt;
+
+        try {
+          saveProject(proj, filePath || undefined);
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ ok: true }));
+        } catch (err) {
+          res.writeHead(500, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ error: err.message }));
+        }
+        return;
+      }
+
       // Parse JSON body for POST routes
       if (req.method === 'POST') {
         // Check for API key before processing any API calls
@@ -263,6 +368,87 @@ function createPlaygroundServer() {
           return;
         }
 
+        // API: Chat message (streaming SSE)
+        if (req.url === '/api/chat/message') {
+          const { query, db, collection, provider, model, maxDocs, rerank, systemPrompt } = parsed;
+          if (!query) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'query is required' }));
+            return;
+          }
+          if (!db || !collection) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'db and collection are required' }));
+            return;
+          }
+
+          const { createLLMProvider } = require('../lib/llm');
+          const { chatTurn } = require('../lib/chat');
+          const { ChatHistory } = require('../lib/history');
+
+          let llm;
+          try {
+            llm = createLLMProvider({
+              llmProvider: provider || undefined,
+              llmModel: model || undefined,
+            });
+          } catch (err) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: err.message }));
+            return;
+          }
+
+          if (!llm) {
+            res.writeHead(400, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: 'No LLM provider configured. Use vai config set llm-provider <name>' }));
+            return;
+          }
+
+          // Use in-memory history for playground (no session persistence)
+          if (!_chatHistory) _chatHistory = new ChatHistory({ maxTurns: 20 });
+          const history = _chatHistory;
+
+          // Stream response as SSE
+          res.writeHead(200, {
+            'Content-Type': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+            'Connection': 'keep-alive',
+          });
+
+          try {
+            for await (const event of chatTurn({
+              query, db, collection, llm, history,
+              opts: {
+                maxDocs: maxDocs || 5,
+                rerank: rerank !== false,
+                stream: true,
+                systemPrompt,
+              },
+            })) {
+              if (event.type === 'retrieval') {
+                res.write(`event: retrieval\ndata: ${JSON.stringify(event.data)}\n\n`);
+              } else if (event.type === 'chunk') {
+                res.write(`event: chunk\ndata: ${JSON.stringify({ text: event.data })}\n\n`);
+              } else if (event.type === 'done') {
+                res.write(`event: done\ndata: ${JSON.stringify(event.data)}\n\n`);
+              }
+            }
+          } catch (err) {
+            res.write(`event: error\ndata: ${JSON.stringify({ error: err.message })}\n\n`);
+          }
+
+          res.end();
+          return;
+        }
+
+        // API: Chat clear
+        if (req.url === '/api/chat/clear') {
+          if (_chatHistory) _chatHistory.clear();
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ ok: true }));
+          return;
+        }
+
         // API: Embed
         if (req.url === '/api/embed') {
           const { texts, model, inputType, dimensions, output_dtype } = parsed;

package/src/commands/purge.js

@@ -2,7 +2,8 @@
 
 const fs = require('fs');
 const path = require('path');
-const p = require('@clack/prompts');
+let p;
+function clack() { if (!p) p = require('@clack/prompts'); return p; }
 const { loadProject } = require('../lib/project');
 const { connect, close } = require('../lib/mongo');
 const ui = require('../lib/ui');
@@ -95,6 +96,7 @@ function formatSample(docs, limit = 5) {
  * Execute the purge command.
  */
 async function purge(options = {}) {
+  clack(); // lazy-load @clack/prompts
   const quiet = options.quiet || options.json;
 
   // Load project config

package/src/commands/refresh.js

@@ -1,6 +1,7 @@
 'use strict';
 
-const p = require('@clack/prompts');
+let p;
+function clack() { if (!p) p = require('@clack/prompts'); return p; }
 const { loadProject, saveProject } = require('../lib/project');
 const { connect, close } = require('../lib/mongo');
 const { generateEmbeddings } = require('../lib/api');
@@ -53,6 +54,7 @@ function rechunkDocument(doc, options) {
  * Execute the refresh command.
  */
 async function refresh(options = {}) {
+  clack(); // lazy-load @clack/prompts
   const quiet = options.quiet || options.json;
 
   // Load project config

package/src/commands/rerank.js

@@ -21,6 +21,7 @@ function registerRerank(program) {
     .option('--truncation', 'Enable truncation for long inputs')
     .option('--no-truncation', 'Disable truncation')
     .option('--return-documents', 'Return document text in results')
+    .option('--estimate', 'Show estimated tokens and cost without calling the API')
     .option('--json', 'Machine-readable JSON output')
     .option('-q, --quiet', 'Suppress non-essential output')
     .action(async (opts) => {
@@ -71,6 +72,15 @@ function registerRerank(program) {
           process.exit(1);
         }
 
+        // --estimate: show cost comparison, optionally switch model
+        if (opts.estimate) {
+          const { estimateTokens, confirmOrSwitchModel } = require('../lib/cost');
+          const tokens = estimateTokens(opts.query) + documents.reduce((s, d) => s + estimateTokens(d), 0);
+          const chosenModel = await confirmOrSwitchModel(tokens, opts.model, { json: opts.json });
+          if (!chosenModel) return; // cancelled
+          opts.model = chosenModel;
+        }
+
         const body = {
           query: opts.query,
           documents,

package/src/commands/scaffold.js

@@ -2,7 +2,6 @@
 
 const fs = require('fs');
 const path = require('path');
-const p = require('@clack/prompts');
 const { loadProject } = require('../lib/project');
 const { renderTemplate, buildContext, listTemplates } = require('../lib/codegen');
 const { PROJECT_STRUCTURE } = require('../lib/scaffold-structure');
@@ -193,7 +192,7 @@ function registerScaffold(program) {
             structure.startCommand,
           ];
           
-          p.note(steps.join('\n'), 'Next steps');
+          require('@clack/prompts').note(steps.join('\n'), 'Next steps');
           
           console.log('');
           console.log(ui.dim('Configuration:'));