voyageai-cli 1.20.5 → 1.21.0

package/src/commands/playground.js

@@ -114,6 +114,99 @@ function createPlaygroundServer() {
         return;
       }
 
+      // API: Generate code
+      if (req.method === 'POST' && req.url === '/api/generate') {
+        let body = '';
+        req.on('data', chunk => { body += chunk; });
+        req.on('end', () => {
+          try {
+            const { target, component, config } = JSON.parse(body);
+            const codegen = require('../lib/codegen');
+            
+            const templateMap = {
+              vanilla: { client: 'client.js', connection: 'connection.js', retrieval: 'retrieval.js', ingest: 'ingest.js', 'search-api': 'search-api.js' },
+              nextjs: { client: 'lib-voyage.js', connection: 'lib-mongo.js', retrieval: 'route-search.js', ingest: 'route-ingest.js', 'search-page': 'page-search.jsx' },
+              python: { client: 'voyage_client.py', connection: 'mongo_client.py', retrieval: 'app.py', ingest: 'chunker.py' },
+            };
+            
+            const templateName = (templateMap[target] || {})[component];
+            if (!templateName) {
+              res.writeHead(400, { 'Content-Type': 'application/json' });
+              res.end(JSON.stringify({ error: `Unknown component: ${component}` }));
+              return;
+            }
+            
+            const context = codegen.buildContext(config || {}, { projectName: 'my-app' });
+            const code = codegen.renderTemplate(target, templateName.replace(/\.(js|jsx|py)$/, ''), context);
+            
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ code, filename: templateName }));
+          } catch (err) {
+            res.writeHead(500, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      // API: Scaffold project (returns ZIP for web mode)
+      if (req.method === 'POST' && req.url === '/api/scaffold') {
+        let body = '';
+        req.on('data', chunk => { body += chunk; });
+        req.on('end', () => {
+          try {
+            const { projectName, target, config } = JSON.parse(body);
+            const codegen = require('../lib/codegen');
+            const { PROJECT_STRUCTURE } = require('../lib/scaffold-structure');
+            const { createZip } = require('../lib/zip');
+            
+            const structure = PROJECT_STRUCTURE[target];
+            if (!structure) {
+              res.writeHead(400, { 'Content-Type': 'application/json' });
+              res.end(JSON.stringify({ error: `Unknown target: ${target}` }));
+              return;
+            }
+            
+            const context = codegen.buildContext(config || {}, { projectName: projectName || 'my-app' });
+            const files = [];
+            
+            // Render template files
+            for (const file of structure.files) {
+              const content = codegen.renderTemplate(target, file.template.replace(/\.(js|jsx|py|json|md|txt)$/, ''), context);
+              files.push({
+                name: `${projectName}/${file.output}`,
+                content,
+              });
+            }
+            
+            // Add extra static files
+            if (structure.extraFiles) {
+              for (const file of structure.extraFiles) {
+                const content = typeof file.content === 'function' ? file.content(context) : file.content;
+                files.push({
+                  name: `${projectName}/${file.output}`,
+                  content,
+                });
+              }
+            }
+            
+            // Create ZIP
+            const zipBuffer = createZip(files);
+            
+            res.writeHead(200, {
+              'Content-Type': 'application/zip',
+              'Content-Disposition': `attachment; filename="${projectName}.zip"`,
+              'Content-Length': zipBuffer.length,
+            });
+            res.end(zipBuffer);
+          } catch (err) {
+            res.writeHead(500, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
       // API: Concepts (from vai explain)
       if (req.method === 'GET' && req.url === '/api/concepts') {
         const { concepts } = require('../lib/explanations');

package/src/commands/purge.js

@@ -0,0 +1,271 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const p = require('@clack/prompts');
+const { loadProject } = require('../lib/project');
+const { connect, close } = require('../lib/mongo');
+const ui = require('../lib/ui');
+
+/**
+ * Build a MongoDB filter from the provided criteria.
+ */
+function buildFilter(options) {
+  const conditions = [];
+
+  // Filter by source pattern (glob-like)
+  if (options.source) {
+    // Convert glob pattern to regex
+    const pattern = options.source
+      .replace(/\./g, '\\.')
+      .replace(/\*/g, '.*')
+      .replace(/\?/g, '.');
+    conditions.push({ 'metadata.source': { $regex: pattern } });
+  }
+
+  // Filter by embedded date
+  if (options.before) {
+    const date = new Date(options.before);
+    if (isNaN(date.getTime())) {
+      throw new Error(`Invalid date format: ${options.before}`);
+    }
+    conditions.push({ _embeddedAt: { $lt: date } });
+  }
+
+  // Filter by model
+  if (options.model) {
+    conditions.push({ _model: options.model });
+  }
+
+  // Raw MongoDB filter
+  if (options.filter) {
+    try {
+      const rawFilter = JSON.parse(options.filter);
+      conditions.push(rawFilter);
+    } catch (err) {
+      throw new Error(`Invalid JSON filter: ${err.message}`);
+    }
+  }
+
+  // Combine conditions with $and
+  if (conditions.length === 0) {
+    return {};
+  } else if (conditions.length === 1) {
+    return conditions[0];
+  } else {
+    return { $and: conditions };
+  }
+}
+
+/**
+ * Check which documents have stale source files (file no longer exists on disk).
+ */
+async function findStaleDocuments(collection, baseDir) {
+  const docs = await collection.find({ 'metadata.source': { $exists: true } }).toArray();
+  const staleIds = [];
+
+  for (const doc of docs) {
+    const source = doc.metadata?.source;
+    if (source) {
+      // Resolve relative to baseDir or treat as absolute
+      const filePath = path.isAbsolute(source) ? source : path.join(baseDir, source);
+      if (!fs.existsSync(filePath)) {
+        staleIds.push(doc._id);
+      }
+    }
+  }
+
+  return staleIds;
+}
+
+/**
+ * Format a sample of documents for display.
+ */
+function formatSample(docs, limit = 5) {
+  const sample = docs.slice(0, limit);
+  return sample.map(doc => {
+    const source = doc.metadata?.source || doc._id?.toString() || 'unknown';
+    const model = doc._model || 'unknown';
+    const date = doc._embeddedAt ? new Date(doc._embeddedAt).toISOString().split('T')[0] : 'unknown';
+    return `  • ${source} (model: ${model}, date: ${date})`;
+  }).join('\n');
+}
+
+/**
+ * Execute the purge command.
+ */
+async function purge(options = {}) {
+  const quiet = options.quiet || options.json;
+
+  // Load project config
+  const project = loadProject();
+  const db = options.db || project.db || process.env.VAI_DB || 'vai';
+  const collectionName = options.collection || project.collection || process.env.VAI_COLLECTION || 'embeddings';
+
+  if (!quiet) {
+    p.intro(ui.title('vai purge'));
+  }
+
+  // Validate that at least one filter is provided
+  if (!options.source && !options.before && !options.model && !options.filter && !options.stale) {
+    if (options.json) {
+      console.log(JSON.stringify({ error: 'No filter criteria provided. Use --source, --before, --model, --filter, or --stale.' }));
+    } else {
+      p.log.error('No filter criteria provided.');
+      p.log.info('Use --source, --before, --model, --filter, or --stale to specify what to purge.');
+    }
+    return { success: false, error: 'No filter criteria' };
+  }
+
+  let client;
+  try {
+    // Connect to MongoDB
+    if (!quiet) {
+      p.log.step(`Connecting to database: ${db}`);
+    }
+    client = await connect(db);
+    const collection = client.db(db).collection(collectionName);
+
+    let filter = {};
+    let staleIds = [];
+
+    if (options.stale) {
+      // Find documents with stale source files
+      if (!quiet) {
+        p.log.step('Scanning for stale documents (source files that no longer exist)...');
+      }
+      const baseDir = project.root || process.cwd();
+      staleIds = await findStaleDocuments(collection, baseDir);
+      
+      if (staleIds.length === 0) {
+        if (options.json) {
+          console.log(JSON.stringify({ success: true, count: 0, message: 'No stale documents found' }));
+        } else {
+          p.log.success('No stale documents found.');
+          p.outro('Nothing to purge.');
+        }
+        return { success: true, count: 0 };
+      }
+      
+      filter = { _id: { $in: staleIds } };
+    } else {
+      // Build filter from criteria
+      filter = buildFilter(options);
+    }
+
+    // Count matching documents
+    const count = options.stale ? staleIds.length : await collection.countDocuments(filter);
+
+    if (count === 0) {
+      if (options.json) {
+        console.log(JSON.stringify({ success: true, count: 0, message: 'No matching documents found' }));
+      } else {
+        p.log.success('No matching documents found.');
+        p.outro('Nothing to purge.');
+      }
+      return { success: true, count: 0 };
+    }
+
+    // Get sample for display
+    const sampleDocs = await collection.find(filter).limit(5).toArray();
+
+    if (options.json) {
+      if (options.dryRun) {
+        console.log(JSON.stringify({
+          dryRun: true,
+          count,
+          sample: sampleDocs.map(d => ({
+            id: d._id?.toString(),
+            source: d.metadata?.source,
+            model: d._model,
+            embeddedAt: d._embeddedAt,
+          })),
+        }));
+        return { success: true, dryRun: true, count };
+      }
+    } else {
+      // Show what will be deleted
+      p.log.warn(`Found ${count} document${count === 1 ? '' : 's'} matching criteria:`);
+      console.log(formatSample(sampleDocs));
+      if (count > 5) {
+        console.log(`  ... and ${count - 5} more`);
+      }
+      console.log();
+    }
+
+    // Dry run - stop here
+    if (options.dryRun) {
+      if (!quiet) {
+        p.log.info('Dry run - no documents deleted.');
+        p.outro(`Would delete ${count} document${count === 1 ? '' : 's'}.`);
+      }
+      return { success: true, dryRun: true, count };
+    }
+
+    // Confirm unless --force
+    if (!options.force && !options.json) {
+      const confirmed = await p.confirm({
+        message: `Delete ${count} document${count === 1 ? '' : 's'}? This cannot be undone.`,
+        initialValue: false,
+      });
+
+      if (p.isCancel(confirmed) || !confirmed) {
+        p.log.info('Purge cancelled.');
+        p.outro('No documents deleted.');
+        return { success: false, cancelled: true };
+      }
+    }
+
+    // Delete documents
+    if (!quiet) {
+      p.log.step('Deleting documents...');
+    }
+    
+    const result = await collection.deleteMany(filter);
+    const deleted = result.deletedCount;
+
+    if (options.json) {
+      console.log(JSON.stringify({ success: true, deleted }));
+    } else {
+      p.log.success(`Deleted ${deleted} document${deleted === 1 ? '' : 's'}.`);
+      p.outro('Purge complete.');
+    }
+
+    return { success: true, deleted };
+
+  } catch (err) {
+    if (options.json) {
+      console.log(JSON.stringify({ error: err.message }));
+    } else {
+      p.log.error(`Purge failed: ${err.message}`);
+    }
+    return { success: false, error: err.message };
+  } finally {
+    if (client) {
+      await close();
+    }
+  }
+}
+
+/**
+ * Register the purge command with Commander.
+ */
+function register(program) {
+  program
+    .command('purge')
+    .description('Remove embeddings from MongoDB based on criteria')
+    .option('--db <database>', 'Database name')
+    .option('--collection <name>', 'Collection name')
+    .option('--source <glob>', 'Filter by metadata.source pattern')
+    .option('--before <date>', 'Filter by _embeddedAt before date (ISO 8601)')
+    .option('-m, --model <model>', 'Filter by _model field')
+    .option('--filter <json>', 'Raw MongoDB filter (JSON)')
+    .option('--stale', 'Remove docs whose source files no longer exist')
+    .option('--force', 'Skip confirmation prompt')
+    .option('--dry-run', 'Show what would be deleted without acting')
+    .option('--json', 'Machine-readable output')
+    .option('-q, --quiet', 'Suppress non-essential output')
+    .action(purge);
+}
+
+module.exports = { register, purge, buildFilter };

package/src/commands/quickstart.js

@@ -0,0 +1,203 @@
+'use strict';
+
+const pc = require('picocolors');
+const readline = require('readline');
+const { getConfigValue } = require('../lib/config');
+const { embed } = require('../lib/api');
+
+/**
+ * vai quickstart — Zero-to-search interactive tutorial
+ * Gets developers from nothing to their first semantic search in minutes.
+ */
+
+const SAMPLE_DOCS = [
+  "MongoDB Atlas Vector Search enables semantic search on your data using machine learning embeddings.",
+  "Voyage AI provides state-of-the-art embedding models with the best quality-to-cost ratio.",
+  "RAG (Retrieval-Augmented Generation) combines vector search with LLMs for accurate AI responses.",
+  "The shared embedding space in Voyage 4 models lets you embed queries and documents with different models.",
+  "Reranking improves search precision by re-scoring results with a cross-encoder model.",
+];
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function createPrompt() {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+  
+  return {
+    ask: (question) => new Promise((resolve) => {
+      rl.question(question, (answer) => resolve(answer));
+    }),
+    close: () => rl.close(),
+  };
+}
+
+async function runQuickstart(options = {}) {
+  const { skip } = options;
+  
+  console.log(pc.bold('\n🚀 Voyage AI CLI Quickstart\n'));
+  console.log(pc.dim('This tutorial will get you from zero to semantic search in 2 minutes.\n'));
+  
+  // Check for API key
+  const apiKey = process.env.VOYAGE_API_KEY || getConfigValue('apiKey');
+  if (!apiKey) {
+    console.log(pc.red('✗ No API key found.\n'));
+    console.log('  First, get a free API key:');
+    console.log(pc.cyan('  → https://dash.voyageai.com/api-keys\n'));
+    console.log('  Then configure it:');
+    console.log(pc.cyan('  → vai config set api-key YOUR_KEY\n'));
+    console.log('  Or set the environment variable:');
+    console.log(pc.cyan('  → export VOYAGE_API_KEY=YOUR_KEY\n'));
+    return 1;
+  }
+  
+  console.log(pc.green('✓') + ' API key configured\n');
+  
+  // Step 1: Explain what we're doing
+  console.log(pc.bold('Step 1: Understanding Embeddings'));
+  console.log(pc.dim('─'.repeat(40)));
+  console.log(`
+Embeddings turn text into ${pc.cyan('vectors')} (arrays of numbers) that capture
+meaning. Similar texts have similar vectors, enabling semantic search.
+
+We'll embed these ${SAMPLE_DOCS.length} sample documents:
+`);
+  
+  SAMPLE_DOCS.forEach((doc, i) => {
+    const preview = doc.length > 70 ? doc.slice(0, 70) + '...' : doc;
+    console.log(`  ${i + 1}. ${pc.dim(preview)}`);
+  });
+  console.log('');
+  
+  if (!skip) {
+    const prompt = createPrompt();
+    await prompt.ask(pc.dim('Press Enter to continue...'));
+    prompt.close();
+  }
+  
+  // Step 2: Embed the documents
+  console.log(pc.bold('\nStep 2: Embedding Documents'));
+  console.log(pc.dim('─'.repeat(40)));
+  console.log(`
+Running: ${pc.cyan('vai embed --model voyage-3-lite')}
+`);
+  
+  let embeddings;
+  try {
+    process.stdout.write('  Embedding documents... ');
+    const result = await embed({
+      texts: SAMPLE_DOCS,
+      model: 'voyage-3-lite',
+      inputType: 'document',
+    });
+    embeddings = result.embeddings;
+    console.log(pc.green('✓'));
+    console.log(`
+  ${pc.green('✓')} Created ${embeddings.length} embeddings
+  ${pc.dim(`  Dimensions: ${embeddings[0].length}`)}
+  ${pc.dim(`  Model: voyage-3-lite`)}
+`);
+  } catch (err) {
+    console.log(pc.red('✗'));
+    console.log(pc.red(`\n  Error: ${err.message}\n`));
+    console.log('  Check your API key with: vai doctor\n');
+    return 1;
+  }
+  
+  // Step 3: Search
+  console.log(pc.bold('Step 3: Semantic Search'));
+  console.log(pc.dim('─'.repeat(40)));
+  
+  const query = 'How do I improve search accuracy?';
+  console.log(`
+Now let's search! We'll embed a query and find the most similar documents.
+
+Query: "${pc.cyan(query)}"
+`);
+  
+  try {
+    process.stdout.write('  Embedding query... ');
+    const queryResult = await embed({
+      texts: [query],
+      model: 'voyage-3-lite',
+      inputType: 'query',
+    });
+    const queryEmbedding = queryResult.embeddings[0];
+    console.log(pc.green('✓'));
+    
+    // Calculate similarities
+    const similarities = embeddings.map((docEmb, i) => {
+      const dotProduct = docEmb.reduce((sum, val, j) => sum + val * queryEmbedding[j], 0);
+      const normA = Math.sqrt(docEmb.reduce((sum, val) => sum + val * val, 0));
+      const normB = Math.sqrt(queryEmbedding.reduce((sum, val) => sum + val * val, 0));
+      return {
+        index: i,
+        score: dotProduct / (normA * normB),
+        text: SAMPLE_DOCS[i],
+      };
+    });
+    
+    // Sort by similarity
+    similarities.sort((a, b) => b.score - a.score);
+    
+    console.log(`
+  ${pc.bold('Results (ranked by similarity):')}
+`);
+    
+    similarities.forEach((item, rank) => {
+      const scoreColor = item.score > 0.5 ? pc.green : item.score > 0.3 ? pc.yellow : pc.dim;
+      const preview = item.text.length > 60 ? item.text.slice(0, 60) + '...' : item.text;
+      console.log(`  ${rank + 1}. ${scoreColor(item.score.toFixed(3))} ${preview}`);
+    });
+    
+  } catch (err) {
+    console.log(pc.red('✗'));
+    console.log(pc.red(`\n  Error: ${err.message}\n`));
+    return 1;
+  }
+  
+  // Success!
+  console.log(pc.bold('\n✨ Congratulations!'));
+  console.log(pc.dim('─'.repeat(40)));
+  console.log(`
+You just performed your first semantic search with Voyage AI!
+
+The top result about ${pc.cyan('reranking')} is relevant because it discusses
+improving search ${pc.cyan('precision')} — even though it doesn't contain the
+exact words "improve" or "accuracy". That's the power of embeddings!
+
+${pc.bold('Why Voyage AI?')}
+  • ${pc.cyan('Best quality-to-cost ratio')} — SOTA quality at lower prices
+  • ${pc.cyan('Shared embedding space')} — mix models for cost optimization
+  • ${pc.cyan('Domain-specific models')} — code, finance, law, multilingual
+  • ${pc.cyan('Reranking')} — boost precision with two-stage retrieval
+
+${pc.bold('Next Steps:')}
+  ${pc.cyan('vai explain embeddings')}  — Learn more about how embeddings work
+  ${pc.cyan('vai explain reranking')}   — Understand two-stage retrieval
+  ${pc.cyan('vai demo')}                — Full interactive walkthrough
+  ${pc.cyan('vai pipeline')}            — Build a complete RAG pipeline
+  ${pc.cyan('vai playground')}          — Visual exploration in your browser
+
+${pc.dim('Docs: https://docs.voyageai.com | Dashboard: https://dash.voyageai.com')}
+`);
+  
+  return 0;
+}
+
+function register(program) {
+  program
+    .command('quickstart')
+    .description('Zero-to-search tutorial — learn semantic search in 2 minutes')
+    .option('--skip', 'Skip interactive prompts')
+    .action(async (options) => {
+      const exitCode = await runQuickstart(options);
+      process.exit(exitCode);
+    });
+}
+
+module.exports = { register, runQuickstart };