voyageai-cli 1.20.6 → 1.22.0

package/src/commands/playground.js

@@ -114,6 +114,99 @@ function createPlaygroundServer() {
         return;
       }
 
+      // API: Generate code
+      if (req.method === 'POST' && req.url === '/api/generate') {
+        let body = '';
+        req.on('data', chunk => { body += chunk; });
+        req.on('end', () => {
+          try {
+            const { target, component, config } = JSON.parse(body);
+            const codegen = require('../lib/codegen');
+            
+            const templateMap = {
+              vanilla: { client: 'client.js', connection: 'connection.js', retrieval: 'retrieval.js', ingest: 'ingest.js', 'search-api': 'search-api.js' },
+              nextjs: { client: 'lib-voyage.js', connection: 'lib-mongo.js', retrieval: 'route-search.js', ingest: 'route-ingest.js', 'search-page': 'page-search.jsx' },
+              python: { client: 'voyage_client.py', connection: 'mongo_client.py', retrieval: 'app.py', ingest: 'chunker.py' },
+            };
+            
+            const templateName = (templateMap[target] || {})[component];
+            if (!templateName) {
+              res.writeHead(400, { 'Content-Type': 'application/json' });
+              res.end(JSON.stringify({ error: `Unknown component: ${component}` }));
+              return;
+            }
+            
+            const context = codegen.buildContext(config || {}, { projectName: 'my-app' });
+            const code = codegen.renderTemplate(target, templateName.replace(/\.(js|jsx|py)$/, ''), context);
+            
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ code, filename: templateName }));
+          } catch (err) {
+            res.writeHead(500, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      // API: Scaffold project (returns ZIP for web mode)
+      if (req.method === 'POST' && req.url === '/api/scaffold') {
+        let body = '';
+        req.on('data', chunk => { body += chunk; });
+        req.on('end', () => {
+          try {
+            const { projectName, target, config } = JSON.parse(body);
+            const codegen = require('../lib/codegen');
+            const { PROJECT_STRUCTURE } = require('../lib/scaffold-structure');
+            const { createZip } = require('../lib/zip');
+            
+            const structure = PROJECT_STRUCTURE[target];
+            if (!structure) {
+              res.writeHead(400, { 'Content-Type': 'application/json' });
+              res.end(JSON.stringify({ error: `Unknown target: ${target}` }));
+              return;
+            }
+            
+            const context = codegen.buildContext(config || {}, { projectName: projectName || 'my-app' });
+            const files = [];
+            
+            // Render template files
+            for (const file of structure.files) {
+              const content = codegen.renderTemplate(target, file.template.replace(/\.(js|jsx|py|json|md|txt)$/, ''), context);
+              files.push({
+                name: `${projectName}/${file.output}`,
+                content,
+              });
+            }
+            
+            // Add extra static files
+            if (structure.extraFiles) {
+              for (const file of structure.extraFiles) {
+                const content = typeof file.content === 'function' ? file.content(context) : file.content;
+                files.push({
+                  name: `${projectName}/${file.output}`,
+                  content,
+                });
+              }
+            }
+            
+            // Create ZIP
+            const zipBuffer = createZip(files);
+            
+            res.writeHead(200, {
+              'Content-Type': 'application/zip',
+              'Content-Disposition': `attachment; filename="${projectName}.zip"`,
+              'Content-Length': zipBuffer.length,
+            });
+            res.end(zipBuffer);
+          } catch (err) {
+            res.writeHead(500, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
       // API: Concepts (from vai explain)
       if (req.method === 'GET' && req.url === '/api/concepts') {
         const { concepts } = require('../lib/explanations');

package/src/commands/purge.js

@@ -0,0 +1,271 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const p = require('@clack/prompts');
+const { loadProject } = require('../lib/project');
+const { connect, close } = require('../lib/mongo');
+const ui = require('../lib/ui');
+
+/**
+ * Build a MongoDB filter from the provided criteria.
+ */
+function buildFilter(options) {
+  const conditions = [];
+
+  // Filter by source pattern (glob-like)
+  if (options.source) {
+    // Convert glob pattern to regex
+    const pattern = options.source
+      .replace(/\./g, '\\.')
+      .replace(/\*/g, '.*')
+      .replace(/\?/g, '.');
+    conditions.push({ 'metadata.source': { $regex: pattern } });
+  }
+
+  // Filter by embedded date
+  if (options.before) {
+    const date = new Date(options.before);
+    if (isNaN(date.getTime())) {
+      throw new Error(`Invalid date format: ${options.before}`);
+    }
+    conditions.push({ _embeddedAt: { $lt: date } });
+  }
+
+  // Filter by model
+  if (options.model) {
+    conditions.push({ _model: options.model });
+  }
+
+  // Raw MongoDB filter
+  if (options.filter) {
+    try {
+      const rawFilter = JSON.parse(options.filter);
+      conditions.push(rawFilter);
+    } catch (err) {
+      throw new Error(`Invalid JSON filter: ${err.message}`);
+    }
+  }
+
+  // Combine conditions with $and
+  if (conditions.length === 0) {
+    return {};
+  } else if (conditions.length === 1) {
+    return conditions[0];
+  } else {
+    return { $and: conditions };
+  }
+}
+
+/**
+ * Check which documents have stale source files (file no longer exists on disk).
+ */
+async function findStaleDocuments(collection, baseDir) {
+  const docs = await collection.find({ 'metadata.source': { $exists: true } }).toArray();
+  const staleIds = [];
+
+  for (const doc of docs) {
+    const source = doc.metadata?.source;
+    if (source) {
+      // Resolve relative to baseDir or treat as absolute
+      const filePath = path.isAbsolute(source) ? source : path.join(baseDir, source);
+      if (!fs.existsSync(filePath)) {
+        staleIds.push(doc._id);
+      }
+    }
+  }
+
+  return staleIds;
+}
+
+/**
+ * Format a sample of documents for display.
+ */
+function formatSample(docs, limit = 5) {
+  const sample = docs.slice(0, limit);
+  return sample.map(doc => {
+    const source = doc.metadata?.source || doc._id?.toString() || 'unknown';
+    const model = doc._model || 'unknown';
+    const date = doc._embeddedAt ? new Date(doc._embeddedAt).toISOString().split('T')[0] : 'unknown';
+    return `  • ${source} (model: ${model}, date: ${date})`;
+  }).join('\n');
+}
+
+/**
+ * Execute the purge command.
+ */
+async function purge(options = {}) {
+  const quiet = options.quiet || options.json;
+
+  // Load project config
+  const project = loadProject();
+  const db = options.db || project.db || process.env.VAI_DB || 'vai';
+  const collectionName = options.collection || project.collection || process.env.VAI_COLLECTION || 'embeddings';
+
+  if (!quiet) {
+    p.intro(ui.title('vai purge'));
+  }
+
+  // Validate that at least one filter is provided
+  if (!options.source && !options.before && !options.model && !options.filter && !options.stale) {
+    if (options.json) {
+      console.log(JSON.stringify({ error: 'No filter criteria provided. Use --source, --before, --model, --filter, or --stale.' }));
+    } else {
+      p.log.error('No filter criteria provided.');
+      p.log.info('Use --source, --before, --model, --filter, or --stale to specify what to purge.');
+    }
+    return { success: false, error: 'No filter criteria' };
+  }
+
+  let client;
+  try {
+    // Connect to MongoDB
+    if (!quiet) {
+      p.log.step(`Connecting to database: ${db}`);
+    }
+    client = await connect(db);
+    const collection = client.db(db).collection(collectionName);
+
+    let filter = {};
+    let staleIds = [];
+
+    if (options.stale) {
+      // Find documents with stale source files
+      if (!quiet) {
+        p.log.step('Scanning for stale documents (source files that no longer exist)...');
+      }
+      const baseDir = project.root || process.cwd();
+      staleIds = await findStaleDocuments(collection, baseDir);
+      
+      if (staleIds.length === 0) {
+        if (options.json) {
+          console.log(JSON.stringify({ success: true, count: 0, message: 'No stale documents found' }));
+        } else {
+          p.log.success('No stale documents found.');
+          p.outro('Nothing to purge.');
+        }
+        return { success: true, count: 0 };
+      }
+      
+      filter = { _id: { $in: staleIds } };
+    } else {
+      // Build filter from criteria
+      filter = buildFilter(options);
+    }
+
+    // Count matching documents
+    const count = options.stale ? staleIds.length : await collection.countDocuments(filter);
+
+    if (count === 0) {
+      if (options.json) {
+        console.log(JSON.stringify({ success: true, count: 0, message: 'No matching documents found' }));
+      } else {
+        p.log.success('No matching documents found.');
+        p.outro('Nothing to purge.');
+      }
+      return { success: true, count: 0 };
+    }
+
+    // Get sample for display
+    const sampleDocs = await collection.find(filter).limit(5).toArray();
+
+    if (options.json) {
+      if (options.dryRun) {
+        console.log(JSON.stringify({
+          dryRun: true,
+          count,
+          sample: sampleDocs.map(d => ({
+            id: d._id?.toString(),
+            source: d.metadata?.source,
+            model: d._model,
+            embeddedAt: d._embeddedAt,
+          })),
+        }));
+        return { success: true, dryRun: true, count };
+      }
+    } else {
+      // Show what will be deleted
+      p.log.warn(`Found ${count} document${count === 1 ? '' : 's'} matching criteria:`);
+      console.log(formatSample(sampleDocs));
+      if (count > 5) {
+        console.log(`  ... and ${count - 5} more`);
+      }
+      console.log();
+    }
+
+    // Dry run - stop here
+    if (options.dryRun) {
+      if (!quiet) {
+        p.log.info('Dry run - no documents deleted.');
+        p.outro(`Would delete ${count} document${count === 1 ? '' : 's'}.`);
+      }
+      return { success: true, dryRun: true, count };
+    }
+
+    // Confirm unless --force
+    if (!options.force && !options.json) {
+      const confirmed = await p.confirm({
+        message: `Delete ${count} document${count === 1 ? '' : 's'}? This cannot be undone.`,
+        initialValue: false,
+      });
+
+      if (p.isCancel(confirmed) || !confirmed) {
+        p.log.info('Purge cancelled.');
+        p.outro('No documents deleted.');
+        return { success: false, cancelled: true };
+      }
+    }
+
+    // Delete documents
+    if (!quiet) {
+      p.log.step('Deleting documents...');
+    }
+    
+    const result = await collection.deleteMany(filter);
+    const deleted = result.deletedCount;
+
+    if (options.json) {
+      console.log(JSON.stringify({ success: true, deleted }));
+    } else {
+      p.log.success(`Deleted ${deleted} document${deleted === 1 ? '' : 's'}.`);
+      p.outro('Purge complete.');
+    }
+
+    return { success: true, deleted };
+
+  } catch (err) {
+    if (options.json) {
+      console.log(JSON.stringify({ error: err.message }));
+    } else {
+      p.log.error(`Purge failed: ${err.message}`);
+    }
+    return { success: false, error: err.message };
+  } finally {
+    if (client) {
+      await close();
+    }
+  }
+}
+
+/**
+ * Register the purge command with Commander.
+ */
+function register(program) {
+  program
+    .command('purge')
+    .description('Remove embeddings from MongoDB based on criteria')
+    .option('--db <database>', 'Database name')
+    .option('--collection <name>', 'Collection name')
+    .option('--source <glob>', 'Filter by metadata.source pattern')
+    .option('--before <date>', 'Filter by _embeddedAt before date (ISO 8601)')
+    .option('-m, --model <model>', 'Filter by _model field')
+    .option('--filter <json>', 'Raw MongoDB filter (JSON)')
+    .option('--stale', 'Remove docs whose source files no longer exist')
+    .option('--force', 'Skip confirmation prompt')
+    .option('--dry-run', 'Show what would be deleted without acting')
+    .option('--json', 'Machine-readable output')
+    .option('-q, --quiet', 'Suppress non-essential output')
+    .action(purge);
+}
+
+module.exports = { register, purge, buildFilter };

package/src/commands/refresh.js

@@ -0,0 +1,322 @@
+'use strict';
+
+const p = require('@clack/prompts');
+const { loadProject, saveProject } = require('../lib/project');
+const { connect, close } = require('../lib/mongo');
+const { generateEmbeddings } = require('../lib/api');
+const { chunkText } = require('../lib/chunker');
+const ui = require('../lib/ui');
+
+/**
+ * Process documents in batches.
+ */
+async function processBatch(docs, embedder, options) {
+  const texts = docs.map(d => d.text);
+  const embeddings = await embedder(texts);
+  
+  return docs.map((doc, i) => ({
+    ...doc,
+    [options.field]: embeddings[i],
+    _model: options.model,
+    _embeddedAt: new Date(),
+  }));
+}
+
+/**
+ * Re-chunk a document's text.
+ */
+function rechunkDocument(doc, options) {
+  const text = doc.text || doc.content || '';
+  if (!text) return [doc];
+
+  const chunks = chunkText(text, {
+    strategy: options.strategy || 'recursive',
+    chunkSize: options.chunkSize || 512,
+    overlap: options.overlap || 50,
+  });
+
+  return chunks.map((chunk, i) => ({
+    ...doc,
+    text: chunk.text,
+    _chunkIndex: i,
+    _chunkCount: chunks.length,
+    metadata: {
+      ...doc.metadata,
+      chunkIndex: i,
+      chunkCount: chunks.length,
+      originalId: doc._id?.toString(),
+    },
+  }));
+}
+
+/**
+ * Execute the refresh command.
+ */
+async function refresh(options = {}) {
+  const quiet = options.quiet || options.json;
+
+  // Load project config
+  const project = loadProject();
+  const db = options.db || project.db || process.env.VAI_DB || 'vai';
+  const collectionName = options.collection || project.collection || process.env.VAI_COLLECTION || 'embeddings';
+  const field = options.field || project.field || 'embedding';
+  const model = options.model || project.model || 'voyage-3.5-lite';
+  const dimensions = options.dimensions || project.dimensions;
+  const batchSize = options.batchSize || 25;
+
+  if (!quiet) {
+    p.intro(ui.title('vai refresh'));
+  }
+
+  let client;
+  try {
+    // Connect to MongoDB
+    if (!quiet) {
+      p.log.step(`Connecting to database: ${db}`);
+    }
+    client = await connect(db);
+    const collection = client.db(db).collection(collectionName);
+
+    // Build filter
+    let filter = {};
+    if (options.filter) {
+      try {
+        filter = JSON.parse(options.filter);
+      } catch (err) {
+        throw new Error(`Invalid JSON filter: ${err.message}`);
+      }
+    }
+
+    // Count documents
+    const totalCount = await collection.countDocuments(filter);
+
+    if (totalCount === 0) {
+      if (options.json) {
+        console.log(JSON.stringify({ success: true, count: 0, message: 'No documents to refresh' }));
+      } else {
+        p.log.success('No documents to refresh.');
+        p.outro('Nothing to do.');
+      }
+      return { success: true, count: 0 };
+    }
+
+    // Show plan
+    const rechunkLabel = options.rechunk ? ` (re-chunking with ${options.strategy || 'recursive'})` : '';
+    const dimLabel = dimensions ? ` @ ${dimensions}d` : '';
+    
+    if (options.json && options.dryRun) {
+      console.log(JSON.stringify({
+        dryRun: true,
+        count: totalCount,
+        model,
+        dimensions: dimensions || 'default',
+        rechunk: !!options.rechunk,
+      }));
+      return { success: true, dryRun: true, count: totalCount };
+    }
+
+    if (!quiet) {
+      p.log.info(`Found ${totalCount} document${totalCount === 1 ? '' : 's'} to refresh`);
+      p.log.info(`Target model: ${model}${dimLabel}${rechunkLabel}`);
+    }
+
+    // Dry run - stop here
+    if (options.dryRun) {
+      if (!quiet) {
+        p.log.info('Dry run - no documents modified.');
+        p.outro(`Would refresh ${totalCount} document${totalCount === 1 ? '' : 's'}.`);
+      }
+      return { success: true, dryRun: true, count: totalCount };
+    }
+
+    // Confirm unless --force
+    if (!options.force && !options.json) {
+      const confirmed = await p.confirm({
+        message: `Re-embed ${totalCount} document${totalCount === 1 ? '' : 's'}? This will update the embeddings in-place.`,
+        initialValue: true,
+      });
+
+      if (p.isCancel(confirmed) || !confirmed) {
+        p.log.info('Refresh cancelled.');
+        p.outro('No documents modified.');
+        return { success: false, cancelled: true };
+      }
+    }
+
+    // Create embedder function
+    const embedder = async (texts) => {
+      const result = await generateEmbeddings(texts, {
+        model,
+        dimensions,
+        inputType: 'document',
+      });
+      return result.embeddings;
+    };
+
+    // Process documents
+    let processed = 0;
+    let errors = 0;
+    const cursor = collection.find(filter);
+    let batch = [];
+
+    const spinner = !quiet ? p.spinner() : null;
+    if (spinner) spinner.start('Processing documents...');
+
+    while (await cursor.hasNext()) {
+      const doc = await cursor.next();
+      
+      if (options.rechunk) {
+        // Re-chunk the document
+        const chunks = rechunkDocument(doc, options);
+        batch.push(...chunks);
+      } else {
+        batch.push(doc);
+      }
+
+      // Process when batch is full
+      if (batch.length >= batchSize) {
+        try {
+          const updated = await processBatch(batch, embedder, { field, model });
+          
+          // Replace documents in database
+          for (const updatedDoc of updated) {
+            if (options.rechunk && updatedDoc.metadata?.originalId) {
+              // For rechunked docs, insert new and delete original later
+              await collection.insertOne(updatedDoc);
+            } else {
+              // Update in place
+              await collection.updateOne(
+                { _id: updatedDoc._id },
+                { $set: { [field]: updatedDoc[field], _model: model, _embeddedAt: new Date() } }
+              );
+            }
+          }
+          
+          processed += batch.length;
+          if (spinner) spinner.message(`Processed ${processed}/${totalCount} documents...`);
+        } catch (err) {
+          errors += batch.length;
+          if (!quiet) {
+            p.log.warn(`Batch error: ${err.message}`);
+          }
+        }
+        batch = [];
+      }
+    }
+
+    // Process remaining batch
+    if (batch.length > 0) {
+      try {
+        const updated = await processBatch(batch, embedder, { field, model });
+        
+        for (const updatedDoc of updated) {
+          if (options.rechunk && updatedDoc.metadata?.originalId) {
+            await collection.insertOne(updatedDoc);
+          } else {
+            await collection.updateOne(
+              { _id: updatedDoc._id },
+              { $set: { [field]: updatedDoc[field], _model: model, _embeddedAt: new Date() } }
+            );
+          }
+        }
+        
+        processed += batch.length;
+      } catch (err) {
+        errors += batch.length;
+        if (!quiet) {
+          p.log.warn(`Batch error: ${err.message}`);
+        }
+      }
+    }
+
+    // If rechunking, delete original documents
+    if (options.rechunk) {
+      const originalIds = await collection.distinct('metadata.originalId', filter);
+      if (originalIds.length > 0) {
+        // Convert string IDs back to ObjectIds for deletion
+        const { ObjectId } = require('mongodb');
+        const objectIds = originalIds
+          .filter(id => id)
+          .map(id => {
+            try { return new ObjectId(id); } catch { return null; }
+          })
+          .filter(id => id);
+        
+        if (objectIds.length > 0) {
+          await collection.deleteMany({ _id: { $in: objectIds } });
+        }
+      }
+    }
+
+    if (spinner) spinner.stop('Processing complete.');
+
+    // Update project config if model/dimensions changed
+    const configUpdated = (model !== project.model) || (dimensions && dimensions !== project.dimensions);
+    if (configUpdated && !options.json) {
+      try {
+        saveProject({
+          ...project,
+          model,
+          ...(dimensions && { dimensions }),
+        });
+        if (!quiet) {
+          p.log.info('Updated .vai.json with new model/dimensions.');
+        }
+      } catch {
+        // Ignore save errors
+      }
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify({ success: true, processed, errors }));
+    } else {
+      if (errors > 0) {
+        p.log.warn(`Refreshed ${processed} documents with ${errors} errors.`);
+      } else {
+        p.log.success(`Refreshed ${processed} document${processed === 1 ? '' : 's'}.`);
+      }
+      p.outro('Refresh complete.');
+    }
+
+    return { success: true, processed, errors };
+
+  } catch (err) {
+    if (options.json) {
+      console.log(JSON.stringify({ error: err.message }));
+    } else {
+      p.log.error(`Refresh failed: ${err.message}`);
+    }
+    return { success: false, error: err.message };
+  } finally {
+    if (client) {
+      await close();
+    }
+  }
+}
+
+/**
+ * Register the refresh command with Commander.
+ */
+function register(program) {
+  program
+    .command('refresh')
+    .description('Re-embed documents with a new model, dimensions, or chunk settings')
+    .option('--db <database>', 'Database name')
+    .option('--collection <name>', 'Collection name')
+    .option('--field <name>', 'Embedding field name')
+    .option('-m, --model <model>', 'New embedding model')
+    .option('-d, --dimensions <n>', 'New dimensions', parseInt)
+    .option('--rechunk', 'Re-chunk text before re-embedding')
+    .option('-s, --strategy <strategy>', 'Chunk strategy (with --rechunk)')
+    .option('-c, --chunk-size <n>', 'Chunk size (with --rechunk)', parseInt)
+    .option('--overlap <n>', 'Chunk overlap (with --rechunk)', parseInt)
+    .option('--batch-size <n>', 'Texts per API call (default: 25)', parseInt)
+    .option('--filter <json>', 'Only refresh matching documents (JSON)')
+    .option('--force', 'Skip confirmation prompt')
+    .option('--dry-run', 'Show plan without executing')
+    .option('--json', 'Machine-readable output')
+    .option('-q, --quiet', 'Suppress non-essential output')
+    .action(refresh);
+}
+
+module.exports = { register, refresh };