voyageai-cli 1.20.6 → 1.22.0

package/src/lib/templates/vanilla/retrieval.js.tpl

@@ -0,0 +1,100 @@
+/**
+ * RAG Retrieval Module
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ * 
+ * Model: {{model}}
+ * Database: {{db}}.{{collection}}
+ * Reranking: {{#if rerank}}enabled ({{rerankModel}}){{else}}disabled{{/if}}
+ */
+
+const { embed{{#if rerank}}, rerank{{/if}} } = require('./client');
+const { vectorSearch } = require('./connection');
+
+/**
+ * Retrieve relevant documents for a query.
+ * 
+ * Pipeline:
+ * 1. Embed the query using Voyage AI
+ * 2. Vector search in MongoDB Atlas
+{{#if rerank}}
+ * 3. Rerank results using Voyage AI
+{{/if}}
+ * 
+ * @param {string} query - User's query
+ * @param {object} options - Retrieval options
+ * @param {number} options.limit - Final number of results (default: 5)
+ * @param {number} options.candidates - Initial candidates for reranking (default: 20)
+ * @param {object} options.filter - MongoDB pre-filter
+ * @returns {Promise<Array<{text: string, score: number, metadata: object}>>}
+ */
+async function retrieve(query, options = {}) {
+  const limit = options.limit || 5;
+{{#if rerank}}
+  const candidates = options.candidates || 20;
+{{/if}}
+
+  // Step 1: Embed the query
+  const { embeddings } = await embed(query, { inputType: 'query' });
+  const queryEmbedding = embeddings[0];
+
+  // Step 2: Vector search
+{{#if rerank}}
+  const searchResults = await vectorSearch(queryEmbedding, {
+    limit: candidates,
+    filter: options.filter,
+  });
+{{else}}
+  const searchResults = await vectorSearch(queryEmbedding, {
+    limit,
+    filter: options.filter,
+  });
+{{/if}}
+
+  if (searchResults.length === 0) {
+    return [];
+  }
+
+{{#if rerank}}
+  // Step 3: Rerank for better relevance
+  const documents = searchResults.map(r => r.document.text);
+  const { results: reranked } = await rerank(query, documents, { topK: limit });
+
+  // Map reranked results back to full documents
+  return reranked.map(r => ({
+    text: searchResults[r.index].document.text,
+    score: r.relevanceScore,
+    metadata: searchResults[r.index].document.metadata,
+  }));
+{{else}}
+  return searchResults.map(r => ({
+    text: r.document.text,
+    score: r.score,
+    metadata: r.document.metadata,
+  }));
+{{/if}}
+}
+
+/**
+ * Format retrieved documents as context for an LLM prompt.
+ * @param {Array<{text: string, score: number}>} results - Retrieved documents
+ * @param {object} options - Formatting options
+ * @param {boolean} options.includeScores - Include relevance scores (default: false)
+ * @param {string} options.separator - Separator between documents (default: '\n\n---\n\n')
+ * @returns {string} Formatted context string
+ */
+function formatContext(results, options = {}) {
+  const separator = options.separator || '\n\n---\n\n';
+  
+  return results.map((r, i) => {
+    let text = r.text;
+    if (options.includeScores) {
+      text = `[Score: ${r.score.toFixed(3)}]\n${text}`;
+    }
+    return text;
+  }).join(separator);
+}
+
+module.exports = {
+  retrieve,
+  formatContext,
+};

package/src/lib/templates/vanilla/search-api.js.tpl

@@ -0,0 +1,175 @@
+/**
+ * Semantic Search API
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ * 
+ * Express router with endpoints:
+ * - POST /api/search - Semantic search
+ * - POST /api/ingest - Document ingestion
+ * - GET /api/health - Health check
+ */
+
+const express = require('express');
+const { retrieve, formatContext } = require('./retrieval');
+const { ingestFile, ingestDirectory } = require('./ingest');
+const { connect, close } = require('./connection');
+
+const router = express.Router();
+
+/**
+ * POST /api/search
+ * 
+ * Request body:
+ * {
+ *   "query": "What is vector search?",
+ *   "limit": 5,
+ *   "includeContext": true
+ * }
+ * 
+ * Response:
+ * {
+ *   "results": [...],
+ *   "context": "...",
+ *   "meta": { "model": "...", "took": 123 }
+ * }
+ */
+router.post('/search', async (req, res) => {
+  const start = Date.now();
+  
+  try {
+    const { query, limit = 5, includeContext = false, filter } = req.body;
+    
+    if (!query || typeof query !== 'string') {
+      return res.status(400).json({ error: 'Query is required' });
+    }
+    
+    const results = await retrieve(query, { limit, filter });
+    
+    const response = {
+      results: results.map(r => ({
+        text: r.text,
+        score: r.score,
+        metadata: r.metadata,
+      })),
+      meta: {
+        model: '{{model}}',
+{{#if rerank}}
+        rerankModel: '{{rerankModel}}',
+{{/if}}
+        took: Date.now() - start,
+      },
+    };
+    
+    if (includeContext) {
+      response.context = formatContext(results);
+    }
+    
+    res.json(response);
+  } catch (err) {
+    console.error('Search error:', err);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+/**
+ * POST /api/ingest
+ * 
+ * Request body (for text):
+ * {
+ *   "text": "Document content...",
+ *   "metadata": { "source": "api", "title": "..." }
+ * }
+ * 
+ * Request body (for file path):
+ * {
+ *   "path": "./docs/guide.md"
+ * }
+ */
+router.post('/ingest', async (req, res) => {
+  const start = Date.now();
+  
+  try {
+    const { text, metadata, path: filePath } = req.body;
+    
+    if (filePath) {
+      const result = await ingestFile(filePath);
+      return res.json({
+        success: true,
+        ...result,
+        took: Date.now() - start,
+      });
+    }
+    
+    if (!text || typeof text !== 'string') {
+      return res.status(400).json({ error: 'Text or path is required' });
+    }
+    
+    // For API-provided text, we need to chunk, embed, and store inline
+    const { embed } = require('./client');
+    const { insertDocuments } = require('./connection');
+    const { chunkText } = require('./ingest');
+    
+    const chunks = chunkText(text);
+    const { embeddings, usage } = await embed(chunks, { inputType: 'document' });
+    
+    const documents = chunks.map((chunk, i) => ({
+      text: chunk,
+      embedding: embeddings[i],
+      metadata: {
+        ...metadata,
+        source: 'api',
+        chunkIndex: i,
+        totalChunks: chunks.length,
+      },
+    }));
+    
+    await insertDocuments(documents);
+    
+    res.json({
+      success: true,
+      chunks: chunks.length,
+      tokens: usage.total_tokens,
+      took: Date.now() - start,
+    });
+  } catch (err) {
+    console.error('Ingest error:', err);
+    res.status(500).json({ error: err.message });
+  }
+});
+
+/**
+ * GET /api/health
+ */
+router.get('/health', async (req, res) => {
+  try {
+    await connect();
+    res.json({
+      status: 'healthy',
+      model: '{{model}}',
+      database: '{{db}}',
+      collection: '{{collection}}',
+    });
+  } catch (err) {
+    res.status(503).json({
+      status: 'unhealthy',
+      error: err.message,
+    });
+  }
+});
+
+module.exports = router;
+
+// Standalone server if run directly
+if (require.main === module) {
+  const app = express();
+  app.use(express.json());
+  app.use('/api', router);
+  
+  const PORT = process.env.PORT || 3000;
+  
+  app.listen(PORT, () => {
+    console.log(`Search API running on http://localhost:${PORT}`);
+    console.log(`  POST /api/search  - Semantic search`);
+    console.log(`  POST /api/ingest  - Document ingestion`);
+    console.log(`  GET  /api/health  - Health check`);
+  });
+}

package/src/lib/templates/vanilla/server.js.tpl

@@ -0,0 +1,81 @@
+/**
+ * Voyage AI RAG Server
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ * 
+ * Endpoints:
+ * - POST /api/search  - Semantic search with optional reranking
+ * - POST /api/ingest  - Document ingestion
+ * - GET  /api/health  - Health check
+ */
+
+require('dotenv').config();
+
+const express = require('express');
+const searchRouter = require('./lib/search-api');
+
+const app = express();
+const PORT = process.env.PORT || 3000;
+
+// Middleware
+app.use(express.json({ limit: '10mb' }));
+
+// CORS for development
+app.use((req, res, next) => {
+  res.header('Access-Control-Allow-Origin', '*');
+  res.header('Access-Control-Allow-Headers', 'Content-Type');
+  res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+  if (req.method === 'OPTIONS') return res.sendStatus(200);
+  next();
+});
+
+// API routes
+app.use('/api', searchRouter);
+
+// Root endpoint
+app.get('/', (req, res) => {
+  res.json({
+    name: '{{projectName}}',
+    description: 'Voyage AI RAG API',
+    model: '{{model}}',
+    database: '{{db}}',
+    collection: '{{collection}}',
+    endpoints: {
+      search: 'POST /api/search',
+      ingest: 'POST /api/ingest',
+      health: 'GET /api/health',
+    },
+  });
+});
+
+// Error handler
+app.use((err, req, res, next) => {
+  console.error('Server error:', err);
+  res.status(500).json({ error: 'Internal server error' });
+});
+
+app.listen(PORT, () => {
+  console.log(`
+🚀 Voyage AI RAG Server
+
+   Model:      {{model}}
+   Database:   {{db}}.{{collection}}
+{{#if rerank}}
+   Reranking:  {{rerankModel}}
+{{/if}}
+
+   Endpoints:
+   POST /api/search  - Semantic search
+   POST /api/ingest  - Document ingestion
+   GET  /api/health  - Health check
+
+   Running on http://localhost:${PORT}
+`);
+});
+
+// Graceful shutdown
+process.on('SIGTERM', async () => {
+  console.log('Shutting down...');
+  const { close } = require('./lib/connection');
+  await close();
+  process.exit(0);
+});

package/src/lib/zip.js

@@ -0,0 +1,130 @@
+'use strict';
+
+/**
+ * Minimal ZIP file creator for text files.
+ * Creates uncompressed (STORE) ZIP archives - perfect for scaffolded code.
+ * No external dependencies.
+ */
+
+/**
+ * Create a ZIP file from an array of file entries.
+ * @param {Array<{name: string, content: string}>} files - Files to include
+ * @returns {Buffer} - ZIP file as a Buffer
+ */
+function createZip(files) {
+  const entries = [];
+  let offset = 0;
+
+  // Process each file
+  for (const file of files) {
+    const content = Buffer.from(file.content, 'utf8');
+    const name = Buffer.from(file.name, 'utf8');
+    const now = new Date();
+    const dosTime = ((now.getHours() << 11) | (now.getMinutes() << 5) | (now.getSeconds() >> 1)) & 0xFFFF;
+    const dosDate = (((now.getFullYear() - 1980) << 9) | ((now.getMonth() + 1) << 5) | now.getDate()) & 0xFFFF;
+    const crc = crc32(content);
+
+    // Local file header (30 bytes + filename)
+    const localHeader = Buffer.alloc(30 + name.length);
+    localHeader.writeUInt32LE(0x04034b50, 0);      // Local file header signature
+    localHeader.writeUInt16LE(20, 4);              // Version needed (2.0)
+    localHeader.writeUInt16LE(0, 6);               // General purpose bit flag
+    localHeader.writeUInt16LE(0, 8);               // Compression method (0 = store)
+    localHeader.writeUInt16LE(dosTime, 10);        // Last mod time
+    localHeader.writeUInt16LE(dosDate, 12);        // Last mod date
+    localHeader.writeUInt32LE(crc, 14);            // CRC-32
+    localHeader.writeUInt32LE(content.length, 18); // Compressed size
+    localHeader.writeUInt32LE(content.length, 22); // Uncompressed size
+    localHeader.writeUInt16LE(name.length, 26);    // Filename length
+    localHeader.writeUInt16LE(0, 28);              // Extra field length
+    name.copy(localHeader, 30);
+
+    entries.push({
+      name,
+      content,
+      crc,
+      dosTime,
+      dosDate,
+      headerOffset: offset,
+      localHeader,
+    });
+
+    offset += localHeader.length + content.length;
+  }
+
+  // Build central directory
+  const centralDir = [];
+  for (const entry of entries) {
+    const cdHeader = Buffer.alloc(46 + entry.name.length);
+    cdHeader.writeUInt32LE(0x02014b50, 0);                   // Central directory signature
+    cdHeader.writeUInt16LE(20, 4);                           // Version made by
+    cdHeader.writeUInt16LE(20, 6);                           // Version needed
+    cdHeader.writeUInt16LE(0, 8);                            // General purpose bit flag
+    cdHeader.writeUInt16LE(0, 10);                           // Compression method
+    cdHeader.writeUInt16LE(entry.dosTime, 12);               // Last mod time
+    cdHeader.writeUInt16LE(entry.dosDate, 14);               // Last mod date
+    cdHeader.writeUInt32LE(entry.crc, 16);                   // CRC-32
+    cdHeader.writeUInt32LE(entry.content.length, 20);        // Compressed size
+    cdHeader.writeUInt32LE(entry.content.length, 24);        // Uncompressed size
+    cdHeader.writeUInt16LE(entry.name.length, 28);           // Filename length
+    cdHeader.writeUInt16LE(0, 30);                           // Extra field length
+    cdHeader.writeUInt16LE(0, 32);                           // Comment length
+    cdHeader.writeUInt16LE(0, 34);                           // Disk number start
+    cdHeader.writeUInt16LE(0, 36);                           // Internal file attributes
+    cdHeader.writeUInt32LE(0, 38);                           // External file attributes
+    cdHeader.writeUInt32LE(entry.headerOffset, 42);          // Relative offset of local header
+    entry.name.copy(cdHeader, 46);
+    centralDir.push(cdHeader);
+  }
+
+  const centralDirBuffer = Buffer.concat(centralDir);
+  const centralDirOffset = offset;
+  const centralDirSize = centralDirBuffer.length;
+
+  // End of central directory record (22 bytes)
+  const eocd = Buffer.alloc(22);
+  eocd.writeUInt32LE(0x06054b50, 0);               // EOCD signature
+  eocd.writeUInt16LE(0, 4);                        // Disk number
+  eocd.writeUInt16LE(0, 6);                        // Disk with central directory
+  eocd.writeUInt16LE(entries.length, 8);           // Entries on this disk
+  eocd.writeUInt16LE(entries.length, 10);          // Total entries
+  eocd.writeUInt32LE(centralDirSize, 12);          // Central directory size
+  eocd.writeUInt32LE(centralDirOffset, 16);        // Central directory offset
+  eocd.writeUInt16LE(0, 20);                       // Comment length
+
+  // Combine all parts
+  const parts = [];
+  for (const entry of entries) {
+    parts.push(entry.localHeader);
+    parts.push(entry.content);
+  }
+  parts.push(centralDirBuffer);
+  parts.push(eocd);
+
+  return Buffer.concat(parts);
+}
+
+/**
+ * CRC-32 calculation (IEEE polynomial).
+ */
+const crcTable = (() => {
+  const table = new Uint32Array(256);
+  for (let i = 0; i < 256; i++) {
+    let c = i;
+    for (let j = 0; j < 8; j++) {
+      c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+    }
+    table[i] = c;
+  }
+  return table;
+})();
+
+function crc32(buffer) {
+  let crc = 0xFFFFFFFF;
+  for (let i = 0; i < buffer.length; i++) {
+    crc = crcTable[(crc ^ buffer[i]) & 0xFF] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+module.exports = { createZip };