voyageai-cli 1.20.6 → 1.22.0

package/src/lib/templates/vanilla/README.md.tpl

@@ -0,0 +1,156 @@
+# {{projectName}}
+
+A semantic search API powered by Voyage AI embeddings and MongoDB Atlas Vector Search.
+
+## Configuration
+
+| Setting | Value |
+|---------|-------|
+| Embedding Model | `{{model}}` |
+| Dimensions | {{dimensions}} |
+| Database | `{{db}}` |
+| Collection | `{{collection}}` |
+| Vector Index | `{{index}}` |
+{{#if rerank}}
+| Rerank Model | `{{rerankModel}}` |
+{{/if}}
+
+## Setup
+
+### 1. Install dependencies
+
+```bash
+npm install
+```
+
+### 2. Configure environment
+
+Copy `.env.example` to `.env` and fill in your credentials:
+
+```bash
+cp .env.example .env
+```
+
+Required variables:
+- `VOYAGE_API_KEY` - Your Voyage AI API key from [dash.voyageai.com](https://dash.voyageai.com)
+- `MONGODB_URI` - Your MongoDB Atlas connection string
+
+### 3. Create vector index
+
+In MongoDB Atlas, create a vector search index on your collection:
+
+```json
+{
+  "fields": [
+    {
+      "type": "vector",
+      "path": "{{field}}",
+      "numDimensions": {{dimensions}},
+      "similarity": "cosine"
+    }
+  ]
+}
+```
+
+Name the index `{{index}}`.
+
+### 4. Ingest documents
+
+```bash
+# Ingest a directory of markdown/text files
+npm run ingest -- ./docs
+
+# Or ingest a single file
+npm run ingest -- ./docs/guide.md
+```
+
+### 5. Start the server
+
+```bash
+npm start
+# or for development with auto-reload
+npm run dev
+```
+
+## API Endpoints
+
+### POST /api/search
+
+Search for relevant documents.
+
+```bash
+curl -X POST http://localhost:3000/api/search \
+  -H "Content-Type: application/json" \
+  -d '{"query": "How does vector search work?", "limit": 5}'
+```
+
+Response:
+```json
+{
+  "results": [
+    {
+      "text": "Vector search uses...",
+      "score": 0.95,
+      "metadata": { "source": "docs/guide.md" }
+    }
+  ],
+  "meta": {
+    "model": "{{model}}",
+    "took": 123
+  }
+}
+```
+
+### POST /api/ingest
+
+Ingest text or a file.
+
+```bash
+# Ingest text
+curl -X POST http://localhost:3000/api/ingest \
+  -H "Content-Type: application/json" \
+  -d '{"text": "Your document content...", "metadata": {"source": "api"}}'
+
+# Ingest a file
+curl -X POST http://localhost:3000/api/ingest \
+  -H "Content-Type: application/json" \
+  -d '{"path": "./docs/new-doc.md"}'
+```
+
+### GET /api/health
+
+Check API health and database connection.
+
+```bash
+curl http://localhost:3000/api/health
+```
+
+## Project Structure
+
+```
+{{projectName}}/
+├── server.js           # Express server entry point
+├── lib/
+│   ├── client.js       # Voyage AI API client
+│   ├── connection.js   # MongoDB connection helper
+│   ├── retrieval.js    # RAG retrieval module
+│   ├── ingest.js       # Document ingestion pipeline
+│   └── search-api.js   # Express API routes
+├── .env.example
+├── package.json
+└── README.md
+```
+
+## Chunking Configuration
+
+Documents are chunked with the following settings:
+
+| Setting | Value |
+|---------|-------|
+| Strategy | `{{chunkStrategy}}` |
+| Chunk Size | {{chunkSize}} characters |
+| Overlap | {{chunkOverlap}} characters |
+
+---
+
+Generated by [vai](https://github.com/mrlynn/voyageai-cli) v{{vaiVersion}}

package/src/lib/templates/vanilla/client.js.tpl

@@ -0,0 +1,103 @@
+/**
+ * Voyage AI API Client
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ * 
+ * Model: {{model}}
+ * Dimensions: {{dimensions}}
+ */
+
+const VOYAGE_API_URL = process.env.VOYAGE_API_URL || 'https://api.voyageai.com/v1';
+const VOYAGE_API_KEY = process.env.VOYAGE_API_KEY;
+
+if (!VOYAGE_API_KEY) {
+  throw new Error('VOYAGE_API_KEY environment variable is required');
+}
+
+/**
+ * Generate embeddings for text(s) using Voyage AI.
+ * @param {string|string[]} input - Text or array of texts to embed
+ * @param {object} options - Optional parameters
+ * @param {string} options.model - Embedding model (default: {{model}})
+ * @param {string} options.inputType - 'document' or 'query' (default: {{inputType}})
+ * @param {number} options.outputDimension - Output dimensions (default: {{dimensions}})
+ * @returns {Promise<{embeddings: number[][], usage: {total_tokens: number}}>}
+ */
+async function embed(input, options = {}) {
+  const texts = Array.isArray(input) ? input : [input];
+  
+  const response = await fetch(`${VOYAGE_API_URL}/embeddings`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${VOYAGE_API_KEY}`,
+    },
+    body: JSON.stringify({
+      model: options.model || '{{model}}',
+      input: texts,
+      input_type: options.inputType || '{{inputType}}',
+      output_dimension: options.outputDimension || {{dimensions}},
+    }),
+  });
+
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`Voyage AI API error: ${response.status} ${error}`);
+  }
+
+  const data = await response.json();
+  return {
+    embeddings: data.data.map(d => d.embedding),
+    usage: data.usage,
+  };
+}
+
+{{#if rerank}}
+/**
+ * Rerank documents by relevance to a query.
+ * @param {string} query - The query to rank against
+ * @param {string[]} documents - Documents to rerank
+ * @param {object} options - Optional parameters
+ * @param {string} options.model - Rerank model (default: {{rerankModel}})
+ * @param {number} options.topK - Number of results to return
+ * @returns {Promise<{results: Array<{index: number, relevanceScore: number, document: string}>}>}
+ */
+async function rerank(query, documents, options = {}) {
+  const response = await fetch(`${VOYAGE_API_URL}/rerank`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${VOYAGE_API_KEY}`,
+    },
+    body: JSON.stringify({
+      model: options.model || '{{rerankModel}}',
+      query,
+      documents,
+      top_k: options.topK,
+      return_documents: true,
+    }),
+  });
+
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`Voyage AI rerank error: ${response.status} ${error}`);
+  }
+
+  const data = await response.json();
+  return {
+    results: data.data.map(d => ({
+      index: d.index,
+      relevanceScore: d.relevance_score,
+      document: d.document,
+    })),
+    usage: data.usage,
+  };
+}
+{{/if}}
+
+module.exports = {
+  embed,
+{{#if rerank}}
+  rerank,
+{{/if}}
+  VOYAGE_API_URL,
+};

package/src/lib/templates/vanilla/connection.js.tpl

@@ -0,0 +1,126 @@
+/**
+ * MongoDB Connection Helper
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ * 
+ * Database: {{db}}
+ * Collection: {{collection}}
+ */
+
+const { MongoClient } = require('mongodb');
+
+const MONGODB_URI = process.env.MONGODB_URI;
+
+if (!MONGODB_URI) {
+  throw new Error('MONGODB_URI environment variable is required');
+}
+
+let client = null;
+let db = null;
+
+/**
+ * Connect to MongoDB and return the database instance.
+ * Reuses existing connection if available.
+ * @returns {Promise<import('mongodb').Db>}
+ */
+async function connect() {
+  if (db) return db;
+
+  client = new MongoClient(MONGODB_URI);
+  await client.connect();
+  db = client.db('{{db}}');
+  
+  console.log('Connected to MongoDB: {{db}}');
+  return db;
+}
+
+/**
+ * Get the documents collection.
+ * @returns {Promise<import('mongodb').Collection>}
+ */
+async function getCollection() {
+  const database = await connect();
+  return database.collection('{{collection}}');
+}
+
+/**
+ * Close the MongoDB connection.
+ */
+async function close() {
+  if (client) {
+    await client.close();
+    client = null;
+    db = null;
+  }
+}
+
+/**
+ * Perform a vector search on the collection.
+ * @param {number[]} embedding - Query embedding vector
+ * @param {object} options - Search options
+ * @param {number} options.limit - Number of results (default: 10)
+ * @param {number} options.numCandidates - Candidates to consider (default: limit * 10)
+ * @param {object} options.filter - Optional pre-filter
+ * @returns {Promise<Array<{document: object, score: number}>>}
+ */
+async function vectorSearch(embedding, options = {}) {
+  const collection = await getCollection();
+  const limit = options.limit || 10;
+  const numCandidates = options.numCandidates || limit * 10;
+
+  const pipeline = [
+    {
+      $vectorSearch: {
+        index: '{{index}}',
+        path: '{{field}}',
+        queryVector: embedding,
+        numCandidates,
+        limit,
+        {{#if filter}}
+        filter: options.filter,
+        {{/if}}
+      },
+    },
+    {
+      $project: {
+        _id: 1,
+        text: 1,
+        metadata: 1,
+        score: { $meta: 'vectorSearchScore' },
+      },
+    },
+  ];
+
+  const results = await collection.aggregate(pipeline).toArray();
+  return results.map(doc => ({
+    document: doc,
+    score: doc.score,
+  }));
+}
+
+/**
+ * Insert documents with embeddings.
+ * @param {Array<{text: string, embedding: number[], metadata?: object}>} docs
+ * @returns {Promise<{insertedCount: number}>}
+ */
+async function insertDocuments(docs) {
+  const collection = await getCollection();
+  
+  const documents = docs.map(doc => ({
+    text: doc.text,
+    {{field}}: doc.embedding,
+    metadata: doc.metadata || {},
+    _embeddedAt: new Date(),
+    _model: '{{model}}',
+  }));
+
+  const result = await collection.insertMany(documents);
+  return { insertedCount: result.insertedCount };
+}
+
+module.exports = {
+  connect,
+  getCollection,
+  close,
+  vectorSearch,
+  insertDocuments,
+};

package/src/lib/templates/vanilla/env.example.tpl

@@ -0,0 +1,11 @@
+# Voyage AI API
+VOYAGE_API_KEY=your_voyage_api_key_here
+
+# MongoDB Atlas
+MONGODB_URI=mongodb+srv://username:password@cluster.mongodb.net/{{db}}?retryWrites=true&w=majority
+
+# Server
+PORT=3000
+
+# Optional: Override defaults
+# VOYAGE_API_URL=https://api.voyageai.com/v1

package/src/lib/templates/vanilla/ingest.js.tpl

@@ -0,0 +1,231 @@
+/**
+ * Document Ingestion Pipeline
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ * 
+ * Model: {{model}}
+ * Chunk Strategy: {{chunkStrategy}}
+ * Chunk Size: {{chunkSize}} characters
+ * Overlap: {{chunkOverlap}} characters
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { embed } = require('./client');
+const { insertDocuments, close } = require('./connection');
+
+/**
+ * Chunk text using {{chunkStrategy}} strategy.
+ * @param {string} text - Text to chunk
+ * @param {object} options - Chunking options
+ * @returns {string[]} Array of chunks
+ */
+function chunkText(text, options = {}) {
+  const size = options.size || {{chunkSize}};
+  const overlap = options.overlap || {{chunkOverlap}};
+  
+{{#if chunkStrategy}}
+{{#unless chunkStrategy}}
+  // Fixed-size chunking
+  const chunks = [];
+  let start = 0;
+  
+  while (start < text.length) {
+    const end = Math.min(start + size, text.length);
+    chunks.push(text.slice(start, end).trim());
+    start = end - overlap;
+    if (start >= text.length) break;
+  }
+  
+  return chunks.filter(c => c.length > 0);
+{{/unless}}
+{{/if}}
+  // Recursive chunking with smart boundaries
+  const separators = ['\n\n', '\n', '. ', ' '];
+  
+  function splitRecursive(text, separatorIndex = 0) {
+    if (text.length <= size) {
+      return [text.trim()].filter(c => c.length > 0);
+    }
+    
+    if (separatorIndex >= separators.length) {
+      // Fall back to fixed-size split
+      const chunks = [];
+      let start = 0;
+      while (start < text.length) {
+        chunks.push(text.slice(start, start + size).trim());
+        start += size - overlap;
+      }
+      return chunks.filter(c => c.length > 0);
+    }
+    
+    const separator = separators[separatorIndex];
+    const parts = text.split(separator);
+    const chunks = [];
+    let current = '';
+    
+    for (const part of parts) {
+      const potential = current ? current + separator + part : part;
+      
+      if (potential.length <= size) {
+        current = potential;
+      } else {
+        if (current) chunks.push(...splitRecursive(current, separatorIndex + 1));
+        current = part;
+      }
+    }
+    
+    if (current) {
+      chunks.push(...splitRecursive(current, separatorIndex + 1));
+    }
+    
+    return chunks;
+  }
+  
+  return splitRecursive(text);
+}
+
+/**
+ * Read and parse a file based on extension.
+ * @param {string} filePath - Path to the file
+ * @returns {{text: string, metadata: object}}
+ */
+function readFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  const content = fs.readFileSync(filePath, 'utf8');
+  
+  const metadata = {
+    source: filePath,
+    filename: path.basename(filePath),
+    extension: ext,
+  };
+  
+  // For markdown, optionally extract frontmatter
+  if (ext === '.md' || ext === '.mdx') {
+    const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+    if (frontmatterMatch) {
+      // Simple YAML-like parsing for common fields
+      const yaml = frontmatterMatch[1];
+      const body = frontmatterMatch[2];
+      
+      const titleMatch = yaml.match(/title:\s*["']?(.+?)["']?\s*$/m);
+      if (titleMatch) metadata.title = titleMatch[1];
+      
+      return { text: body, metadata };
+    }
+  }
+  
+  return { text: content, metadata };
+}
+
+/**
+ * Ingest a single file: read, chunk, embed, and store.
+ * @param {string} filePath - Path to the file
+ * @param {object} options - Ingestion options
+ * @returns {Promise<{chunks: number, tokens: number}>}
+ */
+async function ingestFile(filePath, options = {}) {
+  const { text, metadata } = readFile(filePath);
+  const chunks = chunkText(text, options);
+  
+  if (chunks.length === 0) {
+    return { chunks: 0, tokens: 0 };
+  }
+  
+  // Embed all chunks
+  const { embeddings, usage } = await embed(chunks, { inputType: 'document' });
+  
+  // Prepare documents for insertion
+  const documents = chunks.map((chunk, i) => ({
+    text: chunk,
+    embedding: embeddings[i],
+    metadata: {
+      ...metadata,
+      chunkIndex: i,
+      totalChunks: chunks.length,
+    },
+  }));
+  
+  // Insert into MongoDB
+  await insertDocuments(documents);
+  
+  return {
+    chunks: chunks.length,
+    tokens: usage.total_tokens,
+  };
+}
+
+/**
+ * Ingest multiple files from a directory.
+ * @param {string} dirPath - Directory path
+ * @param {object} options - Ingestion options
+ * @param {string[]} options.extensions - File extensions to include (default: ['.txt', '.md'])
+ * @returns {Promise<{files: number, chunks: number, tokens: number}>}
+ */
+async function ingestDirectory(dirPath, options = {}) {
+  const extensions = options.extensions || ['.txt', '.md', '.mdx'];
+  
+  function findFiles(dir) {
+    const files = [];
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory() && !entry.name.startsWith('.')) {
+        files.push(...findFiles(fullPath));
+      } else if (entry.isFile() && extensions.includes(path.extname(entry.name).toLowerCase())) {
+        files.push(fullPath);
+      }
+    }
+    
+    return files;
+  }
+  
+  const files = findFiles(dirPath);
+  let totalChunks = 0;
+  let totalTokens = 0;
+  
+  for (const file of files) {
+    console.log(`Ingesting: ${file}`);
+    const { chunks, tokens } = await ingestFile(file, options);
+    totalChunks += chunks;
+    totalTokens += tokens;
+  }
+  
+  return {
+    files: files.length,
+    chunks: totalChunks,
+    tokens: totalTokens,
+  };
+}
+
+// CLI entry point
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const target = args[0] || '.';
+  
+  (async () => {
+    try {
+      const stats = fs.statSync(target);
+      
+      if (stats.isDirectory()) {
+        const result = await ingestDirectory(target);
+        console.log(`\n✓ Ingested ${result.files} files (${result.chunks} chunks, ${result.tokens} tokens)`);
+      } else {
+        const result = await ingestFile(target);
+        console.log(`\n✓ Ingested ${result.chunks} chunks (${result.tokens} tokens)`);
+      }
+      
+      await close();
+    } catch (err) {
+      console.error('Error:', err.message);
+      process.exit(1);
+    }
+  })();
+}
+
+module.exports = {
+  chunkText,
+  readFile,
+  ingestFile,
+  ingestDirectory,
+};

package/src/lib/templates/vanilla/package.json.tpl

@@ -0,0 +1,31 @@
+{
+  "name": "{{projectName}}",
+  "version": "1.0.0",
+  "description": "Voyage AI RAG application",
+  "type": "commonjs",
+  "main": "server.js",
+  "scripts": {
+    "start": "node server.js",
+    "ingest": "node lib/ingest.js",
+    "dev": "node --watch server.js"
+  },
+  "dependencies": {
+    "express": "^4.18.2",
+    "mongodb": "^6.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": [
+    "voyage-ai",
+    "rag",
+    "vector-search",
+    "mongodb-atlas"
+  ],
+  "generated": {
+    "by": "vai",
+    "version": "{{vaiVersion}}",
+    "model": "{{model}}",
+    "at": "{{generatedAt}}"
+  }
+}