voyageai-cli 1.20.6 → 1.22.0

package/src/lib/templates/python/app.py.tpl

@@ -0,0 +1,221 @@
+"""
+Voyage AI RAG API (Flask)
+Generated by vai v{{vaiVersion}} on {{generatedAt}}
+
+Endpoints:
+- POST /api/search  - Semantic search with optional reranking
+- POST /api/ingest  - Document ingestion
+- GET  /api/health  - Health check
+"""
+
+import os
+import time
+from flask import Flask, request, jsonify
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from voyage_client import embed_query{{#if rerank}}, rerank{{/if}}
+from mongo_client import connect, vector_search, insert_documents, close
+from chunker import chunk_text
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def index():
+    """API information."""
+    return jsonify({
+        "name": "{{projectName}}",
+        "description": "Voyage AI RAG API",
+        "model": "{{model}}",
+        "database": "{{db}}",
+        "collection": "{{collection}}",
+        "endpoints": {
+            "search": "POST /api/search",
+            "ingest": "POST /api/ingest",
+            "health": "GET /api/health",
+        },
+    })
+
+
+@app.route("/api/search", methods=["POST"])
+def search():
+    """
+    Semantic search endpoint.
+    
+    Request body:
+    {
+        "query": "What is vector search?",
+        "limit": 5,
+        "include_context": true
+    }
+    """
+    start = time.time()
+    data = request.get_json()
+    
+    query = data.get("query")
+    if not query:
+        return jsonify({"error": "Query is required"}), 400
+    
+    limit = data.get("limit", 5)
+{{#if rerank}}
+    candidates = data.get("candidates", 20)
+{{/if}}
+    include_context = data.get("include_context", False)
+    filter_query = data.get("filter")
+    
+    # Step 1: Embed the query
+    query_embedding = embed_query(query)
+    
+    # Step 2: Vector search
+{{#if rerank}}
+    search_results = vector_search(query_embedding, limit=candidates, filter_query=filter_query)
+{{else}}
+    search_results = vector_search(query_embedding, limit=limit, filter_query=filter_query)
+{{/if}}
+    
+    if not search_results:
+        return jsonify({
+            "results": [],
+            "meta": {"model": "{{model}}", "took": int((time.time() - start) * 1000)},
+        })
+    
+{{#if rerank}}
+    # Step 3: Rerank for better relevance
+    documents = [r["document"]["text"] for r in search_results]
+    rerank_result = rerank(query, documents, top_k=limit)
+    
+    results = [
+        {
+            "text": search_results[r["index"]]["document"]["text"],
+            "score": r["relevance_score"],
+            "metadata": search_results[r["index"]]["document"].get("metadata", {}),
+        }
+        for r in rerank_result["results"]
+    ]
+{{else}}
+    results = [
+        {
+            "text": r["document"]["text"],
+            "score": r["score"],
+            "metadata": r["document"].get("metadata", {}),
+        }
+        for r in search_results
+    ]
+{{/if}}
+    
+    response = {
+        "results": results,
+        "meta": {
+            "model": "{{model}}",
+{{#if rerank}}
+            "rerank_model": "{{rerankModel}}",
+{{/if}}
+            "took": int((time.time() - start) * 1000),
+        },
+    }
+    
+    if include_context:
+        response["context"] = "\n\n---\n\n".join(r["text"] for r in results)
+    
+    return jsonify(response)
+
+
+@app.route("/api/ingest", methods=["POST"])
+def ingest():
+    """
+    Document ingestion endpoint.
+    
+    Request body:
+    {
+        "text": "Document content...",
+        "metadata": {"source": "api"}
+    }
+    """
+    start = time.time()
+    data = request.get_json()
+    
+    text = data.get("text")
+    if not text:
+        return jsonify({"error": "Text is required"}), 400
+    
+    metadata = data.get("metadata", {})
+    metadata["source"] = metadata.get("source", "api")
+    
+    # Chunk the text
+    chunks = chunk_text(text)
+    
+    # Embed all chunks
+    from voyage_client import embed
+    result = embed(chunks, input_type="document")
+    embeddings = result["embeddings"]
+    
+    # Prepare documents
+    documents = [
+        {
+            "text": chunk,
+            "embedding": embeddings[i],
+            "metadata": {
+                **metadata,
+                "chunk_index": i,
+                "total_chunks": len(chunks),
+            },
+        }
+        for i, chunk in enumerate(chunks)
+    ]
+    
+    # Insert into MongoDB
+    insert_result = insert_documents(documents)
+    
+    return jsonify({
+        "success": True,
+        "chunks": len(chunks),
+        "tokens": result["usage"]["total_tokens"],
+        "took": int((time.time() - start) * 1000),
+    })
+
+
+@app.route("/api/health", methods=["GET"])
+def health():
+    """Health check endpoint."""
+    try:
+        connect()
+        return jsonify({
+            "status": "healthy",
+            "model": "{{model}}",
+            "database": "{{db}}",
+            "collection": "{{collection}}",
+        })
+    except Exception as e:
+        return jsonify({
+            "status": "unhealthy",
+            "error": str(e),
+        }), 503
+
+
+@app.teardown_appcontext
+def shutdown(exception=None):
+    """Clean up on shutdown."""
+    pass  # Connection pooling handles this
+
+
+if __name__ == "__main__":
+    port = int(os.getenv("PORT", 3000))
+    print(f"""
+🚀 Voyage AI RAG Server (Flask)
+
+   Model:      {{model}}
+   Database:   {{db}}.{{collection}}
+{{#if rerank}}
+   Reranking:  {{rerankModel}}
+{{/if}}
+
+   Endpoints:
+   POST /api/search  - Semantic search
+   POST /api/ingest  - Document ingestion
+   GET  /api/health  - Health check
+
+   Running on http://localhost:{port}
+""")
+    app.run(host="0.0.0.0", port=port, debug=os.getenv("FLASK_DEBUG", "0") == "1")

package/src/lib/templates/python/chunker.py.tpl

@@ -0,0 +1,127 @@
+"""
+Text Chunker
+Generated by vai v{{vaiVersion}} on {{generatedAt}}
+
+Strategy: {{chunkStrategy}}
+Size: {{chunkSize}} characters
+Overlap: {{chunkOverlap}} characters
+"""
+
+from typing import List
+
+
+def chunk_text(
+    text: str,
+    size: int = {{chunkSize}},
+    overlap: int = {{chunkOverlap}},
+) -> List[str]:
+    """
+    Chunk text using recursive splitting with smart boundaries.
+    
+    Args:
+        text: Text to chunk
+        size: Maximum chunk size in characters
+        overlap: Overlap between chunks
+        
+    Returns:
+        List of text chunks
+    """
+    separators = ["\n\n", "\n", ". ", " "]
+    
+    def split_recursive(text: str, sep_index: int = 0) -> List[str]:
+        if len(text) <= size:
+            stripped = text.strip()
+            return [stripped] if stripped else []
+        
+        if sep_index >= len(separators):
+            # Fall back to fixed-size split
+            chunks = []
+            start = 0
+            while start < len(text):
+                chunk = text[start:start + size].strip()
+                if chunk:
+                    chunks.append(chunk)
+                start += size - overlap
+            return chunks
+        
+        separator = separators[sep_index]
+        parts = text.split(separator)
+        chunks = []
+        current = ""
+        
+        for part in parts:
+            potential = f"{current}{separator}{part}" if current else part
+            
+            if len(potential) <= size:
+                current = potential
+            else:
+                if current:
+                    chunks.extend(split_recursive(current, sep_index + 1))
+                current = part
+        
+        if current:
+            chunks.extend(split_recursive(current, sep_index + 1))
+        
+        return chunks
+    
+    return split_recursive(text)
+
+
+def chunk_markdown(text: str, size: int = {{chunkSize}}, overlap: int = {{chunkOverlap}}) -> List[str]:
+    """
+    Chunk markdown with header awareness.
+    Tries to keep sections together when possible.
+    """
+    import re
+    
+    # Split by headers (## or ###)
+    header_pattern = r"(^#{1,3}\s+.+$)"
+    parts = re.split(header_pattern, text, flags=re.MULTILINE)
+    
+    chunks = []
+    current = ""
+    
+    for part in parts:
+        if not part.strip():
+            continue
+        
+        potential = f"{current}\n\n{part}" if current else part
+        
+        if len(potential) <= size:
+            current = potential
+        else:
+            if current:
+                # Chunk the current section
+                chunks.extend(chunk_text(current, size, overlap))
+            current = part
+    
+    if current:
+        chunks.extend(chunk_text(current, size, overlap))
+    
+    return chunks
+
+
+if __name__ == "__main__":
+    # Test chunking
+    sample = """
+    # Introduction
+    
+    This is a sample document for testing the chunker.
+    It contains multiple paragraphs and sections.
+    
+    ## Section 1
+    
+    Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+    Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+    
+    ## Section 2
+    
+    Ut enim ad minim veniam, quis nostrud exercitation ullamco.
+    Duis aute irure dolor in reprehenderit in voluptate velit.
+    """
+    
+    chunks = chunk_text(sample)
+    print(f"Created {len(chunks)} chunks:")
+    for i, chunk in enumerate(chunks):
+        print(f"\n--- Chunk {i + 1} ({len(chunk)} chars) ---")
+        print(chunk[:200] + "..." if len(chunk) > 200 else chunk)

package/src/lib/templates/python/env.example.tpl

@@ -0,0 +1,12 @@
+# Voyage AI API
+VOYAGE_API_KEY=your_voyage_api_key_here
+
+# MongoDB Atlas
+MONGODB_URI=mongodb+srv://username:password@cluster.mongodb.net/{{db}}?retryWrites=true&w=majority
+
+# Server
+PORT=3000
+FLASK_DEBUG=0
+
+# Optional: Override defaults
+# VOYAGE_API_URL=https://api.voyageai.com/v1

package/src/lib/templates/python/mongo_client.py.tpl

@@ -0,0 +1,125 @@
+"""
+MongoDB Connection Helper
+Generated by vai v{{vaiVersion}} on {{generatedAt}}
+
+Database: {{db}}
+Collection: {{collection}}
+"""
+
+import os
+from datetime import datetime
+from typing import List, Dict, Any, Optional
+from pymongo import MongoClient
+
+MONGODB_URI = os.getenv("MONGODB_URI")
+
+if not MONGODB_URI:
+    raise ValueError("MONGODB_URI environment variable is required")
+
+# Module-level connection (singleton pattern)
+_client: Optional[MongoClient] = None
+_db = None
+
+
+def connect():
+    """Connect to MongoDB and return the database instance."""
+    global _client, _db
+    
+    if _db is not None:
+        return _db
+    
+    _client = MongoClient(MONGODB_URI)
+    _db = _client["{{db}}"]
+    print(f"Connected to MongoDB: {{db}}")
+    return _db
+
+
+def get_collection():
+    """Get the documents collection."""
+    db = connect()
+    return db["{{collection}}"]
+
+
+def close():
+    """Close the MongoDB connection."""
+    global _client, _db
+    if _client:
+        _client.close()
+        _client = None
+        _db = None
+
+
+def vector_search(
+    embedding: List[float],
+    limit: int = 10,
+    num_candidates: Optional[int] = None,
+    filter_query: Optional[Dict] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Perform a vector search on the collection.
+    
+    Args:
+        embedding: Query embedding vector
+        limit: Number of results to return
+        num_candidates: Candidates to consider (default: limit * 10)
+        filter_query: Optional MongoDB filter
+        
+    Returns:
+        List of documents with scores
+    """
+    collection = get_collection()
+    num_candidates = num_candidates or limit * 10
+    
+    pipeline = [
+        {
+            "$vectorSearch": {
+                "index": "{{index}}",
+                "path": "{{field}}",
+                "queryVector": embedding,
+                "numCandidates": num_candidates,
+                "limit": limit,
+            }
+        },
+        {
+            "$project": {
+                "_id": 1,
+                "text": 1,
+                "metadata": 1,
+                "score": {"$meta": "vectorSearchScore"},
+            }
+        },
+    ]
+    
+    # Add filter if provided
+    if filter_query:
+        pipeline[0]["$vectorSearch"]["filter"] = filter_query
+    
+    results = list(collection.aggregate(pipeline))
+    return [{"document": doc, "score": doc.pop("score")} for doc in results]
+
+
+def insert_documents(docs: List[Dict[str, Any]]) -> Dict[str, int]:
+    """
+    Insert documents with embeddings.
+    
+    Args:
+        docs: List of dicts with 'text', 'embedding', and optional 'metadata'
+        
+    Returns:
+        Dict with 'inserted_count'
+    """
+    collection = get_collection()
+    
+    documents = [
+        {
+            "text": doc["text"],
+            "{{field}}": doc["embedding"],
+            "metadata": doc.get("metadata", {}),
+            "_embeddedAt": datetime.utcnow(),
+            "_model": "{{model}}",
+        }
+        for doc in docs
+    ]
+    
+    result = collection.insert_many(documents)
+    return {"inserted_count": len(result.inserted_ids)}

package/src/lib/templates/python/requirements.txt.tpl

@@ -0,0 +1,10 @@
+# Voyage AI RAG Application
+# Generated by vai v{{vaiVersion}}
+
+flask>=2.3.0
+pymongo>=4.6.0
+requests>=2.31.0
+python-dotenv>=1.0.0
+
+# Optional: for production
+# gunicorn>=21.0.0

package/src/lib/templates/python/voyage_client.py.tpl

@@ -0,0 +1,124 @@
+"""
+Voyage AI API Client
+Generated by vai v{{vaiVersion}} on {{generatedAt}}
+
+Model: {{model}}
+Dimensions: {{dimensions}}
+"""
+
+import os
+import requests
+from typing import List, Optional, Dict, Any
+
+VOYAGE_API_URL = os.getenv("VOYAGE_API_URL", "https://api.voyageai.com/v1")
+VOYAGE_API_KEY = os.getenv("VOYAGE_API_KEY")
+
+if not VOYAGE_API_KEY:
+    raise ValueError("VOYAGE_API_KEY environment variable is required")
+
+
+def embed(
+    texts: List[str],
+    model: str = "{{model}}",
+    input_type: str = "{{inputType}}",
+    output_dimension: int = {{dimensions}},
+) -> Dict[str, Any]:
+    """
+    Generate embeddings for a list of texts using Voyage AI.
+    
+    Args:
+        texts: List of texts to embed
+        model: Embedding model name
+        input_type: 'document' or 'query'
+        output_dimension: Output vector dimensions
+        
+    Returns:
+        Dict with 'embeddings' (list of vectors) and 'usage' (token counts)
+    """
+    response = requests.post(
+        f"{VOYAGE_API_URL}/embeddings",
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {VOYAGE_API_KEY}",
+        },
+        json={
+            "model": model,
+            "input": texts,
+            "input_type": input_type,
+            "output_dimension": output_dimension,
+        },
+    )
+    
+    response.raise_for_status()
+    data = response.json()
+    
+    return {
+        "embeddings": [d["embedding"] for d in data["data"]],
+        "usage": data["usage"],
+    }
+
+
+def embed_query(query: str, **kwargs) -> List[float]:
+    """Embed a single query, returning the embedding vector."""
+    result = embed([query], input_type="query", **kwargs)
+    return result["embeddings"][0]
+
+
+def embed_documents(documents: List[str], **kwargs) -> List[List[float]]:
+    """Embed multiple documents, returning list of embedding vectors."""
+    result = embed(documents, input_type="document", **kwargs)
+    return result["embeddings"]
+
+
+{{#if rerank}}
+def rerank(
+    query: str,
+    documents: List[str],
+    model: str = "{{rerankModel}}",
+    top_k: Optional[int] = None,
+) -> Dict[str, Any]:
+    """
+    Rerank documents by relevance to a query.
+    
+    Args:
+        query: The query to rank against
+        documents: List of documents to rerank
+        model: Rerank model name
+        top_k: Number of top results to return
+        
+    Returns:
+        Dict with 'results' containing index, relevance_score, and document
+    """
+    payload = {
+        "model": model,
+        "query": query,
+        "documents": documents,
+        "return_documents": True,
+    }
+    if top_k:
+        payload["top_k"] = top_k
+    
+    response = requests.post(
+        f"{VOYAGE_API_URL}/rerank",
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {VOYAGE_API_KEY}",
+        },
+        json=payload,
+    )
+    
+    response.raise_for_status()
+    data = response.json()
+    
+    return {
+        "results": [
+            {
+                "index": d["index"],
+                "relevance_score": d["relevance_score"],
+                "document": d.get("document"),
+            }
+            for d in data["data"]
+        ],
+        "usage": data.get("usage"),
+    }
+{{/if}}