npm - voyageai-cli - Versions diffs - 1.20.5 → 1.21.0 - Mend

voyageai-cli 1.20.5 → 1.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

package/CHANGELOG.md +142 -26
package/README.md +153 -2
package/package.json +1 -1
package/src/cli.js +12 -0
package/src/commands/about.js +39 -3
package/src/commands/doctor.js +325 -0
package/src/commands/eval.js +420 -10
package/src/commands/generate.js +220 -0
package/src/commands/playground.js +93 -0
package/src/commands/purge.js +271 -0
package/src/commands/quickstart.js +203 -0
package/src/commands/refresh.js +322 -0
package/src/commands/scaffold.js +217 -0
package/src/lib/codegen.js +313 -0
package/src/lib/explanations.js +163 -1
package/src/lib/scaffold-structure.js +114 -0
package/src/lib/templates/nextjs/README.md.tpl +106 -0
package/src/lib/templates/nextjs/env.example.tpl +8 -0
package/src/lib/templates/nextjs/layout.jsx.tpl +29 -0
package/src/lib/templates/nextjs/lib-mongo.js.tpl +111 -0
package/src/lib/templates/nextjs/lib-voyage.js.tpl +103 -0
package/src/lib/templates/nextjs/package.json.tpl +33 -0
package/src/lib/templates/nextjs/page-search.jsx.tpl +147 -0
package/src/lib/templates/nextjs/route-ingest.js.tpl +114 -0
package/src/lib/templates/nextjs/route-search.js.tpl +97 -0
package/src/lib/templates/nextjs/theme.js.tpl +84 -0
package/src/lib/templates/python/README.md.tpl +145 -0
package/src/lib/templates/python/app.py.tpl +221 -0
package/src/lib/templates/python/chunker.py.tpl +127 -0
package/src/lib/templates/python/env.example.tpl +12 -0
package/src/lib/templates/python/mongo_client.py.tpl +125 -0
package/src/lib/templates/python/requirements.txt.tpl +10 -0
package/src/lib/templates/python/voyage_client.py.tpl +124 -0
package/src/lib/templates/vanilla/README.md.tpl +156 -0
package/src/lib/templates/vanilla/client.js.tpl +103 -0
package/src/lib/templates/vanilla/connection.js.tpl +126 -0
package/src/lib/templates/vanilla/env.example.tpl +11 -0
package/src/lib/templates/vanilla/ingest.js.tpl +231 -0
package/src/lib/templates/vanilla/package.json.tpl +31 -0
package/src/lib/templates/vanilla/retrieval.js.tpl +100 -0
package/src/lib/templates/vanilla/search-api.js.tpl +175 -0
package/src/lib/templates/vanilla/server.js.tpl +81 -0
package/src/lib/zip.js +130 -0
package/src/playground/index.html +772 -33

package/src/lib/templates/nextjs/README.md.tpl ADDED Viewed

@@ -0,0 +1,106 @@
+# {{projectName}}
+A semantic search application powered by Voyage AI embeddings, MongoDB Atlas Vector Search, and Next.js with Material UI.
+## Configuration
+| Setting | Value |
+|---------|-------|
+| Embedding Model | `{{model}}` |
+| Dimensions | {{dimensions}} |
+| Database | `{{db}}` |
+| Collection | `{{collection}}` |
+| Vector Index | `{{index}}` |
+{{#if rerank}}
+| Rerank Model | `{{rerankModel}}` |
+{{/if}}
+## Setup
+### 1. Install dependencies
+```bash
+npm install
+```
+### 2. Configure environment
+Copy `.env.example` to `.env.local` and fill in your credentials:
+```bash
+cp .env.example .env.local
+```
+Required variables:
+- `VOYAGE_API_KEY` - Your Voyage AI API key from [dash.voyageai.com](https://dash.voyageai.com)
+- `MONGODB_URI` - Your MongoDB Atlas connection string
+### 3. Create vector index
+In MongoDB Atlas, create a vector search index on your collection:
+```json
+{
+  "fields": [
+    {
+      "type": "vector",
+      "path": "{{field}}",
+      "numDimensions": {{dimensions}},
+      "similarity": "cosine"
+    }
+  ]
+}
+```
+Name the index `{{index}}`.
+### 4. Start the development server
+```bash
+npm run dev
+```
+Open [http://localhost:3000](http://localhost:3000) to see the search interface.
+## Project Structure
+```
+{{projectName}}/
+├── app/
+│   ├── layout.jsx          # Root layout with MUI theme
+│   ├── page.jsx            # Home page
+│   ├── search/
+│   │   └── page.jsx        # Search page (MUI)
+│   └── api/
+│       ├── search/route.js # Search endpoint
+│       └── ingest/route.js # Ingest endpoint
+├── lib/
+│   ├── voyage.js           # Voyage AI client
+│   ├── mongodb.js          # MongoDB connection
+│   └── theme.js            # MUI theme
+├── .env.example
+├── package.json
+└── README.md
+```
+## API Endpoints
+### POST /api/search
+```bash
+curl -X POST http://localhost:3000/api/search \
+  -H "Content-Type: application/json" \
+  -d '{"query": "How does vector search work?", "limit": 5}'
+```
+### POST /api/ingest
+```bash
+curl -X POST http://localhost:3000/api/ingest \
+  -H "Content-Type: application/json" \
+  -d '{"text": "Your document content...", "metadata": {"source": "api"}}'
+```
+---
+Generated by [vai](https://github.com/mrlynn/voyageai-cli) v{{vaiVersion}}

package/src/lib/templates/nextjs/env.example.tpl ADDED Viewed

@@ -0,0 +1,8 @@
+# Voyage AI API
+VOYAGE_API_KEY=your_voyage_api_key_here
+# MongoDB Atlas
+MONGODB_URI=mongodb+srv://username:password@cluster.mongodb.net/{{db}}?retryWrites=true&w=majority
+# Optional: Override defaults
+# VOYAGE_API_URL=https://api.voyageai.com/v1

package/src/lib/templates/nextjs/layout.jsx.tpl ADDED Viewed

@@ -0,0 +1,29 @@
+/**
+ * Root Layout with MUI Theme
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ */
+import { AppRouterCacheProvider } from '@mui/material-nextjs/v14-appRouter';
+import { ThemeProvider } from '@mui/material/styles';
+import CssBaseline from '@mui/material/CssBaseline';
+import { theme } from '@/lib/theme';
+export const metadata = {
+  title: '{{projectName}}',
+  description: 'Semantic search powered by Voyage AI',
+};
+export default function RootLayout({ children }) {
+  return (
+    <html lang="en">
+      <body>
+        <AppRouterCacheProvider>
+          <ThemeProvider theme={theme}>
+            <CssBaseline />
+            {children}
+          </ThemeProvider>
+        </AppRouterCacheProvider>
+      </body>
+    </html>
+  );
+}

package/src/lib/templates/nextjs/lib-mongo.js.tpl ADDED Viewed

@@ -0,0 +1,111 @@
+/**
+ * MongoDB Connection Helper (Next.js)
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ *
+ * Uses connection caching for serverless environments.
+ */
+import { MongoClient } from 'mongodb';
+const MONGODB_URI = process.env.MONGODB_URI;
+if (!MONGODB_URI) {
+  throw new Error('MONGODB_URI environment variable is required');
+}
+// Cache the client promise for connection reuse in serverless
+let cached = global.mongo;
+if (!cached) {
+  cached = global.mongo = { conn: null, promise: null };
+}
+/**
+ * Get a cached MongoDB client connection.
+ * Reuses existing connection in serverless environment.
+ */
+export async function getMongoClient() {
+  if (cached.conn) {
+    return cached.conn;
+  }
+  if (!cached.promise) {
+    cached.promise = MongoClient.connect(MONGODB_URI);
+  }
+  cached.conn = await cached.promise;
+  return cached.conn;
+}
+/**
+ * Get the database instance.
+ */
+export async function getDb() {
+  const client = await getMongoClient();
+  return client.db('{{db}}');
+}
+/**
+ * Get the documents collection.
+ */
+export async function getCollection() {
+  const db = await getDb();
+  return db.collection('{{collection}}');
+}
+/**
+ * Perform a vector search.
+ */
+export async function vectorSearch(embedding, options = {}) {
+  const collection = await getCollection();
+  const limit = options.limit || 10;
+  const numCandidates = options.numCandidates || limit * 10;
+  const pipeline = [
+    {
+      $vectorSearch: {
+        index: '{{index}}',
+        path: '{{field}}',
+        queryVector: embedding,
+        numCandidates,
+        limit,
+      },
+    },
+    {
+      $project: {
+        _id: 1,
+        text: 1,
+        metadata: 1,
+        score: { $meta: 'vectorSearchScore' },
+      },
+    },
+  ];
+  if (options.filter) {
+    pipeline[0].$vectorSearch.filter = options.filter;
+  }
+  const results = await collection.aggregate(pipeline).toArray();
+  return results.map(doc => ({
+    document: doc,
+    score: doc.score,
+  }));
+}
+/**
+ * Insert documents with embeddings.
+ */
+export async function insertDocuments(docs) {
+  const collection = await getCollection();
+  const documents = docs.map(doc => ({
+    text: doc.text,
+    {{field}}: doc.embedding,
+    metadata: doc.metadata || {},
+    _embeddedAt: new Date(),
+    _model: '{{model}}',
+  }));
+  const result = await collection.insertMany(documents);
+  return { insertedCount: result.insertedCount };
+}

package/src/lib/templates/nextjs/lib-voyage.js.tpl ADDED Viewed

@@ -0,0 +1,103 @@
+/**
+ * Voyage AI API Client (Next.js)
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ *
+ * Model: {{model}}
+ * Dimensions: {{dimensions}}
+ */
+const VOYAGE_API_URL = process.env.VOYAGE_API_URL || 'https://api.voyageai.com/v1';
+const VOYAGE_API_KEY = process.env.VOYAGE_API_KEY;
+/**
+ * Generate embeddings for text(s) using Voyage AI.
+ */
+export async function embed(input, options = {}) {
+  if (!VOYAGE_API_KEY) {
+    throw new Error('VOYAGE_API_KEY environment variable is required');
+  }
+  const texts = Array.isArray(input) ? input : [input];
+  const response = await fetch(`${VOYAGE_API_URL}/embeddings`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${VOYAGE_API_KEY}`,
+    },
+    body: JSON.stringify({
+      model: options.model || '{{model}}',
+      input: texts,
+      input_type: options.inputType || '{{inputType}}',
+      output_dimension: options.outputDimension || {{dimensions}},
+    }),
+  });
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`Voyage AI API error: ${response.status} ${error}`);
+  }
+  const data = await response.json();
+  return {
+    embeddings: data.data.map(d => d.embedding),
+    usage: data.usage,
+  };
+}
+/**
+ * Embed a single query.
+ */
+export async function embedQuery(query, options = {}) {
+  const result = await embed(query, { ...options, inputType: 'query' });
+  return result.embeddings[0];
+}
+/**
+ * Embed multiple documents.
+ */
+export async function embedDocuments(documents, options = {}) {
+  const result = await embed(documents, { ...options, inputType: 'document' });
+  return result.embeddings;
+}
+{{#if rerank}}
+/**
+ * Rerank documents by relevance to a query.
+ */
+export async function rerank(query, documents, options = {}) {
+  if (!VOYAGE_API_KEY) {
+    throw new Error('VOYAGE_API_KEY environment variable is required');
+  }
+  const response = await fetch(`${VOYAGE_API_URL}/rerank`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${VOYAGE_API_KEY}`,
+    },
+    body: JSON.stringify({
+      model: options.model || '{{rerankModel}}',
+      query,
+      documents,
+      top_k: options.topK,
+      return_documents: true,
+    }),
+  });
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`Voyage AI rerank error: ${response.status} ${error}`);
+  }
+  const data = await response.json();
+  return {
+    results: data.data.map(d => ({
+      index: d.index,
+      relevanceScore: d.relevance_score,
+      document: d.document,
+    })),
+    usage: data.usage,
+  };
+}
+{{/if}}

package/src/lib/templates/nextjs/package.json.tpl ADDED Viewed

@@ -0,0 +1,33 @@
+{
+  "name": "{{projectName}}",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@emotion/cache": "^11.11.0",
+    "@emotion/react": "^11.11.3",
+    "@emotion/styled": "^11.11.0",
+    "@mui/icons-material": "^5.15.0",
+    "@mui/material": "^5.15.0",
+    "@mui/material-nextjs": "^5.15.0",
+    "mongodb": "^6.3.0",
+    "next": "^14.1.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "devDependencies": {
+    "eslint": "^8.56.0",
+    "eslint-config-next": "^14.1.0"
+  },
+  "generated": {
+    "by": "vai",
+    "version": "{{vaiVersion}}",
+    "model": "{{model}}",
+    "at": "{{generatedAt}}"
+  }
+}

package/src/lib/templates/nextjs/page-search.jsx.tpl ADDED Viewed

@@ -0,0 +1,147 @@
+/**
+ * Search Page (MUI)
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ */
+'use client';
+import { useState } from 'react';
+import {
+  Box,
+  Container,
+  TextField,
+  Button,
+  Card,
+  CardContent,
+  Typography,
+  List,
+  ListItem,
+  Chip,
+  CircularProgress,
+  Alert,
+  InputAdornment,
+} from '@mui/material';
+import SearchIcon from '@mui/icons-material/Search';
+export default function SearchPage() {
+  const [query, setQuery] = useState('');
+  const [results, setResults] = useState([]);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState(null);
+  const [meta, setMeta] = useState(null);
+  const handleSearch = async (e) => {
+    e.preventDefault();
+    if (!query.trim()) return;
+    setLoading(true);
+    setError(null);
+    try {
+      const response = await fetch('/api/search', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ query, limit: 5 }),
+      });
+      if (!response.ok) {
+        const data = await response.json();
+        throw new Error(data.error || 'Search failed');
+      }
+      const data = await response.json();
+      setResults(data.results);
+      setMeta(data.meta);
+    } catch (err) {
+      setError(err.message);
+      setResults([]);
+    } finally {
+      setLoading(false);
+    }
+  };
+  return (
+    <Container maxWidth="md" sx={{ py: 4 }}>
+      <Typography variant="h4" component="h1" gutterBottom>
+        Semantic Search
+      </Typography>
+      <Typography variant="body1" color="text.secondary" sx={{ mb: 3 }}>
+        Search your documents using {{model}} embeddings
+        {{#if rerank}} with {{rerankModel}} reranking{{/if}}.
+      </Typography>
+      <Box component="form" onSubmit={handleSearch} sx={{ mb: 4 }}>
+        <TextField
+          fullWidth
+          value={query}
+          onChange={(e) => setQuery(e.target.value)}
+          placeholder="Enter your search query..."
+          variant="outlined"
+          InputProps={{
+            startAdornment: (
+              <InputAdornment position="start">
+                <SearchIcon />
+              </InputAdornment>
+            ),
+            endAdornment: (
+              <InputAdornment position="end">
+                <Button
+                  type="submit"
+                  variant="contained"
+                  disabled={loading || !query.trim()}
+                >
+                  {loading ? <CircularProgress size={24} /> : 'Search'}
+                </Button>
+              </InputAdornment>
+            ),
+          }}
+        />
+      </Box>
+      {error && (
+        <Alert severity="error" sx={{ mb: 3 }}>
+          {error}
+        </Alert>
+      )}
+      {meta && (
+        <Box sx={{ mb: 2, display: 'flex', gap: 1 }}>
+          <Chip label={`${results.length} results`} size="small" />
+          <Chip label={`${meta.took}ms`} size="small" variant="outlined" />
+          <Chip label={meta.model} size="small" variant="outlined" />
+        </Box>
+      )}
+      <List>
+        {results.map((result, index) => (
+          <ListItem key={index} sx={{ px: 0 }}>
+            <Card sx={{ width: '100%' }}>
+              <CardContent>
+                <Box sx={{ display: 'flex', justifyContent: 'space-between', mb: 1 }}>
+                  <Typography variant="subtitle2" color="text.secondary">
+                    {result.metadata?.source || 'Document'}
+                  </Typography>
+                  <Chip
+                    label={`Score: ${result.score.toFixed(3)}`}
+                    size="small"
+                    color={result.score > 0.8 ? 'success' : 'default'}
+                  />
+                </Box>
+                <Typography variant="body2" sx={{ whiteSpace: 'pre-wrap' }}>
+                  {result.text}
+                </Typography>
+              </CardContent>
+            </Card>
+          </ListItem>
+        ))}
+      </List>
+      {results.length === 0 && !loading && query && !error && (
+        <Typography color="text.secondary" align="center">
+          No results found for "{query}"
+        </Typography>
+      )}
+    </Container>
+  );
+}

package/src/lib/templates/nextjs/route-ingest.js.tpl ADDED Viewed

@@ -0,0 +1,114 @@
+/**
+ * Ingest API Route Handler
+ * Generated by vai v{{vaiVersion}} on {{generatedAt}}
+ *
+ * POST /api/ingest
+ */
+import { NextResponse } from 'next/server';
+import { embed } from '@/lib/voyage';
+import { insertDocuments } from '@/lib/mongodb';
+/**
+ * Chunk text using recursive splitting.
+ */
+function chunkText(text, size = {{chunkSize}}, overlap = {{chunkOverlap}}) {
+  const separators = ['\n\n', '\n', '. ', ' '];
+  function splitRecursive(text, sepIndex = 0) {
+    if (text.length <= size) {
+      const trimmed = text.trim();
+      return trimmed ? [trimmed] : [];
+    }
+    if (sepIndex >= separators.length) {
+      const chunks = [];
+      let start = 0;
+      while (start < text.length) {
+        const chunk = text.slice(start, start + size).trim();
+        if (chunk) chunks.push(chunk);
+        start += size - overlap;
+      }
+      return chunks;
+    }
+    const separator = separators[sepIndex];
+    const parts = text.split(separator);
+    const chunks = [];
+    let current = '';
+    for (const part of parts) {
+      const potential = current ? current + separator + part : part;
+      if (potential.length <= size) {
+        current = potential;
+      } else {
+        if (current) chunks.push(...splitRecursive(current, sepIndex + 1));
+        current = part;
+      }
+    }
+    if (current) chunks.push(...splitRecursive(current, sepIndex + 1));
+    return chunks;
+  }
+  return splitRecursive(text);
+}
+export async function POST(request) {
+  const start = Date.now();
+  try {
+    const body = await request.json();
+    const { text, metadata = {} } = body;
+    if (!text || typeof text !== 'string') {
+      return NextResponse.json(
+        { error: 'Text is required' },
+        { status: 400 }
+      );
+    }
+    // Chunk the text
+    const chunks = chunkText(text);
+    if (chunks.length === 0) {
+      return NextResponse.json({
+        success: true,
+        chunks: 0,
+        tokens: 0,
+        took: Date.now() - start,
+      });
+    }
+    // Embed all chunks
+    const { embeddings, usage } = await embed(chunks, { inputType: 'document' });
+    // Prepare documents
+    const documents = chunks.map((chunk, i) => ({
+      text: chunk,
+      embedding: embeddings[i],
+      metadata: {
+        ...metadata,
+        source: metadata.source || 'api',
+        chunkIndex: i,
+        totalChunks: chunks.length,
+      },
+    }));
+    // Insert into MongoDB
+    await insertDocuments(documents);
+    return NextResponse.json({
+      success: true,
+      chunks: chunks.length,
+      tokens: usage.total_tokens,
+      took: Date.now() - start,
+    });
+  } catch (error) {
+    console.error('Ingest error:', error);
+    return NextResponse.json(
+      { error: error.message },
+      { status: 500 }
+    );
+  }
+}