voyageai-cli 1.22.1 → 1.23.1

package/src/lib/wizard-steps-init.js

@@ -0,0 +1,174 @@
+'use strict';
+
+/**
+ * Project init wizard step definitions.
+ *
+ * Surface-agnostic — consumed by CLI, Playground, and Desktop.
+ * Replaces the raw readline prompts in the old init command.
+ */
+
+const { MODEL_CATALOG } = require('./catalog');
+const { STRATEGIES } = require('./chunker');
+
+/**
+ * Get available embedding models (non-legacy, non-unreleased).
+ */
+function getEmbeddingModelOptions() {
+  return MODEL_CATALOG
+    .filter(m => m.type === 'embedding' && !m.legacy && !m.unreleased)
+    .map(m => ({
+      value: m.name,
+      label: m.name,
+      hint: `${m.shortFor || m.bestFor} — ${m.price}`,
+    }));
+}
+
+/**
+ * Get chunk strategy options.
+ */
+function getStrategyOptions() {
+  const descriptions = {
+    fixed: 'Fixed character count',
+    sentence: 'Split on sentence boundaries',
+    paragraph: 'Split on paragraph boundaries',
+    recursive: 'Recursive splitting (recommended)',
+    markdown: 'Markdown-aware splitting',
+  };
+  return STRATEGIES.map(s => ({
+    value: s,
+    label: s,
+    hint: descriptions[s] || '',
+  }));
+}
+
+/**
+ * Get dimension options for the selected model.
+ */
+function getDimensionOptions(answers) {
+  const model = answers.model;
+  const info = MODEL_CATALOG.find(m => m.name === model);
+  if (!info || !info.dimensions) {
+    return [
+      { value: '1024', label: '1024', hint: 'default' },
+      { value: '512', label: '512' },
+      { value: '256', label: '256' },
+    ];
+  }
+  // Parse dimensions string like "1024 (default), 256, 512, 2048"
+  const dims = info.dimensions.split(',').map(d => d.trim());
+  return dims.map(d => {
+    const isDefault = d.includes('default');
+    const val = d.replace(/[^0-9]/g, '');
+    return {
+      value: val,
+      label: val,
+      hint: isDefault ? 'default' : undefined,
+    };
+  });
+}
+
+const initSteps = [
+  // Embedding model
+  {
+    id: 'model',
+    label: 'Embedding model',
+    type: 'select',
+    options: () => getEmbeddingModelOptions(),
+    defaultValue: 'voyage-4-large',
+    required: true,
+    group: 'Embedding',
+  },
+
+  // MongoDB
+  {
+    id: 'db',
+    label: 'Database name',
+    type: 'text',
+    defaultValue: 'myapp',
+    placeholder: 'myapp',
+    required: true,
+    group: 'MongoDB Atlas',
+  },
+  {
+    id: 'collection',
+    label: 'Collection name',
+    type: 'text',
+    defaultValue: 'documents',
+    placeholder: 'documents',
+    required: true,
+    group: 'MongoDB Atlas',
+  },
+  {
+    id: 'field',
+    label: 'Embedding field',
+    type: 'text',
+    defaultValue: 'embedding',
+    placeholder: 'embedding',
+    group: 'MongoDB Atlas',
+  },
+  {
+    id: 'index',
+    label: 'Vector index name',
+    type: 'text',
+    defaultValue: 'vector_index',
+    placeholder: 'vector_index',
+    group: 'MongoDB Atlas',
+  },
+
+  // Dimensions
+  {
+    id: 'dimensions',
+    label: 'Dimensions',
+    type: 'select',
+    options: (answers) => getDimensionOptions(answers),
+    getDefault: (answers) => {
+      const info = MODEL_CATALOG.find(m => m.name === answers.model);
+      if (info && info.dimensions && info.dimensions.includes('1024')) return '1024';
+      return '512';
+    },
+    group: 'Embedding',
+  },
+
+  // Chunking
+  {
+    id: 'chunkStrategy',
+    label: 'Chunk strategy',
+    type: 'select',
+    options: () => getStrategyOptions(),
+    defaultValue: 'recursive',
+    group: 'Chunking',
+  },
+  {
+    id: 'chunkSize',
+    label: 'Chunk size (chars)',
+    type: 'text',
+    defaultValue: '512',
+    placeholder: '512',
+    validate: (v) => {
+      const n = parseInt(v, 10);
+      if (isNaN(n) || n < 50) return 'Must be a number ≥ 50';
+      return true;
+    },
+    group: 'Chunking',
+  },
+  {
+    id: 'chunkOverlap',
+    label: 'Chunk overlap (chars)',
+    type: 'text',
+    defaultValue: '50',
+    placeholder: '50',
+    validate: (v) => {
+      const n = parseInt(v, 10);
+      if (isNaN(n) || n < 0) return 'Must be a non-negative number';
+      return true;
+    },
+    group: 'Chunking',
+  },
+];
+
+module.exports = {
+  initSteps,
+  getEmbeddingModelOptions,
+  getStrategyOptions,
+  getDimensionOptions,
+};

package/src/lib/wizard.js

@@ -0,0 +1,222 @@
+'use strict';
+
+/**
+ * Surface-agnostic wizard engine.
+ *
+ * Defines step schemas, validation, flow control (next/back/skip),
+ * and config resolution. UI renderers (CLI, Playground, Desktop)
+ * consume the same step definitions.
+ *
+ * A wizard is an ordered array of Step objects. The engine walks
+ * them forward/backward, skipping steps whose `skip` predicate
+ * returns true, and validating answers before advancing.
+ */
+
+/**
+ * @typedef {Object} StepOption
+ * @property {string} value   - stored value
+ * @property {string} label   - display label
+ * @property {string} [hint]  - secondary description
+ */
+
+/**
+ * @typedef {Object} Step
+ * @property {string}   id         - unique key (becomes the answer key)
+ * @property {string}   label      - human-readable prompt
+ * @property {'select'|'text'|'password'|'confirm'} type
+ * @property {StepOption[]|function(answers,config):StepOption[]} [options]  - for select type
+ * @property {*}        [defaultValue]       - static default
+ * @property {function(answers,config):*}   [getDefault] - dynamic default
+ * @property {boolean}  [required]           - must have a value to advance
+ * @property {function(value,answers):true|string} [validate] - return true or error message
+ * @property {function(answers,config):boolean}    [skip]     - skip this step entirely
+ * @property {string}   [group]              - visual grouping label
+ * @property {string}   [placeholder]        - input placeholder / hint text
+ */
+
+/**
+ * Resolve the effective options array for a step.
+ * @param {Step} step
+ * @param {object} answers   - answers collected so far
+ * @param {object} config    - existing configuration
+ * @returns {StepOption[]}
+ */
+function resolveOptions(step, answers, config) {
+  if (typeof step.options === 'function') return step.options(answers, config);
+  return step.options || [];
+}
+
+/**
+ * Resolve the default value for a step.
+ * @param {Step} step
+ * @param {object} answers
+ * @param {object} config
+ * @returns {*}
+ */
+function resolveDefault(step, answers, config) {
+  if (typeof step.getDefault === 'function') return step.getDefault(answers, config);
+  return step.defaultValue;
+}
+
+/**
+ * Determine whether a step should be skipped.
+ * @param {Step} step
+ * @param {object} answers
+ * @param {object} config
+ * @returns {boolean}
+ */
+function shouldSkip(step, answers, config) {
+  if (typeof step.skip === 'function') return step.skip(answers, config);
+  return false;
+}
+
+/**
+ * Validate a step's answer.
+ * @param {Step} step
+ * @param {*} value
+ * @param {object} answers
+ * @returns {true|string}  true if valid, or error message string
+ */
+function validateStep(step, value, answers) {
+  if (step.required && (value === undefined || value === null || value === '')) {
+    return `${step.label} is required`;
+  }
+  if (typeof step.validate === 'function') {
+    return step.validate(value, answers);
+  }
+  return true;
+}
+
+/**
+ * Compute the ordered list of non-skipped step indices.
+ * @param {Step[]} steps
+ * @param {object} answers
+ * @param {object} config
+ * @returns {number[]}
+ */
+function activeIndices(steps, answers, config) {
+  const indices = [];
+  for (let i = 0; i < steps.length; i++) {
+    if (!shouldSkip(steps[i], answers, config)) {
+      indices.push(i);
+    }
+  }
+  return indices;
+}
+
+/**
+ * Walk a wizard definition and collect answers.
+ *
+ * This is the **headless engine**. It calls `renderer.prompt(step, context)`
+ * for each active step. The renderer returns:
+ *   - A value (answer)
+ *   - Symbol.for('back')  → go to previous step
+ *   - Symbol.for('cancel') → abort the wizard
+ *
+ * @param {Object} params
+ * @param {Step[]}  params.steps    - wizard step definitions
+ * @param {object}  params.config   - existing config (for skip/default resolution)
+ * @param {object}  params.renderer - { prompt(step, ctx) → value|symbol, intro?, outro? }
+ * @param {object}  [params.initial] - pre-filled answers
+ * @returns {Promise<{answers: object, cancelled: boolean}>}
+ */
+async function runWizard({ steps, config = {}, renderer, initial = {} }) {
+  const answers = { ...initial };
+  const active = activeIndices(steps, answers, config);
+
+  if (active.length === 0) {
+    return { answers, cancelled: false };
+  }
+
+  if (renderer.intro) await renderer.intro(steps, config);
+
+  let pos = 0; // position within the active array
+
+  while (pos < active.length) {
+    const stepIdx = active[pos];
+    const step = steps[stepIdx];
+
+    // Recompute active list (answers may change skip predicates)
+    const currentActive = activeIndices(steps, answers, config);
+    if (!currentActive.includes(stepIdx)) {
+      pos++;
+      continue;
+    }
+
+    const options = resolveOptions(step, answers, config);
+    const defaultValue = answers[step.id] !== undefined
+      ? answers[step.id]
+      : resolveDefault(step, answers, config);
+
+    const result = await renderer.prompt(step, {
+      options,
+      defaultValue,
+      stepNumber: pos + 1,
+      totalSteps: currentActive.length,
+      isFirst: pos === 0,
+      isLast: pos === currentActive.length - 1,
+      answers,
+      config,
+    });
+
+    // Handle navigation
+    if (result === Symbol.for('cancel')) {
+      if (renderer.cancel) await renderer.cancel();
+      return { answers, cancelled: true };
+    }
+
+    if (result === Symbol.for('back')) {
+      if (pos > 0) pos--;
+      continue;
+    }
+
+    // Validate
+    const valid = validateStep(step, result, answers);
+    if (valid !== true) {
+      if (renderer.error) await renderer.error(valid);
+      continue; // re-prompt same step
+    }
+
+    answers[step.id] = result;
+    pos++;
+  }
+
+  if (renderer.outro) await renderer.outro(answers);
+
+  return { answers, cancelled: false };
+}
+
+/**
+ * Export step definitions as a plain serializable array
+ * (for web/desktop consumption). Strips functions, resolves
+ * options and defaults against the provided config.
+ *
+ * @param {Step[]} steps
+ * @param {object} config
+ * @returns {object[]}
+ */
+function serializeSteps(steps, config = {}) {
+  const answers = {};
+  return steps
+    .filter(s => !shouldSkip(s, answers, config))
+    .map(s => ({
+      id: s.id,
+      label: s.label,
+      type: s.type,
+      required: !!s.required,
+      group: s.group || null,
+      placeholder: s.placeholder || null,
+      options: resolveOptions(s, answers, config),
+      defaultValue: resolveDefault(s, answers, config),
+    }));
+}
+
+module.exports = {
+  runWizard,
+  serializeSteps,
+  resolveOptions,
+  resolveDefault,
+  shouldSkip,
+  validateStep,
+  activeIndices,
+};

package/src/mcp/schemas/index.js

@@ -0,0 +1,102 @@
+'use strict';
+
+const { z } = require('zod');
+
+/** vai_query input schema */
+const querySchema = {
+  query: z.string().min(1).max(5000).describe('The question or search query in natural language'),
+  db: z.string().optional().describe('MongoDB database name. Uses vai config default if omitted.'),
+  collection: z.string().optional().describe('Collection with embedded documents. Uses vai config default if omitted.'),
+  limit: z.number().int().min(1).max(50).default(5).describe('Maximum number of results to return'),
+  model: z.string().optional().describe('Voyage AI embedding model. Default: voyage-4-large'),
+  rerank: z.boolean().default(true).describe('Whether to rerank results with Voyage AI reranker'),
+  filter: z.record(z.string(), z.unknown()).optional().describe("MongoDB pre-filter for vector search (e.g., { 'metadata.type': 'api-doc' })"),
+};
+
+/** vai_search input schema */
+const searchSchema = {
+  query: z.string().min(1).max(5000).describe('Search query text'),
+  db: z.string().optional().describe('MongoDB database name'),
+  collection: z.string().optional().describe('Collection with embedded documents'),
+  limit: z.number().int().min(1).max(100).default(10).describe('Maximum results to return'),
+  model: z.string().optional().describe('Voyage AI embedding model'),
+  filter: z.record(z.string(), z.unknown()).optional().describe('MongoDB pre-filter for vector search'),
+};
+
+/** vai_rerank input schema */
+const rerankSchema = {
+  query: z.string().min(1).max(5000).describe('The query to rank documents against'),
+  documents: z.array(z.string()).min(1).max(100).describe('Array of document texts to rerank'),
+  model: z.enum(['rerank-2.5', 'rerank-2.5-lite']).default('rerank-2.5')
+    .describe('Reranking model: rerank-2.5 (accurate) or rerank-2.5-lite (fast)'),
+};
+
+/** vai_embed input schema */
+const embedSchema = {
+  text: z.string().min(1).max(32000).describe('Text to embed'),
+  model: z.string().default('voyage-4-large').describe('Voyage AI embedding model'),
+  inputType: z.enum(['document', 'query']).default('query')
+    .describe('Whether this text is a document or a query (affects embedding)'),
+  dimensions: z.number().int().optional().describe('Output dimensions (512 or 1024 for Matryoshka models)'),
+};
+
+/** vai_similarity input schema */
+const similaritySchema = {
+  text1: z.string().min(1).max(32000).describe('First text'),
+  text2: z.string().min(1).max(32000).describe('Second text'),
+  model: z.string().default('voyage-4-large').describe('Voyage AI embedding model'),
+};
+
+/** vai_collections input schema */
+const collectionsSchema = {
+  db: z.string().optional().describe('Database to list collections from. Uses vai config default if omitted.'),
+};
+
+/** vai_models input schema */
+const modelsSchema = {
+  category: z.enum(['embedding', 'rerank', 'all']).default('all').describe('Filter by model category'),
+};
+
+/** vai_topics input schema */
+const topicsSchema = {
+  search: z.string().optional().describe('Optional search term to filter topics. Omit to list all topics.'),
+};
+
+/** vai_explain input schema */
+const explainSchema = {
+  topic: z.string().describe('Topic to explain — supports fuzzy matching. Use vai_topics to discover all available topics.'),
+};
+
+/** vai_estimate input schema */
+const estimateSchema = {
+  docs: z.number().int().min(1).describe('Number of documents to embed'),
+  queries: z.number().int().min(0).default(0).describe('Number of queries per month'),
+  months: z.number().int().min(1).max(60).default(12).describe('Time horizon in months'),
+};
+
+/** vai_ingest input schema */
+const ingestSchema = {
+  text: z.string().min(1).describe('Document text to ingest'),
+  db: z.string().optional().describe('MongoDB database name'),
+  collection: z.string().optional().describe('Collection to store documents in'),
+  source: z.string().optional().describe('Source identifier (e.g., filename, URL) for citation purposes'),
+  metadata: z.record(z.string(), z.unknown()).optional().describe('Additional metadata to store with the document'),
+  chunkStrategy: z.enum(['fixed', 'sentence', 'paragraph', 'recursive', 'markdown']).default('recursive')
+    .describe('Text chunking strategy'),
+  chunkSize: z.number().int().min(100).max(8000).default(512).describe('Target chunk size in characters'),
+  model: z.string().default('voyage-4-large').describe('Voyage AI embedding model'),
+};
+
+module.exports = {
+  querySchema,
+  searchSchema,
+  rerankSchema,
+  embedSchema,
+  similaritySchema,
+  collectionsSchema,
+  modelsSchema,
+  topicsSchema,
+  explainSchema,
+  estimateSchema,
+  ingestSchema,
+};

package/src/mcp/server.js

@@ -0,0 +1,162 @@
+'use strict';
+
+const { McpServer } = require('@modelcontextprotocol/sdk/server/mcp.js');
+const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js');
+const schemas = require('./schemas');
+const { registerRetrievalTools } = require('./tools/retrieval');
+const { registerEmbeddingTools } = require('./tools/embedding');
+const { registerManagementTools } = require('./tools/management');
+const { registerUtilityTools } = require('./tools/utility');
+const { registerIngestTool } = require('./tools/ingest');
+
+const VERSION = require('../../package.json').version;
+
+/**
+ * Create and configure the MCP server with all tools registered.
+ * @returns {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer}
+ */
+function createServer() {
+  const server = new McpServer({
+    name: 'vai-mcp-server',
+    version: VERSION,
+  });
+
+  // Register all tool domains
+  registerRetrievalTools(server, schemas);
+  registerEmbeddingTools(server, schemas);
+  registerManagementTools(server, schemas);
+  registerUtilityTools(server, schemas);
+  registerIngestTool(server, schemas);
+
+  return server;
+}
+
+/**
+ * Run the MCP server with stdio transport.
+ * The server reads JSON-RPC from stdin and writes to stdout.
+ */
+async function runStdioServer() {
+  const server = createServer();
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+
+  if (process.env.VAI_MCP_VERBOSE) {
+    process.stderr.write(`vai MCP server v${VERSION} running on stdio\n`);
+  }
+}
+
+/**
+ * Run the MCP server with HTTP transport (Streamable HTTP).
+ * @param {object} options
+ * @param {number} options.port
+ * @param {string} options.host
+ */
+async function runHttpServer({ port = 3100, host = '127.0.0.1' } = {}) {
+  const express = require('express');
+  const { StreamableHTTPServerTransport } = require('@modelcontextprotocol/sdk/server/streamableHttp.js');
+  const { getConfigValue } = require('../lib/config');
+  const crypto = require('crypto');
+
+  const app = express();
+  app.use(express.json({ limit: '5mb' }));
+
+  // Load server API keys
+  const serverKeys = getConfigValue('mcp-server-keys') || [];
+  const envKey = process.env.VAI_MCP_SERVER_KEY;
+  const allKeys = envKey ? [...serverKeys, envKey] : serverKeys;
+  const requireAuth = allKeys.length > 0;
+
+  /** Bearer token authentication middleware */
+  function authenticateRequest(req, res, next) {
+    if (!requireAuth) return next();
+    const authHeader = req.headers.authorization;
+    if (!authHeader || !authHeader.startsWith('Bearer ')) {
+      return res.status(401).json({ error: 'Missing or invalid Authorization header' });
+    }
+    const token = authHeader.slice(7);
+    if (!allKeys.includes(token)) {
+      return res.status(401).json({ error: 'Invalid API key' });
+    }
+    next();
+  }
+
+  // Health endpoint (unauthenticated)
+  const startTime = Date.now();
+  app.get('/health', async (_req, res) => {
+    const health = {
+      status: 'ok',
+      version: VERSION,
+      uptime: Math.floor((Date.now() - startTime) / 1000),
+      voyageAi: 'unknown',
+      mongodb: 'unknown',
+    };
+
+    // Check Voyage AI connectivity
+    try {
+      const { getConfigValue } = require('../lib/config');
+      const hasKey = !!(process.env.VOYAGE_API_KEY || getConfigValue('apiKey'));
+      health.voyageAi = hasKey ? 'configured' : 'not configured';
+    } catch {
+      health.voyageAi = 'not configured';
+    }
+
+    // Check MongoDB connectivity
+    try {
+      const { getConfigValue } = require('../lib/config');
+      const hasUri = !!(process.env.MONGODB_URI || getConfigValue('mongodbUri'));
+      health.mongodb = hasUri ? 'configured' : 'not configured';
+    } catch {
+      health.mongodb = 'not configured';
+    }
+
+    res.json(health);
+  });
+
+  // MCP endpoint — stateless per-request transport
+  app.post('/mcp', authenticateRequest, async (req, res) => {
+    const server = createServer();
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: undefined, // stateless
+    });
+    res.on('close', () => transport.close());
+    await server.connect(transport);
+    await transport.handleRequest(req, res, req.body);
+  });
+
+  // Handle GET/DELETE for SSE (required by MCP spec for session management)
+  app.get('/mcp', (_req, res) => {
+    res.status(405).json({ error: 'Method not allowed. Use POST for MCP requests.' });
+  });
+  app.delete('/mcp', (_req, res) => {
+    res.status(405).json({ error: 'Method not allowed. Stateless server — no sessions to delete.' });
+  });
+
+  app.listen(port, host, () => {
+    const msg = `vai MCP server v${VERSION} running on http://${host}:${port}/mcp`;
+    if (process.env.VAI_MCP_VERBOSE) {
+      process.stderr.write(msg + '\n');
+      process.stderr.write(`Authentication: ${requireAuth ? 'enabled' : 'disabled (no keys configured)'}\n`);
+      process.stderr.write(`Health check: http://${host}:${port}/health\n`);
+    }
+    console.log(msg);
+  });
+}
+
+/**
+ * Generate a new MCP server API key and store it in config.
+ */
+function generateKey() {
+  const crypto = require('crypto');
+  const { getConfigValue, setConfigValue } = require('../lib/config');
+
+  const key = 'vai-mcp-key-' + crypto.randomBytes(24).toString('hex');
+  const keys = getConfigValue('mcp-server-keys') || [];
+  keys.push(key);
+  setConfigValue('mcp-server-keys', keys);
+
+  console.log(key);
+  console.log(`\nStored in ~/.vai/config.json. Total keys: ${keys.length}`);
+  console.log('Set as VAI_MCP_SERVER_KEY env var or use in client Authorization header.');
+}
+
+module.exports = { createServer, runStdioServer, runHttpServer, generateKey };