voyageai-cli 1.30.1 → 1.30.3

package/src/lib/workflow.js

@@ -16,6 +16,7 @@ const {
 const VAI_TOOLS = new Set([
   'query', 'search', 'rerank', 'embed', 'similarity',
   'ingest', 'collections', 'models', 'explain', 'estimate',
+  'code_index', 'code_search', 'code_query', 'code_find_similar', 'code_status',
 ]);
 
 const CONTROL_FLOW_TOOLS = new Set(['merge', 'filter', 'transform', 'generate', 'conditional', 'loop', 'template']);
@@ -98,6 +99,25 @@ function validateWorkflow(definition, { mode = 'strict' } = {}) {
     }
   }
 
+  // Validate formatters section (optional)
+  if (definition.formatters) {
+    if (typeof definition.formatters !== 'object' || Array.isArray(definition.formatters)) {
+      addIssue('error', null, 'INVALID_FORMATTERS', '"formatters" must be a plain object');
+    } else {
+      const f = definition.formatters;
+      const validFormats = ['json', 'table', 'markdown', 'text', 'csv'];
+      if (f.default && !validFormats.includes(f.default)) {
+        addIssue('warning', null, 'INVALID_FORMATTER_DEFAULT', `formatters.default "${f.default}" is not a recognized format (${validFormats.join(', ')})`);
+      }
+      if (f.columns && !Array.isArray(f.columns)) {
+        addIssue('error', null, 'INVALID_FORMATTER_COLUMNS', 'formatters.columns must be an array of strings');
+      }
+      if (f.title && typeof f.title !== 'string') {
+        addIssue('error', null, 'INVALID_FORMATTER_TITLE', 'formatters.title must be a string');
+      }
+    }
+  }
+
   // Step-level validation
   const stepIds = new Set();
   const duplicateIds = new Set();
@@ -1238,6 +1258,9 @@ async function executeQuery(inputs, defaults) {
       },
     ]).toArray();
 
+    // Track embed usage
+    const _usage = [{ op: 'embed', model: embRes.model, tokens: embRes.usage?.total_tokens || 0 }];
+
     // Rerank if requested and results exist
     if (doRerank && results.length > 0) {
       const documents = results.map(r => r.text || r.content || '');
@@ -1248,15 +1271,17 @@ async function executeQuery(inputs, defaults) {
         documents,
       });
 
+      _usage.push({ op: 'rerank', model: rerankRes.model || inputs.rerankModel || DEFAULT_RERANK_MODEL, tokens: rerankRes.usage?.total_tokens || 0 });
+
       const reranked = (rerankRes.data || []).map(r => ({
         ...results[r.index],
         score: r.relevance_score,
       }));
 
-      return { results: reranked, resultCount: reranked.length };
+      return { results: reranked, resultCount: reranked.length, _usage };
     }
 
-    return { results, resultCount: results.length };
+    return { results, resultCount: results.length, _usage };
   } finally {
     await client.close();
   }
@@ -1295,7 +1320,8 @@ async function executeRerank(inputs) {
     score: r.relevance_score,
   }));
 
-  return { results, resultCount: results.length };
+  const _usage = [{ op: 'rerank', model: res.model || model, tokens: res.usage?.total_tokens || 0 }];
+  return { results, resultCount: results.length, _usage };
 }
 
 /**
@@ -1319,6 +1345,7 @@ async function executeEmbed(inputs, defaults) {
     embedding: res.data[0].embedding,
     model: res.model,
     dimensions: res.data[0].embedding.length,
+    _usage: [{ op: 'embed', model: res.model, tokens: res.usage?.total_tokens || 0 }],
   };
 }
 
@@ -1340,7 +1367,11 @@ async function executeSimilarity(inputs, defaults) {
   const res = await generateEmbeddings([text1, text2], opts);
   const similarity = cosineSimilarity(res.data[0].embedding, res.data[1].embedding);
 
-  return { similarity, model: res.model };
+  return {
+    similarity,
+    model: res.model,
+    _usage: [{ op: 'similarity', model: res.model, tokens: res.usage?.total_tokens || 0 }],
+  };
 }
 
 /**
@@ -1421,6 +1452,7 @@ async function executeIngest(inputs, defaults) {
       source,
       model: embRes.model,
       indexCreated,
+      _usage: [{ op: 'ingest', model: embRes.model, tokens: embRes.usage?.total_tokens || 0 }],
     };
   } finally {
     await client.close();
@@ -1565,17 +1597,72 @@ async function executeGenerate(inputs) {
 
   // Collect streaming response
   let text = '';
+  let llmUsage = { inputTokens: 0, outputTokens: 0 };
   for await (const chunk of provider.chat(messages, { stream: true })) {
-    text += chunk;
+    if (chunk && typeof chunk === 'object' && chunk.__usage) {
+      llmUsage = chunk.__usage;
+    } else {
+      text += chunk;
+    }
   }
 
   return {
     text,
     model: provider.model,
     provider: provider.name,
+    _usage: [{ op: 'llm', model: provider.model, provider: provider.name, inputTokens: llmUsage.inputTokens, outputTokens: llmUsage.outputTokens }],
   };
 }
 
+// ════════════════════════════════════════════════════════════════════
+// Code Search Tool Executors
+// ════════════════════════════════════════════════════════════════════
+
+/**
+ * Execute a code_index step: index a local directory or GitHub repo.
+ */
+async function executeCodeIndex(inputs) {
+  const { handleCodeIndex } = require('../mcp/tools/code-search');
+  const result = await handleCodeIndex(inputs);
+  return result.structuredContent;
+}
+
+/**
+ * Execute a code_search step: semantic search across an indexed codebase.
+ */
+async function executeCodeSearch(inputs) {
+  const { handleCodeSearch } = require('../mcp/tools/code-search');
+  const result = await handleCodeSearch(inputs);
+  return result.structuredContent;
+}
+
+/**
+ * Execute a code_query step: RAG query against indexed code.
+ */
+async function executeCodeQuery(inputs) {
+  const { handleCodeQuery } = require('../mcp/tools/code-search');
+  const result = await handleCodeQuery(inputs);
+  return result.structuredContent;
+}
+
+/**
+ * Execute a code_find_similar step: find code similar to a snippet.
+ */
+async function executeCodeFindSimilar(inputs) {
+  const { handleCodeFindSimilar } = require('../mcp/tools/code-search');
+  const result = await handleCodeFindSimilar(inputs);
+  return result.structuredContent;
+}
+
+/**
+ * Execute a code_status step: check index health.
+ */
+async function executeCodeStatus(inputs) {
+  const { handleCodeStatus } = require('../mcp/tools/code-search');
+  const result = await handleCodeStatus(inputs);
+  return result.structuredContent;
+}
+
 // ════════════════════════════════════════════════════════════════════
 // Step Dispatcher
 // ════════════════════════════════════════════════════════════════════
@@ -1635,6 +1722,18 @@ async function executeStep(step, resolvedInputs, defaults, context) {
     case 'estimate':
       return executeEstimate(resolvedInputs);
 
+    // Code search tools
+    case 'code_index':
+      return executeCodeIndex(resolvedInputs);
+    case 'code_search':
+      return executeCodeSearch(resolvedInputs);
+    case 'code_query':
+      return executeCodeQuery(resolvedInputs);
+    case 'code_find_similar':
+      return executeCodeFindSimilar(resolvedInputs);
+    case 'code_status':
+      return executeCodeStatus(resolvedInputs);
+
     default:
       throw new Error(`Unknown tool: "${step.tool}"`);
   }
@@ -1825,14 +1924,22 @@ async function executeWorkflow(definition, opts = {}) {
         }
 
         const durationMs = Date.now() - stepStart;
-        context[stepId] = { output };
 
+        // Pass full output (with _usage) to onStepComplete for cost tracking
         if (opts.onStepComplete) opts.onStepComplete(stepId, output, durationMs);
 
+        // Strip _usage from context so downstream steps don't receive it
+        let cleanOutput = output;
+        if (output && output._usage) {
+          cleanOutput = { ...output };
+          delete cleanOutput._usage;
+        }
+        context[stepId] = { output: cleanOutput };
+
         stepResults.push({
           id: stepId,
           tool: step.tool,
-          output,
+          output: cleanOutput,
           durationMs,
         });
       } catch (err) {
@@ -1867,6 +1974,7 @@ async function executeWorkflow(definition, opts = {}) {
     steps: stepResults,
     totalTimeMs: Date.now() - startTime,
     layers,
+    formatters: definition.formatters || null,
   };
 }
 
@@ -1894,9 +2002,10 @@ function coerceInput(value, type) {
  * prompt users for missing inputs before execution.
  *
  * @param {object} definition - Workflow definition with an `inputs` property
+ * @param {object} [cachedInputs] - Previously cached input values (from last run)
  * @returns {import('./wizard').Step[]}
  */
-function buildInputSteps(definition) {
+function buildInputSteps(definition, cachedInputs = {}) {
   if (!definition.inputs) return [];
   return Object.entries(definition.inputs).map(([key, spec]) => ({
     id: key,
@@ -1905,6 +2014,10 @@ function buildInputSteps(definition) {
     required: !!spec.required,
     placeholder: spec.type === 'number' ? 'number' : (spec.type || 'string'),
     defaultValue: spec.default !== undefined ? String(spec.default) : undefined,
+    getDefault: () => {
+      if (key in cachedInputs) return String(cachedInputs[key]);
+      return spec.default !== undefined ? String(spec.default) : undefined;
+    },
     validate: (val) => {
       if (spec.type === 'number' && val && isNaN(Number(val))) {
         return 'Must be a number';
@@ -2063,6 +2176,9 @@ module.exports = {
   executeIngest,
   executeAggregate,
 
+  // Dependency graph
+  buildDependencyGraph,
+
   // Main execution
   executeStep,
   executeWorkflow,

package/src/mcp/schemas/index.js

@@ -124,6 +124,140 @@ const explainCodeSchema = {
   model: z.string().optional().describe('Voyage AI embedding model'),
 };
 
+/** vai_code_index input schema */
+const codeIndexSchema = {
+  source: z.string().min(1).describe(
+    'Local directory path or GitHub repo URL (e.g., "/path/to/project" or "https://github.com/org/repo")'
+  ),
+  db: z.string().optional().describe('MongoDB database name. Default: "vai_code_search"'),
+  collection: z.string().optional().describe(
+    'Collection name. Auto-derived from project name if omitted.'
+  ),
+  model: z.string().optional().describe(
+    'Embedding model. Default: auto-detected (voyage-code-3 for code, voyage-4-large for docs)'
+  ),
+  branch: z.string().default('main').describe('Git branch for remote repos'),
+  maxFiles: z.number().int().min(1).max(10000).default(5000)
+    .describe('Maximum files to index'),
+  maxFileSize: z.number().int().min(1000).max(1000000).default(100000)
+    .describe('Maximum file size in bytes'),
+  chunkSize: z.number().int().min(100).max(4000).default(512)
+    .describe('Target chunk size in characters'),
+  chunkOverlap: z.number().int().min(0).max(500).default(50)
+    .describe('Overlap between chunks in characters'),
+  batchSize: z.number().int().min(1).max(50).default(20)
+    .describe('Files per embedding batch'),
+  refresh: z.boolean().default(false)
+    .describe('Incremental refresh: only re-index changed files'),
+  contentType: z.enum(['code', 'docs', 'config', 'all']).default('code')
+    .describe('Type of content to index'),
+};
+
+/** vai_code_search input schema */
+const codeSearchSchema = {
+  query: z.string().min(1).max(5000).describe(
+    'Natural language search query (e.g., "where do we handle auth timeouts")'
+  ),
+  db: z.string().optional().describe('MongoDB database name'),
+  collection: z.string().optional().describe('Collection with indexed code'),
+  limit: z.number().int().min(1).max(50).default(10)
+    .describe('Maximum number of results'),
+  language: z.string().optional()
+    .describe('Filter by programming language (e.g., "js", "py", "go")'),
+  category: z.enum(['code', 'docs', 'config']).optional()
+    .describe('Filter by content category'),
+  rerank: z.boolean().default(true)
+    .describe('Rerank results with Voyage AI reranker for better relevance'),
+  rerankModel: z.enum(['rerank-2.5', 'rerank-2.5-lite']).default('rerank-2.5')
+    .describe('Reranking model'),
+  model: z.string().optional()
+    .describe('Embedding model for query. Default: voyage-code-3'),
+  filter: z.record(z.string(), z.unknown()).optional()
+    .describe('Additional MongoDB filter on metadata fields'),
+};
+
+/** vai_code_query input schema */
+const codeQuerySchema = {
+  query: z.string().min(1).max(5000).describe(
+    'Question about the codebase (e.g., "how does the auth middleware work")'
+  ),
+  db: z.string().optional().describe('MongoDB database name'),
+  collection: z.string().optional().describe('Collection with indexed code'),
+  limit: z.number().int().min(1).max(20).default(5)
+    .describe('Maximum results (fewer, higher quality)'),
+  language: z.string().optional()
+    .describe('Filter by programming language'),
+  model: z.string().optional()
+    .describe('Embedding model. Default: voyage-code-3'),
+  filter: z.record(z.string(), z.unknown()).optional()
+    .describe('Additional MongoDB filter'),
+};
+
+/** vai_code_find_similar input schema */
+const codeFindSimilarSchema = {
+  code: z.string().min(1).max(10000).describe(
+    'Code snippet to find similar implementations for'
+  ),
+  db: z.string().optional().describe('MongoDB database name'),
+  collection: z.string().optional().describe('Collection with indexed code'),
+  limit: z.number().int().min(1).max(50).default(10)
+    .describe('Maximum results'),
+  language: z.string().optional()
+    .describe('Filter by programming language'),
+  model: z.string().optional()
+    .describe('Embedding model. Default: voyage-code-3'),
+  threshold: z.number().min(0).max(1).default(0.5)
+    .describe('Minimum similarity score (0-1)'),
+  filter: z.record(z.string(), z.unknown()).optional()
+    .describe('Additional MongoDB filter'),
+};
+
+/** vai_code_status input schema */
+const codeStatusSchema = {
+  db: z.string().optional().describe('MongoDB database name'),
+  collection: z.string().optional().describe('Collection to check'),
+};
+
+/** vai_generate_workflow input schema */
+const generateWorkflowSchema = {
+  description: z.string().min(1).max(500).describe('Natural language description of the workflow to generate'),
+  category: z.enum(['retrieval', 'analysis', 'ingestion', 'domain-specific', 'utility', 'integration']).optional()
+    .describe('Workflow category'),
+  tools: z.array(z.string()).optional()
+    .describe('Explicit list of tools to include (e.g., ["query", "rerank", "generate"]). If omitted, tools are inferred from the description.'),
+};
+
+/** vai_multimodal_embed input schema */
+const multimodalEmbedSchema = {
+  text: z.string().max(32000).optional().describe('Optional text content to embed alongside media'),
+  image_base64: z.string().optional().describe('Base64 data URL for an image (e.g., data:image/jpeg;base64,...)'),
+  video_base64: z.string().optional().describe('Base64 data URL for a video (e.g., data:video/mp4;base64,...)'),
+  model: z.string().default('voyage-multimodal-3.5').describe('Multimodal embedding model'),
+  inputType: z.enum(['document', 'query']).optional()
+    .describe('Whether this input is a document or a query (affects embedding)'),
+  outputDimension: z.number().int().optional().describe('Output dimensions (256, 512, 1024, or 2048)'),
+};
+
+/** vai_validate_workflow input schema */
+const validateWorkflowSchema = {
+  workflow: z.object({
+    name: z.string().optional(),
+    description: z.string().optional(),
+    version: z.string().optional(),
+    inputs: z.record(z.string(), z.unknown()).optional(),
+    defaults: z.record(z.string(), z.unknown()).optional(),
+    steps: z.array(z.object({
+      id: z.string(),
+      tool: z.string(),
+      name: z.string().optional(),
+      inputs: z.record(z.string(), z.unknown()).optional(),
+      condition: z.string().optional(),
+      forEach: z.string().optional(),
+    })),
+    output: z.record(z.string(), z.unknown()).optional(),
+  }).describe('The workflow JSON definition to validate'),
+};
+
 module.exports = {
   querySchema,
   searchSchema,
@@ -139,4 +273,12 @@ module.exports = {
   indexWorkspaceSchema,
   searchCodeSchema,
   explainCodeSchema,
+  codeIndexSchema,
+  codeSearchSchema,
+  codeQuerySchema,
+  codeFindSimilarSchema,
+  codeStatusSchema,
+  multimodalEmbedSchema,
+  generateWorkflowSchema,
+  validateWorkflowSchema,
 };

package/src/mcp/server.js

@@ -9,6 +9,8 @@ const { registerManagementTools } = require('./tools/management');
 const { registerUtilityTools } = require('./tools/utility');
 const { registerIngestTool } = require('./tools/ingest');
 const { registerWorkspaceTools } = require('./tools/workspace');
+const { registerCodeSearchTools } = require('./tools/code-search');
+const { registerAuthoringTools } = require('./tools/authoring');
 
 const VERSION = require('../../package.json').version;
 
@@ -29,6 +31,8 @@ function createServer() {
   registerUtilityTools(server, schemas);
   registerIngestTool(server, schemas);
   registerWorkspaceTools(server, schemas);
+  registerCodeSearchTools(server, schemas);
+  registerAuthoringTools(server, schemas);
 
   return server;
 }
@@ -68,7 +72,7 @@ async function runHttpServer({ port = 3100, host = '127.0.0.1', sse = false } =
   const allKeys = envKey ? [...serverKeys, envKey] : serverKeys;
   const requireAuth = allKeys.length > 0;
 
-  /** Bearer token authentication middleware */
+  /** Bearer token authentication middleware (timing-safe comparison) */
   function authenticateRequest(req, res, next) {
     if (!requireAuth) return next();
     const authHeader = req.headers.authorization;
@@ -76,7 +80,13 @@ async function runHttpServer({ port = 3100, host = '127.0.0.1', sse = false } =
       return res.status(401).json({ error: 'Missing or invalid Authorization header' });
     }
     const token = authHeader.slice(7);
-    if (!allKeys.includes(token)) {
+    const tokenBuf = Buffer.from(token);
+    const match = allKeys.some(key => {
+      const keyBuf = Buffer.from(key);
+      if (keyBuf.length !== tokenBuf.length) return false;
+      return crypto.timingSafeEqual(keyBuf, tokenBuf);
+    });
+    if (!match) {
       return res.status(401).json({ error: 'Invalid API key' });
     }
     next();
@@ -95,7 +105,6 @@ async function runHttpServer({ port = 3100, host = '127.0.0.1', sse = false } =
 
     // Check Voyage AI connectivity
     try {
-      const { getConfigValue } = require('../lib/config');
       const hasKey = !!(process.env.VOYAGE_API_KEY || getConfigValue('apiKey'));
       health.voyageAi = hasKey ? 'configured' : 'not configured';
     } catch {
@@ -104,7 +113,6 @@ async function runHttpServer({ port = 3100, host = '127.0.0.1', sse = false } =
 
     // Check MongoDB connectivity
     try {
-      const { getConfigValue } = require('../lib/config');
       const hasUri = !!(process.env.MONGODB_URI || getConfigValue('mongodbUri'));
       health.mongodb = hasUri ? 'configured' : 'not configured';
     } catch {
@@ -175,3 +183,8 @@ function generateKey() {
 }
 
 module.exports = { createServer, runStdioServer, runHttpServer, generateKey };
+
+// Allow direct execution: `node src/mcp/server.js`
+if (require.main === module) {
+  runStdioServer();
+}