voyageai-cli 1.6.1 → 1.8.0

package/test/commands/benchmark.test.js

@@ -0,0 +1,252 @@
+'use strict';
+
+const { describe, it } = require('node:test');
+const assert = require('node:assert/strict');
+const { Command } = require('commander');
+const { registerBenchmark } = require('../../src/commands/benchmark');
+
+describe('benchmark command', () => {
+  it('registers as benchmark with bench alias', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    assert.ok(benchCmd, 'benchmark command should be registered');
+    assert.ok(benchCmd.aliases().includes('bench'), 'should have "bench" alias');
+  });
+
+  it('has embed subcommand', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    assert.ok(embedSub, 'embed subcommand should be registered');
+  });
+
+  it('embed has --models option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--models'), 'should have --models option');
+  });
+
+  it('embed has --rounds option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--rounds'), 'should have --rounds option');
+  });
+
+  it('embed has --input option for custom text', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--input'), 'should have --input option');
+  });
+
+  it('embed has --file option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--file'), 'should have --file option');
+  });
+
+  it('embed has --json and --quiet options', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--json'), 'should have --json option');
+    assert.ok(optionNames.includes('--quiet'), 'should have --quiet option');
+  });
+
+  it('embed has --dimensions option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--dimensions'), 'should have --dimensions option');
+  });
+
+  it('has rerank subcommand', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const rerankSub = benchCmd.commands.find(c => c.name() === 'rerank');
+    assert.ok(rerankSub, 'rerank subcommand should be registered');
+  });
+
+  it('rerank has --models and --rounds options', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const rerankSub = benchCmd.commands.find(c => c.name() === 'rerank');
+    const optionNames = rerankSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--models'), 'should have --models option');
+    assert.ok(optionNames.includes('--rounds'), 'should have --rounds option');
+  });
+
+  it('rerank has --query option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const rerankSub = benchCmd.commands.find(c => c.name() === 'rerank');
+    const optionNames = rerankSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--query'), 'should have --query option');
+  });
+
+  it('rerank has --documents-file and --top-k options', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const rerankSub = benchCmd.commands.find(c => c.name() === 'rerank');
+    const optionNames = rerankSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--documents-file'), 'should have --documents-file option');
+    assert.ok(optionNames.includes('--top-k'), 'should have --top-k option');
+  });
+
+  it('has similarity subcommand', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const simSub = benchCmd.commands.find(c => c.name() === 'similarity');
+    assert.ok(simSub, 'similarity subcommand should be registered');
+  });
+
+  it('similarity has --models, --query, --file, --top-k options', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const simSub = benchCmd.commands.find(c => c.name() === 'similarity');
+    const optionNames = simSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--models'), 'should have --models');
+    assert.ok(optionNames.includes('--query'), 'should have --query');
+    assert.ok(optionNames.includes('--file'), 'should have --file');
+    assert.ok(optionNames.includes('--top-k'), 'should have --top-k');
+  });
+
+  it('has cost subcommand', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const costSub = benchCmd.commands.find(c => c.name() === 'cost');
+    assert.ok(costSub, 'cost subcommand should be registered');
+  });
+
+  it('cost has --models, --tokens, --volumes options', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const costSub = benchCmd.commands.find(c => c.name() === 'cost');
+    const optionNames = costSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--models'), 'should have --models');
+    assert.ok(optionNames.includes('--tokens'), 'should have --tokens');
+    assert.ok(optionNames.includes('--volumes'), 'should have --volumes');
+  });
+
+  it('has batch subcommand', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const batchSub = benchCmd.commands.find(c => c.name() === 'batch');
+    assert.ok(batchSub, 'batch subcommand should be registered');
+  });
+
+  it('batch has --model, --batch-sizes, --rounds options', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const batchSub = benchCmd.commands.find(c => c.name() === 'batch');
+    const optionNames = batchSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--model'), 'should have --model');
+    assert.ok(optionNames.includes('--batch-sizes'), 'should have --batch-sizes');
+    assert.ok(optionNames.includes('--rounds'), 'should have --rounds');
+  });
+
+  it('all subcommands have --json output option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+
+    for (const sub of benchCmd.commands) {
+      if (sub.name() === 'help') continue;
+      const optionNames = sub.options.map(o => o.long);
+      assert.ok(optionNames.includes('--json'), `${sub.name()} should have --json option`);
+    }
+  });
+
+  it('all subcommands have --quiet option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+
+    for (const sub of benchCmd.commands) {
+      if (sub.name() === 'help') continue;
+      const optionNames = sub.options.map(o => o.long);
+      assert.ok(optionNames.includes('--quiet'), `${sub.name()} should have --quiet option`);
+    }
+  });
+
+  it('embed defaults models to voyage-4-large,voyage-4,voyage-4-lite', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const modelsOpt = embedSub.options.find(o => o.long === '--models');
+    assert.equal(modelsOpt.defaultValue, 'voyage-4-large,voyage-4,voyage-4-lite');
+  });
+
+  it('rerank defaults models to rerank-2.5,rerank-2.5-lite', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const rerankSub = benchCmd.commands.find(c => c.name() === 'rerank');
+    const modelsOpt = rerankSub.options.find(o => o.long === '--models');
+    assert.equal(modelsOpt.defaultValue, 'rerank-2.5,rerank-2.5-lite');
+  });
+
+  it('cost defaults volumes to 100,1000,10000,100000', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const costSub = benchCmd.commands.find(c => c.name() === 'cost');
+    const volOpt = costSub.options.find(o => o.long === '--volumes');
+    assert.equal(volOpt.defaultValue, '100,1000,10000,100000');
+  });
+
+  it('embed has --save option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const embedSub = benchCmd.commands.find(c => c.name() === 'embed');
+    const optionNames = embedSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--save'), 'should have --save option');
+  });
+
+  it('rerank has --save option', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const rerankSub = benchCmd.commands.find(c => c.name() === 'rerank');
+    const optionNames = rerankSub.options.map(o => o.long);
+    assert.ok(optionNames.includes('--save'), 'should have --save option');
+  });
+
+  it('batch defaults batch-sizes to 1,5,10,25,50', () => {
+    const program = new Command();
+    registerBenchmark(program);
+    const benchCmd = program.commands.find(c => c.name() === 'benchmark');
+    const batchSub = benchCmd.commands.find(c => c.name() === 'batch');
+    const sizesOpt = batchSub.options.find(o => o.long === '--batch-sizes');
+    assert.equal(sizesOpt.defaultValue, '1,5,10,25,50');
+  });
+});

package/test/commands/playground.test.js

@@ -0,0 +1,137 @@
+'use strict';
+
+const { describe, it, after } = require('node:test');
+const assert = require('node:assert/strict');
+const http = require('http');
+const { Command } = require('commander');
+const { registerPlayground, createPlaygroundServer } = require('../../src/commands/playground');
+
+describe('playground command', () => {
+  let server;
+  let port;
+
+  // Start server once for all tests
+  const serverReady = new Promise((resolve) => {
+    server = createPlaygroundServer();
+    server.listen(0, () => {
+      port = server.address().port;
+      resolve();
+    });
+  });
+
+  after(() => {
+    return new Promise((resolve) => {
+      if (server) server.close(resolve);
+      else resolve();
+    });
+  });
+
+  it('registers correctly on a program', () => {
+    const program = new Command();
+    registerPlayground(program);
+    const cmd = program.commands.find(c => c.name() === 'playground');
+    assert.ok(cmd, 'playground command should be registered');
+    assert.ok(cmd.description().includes('playground') || cmd.description().includes('Playground') || cmd.description().includes('web'),
+      'should have a relevant description');
+  });
+
+  it('serves HTML on GET /', async () => {
+    await serverReady;
+    const body = await httpGet(`http://localhost:${port}/`);
+    assert.ok(body.includes('<!DOCTYPE html>'), 'should return HTML');
+    assert.ok(body.includes('Voyage AI Playground'), 'should include playground title');
+  });
+
+  it('returns JSON from GET /api/models', async () => {
+    await serverReady;
+    const body = await httpGet(`http://localhost:${port}/api/models`);
+    const data = JSON.parse(body);
+    assert.ok(Array.isArray(data.models), 'models should be an array');
+    assert.ok(data.models.length > 0, 'should have at least one model');
+    assert.ok(data.models.every(m => !m.legacy), 'should not include legacy models');
+    // Check a known model exists
+    const names = data.models.map(m => m.name);
+    assert.ok(names.includes('voyage-4-large'), 'should include voyage-4-large');
+  });
+
+  it('returns config with hasKey boolean from GET /api/config', async () => {
+    await serverReady;
+    const body = await httpGet(`http://localhost:${port}/api/config`);
+    const data = JSON.parse(body);
+    assert.ok(typeof data.hasKey === 'boolean', 'hasKey should be a boolean');
+    assert.ok(typeof data.baseUrl === 'string', 'baseUrl should be a string');
+    // Should never expose the actual key
+    assert.ok(!data.apiKey, 'should not expose apiKey');
+    assert.ok(!data.key, 'should not expose key');
+  });
+
+  it('returns 404 for unknown routes', async () => {
+    await serverReady;
+    const { statusCode } = await httpGetFull(`http://localhost:${port}/nonexistent`);
+    assert.equal(statusCode, 404);
+  });
+
+  it('returns 400 for POST /api/embed with invalid body', async () => {
+    await serverReady;
+    const { statusCode, body } = await httpPostFull(`http://localhost:${port}/api/embed`, { texts: 'not-an-array' });
+    assert.equal(statusCode, 400);
+    const data = JSON.parse(body);
+    assert.ok(data.error, 'should return an error message');
+  });
+});
+
+/**
+ * Simple HTTP GET that returns the body string.
+ */
+function httpGet(url) {
+  return new Promise((resolve, reject) => {
+    http.get(url, (res) => {
+      const chunks = [];
+      res.on('data', c => chunks.push(c));
+      res.on('end', () => resolve(Buffer.concat(chunks).toString()));
+      res.on('error', reject);
+    }).on('error', reject);
+  });
+}
+
+/**
+ * HTTP GET returning { statusCode, body }.
+ */
+function httpGetFull(url) {
+  return new Promise((resolve, reject) => {
+    http.get(url, (res) => {
+      const chunks = [];
+      res.on('data', c => chunks.push(c));
+      res.on('end', () => resolve({ statusCode: res.statusCode, body: Buffer.concat(chunks).toString() }));
+      res.on('error', reject);
+    }).on('error', reject);
+  });
+}
+
+/**
+ * HTTP POST returning { statusCode, body }.
+ */
+function httpPostFull(url, data) {
+  return new Promise((resolve, reject) => {
+    const payload = JSON.stringify(data);
+    const parsed = new URL(url);
+    const req = http.request({
+      hostname: parsed.hostname,
+      port: parsed.port,
+      path: parsed.pathname,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(payload),
+      },
+    }, (res) => {
+      const chunks = [];
+      res.on('data', c => chunks.push(c));
+      res.on('end', () => resolve({ statusCode: res.statusCode, body: Buffer.concat(chunks).toString() }));
+      res.on('error', reject);
+    });
+    req.on('error', reject);
+    req.write(payload);
+    req.end();
+  });
+}

package/test/lib/explanations.test.js

@@ -17,6 +17,7 @@ describe('explanations', () => {
     'api-keys',
     'api-access',
     'batch-processing',
+    'benchmarking',
   ];
 
   it('has all expected concepts', () => {