ruvector 0.2.21 → 0.2.23

package/README.md

@@ -10,7 +10,7 @@
 
 **The fastest vector database for Node.js—built in Rust, runs everywhere**
 
-Ruvector is a self-learning vector database with **enterprise-grade semantic search**, hybrid retrieval (sparse + dense), Graph RAG, FlashAttention-3, and billion-scale DiskANN — all in a single npm package. Unlike cloud-only solutions or Python-first databases, Ruvector is designed for JavaScript/TypeScript developers who need **blazing-fast vector search** without external services.
+Ruvector is a self-learning vector database with **enterprise-grade semantic search**, hybrid retrieval (sparse + dense), Graph RAG, FlashAttention-3, and DiskANN — all in a single npm package. Unlike cloud-only solutions or Python-first databases, Ruvector is designed for JavaScript/TypeScript developers who need **blazing-fast vector search** without external services.
 
 > 🚀 **Sub-millisecond queries** • 🎯 **52,000+ inserts/sec** • 💾 **~50 bytes per vector** • 🌍 **Runs anywhere** • 🧠 **859 tests passing**
 
@@ -40,7 +40,7 @@ npx ruvector hooks init --pretrain --build-agents quality
 - **FlashAttention-3** — IO-aware tiled attention, O(N) memory instead of O(N^2)
 - **Graph RAG** — Knowledge graph + community detection for multi-hop queries (30-60% improvement)
 - **Hybrid Search** — Sparse + dense vectors with RRF fusion (20-49% better retrieval)
-- **DiskANN / Vamana** — Billion-scale SSD-backed ANN with <10ms latency
+- **DiskANN / Vamana** — SSD-friendly ANN graph with PQ compression for large-scale search
 - **ColBERT Multi-Vector** — Per-token late interaction retrieval (MaxSim)
 - **Matryoshka Embeddings** — Adaptive-dimension search with funnel/cascade modes
 - **MLA** — Multi-Head Latent Attention with ~93% KV-cache compression (DeepSeek-V2/V3)

package/bin/cli.js

@@ -8935,13 +8935,52 @@ const decompileCmd = program
   .option('-q, --quiet', 'Suppress progress output')
   .option('--version-pkg <ver>', 'Package version (alternative to @version syntax)')
   .option('--diff <version>', 'Compare against another version')
+  .option('--model <file>', 'Decompile LLM model weight file (.gguf, .safetensors)')
+  .option('--api <model-id>', 'Probe remote LLM API to discover architecture')
+  .option('--api-key <key>', 'API key for --api mode (or use env vars)')
   .action(async (target, opts) => {
+    // Model weight decompilation mode (ADR-138)
+    if (opts.model) {
+      try {
+        const modelDecompiler = require('../src/decompiler/model-decompiler.js');
+        const result = await modelDecompiler.decompileModelFile(opts.model);
+        if (opts.json) {
+          console.log(JSON.stringify(result, null, 2));
+        } else {
+          modelDecompiler.printModelResult(result);
+        }
+      } catch (err) {
+        console.error(chalk.red(`Model decompilation failed: ${err.message}`));
+        process.exit(1);
+      }
+      return;
+    }
+
+    // API probing mode (ADR-138)
+    if (opts.api) {
+      try {
+        const apiProber = require('../src/decompiler/api-prober.js');
+        const result = await apiProber.probeModel(opts.api, { apiKey: opts.apiKey });
+        if (opts.json) {
+          console.log(JSON.stringify(result, null, 2));
+        } else {
+          apiProber.printProbeResult(result);
+        }
+      } catch (err) {
+        console.error(chalk.red(`API probe failed: ${err.message}`));
+        process.exit(1);
+      }
+      return;
+    }
+
     if (!target) {
       console.log(chalk.cyan('\nUsage:'));
       console.log(chalk.white('  ruvector decompile <package>           Decompile npm package'));
       console.log(chalk.white('  ruvector decompile <pkg>@<ver>         Specific version'));
       console.log(chalk.white('  ruvector decompile ./bundle.js         Local file'));
       console.log(chalk.white('  ruvector decompile https://unpkg.com/x URL'));
+      console.log(chalk.white('  ruvector decompile --model <file.gguf> LLM weight file'));
+      console.log(chalk.white('  ruvector decompile --api <model-id>    Probe remote API'));
       console.log(chalk.dim('\nOptions:'));
       console.log(chalk.dim('  -o, --output <dir>     Output directory'));
       console.log(chalk.dim('  -f, --format <type>    modules | single | json'));
@@ -8949,6 +8988,9 @@ const decompileCmd = program
       console.log(chalk.dim('  --no-witness           Skip witness chain'));
       console.log(chalk.dim('  --json                 JSON to stdout'));
       console.log(chalk.dim('  --diff <version>       Diff against another version'));
+      console.log(chalk.dim('  --model <file>         Decompile .gguf/.safetensors'));
+      console.log(chalk.dim('  --api <model-id>       Probe LLM API'));
+      console.log(chalk.dim('  --api-key <key>        API key (or set env var)'));
       console.log('');
       return;
     }
@@ -9061,6 +9103,124 @@ const decompileCmd = program
     }
   });
 
+// =============================================================================
+// Optimize Commands — Claude Code profile optimization (ADR-139)
+// =============================================================================
+
+const optimizeCmd = program.command('optimize')
+  .description('Optimize Claude Code configuration per task type (ADR-139)')
+  .option('-p, --profile <type>', 'Task profile: coding|research|quickfix|planning|background|swarm|review|ci')
+  .option('-s, --show', 'Show current optimization status')
+  .option('-l, --list', 'List all available profiles')
+  .option('--generate-settings', 'Output optimal .claude/settings.json')
+  .option('--detect <prompt>', 'Auto-detect task type from a prompt')
+  .option('--apply', 'Apply profile env vars to current process (for hooks)')
+  .option('--json', 'JSON output')
+  .action(async (opts) => {
+    let optimizerMod;
+    try {
+      optimizerMod = require('../src/optimizer/index.js');
+    } catch (e) {
+      console.error(chalk.red('Error: Failed to load optimizer module.'));
+      console.error(chalk.dim(`  ${e.message}`));
+      process.exit(1);
+    }
+
+    // --list: show all profiles
+    if (opts.list) {
+      const profiles = optimizerMod.listProfiles();
+      if (opts.json) {
+        const data = {};
+        for (const name of profiles) {
+          data[name] = optimizerMod.getProfile(name);
+        }
+        console.log(JSON.stringify(data, null, 2));
+        return;
+      }
+      console.log(chalk.bold.cyan('\n  RVAgent Optimizer Profiles (ADR-139)\n'));
+      console.log(chalk.dim('  Based on decompiled Claude Code v2.1.91 intelligence\n'));
+      for (const name of profiles) {
+        const p = optimizerMod.getProfile(name);
+        const envCount = Object.keys(p.env).length;
+        console.log(`  ${chalk.bold.white(name.padEnd(12))} ${chalk.dim(p.description)}`);
+        console.log(chalk.dim(`${''.padEnd(14)}Permission: ${p.permissionMode}, Env vars: ${envCount}`));
+      }
+      console.log('');
+      console.log(chalk.dim('  Usage: ruvector optimize --profile <type>'));
+      console.log(chalk.dim('         ruvector optimize --generate-settings --profile coding'));
+      console.log('');
+      return;
+    }
+
+    // --detect: infer task type from prompt
+    if (opts.detect) {
+      const detected = optimizerMod.detectTaskType(opts.detect);
+      if (opts.json) {
+        console.log(JSON.stringify({ prompt: opts.detect, taskType: detected }));
+        return;
+      }
+      console.log(chalk.cyan(`  Detected task type: ${chalk.bold(detected)}`));
+      return;
+    }
+
+    // Determine profile to use
+    const profileName = opts.profile || 'coding';
+    const profile = optimizerMod.getProfile(profileName);
+
+    if (!profile) {
+      console.error(chalk.red(`  Unknown profile: ${profileName}`));
+      console.error(chalk.yellow(`  Available: ${optimizerMod.listProfiles().join(', ')}`));
+      process.exit(1);
+    }
+
+    // --generate-settings: output settings.json
+    if (opts.generateSettings) {
+      const { generateSettings, formatSettings } = require('../src/optimizer/settings-generator.js');
+      const settings = generateSettings({ ...profile, taskType: profileName });
+      if (opts.json) {
+        console.log(formatSettings(settings));
+      } else {
+        console.log(chalk.bold.cyan(`\n  Generated settings.json for profile: ${profileName}\n`));
+        console.log(formatSettings(settings));
+        console.log('');
+        console.log(chalk.dim('  Save to .claude/settings.json to activate.'));
+        console.log('');
+      }
+      return;
+    }
+
+    // --show: display profile details
+    if (opts.show) {
+      if (opts.json) {
+        console.log(JSON.stringify({ profile: profileName, ...profile }, null, 2));
+        return;
+      }
+      console.log(chalk.bold.cyan(`\n  Profile: ${profileName}\n`));
+      console.log(`  ${chalk.dim('Description:')} ${profile.description}`);
+      console.log(`  ${chalk.dim('Permission:')}  ${profile.permissionMode}`);
+      console.log(`  ${chalk.dim('Env vars:')}`);
+      for (const [key, val] of Object.entries(profile.env)) {
+        console.log(`    ${chalk.white(key)}=${chalk.green(val)}`);
+      }
+      console.log('');
+      return;
+    }
+
+    // --apply or default: apply env vars
+    const result = optimizerMod.applyProfile(profileName);
+    if (opts.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    console.log(chalk.bold.cyan(`\n  Applied profile: ${profileName}`));
+    console.log(chalk.dim(`  ${profile.description}\n`));
+    for (const [key, val] of Object.entries(result.applied)) {
+      console.log(`  ${chalk.green('+')} ${key}=${val}`);
+    }
+    console.log(`\n  ${chalk.dim('Permission mode:')} ${result.permissionMode}`);
+    console.log('');
+  });
+
 program.parse();
 
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "ruvector",
-  "version": "0.2.21",
-  "description": "Self-learning vector database for Node.js — hybrid search, Graph RAG, FlashAttention-3, DiskANN, 50+ attention mechanisms",
+  "version": "0.2.23",
+  "description": "Self-learning vector database for Node.js \u2014 hybrid search, Graph RAG, FlashAttention-3, HNSW, 50+ attention mechanisms",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "bin": {
@@ -47,7 +47,7 @@
     "mcp",
     "edge-computing",
     "graph-rag",
-    "diskann",
+    "hnsw",
     "hybrid-search",
     "colbert",
     "turboquant",
@@ -97,7 +97,8 @@
   "peerDependencies": {
     "@ruvector/pi-brain": ">=0.1.0",
     "@ruvector/router": ">=0.1.0",
-    "@ruvector/ruvllm": ">=2.0.0"
+    "@ruvector/ruvllm": ">=2.0.0",
+    "@ruvector/diskann": ">=0.1.0"
   },
   "peerDependenciesMeta": {
     "@ruvector/pi-brain": {
@@ -108,9 +109,12 @@
     },
     "@ruvector/router": {
       "optional": true
+    },
+    "@ruvector/diskann": {
+      "optional": true
     }
   },
   "engines": {
     "node": ">=18.0.0"
   }
-}
+}

package/src/decompiler/api-prober.js

@@ -0,0 +1,302 @@
+'use strict';
+
+/**
+ * LLM API prober -- discovers model architecture by probing remote APIs.
+ * Detects capabilities, token limits, tokenizer behavior, and model fingerprints.
+ * See ADR-138.
+ */
+
+// ── Provider detection ───────────────────────────────────────────────────
+
+const PROVIDERS = {
+  anthropic: {
+    endpoint: 'https://api.anthropic.com/v1/messages',
+    envKey: 'ANTHROPIC_API_KEY',
+    models: ['claude-sonnet-4-6', 'claude-sonnet-4-20250514', 'claude-haiku-4-20250414', 'claude-opus-4-20250514'],
+  },
+  openai: {
+    endpoint: 'https://api.openai.com/v1/chat/completions',
+    envKey: 'OPENAI_API_KEY',
+    models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo', 'o1', 'o1-mini'],
+  },
+  google: {
+    endpoint: 'https://generativelanguage.googleapis.com/v1beta/models',
+    envKey: 'GOOGLE_AI_API_KEY',
+    models: ['gemini-2.5-flash', 'gemini-2.5-pro', 'gemini-2.0-flash'],
+  },
+};
+
+function detectProvider(modelId) {
+  modelId = modelId.toLowerCase();
+  if (modelId.startsWith('claude')) return 'anthropic';
+  if (modelId.startsWith('gpt') || modelId.startsWith('o1') || modelId.startsWith('o3')) return 'openai';
+  if (modelId.startsWith('gemini')) return 'google';
+  return 'unknown';
+}
+
+// ── Main probe ───────────────────────────────────────────────────────────
+
+async function probeModel(modelId, opts = {}) {
+  const provider = detectProvider(modelId);
+  const providerConfig = PROVIDERS[provider];
+  if (!providerConfig && provider === 'unknown') {
+    throw new Error(`Unknown provider for model: ${modelId}. Supported: claude-*, gpt-*, gemini-*`);
+  }
+
+  const apiKey = opts.apiKey || process.env[providerConfig?.envKey || ''];
+  if (!apiKey) {
+    throw new Error(
+      `No API key found. Set ${providerConfig?.envKey || 'API_KEY'} env var or pass --api-key`
+    );
+  }
+
+  const result = {
+    model: modelId,
+    provider,
+    capabilities: {},
+    tokenizer: {},
+    limits: {},
+    fingerprint: {},
+    latency: {},
+  };
+
+  const send = buildSender(provider, modelId, apiKey);
+
+  // 1. Basic probe -- verify model is reachable and measure latency
+  const start = Date.now();
+  const basicResp = await send('Say exactly: PROBE_OK');
+  result.latency.first_token_ms = Date.now() - start;
+  result.capabilities.reachable = !!basicResp;
+
+  if (!basicResp) {
+    result.capabilities.error = 'Model unreachable or invalid API key';
+    return result;
+  }
+
+  // 2. Capability probes (run in parallel for speed)
+  const [streamResp, toolResp, sysResp] = await Promise.allSettled([
+    testStreaming(send),
+    testToolUse(provider, modelId, apiKey),
+    send('What is 2+2? Reply with just the number.', { systemPrompt: 'You are a calculator.' }),
+  ]);
+
+  result.capabilities.streaming = streamResp.status === 'fulfilled' && streamResp.value;
+  result.capabilities.tools = toolResp.status === 'fulfilled' && toolResp.value;
+  result.capabilities.system_prompt = sysResp.status === 'fulfilled' && !!sysResp.value;
+
+  // 3. Tokenizer probe -- send known strings, analyze responses
+  const tokenizerResult = await probeTokenizer(send);
+  result.tokenizer = tokenizerResult;
+
+  // 4. Model fingerprint -- specific prompts that distinguish families
+  const fingerprint = await fingerprintModel(send, provider);
+  result.fingerprint = fingerprint;
+
+  // 5. Measure response speed
+  const speedStart = Date.now();
+  const longResp = await send('Count from 1 to 20, one per line.');
+  const speedMs = Date.now() - speedStart;
+  const outputTokens = longResp ? longResp.split(/\s+/).length : 0;
+  result.latency.generation_ms = speedMs;
+  result.latency.est_tokens_per_sec = speedMs > 0 ? Math.round((outputTokens / speedMs) * 1000) : 0;
+
+  return result;
+}
+
+// ── Provider-specific request builders ───────────────────────────────────
+
+function buildSender(provider, modelId, apiKey) {
+  return async (prompt, opts = {}) => {
+    try {
+      if (provider === 'anthropic') return await sendAnthropic(modelId, apiKey, prompt, opts);
+      if (provider === 'openai') return await sendOpenAI(modelId, apiKey, prompt, opts);
+      if (provider === 'google') return await sendGoogle(modelId, apiKey, prompt, opts);
+      throw new Error(`Unsupported provider: ${provider}`);
+    } catch (err) {
+      // Return null on API errors (model may not support the feature)
+      if (err.message?.includes('API error')) return null;
+      throw err;
+    }
+  };
+}
+
+async function sendAnthropic(model, apiKey, prompt, opts = {}) {
+  const body = {
+    model,
+    max_tokens: opts.maxTokens || 100,
+    messages: [{ role: 'user', content: prompt }],
+  };
+  if (opts.systemPrompt) body.system = opts.systemPrompt;
+
+  const resp = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+    },
+    body: JSON.stringify(body),
+  });
+  if (!resp.ok) throw new Error(`API error ${resp.status}: ${await resp.text()}`);
+  const data = await resp.json();
+  return data.content?.[0]?.text || '';
+}
+
+async function sendOpenAI(model, apiKey, prompt, opts = {}) {
+  const messages = [];
+  if (opts.systemPrompt) messages.push({ role: 'system', content: opts.systemPrompt });
+  messages.push({ role: 'user', content: prompt });
+
+  const resp = await fetch('https://api.openai.com/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({ model, messages, max_tokens: opts.maxTokens || 100 }),
+  });
+  if (!resp.ok) throw new Error(`API error ${resp.status}: ${await resp.text()}`);
+  const data = await resp.json();
+  return data.choices?.[0]?.message?.content || '';
+}
+
+async function sendGoogle(model, apiKey, prompt, opts = {}) {
+  const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`;
+  const body = {
+    contents: [{ parts: [{ text: prompt }] }],
+    generationConfig: { maxOutputTokens: opts.maxTokens || 100 },
+  };
+  if (opts.systemPrompt) {
+    body.systemInstruction = { parts: [{ text: opts.systemPrompt }] };
+  }
+
+  const resp = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  });
+  if (!resp.ok) throw new Error(`API error ${resp.status}: ${await resp.text()}`);
+  const data = await resp.json();
+  return data.candidates?.[0]?.content?.parts?.[0]?.text || '';
+}
+
+// ── Feature probes ───────────────────────────────────────────────────────
+
+async function testStreaming(send) {
+  // Streaming support is provider-dependent; we just check if the model responds
+  const resp = await send('Say "stream test"');
+  return !!resp;
+}
+
+async function testToolUse(provider, modelId, apiKey) {
+  try {
+    if (provider === 'anthropic') {
+      const resp = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': apiKey,
+          'anthropic-version': '2023-06-01',
+        },
+        body: JSON.stringify({
+          model: modelId,
+          max_tokens: 100,
+          messages: [{ role: 'user', content: 'What is the weather in SF?' }],
+          tools: [{
+            name: 'get_weather',
+            description: 'Get weather for a location',
+            input_schema: { type: 'object', properties: { location: { type: 'string' } } },
+          }],
+        }),
+      });
+      return resp.ok;
+    }
+    return true; // Assume supported for other providers
+  } catch {
+    return false;
+  }
+}
+
+// ── Tokenizer probing ────────────────────────────────────────────────────
+
+async function probeTokenizer(send) {
+  // Send known strings and analyze how the model interprets them
+  const testStr = 'antidisestablishmentarianism';
+  const resp = await send(
+    `How many tokens does the word "${testStr}" require? Just give the number.`
+  );
+  const tokenCount = resp ? parseInt(resp.match(/\d+/)?.[0] || '0', 10) : 0;
+
+  // Detect BPE vs SentencePiece by checking token boundary behavior
+  const bpeResp = await send(
+    'Split "unhappiness" into its BPE tokens. List each token on a line.'
+  );
+
+  let type = 'unknown';
+  if (bpeResp) {
+    if (bpeResp.includes('un') && bpeResp.includes('happiness')) type = 'BPE';
+    if (bpeResp.includes('_un') || bpeResp.includes('\u2581un')) type = 'SentencePiece';
+  }
+
+  return {
+    type,
+    estimated_tokens_for_test_word: tokenCount,
+    test_word: testStr,
+  };
+}
+
+// ── Model fingerprinting ─────────────────────────────────────────────────
+
+async function fingerprintModel(send, provider) {
+  // Ask the model to identify itself
+  const identResp = await send(
+    'What LLM are you? Reply in format: "I am [model name] by [company]"'
+  );
+
+  // Test for specific behaviors
+  const mathResp = await send('What is 7 * 8? Reply with just the number.');
+
+  return {
+    self_identification: identResp || 'unknown',
+    provider_detected: provider,
+    math_correct: mathResp?.trim() === '56',
+    timestamp: new Date().toISOString(),
+  };
+}
+
+// ── Pretty printer ───────────────────────────────────────────────────────
+
+function printProbeResult(result) {
+  const _chalk = require('chalk');
+  const chalk = _chalk.default || _chalk;
+
+  console.log(chalk.bold.cyan('\n  LLM API Probe Results'));
+  console.log(chalk.white(`  Model:    ${result.model}`));
+  console.log(chalk.white(`  Provider: ${result.provider}`));
+  console.log('');
+
+  console.log(chalk.bold('  Capabilities:'));
+  console.log(chalk.white(`    Reachable:      ${result.capabilities.reachable ? 'Yes' : 'No'}`));
+  console.log(chalk.white(`    Streaming:      ${result.capabilities.streaming ? 'Yes' : 'No'}`));
+  console.log(chalk.white(`    Tool use:       ${result.capabilities.tools ? 'Yes' : 'No'}`));
+  console.log(chalk.white(`    System prompt:  ${result.capabilities.system_prompt ? 'Yes' : 'No'}`));
+  console.log('');
+
+  console.log(chalk.bold('  Latency:'));
+  console.log(chalk.white(`    First token:    ${result.latency.first_token_ms} ms`));
+  console.log(chalk.white(`    Generation:     ${result.latency.generation_ms} ms`));
+  console.log(chalk.white(`    Est. tok/sec:   ${result.latency.est_tokens_per_sec}`));
+  console.log('');
+
+  console.log(chalk.bold('  Tokenizer:'));
+  console.log(chalk.white(`    Type:           ${result.tokenizer.type}`));
+  console.log(chalk.white(`    Test word:      "${result.tokenizer.test_word}" -> ${result.tokenizer.estimated_tokens_for_test_word} tokens`));
+  console.log('');
+
+  console.log(chalk.bold('  Fingerprint:'));
+  console.log(chalk.white(`    Self-ID:        ${result.fingerprint.self_identification?.slice(0, 80)}`));
+  console.log(chalk.white(`    Math correct:   ${result.fingerprint.math_correct ? 'Yes' : 'No'}`));
+  console.log('');
+}
+
+module.exports = { probeModel, printProbeResult, detectProvider };