provider-kit 1.1.1 → 1.2.0

package/README.md

@@ -54,26 +54,35 @@ const provider = providerRegistry.getProvider('openai')
 const reply = await provider.chat('gpt-4o-mini', [{ role: 'user', content: 'Hi' }])
 ```
 
-### Model fallback (like OpenRouter auto-failover)
+### Model auto-routing (health probes + real-time switching)
 
-Try GPT-4 first. If it's rate-limited or times out, fall back to Claude, then Ollama.
+Periodically detects which models are available and routes to the best one. Changes take effect immediately — no restart needed.
 
 ```js
 import { createRouter } from 'provider-kit'
 
-const router = createRouter([
-  { provider: 'openai',   model: 'gpt-4',        apiKey: process.env.OPENAI_API_KEY },
-  { provider: 'openai',   model: 'gpt-4o-mini',  apiKey: process.env.OPENAI_API_KEY },
-  { provider: 'anthropic', model: 'claude-3-haiku', apiKey: process.env.ANTHROPIC_API_KEY },
-  { provider: 'ollama',   model: 'llama3.2',     baseUrl: 'http://localhost:11434' },
-])
+const router = createRouter({
+  probes: [
+    { provider: 'openai',   model: 'gpt-4',        apiKey: process.env.OPENAI_API_KEY },
+    { provider: 'openai',   model: 'gpt-4o-mini',  apiKey: process.env.OPENAI_API_KEY },
+    { provider: 'anthropic', model: 'claude-3-haiku', apiKey: process.env.ANTHROPIC_API_KEY },
+    { provider: 'ollama',   model: 'llama3.2',     baseUrl: 'http://localhost:11434' },
+  ],
+  strategy: 'latency',          // 'latency' | 'failover' | 'round-robin'
+  probeInterval: 30000,          // ping every 30s (0 = manual only)
+  onProbeResult: (results) => console.log(results),
+})
 
-const reply = await router.chat([{ role: 'user', content: 'Explain quantum computing' }])
-// Automatically tries gpt-4 → gpt-4o-mini → claude-haiku → llama3.2
-```
+const reply = await router.chat([{ role: 'user', content: 'Hello' }])
+// → auto-routed to the healthiest model
 
-Router skips auth/bad_request/quota errors (those won't work on another model either).
-Only retries on rate_limit, timeout, server_error, and network errors.
+// Manual probe
+const status = await router.checkNow()
+// → [{ provider, model, ok, latency, error }]
+
+// Stop auto-probe
+router.stop()
+```
 
 ### Streaming
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "provider-kit",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "description": "42 LLM provider unified API — one interface for OpenAI, Anthropic, Ollama, OpenRouter, and 38 more. Built-in retry, timeout, and error handling.",
   "type": "module",
   "main": "./src/index.js",

package/src/providers/provider-error-adapter.js

@@ -103,43 +103,129 @@ export function createCancelSignal() {
 }
 
 /**
- * createRouter — model fallback router
+ * createRouter — model health probe + auto-routing
  *
- * Tries models in order. If one fails (rate_limit/timeout/server_error),
- * automatically falls back to the next.
+ * Periodically checks each model's availability and latency.
+ * Routes requests to the best available model in real time.
+ * Changes take effect immediately — no restart needed.
  *
  * Usage:
- *   const router = createRouter([
- *     { provider: 'openai', model: 'gpt-4', apiKey: 'sk-...' },
- *     { provider: 'openai', model: 'gpt-4o-mini', apiKey: 'sk-...' },
- *     { provider: 'anthropic', model: 'claude-3-haiku', apiKey: 'sk-...' },
- *   ]);
+ *   const router = createRouter({
+ *     probes: [
+ *       { provider: 'openai',   model: 'gpt-4',        apiKey: 'sk-...' },
+ *       { provider: 'openai',   model: 'gpt-4o-mini',  apiKey: 'sk-...' },
+ *       { provider: 'anthropic', model: 'claude-3-haiku', apiKey: 'sk-...' },
+ *       { provider: 'ollama',   model: 'llama3',       baseUrl: 'http://localhost:11434' },
+ *     ],
+ *     strategy: 'latency',       // 'latency' | 'failover' | 'round-robin' | 'cheapest'
+ *     probeInterval: 30000,       // check every 30s (0 = no auto-probe)
+ *     probeTimeout: 5000,         // per-probe timeout
+ *     onProbeResult: (results) => console.log(results),
+ *   });
+ *
  *   const reply = await router.chat([{ role: 'user', content: 'Hi' }]);
+ *   // → routed to the best available model
  */
-export function createRouter(strategies) {
-  if (!Array.isArray(strategies) || strategies.length === 0) {
-    throw new ProviderError('createRouter requires a non-empty array of strategies', { type: 'bad_request' });
+export function createRouter(opts) {
+  const probes = Array.isArray(opts) ? opts : opts.probes || opts.strategies || [];
+  if (!Array.isArray(probes) || probes.length === 0) {
+    throw new ProviderError('createRouter requires probes array', { type: 'bad_request' });
   }
 
-  async function chat(messages) {
-    const errors = [];
-    for (const entry of strategies) {
-      try {
-        const { createProvider } = await import('./openai-compatible.js');
-        const provider = await createProvider(entry.provider, entry.apiKey, { baseUrl: entry.baseUrl });
-        return await safeProviderCall(
-          () => provider.chat(entry.model, messages),
-          { provider: entry.provider, retries: 1, timeout: 15000 }
-        );
-      } catch (e) {
-        const ce = classifyError(e, entry.provider);
-        if (ce.type === 'auth' || ce.type === 'bad_request' || ce.type === 'quota') throw ce;
-        errors.push({ provider: entry.provider, model: entry.model, error: ce.message });
-        continue;
+  const strategy = opts.strategy || 'latency';
+  const probeInterval = opts.probeInterval ?? 0;
+  const probeTimeout = opts.probeTimeout ?? 5000;
+  const onProbeResult = opts.onProbeResult || null;
+
+  // Probe results: { provider, model, ok, latency, error, timestamp }
+  let results = probes.map(p => ({ provider: p.provider, model: p.model, ok: true, latency: 0, error: null, timestamp: 0 }));
+  let rrIndex = 0;
+  let probeTimer = null;
+
+  // Probe a single model
+  async function probeOne(entry) {
+    const start = Date.now();
+    try {
+      const { createProvider } = await import('./openai-compatible.js');
+      const provider = await createProvider(entry.provider, entry.apiKey, { baseUrl: entry.baseUrl });
+      await withTimeout(
+        () => provider.chat(entry.model, [{ role: 'user', content: 'Hi' }], { max_tokens: 1 }),
+        probeTimeout
+      );
+      return { ok: true, latency: Date.now() - start, error: null };
+    } catch (e) {
+      const ce = classifyError(e, entry.provider);
+      return { ok: false, latency: Date.now() - start, error: ce.type === 'auth' ? 'auth_failed' : ce.message };
+    }
+  }
+
+  // Probe all models, update results
+  async function probeAll() {
+    const newResults = await Promise.all(probes.map(async (entry, i) => {
+      const { ok, latency, error } = await probeOne(entry);
+      return { provider: entry.provider, model: entry.model, ok, latency, error, timestamp: Date.now() };
+    }));
+    results = newResults;
+    if (onProbeResult) onProbeResult(results);
+  }
+
+  // Pick the best model based on strategy
+  function pick() {
+    const alive = results.filter(r => r.ok);
+    if (alive.length === 0) return probes[0]; // all dead → try first anyway
+
+    switch (strategy) {
+      case 'latency':
+        alive.sort((a, b) => a.latency - b.latency);
+        return probes[results.indexOf(alive[0])];
+      case 'round-robin': {
+        const idx = rrIndex % alive.length;
+        rrIndex++;
+        return probes[results.indexOf(alive[idx])];
       }
+      case 'failover': {
+        const preferred = probes.map((p, i) => ({ p, i })).sort((a, b) => a.i - b.i);
+        for (const { p, i } of preferred) {
+          if (results[i]?.ok) return p;
+        }
+        return probes[0];
+      }
+      default:
+        return probes[probes.indexOf(alive[0])];
     }
-    throw new ProviderError(`All models failed: ${errors.map(e => `${e.provider}/${e.model}`).join(', ')}`, { type: 'server_error' });
   }
 
-  return { chat, strategies };
+  // Start auto-probe
+  if (probeInterval > 0) {
+    probeAll(); // first probe immediately
+    probeTimer = setInterval(probeAll, probeInterval);
+  }
+
+  // Chat — route to best model
+  async function chat(messages) {
+    const entry = pick();
+    const idx = probes.indexOf(entry);
+    const r = results[idx];
+
+    if (!r?.ok && strategy !== 'failover') {
+      // All dead — force probe once
+      const probeResult = await probeOne(entry);
+      results[idx] = { ...results[idx], ...probeResult, timestamp: Date.now() };
+    }
+
+    const { createProvider } = await import('./openai-compatible.js');
+    const provider = await createProvider(entry.provider, entry.apiKey, { baseUrl: entry.baseUrl });
+    return safeProviderCall(
+      () => provider.chat(entry.model, messages),
+      { provider: entry.provider, retries: 1, timeout: 30000 }
+    );
+  }
+
+  // Manual probe trigger
+  async function checkNow() { await probeAll(); return results; }
+
+  // Stop auto-probe
+  function stop() { if (probeTimer) { clearInterval(probeTimer); probeTimer = null; } }
+
+  return { chat, probes, results: () => results, strategy, checkNow, stop };
 }