search-mcp-rotator 1.0.0

package/README.md

@@ -0,0 +1,177 @@
+# Search MCP Rotator
+
+![Search MCP Rotator](./poster.png)
+
+A local MCP (Model Context Protocol) proxy server that provides transparent API key rotation for search providers. When one key is exhausted, rate-limited, or returns any server-side error, the proxy automatically rotates to the next available key and retries the request — transparently, without the MCP client knowing a rotation occurred.
+
+## Features
+
+- **Transparent Key Rotation**: Automatic failover between API keys
+- **Multi-Provider Support**: Supports 9 search providers with different auth patterns
+- **Flexible Rotation Strategies**: Round-robin, priority, and random selection
+- **Circuit Breaker Pattern**: Prevents cascading failures
+- **Configurable Cooldowns**: Smart recovery timing per provider
+- **Strategy-as-Tool-Argument**: LLMs can choose rotation strategy per request
+- **Comprehensive Error Detection**: Provider-specific exhaustion patterns
+
+## Supported Providers
+
+| Provider | Auth Pattern | Specialty |
+|----------|--------------|-----------|
+| **Exa** | Bearer header | Neural/semantic web search, code search |
+| **Firecrawl** | Bearer header | Web scraping, deep crawl, structured extraction |
+| **Linkup** | Bearer header | Real-time web search, source-cited answers |
+| **Bright Data** | Bearer header | 40+ scraping tools, Google SERP, Amazon |
+| **Olostep** | Bearer header | Search + extract + AI answers with citations |
+| **Tavily** | Query param | Real-time web search, extract, map, crawl |
+| **Dappier** | Query param | Real-time news, finance, sports, weather |
+| **Parallel** | Custom header | Highest-accuracy general web search |
+
+## Installation
+
+```bash
+npm install
+npm run build
+```
+
+## Configuration
+
+Copy `config.example.json` to `config.json` and add your API keys:
+
+```json
+{
+  "logLevel": "info",
+  "providers": {
+    "exa": {
+      "enabled": true,
+      "url": "https://mcp.exa.ai/mcp",
+      "authPattern": "bearer",
+      "keys": ["your_exa_key_1", "your_exa_key_2"],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    }
+  }
+}
+```
+
+### Environment Variable Overrides
+
+Keys can be overridden via environment variables:
+
+```bash
+export EXA_KEYS="key1,key2,key3"
+export FIRECRAWL_KEYS="key1,key2"
+```
+
+## Usage
+
+Start a rotator for a specific provider:
+
+```bash
+# Using built binary
+node dist/index.js --provider=exa --config=config.json
+
+# Or using npm script
+npm run dev -- --provider=exa --config=config.json
+```
+
+## OpenCode Integration
+
+Register each rotator as a local MCP server in `~/.config/opencode/opencode.json`:
+
+```json
+{
+  "mcpServers": {
+    "exa-rotator": {
+      "type": "local",
+      "command": "node",
+      "args": [
+        "/path/to/search-mcp-rotator/dist/index.js",
+        "--provider=exa"
+      ],
+      "env": {
+        "MCP_ROTATOR_CONFIG": "/path/to/config.json"
+      },
+      "enabled": true
+    }
+  }
+}
+```
+
+## Rotation Strategies
+
+Every tool call accepts an optional `strategy` parameter:
+
+- **`round-robin`** (default): Distribute requests evenly across keys
+- **`priority`**: Always use first healthy key, fallback to others
+- **`random`**: Random key selection to avoid patterns
+
+```typescript
+// Example tool call with strategy override
+{
+  "query": "AI news",
+  "strategy": "priority"  // Optional: overrides provider default
+}
+```
+
+## Architecture
+
+```
+OpenCode (MCP client)
+        │  stdio transport
+        ▼
+┌─────────────────────────┐
+│   Search MCP Rotator Proxy     │
+│  (local stdio server)   │
+│                         │
+│  ┌─────────────────┐    │
+│  │   KeyPool       │    │
+│  │  [key1, key2,   │    │
+│  │   key3, key4]   │    │
+│  │  activeIdx = 0  │    │
+│  │  degraded: Map  │    │
+│  └─────────────────┘    │
+│                         │
+│  On tool call:          │
+│  1. pick active key     │
+│  2. inject into request │
+│  3. forward upstream    │
+│  4. on rate-limit:      │
+│     mark degraded       │
+│     rotate key          │
+│     retry same call     │
+└─────────────────────────┘
+        │  HTTP (Streamable HTTP transport)
+        ▼
+  Remote MCP Provider
+  (Exa / Firecrawl / etc.)
+```
+
+## Error Detection
+
+The rotator detects exhaustion signals specific to each provider:
+
+- **HTTP Status Codes**: 401, 402, 429, etc.
+- **JSON-RPC Error Codes**: Provider-specific codes
+- **Message Patterns**: "rate limit", "quota", "credits exhausted"
+- **Special Cases**: Provider-specific error formats
+
+## Development
+
+```bash
+# Install dependencies
+npm install
+
+# Development with auto-reload
+npm run dev -- --provider=exa
+
+# Type checking
+npm run typecheck
+
+# Build for production
+npm run build
+```
+
+## License
+
+MIT

package/config.example.json

@@ -0,0 +1,97 @@
+{
+  "logLevel": "info",
+  "providers": {
+    "exa": {
+      "enabled": true,
+      "url": "https://mcp.exa.ai/mcp",
+      "authPattern": "bearer",
+      "keys": [
+        "exa_key_1",
+        "exa_key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    },
+    "firecrawl": {
+      "enabled": true,
+      "url": "https://mcp.firecrawl.dev/mcp",
+      "authPattern": "bearer",
+      "keys": [
+        "fc-key_1",
+        "fc-key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    },
+    "tavily": {
+      "enabled": true,
+      "url": "https://mcp.tavily.com/mcp/",
+      "authPattern": "queryparam",
+      "queryParamName": "tavilyApiKey",
+      "keys": [
+        "tvly-key_1",
+        "tvly-key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    },
+    "linkup": {
+      "enabled": true,
+      "url": "https://mcp.linkup.so/mcp",
+      "authPattern": "bearer",
+      "keys": [
+        "linkup_key_1",
+        "linkup_key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    },
+    "brightdata": {
+      "enabled": true,
+      "url": "https://mcp.brightdata.com/mcp",
+      "authPattern": "bearer",
+      "keys": [
+        "bd_key_1",
+        "bd_key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    },
+    "olostep": {
+      "enabled": true,
+      "url": "https://mcp.olostep.com/mcp",
+      "authPattern": "bearer",
+      "keys": [
+        "olostep_key_1",
+        "olostep_key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 300000
+    },
+
+    "dappier": {
+      "enabled": true,
+      "url": "https://mcp.dappier.com/mcp",
+      "authPattern": "queryparam",
+      "queryParamName": "apiKey",
+      "keys": [
+        "dappier_key_1",
+        "dappier_key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 300000
+    },
+    "parallel": {
+      "enabled": true,
+      "url": "https://search.parallel.ai/mcp",
+      "authPattern": "customheader",
+      "headerName": "x-api-key",
+      "keys": [
+        "parallel_key_1",
+        "parallel_key_2"
+      ],
+      "strategy": "round-robin",
+      "cooldownMs": 60000
+    }
+  }
+}

package/dist/auth-injector.d.ts

@@ -0,0 +1,15 @@
+import type { ProviderConfig, AuthInjectionResult } from './types.js';
+export declare class AuthInjector {
+    private authPattern;
+    private headerName?;
+    private queryParamName?;
+    constructor(config: Pick<ProviderConfig, 'authPattern' | 'headerName' | 'queryParamName'>);
+    /**
+     * Inject key into HTTP headers or URL
+     */
+    inject(key: string, baseUrl: string): AuthInjectionResult;
+    /**
+     * For query-param providers: rebuild URL with new key (for rotation)
+     */
+    updateKey(currentUrl: string, newKey: string): string;
+}

package/dist/auth-injector.js

@@ -0,0 +1,47 @@
+export class AuthInjector {
+    authPattern;
+    headerName;
+    queryParamName;
+    constructor(config) {
+        this.authPattern = config.authPattern;
+        this.headerName = config.headerName;
+        this.queryParamName = config.queryParamName;
+    }
+    /**
+     * Inject key into HTTP headers or URL
+     */
+    inject(key, baseUrl) {
+        const headers = {};
+        let url = baseUrl;
+        switch (this.authPattern) {
+            case 'bearer':
+                headers['Authorization'] = `Bearer ${key}`;
+                break;
+            case 'queryparam':
+                if (!this.queryParamName) {
+                    throw new Error('queryParamName required for queryparam auth pattern');
+                }
+                const separator = url.includes('?') ? '&' : '?';
+                url = `${url}${separator}${this.queryParamName}=${key}`;
+                break;
+            case 'customheader':
+                if (!this.headerName) {
+                    throw new Error('headerName required for customheader auth pattern');
+                }
+                headers[this.headerName] = key;
+                break;
+        }
+        return { url, headers };
+    }
+    /**
+     * For query-param providers: rebuild URL with new key (for rotation)
+     */
+    updateKey(currentUrl, newKey) {
+        if (this.authPattern !== 'queryparam' || !this.queryParamName) {
+            return currentUrl;
+        }
+        const url = new URL(currentUrl);
+        url.searchParams.set(this.queryParamName, newKey);
+        return url.toString();
+    }
+}

package/dist/config.d.ts

@@ -0,0 +1,3 @@
+import type { Config, ProviderConfig, AuthPattern, RotationStrategy, LogLevel } from './types.js';
+export declare function loadConfig(filePath: string): Promise<Config>;
+export type { Config, ProviderConfig, AuthPattern, RotationStrategy, LogLevel };

package/dist/config.js

@@ -0,0 +1,53 @@
+import { readFile } from 'fs/promises';
+import { z } from 'zod';
+const AuthPatternSchema = z.enum(['bearer', 'queryparam', 'customheader']);
+const RotationStrategySchema = z.enum(['round-robin', 'priority', 'random']);
+const LogLevelSchema = z.enum(['debug', 'info', 'warn', 'error']);
+// JSON object keys are always strings — coerce them to numbers at runtime
+const CooldownOverridesSchema = z.record(z.string(), z.number()).transform((obj) => Object.fromEntries(Object.entries(obj).map(([k, v]) => [Number(k), v])));
+const ExhaustionPatternSchema = z.object({
+    httpStatusCodes: z.array(z.number()).default([]),
+    jsonRpcErrorCodes: z.array(z.number()).default([]),
+    messagePatterns: z.array(z.string()).default([]),
+    cooldownOverrides: CooldownOverridesSchema.default({}),
+    hasRetryAfterHeader: z.boolean().default(false)
+});
+const ProviderConfigSchema = z.object({
+    enabled: z.boolean().default(true),
+    url: z.string().url(),
+    authPattern: AuthPatternSchema,
+    // Auth pattern specific fields
+    headerName: z.string().optional(), // For 'customheader'
+    queryParamName: z.string().optional(), // For 'queryparam'
+    keys: z.array(z.string()).min(1),
+    strategy: RotationStrategySchema.default('round-robin'),
+    cooldownMs: z.number().default(60000),
+    exhaustionPatterns: ExhaustionPatternSchema.optional()
+});
+const ConfigSchema = z.object({
+    logLevel: LogLevelSchema.default('info'),
+    providers: z.record(ProviderConfigSchema)
+});
+export async function loadConfig(filePath) {
+    // 1. Read config file
+    const fileContent = await readFile(filePath, 'utf-8');
+    const rawConfig = JSON.parse(fileContent);
+    // 2. Merge with env vars (keys can be overridden via env)
+    const mergedConfig = mergeWithEnv(rawConfig);
+    // 3. Validate with Zod
+    const config = ConfigSchema.parse(mergedConfig);
+    return config;
+}
+function mergeWithEnv(config) {
+    // Allow keys to be overridden via env vars like:
+    // EXA_KEYS="key1,key2,key3"
+    // FIRECRAWL_KEYS="key1,key2"
+    for (const providerName in config.providers) {
+        const envVarName = `${providerName.toUpperCase()}_KEYS`;
+        const envValue = process.env[envVarName];
+        if (envValue) {
+            config.providers[providerName].keys = envValue.split(',').map((k) => k.trim());
+        }
+    }
+    return config;
+}

package/dist/detector.d.ts

@@ -0,0 +1,91 @@
+import type { ExhaustionError, ExhaustionPatternConfig } from './types.js';
+export declare const PROVIDER_EXHAUSTION_CONFIG: {
+    readonly exa: {
+        readonly httpStatusCodes: readonly [401, 402, 429];
+        readonly jsonRpcErrorCodes: readonly [-32000];
+        readonly messagePatterns: readonly ["invalid api key", "no_more_credits", "api_key_budget_exceeded", "team_budget_exceeded", "credits exhausted", "payment required", "account credits", "spending budget", "rate limit", "too many requests", "you've exceeded your exa rate limit", "you've hit exa's free mcp rate limit", "quota"];
+        readonly cooldownOverrides: {
+            readonly 402: 3600000;
+            readonly 401: 31536000000;
+        };
+        readonly hasRetryAfterHeader: true;
+    };
+    readonly firecrawl: {
+        readonly httpStatusCodes: readonly [401, 402, 429, 500, 502, 503, 504];
+        readonly jsonRpcErrorCodes: readonly [];
+        readonly messagePatterns: readonly ["rate limit exceeded", "request rate limit exceeded", "concurrency limit reached", "payment required", "insufficient credits", "payment required to access", "unauthorized: invalid token", "unauthorized: token missing", "unauthorized", "retry after", "error creating server"];
+        readonly cooldownOverrides: {
+            readonly 402: 86400000;
+            readonly 401: 604800000;
+        };
+        readonly hasRetryAfterHeader: true;
+    };
+    readonly tavily: {
+        readonly httpStatusCodes: readonly [401, 429, 432, 433];
+        readonly jsonRpcErrorCodes: readonly [];
+        readonly messagePatterns: readonly ["authentication required", "unauthorized", "invalid api key", "missing or invalid", "excessive requests", "rate of requests", "usage limit", "pay-as-you-go", "paygo limit", "search failed"];
+        readonly cooldownOverrides: {
+            readonly 432: 86400000;
+            readonly 433: 3600000;
+            readonly 401: 31536000000;
+        };
+        readonly hasRetryAfterHeader: true;
+    };
+    readonly linkup: {
+        readonly httpStatusCodes: readonly [401, 403, 429];
+        readonly jsonRpcErrorCodes: readonly [-32000];
+        readonly messagePatterns: readonly ["unauthorized action", "api key is required", "insufficient", "too many requests", "out of credit", "credit", "rate limit"];
+        readonly cooldownOverrides: {};
+        readonly hasRetryAfterHeader: false;
+    };
+    readonly brightdata: {
+        readonly httpStatusCodes: readonly [401, 402, 407, 429];
+        readonly jsonRpcErrorCodes: readonly [];
+        readonly messagePatterns: readonly ["http 401: auth method is not supported", "http 401: token expired", "http 401", "5,000 request monthly limit", "monthly limit for bright data mcp", "usage limit", "zone has reached usage limit", "http 502", "http 429", "http 407", "account is suspended", "kyc required", "http 402"];
+        readonly cooldownOverrides: {};
+        readonly hasRetryAfterHeader: false;
+    };
+    readonly olostep: {
+        readonly httpStatusCodes: readonly [401, 402, 403];
+        readonly jsonRpcErrorCodes: readonly [];
+        readonly messagePatterns: readonly ["invalid_api_key", "invalid api key", "your api key is invalid", "credits exhausted", "payment required", "olostep api error: 401", "olostep api error: 402", "olostep api error: 403"];
+        readonly cooldownOverrides: {};
+        readonly hasRetryAfterHeader: false;
+    };
+    readonly dappier: {
+        readonly httpStatusCodes: readonly [401, 402];
+        readonly jsonRpcErrorCodes: readonly [];
+        readonly messagePatterns: readonly ["error: failed to retrieve real-time information", "error: failed to retrieve", "error: unable to retrieve", "error: authentication", "error: unauthorized", "missing authentication"];
+        readonly cooldownOverrides: {};
+        readonly hasRetryAfterHeader: false;
+    };
+    readonly parallel: {
+        readonly httpStatusCodes: readonly [401, 402, 429];
+        readonly jsonRpcErrorCodes: readonly [];
+        readonly messagePatterns: readonly ["oauth.v2.invalidapikey", "steps.oauth.v2.failedtoresolveapikey", "oauth.v2.apikeyexpired", "oauth.v2.apikeynotapproved", "invalid apikey", "failedtoresolveapikey", "invalid api key", "rate limit", "too many requests", "insufficient credit", "quota exceeded", "code\":16"];
+        readonly cooldownOverrides: {};
+        readonly hasRetryAfterHeader: false;
+    };
+};
+export declare class ExhaustionDetector {
+    private patterns;
+    private providerName;
+    constructor(providerName: string, customPatterns?: Partial<ExhaustionPatternConfig>);
+    /**
+     * Determine if an error indicates key exhaustion
+     */
+    isExhausted(error: ExhaustionError): boolean;
+    private isExaExhausted;
+    private isFirecrawlExhausted;
+    private isTavilyExhausted;
+    private isLinkupExhausted;
+    private isBrightDataExhausted;
+    private isOlostepExhausted;
+    private isDappierExhausted;
+    private isParallelExhausted;
+    private isGenericExhausted;
+}
+/**
+ * Extract cooldown duration from error (if provider specifies it)
+ */
+export declare function extractCooldown(error: ExhaustionError, providerName: string, defaultCooldownMs: number): number;