rrcrawl 0.1.0

package/.env.example

@@ -0,0 +1,21 @@
+# auto uses OneCLI when ONECLI_URL is present, otherwise provider API keys.
+RRCRAWL_AUTH_MODE=auto
+
+# Optional comma-separated provider allowlist.
+# In env mode the default is every provider with a configured key.
+# In onecli mode the default is all providers.
+RRCRAWL_PROVIDERS=firecrawl,tavily,scrapedo
+
+FIRECRAWL_API_KEY=
+TAVILY_API_KEY=
+SCRAPEDO_API_TOKEN=
+
+# Optional operational tuning.
+RRCRAWL_REQUEST_TIMEOUT_MS=60000
+RRCRAWL_CRAWL_TIMEOUT_MS=180000
+RRCRAWL_POLL_INTERVAL_MS=2000
+
+# Optional API endpoint overrides.
+FIRECRAWL_API_URL=https://api.firecrawl.dev
+TAVILY_API_URL=https://api.tavily.com
+SCRAPEDO_API_URL=https://api.scrape.do/

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rrcrawl contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,176 @@
+# rrcrawl
+
+A tiny stdio MCP server that gives agents two web-content tools:
+
+- `scrape`: fetch one URL as Markdown, round-robin across Firecrawl, Tavily,
+  and Scrape.do.
+- `crawl`: crawl multiple pages as Markdown, round-robin across Firecrawl and
+  Tavily. Scrape.do is intentionally excluded because it has no native
+  multi-page crawl API.
+
+Each call advances its tool's round-robin cursor. If the selected provider
+fails, rrcrawl tries every other provider in that tool's pool once.
+
+## Requirements
+
+- Node.js 22 or newer
+- At least one provider credential, or a OneCLI gateway configuration
+
+## Run with npx
+
+Once the package is published, no local installation is needed:
+
+```bash
+npx -y rrcrawl@latest
+```
+
+An MCP client configuration can launch it directly:
+
+```json
+{
+  "mcpServers": {
+    "rrcrawl": {
+      "command": "npx",
+      "args": ["-y", "rrcrawl@latest"],
+      "env": {
+        "FIRECRAWL_API_KEY": "fc-...",
+        "TAVILY_API_KEY": "tvly-...",
+        "SCRAPEDO_API_TOKEN": "..."
+      }
+    }
+  }
+}
+```
+
+With OneCLI:
+
+```bash
+RRCRAWL_AUTH_MODE=onecli onecli run -- npx -y rrcrawl@latest
+```
+
+## Local development
+
+```bash
+npm install
+npm run build
+```
+
+Copy `.env.example` to `.env` for local configuration. `.env` is loaded
+automatically and is ignored by Git.
+
+## Authentication modes
+
+`RRCRAWL_AUTH_MODE` accepts:
+
+- `env`: read `FIRECRAWL_API_KEY`, `TAVILY_API_KEY`, and
+  `SCRAPEDO_API_TOKEN`.
+- `onecli`: omit provider credentials from requests and rely on OneCLI's
+  transparent gateway to inject them.
+- `auto` (default): use `onecli` when `ONECLI_URL` is present; otherwise use
+  `env`.
+
+`RRCRAWL_PROVIDERS` optionally restricts the active providers:
+
+```dotenv
+RRCRAWL_PROVIDERS=firecrawl,tavily,scrapedo
+```
+
+In `env` mode, providers without credentials are disabled unless explicitly
+listed, in which case startup fails with a useful configuration error.
+
+### OneCLI
+
+Configure OneCLI secrets for the provider API hosts:
+
+- `api.firecrawl.dev`: inject `Authorization: Bearer {secret}` for `/v2/*`.
+- `api.tavily.com`: inject `Authorization: Bearer {secret}` for `/extract` and
+  `/crawl`.
+- `api.scrape.do`: inject the `token` query parameter for `/*`.
+
+Then run the built server through the gateway:
+
+```bash
+RRCRAWL_AUTH_MODE=onecli onecli run -- node dist/index.js
+```
+
+No placeholder provider keys are required: rrcrawl omits the credential fields
+in OneCLI mode.
+
+When `HTTPS_PROXY` or `HTTP_PROXY` is set (as OneCLI's gateway does), rrcrawl
+installs a matching proxy dispatcher on startup so all provider calls route
+through the gateway. Node's global `fetch` does not honor these variables on
+its own, so this is required behind an egress-locked gateway. `NODE_EXTRA_CA_CERTS`
+(also injected by the gateway) is honored automatically for the gateway's CA.
+
+## Local MCP client configuration
+
+After `npm run build`, configure an MCP client with:
+
+```json
+{
+  "mcpServers": {
+    "rrcrawl": {
+      "command": "node",
+      "args": ["/absolute/path/to/rrcrawl/dist/index.js"],
+      "env": {
+        "RRCRAWL_AUTH_MODE": "env",
+        "FIRECRAWL_API_KEY": "fc-...",
+        "TAVILY_API_KEY": "tvly-...",
+        "SCRAPEDO_API_TOKEN": "..."
+      }
+    }
+  }
+}
+```
+
+For `.env` configuration, set the MCP server's working directory to this
+project or pass the variables explicitly.
+
+## Tool schemas
+
+### `scrape`
+
+```json
+{ "url": "https://example.com/article" }
+```
+
+Returns the selected provider, canonical URL when available, optional title,
+and Markdown.
+
+### `crawl`
+
+```json
+{
+  "url": "https://example.com/docs",
+  "limit": 10,
+  "maxDepth": 1,
+  "includePaths": ["/docs/.*"],
+  "allowExternal": false,
+  "instructions": "Return API reference pages"
+}
+```
+
+`limit` is capped at 100 and `maxDepth` at 5 to bound cost and response size.
+
+## Development
+
+```bash
+npm test
+npm run check
+npm run build
+```
+
+Tests use injected HTTP fakes and never call paid provider APIs.
+
+## Publishing
+
+The package name `rrcrawl` was unclaimed on npm when this project was created.
+Availability is only guaranteed after the first successful publish.
+
+```bash
+npm login
+npm publish
+```
+
+`prepublishOnly` runs tests and type-checking, while `prepack` always rebuilds
+the distributable executable.

package/dist/config.d.ts

@@ -0,0 +1,24 @@
+import type { ProviderName } from "./types.js";
+export type AuthMode = "env" | "onecli";
+export interface AppConfig {
+    authMode: AuthMode;
+    providers: ProviderName[];
+    requestTimeoutMs: number;
+    crawlTimeoutMs: number;
+    pollIntervalMs: number;
+    firecrawl: {
+        apiUrl: string;
+        apiKey?: string;
+    };
+    tavily: {
+        apiUrl: string;
+        apiKey?: string;
+    };
+    scrapedo: {
+        apiUrl: string;
+        apiToken?: string;
+    };
+}
+type Environment = Record<string, string | undefined>;
+export declare function loadConfig(env?: Environment): AppConfig;
+export {};

package/dist/config.js

@@ -0,0 +1,96 @@
+const providerNames = new Set([
+    "firecrawl",
+    "tavily",
+    "scrapedo",
+]);
+function positiveInteger(env, name, fallback) {
+    const value = env[name];
+    if (!value)
+        return fallback;
+    // Plain base-10 digits only: reject hex, exponent, leading +/- and whitespace
+    // that Number() would silently accept.
+    if (!/^[0-9]+$/.test(value.trim())) {
+        throw new Error(`${name} must be a positive integer`);
+    }
+    const parsed = Number.parseInt(value.trim(), 10);
+    if (!Number.isInteger(parsed) || parsed <= 0) {
+        throw new Error(`${name} must be a positive integer`);
+    }
+    return parsed;
+}
+function parseAuthMode(env) {
+    const configured = env.RRCRAWL_AUTH_MODE ?? "auto";
+    if (configured === "auto") {
+        return env.ONECLI_URL ? "onecli" : "env";
+    }
+    if (configured !== "env" && configured !== "onecli") {
+        throw new Error("RRCRAWL_AUTH_MODE must be auto, env, or onecli");
+    }
+    return configured;
+}
+function parseProviderList(value) {
+    const result = value
+        .split(",")
+        .map((provider) => provider.trim().toLowerCase())
+        .filter(Boolean);
+    for (const provider of result) {
+        if (!providerNames.has(provider)) {
+            throw new Error(`Unknown provider in RRCRAWL_PROVIDERS: ${provider}`);
+        }
+    }
+    return [...new Set(result)];
+}
+export function loadConfig(env = process.env) {
+    const authMode = parseAuthMode(env);
+    const firecrawlKey = env.FIRECRAWL_API_KEY;
+    const tavilyKey = env.TAVILY_API_KEY;
+    const scrapedoToken = env.SCRAPEDO_API_TOKEN ?? env.SCRAPE_DO_API_TOKEN ?? env.SCRAPE_DO_TOKEN;
+    const configuredProviders = env.RRCRAWL_PROVIDERS
+        ? parseProviderList(env.RRCRAWL_PROVIDERS)
+        : undefined;
+    let providers;
+    if (configuredProviders) {
+        providers = configuredProviders;
+    }
+    else if (authMode === "onecli") {
+        providers = ["firecrawl", "tavily", "scrapedo"];
+    }
+    else {
+        providers = [
+            ...(firecrawlKey ? ["firecrawl"] : []),
+            ...(tavilyKey ? ["tavily"] : []),
+            ...(scrapedoToken ? ["scrapedo"] : []),
+        ];
+    }
+    if (providers.length === 0) {
+        throw new Error("No providers configured. Set provider API keys or use RRCRAWL_AUTH_MODE=onecli.");
+    }
+    if (authMode === "env") {
+        const missing = providers.filter((provider) => (provider === "firecrawl" && !firecrawlKey) ||
+            (provider === "tavily" && !tavilyKey) ||
+            (provider === "scrapedo" && !scrapedoToken));
+        if (missing.length > 0) {
+            throw new Error(`Missing credentials for configured providers: ${missing.join(", ")}`);
+        }
+    }
+    return {
+        authMode,
+        providers,
+        requestTimeoutMs: positiveInteger(env, "RRCRAWL_REQUEST_TIMEOUT_MS", 60_000),
+        crawlTimeoutMs: positiveInteger(env, "RRCRAWL_CRAWL_TIMEOUT_MS", 180_000),
+        pollIntervalMs: positiveInteger(env, "RRCRAWL_POLL_INTERVAL_MS", 2_000),
+        firecrawl: {
+            apiUrl: (env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, ""),
+            apiKey: authMode === "env" ? firecrawlKey : undefined,
+        },
+        tavily: {
+            apiUrl: (env.TAVILY_API_URL ?? "https://api.tavily.com").replace(/\/$/, ""),
+            apiKey: authMode === "env" ? tavilyKey : undefined,
+        },
+        scrapedo: {
+            apiUrl: env.SCRAPEDO_API_URL ?? "https://api.scrape.do/",
+            apiToken: authMode === "env" ? scrapedoToken : undefined,
+        },
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,IAAI,GAAG,CAAe;IAC1C,WAAW;IACX,QAAQ;IACR,UAAU;CACX,CAAC,CAAC;AAEH,SAAS,eAAe,CACtB,GAAgB,EAChB,IAAY,EACZ,QAAgB;IAEhB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,8EAA8E;IAC9E,uCAAuC;IACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,GAAgB;IACrC,MAAM,UAAU,GAAG,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC;IACnD,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3C,CAAC;IACD,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,MAAM,GAAG,KAAK;SACjB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,MAAM,EAAE,CAAC;QAC9B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAwB,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAmB,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAmB,OAAO,CAAC,GAAG;IACvD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,GAAG,CAAC,iBAAiB,CAAC;IAC3C,MAAM,SAAS,GAAG,GAAG,CAAC,cAAc,CAAC;IACrC,MAAM,aAAa,GACjB,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC,mBAAmB,IAAI,GAAG,CAAC,eAAe,CAAC;IAE3E,MAAM,mBAAmB,GAAG,GAAG,CAAC,iBAAiB;QAC/C,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,iBAAiB,CAAC;QAC1C,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,SAAyB,CAAC;IAC9B,IAAI,mBAAmB,EAAE,CAAC;QACxB,SAAS,GAAG,mBAAmB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,SAAS,GAAG,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,SAAS,GAAG;YACV,GAAG,CAAC,YAAY,CAAC,CAAC,CAAE,CAAC,WAAW,CAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,aAAa,CAAC,CAAC,CAAE,CAAC,UAAU,CAAW,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAC9B,CAAC,QAAQ,EAAE,EAAE,CACX,CAAC,QAAQ,KAAK,WAAW,IAAI,CAAC,YAAY,CAAC;YAC3C,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC;YACrC,CAAC,QAAQ,KAAK,UAAU,IAAI,CAAC,aAAa,CAAC,CAC9C,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,iDAAiD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,gBAAgB,EAAE,eAAe,CAC/B,GAAG,EACH,4BAA4B,EAC5B,MAAM,CACP;QACD,cAAc,EAAE,eAAe,CAC7B,GAAG,EACH,0BAA0B,EAC1B,OAAO,CACR;QACD,cAAc,EAAE,eAAe,CAC7B,GAAG,EACH,0BAA0B,EAC1B,KAAK,CACN;QACD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,GAAG,CAAC,iBAAiB,IAAI,2BAA2B,CAAC,CAAC,OAAO,CACpE,KAAK,EACL,EAAE,CACH;YACD,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;SACtD;QACD,MAAM,EAAE;YACN,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,IAAI,wBAAwB,CAAC,CAAC,OAAO,CAC9D,KAAK,EACL,EAAE,CACH;YACD,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACnD;QACD,QAAQ,EAAE;YACR,MAAM,EAAE,GAAG,CAAC,gBAAgB,IAAI,wBAAwB;YACxD,QAAQ,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;SACzD;KACF,CAAC;AACJ,CAAC"}

package/dist/http.d.ts

@@ -0,0 +1,15 @@
+export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+export declare class HttpError extends Error {
+    readonly status: number;
+    readonly responseBody: string;
+    constructor(status: number, responseBody: string, url: string);
+}
+export declare class QuotaExceededError extends Error {
+    readonly provider: string;
+    readonly status: number;
+    constructor(provider: string, status: number);
+}
+export declare function rethrowQuota(provider: string, error: unknown, quotaStatuses: readonly number[]): never;
+export declare function isAbsoluteHttpUrl(value: string | undefined): value is string;
+export declare function requestJson<T>(fetchFn: FetchLike, url: string, init: RequestInit, timeoutMs: number): Promise<T>;
+export declare function requestText(fetchFn: FetchLike, url: string, init: RequestInit, timeoutMs: number): Promise<string>;

package/dist/http.js

@@ -0,0 +1,78 @@
+export class HttpError extends Error {
+    status;
+    responseBody;
+    constructor(status, responseBody, url) {
+        super(`HTTP ${status} from ${url}: ${responseBody.slice(0, 500)}`);
+        this.status = status;
+        this.responseBody = responseBody;
+        this.name = "HttpError";
+    }
+}
+// Signals that a provider has exhausted its quota/credits (as opposed to a
+// transient rate limit). The router disables the provider for the rest of the
+// process when it sees this.
+export class QuotaExceededError extends Error {
+    provider;
+    status;
+    constructor(provider, status) {
+        super(`${provider} quota exhausted (HTTP ${status})`);
+        this.provider = provider;
+        this.status = status;
+        this.name = "QuotaExceededError";
+    }
+}
+// Rethrows an error, converting an HttpError with a hard quota/credit status
+// into a QuotaExceededError; every other error is rethrown unchanged.
+export function rethrowQuota(provider, error, quotaStatuses) {
+    if (error instanceof HttpError && quotaStatuses.includes(error.status)) {
+        throw new QuotaExceededError(provider, error.status);
+    }
+    throw error;
+}
+// True only for an absolute http(s) URL — the shape pageSchema.url() accepts.
+// Guards against relative/garbage provider URLs that would otherwise fail
+// output validation outside the tool handler as an unrecoverable McpError.
+export function isAbsoluteHttpUrl(value) {
+    if (!value)
+        return false;
+    try {
+        const { protocol } = new URL(value);
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+// Reads the full body under the same timeout/abort as the request so a stalled
+// body can't hang forever after the headers arrive.
+async function requestBody(fetchFn, url, init, timeoutMs) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const response = await fetchFn(url, {
+            ...init,
+            signal: controller.signal,
+        });
+        const body = await response.text();
+        if (!response.ok) {
+            throw new HttpError(response.status, body, url);
+        }
+        return body;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function requestJson(fetchFn, url, init, timeoutMs) {
+    const body = await requestBody(fetchFn, url, init, timeoutMs);
+    try {
+        return JSON.parse(body);
+    }
+    catch {
+        throw new Error(`Invalid JSON from ${url}: ${body.slice(0, 500)}`);
+    }
+}
+export async function requestText(fetchFn, url, init, timeoutMs) {
+    return requestBody(fetchFn, url, init, timeoutMs);
+}
+//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,SAAU,SAAQ,KAAK;IAEvB;IACA;IAFX,YACW,MAAc,EACd,YAAoB,EAC7B,GAAW;QAEX,KAAK,CAAC,QAAQ,MAAM,SAAS,GAAG,KAAK,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAJ1D,WAAM,GAAN,MAAM,CAAQ;QACd,iBAAY,GAAZ,YAAY,CAAQ;QAI7B,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IAC1B,CAAC;CACF;AAED,2EAA2E;AAC3E,8EAA8E;AAC9E,6BAA6B;AAC7B,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAEhC;IACA;IAFX,YACW,QAAgB,EAChB,MAAc;QAEvB,KAAK,CAAC,GAAG,QAAQ,0BAA0B,MAAM,GAAG,CAAC,CAAC;QAH7C,aAAQ,GAAR,QAAQ,CAAQ;QAChB,WAAM,GAAN,MAAM,CAAQ;QAGvB,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,6EAA6E;AAC7E,sEAAsE;AACtE,MAAM,UAAU,YAAY,CAC1B,QAAgB,EAChB,KAAc,EACd,aAAgC;IAEhC,IAAI,KAAK,YAAY,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,0EAA0E;AAC1E,2EAA2E;AAC3E,MAAM,UAAU,iBAAiB,CAAC,KAAyB;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,oDAAoD;AACpD,KAAK,UAAU,WAAW,CACxB,OAAkB,EAClB,GAAW,EACX,IAAiB,EACjB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YAClC,GAAG,IAAI;YACP,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAkB,EAClB,GAAW,EACX,IAAiB,EACjB,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAkB,EAClB,GAAW,EACX,IAAiB,EACjB,SAAiB;IAEjB,OAAO,WAAW,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;AACpD,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "dotenv/config";

package/dist/index.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+import "dotenv/config";
+import { createRequire } from "node:module";
+import { loadConfig } from "./config.js";
+import { configureProxy } from "./proxy.js";
+import { runServer } from "./server.js";
+const require = createRequire(import.meta.url);
+const { version } = require("../package.json");
+async function main() {
+    const arguments_ = new Set(process.argv.slice(2));
+    if (arguments_.has("--version") || arguments_.has("-v")) {
+        console.log(version);
+        return;
+    }
+    if (arguments_.has("--help") || arguments_.has("-h")) {
+        console.log(`rrcrawl ${version}
+
+Tiny stdio MCP server for round-robin web scraping and crawling.
+
+Usage:
+  rrcrawl
+  rrcrawl --help
+  rrcrawl --version
+
+Configuration is read from environment variables and .env.
+See https://www.npmjs.com/package/rrcrawl for details.`);
+        return;
+    }
+    configureProxy();
+    await runServer(loadConfig());
+}
+main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAwB,CAAC;AAEtE,KAAK,UAAU,IAAI;IACjB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,IAAI,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO;;;;;;;;;;uDAUqB,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,cAAc,EAAE,CAAC;IACjB,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}

package/dist/providers/firecrawl.d.ts

@@ -0,0 +1,25 @@
+import { type FetchLike } from "../http.js";
+import type { CrawlProvider, CrawlRequest, CrawlResult, ScrapeProvider, ScrapeRequest, ScrapeResult } from "../types.js";
+export interface FirecrawlOptions {
+    apiUrl: string;
+    apiKey?: string;
+    requestTimeoutMs: number;
+    crawlTimeoutMs: number;
+    pollIntervalMs: number;
+    fetchFn?: FetchLike;
+    sleep?: (milliseconds: number) => Promise<void>;
+}
+export declare class FirecrawlProvider implements ScrapeProvider, CrawlProvider {
+    private readonly options;
+    readonly name: "firecrawl";
+    private readonly fetchFn;
+    private readonly sleep;
+    constructor(options: FirecrawlOptions);
+    private headers;
+    scrape(request: ScrapeRequest): Promise<ScrapeResult>;
+    crawl(request: CrawlRequest): Promise<CrawlResult>;
+    private collectDocuments;
+    private json;
+    private documentUrl;
+    private page;
+}

package/dist/providers/firecrawl.js

@@ -0,0 +1,135 @@
+import { isAbsoluteHttpUrl, requestJson, rethrowQuota, } from "../http.js";
+export class FirecrawlProvider {
+    options;
+    name = "firecrawl";
+    fetchFn;
+    sleep;
+    constructor(options) {
+        this.options = options;
+        this.fetchFn = options.fetchFn ?? fetch;
+        this.sleep =
+            options.sleep ??
+                ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
+    }
+    headers() {
+        return {
+            "content-type": "application/json",
+            ...(this.options.apiKey
+                ? { authorization: `Bearer ${this.options.apiKey}` }
+                : {}),
+        };
+    }
+    async scrape(request) {
+        const response = await this.json(`${this.options.apiUrl}/v2/scrape`, {
+            method: "POST",
+            headers: this.headers(),
+            body: JSON.stringify({
+                url: request.url,
+                formats: ["markdown"],
+                onlyMainContent: true,
+            }),
+        });
+        if (response.success === false) {
+            throw new Error(`Firecrawl scrape failed: ${response.error ?? "unknown error"}`);
+        }
+        if (!response.data?.markdown) {
+            throw new Error("Firecrawl returned no markdown content");
+        }
+        const url = this.documentUrl(response.data);
+        const page = this.page(response.data, isAbsoluteHttpUrl(url) ? url : request.url);
+        return { provider: this.name, ...page };
+    }
+    async crawl(request) {
+        const started = await this.json(`${this.options.apiUrl}/v2/crawl`, {
+            method: "POST",
+            headers: this.headers(),
+            body: JSON.stringify({
+                url: request.url,
+                limit: request.limit,
+                maxDiscoveryDepth: request.maxDepth,
+                includePaths: request.includePaths.length > 0 ? request.includePaths : undefined,
+                allowExternalLinks: request.allowExternal,
+                prompt: request.instructions,
+                scrapeOptions: {
+                    formats: ["markdown"],
+                    onlyMainContent: true,
+                },
+            }),
+        });
+        if (started.success === false) {
+            throw new Error(`Firecrawl crawl failed to start: ${started.error ?? "unknown error"}`);
+        }
+        if (!started.id) {
+            throw new Error("Firecrawl did not return a crawl job id");
+        }
+        const deadline = Date.now() + this.options.crawlTimeoutMs;
+        while (Date.now() < deadline) {
+            const status = await this.json(`${this.options.apiUrl}/v2/crawl/${encodeURIComponent(started.id)}`, { method: "GET", headers: this.headers() });
+            if (status.status === "completed") {
+                const documents = await this.collectDocuments(status, request.limit);
+                // Each crawl page must carry its own absolute URL. If Firecrawl omits
+                // it we drop the page rather than mislabel every one with the crawl root.
+                const pages = documents
+                    .map((document) => {
+                    const url = this.documentUrl(document);
+                    return document.markdown && isAbsoluteHttpUrl(url)
+                        ? this.page(document, url)
+                        : undefined;
+                })
+                    .filter((page) => page !== undefined);
+                if (pages.length === 0) {
+                    throw new Error("Firecrawl crawl completed without page content");
+                }
+                return { provider: this.name, pages };
+            }
+            if (status.status === "failed" || status.status === "cancelled") {
+                throw new Error(`Firecrawl crawl ${status.status}: ${status.error ?? "unknown error"}`);
+            }
+            await this.sleep(this.options.pollIntervalMs);
+        }
+        throw new Error(`Firecrawl crawl timed out after ${this.options.crawlTimeoutMs}ms`);
+    }
+    // Firecrawl paginates crawl results: `next` holds the URL of the following
+    // batch (returned when the response exceeds ~10MB). Follow it until null so
+    // large crawls don't silently return only the first batch. Bounded by the
+    // requested page limit — there can't be more result pages than documents.
+    async collectDocuments(first, limit) {
+        const documents = [...(first.data ?? [])];
+        let next = first.next;
+        for (let i = 0; next && documents.length < limit && i < limit; i += 1) {
+            const batch = await this.json(next, {
+                method: "GET",
+                headers: this.headers(),
+            });
+            documents.push(...(batch.data ?? []));
+            next = batch.next;
+        }
+        return documents;
+    }
+    // 402 Payment Required is Firecrawl's insufficient-credits signal (permanent).
+    // 429 rate limits are transient and deliberately left to normal failover.
+    async json(url, init) {
+        try {
+            return await requestJson(this.fetchFn, url, init, this.options.requestTimeoutMs);
+        }
+        catch (error) {
+            rethrowQuota(this.name, error, [402]);
+        }
+    }
+    documentUrl(document) {
+        return (document.metadata?.sourceURL ??
+            document.metadata?.sourceUrl ??
+            document.metadata?.url ??
+            undefined);
+    }
+    page(document, url) {
+        return {
+            url,
+            ...(document.metadata?.title
+                ? { title: document.metadata.title }
+                : {}),
+            markdown: document.markdown ?? "",
+        };
+    }
+}
+//# sourceMappingURL=firecrawl.js.map

package/dist/providers/firecrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../src/providers/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,YAAY,GAEb,MAAM,YAAY,CAAC;AAkDpB,MAAM,OAAO,iBAAiB;IAKC;IAJpB,IAAI,GAAG,WAAoB,CAAC;IACpB,OAAO,CAAY;IACnB,KAAK,CAA0C;IAEhE,YAA6B,OAAyB;QAAzB,YAAO,GAAP,OAAO,CAAkB;QACpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACxC,IAAI,CAAC,KAAK;YACR,OAAO,CAAC,KAAK;gBACb,CAAC,CAAC,YAAY,EAAE,EAAE,CAChB,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,OAAO;QACb,OAAO;YACL,cAAc,EAAE,kBAAkB;YAClC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM;gBACrB,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE;gBACpD,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAC9B,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,YAAY,EAClC;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACvB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,eAAe,EAAE,IAAI;aACtB,CAAC;SACH,CACF,CAAC;QAEF,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,4BAA4B,QAAQ,CAAC,KAAK,IAAI,eAAe,EAAE,CAChE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CACpB,QAAQ,CAAC,IAAI,EACb,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAC3C,CAAC;QACF,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAC7B,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,WAAW,EACjC;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACvB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,iBAAiB,EAAE,OAAO,CAAC,QAAQ;gBACnC,YAAY,EACV,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;gBACpE,kBAAkB,EAAE,OAAO,CAAC,aAAa;gBACzC,MAAM,EAAE,OAAO,CAAC,YAAY;gBAC5B,aAAa,EAAE;oBACb,OAAO,EAAE,CAAC,UAAU,CAAC;oBACrB,eAAe,EAAE,IAAI;iBACtB;aACF,CAAC;SACH,CACF,CAAC;QAEF,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,oCAAoC,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CACvE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAC1D,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAC5B,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,aAAa,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,EACnE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,CAC3C,CAAC;YAEF,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACrE,sEAAsE;gBACtE,0EAA0E;gBAC1E,MAAM,KAAK,GAAG,SAAS;qBACpB,GAAG,CAAC,CAAC,QAAQ,EAAoB,EAAE;oBAClC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBACvC,OAAO,QAAQ,CAAC,QAAQ,IAAI,iBAAiB,CAAC,GAAG,CAAC;wBAChD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,CAAC,CAAC,SAAS,CAAC;gBAChB,CAAC,CAAC;qBACD,MAAM,CAAC,CAAC,IAAI,EAAgB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;gBACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;gBACpE,CAAC;gBACD,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YACxC,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAChE,MAAM,IAAI,KAAK,CACb,mBAAmB,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CACvE,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,IAAI,KAAK,CACb,mCAAmC,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CACnE,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,4EAA4E;IAC5E,0EAA0E;IAC1E,0EAA0E;IAClE,KAAK,CAAC,gBAAgB,CAC5B,KAA8B,EAC9B,KAAa;QAEb,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,SAAS,CAAC,MAAM,GAAG,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACtE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAA0B,IAAI,EAAE;gBAC3D,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;aACxB,CAAC,CAAC;YACH,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;YACtC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACpB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,+EAA+E;IAC/E,0EAA0E;IAClE,KAAK,CAAC,IAAI,CAAI,GAAW,EAAE,IAAiB;QAClD,IAAI,CAAC;YACH,OAAO,MAAM,WAAW,CACtB,IAAI,CAAC,OAAO,EACZ,GAAG,EACH,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,QAA2B;QAC7C,OAAO,CACL,QAAQ,CAAC,QAAQ,EAAE,SAAS;YAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS;YAC5B,QAAQ,CAAC,QAAQ,EAAE,GAAG;YACtB,SAAS,CACV,CAAC;IACJ,CAAC;IAEO,IAAI,CAAC,QAA2B,EAAE,GAAW;QACnD,OAAO;YACL,GAAG;YACH,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK;gBAC1B,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE;gBACpC,CAAC,CAAC,EAAE,CAAC;YACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;SAClC,CAAC;IACJ,CAAC;CACF"}

package/dist/providers/scrapedo.d.ts

@@ -0,0 +1,15 @@
+import { type FetchLike } from "../http.js";
+import type { ScrapeProvider, ScrapeRequest, ScrapeResult } from "../types.js";
+export interface ScrapeDoOptions {
+    apiUrl: string;
+    apiToken?: string;
+    requestTimeoutMs: number;
+    fetchFn?: FetchLike;
+}
+export declare class ScrapeDoProvider implements ScrapeProvider {
+    private readonly options;
+    readonly name: "scrapedo";
+    private readonly fetchFn;
+    constructor(options: ScrapeDoOptions);
+    scrape(request: ScrapeRequest): Promise<ScrapeResult>;
+}

package/dist/providers/scrapedo.js

@@ -0,0 +1,51 @@
+import { HttpError, QuotaExceededError, requestText, } from "../http.js";
+export class ScrapeDoProvider {
+    options;
+    name = "scrapedo";
+    fetchFn;
+    constructor(options) {
+        this.options = options;
+        this.fetchFn = options.fetchFn ?? fetch;
+    }
+    async scrape(request) {
+        const endpoint = new URL(this.options.apiUrl);
+        endpoint.searchParams.set("url", request.url);
+        endpoint.searchParams.set("output", "markdown");
+        if (this.options.apiToken) {
+            endpoint.searchParams.set("token", this.options.apiToken);
+        }
+        // Scrape.do returns the *target's* HTTP status. Distinguish its own
+        // control-plane errors from the scraped page's status:
+        //   402 -> out of credits (permanent quota)
+        //   400/401/429 -> Scrape.do rejected the request (transient failover)
+        //   everything else (403/404/410/5xx) is the target page's own status;
+        //   Scrape.do still returns its rendered content, so use it.
+        let markdown;
+        try {
+            markdown = await requestText(this.fetchFn, endpoint.toString(), { method: "GET", headers: { accept: "text/markdown, text/plain, */*" } }, this.options.requestTimeoutMs);
+        }
+        catch (error) {
+            if (error instanceof HttpError) {
+                if (error.status === 402) {
+                    throw new QuotaExceededError(this.name, 402);
+                }
+                if (error.status === 400 || error.status === 401 || error.status === 429) {
+                    throw error;
+                }
+                markdown = error.responseBody;
+            }
+            else {
+                throw error;
+            }
+        }
+        if (!markdown.trim()) {
+            throw new Error("Scrape.do returned no markdown content");
+        }
+        return {
+            provider: this.name,
+            url: request.url,
+            markdown,
+        };
+    }
+}
+//# sourceMappingURL=scrapedo.js.map

package/dist/providers/scrapedo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrapedo.js","sourceRoot":"","sources":["../../src/providers/scrapedo.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,WAAW,GAEZ,MAAM,YAAY,CAAC;AAcpB,MAAM,OAAO,gBAAgB;IAIE;IAHpB,IAAI,GAAG,UAAmB,CAAC;IACnB,OAAO,CAAY;IAEpC,YAA6B,OAAwB;QAAxB,YAAO,GAAP,OAAO,CAAiB;QACnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9C,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9C,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1B,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC;QAED,oEAAoE;QACpE,uDAAuD;QACvD,4CAA4C;QAC5C,uEAAuE;QACvE,uEAAuE;QACvE,6DAA6D;QAC7D,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,WAAW,CAC1B,IAAI,CAAC,OAAO,EACZ,QAAQ,CAAC,QAAQ,EAAE,EACnB,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,gCAAgC,EAAE,EAAE,EACxE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;gBAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACzB,MAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC/C,CAAC;gBACD,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACzE,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}

package/dist/providers/tavily.d.ts

@@ -0,0 +1,18 @@
+import { type FetchLike } from "../http.js";
+import type { CrawlProvider, CrawlRequest, CrawlResult, ScrapeProvider, ScrapeRequest, ScrapeResult } from "../types.js";
+export interface TavilyOptions {
+    apiUrl: string;
+    apiKey?: string;
+    requestTimeoutMs: number;
+    fetchFn?: FetchLike;
+}
+export declare class TavilyProvider implements ScrapeProvider, CrawlProvider {
+    private readonly options;
+    readonly name: "tavily";
+    private readonly fetchFn;
+    constructor(options: TavilyOptions);
+    private headers;
+    scrape(request: ScrapeRequest): Promise<ScrapeResult>;
+    crawl(request: CrawlRequest): Promise<CrawlResult>;
+    private post;
+}

package/dist/providers/tavily.js

@@ -0,0 +1,74 @@
+import { isAbsoluteHttpUrl, requestJson, rethrowQuota, } from "../http.js";
+export class TavilyProvider {
+    options;
+    name = "tavily";
+    fetchFn;
+    constructor(options) {
+        this.options = options;
+        this.fetchFn = options.fetchFn ?? fetch;
+    }
+    headers() {
+        return {
+            "content-type": "application/json",
+            accept: "application/json",
+            ...(this.options.apiKey
+                ? { authorization: `Bearer ${this.options.apiKey}` }
+                : {}),
+        };
+    }
+    async scrape(request) {
+        const response = await this.post("/extract", {
+            urls: [request.url],
+            format: "markdown",
+            extract_depth: "basic",
+        });
+        const result = response.results?.[0];
+        if (!result?.raw_content) {
+            throw new Error("Tavily returned no markdown content");
+        }
+        return {
+            provider: this.name,
+            url: isAbsoluteHttpUrl(result.url) ? result.url : request.url,
+            markdown: result.raw_content,
+        };
+    }
+    async crawl(request) {
+        const response = await this.post("/crawl", {
+            url: request.url,
+            max_depth: request.maxDepth,
+            limit: request.limit,
+            select_paths: request.includePaths.length > 0 ? request.includePaths : undefined,
+            allow_external: request.allowExternal,
+            instructions: request.instructions,
+            format: "markdown",
+            extract_depth: "basic",
+        });
+        // Each crawl page must carry its own absolute URL; drop malformed ones
+        // rather than emit a relative URL (crashes output validation) or duplicate
+        // the crawl root across pages.
+        const pages = (response.results ?? [])
+            .map((result) => result.raw_content && isAbsoluteHttpUrl(result.url)
+            ? { url: result.url, markdown: result.raw_content }
+            : undefined)
+            .filter((page) => page !== undefined);
+        if (pages.length === 0) {
+            throw new Error("Tavily crawl returned no page content");
+        }
+        return { provider: this.name, pages };
+    }
+    // 432 (plan usage exceeded) and 433 (out of credits), plus 402, are Tavily's
+    // permanent quota signals. 429 is a transient rate limit, left to failover.
+    async post(path, body) {
+        try {
+            return await requestJson(this.fetchFn, `${this.options.apiUrl}${path}`, {
+                method: "POST",
+                headers: this.headers(),
+                body: JSON.stringify(body),
+            }, this.options.requestTimeoutMs);
+        }
+        catch (error) {
+            rethrowQuota(this.name, error, [402, 432, 433]);
+        }
+    }
+}
+//# sourceMappingURL=tavily.js.map

package/dist/providers/tavily.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tavily.js","sourceRoot":"","sources":["../../src/providers/tavily.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,YAAY,GAEb,MAAM,YAAY,CAAC;AA2BpB,MAAM,OAAO,cAAc;IAII;IAHpB,IAAI,GAAG,QAAiB,CAAC;IACjB,OAAO,CAAY;IAEpC,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC1C,CAAC;IAEO,OAAO;QACb,OAAO;YACL,cAAc,EAAE,kBAAkB;YAClC,MAAM,EAAE,kBAAkB;YAC1B,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM;gBACrB,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE;gBACpD,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAiB,UAAU,EAAE;YAC3D,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;YACnB,MAAM,EAAE,UAAU;YAClB,aAAa,EAAE,OAAO;SACvB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,GAAG,EAAE,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG;YAC7D,QAAQ,EAAE,MAAM,CAAC,WAAW;SAC7B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAiB,QAAQ,EAAE;YACzD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,OAAO,CAAC,QAAQ;YAC3B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EACV,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;YACpE,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM,EAAE,UAAU;YAClB,aAAa,EAAE,OAAO;SACvB,CAAC,CAAC;QACH,uEAAuE;QACvE,2EAA2E;QAC3E,+BAA+B;QAC/B,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;aACnC,GAAG,CAAC,CAAC,MAAM,EAAoB,EAAE,CAChC,MAAM,CAAC,WAAW,IAAI,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC;YACjD,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE;YACnD,CAAC,CAAC,SAAS,CACd;aACA,MAAM,CAAC,CAAC,IAAI,EAAgB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;IACxC,CAAC;IAED,6EAA6E;IAC7E,4EAA4E;IACpE,KAAK,CAAC,IAAI,CAChB,IAAY,EACZ,IAA6B;QAE7B,IAAI,CAAC;YACH,OAAO,MAAM,WAAW,CACtB,IAAI,CAAC,OAAO,EACZ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,EAC/B;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;gBACvB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF"}

package/dist/proxy.d.ts

@@ -0,0 +1,4 @@
+type Environment = Record<string, string | undefined>;
+export declare function proxyUrl(env: Environment): string | undefined;
+export declare function configureProxy(env?: Environment): string | undefined;
+export {};

package/dist/proxy.js

@@ -0,0 +1,23 @@
+import { ProxyAgent, setGlobalDispatcher } from "undici";
+// Node's global fetch (undici) ignores HTTPS_PROXY/HTTP_PROXY. Behind an egress
+// proxy (e.g. the OneCLI gateway) every request must be routed through a
+// ProxyAgent explicitly, or it bypasses the proxy and gets blocked.
+// ponytail: NO_PROXY is not honored — this server only talks to provider hosts,
+// all of which go through the gateway. Parse NO_PROXY if that ever changes.
+export function proxyUrl(env) {
+    return (env.HTTPS_PROXY ??
+        env.https_proxy ??
+        env.HTTP_PROXY ??
+        env.http_proxy ??
+        undefined);
+}
+// Installs a global proxy dispatcher when a proxy is configured. Returns the
+// proxy URL that was applied, or undefined when none is set.
+export function configureProxy(env = process.env) {
+    const url = proxyUrl(env);
+    if (url) {
+        setGlobalDispatcher(new ProxyAgent(url));
+    }
+    return url;
+}
+//# sourceMappingURL=proxy.js.map

package/dist/proxy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy.js","sourceRoot":"","sources":["../src/proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,QAAQ,CAAC;AAIzD,gFAAgF;AAChF,yEAAyE;AACzE,oEAAoE;AACpE,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,UAAU,QAAQ,CAAC,GAAgB;IACvC,OAAO,CACL,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,UAAU;QACd,GAAG,CAAC,UAAU;QACd,SAAS,CACV,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAC5B,MAAmB,OAAO,CAAC,GAAG;IAE9B,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,EAAE,CAAC;QACR,mBAAmB,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/router.d.ts

@@ -0,0 +1,26 @@
+import type { CrawlProvider, CrawlRequest, CrawlResult, ProviderName, ScrapeProvider, ScrapeRequest, ScrapeResult } from "./types.js";
+export declare class AllProvidersFailedError extends Error {
+    readonly operation: "scrape" | "crawl";
+    readonly failures: Array<{
+        provider: ProviderName;
+        message: string;
+    }>;
+    constructor(operation: "scrape" | "crawl", failures: Array<{
+        provider: ProviderName;
+        message: string;
+    }>);
+}
+export declare class ServiceUnavailableError extends Error {
+    readonly operation: "scrape" | "crawl";
+    constructor(operation: "scrape" | "crawl");
+}
+export declare class RoundRobinRouter {
+    private readonly scrapeProviders;
+    private readonly crawlProviders;
+    private readonly cursor;
+    private readonly disabled;
+    constructor(scrapeProviders: ScrapeProvider[], crawlProviders: CrawlProvider[]);
+    scrape(request: ScrapeRequest): Promise<ScrapeResult>;
+    crawl(request: CrawlRequest): Promise<CrawlResult>;
+    private tryProviders;
+}

package/dist/router.js

@@ -0,0 +1,81 @@
+import { QuotaExceededError } from "./http.js";
+export class AllProvidersFailedError extends Error {
+    operation;
+    failures;
+    constructor(operation, failures) {
+        super(`All ${operation} providers failed: ${failures
+            .map(({ provider, message }) => `${provider}: ${message}`)
+            .join("; ")}`);
+        this.operation = operation;
+        this.failures = failures;
+        this.name = "AllProvidersFailedError";
+    }
+}
+// No provider is available to serve the request: every configured provider has
+// been permanently disabled after exhausting its quota.
+export class ServiceUnavailableError extends Error {
+    operation;
+    constructor(operation) {
+        super(`Service unavailable: all ${operation} providers have exhausted their quota`);
+        this.operation = operation;
+        this.name = "ServiceUnavailableError";
+    }
+}
+export class RoundRobinRouter {
+    scrapeProviders;
+    crawlProviders;
+    cursor = {
+        scrape: 0,
+        crawl: 0,
+    };
+    // Providers that hit a hard quota. Account-level: disabling here removes the
+    // provider from both scrape and crawl rotations until the process restarts.
+    disabled = new Set();
+    constructor(scrapeProviders, crawlProviders) {
+        this.scrapeProviders = scrapeProviders;
+        this.crawlProviders = crawlProviders;
+        if (scrapeProviders.length === 0) {
+            throw new Error("At least one scrape provider is required");
+        }
+    }
+    scrape(request) {
+        return this.tryProviders("scrape", this.scrapeProviders, (provider) => provider.scrape(request));
+    }
+    crawl(request) {
+        if (this.crawlProviders.length === 0) {
+            throw new Error("No crawl-capable provider configured; enable Firecrawl or Tavily");
+        }
+        return this.tryProviders("crawl", this.crawlProviders, (provider) => provider.crawl(request));
+    }
+    async tryProviders(operation, providers, execute) {
+        const active = providers.filter((provider) => !this.disabled.has(provider.name));
+        if (active.length === 0) {
+            throw new ServiceUnavailableError(operation);
+        }
+        const start = this.cursor[operation] % active.length;
+        this.cursor[operation] += 1;
+        const failures = [];
+        for (let offset = 0; offset < active.length; offset += 1) {
+            const provider = active[(start + offset) % active.length];
+            try {
+                return await execute(provider);
+            }
+            catch (error) {
+                if (error instanceof QuotaExceededError) {
+                    this.disabled.add(provider.name);
+                }
+                failures.push({
+                    provider: provider.name,
+                    message: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        // If the quota errors just disabled the last provider, surface unavailable
+        // rather than a generic all-failed error.
+        if (providers.every((provider) => this.disabled.has(provider.name))) {
+            throw new ServiceUnavailableError(operation);
+        }
+        throw new AllProvidersFailedError(operation, failures);
+    }
+}
+//# sourceMappingURL=router.js.map

package/dist/router.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAW/C,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAErC;IACA;IAFX,YACW,SAA6B,EAC7B,QAA4D;QAErE,KAAK,CACH,OAAO,SAAS,sBAAsB,QAAQ;aAC3C,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,GAAG,QAAQ,KAAK,OAAO,EAAE,CAAC;aACzD,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;QAPO,cAAS,GAAT,SAAS,CAAoB;QAC7B,aAAQ,GAAR,QAAQ,CAAoD;QAOrE,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED,+EAA+E;AAC/E,wDAAwD;AACxD,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAC3B;IAArB,YAAqB,SAA6B;QAChD,KAAK,CACH,4BAA4B,SAAS,uCAAuC,CAC7E,CAAC;QAHiB,cAAS,GAAT,SAAS,CAAoB;QAIhD,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED,MAAM,OAAO,gBAAgB;IAUR;IACA;IAVF,MAAM,GAAuC;QAC5D,MAAM,EAAE,CAAC;QACT,KAAK,EAAE,CAAC;KACT,CAAC;IACF,6EAA6E;IAC7E,4EAA4E;IAC3D,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IAEpD,YACmB,eAAiC,EACjC,cAA+B;QAD/B,oBAAe,GAAf,eAAe,CAAkB;QACjC,mBAAc,GAAd,cAAc,CAAiB;QAEhD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,CAAC,OAAsB;QAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,QAAQ,EAAE,EAAE,CACpE,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CACzB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAqB;QACzB,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE,CAClE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CACxB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,SAA6B,EAC7B,SAAsB,EACtB,OAAkD;QAElD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAC7B,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAChD,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACrD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAuD,EAAE,CAAC;QAExE,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;oBACxC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACnC,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,QAAQ,CAAC,IAAI;oBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAChE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,0CAA0C;QAC1C,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,IAAI,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzD,CAAC;CACF"}

package/dist/server.d.ts

@@ -0,0 +1,6 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { AppConfig } from "./config.js";
+import { RoundRobinRouter } from "./router.js";
+export declare function createRouter(config: AppConfig): RoundRobinRouter;
+export declare function createServer(router: RoundRobinRouter): McpServer;
+export declare function runServer(config: AppConfig): Promise<void>;

package/dist/server.js

@@ -0,0 +1,160 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+import { FirecrawlProvider } from "./providers/firecrawl.js";
+import { ScrapeDoProvider } from "./providers/scrapedo.js";
+import { TavilyProvider } from "./providers/tavily.js";
+import { RoundRobinRouter } from "./router.js";
+const pageSchema = z.object({
+    url: z.string().url(),
+    title: z.string().optional(),
+    markdown: z.string(),
+});
+function toErrorResult(error) {
+    return {
+        content: [
+            {
+                type: "text",
+                text: error instanceof Error ? error.message : String(error),
+            },
+        ],
+        isError: true,
+    };
+}
+export function createRouter(config) {
+    const scrapeProviders = [];
+    const crawlProviders = [];
+    if (config.providers.includes("firecrawl")) {
+        const provider = new FirecrawlProvider({
+            ...config.firecrawl,
+            requestTimeoutMs: config.requestTimeoutMs,
+            crawlTimeoutMs: config.crawlTimeoutMs,
+            pollIntervalMs: config.pollIntervalMs,
+        });
+        scrapeProviders.push(provider);
+        crawlProviders.push(provider);
+    }
+    if (config.providers.includes("tavily")) {
+        const provider = new TavilyProvider({
+            ...config.tavily,
+            requestTimeoutMs: config.requestTimeoutMs,
+        });
+        scrapeProviders.push(provider);
+        crawlProviders.push(provider);
+    }
+    if (config.providers.includes("scrapedo")) {
+        scrapeProviders.push(new ScrapeDoProvider({
+            ...config.scrapedo,
+            requestTimeoutMs: config.requestTimeoutMs,
+        }));
+    }
+    return new RoundRobinRouter(scrapeProviders, crawlProviders);
+}
+export function createServer(router) {
+    const server = new McpServer({
+        name: "rrcrawl",
+        version: "0.1.0",
+    });
+    server.registerTool("scrape", {
+        title: "Scrape one URL",
+        description: "Fetch one URL as normalized Markdown. Requests rotate across all configured providers and fail over on errors.",
+        inputSchema: {
+            url: z.string().url().describe("HTTP or HTTPS URL to scrape"),
+        },
+        outputSchema: pageSchema.extend({
+            provider: z.enum(["firecrawl", "tavily", "scrapedo"]),
+        }),
+        annotations: {
+            readOnlyHint: true,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
+    }, async ({ url }) => {
+        try {
+            const result = await router.scrape({ url });
+            return {
+                content: [{ type: "text", text: JSON.stringify(result) }],
+                structuredContent: {
+                    provider: result.provider,
+                    url: result.url,
+                    ...(result.title ? { title: result.title } : {}),
+                    markdown: result.markdown,
+                },
+            };
+        }
+        catch (error) {
+            return toErrorResult(error);
+        }
+    });
+    server.registerTool("crawl", {
+        title: "Crawl a website",
+        description: "Crawl multiple pages as normalized Markdown. Requests rotate between configured crawl-capable providers (Firecrawl and Tavily) and fail over on errors.",
+        inputSchema: {
+            url: z.string().url().describe("Root HTTP or HTTPS URL"),
+            limit: z
+                .number()
+                .int()
+                .min(1)
+                .max(100)
+                .default(10)
+                .describe("Maximum number of pages"),
+            maxDepth: z
+                .number()
+                .int()
+                .min(1)
+                .max(5)
+                .default(1)
+                .describe("Maximum link depth from the root URL"),
+            includePaths: z
+                .array(z.string())
+                .max(20)
+                .default([])
+                .describe("Provider-compatible regex path filters"),
+            allowExternal: z
+                .boolean()
+                .default(false)
+                .describe("Allow pages outside the root domain"),
+            instructions: z
+                .string()
+                .max(2_000)
+                .optional()
+                .describe("Natural-language guidance for selecting pages"),
+        },
+        outputSchema: z.object({
+            provider: z.enum(["firecrawl", "tavily"]),
+            pages: z.array(pageSchema),
+        }),
+        annotations: {
+            readOnlyHint: true,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
+    }, async ({ url, limit, maxDepth, includePaths, allowExternal, instructions, }) => {
+        try {
+            const result = await router.crawl({
+                url,
+                limit,
+                maxDepth,
+                includePaths,
+                allowExternal,
+                instructions,
+            });
+            return {
+                content: [{ type: "text", text: JSON.stringify(result) }],
+                structuredContent: {
+                    provider: result.provider,
+                    pages: result.pages,
+                },
+            };
+        }
+        catch (error) {
+            return toErrorResult(error);
+        }
+    });
+    return server;
+}
+export async function runServer(config) {
+    const server = createServer(createRouter(config));
+    await server.connect(new StdioServerTransport());
+}
+//# sourceMappingURL=server.js.map

package/dist/server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAM/C,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACrB,CAAC,CAAC;AAEH,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D;SACF;QACD,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAiB;IAC5C,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;YACrC,GAAG,MAAM,CAAC,SAAS;YACnB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,GAAG,MAAM,CAAC,MAAM;YAChB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,eAAe,CAAC,IAAI,CAClB,IAAI,gBAAgB,CAAC;YACnB,GAAG,MAAM,CAAC,QAAQ;YAClB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,gBAAgB,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACnD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,OAAO;KACjB,CAAC,CAAC;IAEH,MAAM,CAAC,YAAY,CACjB,QAAQ,EACR;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,gHAAgH;QAClH,WAAW,EAAE;YACX,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;SAC9D;QACD,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC;YAC9B,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;SACtD,CAAC;QACF,WAAW,EAAE;YACX,YAAY,EAAE,IAAI;YAClB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;KACF,EACD,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5C,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,iBAAiB,EAAE;oBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChD,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,OAAO,EACP;QACE,KAAK,EAAE,iBAAiB;QACxB,WAAW,EACT,yJAAyJ;QAC3J,WAAW,EAAE;YACX,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;YACxD,KAAK,EAAE,CAAC;iBACL,MAAM,EAAE;iBACR,GAAG,EAAE;iBACL,GAAG,CAAC,CAAC,CAAC;iBACN,GAAG,CAAC,GAAG,CAAC;iBACR,OAAO,CAAC,EAAE,CAAC;iBACX,QAAQ,CAAC,yBAAyB,CAAC;YACtC,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,GAAG,EAAE;iBACL,GAAG,CAAC,CAAC,CAAC;iBACN,GAAG,CAAC,CAAC,CAAC;iBACN,OAAO,CAAC,CAAC,CAAC;iBACV,QAAQ,CAAC,sCAAsC,CAAC;YACnD,YAAY,EAAE,CAAC;iBACZ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,GAAG,CAAC,EAAE,CAAC;iBACP,OAAO,CAAC,EAAE,CAAC;iBACX,QAAQ,CAAC,wCAAwC,CAAC;YACrD,aAAa,EAAE,CAAC;iBACb,OAAO,EAAE;iBACT,OAAO,CAAC,KAAK,CAAC;iBACd,QAAQ,CAAC,qCAAqC,CAAC;YAClD,YAAY,EAAE,CAAC;iBACZ,MAAM,EAAE;iBACR,GAAG,CAAC,KAAK,CAAC;iBACV,QAAQ,EAAE;iBACV,QAAQ,CAAC,+CAA+C,CAAC;SAC7D;QACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC;YACrB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YACzC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;SAC3B,CAAC;QACF,WAAW,EAAE;YACX,YAAY,EAAE,IAAI;YAClB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;KACF,EACD,KAAK,EAAE,EACL,GAAG,EACH,KAAK,EACL,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,YAAY,GACb,EAAE,EAAE;QACH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;gBAChC,GAAG;gBACH,KAAK;gBACL,QAAQ;gBACR,YAAY;gBACZ,aAAa;gBACb,YAAY;aACb,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,iBAAiB,EAAE;oBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAiB;IAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAClD,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;AACnD,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,32 @@
+export type ProviderName = "firecrawl" | "tavily" | "scrapedo";
+export interface Page {
+    url: string;
+    title?: string;
+    markdown: string;
+}
+export interface ScrapeRequest {
+    url: string;
+}
+export interface ScrapeResult extends Page {
+    provider: ProviderName;
+}
+export interface CrawlRequest {
+    url: string;
+    limit: number;
+    maxDepth: number;
+    includePaths: string[];
+    allowExternal: boolean;
+    instructions?: string;
+}
+export interface CrawlResult {
+    provider: ProviderName;
+    pages: Page[];
+}
+export interface ScrapeProvider {
+    readonly name: ProviderName;
+    scrape(request: ScrapeRequest): Promise<ScrapeResult>;
+}
+export interface CrawlProvider {
+    readonly name: ProviderName;
+    crawl(request: CrawlRequest): Promise<CrawlResult>;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "rrcrawl",
+  "version": "0.1.0",
+  "description": "Tiny round-robin crawl MCP backed by Firecrawl, Tavily, and Scrape.do",
+  "license": "MIT",
+  "author": "Roman Shterenzon <roman.shterenzon@gmail.com>",
+  "homepage": "https://github.com/romanbsd/rrcrawl#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/romanbsd/rrcrawl.git"
+  },
+  "bugs": {
+    "url": "https://github.com/romanbsd/rrcrawl/issues"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "scrape",
+    "crawl",
+    "firecrawl",
+    "tavily",
+    "scrape.do"
+  ],
+  "type": "module",
+  "bin": {
+    "rrcrawl": "./dist/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    ".env.example"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "check": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.test.json --noEmit",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test": "vitest run",
+    "smoke": "tsx scripts/smoke.ts",
+    "prepack": "npm run build",
+    "prepublishOnly": "npm test && npm run check"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "1.29.0",
+    "dotenv": "^17.2.3",
+    "undici": "^8.8.0",
+    "zod": "^3.25.76"
+  },
+  "devDependencies": {
+    "@types/node": "^24.10.1",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
+  },
+  "overrides": {
+    "@hono/node-server": "2.0.11"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}