scoutline 0.9.0 → 0.10.0

package/dist/providers/firecrawl/adapter.js

@@ -0,0 +1,962 @@
+/**
+ * Firecrawl Provider Adapter (firecrawl tech-plan §1, D3–D6).
+ *
+ * Implements the Firecrawl Provider Descriptor with Search, Reader, Map,
+ * async Crawl, Quota, and Diagnostics capabilities on top of the
+ * direct-HTTP transport (`./client.ts`). The Adapter owns credentials,
+ * transport lifecycle, Provider field mapping, and failure normalization;
+ * shared execution owns cache and retry policy.
+ *
+ * Boundary rules (ARCHITECTURE.md §2):
+ *   - May import capability types, normalized errors, Provider identity
+ *     types, and the Adapter-local credential and transport Modules.
+ *   - Must NOT import command presentation, output mode, or another
+ *     Provider's Adapter.
+ *
+ * Field mapping (tech-plan D4/D5/D6; investigation §3.1–§3.4):
+ *   Search data[].title       -> title
+ *   Search data[].url         -> url
+ *   Search data[].markdown    -> summary  (high content-size; else description)
+ *   Search data[].description -> summary  (medium content-size fallback)
+ *
+ *   Scrape data.markdown|text -> content
+ *   Scrape data.metadata.sourceURL -> finalUrl
+ *   Scrape data.metadata.title     -> title  (better than Tavily's null)
+ *
+ *   Map links[]               -> urls
+ */
+import crypto from "node:crypto";
+import * as fs from "node:fs/promises";
+import path from "node:path";
+import { decodeReaderFetchResult } from "../../capabilities/reader.js";
+import { decodeMapResult } from "../../capabilities/map.js";
+import { decodeCrawlResult } from "../../capabilities/crawl.js";
+import { computeAsyncJobStateHash, createProductionAsyncJobStateFile, } from "../../lib/async-job-state.js";
+import { asyncJobStateDir } from "../../lib/cache.js";
+import { ApiError, AuthError, ConfigurationError, NetworkError, QuotaError, TimeoutError, UnsupportedOptionError, ValidationError, } from "../../lib/errors.js";
+import { createFirecrawlCrawl, fetchFirecrawlCrawlNext, fetchFirecrawlMap, fetchFirecrawlScrape, fetchFirecrawlSearch, listActiveFirecrawlCrawls, pollFirecrawlCrawl, } from "./client.js";
+import { isFirecrawlConfigured, requireFirecrawlApiKey } from "./credentials.js";
+import { createFirecrawlQuotaCapability } from "./quota.js";
+import { createFirecrawlDiagnosticsCapability } from "./diagnostics.js";
+// ---------------------------------------------------------------------------
+// Credential + shape helpers
+// ---------------------------------------------------------------------------
+function credentialFingerprint(apiKey) {
+    return crypto.createHash("sha256").update(apiKey).digest("hex");
+}
+function resolveApiKey(env) {
+    return requireFirecrawlApiKey(env);
+}
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function assertHttpUrl(url) {
+    if (typeof url !== "string" || url.length === 0) {
+        throw new ValidationError("Firecrawl reader URL must be a non-empty string");
+    }
+    if (!/^https?:\/\//.test(url)) {
+        throw new ValidationError("URL must start with http:// or https://");
+    }
+}
+// ---------------------------------------------------------------------------
+// Search control mapping (SearchControls → Firecrawl-native API params)
+// ---------------------------------------------------------------------------
+/**
+ * Map `--recency` to Firecrawl's Google-style `tbs` time filter
+ * (investigation §3.2). `noLimit` and absent recency omit the param.
+ */
+function mapRecencyToTbs(recency) {
+    switch (recency) {
+        case "oneDay":
+            return "qdr:d";
+        case "oneWeek":
+            return "qdr:w";
+        case "oneMonth":
+            return "qdr:m";
+        case "oneYear":
+            return "qdr:y";
+        case "noLimit":
+            return undefined;
+        default:
+            return undefined;
+    }
+}
+/**
+ * Map Provider-neutral `SearchControls` to Firecrawl-native search params.
+ *
+ *   domain      -> includeDomains: [domain]
+ *   recency     -> tbs (qdr:*)
+ *   contentSize -> scrapeOptions.formats:["markdown"]  (high only)
+ *   topic       -> sources: news → [{type:"news"}], else [{type:"web"}]
+ *
+ * `location` is rejected in `validate` (Z.AI-specific). `sources` always
+ * carries a default of `[{type:"web"}]` per the investigation's
+ * recommended default.
+ */
+function mapSearchControls(controls) {
+    const topic = controls?.topic;
+    const sources = [{ type: topic === "news" ? "news" : "web" }];
+    const params = { sources };
+    if (controls?.domain) {
+        params.includeDomains = [controls.domain];
+    }
+    if (controls?.recency) {
+        const tbs = mapRecencyToTbs(controls.recency);
+        if (tbs)
+            params.tbs = tbs;
+    }
+    if (controls?.contentSize === "high") {
+        params.scrapeOptions = { formats: ["markdown"] };
+    }
+    return params;
+}
+// ---------------------------------------------------------------------------
+// Response normalization
+// ---------------------------------------------------------------------------
+/**
+ * Normalize a raw Firecrawl search response into `SearchSource[]`.
+ *
+ * Firecrawl nests the result list under a source-type key inside `data`
+ * (`data.web` for a web search, `data.news` for a news-topic search) rather
+ * than as a flat `data` array. A flat `data` array is also accepted for
+ * forward-compat.
+ *
+ *   data.web[].title        -> title
+ *   data.web[].url          -> url
+ *   data.web[].markdown     -> summary  (high content-size — richer)
+ *   data.web[].description  -> summary  (medium fallback)
+ *   data.web[].source|category -> source
+ *
+ * Any malformed shape is a retryable `ApiError` 500.
+ */
+function normalizeFirecrawlSearchResults(raw) {
+    if (!isPlainObject(raw)) {
+        throw new ApiError("Firecrawl search returned a malformed response", 500);
+    }
+    const data = raw.data;
+    let results;
+    if (Array.isArray(data)) {
+        results = data;
+    }
+    else if (isPlainObject(data)) {
+        // Results are keyed by source type (web for the default, news for a
+        // news-topic search). Collect from web, falling back to news.
+        const web = data.web;
+        results =
+            Array.isArray(web) && web.length > 0 ? web : Array.isArray(data.news) ? data.news : undefined;
+    }
+    if (!Array.isArray(results)) {
+        throw new ApiError("Firecrawl search returned a malformed response", 500);
+    }
+    const out = [];
+    for (const entry of results) {
+        if (!isPlainObject(entry)) {
+            throw new ApiError("Firecrawl search returned a malformed response", 500);
+        }
+        const title = entry.title;
+        const url = entry.url;
+        if (typeof title !== "string" || typeof url !== "string") {
+            throw new ApiError("Firecrawl search returned a malformed response", 500);
+        }
+        // Prefer the scraped markdown (high content-size) over the bare
+        // description (medium). An absent description is an empty summary.
+        const markdown = typeof entry.markdown === "string" ? entry.markdown : undefined;
+        const description = typeof entry.description === "string" ? entry.description : undefined;
+        const summary = markdown ?? description ?? "";
+        const source = typeof entry.source === "string"
+            ? entry.source
+            : typeof entry.category === "string"
+                ? entry.category
+                : undefined;
+        const result = { title, url, summary };
+        if (source !== undefined)
+            result.source = source;
+        out.push(result);
+    }
+    return out;
+}
+/**
+ * Normalize a raw Firecrawl scrape response into a `ReaderFetchResult`.
+ *
+ *   data.markdown|text          -> content  (per requested format)
+ *   data.metadata.sourceURL     -> finalUrl
+ *   data.metadata.title         -> title    (null when absent/blank)
+ *
+ * Firecrawl returns a genuine title (better than Tavily's null). Any
+ * malformed shape is a retryable `ApiError` 500.
+ */
+function normalizeFirecrawlScrapeResult(raw, request) {
+    if (!isPlainObject(raw)) {
+        throw new ApiError("Firecrawl scrape returned a malformed response", 500);
+    }
+    const data = raw.data;
+    if (!isPlainObject(data)) {
+        throw new ApiError("Firecrawl scrape returned a malformed response", 500);
+    }
+    const contentFormat = request.format ?? "markdown";
+    const contentField = contentFormat === "text" ? "text" : "markdown";
+    const content = data[contentField];
+    if (typeof content !== "string" || content.length === 0) {
+        throw new ApiError("Firecrawl scrape returned a malformed response", 500);
+    }
+    const metadata = isPlainObject(data.metadata) ? data.metadata : {};
+    const sourceURL = typeof metadata.sourceURL === "string" && metadata.sourceURL.length > 0
+        ? metadata.sourceURL
+        : undefined;
+    const finalUrl = sourceURL ?? request.url;
+    const rawTitle = typeof metadata.title === "string" ? metadata.title : undefined;
+    const title = rawTitle !== undefined && rawTitle.trim().length > 0 ? rawTitle : null;
+    return {
+        schemaVersion: 1,
+        url: request.url,
+        finalUrl,
+        title,
+        content,
+        contentFormat,
+    };
+}
+/**
+ * Normalize a raw Firecrawl map response into a `MapResult`.
+ *
+ *   links[].url -> urls   (links are objects {url, title}, not strings)
+ *   baseUrl     -> the request URL
+ *   totalUrls   -> links array length
+ *
+ * A bare-string `links[]` entry is also accepted for forward-compat. Any
+ * malformed shape is a retryable `ApiError` 500.
+ */
+function normalizeFirecrawlMapResult(raw, request) {
+    if (!isPlainObject(raw)) {
+        throw new ApiError("Firecrawl map returned a malformed response", 500);
+    }
+    const links = raw.links;
+    if (!Array.isArray(links)) {
+        throw new ApiError("Firecrawl map returned a malformed response", 500);
+    }
+    const urls = [];
+    for (const entry of links) {
+        let url;
+        if (typeof entry === "string") {
+            url = entry;
+        }
+        else if (isPlainObject(entry)) {
+            url = entry.url;
+        }
+        if (typeof url !== "string" || url.length === 0) {
+            throw new ApiError("Firecrawl map returned a malformed response", 500);
+        }
+        urls.push(url);
+    }
+    return {
+        schemaVersion: 1,
+        baseUrl: request.url,
+        urls,
+        totalUrls: urls.length,
+    };
+}
+// ---------------------------------------------------------------------------
+// Failure normalization: stable public codes, no raw payloads (NFR-006)
+// ---------------------------------------------------------------------------
+/**
+ * Resolve a stable HTTP-style status code for retry classification.
+ * Explicit typed errors carry their own status; the fallback default is
+ * 500 (transient). Mirrors the Tavily helper.
+ */
+function inferStatusCode(known) {
+    if (typeof known === "number" && Number.isFinite(known))
+        return known;
+    return 500;
+}
+/**
+ * Status-keyed outward message for rewrapped Firecrawl ApiErrors. Curated
+ * constants only — a raw Provider body embedded upstream never survives.
+ */
+function firecrawlApiErrorMessage(statusCode) {
+    if (statusCode === 429)
+        return "Firecrawl rate limit exceeded";
+    return "Firecrawl request failed";
+}
+/**
+ * Normalize a Provider failure with sanitized messages. Raw response
+ * bodies never cross the adapter boundary. Mirrors `normalizeTavilyError`.
+ */
+function normalizeFirecrawlError(error) {
+    // QuotaError pass-through — terminal retry guarantee preserved.
+    if (error instanceof QuotaError)
+        return error;
+    // Configuration/option/validation errors carry clean, human-authored
+    // messages and are safe to surface verbatim.
+    if (error instanceof ValidationError ||
+        error instanceof UnsupportedOptionError ||
+        error instanceof ConfigurationError) {
+        return error;
+    }
+    // Re-wrap typed transport errors with sanitized messages so a raw
+    // Provider response body embedded upstream never survives.
+    if (error instanceof AuthError) {
+        return new AuthError("Firecrawl authentication failed", "FIRECRAWL_API_KEY");
+    }
+    if (error instanceof NetworkError) {
+        return new NetworkError("Firecrawl network error");
+    }
+    if (error instanceof TimeoutError) {
+        return new TimeoutError(error.durationMs, "Try again or increase timeout with FIRECRAWL_TIMEOUT env var");
+    }
+    if (error instanceof ApiError) {
+        const statusCode = inferStatusCode(error.statusCode);
+        return new ApiError(firecrawlApiErrorMessage(statusCode), statusCode);
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    const lower = message.toLowerCase();
+    if (lower.includes("401") ||
+        lower.includes("403") ||
+        lower.includes("unauthorized") ||
+        lower.includes("forbidden")) {
+        return new AuthError("Firecrawl authentication failed");
+    }
+    if (lower.includes("timeout") || lower.includes("timed out") || lower.includes("etimedout")) {
+        return new TimeoutError(30000);
+    }
+    if (lower.includes("econnrefused") ||
+        lower.includes("econnreset") ||
+        lower.includes("network") ||
+        lower.includes("enotfound") ||
+        lower.includes("fetch failed")) {
+        return new NetworkError("Firecrawl network error");
+    }
+    if (lower.includes("429") || lower.includes("rate limit")) {
+        return new ApiError("Firecrawl rate limit exceeded", 429);
+    }
+    return new ApiError("Firecrawl request failed", 500);
+}
+// ---------------------------------------------------------------------------
+// Reader validation helpers
+// ---------------------------------------------------------------------------
+/** Z.AI-only reader options that Firecrawl does not accept. */
+const UNSUPPORTED_READER_OPTIONS = [
+    "withLinksSummary",
+    "noGfm",
+    "keepImgDataUrl",
+    "withImagesSummary",
+];
+function assertNoUnsupportedReaderOptions(request) {
+    for (const key of UNSUPPORTED_READER_OPTIONS) {
+        // Only reject when the user explicitly enabled the option (`true`).
+        // The read command handler sets boolean options to `false` (not
+        // `undefined`) when the flag is absent, so `!== undefined` would
+        // over-reject. `false` means "user didn't pass the flag" → accept.
+        if (request[key] === true) {
+            throw new UnsupportedOptionError("firecrawl", "reader", key);
+        }
+    }
+}
+function createFirecrawlSearchCapability(options) {
+    const { env, transport } = options;
+    const capability = {
+        validate(request) {
+            if (!request || typeof request.query !== "string" || request.query.trim() === "") {
+                throw new ValidationError("Search query must contain at least one non-whitespace character");
+            }
+            // Firecrawl supports domain, recency, contentSize, and topic.
+            // location is Z.AI-specific and rejected before any transport call.
+            if (request.controls?.location !== undefined) {
+                throw new UnsupportedOptionError("firecrawl", "search", "location");
+            }
+            // Firecrawl sources are web/news only; --topic finance has no native
+            // source mapping — reject explicitly rather than silently returning
+            // generic web results (supported topics: general, news).
+            if (request.controls?.topic === "finance") {
+                throw new ValidationError("Firecrawl search does not support --topic finance (supported: general, news)");
+            }
+        },
+        cacheIdentity(request) {
+            const apiKey = resolveApiKey(env);
+            const identityRequest = {
+                query: request.query,
+            };
+            if (request.controls) {
+                identityRequest.controls = request.controls;
+            }
+            return {
+                provider: "firecrawl",
+                capability: "search",
+                credentialFingerprint: credentialFingerprint(apiKey),
+                request: identityRequest,
+            };
+        },
+        async invoke(request) {
+            capability.validate(request);
+            const apiKey = resolveApiKey(env);
+            try {
+                const params = mapSearchControls(request.controls);
+                const raw = await fetchFirecrawlSearch(apiKey, request.query, params, transport);
+                return normalizeFirecrawlSearchResults(raw);
+            }
+            catch (error) {
+                throw normalizeFirecrawlError(error);
+            }
+        },
+    };
+    return capability;
+}
+function createFirecrawlReaderCapability(options) {
+    const { env, transport } = options;
+    const fetch = {
+        kind: "reader-fetch",
+        validate(request) {
+            assertHttpUrl(request.url);
+            assertNoUnsupportedReaderOptions(request);
+        },
+        cacheIdentity(request) {
+            const apiKey = resolveApiKey(env);
+            return {
+                provider: "firecrawl",
+                capability: "reader",
+                operation: "reader-fetch",
+                credentialFingerprint: credentialFingerprint(apiKey),
+                request,
+                legacyCandidates: [],
+            };
+        },
+        decodeCached(value) {
+            return decodeReaderFetchResult(value);
+        },
+        async invoke(request) {
+            fetch.validate(request);
+            const apiKey = resolveApiKey(env);
+            try {
+                const formats = request.format === "text" ? ["text"] : ["markdown"];
+                // retainImages is set only when --no-images is passed (read.ts),
+                // so `=== false` is exactly the "strip images" case. Inverted to
+                // Firecrawl's removeBase64Images.
+                const params = {
+                    formats,
+                    proxy: "basic",
+                    ...(request.retainImages === false ? { removeBase64Images: true } : {}),
+                };
+                const raw = await fetchFirecrawlScrape(apiKey, request.url, params, transport);
+                return normalizeFirecrawlScrapeResult(raw, request);
+            }
+            catch (error) {
+                throw normalizeFirecrawlError(error);
+            }
+        },
+    };
+    return { fetch };
+}
+// ---------------------------------------------------------------------------
+// Map Capability
+// ---------------------------------------------------------------------------
+/**
+ * Map Provider-neutral `MapRequest` to Firecrawl-native map params.
+ *
+ *   limit           -> limit
+ *   instructions    -> search
+ *
+ * `depth`, `breadth`, `selectPaths`, and `excludePaths` have NO native
+ * Firecrawl /v2/map equivalent (map is discovery-only) and are omitted
+ * (tech-plan D6).
+ */
+function mapMapControls(request) {
+    const params = {};
+    if (request.limit !== undefined)
+        params.limit = request.limit;
+    if (request.instructions !== undefined)
+        params.search = request.instructions;
+    return params;
+}
+function createFirecrawlMapCapability(options) {
+    const { env, transport } = options;
+    const fetch = {
+        kind: "map-fetch",
+        validate(request) {
+            assertHttpUrl(request.url);
+            if (request.limit !== undefined && request.limit <= 0) {
+                throw new ValidationError("Map limit must be greater than 0");
+            }
+            // Firecrawl /v2/map supports only url, limit, and search(instructions).
+            // depth/breadth/selectPaths/excludePaths have no native map equivalent
+            // (Tavily map supports them) — reject rather than silently discard.
+            if (request.depth !== undefined) {
+                throw new UnsupportedOptionError("firecrawl", "map", "depth");
+            }
+            if (request.breadth !== undefined) {
+                throw new UnsupportedOptionError("firecrawl", "map", "breadth");
+            }
+            if (request.selectPaths !== undefined) {
+                throw new UnsupportedOptionError("firecrawl", "map", "selectPaths");
+            }
+            if (request.excludePaths !== undefined) {
+                throw new UnsupportedOptionError("firecrawl", "map", "excludePaths");
+            }
+        },
+        cacheIdentity(request) {
+            const apiKey = resolveApiKey(env);
+            return {
+                provider: "firecrawl",
+                capability: "map",
+                credentialFingerprint: credentialFingerprint(apiKey),
+                request,
+            };
+        },
+        decodeCached(value) {
+            return decodeMapResult(value);
+        },
+        async invoke(request) {
+            fetch.validate(request);
+            const apiKey = resolveApiKey(env);
+            try {
+                const params = mapMapControls(request);
+                const raw = await fetchFirecrawlMap(apiKey, request.url, params, transport);
+                return normalizeFirecrawlMapResult(raw, request);
+            }
+            catch (error) {
+                throw normalizeFirecrawlError(error);
+            }
+        },
+    };
+    return { fetch };
+}
+// ---------------------------------------------------------------------------
+// Crawl Capability — async create→poll→resume (tech-plan D2)
+// ---------------------------------------------------------------------------
+const DEFAULT_CRAWL_POLL_INTERVAL_MS = 2000;
+/** Safety bound on `next`-cursor traversal for very large crawls. */
+const MAX_CRAWL_NEXT_ITERATIONS = 500;
+/** Reclaim-on-miss staleness guard — don't adopt jobs older than this. */
+const CRAWL_RECLAIM_STALE_MS = 24 * 60 * 60 * 1000;
+/** Split a comma-separated path-pattern string into a trimmed array. */
+function splitPathPatterns(value) {
+    if (value === undefined || value.trim() === "")
+        return undefined;
+    return value
+        .split(",")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+function isEexistError(err) {
+    return (typeof err === "object" &&
+        err !== null &&
+        "code" in err &&
+        err.code === "EEXIST");
+}
+function resolveCrawlPollIntervalMs(env) {
+    const raw = env?.FIRECRAWL_CRAWL_POLL_INTERVAL_MS;
+    const parsed = parseInt(raw ?? "", 10);
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_CRAWL_POLL_INTERVAL_MS;
+}
+/**
+ * Abortable sleep built from the injected timers. When `signal` aborts
+ * (the command handler's `--timeout`), the pending timer is cleared and
+ * the promise rejects with a `TimeoutError` so the poll loop unwinds
+ * promptly. Mirrors the research poll loop's `makeSleep`.
+ */
+function makeCrawlSleep(deps, signal) {
+    const setT = deps?.setTimeout ?? setTimeout;
+    const clearT = deps?.clearTimeout ?? clearTimeout;
+    return (ms) => new Promise((resolve, reject) => {
+        if (signal?.aborted) {
+            reject(new TimeoutError(0, "Crawl polling aborted"));
+            return;
+        }
+        if (ms <= 0) {
+            setImmediate(() => {
+                if (signal?.aborted) {
+                    reject(new TimeoutError(0, "Crawl polling aborted"));
+                    return;
+                }
+                resolve();
+            });
+            return;
+        }
+        const onAbort = () => {
+            clearT(id);
+            reject(new TimeoutError(0, "Crawl polling aborted"));
+        };
+        const id = setT(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener("abort", onAbort);
+    });
+}
+/**
+ * Map a Provider-neutral `CrawlRequest` into Firecrawl-native /v2/crawl
+ * body fields. `breadth` has no Firecrawl equivalent (rejected in
+ * `validate`); `proxy` is pinned to `"basic"` (D9 cost-safety) and nests
+ * under `scrapeOptions` (crawl nests scrape fields; `/scrape` takes
+ * `proxy` top-level). `format` nests under `scrapeOptions.formats`.
+ */
+function mapCrawlControls(request) {
+    const contentFormat = request.format ?? "markdown";
+    const params = { scrapeOptions: { formats: [contentFormat], proxy: "basic" } };
+    if (request.depth !== undefined)
+        params.maxDepth = request.depth;
+    if (request.limit !== undefined)
+        params.limit = request.limit;
+    const includePaths = splitPathPatterns(request.selectPaths);
+    if (includePaths !== undefined)
+        params.includePaths = includePaths;
+    const excludePaths = splitPathPatterns(request.excludePaths);
+    if (excludePaths !== undefined)
+        params.excludePaths = excludePaths;
+    return params;
+}
+/**
+ * Normalize a batch of Firecrawl crawl page objects into `CrawlPage[]`.
+ *
+ *   data[].metadata.sourceURL -> url
+ *   data[].markdown|text      -> content
+ *
+ * Any malformed entry is a retryable `ApiError` 500.
+ */
+function normalizeCrawlPages(data, request) {
+    const contentFormat = request.format ?? "markdown";
+    const contentField = contentFormat === "text" ? "text" : "markdown";
+    const pages = [];
+    for (const entry of data) {
+        if (!isPlainObject(entry)) {
+            throw new ApiError("Firecrawl crawl returned a malformed response", 500);
+        }
+        const metadata = isPlainObject(entry.metadata) ? entry.metadata : {};
+        const url = metadata.sourceURL;
+        const content = entry[contentField];
+        if (typeof url !== "string" ||
+            url.length === 0 ||
+            typeof content !== "string" ||
+            content.length === 0) {
+            throw new ApiError("Firecrawl crawl returned a malformed response", 500);
+        }
+        pages.push({ url, content, contentFormat });
+    }
+    return pages;
+}
+/**
+ * Collect the full crawl result from a completed poll, following the
+ * pagination cursor `next` to exhaustion for large sets (each batch
+ * distinct — no dedup needed; tech-plan D2 / G0 #1). Bounded by a
+ * max-iteration guard against a runaway cursor.
+ */
+async function collectCrawlResult(poll, apiKey, request, transport) {
+    const pages = [];
+    if (poll.data)
+        pages.push(...normalizeCrawlPages(poll.data, request));
+    let next = poll.next;
+    let guard = 0;
+    while (next !== undefined) {
+        guard += 1;
+        if (guard > MAX_CRAWL_NEXT_ITERATIONS) {
+            throw new ApiError("Firecrawl crawl pagination exceeded the safety limit", 500);
+        }
+        const page = await fetchFirecrawlCrawlNext(apiKey, next, transport);
+        if (page.data)
+            pages.push(...normalizeCrawlPages(page.data, request));
+        next = page.next;
+    }
+    return { schemaVersion: 1, baseUrl: request.url, pages, totalPages: pages.length };
+}
+/**
+ * Unordered array equality for path-filter matching. `undefined` expected
+ * means "no constraint requested" (always compatible).
+ */
+function stringArrayEqualUnordered(a, expected) {
+    if (expected === undefined)
+        return true;
+    if (!Array.isArray(a))
+        return false;
+    return JSON.stringify([...a].sort()) === JSON.stringify([...expected].sort());
+}
+/**
+ * Best-effort compatibility check between an active-job `options` blob and
+ * the params this request would send. The server echoes the request
+ * options; verifying the cost-bearing fields (limit, maxDepth, path
+ * filters, scrapeOptions.formats) is a strong signal it is the same job.
+ * Missing or differently-shaped options → not compatible (safer to create
+ * fresh than to mis-adopt a different crawl and return the wrong pages).
+ */
+function crawlOptionsCompatible(options, params) {
+    if (!isPlainObject(options))
+        return false;
+    if (params.maxDepth !== undefined && options.maxDepth !== params.maxDepth)
+        return false;
+    if (params.limit !== undefined && options.limit !== params.limit)
+        return false;
+    const expectedFormats = params.scrapeOptions?.formats;
+    const so = options.scrapeOptions;
+    if (expectedFormats !== undefined && isPlainObject(so) && Array.isArray(so.formats)) {
+        const got = JSON.stringify([...so.formats].sort());
+        const want = JSON.stringify([...expectedFormats].sort());
+        if (got !== want)
+            return false;
+    }
+    // Path filters determine which pages are crawled (cost AND result
+    // content) — a mismatched filter would adopt a differently-scoped crawl.
+    if (!stringArrayEqualUnordered(options.includePaths, params.includePaths))
+        return false;
+    if (!stringArrayEqualUnordered(options.excludePaths, params.excludePaths))
+        return false;
+    return true;
+}
+/**
+ * Reclaim-on-miss: find an in-flight job matching this request by `url`
+ * (with a `created_at` recency guard against adopting stale jobs, and a
+ * best-effort options check when the server echoes them). A missing or
+ * unparseable `created_at` is treated as stale (skipped) so a zombie entry
+ * is never adopted. Returns the job id, or `undefined` when no match exists.
+ */
+function matchActiveCrawl(active, request, params) {
+    const now = Date.now();
+    for (const entry of active) {
+        if (entry.url !== request.url)
+            continue;
+        const ts = entry.created_at !== undefined ? Date.parse(entry.created_at) : NaN;
+        if (!Number.isFinite(ts) || now - ts > CRAWL_RECLAIM_STALE_MS)
+            continue;
+        if (entry.options !== undefined && !crawlOptionsCompatible(entry.options, params))
+            continue;
+        return entry.id;
+    }
+    return undefined;
+}
+/**
+ * Persist a crawl job id in the state file (atomic `wx` create). On EEXIST
+ * (a concurrent invocation already persisted a job for this request), read
+ * and return its id instead — the concurrent job is the one to poll.
+ */
+async function persistCrawlId(stateFile, identityHash, id) {
+    const state = {
+        requestId: id,
+        identityHash,
+        createdAt: new Date().toISOString(),
+        status: "pending",
+    };
+    try {
+        await stateFile.write(identityHash, state);
+        return id;
+    }
+    catch (err) {
+        if (isEexistError(err)) {
+            const existing = await stateFile.read(identityHash);
+            return existing !== null ? existing.requestId : id;
+        }
+        throw err;
+    }
+}
+// ---------------------------------------------------------------------------
+// Concurrent-create lock (cost-safety — review C3)
+// ---------------------------------------------------------------------------
+/** How long to wait for a contended crawl create-lock before giving up. */
+const CRAWL_LOCK_TIMEOUT_MS = 30000;
+/** A lock older than this is treated as stale (holder died) and broken. */
+const CRAWL_LOCK_STALE_MS = 10 * 60 * 1000;
+/**
+ * Serialize the listActive→create→persist critical section per request so two
+ * concurrent identical crawls can't both POST (and charge) a job — the second
+ * waits, then reclaims the first's job from `/active` instead of re-POSTing.
+ * Uses an exclusive `wx`-create lockfile sibling to the state file; a stale
+ * lock (holder crashed) is broken after {@link CRAWL_LOCK_STALE_MS}.
+ *
+ * When `stateDir` is undefined (in-memory test mode), the lock is a no-op —
+ * tests are single-process and need no cross-process serialization.
+ */
+async function withCrawlLock(stateDir, identityHash, fn, deps) {
+    if (stateDir === undefined)
+        return fn();
+    const setT = deps?.setTimeout ?? setTimeout;
+    const sleep = (ms) => new Promise((r) => setT(() => r(), ms));
+    await fs.mkdir(stateDir, { recursive: true }).catch(() => { });
+    const lockPath = path.join(stateDir, `${identityHash}.lock`);
+    const deadline = Date.now() + CRAWL_LOCK_TIMEOUT_MS;
+    for (;;) {
+        try {
+            const handle = await fs.open(lockPath, "wx");
+            try {
+                return await fn();
+            }
+            finally {
+                await handle.close().catch(() => { });
+                await fs.unlink(lockPath).catch(() => { });
+            }
+        }
+        catch (err) {
+            if (!isEexistError(err))
+                throw err;
+            if (Date.now() > deadline) {
+                throw new ApiError("Firecrawl crawl create-lock timed out", 500);
+            }
+            // Break a stale lock (the holder died without releasing).
+            const stat = await fs.stat(lockPath).catch(() => null);
+            if (stat && Date.now() - stat.mtimeMs > CRAWL_LOCK_STALE_MS) {
+                await fs.unlink(lockPath).catch(() => { });
+                continue;
+            }
+            await sleep(500);
+        }
+    }
+}
+/**
+ * Reclaim an in-flight job whose create-POST response was lost, or create a
+ * fresh one. On a state-file miss, GET /v2/crawl/active and adopt a matching
+ * job; else POST /v2/crawl and persist the id. The whole listActive→create→
+ * persist sequence runs under {@link withCrawlLock} so concurrent identical
+ * invocations serialize (the second reclaims the first's job instead of
+ * re-POSTing). The create POST is zero-retry at the shared-execution layer.
+ */
+async function reclaimOrCreateCrawlJob(apiKey, request, identityHash, params, stateFile, transport, stateDir) {
+    return withCrawlLock(stateDir, identityHash, async () => {
+        const active = await listActiveFirecrawlCrawls(apiKey, transport);
+        const matched = matchActiveCrawl(active, request, params);
+        if (matched !== undefined) {
+            return persistCrawlId(stateFile, identityHash, matched);
+        }
+        const created = await createFirecrawlCrawl(apiKey, request.url, params, transport);
+        return persistCrawlId(stateFile, identityHash, created.id);
+    }, transport);
+}
+function createFirecrawlCrawlCapability(options) {
+    const { env, transport, stateFile, stateDir } = options;
+    const fetch = {
+        kind: "crawl-fetch",
+        validate(request) {
+            assertHttpUrl(request.url);
+            // breadth has no Firecrawl equivalent (D9) — reject, don't ignore.
+            if (request.breadth !== undefined) {
+                throw new UnsupportedOptionError("firecrawl", "crawl", "breadth");
+            }
+            if (request.depth !== undefined) {
+                if (!Number.isInteger(request.depth) || request.depth < 1 || request.depth > 5) {
+                    throw new ValidationError("Crawl depth must be an integer between 1 and 5");
+                }
+            }
+            if (request.limit !== undefined && request.limit <= 0) {
+                throw new ValidationError("Crawl limit must be greater than 0");
+            }
+        },
+        cacheIdentity(request) {
+            const apiKey = resolveApiKey(env);
+            return {
+                provider: "firecrawl",
+                capability: "crawl",
+                credentialFingerprint: credentialFingerprint(apiKey),
+                request,
+            };
+        },
+        decodeCached(value) {
+            return decodeCrawlResult(value);
+        },
+        async invoke(request, signal) {
+            fetch.validate(request);
+            const apiKey = resolveApiKey(env);
+            const credFingerprint = credentialFingerprint(apiKey);
+            const identityHash = computeAsyncJobStateHash({
+                provider: "firecrawl",
+                capability: "crawl",
+                credentialFingerprint: credFingerprint,
+                request,
+            });
+            const params = mapCrawlControls(request);
+            const pollIntervalMs = resolveCrawlPollIntervalMs(env);
+            const sleep = makeCrawlSleep(transport, signal);
+            try {
+                // 1. Resume: an in-flight job for this request was already persisted
+                //    (Ctrl-C / crash mid-poll). Poll it instead of creating a second
+                //    one (double-charge prevention).
+                const existing = await stateFile.read(identityHash);
+                let id;
+                if (existing !== null) {
+                    id = existing.requestId;
+                }
+                else {
+                    // 2. Reclaim-on-miss or create (zero-retry create; wx-flag write).
+                    id = await reclaimOrCreateCrawlJob(apiKey, request, identityHash, params, stateFile, transport, stateDir);
+                }
+                // 3. Poll status to terminal, then collect the result once. We do
+                //    NOT accumulate data[] across pre-completion polls — the
+                //    completed response carries the full set (G0 #1).
+                let recreated = false;
+                for (;;) {
+                    if (signal?.aborted) {
+                        throw new TimeoutError(0, "Crawl polling aborted");
+                    }
+                    const poll = await pollFirecrawlCrawl(apiKey, id, transport);
+                    if (poll.status === "completed") {
+                        // Collect FIRST, remove only on success. If collection throws
+                        // (a malformed page, a pagination/network blip), the state file
+                        // stays so a re-run re-polls this already-completed (and paid-
+                        // for) job instead of POSTing a second, doubly-charging one.
+                        const result = await collectCrawlResult(poll, apiKey, request, transport);
+                        await stateFile.remove(identityHash);
+                        return result;
+                    }
+                    if (poll.status === "failed") {
+                        await stateFile.remove(identityHash);
+                        throw new ApiError("Firecrawl crawl job failed", 500);
+                    }
+                    if (poll.status === "not_found") {
+                        // Server-side job disappeared — drop state and create fresh,
+                        // but at most once. A pathological server that 404s every
+                        // freshly-created job would otherwise loop and re-POST
+                        // indefinitely (each POST charges credits).
+                        if (recreated) {
+                            throw new ApiError("Firecrawl crawl job could not be found after recreate", 500);
+                        }
+                        recreated = true;
+                        await stateFile.remove(identityHash);
+                        id = await reclaimOrCreateCrawlJob(apiKey, request, identityHash, params, stateFile, transport, stateDir);
+                        continue;
+                    }
+                    // scraping: sleep and poll again. The state file already holds
+                    // the id (the load-bearing field for resume).
+                    await sleep(pollIntervalMs);
+                }
+            }
+            catch (error) {
+                throw normalizeFirecrawlError(error);
+            }
+        },
+    };
+    return { fetch };
+}
+/**
+ * Build the Firecrawl Provider Descriptor. Advertises the full Firecrawl
+ * capability set. `create()` wires all six capabilities — Search, Reader,
+ * Map, async Crawl (create→poll→resume + reclaim-on-miss), Quota
+ * (`unit:"credits"`), and Diagnostics (single-scrape probe). Construction
+ * is side-effect-free; the transport is invoked per Capability call.
+ */
+export function createFirecrawlDescriptor(dependencies) {
+    const transport = dependencies?.transport;
+    const crawlStateDir = dependencies?.crawlStateDir ?? asyncJobStateDir("crawl");
+    const crawlStateFile = dependencies?.crawlStateFile ?? createProductionAsyncJobStateFile(crawlStateDir);
+    return {
+        id: "firecrawl",
+        isConfigured(env) {
+            return isFirecrawlConfigured(env);
+        },
+        capabilities() {
+            return new Set([
+                "search",
+                "reader",
+                "crawl",
+                "map",
+                "quota",
+                "diagnostics",
+            ]);
+        },
+        create(context) {
+            const search = createFirecrawlSearchCapability({ env: context.env, transport });
+            const reader = createFirecrawlReaderCapability({ env: context.env, transport });
+            const crawl = createFirecrawlCrawlCapability({
+                env: context.env,
+                transport,
+                stateFile: crawlStateFile,
+                stateDir: crawlStateDir,
+            });
+            const map = createFirecrawlMapCapability({ env: context.env, transport });
+            const quota = createFirecrawlQuotaCapability({ env: context.env, transport });
+            const diagnostics = createFirecrawlDiagnosticsCapability({ env: context.env, transport });
+            return { id: "firecrawl", search, reader, crawl, map, quota, diagnostics };
+        },
+    };
+}
+//# sourceMappingURL=adapter.js.map