spiderlint 0.0.0 → 0.143.0

package/dist/crawl/browser.js

@@ -0,0 +1,498 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { load } from "cheerio";
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
+import { createServer } from "node:net";
+import path from "node:path";
+import { MIMEType } from "node:util";
+import { Configuration, PlaywrightCrawler } from "crawlee";
+import { chromium, firefox, webkit } from "playwright";
+import { browserContext } from "../agent.js";
+import { ConfigError } from "../config/index.js";
+import { COOKIE_WRITES, headerFacts, observedResources, scriptCookies, redirectFacts, remoteFacts, timingFacts, tlsFacts, weightFacts, wireSize, withProbe } from "../facts/browser.js";
+import { declaredHtml, extractHtml, HTML_TYPES } from "../facts/html.js";
+import { parityFacts } from "../facts/parity.js";
+import { extractResources } from "../facts/resources.js";
+import { cookieFacts, dateSkew, redactHeaders } from "../facts/transport.js";
+import { log } from "../logger.js";
+import { isParsed } from "./body.js";
+import { reason } from "./fetch.js";
+import { width } from "./resources.js";
+import { chromiumArguments } from "./resolve.js";
+import { TlsProber } from "./tls-probe.js";
+const SETTLE_MS = 5000;
+// Pages a browser renders before a fresh one replaces it.
+const RETIRE_AFTER_PAGES = 1000;
+const LAUNCHERS = { chromium, firefox, webkit };
+const PORT_ARGUMENT = "--remote-debugging-port=";
+// Chromium past Crawlee’s loopback proxy, so it reports the server it reached.
+const DIRECT = "--proxy-server=direct://";
+// Each rendered page’s browser DevTools port, when an extractor asked for one.
+const ports = new WeakMap();
+const loaded = new WeakMap();
+// The DevTools port of the browser rendering `page`; undefined unless an active extractor is `debugging`.
+export function debuggingPort(page) {
+    return ports.get(page);
+}
+// The digest of every sub-resource body `page` loaded; undefined when one failed, was not read, or a frame navigated.
+export function loadedDigest(page) {
+    return loaded.get(page);
+}
+// A loopback port free right now, for the next browser to listen on.
+async function freePort() {
+    const server = createServer();
+    await new Promise((resolve, reject) => server.once("error", reject).listen(0, "127.0.0.1", resolve));
+    const { port } = server.address();
+    await new Promise((resolve) => server.close(resolve));
+    return port;
+}
+// Headers, address and certificate of a document response; an unreadable one keeps no headers rather than failing the page.
+async function readTransport(response) {
+    try {
+        const [headers, address, security] = await Promise.all([response.headersArray(), response.serverAddr(), response.securityDetails()]);
+        const tls = tlsFacts(security);
+        return { headers: headerFacts(headers), remote: remoteFacts(address), ...(tls && { tls }), ...(security?.protocol && { security: security.protocol }) };
+    }
+    catch (error) {
+        log.warn({ url: response.url(), error: String(error) }, "document transport unreadable");
+        return { headers: {} };
+    }
+}
+// Listens to a page before it navigates, so a download that aborts the navigation still leaves its response behind.
+function observe(page) {
+    const seen = { protocols: new Map(), console: { errors: [], warnings: [] }, requests: [] };
+    const errors = new Set();
+    const warnings = new Set();
+    page.on("response", (response) => {
+        if (!response.request().isNavigationRequest() || response.frame() !== page.mainFrame())
+            return;
+        seen.document = response;
+        seen.transport = readTransport(response);
+    });
+    page.on("console", (message) => {
+        const bucket = message.type() === "error" ? errors : message.type() === "warning" ? warnings : undefined;
+        bucket?.add(message.text());
+        seen.console = { errors: [...errors], warnings: [...warnings] };
+    });
+    page.on("pageerror", (error) => {
+        errors.add(error.message);
+        seen.console = { errors: [...errors], warnings: [...warnings] };
+    });
+    const record = (request) => {
+        seen.requests.push(request);
+    };
+    page.on("requestfinished", record);
+    page.on("requestfailed", record);
+    return seen;
+}
+// Records per document URL the protocol Chromium’s network log names, `h3`, `h2` or `http/1.1`; other browsers record nothing.
+async function watchProtocols(page, seen) {
+    try {
+        const cdp = await page.context().newCDPSession(page);
+        cdp.on("Network.responseReceived", ({ type, response }) => type === "Document" && response.protocol && seen.protocols.set(response.url, response.protocol));
+        await cdp.send("Network.enable");
+    }
+    catch (error) {
+        log.debug({ url: page.url(), error: String(error) }, "document protocol unobservable");
+    }
+}
+// `type/subtype` and charset of a Content-Type header; an unparsable one is its raw essence.
+function contentTypeOf(header) {
+    const raw = [header ?? ""].flat()[0] ?? "";
+    try {
+        const mime = new MIMEType(raw);
+        const charset = mime.params.get("charset");
+        return { type: mime.essence, ...(charset && { charset }) };
+    }
+    catch {
+        return { type: raw.split(";", 1)[0]?.trim().toLowerCase() ?? "" };
+    }
+}
+// Http and tls facts of a document response, the prober filling what Chromium does not say; `size` is what the caller could measure of its body.
+async function transportFacts(observation, size, timing, isDirect, prober, hop) {
+    const response = observation.document;
+    const { headers, remote: seenRemote, tls: seen, security } = await observation.transport;
+    const remote = isDirect ? seenRemote : undefined;
+    const { type, charset } = contentTypeOf(headers["content-type"]);
+    const url = new URL(response.url());
+    const { tls, version } = withProbe(url.href, seen, security, await prober?.facts(url, remote?.address), hop);
+    return {
+        http: {
+            status: response.status(),
+            ...(version && { version }),
+            redirects: await redirectFacts(response.request()),
+            headers: redactHeaders(headers),
+            ...(remote && { remote }),
+            timing,
+            cookies: cookieFacts(headers["set-cookie"], headers.date),
+            size,
+            "content-type": type,
+            ...(charset && { charset }),
+        },
+        ...(tls && { tls }),
+    };
+}
+// The byte count a Content-Length header declares, when it is a whole number.
+function declaredSize(headers) {
+    const declared = Number(headers["content-length"]);
+    return Number.isSafeInteger(declared) ? { declared } : {};
+}
+// A milestone the page reached, in whole milliseconds; zero means it never fired.
+function reached(value) {
+    return value && value > 0 ? Math.round(value) : undefined;
+}
+// Navigation Timing milestones, measured from the start of the navigation.
+async function milestones(page) {
+    const entry = await page.evaluate(() => performance.getEntriesByType("navigation")[0]?.toJSON());
+    const domContentLoaded = reached(entry?.domContentLoadedEventEnd);
+    const load = reached(entry?.loadEventEnd);
+    return { ...(domContentLoaded !== undefined && { "dom-content-loaded": domContentLoaded }), ...(load !== undefined && { load }) };
+}
+// Waits for the network to go quiet so client-rendered tags land; a page that never settles is read as it stands.
+async function didSettle(page) {
+    try {
+        await page.waitForLoadState("networkidle", { timeout: SETTLE_MS });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// A navigation Chromium turned into a download.
+function isDownload(error) {
+    return error instanceof Error && /Download is starting|net::ERR_ABORTED/.test(error.message);
+}
+// Chromium aborts a navigation that becomes a download; the response it already had stands in as the page.
+class Crawler extends PlaywrightCrawler {
+    #observations;
+    constructor(options, config, observations) {
+        super(options, config);
+        this.#observations = observations;
+    }
+    async _navigationHandler(context, gotoOptions) {
+        try {
+            return await super._navigationHandler(context, gotoOptions);
+        }
+        catch (error) {
+            const observation = this.#observations.get(context.request);
+            if (!isDownload(error) || !observation?.document)
+                throw error;
+            log.debug({ url: context.request.url, status: observation.document.status() }, "navigation became a download");
+            observation.isDownload = true;
+            return observation.document;
+        }
+    }
+}
+// The response’s own bytes; for a document an XSLT stylesheet turned into another type Chromium answers the output, so those bytes are fetched once more.
+async function rawOf(page, response, type, timeout) {
+    const raw = await response.body();
+    if (HTML_TYPES.has(type) || !isParsed(type))
+        return raw;
+    try {
+        const rendered = (await page.evaluate("document.contentType"));
+        if (rendered === type)
+            return raw;
+        const again = await page.request.fetch(response.request(), { timeout, failOnStatusCode: false });
+        const body = await again.body();
+        log.debug({ url: response.url(), type, rendered, transformed: raw.length, bytes: body.length, status: again.status() }, "transformed document read again");
+        return body;
+    }
+    catch (error) {
+        log.debug({ url: response.url(), type, error: reason(error) }, "transformed document not read again, its output kept");
+        return raw;
+    }
+}
+// The body as the facts see it: rendered DOM for HTML, raw text for other parsed types, nothing for a download or a binary.
+async function bodyOf(page, response, observation, type, timeout) {
+    if (observation.isDownload)
+        return { raw: Buffer.alloc(0), text: "" };
+    const raw = await rawOf(page, response, type, timeout);
+    return { raw, text: HTML_TYPES.has(type) ? await page.content() : isParsed(type) ? raw.toString("utf8") : "" };
+}
+// The document as served, before any script ran, and its html facts.
+function staticHtml(raw, url, scope) {
+    const source = raw.toString("utf8");
+    const $ = load(source);
+    return { $, html: extractHtml($, source, url, scope) };
+}
+// The first URL of a redirect chain, without its fragment.
+function requestedUrl(request) {
+    let first = request;
+    for (let previous = first.redirectedFrom(); previous; previous = previous.redirectedFrom())
+        first = previous;
+    const url = new URL(first.url());
+    url.hash = "";
+    return url.href;
+}
+// Logs each finished sub-request’s final response once per URL, keeping a body under `max` only where `isKeptType` takes its type.
+async function logResponses(requests, responses, max, isKeptType) {
+    const finished = requests.filter((request) => !request.isNavigationRequest() && request.failure() === null && request.redirectedTo() === null && /^https?:/.test(request.url()));
+    await Promise.all(finished.map(async (request) => {
+        const url = requestedUrl(request);
+        if (responses.has(url))
+            return;
+        try {
+            const response = await request.response();
+            if (!response)
+                return;
+            const [headers, body] = await Promise.all([response.allHeaders(), response.body()]);
+            const isKept = body.length < max && isKeptType(contentTypeOf(headers["content-type"]).type);
+            const { responseEnd } = request.timing();
+            const digest = createHash("sha256").update(body).digest("hex");
+            responses.set(url, { status: response.status(), headers, bytes: body.length, ...(isKept && { body }), digest, ...(responseEnd >= 0 && { ms: Math.round(responseEnd) }) });
+        }
+        catch (error) {
+            log.debug({ url, error: String(error) }, "browser response unreadable, fetched again later");
+        }
+    }));
+    log.debug({ finished: finished.length, logged: responses.size }, "browser responses logged");
+}
+// sha256 over each finished sub-request’s origin, path and body digest, sorted and distinct, so a cache-busting query or a repeated beacon keys nothing; undefined when one is unknown.
+function resourcesDigest(requests, responses) {
+    const pairs = new Set();
+    for (const request of requests) {
+        const isMainDocument = request.isNavigationRequest() && request.frame().parentFrame() === null;
+        if (isMainDocument || request.redirectedTo() !== null || !/^https?:/.test(request.url()))
+            continue;
+        const url = requestedUrl(request);
+        const digest = request.isNavigationRequest() || request.failure() !== null ? undefined : responses.get(url)?.digest;
+        if (digest === undefined) {
+            log.debug({ url, failure: request.failure()?.errorText, isFrame: request.isNavigationRequest() }, "page input unknown, extractors run uncached");
+            return undefined;
+        }
+        const { origin, pathname } = new URL(url);
+        pairs.add(`${origin}${pathname} ${digest}`);
+    }
+    return createHash("sha256")
+        .update([...pairs].toSorted((a, b) => a.localeCompare(b)).join("\n"))
+        .digest("hex");
+}
+// Minutes a browser download may take before the run gives up on it.
+const INSTALL_TIMEOUT_MS = 600_000;
+// Install output lines kept for the error when the download fails.
+const INSTALL_TAIL_LINES = 20;
+// Whether `error` is Playwright reporting the browser binary is not downloaded.
+export function isMissingExecutable(error) {
+    return reason(error).includes("Executable doesn't exist");
+}
+// The `playwright install` arguments downloading `name`; headless runs never launch the full Chromium.
+export function installArguments(name) {
+    return ["install", name, ...(name === "chromium" ? ["--only-shell"] : [])];
+}
+// The install command a ConfigError names when spiderlint may not download the browser itself.
+export function installCommand(name) {
+    return `npx playwright ${installArguments(name).join(" ")}`;
+}
+// Downloads `name` with the pinned Playwright CLI beside this file, which no `npx` lookup can miss.
+export async function installBrowser(name) {
+    const command = installCommand(name);
+    let cli;
+    try {
+        cli = path.join(path.dirname(createRequire(import.meta.url).resolve("playwright/package.json")), "cli.js");
+        if (!existsSync(cli))
+            throw new Error("no cli.js");
+    }
+    catch {
+        throw new ConfigError(`browser ${name} is not installed; install it with: ${command}`);
+    }
+    const started = Date.now();
+    log.info({ browser: name, command }, "browser is not installed, downloading it");
+    const child = spawn(process.execPath, [cli, ...installArguments(name)], { stdio: ["ignore", "pipe", "pipe"] });
+    let output = "";
+    child.stdout.on("data", (chunk) => {
+        output += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+        output += chunk;
+    });
+    let isTimedOut = false;
+    const code = await new Promise((resolve) => {
+        const timer = setTimeout(() => {
+            isTimedOut = true;
+            child.kill("SIGKILL");
+            resolve(-1);
+        }, INSTALL_TIMEOUT_MS);
+        child.once("error", () => {
+            clearTimeout(timer);
+            resolve(-1);
+        });
+        child.once("close", (status) => {
+            clearTimeout(timer);
+            resolve(status ?? -1);
+        });
+    });
+    const seconds = Math.round((Date.now() - started) / 1000);
+    if (code !== 0)
+        throw new ConfigError(`browser ${name} could not be downloaded (${isTimedOut ? `timed out after ${seconds} s` : `exited ${code}`}): ${output.trim().split("\n").slice(-INSTALL_TAIL_LINES).join("\n") || command}`);
+    log.info({ browser: name, seconds }, "browser downloaded");
+}
+// Launches `name` once to prove it is installed, downloading it first when it is missing and allowed to.
+export async function ensureBrowser(name, launcher, check, install) {
+    try {
+        const launched = await launcher.launch();
+        await launched.close();
+        return;
+    }
+    catch (error) {
+        if (!isMissingExecutable(error))
+            throw error;
+    }
+    if (!check.browserInstall)
+        throw new ConfigError(`browser ${name} is not installed; install it with: ${installCommand(name)}`);
+    if (check.cacheMode === "offline")
+        throw new ConfigError(`browser ${name} is not installed, and --offline cannot download it; install it with: ${installCommand(name)}`);
+    await install();
+    try {
+        const launched = await launcher.launch();
+        await launched.close();
+    }
+    catch (error) {
+        throw new ConfigError(`browser ${name} still does not launch after downloading it: ${reason(error)}`);
+    }
+}
+// Makes sure `name` launches before any scan, downloading it into the cache when missing.
+export async function prepareBrowser(name) {
+    await ensureBrowser(name, launcherOf(name), { browserInstall: true, cacheMode: "use" }, () => installBrowser(name));
+}
+// The UA the browser sends unmodified, read off a throwaway launch so rendered pages carry it with the token appended.
+async function realUserAgent(launcher) {
+    const launched = await launcher.launch();
+    try {
+        const context = await launched.newContext();
+        const page = await context.newPage();
+        const ua = (await page.evaluate("navigator.userAgent"));
+        await context.close();
+        log.debug({ ua }, "browser default user agent read");
+        return ua;
+    }
+    finally {
+        await launched.close();
+    }
+}
+// The Playwright launcher for `name`; anything else is a config error.
+function launcherOf(name) {
+    const launcher = Object.hasOwn(LAUNCHERS, name) ? LAUNCHERS[name] : undefined;
+    if (!launcher)
+        throw new ConfigError(`browser ${name}: expected chromium, firefox or webkit`);
+    log.debug({ browser: name }, "browser chosen");
+    return launcher;
+}
+// The browser crawler of a frontier: facts come from the rendered DOM and the browser’s own network log; an adaptive group’s rendered page is compared with its static HTML.
+export async function browserCrawler(config, onPage, frontier, router, storage, proxy, isKeptType = () => false, isDebugged = false, isExpensive = false) {
+    const launcher = launcherOf(config.browser);
+    await ensureBrowser(config.browser, launcher, config, () => installBrowser(config.browser));
+    const agent = browserContext(await realUserAgent(launcher));
+    const isPortOpen = isDebugged && config.browser === "chromium";
+    if (isDebugged && !isPortOpen)
+        log.warn({ browser: config.browser }, "only Chromium opens a DevTools port; its extractors add nothing");
+    let pages = 0;
+    const observations = new WeakMap();
+    const responses = new Map();
+    const prober = proxy ? undefined : new TlsProber(config.allowPrivate);
+    if (proxy)
+        log.warn({ fetch: "browser" }, "TLS probes would bypass the proxy; browser pages carry no cipher, ALPN, SAN or HTTP version");
+    const isDirect = !proxy && config.browser === "chromium";
+    log.debug({ browser: config.browser, isDirect }, "browser remote address recorded only when direct");
+    let launches = 0;
+    // Pages rendered at once, all in one browser: `concurrency`, else one beside an expensive extractor, else half of NUMPROCS.
+    const openPages = config.concurrency || (isExpensive ? 1 : Math.ceil(width() / 2));
+    log.debug({ openPages, concurrency: config.concurrency, isExpensive }, "browser concurrency");
+    const crawler = new Crawler({
+        ...frontier.options("browser", storage, proxy),
+        headless: true,
+        // Snapshot with `page.content()`; crawlee’s shadow-root expansion writes `innerHTML` into the live page.
+        ignoreShadowRoots: true,
+        maxConcurrency: openPages,
+        launchContext: { launcher, userAgent: agent.userAgent, launchOptions: { args: config.browser === "chromium" ? [...chromiumArguments(), ...(proxy ? [] : [DIRECT])] : [] } },
+        browserPoolOptions: {
+            useFingerprints: false,
+            maxOpenPagesPerBrowser: openPages,
+            retireBrowserAfterPageCount: RETIRE_AFTER_PAGES,
+            preLaunchHooks: isPortOpen
+                ? [
+                    async (pageId, launchContext) => {
+                        const port = await freePort();
+                        launchContext.launchOptions = { ...launchContext.launchOptions, args: [...(launchContext.launchOptions?.args ?? []), `${PORT_ARGUMENT}${port}`] };
+                        log.debug({ pageId, port }, "browser DevTools port opened on loopback");
+                    },
+                ]
+                : [],
+            postLaunchHooks: [
+                (pageId) => {
+                    launches += 1;
+                    log.debug({ pageId, launches }, "browser launched");
+                },
+            ],
+        },
+        preNavigationHooks: [
+            async ({ page, request }) => {
+                const observation = observe(page);
+                observations.set(request, observation);
+                await watchProtocols(page, observation);
+                await page.addInitScript({ content: COOKIE_WRITES });
+                if (agent.extraHTTPHeaders)
+                    await page.context().setExtraHTTPHeaders(agent.extraHTTPHeaders);
+            },
+        ],
+        async requestHandler({ request, page, parseWithCheerio, enqueueLinks, browserController }) {
+            const observation = observations.get(request);
+            const response = observation?.document;
+            if (!observation || !response)
+                return log.warn({ url: request.url }, "page rendered without a document response");
+            const url = new URL(response.url());
+            if (!frontier.admit(request, url))
+                return;
+            const { headers } = await observation.transport;
+            const { type } = contentTypeOf(headers["content-type"]);
+            const isHtml = HTML_TYPES.has(type) && !observation.isDownload;
+            const settled = isHtml && (await didSettle(page));
+            const { raw, text } = await bodyOf(page, response, observation, type, config.timeout * 1000);
+            const body = text.slice(0, config.maxBodySize);
+            const wire = observation.isDownload ? 0 : await wireSize(response.request(), raw.length);
+            const size = { body: wire, decoded: raw.length, ...declaredSize(headers), ...((body.length < text.length || observation.isDownload) && { truncated: true }) };
+            const timing = observation.isDownload ? {} : timingFacts(response.request().timing());
+            const facts = { ...frontier.identity(request, url, "browser"), ...(await transportFacts(observation, size, timing, isDirect, prober, observation.protocols.get(response.url()))) };
+            const { startTime, requestStart, responseStart } = response.request().timing();
+            const skew = observation.isDownload || responseStart < 0 ? undefined : dateSkew(url.href, headers, startTime + requestStart, startTime + responseStart);
+            if (skew !== undefined)
+                facts.http["date-skew"] = skew;
+            const served = isHtml ? staticHtml(raw, url, config.scope) : undefined;
+            if (isHtml && served) {
+                const $ = await parseWithCheerio();
+                const rendered = extractHtml($, text, url, config.scope);
+                facts.html = declaredHtml(served.html, rendered);
+                log.debug({ url: url.href, rewritten: facts.html.rewritten?.length ?? 0 }, "served declarations kept over the rendered DOM");
+                facts.parity = parityFacts(url.href, served.$, served.html, $, rendered);
+                facts.resources = observedResources(extractResources($, url, config.maxResourcesPerPage), observation.requests, url, config.maxResourcesPerPage);
+                facts.browser = { timing: await milestones(page), console: observation.console, weight: await weightFacts(observation.requests), cookies: await scriptCookies(page) };
+                await logResponses(observation.requests, responses, config.maxBodySize, isKeptType);
+                const digest = resourcesDigest(observation.requests, responses);
+                if (digest)
+                    loaded.set(page, digest);
+            }
+            log.debug({ url: url.href, status: facts.http.status, type, bytes: size.body, depth: facts.crawl.depth, settled, isDownload: observation.isDownload === true, requests: observation.requests.length }, "page rendered");
+            if (router.isDetecting(url.href))
+                router.detected(url.href, served?.html, facts.html);
+            pages += 1;
+            const port = browserController.launchContext.launchOptions?.args?.findLast((argument) => argument.startsWith(PORT_ARGUMENT))?.slice(PORT_ARGUMENT.length);
+            if (port)
+                ports.set(page, Number(port));
+            await onPage(facts, body, isHtml ? page : undefined);
+            if (!isHtml && !facts.feed)
+                return;
+            log.debug({ url: url.href, enqueued: await frontier.enqueue(enqueueLinks, facts, "browser") }, "links enqueued");
+        },
+        async failedRequestHandler({ request }, error) {
+            const facts = frontier.failed(request, reason(error));
+            if (facts)
+                await onPage(facts, "");
+        },
+    }, storage?.config ?? new Configuration({ persistStorage: false, purgeOnStart: false }), observations);
+    if (storage?.earlier)
+        log.debug({ fetch: "browser" }, "browser pages are re-rendered, never revalidated");
+    return { crawler, stats: () => ({ pages, launches, responses, tlsProbes: prober?.probes ?? 0 }) };
+}

package/dist/crawl/crawl.js

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { ConfigError } from "../config/index.js";
+import { interrupted } from "../interrupt.js";
+import { log } from "../logger.js";
+import { browserCrawler } from "./browser.js";
+import { Frontier } from "./frontier.js";
+import { httpCrawler } from "./http.js";
+import { bridgeCrawleeLog } from "./log.js";
+import { singleOrigin } from "./scope.js";
+import { trackProgress } from "../progress.js";
+// Crawls with every crawler some group needs, side by side over one frontier that routes each URL to its group’s.
+export async function crawlSite(config, onPage, cache, router, storage, proxy, isKeptType, isDebugged, isExpensive) {
+    bridgeCrawleeLog();
+    const modes = router.crawlers;
+    if (modes.includes("browser") && !config.allowPrivate)
+        throw new ConfigError("fetch browser: the address guard cannot check what the browser connects to");
+    const frontier = await Frontier.open(config, cache, router);
+    const http = modes.includes("http") ? httpCrawler(config, onPage, frontier, storage, proxy) : undefined;
+    const browser = modes.includes("browser") ? await browserCrawler(config, onPage, frontier, router, storage, proxy, isKeptType, isDebugged, isExpensive) : undefined;
+    log.debug({ crawlers: modes, groups: router.modes }, "crawlers chosen");
+    const crawlers = { ...(http && { http: http.crawler }), ...(browser && { browser: browser.crawler }) };
+    const stop = trackProgress(() => frontier.known(), singleOrigin(config.seeds));
+    const halt = (reason, why) => {
+        frontier.cut(reason);
+        for (const crawler of [http?.crawler, browser?.crawler])
+            crawler?.stop(why);
+    };
+    const interrupt = () => halt("interrupted", `interrupted by ${String(interrupted.reason)}`);
+    interrupted.addEventListener("abort", interrupt, { once: true });
+    const deadline = config.crawlDeadline > 0
+        ? setTimeout(() => {
+            log.warn({ crawlDeadline: config.crawlDeadline }, "crawl deadline reached, stopping after the pages in flight");
+            halt("timeout", `crawl deadline of ${config.crawlDeadline} s reached`);
+        }, config.crawlDeadline * 1000)
+        : undefined;
+    try {
+        await frontier.run(crawlers, cache.robots, storage?.resumed);
+    }
+    finally {
+        clearTimeout(deadline);
+        interrupted.removeEventListener("abort", interrupt);
+        stop();
+    }
+    const [fetched, rendered] = [http?.stats(), browser?.stats()];
+    if (fetched?.revalidated)
+        log.debug({ revalidated: fetched.revalidated }, "pages revalidated");
+    return {
+        site: frontier.site(),
+        pages: { ...(fetched && { http: fetched.pages }), ...(rendered && { browser: rendered.pages }) },
+        revalidated: fetched?.revalidated ?? 0,
+        launches: rendered?.launches ?? 0,
+        responses: rendered?.responses ?? new Map(),
+        tlsProbes: rendered?.tlsProbes ?? 0,
+        modes: router.modes,
+    };
+}