spiderlint 0.0.0 → 0.143.1

package/dist/crawl/frontier.js

@@ -0,0 +1,370 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { ProxyConfiguration } from "crawlee";
+import picomatch from "picomatch";
+import { vendorPath } from "../facts/vendors.js";
+import { log } from "../logger.js";
+import { Spread } from "./diversity.js";
+import { width } from "./resources.js";
+import { crawlDelayOf, robotsFactsOf } from "./robots.js";
+import { isInScope, STRATEGY } from "./scope.js";
+import { loadSitemap } from "./sitemap.js";
+// Feed media types a head `rel=alternate` names; such a feed is crawled as a page.
+const FEED_TYPES = new Set(["application/rss+xml", "application/atom+xml", "application/feed+json"]);
+// Item links, the stylesheet and the archive pages a feed names, which its rules judge against the crawl.
+function feedTargets(facts) {
+    const feed = facts.feed;
+    return [...(feed?.entries ?? []).flatMap((entry) => (entry.link ? [entry.link] : [])), ...(feed?.stylesheet ? [feed.stylesheet] : []), ...Object.values(feed?.archives ?? {})];
+}
+// Reasons a crawl ended early, the one that names it first.
+const CRAWL_ENDS = ["interrupted", "timeout", "max-pages", "max-depth"];
+function globMatchers(config) {
+    return { include: config.includeUrls.map((glob) => picomatch(glob)), exclude: config.excludeUrls.map((glob) => picomatch(glob)) };
+}
+// What every fetch mode shares: seeds, sitemap, globs, robots, the page budget, the visited set, and which crawler each URL goes to.
+export class Frontier {
+    static async open(config, cache, router) {
+        return new Frontier(config, config.sitemap ? await loadSitemap(config.seeds, cache.robots, cache.sitemaps, config.canonicalOrigin) : { index: new Map(), files: [] }, router);
+    }
+    #config;
+    #seeds;
+    #sitemap;
+    #skipped = new Map();
+    // Vendor entries that kept a page out of the crawl, with the pages and the links they took.
+    #vendors = new Map();
+    #visited = new Set();
+    #globs;
+    #robots = [];
+    // Crawlee resets maxRequestsPerCrawl on every run(), so --max-pages needs its own cross-phase tally.
+    #handled = 0;
+    // Candidates pooled while `diversify` holds, and the requests already given to a crawler.
+    #spread;
+    #released = 0;
+    #router;
+    #crawlers = new Map();
+    #hasStraggled = false;
+    // Why pages were left unfetched, most telling first.
+    #cuts = new Set();
+    files;
+    // A vendor-owned path is no page of the site; include and exclude globs then run on `pathname + search`, as group matchers do.
+    transformRequestFunction = (request) => {
+        const vendor = this.#vendorOf(request.url);
+        if (vendor) {
+            const tally = this.#vendors.get(vendor) ?? { pages: new Set(), links: 0 };
+            tally.pages.add(request.url);
+            tally.links += 1;
+            this.#vendors.set(vendor, tally);
+            log.debug({ url: request.url, vendor: vendor.vendor, match: vendor.match }, "link skipped, vendor path");
+            return false;
+        }
+        const { include, exclude } = this.#globs;
+        const url = new URL(request.url);
+        const path = url.pathname + url.search;
+        const reason = include.length > 0 && include.every((match) => !match(path)) ? "include" : exclude.some((match) => match(path)) ? "exclude" : undefined;
+        if (!reason)
+            return request;
+        log.debug({ url: request.url, reason }, "link skipped");
+        this.#skipped.set(request.url, reason);
+        return false;
+    };
+    constructor(config, sitemaps, router) {
+        this.#config = config;
+        this.#router = router;
+        this.#seeds = new Set(config.seeds);
+        this.#sitemap = sitemaps.index;
+        this.#globs = globMatchers(config);
+        this.#spread = config.diversify ? new Spread(config.maxPages || Infinity) : undefined;
+        this.files = sitemaps.files;
+    }
+    // The shipped vendor entry owning `href` as a page, unless `vendor-paths` is off.
+    #vendorOf(href) {
+        return this.#config.vendorPaths ? vendorPath(href, "page") : undefined;
+    }
+    // Whether a URL a sitemap or a stored page names may join the crawl: unvisited, no vendor page, in scope and inside the globs.
+    #isCrawlable(href, reference) {
+        if (this.#visited.has(href) || this.#vendorOf(href))
+            return false;
+        const url = new URL(href);
+        if (!isInScope(url, reference, this.#config.scope))
+            return false;
+        const { include, exclude } = this.#globs;
+        const path = url.pathname + url.search;
+        return (include.length === 0 || include.some((match) => match(path))) && exclude.every((match) => !match(path));
+    }
+    // A sitemap URL still unvisited once the link crawl settles joins the frontier as its own root.
+    #stragglers() {
+        if (this.#sitemap.size === 0 || this.#config.seeds.length === 0 || !this.#config.follow)
+            return [];
+        const reference = new URL(this.#config.seeds[0]);
+        const extra = this.#sitemap
+            .keys()
+            .filter((href) => this.#isCrawlable(href, reference))
+            .toArray();
+        return extra;
+    }
+    // Gives the requests to their crawlers, or pools them while a budget spreads; returns how many the pool took.
+    async #offer(urls, crawlDepth) {
+        if (!this.#spread) {
+            await this.#add(urls.map((url) => ({ url, crawlDepth })));
+            return 0;
+        }
+        const spread = this.#spread;
+        const isDeep = this.#config.maxDepth > 0 && crawlDepth > this.#config.maxDepth;
+        let pooled = 0;
+        for (const url of urls) {
+            if (this.#visited.has(url))
+                continue;
+            const isNew = spread.add({ url, crawlDepth });
+            const isRelinked = !isNew && !isDeep && spread.relink(url, crawlDepth);
+            if (isNew)
+                pooled += 1;
+            if (isRelinked)
+                log.debug({ url, crawlDepth }, "listed candidate reached by a link");
+        }
+        log.debug({ offered: urls.length, pooled, size: this.#spread.size, crawlDepth }, "candidates pooled");
+        await this.#refill();
+        return pooled;
+    }
+    // Counts the seeds and the pages a resumed crawl stored as scheduled, then pools their links and the sitemap, which a spread crawl draws from at once.
+    #prime(resumed) {
+        const spread = this.#spread;
+        for (const page of resumed) {
+            this.#visited.add(page.url.href);
+            if (page.crawl.requested)
+                this.#visited.add(page.crawl.requested);
+            if (spread.note(page.url.href))
+                this.#released += 1;
+        }
+        this.#handled += resumed.length;
+        for (const seed of this.#seeds)
+            if (spread.note(seed))
+                this.#released += 1;
+        const reference = this.#config.seeds[0] ? new URL(this.#config.seeds[0]) : undefined;
+        const links = resumed.flatMap((page) => (page.html?.links.internal ?? []).filter((href) => reference && this.#config.follow && URL.canParse(href) && this.#isCrawlable(href, reference)).map((href) => ({ url: href, crawlDepth: page.crawl.depth + 1 })));
+        const listed = this.#stragglers().map((url) => ({ url, crawlDepth: 0 }));
+        const pooled = [...links, ...listed].filter((candidate) => spread.add(candidate)).length;
+        this.#hasStraggled = true;
+        log.debug({ resumed: resumed.length, seeds: this.#seeds.size, links: links.length, listed: listed.length, pooled, released: this.#released }, "budget primed");
+    }
+    // Gives the crawlers the pool’s most diverse candidates while their queues run short and the budget lasts; a drained queue has nothing pending.
+    async #refill(isDrained = false) {
+        if (!this.#spread)
+            return 0;
+        const pending = isDrained ? 0 : Math.max(0, this.#released - this.#handled);
+        const left = this.#config.maxPages > 0 ? this.#config.maxPages - this.#released : Infinity;
+        const room = Math.min(2 * width(this.#config.concurrency) - pending, left);
+        log.debug({ pending, room, released: this.#released, pooled: this.#spread.size, maxPages: this.#config.maxPages, isDrained }, "refill decided");
+        if (room <= 0)
+            return 0;
+        const batch = this.#spread.take(room);
+        this.#released += batch.length;
+        await this.#add(batch);
+        return batch.length;
+    }
+    // Queues each request on the crawler its group needs.
+    async #add(requests) {
+        const byCrawler = Object.groupBy(requests, (request) => this.#router.queue(request.url));
+        for (const [mode, batch = []] of Object.entries(byCrawler)) {
+            log.debug({ crawler: mode, requests: batch.length }, "requests routed");
+            await this.#crawlers.get(mode)?.addRequests(batch);
+        }
+    }
+    // Finished once every crawler’s queue drains and no sitemap straggler is left to add.
+    async #isFinished() {
+        for (const crawler of this.#crawlers.values()) {
+            const queue = await crawler.getRequestQueue();
+            if (!(await queue.isFinished()))
+                return false;
+        }
+        if ((await this.#refill(true)) > 0)
+            return false;
+        if (this.#hasStraggled)
+            return true;
+        this.#hasStraggled = true;
+        const stragglers = this.#stragglers();
+        const isOverBudget = this.#config.maxPages > 0 && this.#handled >= this.#config.maxPages;
+        log.debug({ listed: this.#sitemap.size, stragglers: stragglers.length, handled: this.#handled, isOverBudget }, "queue drained");
+        if (isOverBudget && stragglers.length > 0)
+            this.cut("max-pages");
+        if (isOverBudget || stragglers.length === 0)
+            return true;
+        await this.#add(stragglers.map((url) => ({ url, crawlDepth: 0 })));
+        return false;
+    }
+    // Each seed’s origin, once.
+    #origins() {
+        return [...new Set(this.#config.seeds.map((seed) => new URL(seed).origin))];
+    }
+    // The longest `Crawl-delay` over the seed origins, as Crawlee applies one delay to every domain.
+    async #crawlDelay(robots) {
+        const origins = this.#origins();
+        const delays = await Promise.all(origins.map(async (origin) => crawlDelayOf(await robots(origin))));
+        const delay = Math.max(0, ...delays);
+        if (delay > 0)
+            log.info({ delay, origins: origins.length }, `robots.txt asks for ${delay} s between requests, honoured`);
+        return delay;
+    }
+    // Why a seed was never fetched, naming the network error behind an unreachable robots.txt.
+    #explainSkippedSeed(seed) {
+        const reason = this.#skipped.get(seed);
+        if (reason === undefined)
+            return;
+        const error = this.#robots.find((facts) => facts.url === `${new URL(seed).origin}/robots.txt`)?.error;
+        if (error)
+            log.error({ url: seed, error }, "seed not crawled: its robots.txt is unreachable, which disallows the whole origin (RFC 9309)");
+        else
+            log.warn({ url: seed, reason }, `seed not crawled, skipped by its ${reason} check:`);
+    }
+    // How the crawl ended: complete when nothing was left unfetched, else the first reason of `CRAWL_ENDS` that applies.
+    #crawl() {
+        if (this.#spread && this.#spread.size > 0)
+            this.#cuts.add("max-pages");
+        const reason = CRAWL_ENDS.find((end) => this.#cuts.has(end));
+        log.debug({ cuts: [...this.#cuts], pooled: this.#spread?.size, reason }, "crawl end decided");
+        return reason ? { complete: false, reason } : { complete: true };
+    }
+    // Crawler options every adapter passes through unchanged; crawlers running side by side split the rate.
+    options(mode, storage, proxy) {
+        return {
+            ...(storage && { requestQueue: storage.queues[mode] }),
+            autoscaledPoolOptions: { isFinishedFunction: () => this.#isFinished() },
+            // A 401, 403 or 429 is a page to lint, never a session to retire and retry.
+            sessionPoolOptions: { blockedStatusCodes: [] },
+            requestHandlerTimeoutSecs: this.#config.timeout,
+            navigationTimeoutSecs: Math.ceil(this.#config.timeout / 2),
+            maxRequestsPerCrawl: this.#config.maxPages || undefined,
+            maxRequestsPerMinute: this.#config.rate ? Math.max(1, Math.floor(this.#config.rate / this.#router.crawlers.length)) : undefined,
+            ...(proxy && { proxyConfiguration: new ProxyConfiguration({ proxyUrls: [proxy] }) }),
+            maxCrawlDepth: this.#config.maxDepth || undefined,
+            respectRobotsTxtFile: this.#config.robots && { userAgent: "spiderlint" },
+            onSkippedRequest: ({ url, reason }) => {
+                log.debug({ url, reason }, "link skipped");
+                this.#skipped.set(url, reason);
+                if (reason === "limit")
+                    this.cut("max-pages");
+                else if (reason === "depth")
+                    this.cut("max-depth");
+                if (this.#spread)
+                    this.#released = Math.max(0, this.#released - 1);
+            },
+        };
+    }
+    // False once --max-pages is spent; otherwise the requested and the loaded URL count as visited.
+    admit(request, loaded) {
+        this.#handled += 1;
+        if (this.#config.maxPages && this.#handled > this.#config.maxPages) {
+            log.debug({ url: request.url, handled: this.#handled, maxPages: this.#config.maxPages }, "page dropped past max-pages");
+            this.cut("max-pages");
+            return false;
+        }
+        this.#visited.add(request.url);
+        this.#visited.add(loaded.href);
+        return true;
+    }
+    // Records that the crawl left pages unfetched for `reason`.
+    cut(reason) {
+        log.debug({ reason, cuts: [...this.#cuts] }, "crawl cut");
+        this.#cuts.add(reason);
+    }
+    // The facts a page owes to its URL and how the crawl reached it, whatever fetched it; the sitemap entry is the loaded URL’s, else the requested one’s.
+    identity(request, url, mode) {
+        const listing = this.#sitemap.get(url.href) ?? this.#sitemap.get(request.url);
+        return {
+            url: { href: url.href, origin: url.origin, protocol: url.protocol, host: url.host, pathname: url.pathname, search: url.search },
+            group: "default",
+            crawl: { depth: request.crawlDepth, "discovered-via": request.crawlDepth > 0 ? "link" : this.#seeds.has(request.url) ? "seed" : "sitemap", referrers: [], ...(request.url !== url.href && { requested: request.url }), at: new Date().toISOString(), ...(mode && { mode }) },
+            ...(this.#sitemap.size > 0 && { sitemap: listing ?? { listed: false } }),
+        };
+    }
+    // A page no attempt fetched, as status 0 and its error so its referrers still see it; undefined past --max-pages.
+    failed(request, error) {
+        const url = new URL(request.url);
+        if (!this.admit(request, url))
+            return undefined;
+        log.warn({ url: url.href, error, retries: request.retryCount }, "page not fetched");
+        return { ...this.identity(request, url), http: { status: 0, redirects: [], headers: {}, timing: {}, cookies: [], size: { body: 0, decoded: 0 }, "content-type": "", error } };
+    }
+    // Hands a request to the browser before any fetch when its group renders there; true once handed.
+    async handOff(request) {
+        if (!this.#crawlers.has("browser") || !(await this.#router.handOff(request.url)))
+            return false;
+        await this.#crawlers.get("browser")?.addRequests([{ url: request.url, crawlDepth: request.crawlDepth }]);
+        return true;
+    }
+    // The page’s anchors and head feeds, or a feed’s targets, queued under the scope and globs on the crawler their group needs, unless the seeds are the whole frontier; returns how many anchors and head feeds were new.
+    async enqueue(enqueueLinks, facts, mode) {
+        if (!this.#config.follow)
+            return 0;
+        const routed = [];
+        const transformRequestFunction = (request) => {
+            const kept = this.transformRequestFunction(request);
+            const isRouted = kept && isInScope(new URL(request.url), new URL(facts.url.href), this.#config.scope) && (this.#spread !== undefined || this.#router.queue(request.url) !== mode);
+            if (!isRouted)
+                return kept;
+            routed.push(request.url);
+            return false;
+        };
+        const options = { strategy: STRATEGY[this.#config.scope], transformRequestFunction };
+        const heads = facts.html?.head.links ?? [];
+        const feeds = heads.filter((link) => /\balternate\b/i.test(link.rel ?? "") && FEED_TYPES.has(link.type?.toLowerCase() ?? "")).flatMap((link) => (link.href ? [link.href] : []));
+        log.debug({ url: facts.url.href, feeds }, "head feeds found");
+        const batches = facts.html ? [await enqueueLinks(options), ...(feeds.length > 0 ? [await enqueueLinks({ ...options, urls: feeds })] : [])] : [];
+        const targets = feedTargets(facts).filter((url) => URL.canParse(url) && isInScope(new URL(url), new URL(facts.url.href), this.#config.scope) && this.transformRequestFunction({ url }) !== false);
+        log.debug({ url: facts.url.href, targets: targets.length }, "feed targets queued");
+        const pooled = await this.#offer([...routed, ...targets], facts.crawl.depth + 1);
+        return batches.flatMap((batch) => batch.processedRequests).filter((entry) => !entry.wasAlreadyPresent).length + (this.#spread ? pooled : routed.length);
+    }
+    // Pages known: every queue’s requests, plus sitemap URLs no link has reached yet, within the page budget.
+    async known() {
+        let total = (this.#spread?.size ?? 0) + (this.#hasStraggled ? 0 : this.#stragglers().length);
+        for (const crawler of this.#crawlers.values()) {
+            const queue = await crawler.getRequestQueue();
+            total += queue.getTotalCount();
+        }
+        return this.#config.maxPages > 0 ? Math.min(total, this.#config.maxPages) : total;
+    }
+    // What the crawl learnt about the site: how it ended, sitemap files, and each seed origin’s robots.txt when one was read.
+    site() {
+        return { sitemaps: this.files, crawl: this.#crawl(), ...(this.#robots.length > 0 && { robots: this.#robots }) };
+    }
+    // Seeds first, each on its group’s crawler, sitemap stragglers while the budget lasts; robots.txt answered through the robots bucket, a crawl-delay stretched over the crawlers sharing it.
+    async run(crawlers, robots, resumed = []) {
+        if (this.#spread)
+            this.#prime(resumed);
+        const answer = (url) => (this.#config.robots ? robots(url) : Promise.resolve(undefined));
+        const delay = this.#config.robots ? await this.#crawlDelay(robots) : 0;
+        const running = Object.entries(crawlers);
+        for (const [mode, crawler] of running) {
+            Object.assign(crawler, { getRobotsTxtFileForUrl: answer });
+            if (delay > 0)
+                Object.assign(crawler, { sameDomainDelayMillis: delay * 1000 * running.length });
+            this.#crawlers.set(mode, crawler);
+        }
+        const seeds = Object.groupBy(this.#config.seeds, (seed) => this.#router.queue(seed));
+        log.debug({ crawlers: running.map(([mode]) => mode), http: seeds.http?.length ?? 0, browser: seeds.browser?.length ?? 0 }, "seeds routed");
+        await Promise.all(running.map(([mode, crawler]) => crawler.run(seeds[mode] ?? [])));
+        for (const [mode, crawler] of running) {
+            const queue = await crawler.getRequestQueue();
+            if (this.#config.maxPages === 0 || (await queue.isFinished()))
+                continue;
+            log.debug({ crawler: mode, handled: this.#handled }, "queue left requests behind");
+            this.cut("max-pages");
+        }
+        const isRead = this.#config.robots || this.#config.sitemap;
+        const files = isRead ? await Promise.all(this.#origins().map(async (origin) => robotsFactsOf(await robots(origin)))) : [];
+        this.#robots = files.filter((facts) => facts !== undefined);
+        log.debug({ isRead, files: this.#robots.length }, "robots.txt facts collected");
+        const reasons = Object.groupBy(this.#skipped.values(), (reason) => reason);
+        if (this.#skipped.size > 0)
+            log.debug({ skipped: this.#skipped.size, ...Object.fromEntries(Object.entries(reasons).map(([reason, all]) => [reason, all?.length])) }, "links skipped");
+        for (const seed of this.#seeds)
+            this.#explainSkippedSeed(seed);
+        const vendors = this.#vendors
+            .entries()
+            .map(([entry, tally]) => ({ vendor: entry.vendor, match: entry.match, pages: tally.pages.size, links: tally.links }))
+            .toArray();
+        if (vendors.length > 0)
+            log.debug({ vendors }, "vendor paths kept out of the crawl");
+    }
+}

package/dist/crawl/guard.js

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import dns, {} from "node:dns";
+import { BlockList, isIP } from "node:net";
+import { log } from "../logger.js";
+// Loopback, private, link-local, CGNAT, unspecified, unique-local and cloud metadata ranges.
+const PRIVATE = new BlockList();
+for (const [network, prefix] of [
+    ["0.0.0.0", 8],
+    ["10.0.0.0", 8],
+    ["100.64.0.0", 10],
+    ["127.0.0.0", 8],
+    ["169.254.0.0", 16],
+    ["172.16.0.0", 12],
+    ["192.168.0.0", 16],
+])
+    PRIVATE.addSubnet(network, prefix, "ipv4");
+for (const [network, prefix] of [
+    ["::", 128],
+    ["::1", 128],
+    ["fc00::", 7],
+    ["fe80::", 10],
+])
+    PRIVATE.addSubnet(network, prefix, "ipv6");
+// An address no public probe may reach; an IPv4-mapped IPv6 address is judged as its IPv4 form.
+export function isPrivate(address) {
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address)?.[1];
+    return mapped ? PRIVATE.check(mapped, "ipv4") : PRIVATE.check(address, isIP(address) === 6 ? "ipv6" : "ipv4");
+}
+// A refused address, named so the log and the fact say why.
+export class PrivateAddress extends Error {
+    code = "EPRIVATE";
+}
+// The address a raw socket to `hostname` may connect to, through the run’s lookup; a private one is refused unless allowed.
+export async function connectable(hostname, isPrivateAllowed) {
+    const address = isIP(hostname) === 0 ? await new Promise((resolve, reject) => dns.lookup(hostname, (error, found) => (error ? reject(error) : resolve(found)))) : hostname;
+    log.debug({ hostname, address, isPrivateAllowed }, "socket address resolved");
+    if (!isPrivateAllowed && isPrivate(address))
+        throw new PrivateAddress(`${hostname} resolves to private address ${address}`);
+    return address;
+}
+// `lookup` failing when any answer is private, so the socket connects only to an address it checked.
+export function guarding(lookup) {
+    return (hostname, options, callback) => {
+        lookup(hostname, { ...options, all: true }, (error, found) => {
+            if (error)
+                return callback(error, "", 0);
+            const addresses = found;
+            const refused = addresses.find((answer) => isPrivate(answer.address));
+            log.debug({ hostname, addresses: addresses.map((answer) => answer.address), refused: refused?.address }, "address checked");
+            if (refused)
+                return callback(new PrivateAddress(`${hostname} resolves to private address ${refused.address}`), "", 0);
+            const [first] = addresses;
+            // eslint-disable-next-line unicorn/no-null -- the lookup callback contract types a success as a null error
+            callback(null, options.all ? addresses : (first?.address ?? ""), first?.family ?? 0);
+        });
+    };
+}
+// The system lookup, guarded.
+export const guardedLookup = guarding((hostname, options, callback) => dns.lookup(hostname, options, callback));
+// Throws for a URL whose host is a private address literal, which no lookup ever sees.
+export function refuseLiteral(url, isPrivateAllowed) {
+    const host = new URL(url).hostname.replaceAll(/^\[|\]$/g, "");
+    if (isPrivateAllowed || isIP(host) === 0 || !isPrivate(host))
+        return;
+    log.warn({ host, url: String(url) }, "private address literal refused");
+    throw new PrivateAddress(`${host} is a private address`);
+}

package/dist/crawl/http.js

@@ -0,0 +1,171 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { CheerioCrawler, Configuration } from "crawlee";
+import { agentHeaders } from "../agent.js";
+import { ACCEPT_ENCODING, capped, isParsed, replayed } from "./body.js";
+import { extractHtml, HTML_TYPES } from "../facts/html.js";
+import { extractResources } from "../facts/resources.js";
+import { cookieFacts, dateSkew, earlyHintsHook, redactHeaders, redirectHook, timingFacts, tlsFacts } from "../facts/transport.js";
+import { log } from "../logger.js";
+import { MISMATCH, reason } from "./fetch.js";
+import { guardUrl } from "./network.js";
+import { width } from "./resources.js";
+// The first value of a header that may repeat.
+function first(value) {
+    return Array.isArray(value) ? value[0] : value;
+}
+// `If-None-Match` and `If-Modified-Since` from a stored page’s `ETag` and `Last-Modified`.
+function validators(headers) {
+    const etag = first(headers.etag);
+    const modified = first(headers["last-modified"]);
+    return { ...(etag && { "if-none-match": etag }), ...(modified && { "if-modified-since": modified }) };
+}
+// A 304 keeps the stored status, size and cookies and merges its headers over the stored ones.
+function revalidated(earlier, fresh) {
+    const cookies = fresh.http.cookies.length > 0 ? fresh.http.cookies : earlier.http.cookies;
+    const headers = { ...earlier.http.headers, ...fresh.http.headers };
+    return { ...fresh.http, status: earlier.http.status, size: earlier.http.size, headers, cookies, revalidated: true };
+}
+// The stored `Content-Type` header, else one rebuilt from the stored type and charset.
+function storedContentType(facts) {
+    return first(facts.http.headers["content-type"]) ?? `${facts.http["content-type"]}${facts.http.charset ? `; charset=${facts.http.charset}` : ""}`;
+}
+// Wire bytes received so far, from got's progress on the original response stream.
+function transferred(source) {
+    return source?.downloadProgress?.transferred;
+}
+// The response's own connection, read while it is still attached.
+function socketOf(source) {
+    const stream = source;
+    return stream.socket ?? stream.request?.socket;
+}
+// Why a page fetch failed, naming both byte counts when the body ended short of `Content-Length`.
+function failure(error, source) {
+    const declared = Number(first(source?.headers?.["content-length"]));
+    const received = transferred(source);
+    log.debug({ error: error.message, declared, received }, "page failure named");
+    if (received !== undefined && received < declared)
+        return `${MISMATCH}: body ended at ${received} of the ${declared} bytes Content-Length declares`;
+    return /content-length/i.test(error.message) ? MISMATCH : reason(error);
+}
+// Hands a request its group renders to the browser before fetching it.
+class Crawler extends CheerioCrawler {
+    #frontier;
+    constructor(options, config, frontier) {
+        super(options, config);
+        this.#frontier = frontier;
+    }
+    async _runRequestHandler(context) {
+        const isHanded = await this.#frontier.handOff(context.request);
+        log.debug({ url: context.request.url, isHanded }, "request crawler chosen");
+        return isHanded ? undefined : super._runRequestHandler(context);
+    }
+}
+// The http crawler of a frontier: fetches and parses without rendering; storage stays in memory.
+export function httpCrawler(config, onPage, frontier, storage, proxy) {
+    let pages = 0;
+    const revalidating = new WeakMap();
+    const hinted = new WeakMap();
+    const hopped = new WeakMap();
+    let revalidatedPages = 0;
+    const bodies = new WeakMap();
+    const crawler = new Crawler({
+        additionalMimeTypes: ["*/*"],
+        ...frontier.options("http", storage, proxy),
+        maxConcurrency: width(config.concurrency),
+        preNavigationHooks: [
+            async ({ request }, gotOptions) => {
+                bodies.delete(request);
+                Object.assign(gotOptions, { decompress: false, headers: { ...gotOptions.headers, ...agentHeaders(), "accept-encoding": ACCEPT_ENCODING } });
+                const hints = [];
+                hinted.set(request, hints);
+                const hops = [];
+                hopped.set(request, hops);
+                Object.assign(gotOptions, {
+                    hooks: { ...gotOptions.hooks, beforeRequest: [...(gotOptions.hooks?.beforeRequest ?? []), (options) => guardUrl(options.url ?? request.url), earlyHintsHook(request.url, hints)], beforeRedirect: [...(gotOptions.hooks?.beforeRedirect ?? []), redirectHook(hops)] },
+                });
+                const earlier = config.cacheMode === "use" ? await storage?.earlier?.(request.url) : undefined;
+                const conditional = earlier ? validators(earlier.facts.http.headers) : {};
+                log.debug({ url: request.url, isStored: earlier !== undefined, conditional: Object.keys(conditional) }, "page revalidation decided");
+                if (earlier && Object.keys(conditional).length > 0) {
+                    revalidating.set(request, earlier);
+                    Object.assign(gotOptions, { headers: { ...gotOptions.headers, ...conditional } });
+                }
+                if (config.keepalive)
+                    return;
+                Object.assign(gotOptions, { http2: false, headers: { ...gotOptions.headers, connection: "close" } });
+            },
+        ],
+        postNavigationHooks: [
+            (context) => {
+                const source = context.response;
+                const contentType = source.headers["content-type"];
+                const max = isParsed(contentType) ? config.maxBodySize : 0;
+                const cap = capped(source, max);
+                log.debug({ url: context.request.url, contentType, max }, "body capped");
+                const socket = socketOf(source);
+                const address = socket?.remoteAddress ?? source.ip;
+                const remote = address ? { address, ...(socket?.remoteFamily && { family: socket.remoteFamily }) } : undefined;
+                bodies.set(context.request, { ...cap, source: source, tls: tlsFacts(socket), remote });
+                const earlier = source.statusCode === 304 ? revalidating.get(context.request) : undefined;
+                log.debug({ url: context.request.url, isReplayed: earlier !== undefined }, "page body chosen");
+                Object.assign(context, { response: earlier ? replayed(cap.stream, earlier.body, storedContentType(earlier.facts)) : cap.stream });
+            },
+        ],
+        async requestHandler({ request, response, body, $, contentType, enqueueLinks }) {
+            const url = new URL(request.loadedUrl ?? request.url);
+            if (!frontier.admit(request, url))
+                return;
+            const pending = revalidating.get(request);
+            const earlier = response.statusCode === 304 ? pending : undefined;
+            // A validator answered 200 with the stored body still byte-identical.
+            const isUnmodified = response.statusCode === 200 && pending !== undefined && body.toString() === pending.body;
+            const isHtml = HTML_TYPES.has(contentType.type);
+            const cap = bodies.get(request);
+            const decoded = Buffer.byteLength(body);
+            const declared = Number(response.headers["content-length"]);
+            const skew = dateSkew(url.href, response.headers, cap?.source.timings?.upload, cap?.source.timings?.response);
+            const facts = {
+                ...frontier.identity(request, url, "http"),
+                http: {
+                    status: response.statusCode ?? 0,
+                    ...(cap?.source.httpVersion && { version: cap.source.httpVersion }),
+                    redirects: (cap?.source.redirectUrls ?? []).map((redirect, index, all) => hopped.get(request)?.at(index - all.length) ?? { url: String(redirect) }),
+                    headers: redactHeaders(response.headers),
+                    ...(cap?.remote && { remote: cap.remote }),
+                    timing: cap ? timingFacts(cap.source) : {},
+                    cookies: cookieFacts(response.headers["set-cookie"], response.headers.date),
+                    ...(hinted.get(request)?.length && { "early-hints": hinted.get(request) }),
+                    size: {
+                        body: transferred(cap?.source) ?? decoded,
+                        decoded,
+                        ...(Number.isSafeInteger(declared) && { declared }),
+                        ...(cap?.isTruncated() && { truncated: true }),
+                    },
+                    "content-type": contentType.type,
+                    ...(contentType.encoding && { charset: contentType.encoding }),
+                    ...(skew !== undefined && { "date-skew": skew }),
+                    ...(isUnmodified && { unmodified: true }),
+                },
+                ...(cap?.tls && { tls: cap.tls }),
+                ...(isHtml && { html: extractHtml($, body.toString(), url, config.scope), resources: extractResources($, url, config.maxResourcesPerPage) }),
+            };
+            if (earlier)
+                facts.http = revalidated(earlier.facts, facts);
+            revalidatedPages += earlier ? 1 : 0;
+            pages += 1;
+            log.debug({ url: url.href, status: facts.http.status, type: facts.http["content-type"], bytes: facts.http.size.body, depth: facts.crawl.depth, revalidated: facts.http.revalidated, unmodified: facts.http.unmodified }, "page fetched");
+            await onPage(facts, body.toString());
+            if (!isHtml && !facts.feed)
+                return;
+            log.debug({ url: url.href, enqueued: await frontier.enqueue(enqueueLinks, facts, "http") }, "links enqueued");
+        },
+        async failedRequestHandler({ request }, error) {
+            const facts = frontier.failed(request, failure(error, bodies.get(request)?.source));
+            if (facts)
+                await onPage(facts, "");
+        },
+    }, storage?.config ?? new Configuration({ persistStorage: false, purgeOnStart: false }), frontier);
+    return { crawler, stats: () => ({ pages, revalidated: revalidatedPages }) };
+}