spiderlint 0.0.0 → 0.143.1

package/dist/crawl/links.js

@@ -0,0 +1,89 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { progressCount, progressPhase } from "../progress.js";
+import { vendorPath } from "../facts/vendors.js";
+import { log } from "../logger.js";
+import { reason } from "./fetch.js";
+import { PrivateAddress } from "./guard.js";
+import { probe } from "./probe.js";
+import { width } from "./resources.js";
+// A Cloudflare challenge, a Cloudflare edge block that never asked the origin, or LinkedIn’s 999: the answer says nothing about the page.
+function isWalled(answer) {
+    const timing = [answer.headers["server-timing"] ?? []].flat().join(",");
+    return answer.headers["cf-mitigated"] === "challenge" || (answer.status === 403 && /(^|,)\s*cfOrigin;dur=0(,|$)/.test(timing)) || answer.status === 999;
+}
+// A HEAD, then a GET when the server refuses HEAD; a network failure is status 0 with its error.
+async function probeOne(href, config, signal) {
+    const options = { host: new URL(href).hostname, allowPrivate: config.allowPrivate, signal };
+    try {
+        const head = await probe(href, { method: "HEAD", redirect: "follow" }, options);
+        const answer = head.status === 405 ? await probe(href, { method: "GET", redirect: "follow" }, options) : head;
+        const isWall = isWalled(answer);
+        log.debug({ url: href, head: head.status, status: answer.status, isWall }, "external link probed");
+        return { status: answer.status, method: head.status === 405 ? "GET" : "HEAD", ...(isWall && { walled: true }) };
+    }
+    catch (error) {
+        log.debug({ url: href, error: reason(error) }, "external link unreachable");
+        return { status: 0, error: reason(error), ...(error instanceof PrivateAddress && { refused: true }) };
+    }
+}
+// Whether an answer says anything about its link: not excluded, a vendor’s, refused, walled or rate limited.
+export function isJudged(answer) {
+    return !answer.excluded && !answer.vendor && !answer.refused && !answer.walled && answer.status !== 429;
+}
+// A healthy or walled answer: the only kind worth keeping.
+function isKept(answer) {
+    return answer.status > 0 && (answer.status < 400 || answer.walled === true);
+}
+// One link’s answer, from `bucket` while fresh and still worth keeping; an excluded host or a vendor path is never asked, and only a healthy or walled answer is stored.
+export async function answerOf(href, config, bucket, signal) {
+    const vendor = config.vendorPaths ? vendorPath(href, "page")?.vendor : undefined;
+    log.debug({ url: href, vendor }, "external link vendor");
+    if (vendor)
+        return { status: 0, vendor };
+    const host = new URL(href).hostname;
+    const isExcluded = config.linkExclude.some((entry) => host === entry || host.endsWith(`.${entry}`));
+    log.debug({ url: href, host, isExcluded }, "external link scoped");
+    if (isExcluded)
+        return { status: 0, excluded: true };
+    const entry = await bucket.get(href);
+    const isReused = entry !== undefined && bucket.isFresh(entry) && isKept(entry.value);
+    log.debug({ url: href, isCached: entry !== undefined, isReused }, "external link cache");
+    if (isReused)
+        return { ...entry.value, cached: true };
+    const answer = { ...(await probeOne(href, config, signal)), checked: new Date().toISOString() };
+    const isStored = isKept(answer);
+    log.debug({ url: href, status: answer.status, isStored }, "external link answer");
+    if (isStored)
+        await bucket.set(href, answer);
+    return answer;
+}
+// Probes every distinct http(s) link once, one request at a time per host, hosts in parallel.
+export async function probeLinks(links, config, bucket) {
+    const hrefs = [...new Set(links)].filter((href) => URL.canParse(href) && /^https?:$/.test(new URL(href).protocol));
+    const hosts = Map.groupBy(hrefs, (href) => new URL(href).hostname).values();
+    log.debug({ links: hrefs.length }, "external links found");
+    const answers = {};
+    const signal = new AbortController().signal;
+    let cached = 0;
+    let probed = 0;
+    progressPhase("probes", hrefs.length);
+    const worker = async () => {
+        for (const group of hosts) {
+            for (const href of group) {
+                const { cached: isCached, ...answer } = await answerOf(href, config, bucket, signal);
+                cached += isCached ? 1 : 0;
+                answers[href] = answer;
+                progressCount((probed += 1));
+            }
+        }
+    };
+    const workers = Array.from({ length: Math.min(width(config.concurrency), hrefs.length) }, worker);
+    await Promise.all(workers);
+    const all = Object.values(answers);
+    const skipped = all.filter((answer) => !isJudged(answer)).length;
+    const failed = all.filter((answer) => isJudged(answer) && (answer.status === 0 || answer.status >= 400)).length;
+    log.debug({ links: hrefs.length, cached, failed, skipped }, "external links probed");
+    return answers;
+}

package/dist/crawl/log.js

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log as crawleeLog, Logger, LogLevel } from "crawlee";
+import { log } from "../logger.js";
+const METHOD = {
+    [LogLevel.ERROR]: "error",
+    [LogLevel.SOFT_FAIL]: "warn",
+    [LogLevel.WARNING]: "warn",
+    [LogLevel.INFO]: "info",
+    [LogLevel.DEBUG]: "debug",
+    [LogLevel.PERF]: "trace",
+};
+// Crawlee lines land in the pino stream with their data fields and a `crawlee` marker; its npm-worded Ctrl-C hint is dropped for spiderlint’s own.
+class PinoLogger extends Logger {
+    _log(level, message, data, exception, options) {
+        if (message.startsWith("Pausing..."))
+            return;
+        const error = exception instanceof Error ? exception.message : exception;
+        log[METHOD[level] ?? "info"]({ ...data, crawlee: options?.prefix ?? true, ...(error !== undefined && { error }) }, message.split("\n    at ", 1)[0]);
+    }
+}
+// Crawlee stays at WARNING unless spiderlint itself is at debug or below.
+export function bridgeCrawleeLog() {
+    const isVerbose = log.levelVal <= log.levels.values.debug;
+    crawleeLog.setOptions({ logger: new PinoLogger({}), level: isVerbose ? LogLevel.DEBUG : LogLevel.WARNING });
+}

package/dist/crawl/network.js

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { setGlobalProxyFromEnv } from "node:http";
+import { setTimeout as sleep } from "node:timers/promises";
+import { Server } from "proxy-chain";
+import { ConfigError, defaults } from "../config/index.js";
+import { log } from "../logger.js";
+import { refuseLiteral } from "./guard.js";
+import { openResolution } from "./resolve.js";
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
+const MAX_REDIRECTS = 20;
+const SOCKS = new Set(["socks:", "socks4:", "socks4a:", "socks5:", "socks5h:"]);
+// Milliseconds between two requests, when the next may start, and the run’s `timeout` over its default.
+const pacing = { spacing: 0, next: 0, scale: 1 };
+// Whether the open run may reach loopback, private and link-local addresses.
+const guard = { isPrivateAllowed: true };
+// Throws for a private address literal while the open run refuses private addresses.
+export function guardUrl(url) {
+    refuseLiteral(url, guard.isPrivateAllowed);
+}
+// `fetch`, following redirects itself while the guard is on, so every hop’s address literal is checked as the first one was.
+export async function guardedFetch(url, init) {
+    guardUrl(url);
+    if (guard.isPrivateAllowed)
+        return fetch(url, init);
+    let current = url;
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
+        const response = await fetch(current, { ...init, redirect: "manual" });
+        const location = response.headers.get("location");
+        if (location === null || !REDIRECTS.has(response.status))
+            return response;
+        await response.body?.cancel();
+        const next = new URL(location, current).href;
+        log.debug({ from: current, to: next, status: response.status, hop }, "redirect followed");
+        guardUrl(next);
+        current = next;
+    }
+    throw new TypeError(`${url}: more than ${MAX_REDIRECTS} redirects`);
+}
+// `ms` stretched by the run’s `timeout`, so a slow network gets as long as a page does.
+export function patient(ms) {
+    return ms * pacing.scale;
+}
+// Waits for the next slot under `rate`; unlimited returns at once.
+export async function pace() {
+    if (pacing.spacing === 0)
+        return;
+    const now = Date.now();
+    const slot = Math.max(now, pacing.next);
+    pacing.next = slot + pacing.spacing;
+    log.trace({ waitMs: slot - now, spacing: pacing.spacing }, "request paced");
+    if (slot > now)
+        await sleep(slot - now);
+}
+// Paces, and routes every HTTP client of one run through `config.proxy`, until `close`.
+export async function openNetwork(config) {
+    pacing.spacing = config.rate > 0 ? 60_000 / config.rate : 0;
+    pacing.next = 0;
+    pacing.scale = config.timeout / defaults().timeout;
+    log.debug({ rate: config.rate, spacing: pacing.spacing, scale: pacing.scale, isProxied: config.proxy !== "" }, "network opened");
+    guard.isPrivateAllowed = config.allowPrivate;
+    const unresolve = await openResolution(config.resolve, config.resolver, config.seeds, config.proxy !== "", config.allowPrivate);
+    const reset = () => {
+        pacing.spacing = 0;
+        pacing.scale = 1;
+        guard.isPrivateAllowed = true;
+        unresolve();
+    };
+    if (!config.proxy)
+        return { close: async () => reset() };
+    if (!config.allowPrivate)
+        throw new ConfigError("proxy: the address guard cannot check what a proxy connects to");
+    const upstream = new URL(config.proxy);
+    const bridge = SOCKS.has(upstream.protocol) ? new Server({ host: "127.0.0.1", port: 0, prepareRequestFunction: () => ({ upstreamProxyUrl: config.proxy }) }) : undefined;
+    await bridge?.listen();
+    const proxy = bridge ? `http://127.0.0.1:${bridge.port}` : config.proxy;
+    const restore = setGlobalProxyFromEnv({ HTTP_PROXY: proxy, HTTPS_PROXY: proxy });
+    log.debug({ proxy: `${upstream.protocol}//${upstream.host}`, isBridged: bridge !== undefined }, "requests go through the proxy");
+    return {
+        proxy,
+        async close() {
+            restore();
+            reset();
+            await bridge?.close(true);
+        },
+    };
+}

package/dist/crawl/probe.js

@@ -0,0 +1,107 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { isIP } from "node:net";
+import { setTimeout as sleep } from "node:timers/promises";
+import { agentHeaders } from "../agent.js";
+import { redactHeaders } from "../facts/transport.js";
+import { log } from "../logger.js";
+import { delay, reason } from "./fetch.js";
+import { guardedLookup, isPrivate, PrivateAddress } from "./guard.js";
+import { pace, patient } from "./network.js";
+import { robotsFactsOf } from "./robots.js";
+const ATTEMPTS = 2;
+const TIMEOUT_MS = 10_000;
+const MAX_HOPS = 10;
+const MAX_BODY = 1_048_576;
+const RETRY_STATUS = new Set([429, 503]);
+// A probe robots.txt disallows for `spiderlint`, never sent; `unreachable` when that is only because the file did not answer.
+export class RobotsDisallowed extends Error {
+    unreachable;
+    constructor(message, unreachable) {
+        super(message);
+        if (unreachable !== undefined)
+            this.unreachable = unreachable;
+    }
+}
+// Reads at most MAX_BODY bytes of a response, as text and as they came.
+async function text(response) {
+    const chunks = [];
+    let bytes = 0;
+    for await (const chunk of response) {
+        chunks.push(chunk);
+        bytes += chunk.byteLength;
+        if (bytes < MAX_BODY)
+            continue;
+        response.destroy();
+        const raw = Buffer.concat(chunks).subarray(0, MAX_BODY);
+        return { body: raw.toString("utf8"), raw, truncated: true };
+    }
+    const raw = Buffer.concat(chunks);
+    return { body: raw.toString("utf8"), raw };
+}
+// One request on a guarded socket, never following a redirect.
+async function once(url, init, options) {
+    const literal = url.hostname.replaceAll(/^\[|\]$/g, "");
+    if (!options.allowPrivate && isIP(literal) && isPrivate(literal))
+        throw new PrivateAddress(`${url.hostname} is a private address`);
+    const request = url.protocol === "https:" ? httpsRequest : httpRequest;
+    await pace();
+    const expiry = AbortSignal.timeout(patient(TIMEOUT_MS));
+    const response = await new Promise((resolve, reject) => {
+        const outgoing = request(url, { method: init.method ?? "GET", headers: { ...init.headers, ...agentHeaders() }, signal: AbortSignal.any([options.signal, expiry]), ...(!options.allowPrivate && { lookup: guardedLookup }) }, resolve);
+        outgoing.on("error", reject);
+        outgoing.end();
+    });
+    const { body, raw, truncated } = await text(response);
+    return { url: url.href, status: response.statusCode ?? 0, headers: redactHeaders(response.headers), body, ...(truncated && { truncated }), ...(init.binary && { bytes: raw }) };
+}
+// One request with a retry on a network error, 429 or 503.
+async function retrying(url, init, options) {
+    for (let attempt = 0;; attempt += 1) {
+        const isLast = attempt === ATTEMPTS - 1;
+        try {
+            const answer = await once(url, init, options);
+            log.debug({ url: url.href, method: init.method ?? "GET", status: answer.status, attempt }, "probed");
+            if (isLast || !RETRY_STATUS.has(answer.status))
+                return answer;
+            const retryAfter = answer.headers["retry-after"];
+            await sleep(delay(attempt, typeof retryAfter === "string" ? retryAfter : undefined), undefined, { signal: options.signal });
+        }
+        catch (error) {
+            log.debug({ url: url.href, attempt, error: reason(error) }, "probe failed");
+            if (isLast || error instanceof PrivateAddress || options.signal.aborted)
+                throw error;
+            await sleep(delay(attempt), undefined, { signal: options.signal });
+        }
+    }
+}
+// A GET or HEAD on `options.host` only, through the address guard unless private addresses are allowed.
+export async function probe(href, init, options) {
+    const started = performance.now();
+    const redirects = [];
+    let url = new URL(href);
+    for (let hop = 0;; hop += 1) {
+        if (url.hostname !== options.host)
+            throw new Error(`probe ${url.href} leaves host ${options.host}`);
+        const robots = await options.robots?.(url.href);
+        if (robots && !robots.isAllowed(url.href, "spiderlint")) {
+            const facts = robotsFactsOf(robots);
+            const unreachable = facts?.error ?? (facts && facts.status >= 500 ? `robots.txt answers ${facts.status}` : undefined);
+            log.info({ url: url.href, unreachable }, "robots.txt disallows the probe; skipped");
+            throw new RobotsDisallowed(`robots.txt disallows ${url.href}`, unreachable);
+        }
+        const answer = await retrying(url, init, options);
+        const location = answer.headers.location;
+        const isRedirect = answer.status >= 300 && answer.status < 400 && typeof location === "string";
+        const next = isRedirect && URL.canParse(location, url) ? new URL(location, url) : undefined;
+        const isFollowed = init.redirect === "follow" && next?.hostname === options.host && hop < MAX_HOPS;
+        log.debug({ url: url.href, status: answer.status, location, hop, isFollowed }, "probe answered");
+        if (!isFollowed)
+            return { ...answer, redirects, ms: Math.round(performance.now() - started) };
+        redirects.push({ url: url.href, status: answer.status });
+        url = next;
+    }
+}

package/dist/crawl/profile.js

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { fetchCached } from "../cache/http.js";
+import { log } from "../logger.js";
+import { RobotsDisallowed } from "./probe.js";
+// Bytes of a page read, as a probe reads at most as many.
+const MAX_BODY = 1_048_576;
+async function read(response) {
+    const chunks = [];
+    let bytes = 0;
+    const stream = response.body;
+    if (stream) {
+        for await (const chunk of stream) {
+            chunks.push(chunk);
+            bytes += chunk.byteLength;
+            if (bytes >= MAX_BODY)
+                break;
+        }
+    }
+    return { url: response.url, body: Buffer.concat(chunks).subarray(0, MAX_BODY).toString("utf8") };
+}
+// A GET answered from `bucket` while fresh, revalidated with its validators once stale; robots.txt applies as to any probe.
+export async function cachedGet(url, bucket, robots) {
+    const file = await robots?.(url);
+    if (file && !file.isAllowed(url, "spiderlint")) {
+        log.info({ url }, "robots.txt disallows the page; skipped");
+        throw new RobotsDisallowed(`robots.txt disallows ${url}`);
+    }
+    const { status, value, at, cached, revalidated } = await fetchCached(bucket, url, read, true);
+    log.debug({ url, status, at, cached, revalidated }, "page read through the profiles bucket");
+    return { url: value.url, status, body: value.body, at: at ?? new Date().toISOString(), ...(cached && { cached }), ...(revalidated && { revalidated }) };
+}

package/dist/crawl/rdap.js

@@ -0,0 +1,86 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import lockfile from "proper-lockfile";
+import { userCacheDirectory, writeAtomic } from "../cache/index.js";
+import { log } from "../logger.js";
+import { reason } from "./fetch.js";
+// RFC 9224 §4: the IANA bootstrap mapping TLDs to RDAP base URLs.
+export const BOOTSTRAP = "https://data.iana.org/rdap/dns.json";
+const FRESH_MS = 86_400_000;
+const STALE_LOCK_MS = 30_000;
+const bootstrapFile = () => path.join(userCacheDirectory(), "rdap", "dns.json");
+// The stored bootstrap, however old; none when absent or unreadable.
+async function stored() {
+    try {
+        return JSON.parse(await readFile(bootstrapFile(), "utf8"));
+    }
+    catch (error) {
+        log.debug({ file: bootstrapFile(), error: reason(error) }, "rdap bootstrap not stored");
+        return undefined;
+    }
+}
+const isFresh = (entry) => Date.now() - Date.parse(entry.fetched) < FRESH_MS;
+// Fetches and stores the bootstrap; the caller holds the lock.
+async function refresh(get) {
+    const answer = await get(BOOTSTRAP, { headers: { accept: "application/json" } });
+    if (answer.status !== 200)
+        throw new Error(`the IANA RDAP bootstrap answered ${answer.status}`);
+    const services = JSON.parse(answer.body).services ?? [];
+    await writeAtomic(bootstrapFile(), JSON.stringify({ fetched: new Date().toISOString(), services }));
+    log.debug({ url: BOOTSTRAP, services: services.length }, "rdap bootstrap refreshed");
+    return services;
+}
+// The bootstrap services from the user cache while under a day old, else refetched by one process under a file lock; a stale copy stands in when the refresh fails.
+export async function bootstrap(get) {
+    const cached = await stored();
+    if (cached && isFresh(cached))
+        return cached.services;
+    await mkdir(path.dirname(bootstrapFile()), { recursive: true });
+    const release = await lockfile.lock(path.dirname(bootstrapFile()), { lockfilePath: `${bootstrapFile()}.lock`, realpath: false, stale: STALE_LOCK_MS, retries: { retries: 10, minTimeout: 200, maxTimeout: 2000, randomize: true } });
+    try {
+        const again = await stored();
+        return again && isFresh(again) ? again.services : await refresh(get);
+    }
+    catch (error) {
+        if (!cached)
+            throw error;
+        log.warn({ url: BOOTSTRAP, fetched: cached.fetched, error: reason(error) }, "rdap bootstrap refresh failed; using the stored copy");
+        return cached.services;
+    }
+    finally {
+        await release();
+    }
+}
+// The RDAP base URL of the longest bootstrap suffix `domain` ends with, https preferred.
+export function serverFor(domain, services) {
+    const labels = domain.toLowerCase().split(".");
+    for (let index = 1; index < labels.length; index += 1) {
+        const suffix = labels.slice(index).join(".");
+        const urls = services.find(([suffixes]) => suffixes.includes(suffix))?.[1];
+        if (urls)
+            return urls.find((url) => url.startsWith("https:")) ?? urls[0];
+    }
+    return undefined;
+}
+// A name server name lower-cased without its root dot.
+export const nsName = (name) => name.toLowerCase().replace(/\.$/, "");
+// The status, expiration and name servers of `domain` from its registry’s RDAP server (RFC 9083); none when no server serves its TLD.
+export async function registration(domain, get) {
+    const base = serverFor(domain, await bootstrap(get));
+    log.debug({ domain, server: base }, "rdap server found");
+    if (!base)
+        return undefined;
+    const url = new URL(`domain/${domain}`, base.endsWith("/") ? base : `${base}/`).href;
+    const answer = await get(url, { headers: { accept: "application/rdap+json" }, redirect: "follow" });
+    if (answer.status !== 200)
+        throw new Error(`RDAP ${url} answered ${answer.status}`);
+    const body = JSON.parse(answer.body);
+    const expires = body.events?.find((event) => event.eventAction === "expiration")?.eventDate;
+    const isDated = expires !== undefined && !Number.isNaN(Date.parse(expires));
+    const nameservers = [...new Set((body.nameservers ?? []).flatMap((server) => (server.ldhName ? nsName(server.ldhName) : [])))].toSorted((a, b) => a.localeCompare(b));
+    log.debug({ domain, url, status: body.status, expires, nameservers: nameservers.length }, "rdap answered");
+    return { server: base, ...(Array.isArray(body.status) && { status: body.status.map((entry) => String(entry).toLowerCase()) }), ...(isDated && { expires: new Date(expires).toISOString() }), nameservers };
+}

package/dist/crawl/resolve.js

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import dns, {} from "node:dns";
+import { syncBuiltinESMExports } from "node:module";
+import { isIP } from "node:net";
+import { Bucket } from "../cache/index.js";
+import { ConfigError } from "../config/index.js";
+import { log } from "../logger.js";
+import { dnsClient } from "./dns.js";
+import { reason } from "./fetch.js";
+import { guarding } from "./guard.js";
+// Chromium’s `--host-resolver-rules` for the open run, empty when nothing is pinned.
+const browser = { hostRules: "" };
+// `host:address`, or curl’s `host:port:address` whose port is accepted and ignored, since a lookup never sees it.
+export function parsePin(raw) {
+    const match = /^([^:[\]\s]+)(?::\d+)?:(\[[^\]]+\]|[^:[\]\s]+)$/.exec(raw.trim());
+    const address = match?.[2]?.replaceAll(/^\[|\]$/g, "") ?? "";
+    if (!match || isIP(address) === 0)
+        throw new ConfigError(`resolve: invalid pin ${raw} (expected host[:port]:address)`);
+    return { host: match[1].toLowerCase(), address };
+}
+function answer(addresses, options, callback) {
+    const [first] = addresses;
+    // eslint-disable-next-line unicorn/no-null -- the lookup callback contract types a success as a null error
+    callback(null, options.all ? addresses : (first?.address ?? ""), first?.family ?? 0);
+}
+function notFound(hostname, why) {
+    return Object.assign(new Error(`getaddrinfo ENOTFOUND ${hostname} (${why})`), { code: "ENOTFOUND", hostname });
+}
+// A and AAAA of `hostname` from `client`, IPv4 first, restricted to `family` when it is 4 or 6.
+async function addressesOf(client, hostname, family) {
+    const types = family === 4 ? ["A"] : family === 6 ? ["AAAA"] : ["A", "AAAA"];
+    const replies = await Promise.all(types.map((type) => client.query(hostname, type)));
+    return replies.flatMap((reply) => reply.answers.flatMap((record) => (record.type === "A" || record.type === "AAAA" ? { address: record.data, family: record.type === "A" ? 4 : 6 } : [])));
+}
+// `dns.lookup` overloads (`(host, cb)`, `(host, family, cb)`) normalised to the three-argument form `lookup` takes.
+function overloaded(lookup) {
+    return ((hostname, options, callback) => {
+        log.debug({ hostname, form: typeof options }, "dns.lookup called");
+        if (typeof options === "function")
+            return lookup(hostname, {}, options);
+        const normal = typeof options === "number" ? { family: options } : (options ?? {});
+        return lookup(hostname, normal, callback);
+    });
+}
+// Pins first, then the configured servers, `localhost` always from the system; every answer is kept for the run.
+function crawlLookup(pins, client, system) {
+    const answers = new Map();
+    return (hostname, options, callback) => {
+        const name = hostname.toLowerCase().replace(/\.$/, "");
+        const pin = pins.find((entry) => entry.host === name);
+        if (pin) {
+            log.debug({ hostname, address: pin.address }, "name pinned by --resolve");
+            answer([{ address: pin.address, family: isIP(pin.address) }], options, callback);
+            return;
+        }
+        if (!client || name === "localhost" || name.endsWith(".localhost")) {
+            system(hostname, options, callback);
+            return;
+        }
+        const family = Number(options.family ?? 0);
+        const key = `${name}\t${family}`;
+        let pending = answers.get(key);
+        if (!pending) {
+            pending = addressesOf(client, name, family);
+            answers.set(key, pending);
+        }
+        const settle = async (asked) => {
+            let addresses;
+            try {
+                addresses = await asked;
+            }
+            catch (error) {
+                log.debug({ hostname, error: reason(error) }, "resolver lookup failed");
+                callback(notFound(hostname, reason(error)), "", 0);
+                return;
+            }
+            log.debug({ hostname, family, addresses: addresses.map((entry) => entry.address) }, "name resolved through the resolver");
+            if (addresses.length === 0)
+                callback(notFound(hostname, "no address from the resolver"), "", 0);
+            else
+                answer(addresses, options, callback);
+        };
+        void settle(pending);
+    };
+}
+// `MAP host address` per pin, then per seed host the resolver answers; Chromium resolves any other name itself.
+async function chromiumRules(pins, seeds, client) {
+    const mapped = new Map(pins.map((pin) => [pin.host, pin.address]));
+    const hosts = [...new Set(seeds.flatMap((seed) => (URL.canParse(seed) ? [new URL(seed).hostname] : [])))].filter((host) => isIP(host.replaceAll(/^\[|\]$/g, "")) === 0 && !mapped.has(host));
+    if (client) {
+        for (const host of hosts) {
+            try {
+                const [first] = await addressesOf(client, host, 0);
+                log.debug({ host, address: first?.address }, "seed host resolved for the browser");
+                if (first)
+                    mapped.set(host, first.address);
+            }
+            catch (error) {
+                log.warn({ host, error: reason(error) }, `${host} unresolved for the browser, which asks the system instead:`);
+            }
+        }
+    }
+    return [...mapped].map(([host, address]) => `MAP ${host} ${isIP(address) === 6 ? `[${address}]` : address}`).join(", ");
+}
+// Chromium launch arguments for the open run’s pins and resolver.
+export function chromiumArguments() {
+    return browser.hostRules ? [`--host-resolver-rules=${browser.hostRules}`] : [];
+}
+// Swaps `dns.lookup` so got, fetch, probes and TLS handshakes resolve through pins and `resolver`, guarded unless private is allowed; returns the restore.
+export async function openResolution(pins, resolver, seeds, isProxied, isPrivateAllowed = true) {
+    const isSystem = resolver === "system";
+    if (isPrivateAllowed && isSystem && pins.length === 0)
+        return () => { };
+    if (isProxied) {
+        log.warn({ pins: pins.length, resolver }, "names resolve inside the proxy; --resolve and --resolver do not reach the crawl");
+        return () => { };
+    }
+    const client = isSystem ? undefined : dnsClient(resolver, new Bucket("dns", undefined, 60, "off"), false);
+    const system = dns.lookup;
+    const lookup = crawlLookup(pins, client, system);
+    dns.lookup = overloaded(isPrivateAllowed ? lookup : guarding(lookup));
+    syncBuiltinESMExports();
+    browser.hostRules = await chromiumRules(pins, seeds, client);
+    log.debug({ pins: pins.map((pin) => pin.host), resolver, isPrivateAllowed, browserRules: browser.hostRules }, "crawl resolves names through the configured resolution");
+    return () => {
+        dns.lookup = system;
+        syncBuiltinESMExports();
+        browser.hostRules = "";
+    };
+}