spiderlint 0.0.0 → 0.143.0

package/dist/crawl/diversity.js

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+// Directory names kept apart per level; later names share one wildcard child.
+const FANOUT = 32;
+const WILDCARD = "*";
+// Pooled candidates in arrival order.
+class Bucket {
+    #items = [];
+    #head = 0;
+    get size() {
+        return this.#items.length - this.#head;
+    }
+    push(candidate) {
+        this.#items.push(candidate);
+    }
+    // The oldest candidate; the array is compacted once half of it is spent.
+    shift() {
+        const candidate = this.#items[this.#head];
+        this.#head += 1;
+        if (this.#head >= 1024 && this.#head * 2 >= this.#items.length) {
+            this.#items = this.#items.slice(this.#head);
+            this.#head = 0;
+        }
+        return candidate;
+    }
+}
+// One directory of the URL tree: its own pages, its subdirectories, and how much of the budget each took.
+class Directory {
+    children = new Map();
+    leaves = new Bucket();
+    // Pages scheduled from this subtree, and from its own pages alone.
+    taken = 0;
+    leafTaken = 0;
+    // Distinct pages this directory itself has held.
+    leafCount = 0;
+    // Candidates still pooled in this subtree.
+    pooled = 0;
+}
+// The URL without its fragment, which names the same page.
+function keyOf(href) {
+    const url = new URL(href);
+    url.hash = "";
+    return url.href;
+}
+// Candidates pooled by directory; each pick descends to the least-scheduled child, so a budget spreads over the site’s sections before it deepens one.
+export class Spread {
+    #root = new Directory();
+    #known = new Set();
+    #pooled = new Map();
+    #limit;
+    // `limit` caps one directory’s pooled pages: a budget never takes more than that from it.
+    constructor(limit) {
+        this.#limit = limit;
+    }
+    // The directories `href` sits under, host first, created on the way.
+    #walk(href) {
+        const url = new URL(href);
+        const names = [url.host, ...url.pathname.split("/").filter(Boolean).slice(0, -1)];
+        const path = [];
+        let node = this.#root;
+        for (const name of names) {
+            const key = node.children.has(name) || node.children.size < FANOUT ? name : WILDCARD;
+            const next = node.children.get(key) ?? new Directory();
+            node.children.set(key, next);
+            path.push(next);
+            node = next;
+        }
+        return path;
+    }
+    // The next candidate: at each level the child, or the directory’s own pages, with the smallest share of the budget so far.
+    #pick() {
+        const path = [];
+        let node = this.#root;
+        for (;;) {
+            let next;
+            let least = node.leaves.size > 0 ? node.leafTaken / Math.min(node.leafCount, FANOUT) : Infinity;
+            for (const child of node.children.values()) {
+                if (child.pooled === 0 || child.taken >= least)
+                    continue;
+                next = child;
+                least = child.taken;
+            }
+            if (!next)
+                break;
+            path.push(next);
+            node = next;
+        }
+        const candidate = node.leaves.shift();
+        this.#pooled.delete(keyOf(candidate.url));
+        node.leafTaken += 1;
+        for (const directory of path) {
+            directory.taken += 1;
+            directory.pooled -= 1;
+        }
+        this.#root.pooled -= 1;
+        return candidate;
+    }
+    get size() {
+        return this.#root.pooled;
+    }
+    // Counts a page scheduled elsewhere, so the pool favours what it has not covered; false for one already known.
+    note(href) {
+        const key = keyOf(href);
+        if (this.#known.has(key))
+            return false;
+        this.#known.add(key);
+        const path = this.#walk(href);
+        for (const directory of path)
+            directory.taken += 1;
+        const holder = path.at(-1);
+        holder.leafTaken += 1;
+        holder.leafCount += 1;
+        return true;
+    }
+    // Pools a candidate; false for one already known or past its directory’s limit.
+    add(candidate) {
+        const key = keyOf(candidate.url);
+        if (this.#known.has(key))
+            return false;
+        const path = this.#walk(candidate.url);
+        const holder = path.at(-1);
+        if (holder.leaves.size >= this.#limit) {
+            log.debug({ url: candidate.url, limit: this.#limit }, "candidate dropped, directory full");
+            return false;
+        }
+        this.#known.add(key);
+        holder.leaves.push(candidate);
+        this.#pooled.set(key, candidate);
+        holder.leafCount += 1;
+        for (const directory of path)
+            directory.pooled += 1;
+        this.#root.pooled += 1;
+        return true;
+    }
+    // Gives a pooled candidate that no link has reached the depth of the page linking to it; true when it did.
+    relink(url, crawlDepth) {
+        const pooled = this.#pooled.get(keyOf(url));
+        if (!pooled || pooled.crawlDepth > 0)
+            return false;
+        pooled.crawlDepth = crawlDepth;
+        return true;
+    }
+    // Up to `count` candidates, each from the least-scheduled directory at every level.
+    take(count) {
+        const taken = [];
+        while (taken.length < count && this.#root.pooled > 0)
+            taken.push(this.#pick());
+        log.debug({ requested: count, taken: taken.length, pooled: this.#root.pooled }, "candidates taken");
+        return taken;
+    }
+}

package/dist/crawl/dns.js

@@ -0,0 +1,194 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { randomInt } from "node:crypto";
+import { createSocket } from "node:dgram";
+import { readFileSync } from "node:fs";
+import { connect, isIP } from "node:net";
+import { setTimeout as sleep } from "node:timers/promises";
+import dnsPacket, {} from "dns-packet";
+import { ConfigError } from "../config/index.js";
+import { log } from "../logger.js";
+import { reason } from "./fetch.js";
+const ATTEMPTS = 2;
+const TIMEOUT_MS = 3000;
+const PAYLOAD = 1232;
+// A client refusing every query, for a proxied run whose queries would bypass the proxy.
+export const PROXIED_DNS = {
+    canQueryDirectly: false,
+    query: async (name) => {
+        throw new Error(`DNS query for ${name} would bypass the proxy`);
+    },
+    validating: async () => false,
+};
+// `address[:port]`, an IPv6 address bracketed when it carries a port.
+function server(entry) {
+    const bracketed = /^\[([^\]]+)\](?::(\d+))?$/.exec(entry);
+    const [address, port] = bracketed ? [bracketed[1], bracketed[2]] : isIP(entry) === 6 ? [entry, undefined] : entry.split(":");
+    const number = port === undefined ? 53 : Number(port);
+    if (isIP(address.split("%", 1)[0]) === 0 || !Number.isSafeInteger(number) || number < 1 || number > 65_535)
+        throw new ConfigError(`resolver: invalid server ${entry} (expected address[:port])`);
+    return { address, port: number };
+}
+// `system`, or a comma list of `address[:port]`; anything else is a config error.
+export function parseResolver(raw) {
+    const value = raw.trim();
+    if (value !== "system")
+        for (const entry of value.split(","))
+            server(entry.trim());
+    return value;
+}
+// The `nameserver` lines of `/etc/resolv.conf`.
+function systemServers() {
+    let text = "";
+    try {
+        text = readFileSync("/etc/resolv.conf", "utf8");
+    }
+    catch (error) {
+        log.warn({ error: reason(error) }, "resolv.conf unreadable");
+    }
+    const servers = text
+        .matchAll(/^\s*nameserver\s+(\S+)/gm)
+        .map((match) => ({ address: match[1], port: 53 }))
+        .toArray();
+    log.debug({ servers: servers.map((entry) => entry.address) }, "system resolvers read");
+    return servers;
+}
+// The servers a resolver setting names.
+export function servers(resolver) {
+    return resolver === "system" ? systemServers() : resolver.split(",").map((entry) => server(entry.trim()));
+}
+function label({ address, port }) {
+    return isIP(address.split("%", 1)[0]) === 6 ? `[${address}]:${port}` : `${address}:${port}`;
+}
+// One query over UDP, answered only by a reply carrying its ID.
+function overUdp(target, message, id, signal) {
+    return new Promise((resolve, reject) => {
+        const socket = createSocket(isIP(target.address.split("%", 1)[0]) === 6 ? "udp6" : "udp4");
+        const done = (error, reply) => {
+            socket.close();
+            signal.removeEventListener("abort", abort);
+            if (error)
+                reject(error);
+            else
+                resolve(reply);
+        };
+        const abort = () => done(new Error(`no answer from ${label(target)} in ${TIMEOUT_MS} ms`));
+        signal.addEventListener("abort", abort, { once: true });
+        socket.on("error", (error) => done(error));
+        socket.on("message", (reply) => {
+            if (reply.length >= 2 && reply.readUInt16BE(0) === id)
+                done(undefined, reply);
+        });
+        socket.send(message, target.port, target.address);
+    });
+}
+// One query over TCP, each message prefixed by its length.
+function overTcp(target, message, signal) {
+    return new Promise((resolve, reject) => {
+        const socket = connect({ host: target.address, port: target.port, signal });
+        const chunks = [];
+        const prefix = Buffer.alloc(2);
+        prefix.writeUInt16BE(message.length);
+        socket.on("connect", () => socket.end(Buffer.concat([prefix, message])));
+        socket.on("data", (chunk) => {
+            chunks.push(chunk);
+        });
+        socket.on("error", reject);
+        socket.on("end", () => {
+            const reply = Buffer.concat(chunks);
+            if (reply.length < 2 || reply.length < 2 + reply.readUInt16BE(0))
+                reject(new Error(`short TCP answer from ${label(target)}`));
+            else
+                resolve(reply.subarray(2, 2 + reply.readUInt16BE(0)));
+        });
+    });
+}
+// Encodes one question with `DO` set and `RD` unless the server is asked directly.
+function encode(name, type, options) {
+    const id = randomInt(65_536);
+    const flags = (options.server === undefined || options.recursion ? dnsPacket.RECURSION_DESIRED : 0) | dnsPacket.AUTHENTIC_DATA | (options.checkingDisabled ? dnsPacket.CHECKING_DISABLED : 0);
+    const message = dnsPacket.encode({ type: "query", id, flags, questions: [{ type: type, name, class: "IN" }], additionals: [{ type: "OPT", name: ".", udpPayloadSize: PAYLOAD, flags: dnsPacket.DNSSEC_OK }] });
+    return { id, message };
+}
+// One exchange with one server: UDP, then TCP when the answer is truncated.
+async function exchange(target, name, type, options, outer) {
+    const { id, message } = encode(name, type, options);
+    const signal = AbortSignal.any([outer, AbortSignal.timeout(TIMEOUT_MS)]);
+    const reply = await overUdp(target, message, id, signal);
+    if ((reply.readUInt16BE(2) & 0x02_00) === 0)
+        return reply;
+    log.debug({ name, type, server: label(target) }, "dns answer truncated, retrying over TCP");
+    return overTcp(target, message, signal);
+}
+// Decodes the wire answer the extractors read.
+function decode(target, wire) {
+    const packet = dnsPacket.decode(wire);
+    return { server: target, rcode: packet.rcode ?? "NOERROR", aa: packet.flag_aa ?? false, ad: packet.flag_ad ?? false, ra: packet.flag_ra ?? false, answers: packet.answers ?? [], authorities: packet.authorities ?? [] };
+}
+// The smallest TTL in an answer, floored at `floor` seconds.
+function lifetime(reply, floor) {
+    const ttls = [...reply.answers, ...reply.authorities].flatMap((record) => ("ttl" in record && record.ttl !== undefined ? [record.ttl] : []));
+    return ttls.length === 0 ? floor : Math.max(floor, Math.min(...ttls));
+}
+// Whether the resolver sets `AD` on the signed root SOA; warns when it does not.
+async function isValidating(client, resolver) {
+    try {
+        const { ad } = await client.query(".", "SOA");
+        log.debug({ resolver, validating: ad }, "resolver validation checked");
+        if (!ad)
+            log.warn({ resolver }, `resolver ${resolver} does not validate DNSSEC; dns/dnssec-bogus skipped`);
+        return ad;
+    }
+    catch (error) {
+        log.warn({ resolver, error: reason(error) }, `resolver ${resolver} unreachable, dns/dnssec-bogus skipped:`);
+        return false;
+    }
+}
+// A resolver over the configured servers, caching every answer for its record TTL.
+export function dnsClient(resolver, bucket, canQueryDirectly, directPort = 53) {
+    const configured = servers(resolver);
+    let validation;
+    const client = {
+        canQueryDirectly,
+        async query(name, type, options = {}) {
+            if (!canQueryDirectly && options.server !== undefined)
+                throw new Error(`direct query to ${options.server} refused`);
+            const targets = options.server === undefined ? configured : [{ address: options.server, port: directPort }];
+            if (targets.length === 0)
+                throw new Error("no resolver configured");
+            const signal = options.signal ?? new AbortController().signal;
+            signal.throwIfAborted();
+            const key = `${targets.map((target) => label(target)).join(",")}\t${name}\t${type}\t${options.checkingDisabled ? "cd" : ""}${options.recursion ? "rd" : ""}`;
+            const entry = await bucket.get(key);
+            if (entry && Date.parse(entry.value.expires) > Date.now())
+                return decode(targets[0] ? label(targets[0]) : "", Buffer.from(entry.value.packet, "base64"));
+            let failure;
+            for (const target of targets) {
+                for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) {
+                    try {
+                        const wire = await exchange(target, name, type, options, signal);
+                        const reply = decode(label(target), wire);
+                        log.debug({ name, type, server: label(target), rcode: reply.rcode, answers: reply.answers.length, ad: reply.ad, attempt }, "dns answered");
+                        await bucket.set(key, { expires: new Date(Date.now() + lifetime(reply, bucket.ttlSeconds) * 1000).toISOString(), packet: wire.toString("base64") });
+                        return reply;
+                    }
+                    catch (error) {
+                        failure = error;
+                        log.debug({ name, type, server: label(target), attempt, error: reason(error) }, "dns query failed");
+                        if (signal.aborted)
+                            throw error;
+                        await sleep(100 * 2 ** attempt + Math.random() * 100);
+                    }
+                }
+            }
+            throw failure instanceof Error ? failure : new Error(String(failure));
+        },
+        // The root zone is signed, so a validating resolver sets `AD` on its SOA; warns once when it does not.
+        validating() {
+            validation ??= isValidating(client, resolver);
+            return validation;
+        },
+    };
+    return client;
+}

package/dist/crawl/fetch.js

@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { setTimeout as sleep } from "node:timers/promises";
+import { agentHeaders } from "../agent.js";
+import { log } from "../logger.js";
+import { PrivateAddress } from "./guard.js";
+import { guardedFetch, pace, patient } from "./network.js";
+const ATTEMPTS = 3;
+const TIMEOUT_MS = 30_000;
+const RETRY_STATUS = new Set([429, 503]);
+// Attempts a final answer of `status` cost; 0 is a network failure.
+export function attemptsFor(status) {
+    return status === 0 || RETRY_STATUS.has(status) ? ATTEMPTS : 1;
+}
+// Exponential backoff with jitter; a Retry-After in seconds wins, capped at the timeout.
+export function delay(attempt, retryAfter) {
+    const seconds = Number(retryAfter ?? NaN);
+    return Number.isFinite(seconds) && seconds >= 0 ? Math.min(seconds * 1000, TIMEOUT_MS) : 500 * 2 ** attempt + Math.random() * 250;
+}
+// The error undici raises for a short body, a duplicate Content-Length, or one beside chunked.
+export const MISMATCH = "Response body length does not match content-length header";
+// The innermost cause of a fetch failure, which names the socket error rather than "fetch failed".
+export function reason(error) {
+    return error instanceof Error ? (error.cause instanceof Error ? error.cause.message : error.message) : String(error);
+}
+// One request with timeout and the spiderlint user agent; network errors, 429 and 503 retry, the last failure or a refused address throws.
+export async function fetchRetrying(url, consume, headers = {}, method = "GET") {
+    for (let attempt = 0;; attempt += 1) {
+        const started = performance.now();
+        const isLast = attempt === ATTEMPTS - 1;
+        try {
+            await pace();
+            const response = await guardedFetch(url, { method, headers: { ...headers, ...agentHeaders() }, signal: AbortSignal.timeout(patient(TIMEOUT_MS)) });
+            const value = await consume(response);
+            log.debug({ url, status: response.status, attempt }, "fetched");
+            if (isLast || !RETRY_STATUS.has(response.status))
+                return { response, value, ms: Math.round(performance.now() - started) };
+            await sleep(delay(attempt, response.headers.get("retry-after") ?? undefined));
+        }
+        catch (error) {
+            log.debug({ url, attempt, error: reason(error) }, "fetch failed");
+            if (isLast || error instanceof PrivateAddress)
+                throw error;
+            await sleep(delay(attempt));
+        }
+    }
+}