spiderlint 0.0.0 → 0.143.0

package/dist/server/jobs.js

@@ -0,0 +1,82 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { randomUUID } from "node:crypto";
+import { normalizeVia } from "../agent.js";
+import { log } from "../logger.js";
+import { formatNames } from "../plugins/index.js";
+import { Buckets, clientOf } from "./clients.js";
+import { admit, Refusal, resolveRules } from "./policy.js";
+import { charge, repeatKey } from "./queue.js";
+import { traceCarrier } from "../telemetry.js";
+// BullMQ states as the API names them; anything else is still waiting its turn.
+const STATUS = { active: "running", completed: "done", failed: "failed" };
+function iso(ms) {
+    return ms ? new Date(ms).toISOString() : undefined;
+}
+// A job as clients see it, with relative links so no Host header is trusted.
+export async function view(job) {
+    const status = STATUS[await job.getState()] ?? "queued";
+    const self = `/v1/jobs/${job.id}`;
+    return {
+        id: job.id,
+        url: job.data.url,
+        policy: job.data.policy,
+        status,
+        ...(typeof job.progress === "object" && { progress: job.progress }),
+        created: iso(job.timestamp),
+        started: iso(job.processedOn),
+        finished: iso(job.finishedOn),
+        ...(status === "failed" && { error: job.failedReason }),
+        ...(status === "done" && { summary: job.returnvalue.summary }),
+        links: { self, page: `/jobs/${job.id}`, events: `${self}/events`, ...(status === "done" && { reports: Object.fromEntries(formatNames().map((name) => [name, `${self}/report/${name}`])) }) },
+    };
+}
+// The job the path names, or a 404.
+export async function jobOf(queue, id) {
+    const job = (await queue.getJob(id));
+    if (!job)
+        throw new Refusal(404, "not-found", `job ${id}: not found or expired`);
+    return job;
+}
+// The job a repeat of this scan returns: queued, running or done inside the policy’s window.
+async function repeated(jobs, key) {
+    const id = await jobs.redis.get(key);
+    const job = id ? (await jobs.queue.getJob(id)) : undefined;
+    const state = await job?.getState();
+    log.debug({ job: id, state }, "repeat looked up");
+    return state && state !== "failed" ? job : undefined;
+}
+// The address a request came from, through the proxies and CDN edges the settings trust.
+export function clientAddress(c, jobs) {
+    const peer = c.env?.incoming?.socket.remoteAddress ?? "";
+    return clientOf(peer, c.req.header("x-forwarded-for"), [jobs.settings().clients.trusted, jobs.edges.list]);
+}
+// Admits `body` and queues its scan, or returns the job it repeats; every refusal comes before the host’s window is charged.
+export async function submit(jobs, body, c) {
+    const current = jobs.settings();
+    const admitted = admit(body, current);
+    await resolveRules(admitted);
+    const key = repeatKey(admitted);
+    const repeat = admitted.policy.repeat ? await repeated(jobs, key) : undefined;
+    if (repeat) {
+        log.info({ job: repeat.id, host: admitted.host, policy: admitted.policy.name }, "scan repeated");
+        return { job: repeat, isRepeat: true };
+    }
+    const waiting = await jobs.queue.getWaitingCount();
+    log.debug({ waiting, maxQueued: current.maxQueued }, "queue depth checked");
+    if (waiting >= current.maxQueued)
+        throw new Refusal(503, "queue-full", `${waiting} scans are waiting; try again later`, 60);
+    const { rate } = current.clients;
+    const wait = rate ? jobs.buckets.take(clientAddress(c, jobs), rate) : 0;
+    if (rate && wait > 0)
+        throw new Refusal(429, "client-rate-limited", `a client may queue ${rate.jobs} scans per ${rate.seconds} s`, wait);
+    await charge(jobs.redis, admitted);
+    const via = normalizeVia(c.req.header("host"));
+    const data = { url: admitted.url, host: admitted.host, policy: admitted.policy.name, settings: admitted.settings, scanTimeout: admitted.policy.scanTimeout, deny: admitted.policy.rules.deny, via, ...(admitted.policy.repeat && { repeatKey: key }), trace: traceCarrier() };
+    const job = (await jobs.queue.add("scan", data, { jobId: randomUUID() }));
+    if (admitted.policy.repeat)
+        await jobs.redis.set(key, job.id, "EX", admitted.policy.repeat);
+    log.info({ job: job.id, host: admitted.host, policy: admitted.policy.name, repeat: admitted.policy.repeat }, "scan queued");
+    return { job, isRepeat: false };
+}

package/dist/server/main.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env -S node --experimental-strip-types
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { readFileSync } from "node:fs";
+import { serve } from "@hono/node-server";
+import { ConfigError } from "../config/index.js";
+import { environmentSettings } from "../config/environment.js";
+import { prepareBrowser } from "../crawl/browser.js";
+import { log } from "../logger.js";
+import { api } from "./api.js";
+import { checkClock } from "./clock.js";
+import { connect, scanQueue } from "./queue.js";
+import { DEFAULT_PATH, watchSettings } from "./settings.js";
+import { startWorker } from "./worker.js";
+import { observeQueue, startTelemetry } from "../telemetry.js";
+const MODES = ["api", "worker", "all"];
+// Downloads Chromium into the cache once, before the worker takes a job; a failure leaves browser scans failing, never the worker.
+async function prepareWorkerBrowser() {
+    const { browserInstall = true } = environmentSettings(process.env);
+    if (!browserInstall) {
+        log.info({ browserInstall }, "browser not prepared");
+        return;
+    }
+    try {
+        await prepareBrowser("chromium");
+        log.info({ browser: "chromium" }, "browser ready");
+    }
+    catch (error) {
+        log.warn({ browser: "chromium", error: error instanceof Error ? error.message : String(error) }, "browser not ready, browser scans will fail");
+    }
+}
+// Starts the API, the worker or both, per SPIDERLINT_MODE, until SIGTERM or SIGINT.
+async function main() {
+    const mode = process.env.SPIDERLINT_MODE ?? "all";
+    if (!MODES.includes(mode))
+        throw new ConfigError(`SPIDERLINT_MODE: invalid value ${mode} (expected: ${MODES.join("|")})`);
+    const path = process.env.SPIDERLINT_SERVER_CONFIG ?? DEFAULT_PATH;
+    const settings = watchSettings(path);
+    const { redis: url, redisPasswordFile, listen, retention, workers, clockReferences } = settings.current();
+    const password = redisPasswordFile ? readFileSync(redisPasswordFile, "utf8").trim() : undefined;
+    log.info({ mode, path, listen, workers, redisPasswordFile }, "server starting");
+    const telemetry = await startTelemetry("spiderlint");
+    void checkClock(clockReferences);
+    const closers = [];
+    if (mode !== "worker") {
+        const redis = connect(url, password);
+        const queue = scanQueue(redis, retention);
+        observeQueue(async () => queue.getJobCounts("waiting", "active", "delayed", "prioritized"));
+        const server = serve({ fetch: api(queue, redis, settings.current).fetch, hostname: listen.host, port: listen.port }, (info) => log.info({ address: info.address, port: info.port }, "api listening"));
+        closers.push(async () => new Promise((resolve) => server.close(resolve)), async () => queue.close(), async () => redis.quit());
+    }
+    if (mode !== "api") {
+        await prepareWorkerBrowser();
+        const redis = connect(url, password);
+        const worker = startWorker(redis, workers, retention);
+        closers.push(async () => worker.close(), async () => redis.quit());
+    }
+    if (telemetry)
+        closers.push(async () => telemetry.shutdown());
+    const stop = async (signal) => {
+        log.info({ signal, closers: closers.length }, "server stopping");
+        settings.close();
+        for (const close of closers) {
+            try {
+                await close();
+            }
+            catch (error) {
+                log.warn({ error: String(error) }, "close failed");
+            }
+        }
+        log.info({ signal }, "server stopped");
+    };
+    for (const signal of ["SIGTERM", "SIGINT"])
+        process.once(signal, (name) => void stop(name));
+}
+try {
+    await main();
+}
+catch (error) {
+    log.fatal({ error: error instanceof Error ? error.message : String(error) }, "server not started");
+    process.exitCode = error instanceof ConfigError ? 2 : 4;
+}

package/dist/server/page.js

@@ -0,0 +1,161 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import { load } from "cheerio";
+import { ConfigError } from "../config/index.js";
+import { LANGUAGES, translator } from "../i18n.js";
+import { log } from "../logger.js";
+const SLOTS = ["head", "header", "footer"];
+const FILE = /^(head|header|footer)(?:\.([a-z]{2,3}))?\.html$/;
+const RELOAD_MS = 5000;
+const MEDIA = {
+    js: "text/javascript",
+    mjs: "text/javascript",
+    css: "text/css",
+    json: "application/json",
+    map: "application/json",
+    txt: "text/plain; charset=utf-8",
+    svg: "image/svg+xml",
+    png: "image/png",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    gif: "image/gif",
+    webp: "image/webp",
+    avif: "image/avif",
+    ico: "image/x-icon",
+    woff2: "font/woff2",
+    woff: "font/woff",
+};
+// The CSP hash source of `text`.
+function hashOf(text) {
+    return `'sha256-${createHash("sha256").update(text).digest("base64")}'`;
+}
+// What a fragment’s scripts and styles ask of the policy: inline ones by hash, external ones by origin; an external one without `integrity` or off https is a ConfigError naming the file.
+function sourcesOf(file, html) {
+    const found = { script: new Set(), style: new Set() };
+    const $ = load(html, undefined, false);
+    for (const element of $("script, style, link[rel~=stylesheet]")) {
+        const name = element.tagName;
+        const kind = name === "script" ? "script" : "style";
+        const attribute = name === "script" ? "src" : "href";
+        const reference = $(element).attr(attribute);
+        if (name === "style" || (name === "script" && reference === undefined)) {
+            found[kind].add(hashOf($(element).html() ?? ""));
+            continue;
+        }
+        if (reference === undefined)
+            continue;
+        if (!$(element).attr("integrity"))
+            throw new ConfigError(`page/${file}: <${name} ${attribute}="${reference}"> needs an integrity attribute`);
+        const isLocal = reference.startsWith("/") && !reference.startsWith("//");
+        if (!isLocal && !(URL.canParse(reference) && new URL(reference).protocol === "https:"))
+            throw new ConfigError(`page/${file}: ${reference} must be an https URL or a path on this server`);
+        found[kind].add(isLocal ? "'self'" : new URL(reference).origin);
+    }
+    return { script: [...found.script], style: [...found.style] };
+}
+// The page fragments of one directory — `head`, `header` and `footer`, each `<slot>.html` or `<slot>.<lang>.html` — rendered for every language when read, so a bad one fails at once.
+export class Fragments {
+    rendered = new Map();
+    signature = "";
+    checked = 0;
+    directory;
+    constructor(directory) {
+        this.directory = directory;
+        this.read();
+    }
+    // The files of the directory with their modification times.
+    scan() {
+        return readdirSync(this.directory)
+            .filter((file) => FILE.test(file))
+            .toSorted((a, b) => a.localeCompare(b))
+            .map((file) => `${file}:${statSync(path.join(this.directory, file)).mtimeMs}`)
+            .join("\n");
+    }
+    // Reads every fragment and renders it for every language; throws ConfigError on a bad one.
+    read() {
+        let files;
+        try {
+            files = readdirSync(this.directory)
+                .filter((file) => FILE.test(file))
+                .toSorted((a, b) => a.localeCompare(b));
+        }
+        catch (error) {
+            throw new ConfigError(`page/directory: ${this.directory} cannot be read (${error instanceof Error ? error.message : String(error)})`);
+        }
+        const texts = new Map(files.map((file) => [file, readFileSync(path.join(this.directory, file), "utf8")]));
+        const rendered = new Map();
+        for (const lang of LANGUAGES) {
+            for (const slot of SLOTS) {
+                const file = texts.has(`${slot}.${lang}.html`) ? `${slot}.${lang}.html` : `${slot}.html`;
+                const text = texts.get(file);
+                if (text === undefined)
+                    continue;
+                const html = text.replaceAll("{lang}", () => lang).replaceAll("{dir}", () => translator(lang).dir);
+                const sources = sourcesOf(file, html);
+                rendered.set(`${slot}.${lang}`, { html, sources });
+                log.debug({ file, lang, scripts: sources.script, styles: sources.style }, "page fragment rendered");
+            }
+        }
+        this.rendered = rendered;
+        this.signature = this.scan();
+        log.info({ directory: this.directory, files, hashes: rendered.values().reduce((sum, entry) => sum + entry.sources.script.length + entry.sources.style.length, 0) }, "page fragments loaded");
+    }
+    // Rereads the files when one changed, at most every few seconds; a bad edit is logged and the last good fragments stay.
+    refresh() {
+        const now = Date.now();
+        if (now - this.checked < RELOAD_MS)
+            return;
+        this.checked = now;
+        try {
+            const current = this.scan();
+            if (current === this.signature)
+                return;
+            try {
+                this.read();
+            }
+            catch (error) {
+                log.error({ directory: this.directory, error: error instanceof Error ? error.message : String(error) }, "page fragments rejected; previous kept");
+                this.signature = current;
+            }
+        }
+        catch (error) {
+            log.error({ directory: this.directory, error: error instanceof Error ? error.message : String(error) }, "page fragments unreadable; previous kept");
+        }
+    }
+    // The fragment for `slot` in `lang`, `{lang}` and `{dir}` filled, and what it adds to the policy.
+    slot(slot, lang) {
+        this.refresh();
+        return this.rendered.get(`${slot}.${lang}`) ?? { html: "", sources: { script: [], style: [] } };
+    }
+}
+// The directory a settings file names, or a ConfigError.
+export function directoryOf(key, value) {
+    try {
+        if (statSync(value).isDirectory())
+            return value;
+    }
+    catch {
+        // falls through to the error below
+    }
+    throw new ConfigError(`${key}: ${value} is not a directory`);
+}
+// The file `relative` names under `root` and its media type; undefined for anything else, a directory, a dotfile or a path out of the root included.
+export async function asset(root, relative) {
+    const segments = relative.split("/").filter(Boolean);
+    if (segments.length === 0 || segments.some((segment) => segment.startsWith(".") || segment.includes("\0") || segment.includes("\\")))
+        return undefined;
+    try {
+        const base = await realpath(root);
+        const file = await realpath(path.join(base, ...segments));
+        const found = await stat(file);
+        return !file.startsWith(`${base}${path.sep}`) || !found.isFile() ? undefined : { body: await readFile(file), type: MEDIA[path.extname(file).slice(1).toLowerCase()] ?? "application/octet-stream" };
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/server/policy.js

@@ -0,0 +1,145 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import picomatch from "picomatch";
+import { ConfigError } from "../config/index.js";
+import { fromSubtree } from "../config/policy.js";
+import { validateSubtree } from "../config/schema.js";
+import { validateRules } from "../index.js";
+import { log } from "../logger.js";
+import { CAPPED } from "./settings.js";
+// `org.spiderlint` keys a request may set; the rest read files, load code, leave the address guard or skip robots.txt.
+const REQUEST_KEYS = new Set(["canonical-origin", "rules", "exclude-rules", "override", "groups", "fetch", "browser", "scope", "concurrency", "rate", "timeout", "max-pages", "max-depth", "max-body-size", "keepalive", "include-urls", "exclude-urls", "resources", "links", "sitemap", "fold"]);
+// Rules whose extractors connect to ports a stranger’s URL must never aim the server at.
+export const NEVER_SERVED = ["sshfp", "sshfp/*"];
+// What the form lets a visitor pick, default first, and the settings each lowers beside its rules; the policy still clamps after, so none raises a cap.
+export const WEB_PRESETS = {
+    recommended: {},
+    "web-quick": { "max-pages": 25, resources: { fetch: false } },
+    "web-comprehensive": {},
+};
+// A request the server turns down, with a stable code a client can translate.
+export class Refusal extends Error {
+    status;
+    code;
+    retryAfter;
+    constructor(status, code, message, retryAfter) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.retryAfter = retryAfter;
+    }
+}
+// The http or https seed, without credentials, and its lowercase host without a trailing dot.
+export function seedOf(raw) {
+    const url = typeof raw === "string" && URL.canParse(raw) ? new URL(raw) : undefined;
+    if (!url || !["http:", "https:"].includes(url.protocol))
+        throw new Refusal(400, "invalid-url", "url: expected an absolute http or https URL");
+    if (url.username || url.password)
+        throw new Refusal(400, "invalid-url", "url: credentials are not accepted");
+    return { url: url.href, host: url.hostname.replace(/\.$/, "") };
+}
+// The first policy with a suffix `host` equals or ends in.
+export function policyFor(host, policies) {
+    const bare = host.replaceAll(/^\[|\]$/g, "");
+    return policies.find((policy) => policy.hosts.some((suffix) => suffix === "*" || bare === suffix || bare.endsWith(`.${suffix}`)));
+}
+// `raw` under the cap, where 0 or unset is unlimited and so takes the cap.
+function capped(raw, cap) {
+    if (cap === undefined)
+        return raw;
+    return raw === undefined || raw === 0 || raw > cap ? cap : raw;
+}
+// Every ruleset or rule ID the scan names, `recommended` when none is named.
+function namedRules(settings) {
+    // The implicit `default` group is always there, as `groupsOf` adds it.
+    const groups = Object.values({ default: {}, ...settings.groups });
+    const named = [...(settings.rules ?? []), ...groups.flatMap((group) => group.rules ?? [])];
+    const isImplicit = settings.rules === undefined && groups.some((group) => group.rules === undefined);
+    return [...new Set([...named, ...(isImplicit ? ["recommended"] : [])])];
+}
+// Each named rule must match an `allow` entry and no `deny` entry.
+function checkRules(settings, policy) {
+    const isDenied = picomatch([...policy.rules.deny, ...NEVER_SERVED]);
+    const isAllowed = policy.rules.allow ? picomatch(policy.rules.allow) : () => true;
+    for (const id of namedRules(settings)) {
+        log.debug({ id, policy: policy.name }, "requested rule checked");
+        if (isDenied(id) || !isAllowed(id))
+            throw new Refusal(403, "forbidden-rule", `rules: ${id} is not allowed by policy ${policy.name}`);
+    }
+}
+// The request settings of the web preset `name`, or a refusal when the form never offered it.
+export function presetSettings(name) {
+    if (!Object.hasOwn(WEB_PRESETS, name))
+        throw new Refusal(400, "unknown-rule", `preset: ${name} is not a web preset`);
+    log.debug({ preset: name }, "web preset chosen");
+    return { rules: [name], ...WEB_PRESETS[name] };
+}
+// The web presets some admitting policy lets a request name, in the form’s order; the host typed picks the policy, so each stays checked on submit.
+export function presetsOffered(server) {
+    const policies = server.policies.filter((policy) => policy.ban === undefined);
+    const offered = Object.keys(WEB_PRESETS).filter((name) => policies.some((policy) => {
+        try {
+            checkRules({ rules: [name] }, policy);
+            return true;
+        }
+        catch (error) {
+            if (error instanceof Refusal)
+                return false;
+            throw error;
+        }
+    }));
+    log.debug({ offered, policies: policies.length }, "web presets offered");
+    return offered;
+}
+// Each fetch mode the scan names must be allowed; unnamed, it takes the policy’s first.
+function checkFetch(settings, policy) {
+    const groups = Object.values((settings.groups ?? {}));
+    const named = [settings.fetch, ...groups.map((group) => group.fetch)].filter((mode) => mode !== undefined);
+    const refused = named.find((mode) => !policy.fetch.includes(mode));
+    if (refused)
+        throw new Refusal(403, "forbidden-fetch", `fetch: ${refused} is not allowed by policy ${policy.name} (allowed: ${policy.fetch.join(", ")})`);
+    return settings.fetch === undefined ? { ...settings, fetch: policy.fetch.includes("auto") ? "auto" : policy.fetch[0] } : settings;
+}
+// The request as the scan will run it, or the Refusal naming why it may not run.
+export function admit(body, server) {
+    const { url: rawUrl, settings: requested = {} } = (body ?? {});
+    const { url, host } = seedOf(rawUrl);
+    const policy = policyFor(host, server.policies);
+    log.info({ host, policy: policy?.name, isBanned: policy?.ban !== undefined }, "policy matched");
+    if (!policy)
+        throw new Refusal(403, "no-policy", `${host}: no policy admits this host`);
+    if (policy.ban)
+        throw new Refusal(403, "banned", policy.ban);
+    if (typeof requested !== "object" || requested === null || Array.isArray(requested))
+        throw new Refusal(400, "invalid-settings", "settings: expected an object");
+    const forbidden = Object.keys(requested).filter((key) => !REQUEST_KEYS.has(key));
+    if (forbidden.length > 0)
+        throw new Refusal(400, "forbidden-setting", `settings: ${forbidden.join(", ")} cannot be set through the server`);
+    let settings;
+    try {
+        settings = validateSubtree({ ...server.defaults, ...requested }, "settings");
+    }
+    catch (error) {
+        if (error instanceof ConfigError)
+            throw new Refusal(400, "invalid-settings", error.message);
+        throw error;
+    }
+    checkRules(settings, policy);
+    settings = checkFetch(settings, policy);
+    const caps = Object.fromEntries(CAPPED.map((key) => [key, capped(settings[key], policy.caps[key])]).filter(([, value]) => value !== undefined));
+    log.debug({ host, policy: policy.name, caps, denied: policy.rules.deny.length }, "request clamped");
+    return { url, host, policy, settings: { ...settings, ...caps, robots: true, "allow-private": server.allowPrivate, "browser-install": false } };
+}
+// Refuses a scan naming a ruleset or rule no configured plugin defines, before any window is charged.
+export async function resolveRules(admitted) {
+    try {
+        await validateRules({ ...fromSubtree(admitted.settings), seeds: [admitted.url] });
+    }
+    catch (error) {
+        if (error instanceof ConfigError)
+            throw new Refusal(400, "unknown-rule", error.message);
+        throw error;
+    }
+    log.debug({ host: admitted.host }, "requested rules resolved");
+}

package/dist/server/providers.js

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { BlockList, isIP } from "node:net";
+import { setTimeout as sleep } from "node:timers/promises";
+import { agentHeaders } from "../agent.js";
+import { log } from "../logger.js";
+const TIMEOUT_MS = 15_000;
+const ATTEMPTS = 3;
+const CIDR = /[\d.:a-f]+\/\d{1,3}/gi;
+// Documents naming each CDN's own edge space, as o9s/traefik reads them.
+export const PROVIDERS = {
+    cloudflare: ["https://www.cloudflare.com/ips-v4", "https://www.cloudflare.com/ips-v6"],
+    akamai: ["https://techdocs.akamai.com/property-manager/pdfs/akamai_ipv4_CIDRs.txt", "https://techdocs.akamai.com/property-manager/pdfs/akamai_ipv6_CIDRs.txt"],
+    fastly: ["https://api.fastly.com/public-ip-list"],
+};
+// One document's text, retried with backoff and jitter; the last failure throws.
+async function fetchText(url) {
+    for (let attempt = 0;; attempt += 1) {
+        try {
+            const response = await fetch(url, { headers: { ...agentHeaders() }, signal: AbortSignal.timeout(TIMEOUT_MS) });
+            if (!response.ok)
+                throw new Error(`${url} answers ${response.status}`);
+            return await response.text();
+        }
+        catch (error) {
+            log.debug({ url, attempt, error: String(error) }, "edge range fetch failed");
+            if (attempt === ATTEMPTS - 1)
+                throw error;
+            await sleep(500 * 2 ** attempt + Math.random() * 250);
+        }
+    }
+}
+// The CIDR-shaped entries of `text` added to `list`; how many were valid.
+export function addRanges(list, text) {
+    let added = 0;
+    for (const [range] of text.matchAll(CIDR)) {
+        const [address = "", prefix = ""] = range.split("/", 2);
+        const family = isIP(address);
+        if (family === 0 || Number(prefix) > (family === 6 ? 128 : 32))
+            continue;
+        list.addSubnet(address, Number(prefix), family === 6 ? "ipv6" : "ipv4");
+        added += 1;
+    }
+    return added;
+}
+// The live edge ranges of `providers`; a provider whose documents fail is skipped with a warning.
+export async function edgeRanges(providers) {
+    const list = new BlockList();
+    for (const provider of providers) {
+        try {
+            const texts = await Promise.all((PROVIDERS[provider] ?? []).map(async (url) => fetchText(url)));
+            log.info({ provider, ranges: addRanges(list, texts.join("\n")) }, "edge ranges trusted");
+        }
+        catch (error) {
+            log.warn({ provider, error: String(error) }, "edge ranges not fetched; keeping the other sources");
+        }
+    }
+    return list;
+}

package/dist/server/queue.js

@@ -0,0 +1,50 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash } from "node:crypto";
+import { Queue } from "bullmq";
+import { Redis } from "ioredis";
+import { log } from "../logger.js";
+import { Refusal } from "./policy.js";
+export const QUEUE = "spiderlint-scans";
+export const PREFIX = "spiderlint";
+// The descriptor a scan runner writes progress to, one JSON line per change.
+export const PROGRESS_FD = 3;
+// One connection per role; BullMQ needs `maxRetriesPerRequest: null` for blocking commands, and retries with backoff itself.
+export function connect(url, password) {
+    // eslint-disable-next-line unicorn/no-null -- BullMQ requires null, not undefined
+    const redis = new Redis(url, { ...(password && { password }), maxRetriesPerRequest: null, retryStrategy: (attempt) => Math.min(250 * 2 ** attempt, 10_000) + Math.random() * 250 });
+    redis.on("error", (error) => log.warn({ error: error.message }, "redis connection error"));
+    return redis;
+}
+// The scan queue, jobs kept `retention` seconds after they settle and never retried.
+export function scanQueue(redis, retention) {
+    return new Queue(QUEUE, { connection: redis, prefix: PREFIX, defaultJobOptions: { attempts: 1, removeOnComplete: { age: retention }, removeOnFail: { age: retention } } });
+}
+// Counts one job against the policy’s window for the host; a full window refuses with the seconds until it opens.
+export async function charge(redis, admitted) {
+    const { rate } = admitted.policy;
+    if (!rate)
+        return;
+    const key = `${PREFIX}:rate:${admitted.policy.name}:${admitted.host}`;
+    const reply = await redis
+        .multi()
+        .incr(key)
+        .pexpire(key, rate.seconds * 1000, "NX")
+        .pttl(key)
+        .exec();
+    const used = Number(reply?.[0]?.[1] ?? 0);
+    const left = Math.ceil(Number(reply?.[2]?.[1] ?? 0) / 1000);
+    log.info({ host: admitted.host, policy: admitted.policy.name, used, jobs: rate.jobs, left }, "rate window charged");
+    if (used > rate.jobs)
+        throw new Refusal(429, "rate-limited", `${admitted.host}: ${rate.jobs} scans per ${rate.seconds} s under policy ${admitted.policy.name}`, Math.max(1, left));
+}
+// The key a repeat of this scan finds its job under: the policy, the seed and the settings with keys sorted.
+export function repeatKey(admitted) {
+    const sorted = JSON.stringify(admitted.settings, (_key, value) => (value && typeof value === "object" && !Array.isArray(value) ? Object.fromEntries(Object.entries(value).toSorted(([a], [b]) => a.localeCompare(b))) : value));
+    return `${PREFIX}:repeat:${admitted.policy.name}:${createHash("sha256").update(`${admitted.url}\n${sorted}`).digest("hex")}`;
+}
+// The key naming the last finished scan of `host`, which the badge reads.
+export function latestKey(host) {
+    return `${PREFIX}:latest:${host}`;
+}

package/dist/server/scan.js

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { writeSync } from "node:fs";
+import { text } from "node:stream/consumers";
+import { getCACertificates, setDefaultCACertificates } from "node:tls";
+import { ConfigError } from "../config/index.js";
+import { fromSubtree } from "../config/policy.js";
+import { validateSubtree } from "../config/schema.js";
+import { audit } from "../index.js";
+import { log } from "../logger.js";
+import { onProgress } from "../progress.js";
+import { formatJson } from "../report/json.js";
+import { inSpan, startTelemetry, withTraceCarrier } from "../telemetry.js";
+import { NEVER_SERVED } from "./policy.js";
+import { PROGRESS_FD } from "./queue.js";
+// Exit codes: 0 the report is on stdout, 2 the settings are invalid, 3 no page was fetched, 4 the run failed.
+async function main() {
+    const { url, settings, deny, via, crawlDeadline, trace } = JSON.parse(await text(process.stdin));
+    onProgress((progress) => writeSync(PROGRESS_FD, `${JSON.stringify(progress)}\n`));
+    const telemetry = await startTelemetry("spiderlint");
+    try {
+        const report = await withTraceCarrier(trace, async () => inSpan("scan", { "url.full": url }, async () => audit({ ...fromSubtree(validateSubtree(settings, "settings")), ...(via && { via }), seeds: [url], crawlDeadline, cacheMode: "off", denyRules: [...deny, ...NEVER_SERVED] })));
+        log.info({ url, pages: report.pages.length, findings: report.findings.length }, "scan finished");
+        if (report.pages.length === 0)
+            return 3;
+        process.stdout.write(formatJson(report));
+        return 0;
+    }
+    catch (error) {
+        log.error({ url, error: error instanceof Error ? error.message : String(error) }, "scan aborted");
+        return error instanceof ConfigError ? 2 : 4;
+    }
+    finally {
+        await telemetry?.shutdown();
+    }
+}
+setDefaultCACertificates([...getCACertificates("default"), ...getCACertificates("system")]);
+process.exitCode = await main();