spiderlint 0.0.0 → 0.143.0

package/dist/rules/run.js

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+import { scoreOf, weight } from "./score.js";
+import { isPageRule } from "./types.js";
+// Key of the (group, rule) cell the fold reads.
+export function cell(group, rule) {
+    return `${group}\t${rule}`;
+}
+// One subject a rule judged, none when it had no pages or is a hint; it fails on any finding above `info`, at the cost of its worst score.
+function judged(run, found, rule, pages = 1) {
+    if (pages === 0 || rule.meta.severity === "hint")
+        return;
+    run.checks.total += 1;
+    if (found.some((finding) => finding.severity === "error"))
+        run.checks.errored += 1;
+    const failing = found.filter((finding) => finding.severity !== "info" && finding.severity !== "hint");
+    const tally = run.perRule.get(rule.meta.id) ?? { checks: 0, failed: 0, pages: 0 };
+    run.perRule.set(rule.meta.id, { checks: tally.checks + 1, failed: tally.failed + (failing.length > 0 ? 1 : 0), pages: tally.pages + pages });
+    if (failing.length === 0)
+        return;
+    run.checks.failed += 1;
+    const worst = Math.max(...failing.map((finding) => scoreOf(finding)));
+    run.checks.cost = Number((run.checks.cost + weight(worst)).toFixed(4));
+}
+// How the crawl read a page: when, by which crawler, and whether the origin only confirmed a stored copy.
+export function pageEvidence(page) {
+    return { bucket: "pages", key: page.url.href, ...(page.crawl.at && { at: page.crawl.at }), via: page.http.revalidated ? "revalidated" : "network", ...(page.crawl.mode && { mode: page.crawl.mode }) };
+}
+// One page rule over one group; `undefined` results are `when`-skips and do not count.
+function runPageRule(rule, members, group, run, site) {
+    let applicable = 0;
+    for (const page of members) {
+        const found = rule.check(page, site);
+        if (found === undefined)
+            continue;
+        applicable += 1;
+        judged(run, found, rule);
+        const read = pageEvidence(page);
+        run.findings.push(...found.map((finding) => ({ ...finding, evidence: [read, ...(finding.evidence ?? [])] })));
+    }
+    run.applicable.set(cell(group, rule.meta.id), applicable);
+}
+// Page and group rules run within their group; site rules run once over the crawl.
+export function runRules(pages, rulesByGroup, facts) {
+    const run = { findings: [], applicable: new Map(), perRule: new Map(), checks: { total: 0, failed: 0, errored: 0, cost: 0 } };
+    const site = new Map();
+    for (const [group, rules] of rulesByGroup) {
+        const members = pages.filter((page) => page.group === group);
+        for (const rule of rules) {
+            const before = run.findings.length;
+            if (isPageRule(rule))
+                runPageRule(rule, members, group, run, facts);
+            else if (rule.meta.scope === "group") {
+                const found = rule.check(members, group, facts) ?? [];
+                judged(run, found, rule, members.length);
+                run.findings.push(...found);
+            }
+            else if (!site.has(rule.meta.id))
+                site.set(rule.meta.id, rule);
+            log.debug({ rule: rule.meta.id, group, pages: members.length, findings: run.findings.length - before }, "rule ran");
+        }
+    }
+    for (const rule of site.values()) {
+        const found = rule.check(pages, undefined, facts);
+        log.debug({ rule: rule.meta.id, scope: "site", pages: pages.length, findings: found?.length, isSkipped: found === undefined }, "rule ran");
+        if (found === undefined)
+            continue;
+        judged(run, found, rule, pages.length);
+        run.findings.push(...found);
+    }
+    log.debug(run.checks, "checks judged");
+    return run;
+}

package/dist/rules/score.js

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+// Each level’s lowest and highest score in tenths, and the score a level alone stands for.
+const BANDS = {
+    hint: { min: 0, max: 0.9, base: 0.5 },
+    info: { min: 1, max: 3.2, base: 2 },
+    warning: { min: 3.3, max: 6.5, base: 5 },
+    error: { min: 6.6, max: 9.9, base: 8 },
+};
+const LEVELS = ["error", "warning", "info", "hint"];
+// A score on the one-decimal grid, 0.0 to 9.9.
+export function round(score) {
+    return Math.min(9.9, Math.max(0, Math.round(score * 10) / 10));
+}
+// The level whose band holds the score.
+export function levelOf(score) {
+    const tenths = round(score);
+    return LEVELS.find((level) => tenths >= BANDS[level].min) ?? "hint";
+}
+// The score a rule gets for a level alone.
+export function baseScore(level) {
+    return BANDS[level].base;
+}
+// The score moved into a level’s band, so a pinned level never leaves it.
+export function pin(level, score) {
+    return Math.min(BANDS[level].max, Math.max(BANDS[level].min, round(score)));
+}
+// The score between the two scale points around `value`, flat beyond the ends.
+export function interpolate(scale, value) {
+    const points = scale.toSorted(([a], [b]) => a - b);
+    const at = Math.min(Math.max(value, points[0][0]), points.at(-1)[0]);
+    const index = Math.max(1, points.findIndex(([x]) => x >= at));
+    const [x0, y0] = points[index - 1];
+    const [x1, y1] = points[index];
+    return round(y0 + ((y1 - y0) * (at - x0)) / (x1 - x0));
+}
+// A finding’s score, the level’s base when no rule computed one.
+export function scoreOf(finding) {
+    return finding.score ?? BANDS[finding.severity].base;
+}
+// What a failed check costs the rating: 1 at the warning base, rising with the square of the score.
+export function weight(score) {
+    return (score / BANDS.warning.base) ** 2;
+}
+// Pages the finding touches: its folded count or listed URLs, else the whole site for a site-wide one, else one.
+export function pagesOf(finding, total) {
+    return finding.occurrences ?? finding.urls?.length ?? (finding.scope === "site" ? total : 1);
+}
+// Score times the pages the finding touches.
+export function impact(finding, total) {
+    return scoreOf(finding) * pagesOf(finding, total);
+}
+// Score times the share of the site’s pages the finding touches; a site-wide finding touches all of them.
+export function importance(finding, total) {
+    return scoreOf(finding) * (total === 0 ? 1 : Math.min(1, pagesOf(finding, total) / total));
+}
+// Orders findings from the most to the least important, then by rule and URL.
+export function byImportance(total) {
+    return (a, b) => importance(b, total) - importance(a, total) || a.rule.localeCompare(b.rule) || a.url.localeCompare(b.url);
+}

package/dist/rules/types.js

@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+// TS does not narrow a union on a nested discriminant, so the guard is explicit.
+export function isPageRule(rule) {
+    return rule.meta.scope === "page";
+}

package/dist/rules/url.js

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import anyAscii from "any-ascii";
+import { log } from "../logger.js";
+import { said } from "./message.js";
+// A slug that is only an identifier: digits, a hex run with a digit, or a UUID.
+const ID_ONLY = /^(?:\d+|(?=[a-f]*\d)[\da-f]{6,}|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})$/i;
+// A last segment naming a file by its extension.
+const FILE = /\.[a-z\d]{1,5}$/i;
+// Lower-case ASCII word tokens of `text`, in any script, percent-decoded.
+export function tokens(text) {
+    let decoded = text;
+    try {
+        decoded = decodeURIComponent(text);
+    }
+    catch {
+        log.debug({ text }, "slug not percent-decodable");
+    }
+    return new Set(anyAscii(decoded.normalize("NFKC"))
+        .toLowerCase()
+        .split(/[^a-z\d]+/)
+        .filter((token) => token.length > 0));
+}
+// The last non-empty path segment without its extension; empty for the root.
+function slugOf(page) {
+    return (page.url.pathname.split("/").findLast((segment) => segment.length > 0) ?? "").replace(FILE, "");
+}
+// A slug that is an identifier or shares no word with the page’s title or first heading.
+const readableSlug = (severity) => ({
+    meta: { id: "url/readable-slug", severity, scope: "page", facts: ["url.pathname", "html.title", "html.h1"], docs: "https://developers.google.com/search/docs/crawling-indexing/url-structure", fix: "Name the page in its URL with words from its title, as /spring-recipes for “Spring recipes”." },
+    check(page) {
+        const slug = slugOf(page);
+        const heading = [page.html?.title ?? "", ...(page.html?.h1 ?? [])].join(" ");
+        if (!slug || !heading.trim())
+            return;
+        const isId = ID_ONLY.test(slug);
+        const shared = tokens(slug).intersection(tokens(heading)).size;
+        log.debug({ rule: "url/readable-slug", url: page.url.href, slug, isId, shared }, "slug read against the title");
+        if (!isId && shared > 0)
+            return [];
+        const title = `“${page.html?.title ?? page.html?.h1[0]}”`;
+        const sentence = said(isId ? "the slug is an identifier, not words" : "the slug shares no word with the page title");
+        return [{ rule: "url/readable-slug", severity, scope: "page", url: page.url.href, group: page.group, ...sentence, ...(!isId && { data: { [page.url.href]: { title } } }), value: slug }];
+    },
+});
+// A site finding naming the majority form and every page in the minority, when both forms appear.
+function minority(id, severity, text, variables, forms) {
+    const ranked = forms
+        .entries()
+        .toArray()
+        .toSorted(([, a], [, b]) => b.length - a.length);
+    const [[major, most] = ["", []], ...rest] = ranked;
+    const urls = rest.flatMap(([, pages]) => pages.map((page) => page.url.href));
+    log.debug({ rule: id, text, major, most: most.length, minor: urls.length }, "url forms counted");
+    const data = Object.fromEntries(rest.flatMap(([form, pages]) => pages.map((page) => [page.url.href, { form }])));
+    return urls.length === 0 ? undefined : { rule: id, severity, scope: "site", url: urls[0], ...said(text, { ...variables, count: most.length, total: most.length + urls.length, major }), data, value: Object.fromEntries(ranked.map(([form, pages]) => [form, pages.length])), urls };
+}
+// Pages whose words join with the separator fewer pages of the site use.
+const separators = (severity) => ({
+    meta: { id: "url/separators", severity, scope: "site", facts: ["url.pathname"], docs: "https://developers.google.com/search/docs/crawling-indexing/url-structure", fix: "Join the words of every URL with hyphens, and redirect the others to them." },
+    check(pages) {
+        const forms = new Map();
+        for (const page of pages) {
+            const path = page.url.pathname.replace(FILE, "");
+            const form = /\w_\w/.test(path) ? (/\w-\w/.test(path) ? "mix both" : "use _") : /\w-\w/.test(path) ? "use -" : undefined;
+            if (form)
+                forms.set(form, [...(forms.get(form) ?? []), page]);
+        }
+        const finding = minority("url/separators", severity, "word separators differ: {count} of {total} pages {major}; the others differ", {}, forms);
+        return finding ? [finding] : [];
+    },
+});
+// Per group, pages whose trailing slash differs from the group’s usual form; the root and file names are left out.
+const trailingSlash = (severity) => ({
+    meta: { id: "url/trailing-slash", severity, scope: "site", facts: ["url.pathname"], docs: "https://developers.google.com/search/blog/2010/04/to-slash-or-not-to-slash", fix: "Pick one form, with or without the trailing slash, and redirect the other to it." },
+    check(pages) {
+        const judged = pages.filter((page) => page.url.pathname !== "/" && !FILE.test(page.url.pathname));
+        return Map.groupBy(judged, (page) => page.group)
+            .entries()
+            .map(([group, members]) => minority("url/trailing-slash", severity, "trailing slashes differ in group {group}: {count} of {total} pages {major}; the others differ", { group }, Map.groupBy(members, (page) => (page.url.pathname.endsWith("/") ? "end with /" : "end without /"))))
+            .filter((finding) => finding !== undefined)
+            .toArray();
+    },
+});
+export const urlRules = {
+    "url/readable-slug": readableSlug,
+    "url/separators": separators,
+    "url/trailing-slash": trailingSlash,
+};

package/dist/server/analytics.js

@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+const TIMEOUT_MS = 3000;
+const PAUSE_MS = 60_000;
+const NAMES = { "/": "Scan form", "/jobs/:id": "Scan report" };
+// Tracking stays off until this time after a failure, so a down Matomo costs a page one timeout, not every page.
+const pause = { until: 0, failures: 0 };
+// Whether the visitor asked not to be tracked.
+function isRefused(c) {
+    return c.req.header("dnt") === "1" || c.req.header("sec-gpc") === "1";
+}
+// Reports the page `route` the visitor just got, without waiting for Matomo; the badge and the API never come here, and a visitor who sent DNT or Sec-GPC is not reported.
+export function track(matomo, c, route, host) {
+    if (!matomo)
+        return;
+    const name = NAMES[route];
+    if (!name || c.req.method !== "GET" || Date.now() < pause.until || isRefused(c)) {
+        log.debug({ route, method: c.req.method, isRefused: isRefused(c), isPaused: Date.now() < pause.until }, "page view not tracked");
+        return;
+    }
+    const address = new URL("matomo.php", matomo.url);
+    const parameters = {
+        idsite: String(matomo.siteId),
+        rec: "1",
+        apiv: "1",
+        send_image: "0",
+        url: new URL(route, matomo.site).href,
+        action_name: host && matomo.isHostNamed ? `${name}: ${host}` : name,
+        ua: c.req.header("user-agent") ?? "",
+        lang: c.req.header("accept-language") ?? "",
+        rand: String(Math.random()).slice(2),
+    };
+    address.search = new URLSearchParams(parameters).toString();
+    void fetch(address, { signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "manual" })
+        .then((response) => {
+        if (response.status >= 500)
+            throw new Error(`answered ${response.status}`);
+        pause.failures = 0;
+        log.debug({ route, status: response.status }, "page view tracked");
+        return response.status;
+    })
+        .catch((error) => {
+        pause.failures += 1;
+        pause.until = Date.now() + PAUSE_MS * Math.min(pause.failures, 10);
+        log.warn({ route, failures: pause.failures, error: error instanceof Error ? error.message : String(error) }, "page view not tracked; tracking paused");
+    });
+}

package/dist/server/api.js

@@ -0,0 +1,134 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { Hono } from "hono";
+import { bodyLimit } from "hono/body-limit";
+import { routePath } from "hono/route";
+import { secureHeaders } from "hono/secure-headers";
+import { streamSSE } from "hono/streaming";
+import { plain } from "../color.js";
+import { negotiate } from "../i18n.js";
+import { log } from "../logger.js";
+import { formatNames, formatter } from "../plugins/index.js";
+import { BlockList } from "node:net";
+import { Buckets } from "./clients.js";
+import { clientAddress, jobOf, submit, view } from "./jobs.js";
+import { Refusal } from "./policy.js";
+import { edgeRanges } from "./providers.js";
+import { PAGE_CSP, web } from "./web.js";
+import { countRefusal, countRequest, inSpan, withTraceCarrier } from "../telemetry.js";
+export const BODY_MAX = 64 * 1024;
+const EVENTS_MS = 1000;
+const EDGES_MS = 24 * 60 * 60 * 1000;
+// Media type and file extension per bundled format; a plugin’s format is plain text.
+const MEDIA = { json: ["application/json", "json"], sarif: ["application/sarif+json", "sarif"], csv: ["text/csv", "csv"], checkstyle: ["application/xml", "xml"], human: ["text/plain", "txt"], html: ["text/html", "html"] };
+// The refusal as `{ error: { code, message } }`, with Retry-After when it has one.
+export function refused(c, refusal) {
+    countRefusal(refusal.code);
+    if (refusal.retryAfter !== undefined)
+        c.header("retry-after", String(refusal.retryAfter));
+    return c.json({ error: { code: refusal.code, message: refusal.message } }, refusal.status);
+}
+// The routes over one queue; `settings` is read per request, so a reloaded policy applies at once.
+export function api(queue, redis, settings) {
+    const jobs = { queue, redis, settings, buckets: new Buckets(), edges: { list: new BlockList() } };
+    // CDN edge ranges, fetched now and daily, so a new edge is trusted without a restart.
+    const refresh = async () => {
+        const { providers } = settings().clients;
+        if (providers.length > 0)
+            jobs.edges.list = await edgeRanges(providers);
+    };
+    void refresh();
+    setInterval(() => void refresh(), EDGES_MS).unref();
+    const app = new Hono();
+    // One span and one count per request, continuing a trace the caller sends in `traceparent`.
+    app.use(async (c, next) => withTraceCarrier(c.req.header(), async () => inSpan(`${c.req.method} ${c.req.path}`, { "http.request.method": c.req.method, "url.path": c.req.path }, async (span) => {
+        await next();
+        const route = routePath(c, -1);
+        span.updateName(`${c.req.method} ${route}`);
+        span.setAttributes({ "http.route": route, "http.response.status_code": c.res.status });
+        countRequest(route, c.req.method, c.res.status);
+    })));
+    // Registered before the rest so it runs last: a badge may be embedded by any site.
+    app.use("/badge/*", async (c, next) => {
+        await next();
+        c.header("cross-origin-resource-policy", "cross-origin");
+    });
+    app.use(secureHeaders({ xXssProtection: false }));
+    app.use(async (c, next) => {
+        const started = performance.now();
+        await next();
+        log.info({ client: clientAddress(c, jobs), method: c.req.method, path: c.req.path, status: c.res.status, ms: Math.round(performance.now() - started) }, "request served");
+    });
+    app.onError((error, c) => {
+        if (error instanceof Refusal)
+            return refused(c, error);
+        log.error({ client: clientAddress(c, jobs), path: c.req.path, error: error.message }, "request failed");
+        return c.json({ error: { code: "internal", message: "internal error" } }, 500);
+    });
+    app.notFound((c) => {
+        if (c.req.path.startsWith("/v1/") || c.req.path === "/healthz")
+            return refused(c, new Refusal(404, "not-found", `${c.req.path}: no such route`));
+        c.header("content-security-policy", PAGE_CSP);
+        c.header("cache-control", "no-cache");
+        return c.html('<!DOCTYPE html>\n<html lang="en" dir="ltr"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Not found</title></head><body><main><h1>Not found</h1><p><a href="/">Start a new scan</a></p></main></body></html>\n', 404);
+    });
+    app.post("/v1/jobs", bodyLimit({ maxSize: BODY_MAX, onError: (c) => refused(c, new Refusal(400, "invalid-body", `the body exceeds ${BODY_MAX} bytes`)) }), async (c) => {
+        let body;
+        try {
+            body = await c.req.json();
+        }
+        catch {
+            throw new Refusal(400, "invalid-body", "the body is not JSON");
+        }
+        const { job, isRepeat } = await submit(jobs, body, c);
+        c.header("location", `/v1/jobs/${job.id}`);
+        return c.json(await view(job), isRepeat ? 200 : 202);
+    });
+    app.get("/v1/jobs/:id", async (c) => {
+        const job = await jobOf(queue, c.req.param("id"));
+        return c.json(await view(job));
+    });
+    app.get("/v1/jobs/:id/events", async (c) => {
+        const id = c.req.param("id");
+        await jobOf(queue, id);
+        return streamSSE(c, async (stream) => {
+            let last = "";
+            while (!stream.aborted) {
+                const job = (await queue.getJob(id));
+                const current = job ? await view(job) : undefined;
+                const data = JSON.stringify(current ?? { id, status: "expired" });
+                const event = current ? (["done", "failed"].includes(current.status) ? current.status : "progress") : "expired";
+                if (data !== last)
+                    await stream.writeSSE({ event, data });
+                log.debug({ job: id, event, isChanged: data !== last }, "job event polled");
+                last = data;
+                if (event !== "progress")
+                    break;
+                await stream.sleep(EVENTS_MS);
+            }
+        });
+    });
+    app.get("/v1/jobs/:id/report/:format", async (c) => {
+        const job = await jobOf(queue, c.req.param("id"));
+        const name = c.req.param("format");
+        const format = formatter(name);
+        if (!format)
+            throw new Refusal(404, "unknown-format", `${name}: not one of ${formatNames().join(", ")}`);
+        if ((await job.getState()) !== "completed")
+            throw new Refusal(409, "not-ready", `job ${job.id}: no report until it is done`);
+        const [media, extension] = MEDIA[name] ?? ["text/plain", "txt"];
+        const report = { pages: [], site: {}, ...job.returnvalue };
+        c.header("content-type", `${media}; charset=utf-8`);
+        c.header("content-security-policy", PAGE_CSP);
+        c.header("content-disposition", `inline; filename="spiderlint-${job.data.host.replaceAll(/[^\w.-]/g, "-")}.${extension}"`);
+        const lang = negotiate(c.req.header("accept-language"));
+        return c.body(format(report, plain, false, lang));
+    });
+    app.get("/healthz", async (c) => {
+        const [pong, counts] = await Promise.all([redis.ping(), queue.getJobCounts("waiting", "active", "delayed", "completed", "failed")]);
+        return c.json({ status: pong === "PONG" ? "ok" : "degraded", queue: counts });
+    });
+    app.route("/", web(jobs));
+    return app;
+}

package/dist/server/clients.js

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { isIP } from "node:net";
+import { log } from "../logger.js";
+const SWEEP_MS = 60_000;
+// An IPv4-mapped IPv6 address as its IPv4 form.
+function plain(address) {
+    return /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address)?.[1] ?? address;
+}
+function isTrusted(address, trusted) {
+    return isIP(address) !== 0 && trusted.some((list) => list.check(address, isIP(address) === 6 ? "ipv6" : "ipv4"));
+}
+// The peer, or while the hop in hand is a trusted proxy, the X-Forwarded-For entry it added, read right to left.
+export function clientOf(peer, forwarded, trusted) {
+    let client = plain(peer);
+    const hops = (forwarded ?? "")
+        .split(",")
+        .map((entry) => plain(entry.trim()))
+        .toReversed();
+    for (const hop of hops) {
+        if (!isTrusted(client, trusted) || isIP(hop) === 0)
+            break;
+        client = hop;
+    }
+    return client;
+}
+// Token buckets per client address, in memory only, so an address never reaches Redis.
+export class Buckets {
+    #buckets = new Map();
+    #swept = 0;
+    // Forgets buckets full again, at most once a minute.
+    #sweep(jobs, perMs, now) {
+        if (now - this.#swept < SWEEP_MS)
+            return;
+        this.#swept = now;
+        for (const [client, bucket] of this.#buckets)
+            if (bucket.tokens + (now - bucket.at) * perMs >= jobs)
+                this.#buckets.delete(client);
+        log.debug({ clients: this.#buckets.size }, "client buckets swept");
+    }
+    // Takes a token from the bucket of `rate.jobs` refilled evenly over `rate.seconds`; 0 when taken, else the seconds until one is back.
+    take(client, rate, now = Date.now()) {
+        const perMs = rate.jobs / (rate.seconds * 1000);
+        this.#sweep(rate.jobs, perMs, now);
+        const bucket = this.#buckets.get(client);
+        const tokens = bucket ? Math.min(rate.jobs, bucket.tokens + (now - bucket.at) * perMs) : rate.jobs;
+        const isTaken = tokens >= 1;
+        this.#buckets.set(client, { tokens: isTaken ? tokens - 1 : tokens, at: now });
+        const wait = isTaken ? 0 : Math.ceil((1 - tokens) / perMs / 1000);
+        log.debug({ tokens: Math.floor(tokens), jobs: rate.jobs, wait }, "client bucket taken");
+        return wait;
+    }
+}

package/dist/server/clock.js

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { fetchRetrying, reason } from "../crawl/fetch.js";
+import { dateSkew } from "../facts/transport.js";
+import { log } from "../logger.js";
+import { isOwnClock } from "../rules/clock.js";
+// Seconds `url`’s `Date` runs ahead of our clock, or undefined when it does not answer or sends none.
+async function skewOf(url) {
+    try {
+        const { value, ms } = await fetchRetrying(url, async (response) => {
+            const firstByte = Date.now();
+            await response.body?.cancel();
+            return { firstByte, headers: Object.fromEntries(response.headers) };
+        });
+        return dateSkew(url, value.headers, value.firstByte - ms, value.firstByte);
+    }
+    catch (error) {
+        log.debug({ url, error: reason(error) }, "clock reference unreachable");
+        return undefined;
+    }
+}
+// Compares our clock with each reference origin’s `Date` and warns when they all disagree with us alike.
+export async function checkClock(references) {
+    if (references.length === 0)
+        return log.debug({ references }, "clock check disabled");
+    const answers = await Promise.all(references.map(async (url) => skewOf(url)));
+    const skews = answers.filter((skew) => skew !== undefined);
+    if (isOwnClock(skews))
+        log.warn({ offset: -skews[0], references: skews.length }, "the local clock is off; clock skew findings are unreliable");
+    else
+        log.info({ skews, references: references.length }, "local clock checked");
+}

package/dist/server/compress.js

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { Duplex } from "node:stream";
+import { createBrotliCompress, createDeflate, createGzip, createZstdCompress } from "node:zlib";
+import { COMPRESSIBLE_CONTENT_TYPE_REGEX } from "hono/compress";
+import { log } from "../logger.js";
+const THRESHOLD = 1024;
+const NO_TRANSFORM = /(?:^|,)\s*?no-transform\s*?(?:,|$)/i;
+const VARY_ENCODING = /(?:^|,)\s*accept-encoding\s*(?:,|$)/i;
+// Codings negotiated, best first; Hono’s own middleware stops at gzip and deflate.
+const CODERS = { zstd: createZstdCompress, br: createBrotliCompress, gzip: createGzip, deflate: createDeflate };
+const OFFERED = Object.keys(CODERS);
+// The best of `offered` the client’s Accept-Encoding names, ties to the earlier one; undefined when it names none.
+function preferred(header, offered) {
+    if (header === undefined)
+        return undefined;
+    const entries = header.split(",").map((part) => {
+        const [token = "", ...parameters] = part.trim().split(";");
+        const q = Number(parameters
+            .map((parameter) => parameter.trim())
+            .find((parameter) => parameter.startsWith("q="))
+            ?.slice(2) ?? 1);
+        return { token: token.trim().toLowerCase(), q: Number.isFinite(q) ? q : 0 };
+    });
+    const wildcard = entries.find((entry) => entry.token === "*")?.q ?? 0;
+    let [best, bestQ] = [undefined, 0];
+    for (const name of offered) {
+        const q = entries.find((entry) => entry.token === name)?.q ?? wildcard;
+        if (q > bestQ)
+            [best, bestQ] = [name, q];
+    }
+    return best;
+}
+// Answers compressible bodies in the client’s best coding, as Hono’s compress does plus zstd and br.
+export function compress(threshold = THRESHOLD) {
+    return async function compress(context, next) {
+        await next();
+        const headers = context.res.headers;
+        const type = headers.get("content-type");
+        const declared = headers.get("content-length");
+        const length = declared === null ? undefined : Number(declared);
+        const isUntouched = context.req.method === "HEAD" || context.res.status === 206 || headers.has("content-encoding") || headers.has("transfer-encoding") || NO_TRANSFORM.test(headers.get("cache-control") ?? "");
+        if (isUntouched || !type || !COMPRESSIBLE_CONTENT_TYPE_REGEX.test(type) || (length !== undefined && (!Number.isFinite(length) || length < threshold)))
+            return;
+        const coding = preferred(context.req.header("accept-encoding"), OFFERED);
+        if (coding === undefined || !context.res.body)
+            return;
+        log.debug({ coding, path: context.req.path }, "response compressed");
+        const pipe = Duplex.toWeb(CODERS[coding]());
+        context.res = new Response(context.res.body.pipeThrough(pipe), context.res);
+        context.res.headers.delete("content-length");
+        context.res.headers.set("content-encoding", coding);
+        const current = context.res.headers.get("vary");
+        if (current !== "*" && !(current && VARY_ENCODING.test(current)))
+            context.res.headers.set("vary", current ? `${current}, Accept-Encoding` : "Accept-Encoding");
+    };
+}

package/dist/server/health.js

@@ -0,0 +1,14 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { DEFAULT_PATH, readSettings } from "./settings.js";
+// Exits 0 when the API answers /healthz on the address the settings name, 1 otherwise.
+const { listen } = readSettings(process.env.SPIDERLINT_SERVER_CONFIG ?? DEFAULT_PATH);
+const host = ["0.0.0.0", "::"].includes(listen.host) ? "127.0.0.1" : listen.host;
+try {
+    const response = await fetch(`http://${host.includes(":") ? `[${host}]` : host}:${listen.port}/healthz`, { signal: AbortSignal.timeout(2000) });
+    process.exitCode = response.ok ? 0 : 1;
+}
+catch {
+    process.exitCode = 1;
+}