spiderlint 0.0.0 → 0.143.0

package/dist/rules/i18n.js

@@ -0,0 +1,131 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+import { resolve } from "./builtin.js";
+import { said } from "./message.js";
+// The fragment-free hreflang targets of a page other than itself, `x-default` included.
+function alternatesOf(page) {
+    const here = selves(page);
+    return [...new Set((page.html?.hreflang ?? []).map((alternate) => resolve(alternate.href, page.url.href)))].filter((href) => !here.has(href));
+}
+// Every URL a page answers to: its own, the one it was requested as and its twin on the canonical origin.
+function selves(page) {
+    return new Set([page.url.href, page.crawl.requested, page.url.twin].flatMap((href) => (href ? [resolve(href, href)] : [])));
+}
+// Whether a page answered 2xx.
+function isOk(page) {
+    return page.http.status >= 200 && page.http.status < 300;
+}
+// The crawled 2xx page behind a URL, through its twin and through a redirect.
+function pageIndex(pages, site) {
+    const index = new Map();
+    for (const page of pages)
+        if (isOk(page))
+            for (const href of selves(page))
+                index.set(href, page);
+    return (href) => index.get(href) ?? index.get(site?.redirects?.[href] ?? "");
+}
+// The primary language subtag, lower-cased.
+function primary(tag) {
+    return tag.trim().split("-", 1)[0]?.toLowerCase() ?? "";
+}
+// Every crawled 2xx alternate that does not name back the 2xx pages listing it, once per alternate.
+const hreflangReciprocal = (severity) => ({
+    meta: { id: "i18n/hreflang-reciprocal", severity, scope: "site", facts: ["html.hreflang", "site.redirects"], docs: "https://developers.google.com/search/docs/specialty/international/localized-versions#html", fix: "List every language version, itself included, in the hreflang links of each version." },
+    check(pages, _group, site) {
+        const find = pageIndex(pages, site);
+        const unanswered = new Map();
+        for (const page of pages) {
+            if (!isOk(page))
+                continue;
+            for (const href of alternatesOf(page)) {
+                const alternate = find(href);
+                const isNamedBack = alternate === undefined || alternatesOf(alternate).some((back) => selves(page).has(back));
+                log.debug({ rule: "i18n/hreflang-reciprocal", url: page.url.href, alternate: href, isCrawled: alternate !== undefined, isNamedBack }, "hreflang alternate judged");
+                if (!isNamedBack)
+                    unanswered.set(alternate.url.href, [...(unanswered.get(alternate.url.href) ?? []), page.url.href]);
+            }
+        }
+        return unanswered
+            .entries()
+            .map(([url, urls]) => ({ rule: "i18n/hreflang-reciprocal", severity, scope: "site", url, ...said("the page does not name back the pages listing it as an hreflang alternate"), value: urls, urls }))
+            .toArray();
+    },
+});
+// Every crawled hreflang target answering outside 2xx or redirecting, once per target, with the pages naming it.
+const hreflangStatus = (severity) => ({
+    meta: { id: "i18n/hreflang-status", severity, scope: "site", facts: ["html.hreflang", "http.status", "site.redirects"], docs: "https://developers.google.com/search/docs/specialty/international/localized-versions#html", fix: "Point each hreflang link at the final URL of a page that answers 200." },
+    check(pages, _group, site) {
+        const status = new Map(pages.flatMap((page) => [...selves(page)].map((href) => [href, page.http.status])));
+        const naming = new Map();
+        for (const page of pages)
+            for (const href of alternatesOf(page))
+                naming.set(href, [...(naming.get(href) ?? []), page.url.href]);
+        const findings = [];
+        for (const [url, urls] of naming) {
+            const [answer, landing] = [status.get(url), site?.redirects?.[url]];
+            log.debug({ rule: "i18n/hreflang-status", url, status: answer, landing, pages: urls.length }, "hreflang target judged");
+            const verdict = landing ? { ...said("the hreflang target redirects; named by these pages"), data: { [url]: { landing } } } : answer !== undefined && (answer < 200 || answer > 299) ? said("the hreflang target answers {status}; named by these pages", { status: String(answer) }) : undefined;
+            if (verdict)
+                findings.push({ rule: "i18n/hreflang-status", severity, scope: "site", url, ...verdict, value: landing ?? answer, urls });
+        }
+        return findings;
+    },
+});
+// A page whose `lang` names a language its `Content-Language` header leaves out; a page lacking either is skipped.
+const contentLanguage = (severity) => ({
+    meta: { id: "i18n/content-language", severity, scope: "page", facts: ["html.lang", "http.headers.content-language"], docs: "https://www.w3.org/International/questions/qa-http-and-lang", fix: "Make the Content-Language header name the language the html lang attribute declares, or drop the header." },
+    check(page) {
+        const [lang, header] = [page.html?.lang, [page.http.headers["content-language"] ?? []].flat().join(",")];
+        if (!lang || header === "")
+            return;
+        const languages = header.split(",").map((tag) => primary(tag));
+        const isListed = languages.includes(primary(lang));
+        log.debug({ rule: "i18n/content-language", url: page.url.href, lang, languages, isListed }, "content language compared");
+        return isListed ? [] : [{ rule: "i18n/content-language", severity, scope: "page", url: page.url.href, group: page.group, ...said("lang and Content-Language name different languages"), data: { [page.url.href]: { lang, "content-language": header } }, value: header }];
+    },
+});
+// `lang href` pairs of a hreflang list, resolved against the page and lower-cased.
+function pairs(alternates, base) {
+    return new Set(alternates.map((alternate) => `${alternate.lang.toLowerCase()} ${resolve(alternate.href, base)}`));
+}
+// A page whose sitemap alternates and hreflang links both exist and disagree.
+const sitemapHreflang = (severity) => ({
+    meta: { id: "sitemap/hreflang", severity, scope: "page", facts: ["sitemap.alternates", "html.hreflang"], docs: "https://developers.google.com/search/docs/specialty/international/localized-versions#sitemap", fix: "Generate the sitemap alternates and the page’s hreflang links from one list." },
+    check(page) {
+        const [listed, linked] = [page.sitemap?.alternates ?? [], page.html?.hreflang ?? []];
+        if (listed.length === 0 || linked.length === 0)
+            return;
+        const [inSitemap, onPage] = [pairs(listed, page.url.href), pairs(linked, page.url.href)];
+        const differ = [...inSitemap.symmetricDifference(onPage)];
+        log.debug({ rule: "sitemap/hreflang", url: page.url.href, sitemap: inSitemap.size, page: onPage.size, differ: differ.length }, "sitemap alternates compared");
+        if (differ.length === 0)
+            return [];
+        const only = (side) => differ.filter((pair) => side.has(pair)).join(", ");
+        return [{ rule: "sitemap/hreflang", severity, scope: "page", url: page.url.href, group: page.group, ...said("sitemap alternates differ from the page’s hreflang"), data: { [page.url.href]: { "sitemap only": only(inSitemap), "page only": only(onPage) } }, value: differ }];
+    },
+});
+// A title or description confidently identified as a language other than the primary subtag of `lang`.
+const metadataLanguage = (severity) => ({
+    meta: { id: "i18n/metadata-language", severity, scope: "page", facts: ["html.detected", "html.lang"], docs: "https://www.w3.org/International/questions/qa-html-language-declarations", fix: "Translate the title and meta description into the language lang declares, or correct lang." },
+    check(page) {
+        const [lang, detected] = [page.html?.lang, page.html?.detected];
+        if (!lang || !detected)
+            return;
+        const declared = primary(lang);
+        const off = ["title", "description"].flatMap((field) => {
+            const guess = detected[field];
+            return guess?.reliable && guess.language !== declared ? [`${field} reads as ${guess.language} (${guess.confidence})`] : [];
+        });
+        log.debug({ rule: "i18n/metadata-language", url: page.url.href, lang, declared, title: detected.title?.language, description: detected.description?.language, off: off.length }, "metadata language compared");
+        return off.length === 0 ? [] : [{ rule: "i18n/metadata-language", severity, scope: "page", url: page.url.href, group: page.group, ...said("the title or description reads as another language than lang declares"), data: { [page.url.href]: { lang } }, locations: off, value: detected }];
+    },
+});
+export const i18nRules = {
+    "i18n/metadata-language": metadataLanguage,
+    "sitemap/hreflang": sitemapHreflang,
+    "i18n/hreflang-reciprocal": hreflangReciprocal,
+    "i18n/hreflang-status": hreflangStatus,
+    "i18n/content-language": contentLanguage,
+};

package/dist/rules/insights.js

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+import { median } from "../report/stats.js";
+import { get } from "./declarative.js";
+import { said } from "./message.js";
+// Iglewicz and Hoaglin’s modified z-score scale and cut-off.
+const Z_SCALE = 0.6745;
+// Their mean absolute deviation fallback when over half the values are equal.
+const MEAN_SCALE = 0.797885;
+// Pages the dominance test weighs the largest value against.
+const NEXT = 10;
+const DOCS = "https://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm";
+const NUMERIC = { facts: ["co2.grams", "http.size.body", "resources.length", "http.timing.total"], z: 3.5, ratio: 3, "min-pages": 20 };
+const MINORITY = { facts: ["http.version", "http.headers.content-encoding", "http.headers.cache-control"], share: 0.1, dominant: 0.8, "min-pages": 20 };
+// Pages carrying a value for `read`, grouped by it in first-seen order.
+export function partition(pages, read) {
+    return Map.groupBy(pages.filter((page) => read(page) !== undefined), (page) => read(page));
+}
+// Pages that answered 2xx, the only ones an insight compares.
+function served(pages) {
+    return pages.filter((page) => page.http.status >= 200 && page.http.status < 300);
+}
+// A fact as one comparable string, repeated headers joined; objects and arrays are not categories.
+function category(page, path) {
+    const value = get(page, path);
+    if (Array.isArray(value) && value.every((entry) => typeof entry === "string"))
+        return value.join(", ");
+    return ["string", "number", "boolean"].includes(typeof value) ? String(value) : undefined;
+}
+// The message sentences; each listed page’s own value rides in the finding’s `data`.
+const SENTENCES = {
+    outlier: "{label} is far above the median {median} of {count} pages",
+    dominates: "{label} is far above the median {median} of {count} pages; the first alone outweighs the next {next} pages combined",
+    minority: "{label} is {common} on {count} of {total} pages, but not on these",
+};
+// The pages whose `path` lies far above the rest by modified z-score and by `ratio` times the median.
+function outliers(pages, path, settings, severity) {
+    const measured = pages.flatMap((page) => {
+        const value = get(page, path);
+        return typeof value === "number" && Number.isFinite(value) ? [{ url: page.url.href, value }] : [];
+    });
+    if (measured.length < settings["min-pages"])
+        return undefined;
+    const sorted = measured.map((entry) => entry.value).toSorted((a, b) => a - b);
+    const middle = median(sorted);
+    const deviations = sorted.map((value) => Math.abs(value - middle));
+    const mad = median(deviations.toSorted((a, b) => a - b));
+    const scale = mad > 0 ? mad / Z_SCALE : deviations.reduce((sum, value) => sum + value, 0) / deviations.length / MEAN_SCALE;
+    const far = scale > 0 ? measured.filter((entry) => (entry.value - middle) / scale > settings.z && entry.value >= middle * settings.ratio).toSorted((a, b) => b.value - a.value) : [];
+    log.debug({ rule: "insight/numeric-outlier", fact: path, pages: measured.length, median: middle, mad, scale, outliers: far.length }, "numeric fact compared");
+    const [top] = far;
+    if (!top)
+        return undefined;
+    const next = sorted.slice(-NEXT - 1, -1).reduce((sum, value) => sum + value, 0);
+    const isDominant = sorted.length > NEXT && top.value > next;
+    const data = Object.fromEntries(far.map((entry) => [entry.url, { value: { fact: path, value: entry.value }, ...(middle > 0 && { ratio: { ratio: entry.value / middle } }) }]));
+    return {
+        rule: "insight/numeric-outlier",
+        severity,
+        scope: "site",
+        url: top.url,
+        ...said(isDominant ? SENTENCES.dominates : SENTENCES.outlier, { label: { name: path }, median: { fact: path, value: middle }, count: measured.length, ...(isDominant && { next: NEXT }) }),
+        data,
+        value: { median: middle, ...Object.fromEntries(far.map((entry) => [entry.url, entry.value])) },
+        urls: far.map((entry) => entry.url),
+    };
+}
+// The pages holding a rare value of `path` while most pages share one other value.
+function minority(pages, path, settings, severity) {
+    const byValue = [...partition(pages, (page) => category(page, path))].toSorted(([, a], [, b]) => b.length - a.length);
+    const total = byValue.reduce((sum, [, members]) => sum + members.length, 0);
+    const [top, ...rest] = byValue;
+    const rare = rest.filter(([, members]) => members.length / total <= settings.share);
+    const isDominant = top !== undefined && top[1].length / total >= settings.dominant;
+    log.debug({ rule: "insight/minority-value", fact: path, pages: total, values: byValue.length, top: top?.[0], rare: rare.length, isDominant }, "categorical fact compared");
+    if (!top || !isDominant || total < settings["min-pages"] || rare.length === 0)
+        return undefined;
+    const urls = rare.flatMap(([, members]) => members.map((page) => page.url.href));
+    const data = Object.fromEntries(rare.flatMap(([value, members]) => members.map((page) => [page.url.href, { value }])));
+    return { rule: "insight/minority-value", severity, scope: "site", url: urls[0], ...said(SENTENCES.minority, { label: { name: path }, common: top[0], count: top[1].length, total }), data, value: Object.fromEntries(byValue.map(([value, members]) => [value, members.length])), urls };
+}
+// One finding per numeric fact whose pages hold robust outliers above the median.
+const numericOutlier = (severity, expect) => {
+    const settings = { ...NUMERIC, ...expect };
+    return {
+        meta: { id: "insight/numeric-outlier", severity, scope: "site", facts: settings.facts, docs: DOCS, fix: "Find what makes the listed pages cost more than the rest, an uncached query or an oversized image, and bring them near the median." },
+        check: (pages) => settings.facts.map((path) => outliers(served(pages), path, settings, severity)).filter((finding) => finding !== undefined),
+    };
+};
+// One finding per categorical fact where a few pages break the value nearly every page shares.
+const minorityValue = (severity, expect) => {
+    const settings = { ...MINORITY, ...expect };
+    return {
+        meta: { id: "insight/minority-value", severity, scope: "site", facts: settings.facts, fix: "Serve the listed pages the way the rest of the site serves them, or keep the difference only where it is deliberate." },
+        check: (pages) => settings.facts.map((path) => minority(served(pages), path, settings, severity)).filter((finding) => finding !== undefined),
+    };
+};
+export const insightRules = { "insight/numeric-outlier": numericOutlier, "insight/minority-value": minorityValue };

package/dist/rules/length.js

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { MISMATCH } from "../crawl/fetch.js";
+import { log } from "../logger.js";
+import { said } from "./message.js";
+// What a resource finding names first.
+const SUBJECT = "{kind}:";
+// What is wrong with a response’s framing headers, or undefined when nothing is.
+export function framingFault(status, headers, error) {
+    const hasLength = headers["content-length"] !== undefined;
+    const detail = error?.startsWith(MISMATCH) ? error.slice(MISMATCH.length + 2) : undefined;
+    if (detail)
+        return { text: "the response body ended short of the bytes Content-Length declares", detail };
+    if (detail !== undefined)
+        return { text: "the response body does not match Content-Length, or Content-Length is sent twice or beside chunked" };
+    if (hasLength && headers["transfer-encoding"] !== undefined)
+        return { text: "the response sends Content-Length beside Transfer-Encoding" };
+    return hasLength && status === 204 ? { text: "a 204 response carries Content-Length" } : undefined;
+}
+// One finding per host whose pages, and per resource whose response, frame the body wrongly.
+const contentLength = (severity) => ({
+    meta: { id: "http/content-length", severity, scope: "site", facts: ["http.headers", "resources"], docs: "https://www.rfc-editor.org/rfc/rfc9112#section-6.3", fix: "Send one Content-Length equal to the body, none beside Transfer-Encoding and none on a 204." },
+    check(pages) {
+        const byFault = new Map();
+        const note = (key, subject, fault, url, kind) => byFault.set(key, { subject, ...(kind && { kind }), fault, urls: (byFault.get(key)?.urls ?? new Set()).add(url) });
+        for (const page of pages) {
+            const fault = framingFault(page.http.status, page.http.headers, page.http.error);
+            log.debug({ rule: "http/content-length", url: page.url.href, fault }, "page framing judged");
+            if (fault)
+                note(`${page.url.host}\t${fault.text}\t${fault.detail}`, page.url.host, fault, page.url.href);
+            const resources = page.resources ?? [];
+            for (const resource of resources) {
+                const http = resource.http;
+                const found = http && framingFault(http.status, http.headers, http.error);
+                if (found)
+                    note(resource.url, resource.url, found, page.url.href, resource.kind);
+            }
+        }
+        const findings = [];
+        for (const [key, { subject, kind, fault, urls }] of byFault) {
+            const listed = [...urls].toSorted((a, b) => a.localeCompare(b));
+            const sentence = kind === undefined ? said(`${fault.text}; on these pages`) : said(`${SUBJECT} ${fault.text}; used by these pages`, { kind });
+            log.debug({ rule: "http/content-length", subject, key, pages: listed.length }, "framing fault grouped");
+            findings.push({ rule: "http/content-length", severity, scope: "site", url: subject, ...sentence, ...(fault.detail && { data: { [subject]: { detail: fault.detail } } }), urls: listed });
+        }
+        return findings;
+    },
+});
+export const lengthRules = {
+    "http/content-length": contentLength,
+};

package/dist/rules/message.js

@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { label, withUnit } from "../facts/labels.js";
+import { utc } from "../facts/read-note.js";
+import { translator } from "../i18n.js";
+const english = translator("en");
+// Numbers as English writes them, one decimal.
+export const inEnglish = (value, options) => english.number(value, options);
+// A datum as text, numbers in `format`, a fact’s label through `translate`.
+export function written(datum, format, translate = (text) => text) {
+    if (typeof datum === "string")
+        return datum;
+    if (typeof datum === "number")
+        return format(datum);
+    if ("fact" in datum)
+        return withUnit(datum.fact, datum.value, format);
+    if ("name" in datum)
+        return translate(label(datum.name) ?? datum.name);
+    return "ratio" in datum ? `${format(datum.ratio)}×` : utc(datum.at);
+}
+// Every datum of a record as text, numbers in `format`, labels through `translate`.
+export function writtenAll(data = {}, format, translate) {
+    return Object.fromEntries(Object.entries(data).map(([key, datum]) => [key, written(datum, format, translate)]));
+}
+// A finding’s sentence in `t`’s language and locale; a finding with no template keeps its English message.
+export function sentence(finding, t = english) {
+    return finding.text === undefined
+        ? finding.message
+        : t._(finding.text, writtenAll(finding.variables, t.number, (text) => t._(text)));
+}
+// The message fields of a finding from its template and shared values.
+export function said(text, variables) {
+    return { message: sentence({ message: "", text, variables }), text, ...(variables && { variables }) };
+}
+// The values measured at `url`, as text in `format`, in the order the rule wrote them.
+export function valuesAt(finding, url, format) {
+    return Object.values(writtenAll(finding.data?.[url], format));
+}
+// Observations the findings rest on that this run took from a cache rather than the origin, once each.
+export function cachedReads(findings) {
+    return new Map(findings
+        .flatMap((finding) => finding.evidence ?? [])
+        .filter((read) => read.via === "cache")
+        .map((read) => [`${read.bucket}:${read.key}`, read]))
+        .values()
+        .toArray();
+}

package/dist/rules/relations.js

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import picomatch from "picomatch";
+import { ConfigError } from "../config/index.js";
+import { REGIONS } from "../facts/html.js";
+import { log } from "../logger.js";
+import { said } from "./message.js";
+// Internal links carrying `nofollow`, which withholds the site’s own ranking from its pages.
+const internalNofollow = (severity) => ({
+    meta: {
+        id: "links/internal-nofollow",
+        severity,
+        scope: "page",
+        facts: ["html.links.internal", "html.links.nofollow"],
+        docs: "https://developers.google.com/search/docs/crawling-indexing/qualify-outbound-links",
+        fix: "Drop rel=nofollow from links to the site’s own pages; keep robots.txt or noindex for pages crawlers should skip.",
+    },
+    check(page) {
+        if (!page.html)
+            return;
+        const internal = new Set(page.html.links.internal);
+        const found = page.html.links.nofollow.filter((href) => internal.has(href));
+        log.debug({ rule: "links/internal-nofollow", url: page.url.href, found: found.length }, "internal nofollow checked");
+        return found.length === 0 ? [] : [{ rule: "links/internal-nofollow", severity, scope: "page", url: page.url.href, group: page.group, ...said("internal links carry rel=nofollow"), data: { [page.url.href]: { links: found.length } }, locations: found, value: found }];
+    },
+});
+// The `expect` of `links/external-rel`: `[region:]host glob` → the rel tokens a link to a matching host, inside that region when named, carries.
+function policyOf(expect) {
+    const entries = Object.entries((expect ?? {}));
+    return entries.map(([key, tokens]) => {
+        const list = [tokens].flat();
+        const [region, glob] = key.includes(":") ? key.split(":", 2) : [undefined, key];
+        if (region !== undefined && !Object.hasOwn(REGIONS, region))
+            throw new ConfigError(`rule links/external-rel: expect ${key} names region ${region}, not one of ${Object.keys(REGIONS).join(", ")}`);
+        if (glob.length === 0 || list.length === 0 || list.some((token) => typeof token !== "string" || !/^[a-z-]+$/.test(token)))
+            throw new ConfigError(`rule links/external-rel: expect ${key || '""'} must name lower-case rel tokens, found ${JSON.stringify(tokens)}`);
+        return { ...(region !== undefined && { region }), isMatch: picomatch(glob, { nocase: true }), tokens: list };
+    });
+}
+// External links whose host a declared glob matches and that lack a token it requires.
+const externalRelation = (severity, expect) => {
+    const policy = policyOf(expect);
+    return {
+        meta: {
+            id: "links/external-rel",
+            severity,
+            scope: "page",
+            facts: ["html.links.external", "html.links.rel"],
+            docs: "https://developers.google.com/search/docs/crawling-indexing/qualify-outbound-links",
+            fix: "Add the rel tokens the policy names, as sponsored on paid links and ugc on user content, rather than nofollow on every external link.",
+            expect: expect,
+        },
+        check(page) {
+            if (!page.html || policy.length === 0)
+                return;
+            const findings = [];
+            const links = page.html.links;
+            const scopes = [
+                [undefined, links.external.map((href) => [href, links.rel?.[href] ?? []])],
+                ...new Set(policy.flatMap(({ region }) => region ?? [])).values().map((region) => [region, Object.entries(links.regions?.[region] ?? {})]),
+            ];
+            for (const [region, hrefs] of scopes) {
+                for (const [href, carried] of hrefs) {
+                    const missing = [...new Set(policy.filter((entry) => entry.region === region && entry.isMatch(new URL(href).hostname)).flatMap(({ tokens }) => tokens))].filter((token) => !carried.includes(token));
+                    if (missing.length === 0)
+                        continue;
+                    log.debug({ rule: "links/external-rel", url: page.url.href, href, region, carried, missing }, "external link lacks declared rel");
+                    findings.push({
+                        rule: "links/external-rel",
+                        severity,
+                        scope: "page",
+                        url: page.url.href,
+                        group: page.group,
+                        ...said("a link lacks the rel tokens declared for its host"),
+                        data: { [page.url.href]: { href, missing: missing.join(" "), ...(carried.length > 0 && { carried: carried.join(" ") }), ...(region !== undefined && { region }) } },
+                        value: { href, missing, ...(region !== undefined && { region }) },
+                    });
+                }
+            }
+            return findings;
+        },
+    };
+};
+export const relationRules = {
+    "links/internal-nofollow": internalNofollow,
+    "links/external-rel": externalRelation,
+};

package/dist/rules/robots.js

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+import { said } from "./message.js";
+// AI crawler product tokens by purpose; `retired` ones no vendor still sends.
+export const AI_CRAWLERS = {
+    gptbot: "training",
+    claudebot: "training",
+    ccbot: "training",
+    "google-extended": "training",
+    "applebot-extended": "training",
+    bytespider: "training",
+    "meta-externalagent": "training",
+    "cohere-training-data-crawler": "training",
+    ai2bot: "training",
+    "oai-searchbot": "search",
+    "claude-searchbot": "search",
+    perplexitybot: "search",
+    "chatgpt-user": "user",
+    "claude-user": "user",
+    "perplexity-user": "user",
+    "anthropic-ai": "retired",
+    "claude-web": "retired",
+    "cohere-ai": "retired",
+};
+// Content Signals keys and the values each takes.
+const SIGNALS = new Set(["search", "ai-input", "ai-train"]);
+const VERDICTS = new Set(["yes", "no"]);
+// A group shutting its agents out of every path.
+function isBlanket(group) {
+    return group.disallow.includes("/") && group.allow.length === 0;
+}
+// One site rule per robots.txt file: `judge` returns the finding’s sentence and value, or nothing.
+function robotsRule(id, documentation, judge, fix) {
+    return (severity) => ({
+        meta: { id, severity, scope: "site", facts: ["site.robots"], docs: documentation, ...(fix && { fix }) },
+        check(_pages, _group, site) {
+            const findings = [];
+            const files = site?.robots ?? [];
+            for (const file of files) {
+                const verdict = judge(file);
+                log.debug({ rule: id, url: file.url, isFinding: verdict !== undefined }, "robots.txt judged");
+                if (verdict)
+                    findings.push({ rule: id, severity, scope: "site", url: file.url, ...verdict });
+            }
+            return findings;
+        },
+    });
+}
+// Every group naming `*` that disallows `/` and allows nothing back.
+const disallowAll = robotsRule("robots/disallow-all", "https://www.rfc-editor.org/rfc/rfc9309#section-2.2.2", (file) => {
+    const blanket = file.groups.filter((group) => group.agents.includes("*") && isBlanket(group));
+    return blanket.length > 0 ? { ...said("User-agent: * is disallowed from every path, so no crawler indexes the site"), value: "/" } : undefined;
+}, "Remove the Disallow: / rule from the User-agent: * group, or add Allow rules for the paths you want crawled.");
+// The AI crawlers the file names, each with its purpose and whether it is shut out.
+const aiCrawlers = robotsRule("robots/ai-crawlers", "https://www.rfc-editor.org/rfc/rfc9309#section-2.2.1", (file) => {
+    const named = file.groups.flatMap((group) => group.agents.flatMap((agent) => (Object.hasOwn(AI_CRAWLERS, agent) ? [{ agent, purpose: AI_CRAWLERS[agent], blocked: isBlanket(group) }] : [])));
+    if (named.length === 0)
+        return;
+    return { ...said("the file names AI crawlers"), locations: named.map(({ agent, purpose, blocked }) => `${agent} (${purpose}${blocked ? ", disallowed" : ""})`), value: named };
+}, "Confirm each named AI crawler is allowed or disallowed as intended.");
+// Every `Content-Signal` line naming an unknown signal, a value other than yes or no, or nothing.
+const contentSignal = robotsRule("robots/content-signal", "https://contentsignals.org/", (file) => {
+    const malformed = file["content-signals"].filter(({ signals }) => {
+        const entries = Object.entries(signals);
+        return entries.length === 0 || entries.some(([key, value]) => !SIGNALS.has(key) || !VERDICTS.has(value));
+    });
+    return malformed.length > 0 ? { ...said("Content-Signal should be search, ai-input or ai-train set to yes or no"), data: { [file.url]: { found: malformed.map((line) => line.value).join(", ") } }, value: malformed.map((line) => line.value) } : undefined;
+}, "Write each Content-Signal line as comma-separated signal=yes|no pairs, e.g. Content-Signal: search=yes, ai-input=no, ai-train=no.");
+export const robotsRules = {
+    "robots/disallow-all": disallowAll,
+    "robots/ai-crawlers": aiCrawlers,
+    "robots/content-signal": contentSignal,
+};

package/dist/rules/rulesets.js

@@ -0,0 +1,134 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { readdirSync, readFileSync } from "node:fs";
+import picomatch from "picomatch";
+import { parse } from "yaml";
+import { ConfigError } from "../config/index.js";
+import { log } from "../logger.js";
+import { pageReader, pluginPreset, pluginPresetNames, ruleMaker } from "../plugins/index.js";
+import { compileRule } from "./declarative.js";
+const PRESETS = new URL("../../presets/", import.meta.url);
+const PREFIX = "spiderlint:";
+const ALL = "all";
+const SERVER = "server";
+const presetCache = new Map();
+// A plugin’s preset, else presets/<name>.yaml read once; undefined when no such preset ships.
+function preset(name) {
+    if (name === ALL)
+        return {
+            description: "Every rule that ships or a loaded plugin adds",
+            extends: presetNames()
+                .filter((other) => other !== ALL && other !== SERVER)
+                .map((other) => `${PREFIX}${other}`),
+        };
+    if (name === SERVER)
+        return serverPreset();
+    const plugged = pluginPreset(name);
+    if (plugged)
+        return plugged;
+    if (!presetCache.has(name)) {
+        try {
+            const text = readFileSync(new URL(`${name}.yaml`, PRESETS), "utf8");
+            presetCache.set(name, parse(text));
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw new ConfigError(`preset ${PREFIX}${name}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    return presetCache.get(name);
+}
+// The `recommended` rules that judge only the seeds’ origins and hosts, so running them alone fetches no page past a seed.
+function serverPreset() {
+    const cached = presetCache.get(SERVER);
+    if (cached)
+        return cached;
+    const specs = Object.entries(resolveRuleset(`${PREFIX}recommended`, {}));
+    const rules = Object.fromEntries(specs.filter(([id, spec]) => spec.severity !== "off" && !pageReader([compileRule(id, spec)])));
+    log.debug({ rules: Object.keys(rules).length, of: specs.length }, "server preset derived");
+    const derived = { description: "The recommended rules about the server, DNS and domain, which fetch only the seed pages", rules };
+    presetCache.set(SERVER, derived);
+    return derived;
+}
+// A bare name is the user's ruleset, else the bundled preset; the prefix forces the preset, and `all` is always every preset.
+export function lookup(name, rulesets) {
+    if (name === ALL && Object.hasOwn(rulesets, ALL))
+        throw new ConfigError(`ruleset ${ALL}: reserved for every shipped rule; rename it`);
+    return name.startsWith(PREFIX) ? preset(name.slice(PREFIX.length)) : (rulesets[name] ?? preset(name));
+}
+// Every preset that ships or a plugin adds, by bare name.
+export function presetNames() {
+    const files = readdirSync(PRESETS)
+        .filter((file) => file.endsWith(".yaml"))
+        .map((file) => file.slice(0, -".yaml".length));
+    return [...files, ALL, SERVER, ...pluginPresetNames()].toSorted((a, b) => a.localeCompare(b));
+}
+// A level pins the rule’s score to its band; a number replaces the level.
+function entryOf(entry, isUser) {
+    if (typeof entry === "number")
+        return { score: entry, severity: undefined, pinned: undefined };
+    const spec = typeof entry === "string" ? { severity: entry } : entry;
+    const isPinned = spec.severity !== undefined && spec.score === undefined && (typeof entry === "string" || isUser);
+    return isPinned ? { ...spec, pinned: true } : spec;
+}
+// Flattens `extends` depth-first; later entries override earlier ones per rule ID, `expect` keyword by keyword.
+export function resolveRuleset(name, rulesets, seen = []) {
+    if (seen.includes(name))
+        throw new ConfigError(`ruleset ${name}: extends itself through ${seen.join(" → ")}`);
+    const config = lookup(name, rulesets);
+    if (!config)
+        return selectRules(name, rulesets, seen);
+    const merged = {};
+    const parents = config.extends ?? [];
+    const isUser = !name.startsWith(PREFIX) && Object.hasOwn(rulesets, name);
+    for (const parent of parents)
+        Object.assign(merged, resolveRuleset(parent, rulesets, [...seen, name]));
+    const own = Object.entries(config.rules ?? {});
+    for (const [id, entry] of own) {
+        if (typeof entry !== "object" && merged[id] === undefined && ruleMaker(id) === undefined)
+            throw new ConfigError(`ruleset ${name}: rule ${id} sets ${entry} but is not defined`);
+        const spec = entryOf(entry, isUser);
+        // A new `expect` drops the inherited sentence, which may state the old bounds.
+        const expect = spec.expect && { expect: { ...merged[id]?.expect, ...spec.expect }, message: spec.message };
+        merged[id] = { ...merged[id], ...spec, ...expect, ...(config.when && { when: { ...config.when, ...merged[id]?.when, ...spec.when } }) };
+    }
+    log.debug({ ruleset: name, rules: Object.keys(merged).length }, "ruleset resolved");
+    return merged;
+}
+// A name no ruleset carries, read as a rule ID or glob over every shipped rule.
+function selectRules(pattern, rulesets, seen) {
+    const shipped = Object.entries(resolveRuleset(`${PREFIX}${ALL}`, rulesets, [...seen, pattern]));
+    const picked = Object.fromEntries(shipped.filter(([id]) => isRuleMatch(id, pattern)));
+    log.debug({ pattern, rules: Object.keys(picked) }, "ruleset name read as rule IDs");
+    if (Object.keys(picked).length === 0)
+        throw new ConfigError(`ruleset ${pattern}: not defined, and no rule ID matches it`);
+    return picked;
+}
+// Whether a rule ID is `pattern` or matches it as a glob (`lighthouse/*`).
+export function isRuleMatch(id, pattern) {
+    return id === pattern || picomatch.isMatch(id, pattern);
+}
+// Every rule ID the named rulesets define, `off` ones included.
+export function ruleIds(names, rulesets) {
+    return new Set(names.flatMap((name) => Object.keys(resolveRuleset(name, rulesets))));
+}
+// Union of the named rulesets, compiled; `off` rules — including `--exclude-rules` and a
+// `--error`/`--warning`/`--info` override landing on `off` — are dropped (AGENTS.md ## Rules).
+export function compileRulesets(names, rulesets, excludeRules = new Set(), overrides = {}) {
+    const specs = {};
+    for (const name of names)
+        Object.assign(specs, resolveRuleset(name, rulesets));
+    const rules = [];
+    for (const [id, spec] of Object.entries(specs)) {
+        const isExcluded = [...excludeRules].some((pattern) => isRuleMatch(id, pattern));
+        const override = overrides[id] ?? Object.entries(overrides).find(([pattern]) => isRuleMatch(id, pattern))?.[1];
+        const severity = isExcluded ? "off" : (override ?? spec.severity);
+        if (severity === "off") {
+            log.debug({ rule: id }, "rule off");
+            continue;
+        }
+        rules.push(compileRule(id, severity === spec.severity ? spec : { ...spec, severity, pinned: true }));
+    }
+    return rules;
+}