spiderlint 0.0.0 → 0.143.0

package/dist/plugins/images.js

@@ -0,0 +1,269 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash } from "node:crypto";
+import { transform } from "esbuild";
+import sharp, {} from "sharp";
+import { optimize } from "svgo";
+import { log } from "../logger.js";
+import { resourceRule } from "../rules/builtin.js";
+import { said } from "../rules/message.js";
+import { definePlugin } from "./types.js";
+const ID = "images";
+const TEXT = "text";
+const FONTS = "fonts";
+const LAYOUT = "imageLayout";
+const TIMEOUT_SECONDS = 30;
+const MAX_PIXELS = 50_000_000;
+// Digests whose facts stay in memory, oldest dropped first.
+const MEASURED_MAX = 1024;
+const SETTINGS = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        saving: { type: "object", additionalProperties: false, default: {}, properties: { share: { type: "number", minimum: 0, maximum: 1, default: 0.2 }, bytes: { type: "integer", minimum: 0, default: 10_000 } } },
+        weight: { type: "integer", minimum: 0, default: 200_000 },
+        oversize: { type: "number", minimum: 1, default: 2 },
+    },
+};
+// Encoders per output format, at the quality a CDN image service defaults to; png is palette-quantised as TinyPNG does.
+const ENCODERS = {
+    jpeg: (image) => image.jpeg({ quality: 80, mozjpeg: true }),
+    png: (image) => image.png({ compressionLevel: 9, palette: true, effort: 7 }),
+    webp: (image) => image.webp({ quality: 80, effort: 4 }),
+    avif: (image) => image.avif({ quality: 50, effort: 4 }),
+};
+const LEGACY = new Set(["jpeg", "png", "gif"]);
+const SCRIPTS = ["text/javascript", "application/javascript", "application/x-javascript", "application/ecmascript", "text/ecmascript"];
+// Font formats by their first four bytes.
+const FONT_TAGS = { wOF2: "woff2", wOFF: "woff", OTTO: "otf", "\0\u{1}\0\0": "ttf", true: "ttf", ttcf: "ttc" };
+// `font-display` values that hide text while the font loads.
+const INVISIBLE = new Set(["auto", "block"]);
+// sharp reports AVIF as `heif` with AV1 compression.
+function formatOf(metadata) {
+    return metadata.format === "heif" && metadata.compression === "av1" ? "avif" : metadata.format;
+}
+// Re-encode targets for `format`: its own encoder, then WebP and AVIF; libvips encodes no animated AVIF.
+function targetsOf(format, isAnimated) {
+    const own = Object.hasOwn(ENCODERS, format) && !(isAnimated && format === "avif") ? [["same", format]] : [];
+    const modern = LEGACY.has(format) ? [["webp", "webp"], ...(isAnimated ? [] : [["avif", "avif"]])] : [];
+    return [...own, ...modern];
+}
+// An SVG’s size, and its size after SVGO when SVGO can parse it.
+function measureSvg(url, body) {
+    const text = Buffer.from(body).toString("utf8");
+    try {
+        return { format: "svg", bytes: body.byteLength, encoded: { same: Buffer.byteLength(optimize(text).data) } };
+    }
+    catch (error) {
+        log.debug({ url, error: error instanceof Error ? error.message : String(error) }, "SVGO cannot parse the SVG");
+        return { format: "svg", bytes: body.byteLength };
+    }
+}
+// Format, size and the bytes each re-encoding would take, on one libvips thread; SVG goes through SVGO, never rasterised.
+async function measure(url, contentType, body) {
+    if (contentType === "image/svg+xml")
+        return measureSvg(url, body);
+    sharp.concurrency(1);
+    sharp.cache(false);
+    const input = () => sharp(body, { animated: true, limitInputPixels: MAX_PIXELS }).timeout({ seconds: TIMEOUT_SECONDS });
+    const metadata = await input().metadata();
+    const format = formatOf(metadata);
+    const isAnimated = (metadata.pages ?? 1) > 1;
+    const facts = { format, bytes: body.byteLength, width: metadata.width, height: isAnimated ? metadata.pageHeight : metadata.height, ...(isAnimated && { animated: true }) };
+    const encoded = {};
+    for (const [target, output] of targetsOf(format, isAnimated)) {
+        const buffer = await ENCODERS[output](input()).toBuffer();
+        encoded[target] = buffer.byteLength;
+    }
+    log.debug({ url, format, bytes: body.byteLength, isAnimated, encoded }, "image measured");
+    return Object.keys(encoded).length > 0 ? { ...facts, encoded } : facts;
+}
+// Facts per content type and body digest, so one image served under several URLs is measured once.
+const measured = new Map();
+// An image’s facts, measured once per digest; a failed measurement is not remembered.
+async function extract(url, contentType, body) {
+    const digest = createHash("sha256").update(contentType).update("\0").update(body).digest("hex");
+    const known = measured.get(digest);
+    log.debug({ url, digest, isKnown: known !== undefined }, "image looked up by digest");
+    if (known)
+        return structuredClone(await known);
+    const facts = measure(url, contentType, body);
+    measured.set(digest, facts);
+    if (measured.size > MEASURED_MAX)
+        measured.delete(measured.keys().next().value);
+    try {
+        return structuredClone(await facts);
+    }
+    catch (error) {
+        measured.delete(digest);
+        throw error;
+    }
+}
+// Each `@font-face` family and its `font-display`, read from esbuild’s minified CSS.
+function fontFaces(css) {
+    return css
+        .matchAll(/@font-face\{([^}]*)\}/g)
+        .map(([, block = ""]) => {
+        const family = /font-family:\s*("[^"]*"|'[^']*'|[^;]*)/.exec(block)?.[1]?.replaceAll(/^["']|["']$/g, "") ?? "";
+        const display = /font-display:\s*([^;]*)/.exec(block)?.[1]?.trim();
+        return { family, ...(display && { display }) };
+    })
+        .toArray();
+}
+// A stylesheet’s or script’s bytes before and after esbuild minifies it.
+async function extractText(url, contentType, body) {
+    const loader = contentType === "text/css" ? "css" : "js";
+    const { code } = await transform(Buffer.from(body).toString("utf8"), { loader, minify: true, logLevel: "silent" });
+    const faces = loader === "css" ? fontFaces(code) : [];
+    log.debug({ url, loader, bytes: body.byteLength, minified: Buffer.byteLength(code), fontFaces: faces.length }, "text asset minified");
+    return { bytes: body.byteLength, minified: Buffer.byteLength(code), ...(faces.length > 0 && { "font-faces": faces }) };
+}
+// A font’s format by its leading bytes, whatever its content type claims; EOT carries `LP` at byte 34.
+async function extractFont(url, _contentType, body) {
+    const tag = Buffer.from(body.subarray(0, 4)).toString("latin1");
+    const format = FONT_TAGS[tag] ?? (body[34] === 0x4c && body[35] === 0x50 ? "eot" : "unknown");
+    log.debug({ url, tag, format }, "font sniffed");
+    return { format, bytes: body.byteLength };
+}
+// Where each rendered `<img>` sits and how many pixels it ships, read in the crawler’s page.
+const LAYOUT_SCRIPT = `(() => ({
+    viewport: { width: innerWidth, height: innerHeight, dpr: devicePixelRatio },
+    images: [...document.images].map((img) => {
+        const box = img.getBoundingClientRect();
+        return { src: img.getAttribute("src") ?? img.currentSrc, top: Math.round(box.top + scrollY), width: Math.round(box.width), height: Math.round(box.height), natural: img.naturalWidth, loading: img.loading };
+    }),
+}))()`;
+// The share saved when `after` beats `before` by both `saving` bounds.
+function saved(saving, before, after) {
+    return after === undefined || before - after < saving.bytes || (before - after) / before < saving.share ? undefined : Math.round((1 - after / before) * 100);
+}
+const imageOf = (resource) => resource[ID];
+const textOf = (resource) => resource[TEXT];
+const fontOf = (resource) => resource[FONTS];
+const isImage = (_page, resource) => imageOf(resource) !== undefined;
+const valueOf = (resource) => imageOf(resource);
+const FACTS = [`resources.${ID}`];
+// A JPEG, PNG or GIF that WebP or AVIF would shrink.
+const modernFormat = (severity, settings) => resourceRule("images/modern-format", isImage, (resource) => {
+    const image = imageOf(resource);
+    const encoded = image.encoded ?? {};
+    const target = ["avif", "webp"].filter((name) => encoded[name] !== undefined).toSorted((a, b) => encoded[a] - encoded[b])[0];
+    const share = target && LEGACY.has(image.format) ? saved(settings.saving, image.bytes, encoded[target]) : undefined;
+    return share === undefined
+        ? undefined
+        : { ...said("a {format} image is smaller as {target}; used by these pages", { format: image.format, target: target }), data: { [resource.url]: { size: { fact: "http.size.body", value: image.bytes }, smaller: { fact: "http.size.body", value: encoded[target] } } } };
+}, FACTS, valueOf, { docs: "https://web.dev/articles/choose-the-right-image-format", fix: "Convert the image to WebP or AVIF." })(severity);
+// An image its own format, or SVGO, re-encodes much smaller.
+const recompress = (severity, settings) => resourceRule("images/recompress", isImage, (resource) => {
+    const image = imageOf(resource);
+    const share = saved(settings.saving, image.bytes, image.encoded?.same);
+    return share === undefined ? undefined : { ...said("a {format} image re-encodes smaller; used by these pages", { format: image.format }), data: { [resource.url]: { size: { fact: "http.size.body", value: image.bytes }, smaller: { fact: "http.size.body", value: image.encoded?.same } } } };
+}, FACTS, valueOf, { docs: "https://web.dev/articles/use-imagemin-to-compress-images", fix: "Re-encode the image with a modern tool at a lower quality or smaller dimensions." })(severity);
+// An image heavier than `weight`.
+const weight = (severity, settings) => resourceRule("images/weight", isImage, (resource) => {
+    const image = imageOf(resource);
+    const limit = settings.weight;
+    return image.bytes > limit ? { ...said("a {format} image is above the weight limit; used by these pages", { format: image.format }), data: { [resource.url]: { size: { fact: "http.size.body", value: image.bytes }, limit: { fact: "http.size.body", value: limit } } } } : undefined;
+}, FACTS, valueOf, { docs: "https://developer.mozilla.org/docs/Learn_web_development/Extensions/Performance/Multimedia", fix: "Resize or re-compress the image so the file is below the weight limit." })(severity);
+// A stylesheet or script minification shrinks by both `saving` bounds.
+const minify = (severity, settings) => resourceRule("images/minify", (_page, resource) => textOf(resource) !== undefined, (resource) => {
+    const text = textOf(resource);
+    const share = saved(settings.saving, text.bytes, text.minified);
+    return share === undefined ? undefined : { ...said("a {kind} minifies smaller; used by these pages", { kind: resource.kind }), data: { [resource.url]: { size: { fact: "http.size.body", value: text.bytes }, smaller: { fact: "http.size.body", value: text.minified } } } };
+}, [`resources.${TEXT}`], textOf, { docs: "https://web.dev/articles/reduce-network-payloads-using-text-compression", fix: "Minify the stylesheet or script in the build." })(severity);
+// A font served in any format but WOFF2.
+const fontFormat = resourceRule("images/font-format", (_page, resource) => fontOf(resource) !== undefined, (resource) => {
+    const font = fontOf(resource);
+    return font.format === "woff2" ? undefined : { ...said("a {format} font is not WOFF2; used by these pages", { format: font.format }), data: { [resource.url]: { size: { fact: "http.size.body", value: font.bytes } } } };
+}, [`resources.${FONTS}`], fontOf, { docs: "https://web.dev/articles/reduce-webfont-size", fix: "Convert the font to WOFF2." });
+// A stylesheet whose `@font-face` rules hide text while their font loads.
+const fontDisplay = resourceRule("images/font-display", (_page, resource) => textOf(resource)?.["font-faces"] !== undefined, (resource) => {
+    const invisible = (textOf(resource)?.["font-faces"] ?? []).filter((face) => INVISIBLE.has(face.display ?? "auto")).map((face) => face.family);
+    return invisible.length === 0 ? undefined : { ...said("@font-face rules without font-display: swap, fallback or optional; used by these pages"), data: { [resource.url]: { faces: invisible.length, families: [...new Set(invisible)].join(", ") } } };
+}, [`resources.${TEXT}`], (resource) => textOf(resource)?.["font-faces"], { docs: "https://developer.mozilla.org/docs/Web/CSS/@font-face/font-display", fix: "Add font-display: swap (or fallback or optional) to each @font-face." });
+// A page rule over its `<img>` elements: one finding listing each offender as a location.
+function imgRule(id, facts, documentation, sentence, offends, fix) {
+    return (severity) => ({
+        meta: { id, severity, scope: "page", facts, docs: documentation, ...(fix && { fix }) },
+        check(page) {
+            if (!page.html)
+                return;
+            const locations = page.html.images.filter((img) => !img.noscript).flatMap((img) => offends(page, img) ?? []);
+            log.debug({ rule: id, url: page.url.href, images: page.html.images.length, offending: locations.length }, "images judged");
+            return locations.length === 0 ? [] : [{ rule: id, severity, scope: "page", url: page.url.href, group: page.group, ...sentence, data: { [page.url.href]: { images: locations.length } }, value: locations, locations }];
+        },
+    });
+}
+// A page rule over its rendered `<img>` boxes, skipped on a page the layout extractor did not sample; hidden images are never judged.
+function layoutRule(id, documentation, sentence, offends, fix) {
+    return (severity, settings) => ({
+        meta: { id, severity, scope: "page", facts: [LAYOUT], docs: documentation, ...(fix && { fix }) },
+        check(page) {
+            const layout = page[LAYOUT];
+            if (!layout)
+                return;
+            const locations = layout.images.filter((box) => box.width > 0 && box.height > 0).flatMap((box) => offends(layout, box, settings) ?? []);
+            log.debug({ rule: id, url: page.url.href, images: layout.images.length, offending: locations.length }, "rendered images judged");
+            return locations.length === 0 ? [] : [{ rule: id, severity, scope: "page", url: page.url.href, group: page.group, ...sentence, data: { [page.url.href]: { images: locations.length } }, value: locations, locations }];
+        },
+    });
+}
+// The image facts of the resource an `<img src>` loads.
+function loaded(page, source) {
+    if (!URL.canParse(source, page.url.href))
+        return undefined;
+    const url = new URL(source, page.url.href);
+    url.hash = "";
+    const resource = page.resources?.find((entry) => entry.url === url.href && imageOf(entry));
+    return resource && imageOf(resource);
+}
+const dimensions = imgRule("images/dimensions", ["html.images"], "https://web.dev/articles/optimize-cls#images-without-dimensions", said("an <img> without width and height shifts the layout as it loads"), (_page, img) => (img.width === undefined || img.height === undefined ? img.src : undefined), "Add width and height attributes to each <img>.");
+const oversized = (severity, settings) => {
+    const { oversize } = settings;
+    return imgRule("images/oversized", ["html.images", `resources.${ID}`], "https://web.dev/articles/serve-responsive-images", said("an <img> without srcset ships over {oversize} the width it displays", { oversize: { ratio: oversize } }), (page, img) => {
+        const declared = Number(img.width);
+        const intrinsic = img.srcset === undefined && /^\d+$/.test(img.width ?? "") ? loaded(page, img.src)?.width : undefined;
+        return intrinsic !== undefined && declared > 0 && intrinsic > declared * oversize ? `${img.src} ${intrinsic} px for width=${declared}` : undefined;
+    }, "Serve a srcset with a width matching what the page renders.")(severity);
+};
+const lazyBelow = layoutRule("images/lazy-below-fold", "https://web.dev/articles/browser-level-image-lazy-loading", said("an <img> below the fold without loading=lazy loads before the reader scrolls"), (layout, box) => (box.top >= layout.viewport.height && box.loading !== "lazy" ? `${box.src} at ${box.top} px` : undefined), "Add loading=lazy to off-screen images.");
+const lazyAbove = layoutRule("images/lazy-above-fold", "https://web.dev/articles/lcp-lazy-loading", said("an <img> in the first viewport with loading=lazy waits for layout before loading"), (layout, box) => (box.top < layout.viewport.height && box.loading === "lazy" ? `${box.src} at ${box.top} px` : undefined), "Remove loading=lazy from images in the first viewport.");
+const renderedOversize = layoutRule("images/rendered-oversize", "https://web.dev/articles/serve-responsive-images", said("an <img> ships far more pixels than the page renders"), (layout, box, settings) => (box.natural > box.width * layout.viewport.dpr * settings.oversize ? `${box.src} ${box.natural} px shown at ${box.width} px` : undefined), "Serve an image whose natural width matches what the layout renders.");
+export default definePlugin({
+    name: "images",
+    settings: SETTINGS,
+    extractors: [{ id: LAYOUT, mode: "browser", cost: "expensive", inputs: ["resources"], extract: async (_page, _body, live) => live && (await live.evaluate(LAYOUT_SCRIPT)) }],
+    resources: [
+        { id: ID, types: ["image/jpeg", "image/png", "image/gif", "image/webp", "image/avif", "image/svg+xml"], extract },
+        { id: TEXT, types: ["text/css", ...SCRIPTS], extract: extractText },
+        { id: FONTS, types: ["font/", "application/font-", "application/x-font-", "application/vnd.ms-fontobject"], extract: extractFont },
+    ],
+    rules: {
+        "images/modern-format": modernFormat,
+        "images/recompress": recompress,
+        "images/weight": weight,
+        "images/dimensions": dimensions,
+        "images/oversized": oversized,
+        "images/minify": minify,
+        "images/font-format": fontFormat,
+        "images/font-display": fontDisplay,
+        "images/lazy-below-fold": lazyBelow,
+        "images/lazy-above-fold": lazyAbove,
+        "images/rendered-oversize": renderedOversize,
+    },
+    presets: {
+        images: {
+            description: "Image weight and markup: bytes a modern format or a re-encode saves, heavy and oversized images, layout shift",
+            rules: { "images/modern-format": { severity: "warning", score: 5.2 }, "images/recompress": { severity: "warning", score: 4.8 }, "images/weight": { severity: "warning", score: 5.6 }, "images/dimensions": { severity: "warning", score: 5.4 }, "images/oversized": { severity: "warning", score: 4.6 } },
+        },
+        "images:assets": {
+            description: "Fonts and text assets: WOFF2, font-display, and bytes minification saves",
+            rules: { "images/font-format": { severity: "warning", score: 4.4 }, "images/font-display": { severity: "warning", score: 5 }, "images/minify": { severity: "warning", score: 4.2 } },
+        },
+        "images:live": {
+            description: "Images as the browser lays them out, on sampled pages: lazy loading against the fold, pixels shipped against pixels shown",
+            rules: { "images/lazy-below-fold": { severity: "warning", score: 4.8 }, "images/lazy-above-fold": { severity: "warning", score: 5.8 }, "images/rendered-oversize": { severity: "warning", score: 4.4 } },
+        },
+    },
+});

package/dist/plugins/index.js

@@ -0,0 +1,286 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { Ajv2020 } from "ajv/dist/2020.js";
+import { loadedDigest } from "../crawl/browser.js";
+import { VERSION } from "../agent.js";
+import { ExtractorCache } from "../cache/extractors.js";
+import { ConfigError } from "../config/index.js";
+import { CORE_KEYS, describe } from "../config/schema.js";
+import { subjectPath } from "../facts/sites.js";
+import { log } from "../logger.js";
+import { progressStep } from "../progress.js";
+import { inSpan, recordExtractor } from "../telemetry.js";
+import { builtin } from "../rules/builtin.js";
+import report from "../report/index.js";
+import { presetNames } from "../rules/rulesets.js";
+import axe from "./axe.js";
+import cookies from "./cookies.js";
+import css from "./css.js";
+import dns from "./dns.js";
+import feeds from "./feeds.js";
+import htmlValidate from "./html-validate.js";
+import htmlhint from "./htmlhint.js";
+import images from "./images.js";
+import keyboard from "./keyboard.js";
+import lighthouse from "./lighthouse.js";
+import linkText from "./link-text.js";
+import profileLinks from "./profile-links.js";
+import list from "./list.js";
+import live from "./live.js";
+import icons from "./icons.js";
+import manifest from "./manifest.js";
+import markup from "./markup.js";
+import network from "./network.js";
+import origin from "./origin.js";
+import structuredData from "./structured-data.js";
+import tlsProbe from "./tls-probe.js";
+import mail from "./mail.js";
+import sshfp from "./sshfp.js";
+import stack from "./stack.js";
+import trackers from "./trackers.js";
+import wellKnown from "./well-known.js";
+const plugins = [report, htmlValidate, htmlhint, axe, keyboard, profileLinks, live, lighthouse, origin, dns, mail, sshfp, network, stack, tlsProbe, cookies, images, css, wellKnown, feeds, structuredData, manifest, icons, linkText, markup, trackers, list];
+// Milliseconds before a source that has not answered aborts the run.
+const SOURCE_MS = 60_000;
+const loaded = new Set(plugins.map((plugin) => plugin.name));
+const bundled = plugins.flatMap((plugin) => [...(plugin.extractors ?? []), ...(plugin.resources ?? []), ...(plugin.sites ?? [])]);
+for (const extractor of bundled)
+    extractor.version ??= VERSION;
+const ajv = new Ajv2020({ strictTypes: false, allErrors: true, useDefaults: true });
+// Each plugin’s validated settings, by plugin name.
+const settings = new Map();
+// `raw` validated against the plugin’s schema, its defaults filled in; a violation names its path.
+function settle(plugin, raw = {}) {
+    if (!plugin.settings)
+        return undefined;
+    const value = structuredClone(raw);
+    const validate = ajv.compile(plugin.settings);
+    if (!validate(value))
+        throw new ConfigError((validate.errors ?? []).map((error) => describe(error, `org.spiderlint/${plugin.name}`)).join("; "));
+    return value;
+}
+// Settles every plugin’s key of `raw`; a key no loaded plugin claims is unknown.
+function configure(raw) {
+    const unclaimed = Object.keys(raw).filter((key) => plugins.every((plugin) => plugin.name !== key || !plugin.settings));
+    if (unclaimed.length > 0)
+        throw new ConfigError(unclaimed.map((key) => `org.spiderlint: unknown key "${key}"`).join("; "));
+    settings.clear();
+    for (const plugin of plugins)
+        if (plugin.settings)
+            settings.set(plugin.name, settle(plugin, raw[plugin.name]));
+    log.debug({ plugins: settings.keys().toArray(), set: Object.keys(raw) }, "plugin settings validated");
+}
+// A TypeScript rule by ID: the core’s, else a plugin’s, handed its plugin’s settings.
+export function ruleMaker(id) {
+    const plugin = plugins.find((candidate) => candidate.rules?.[id]);
+    const make = plugin?.rules?.[id];
+    return builtin[id] ?? (plugin && make && ((severity) => make(severity, settings.has(plugin.name) ? settings.get(plugin.name) : settle(plugin))));
+}
+// A preset a plugin ships.
+export function pluginPreset(name) {
+    return plugins.find((plugin) => plugin.presets?.[name])?.presets?.[name];
+}
+export function pluginPresetNames() {
+    return plugins.flatMap((plugin) => Object.keys(plugin.presets ?? {}));
+}
+// A formatter by format name, bundled or a plugin’s.
+export function formatter(name) {
+    return plugins.find((plugin) => plugin.formatters?.[name])?.formatters?.[name];
+}
+// Every format name, the bundled ones first.
+export function formatNames() {
+    return [...plugins.flatMap((plugin) => Object.keys(plugin.formatters ?? {}))];
+}
+function allSources() {
+    return plugins.flatMap((plugin) => plugin.sources ?? []);
+}
+function allExtractors() {
+    return plugins.flatMap((plugin) => plugin.extractors ?? []);
+}
+function allSiteExtractors() {
+    return plugins.flatMap((plugin) => plugin.sites ?? []);
+}
+function allResourceExtractors() {
+    return plugins.flatMap((plugin) => plugin.resources ?? []);
+}
+// Whether reading `fact` needs a rendered page: `browser.*`, or the key of a browser-mode extractor.
+export function isBrowserFact(fact) {
+    const root = fact.split(".", 1)[0];
+    return root === "browser" || allExtractors().some((extractor) => extractor.mode === "browser" && extractor.id === root);
+}
+// Whether reading `fact` needs an expensive extractor, whose facts only the group’s sample carries.
+export function isSampledFact(fact) {
+    const root = fact.split(".", 1)[0];
+    return allExtractors().some((extractor) => extractor.cost === "expensive" && extractor.id === root);
+}
+// Adds a plugin; a rule, preset or extractor name already taken is a config error.
+function register(plugin) {
+    const taken = new Set(allExtractors().map((extractor) => extractor.id));
+    const shipped = new Set(presetNames());
+    const rules = Object.keys(plugin.rules ?? {}).filter((id) => ruleMaker(id));
+    const presets = Object.keys(plugin.presets ?? {}).filter((name) => shipped.has(name));
+    const extractors = (plugin.extractors ?? []).map((extractor) => extractor.id).filter((id) => taken.has(id));
+    const sites = (plugin.sites ?? []).filter((site) => allSiteExtractors().some((other) => other.id === site.id && other.per === site.per)).map((site) => `site.${site.per}s.*.${site.id}`);
+    const resources = (plugin.resources ?? []).filter((resource) => allResourceExtractors().some((other) => other.id === resource.id)).map((resource) => `resources.${resource.id}`);
+    const formatters = Object.keys(plugin.formatters ?? {})
+        .filter((name) => formatNames().includes(name))
+        .map((name) => `format ${name}`);
+    const sources = (plugin.sources ?? []).filter((source) => allSources().some((other) => other.id === source.id)).map((source) => `source ${source.id}`);
+    const keys = plugin.settings && CORE_KEYS.has(plugin.name) ? [`settings key ${plugin.name}`] : [];
+    const clash = [...rules, ...presets, ...extractors, ...sites, ...resources, ...formatters, ...sources, ...keys];
+    if (clash.length > 0)
+        throw new ConfigError(`plugin ${plugin.name}: ${clash.join(", ")} already defined`);
+    plugins.push(plugin);
+    log.debug({
+        plugin: plugin.name,
+        rules: Object.keys(plugin.rules ?? {}).length,
+        presets: Object.keys(plugin.presets ?? {}),
+        extractors: plugin.extractors?.length ?? 0,
+        sites: plugin.sites?.length ?? 0,
+        resources: plugin.resources?.length ?? 0,
+        formatters: Object.keys(plugin.formatters ?? {}),
+        sources: (plugin.sources ?? []).map((source) => source.id),
+    }, "plugin registered");
+}
+// Imports each named plugin once: a path from the working directory, else a package installed beside spiderlint; then settles every plugin’s settings from `raw`.
+export async function loadPlugins(names, raw = {}) {
+    for (const name of names) {
+        if (loaded.has(name)) {
+            log.debug({ plugin: name }, "plugin bundled or loaded");
+            continue;
+        }
+        const specifier = /^\.{0,2}\//.test(name) ? pathToFileURL(path.resolve(name)).href : name;
+        let module;
+        try {
+            module = (await import(__rewriteRelativeImportExtension(specifier)));
+        }
+        catch (error) {
+            throw new ConfigError(`plugin ${name}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (typeof module.default?.name !== "string")
+            throw new ConfigError(`plugin ${name}: its default export has no name`);
+        if (!loaded.has(module.default.name))
+            register(module.default);
+        loaded.add(name).add(module.default.name);
+        log.debug({ plugin: module.default.name, specifier }, "plugin loaded");
+    }
+    configure(raw);
+}
+// Seeds with each `sources` entry’s URLs appended; one source that does not follow makes them the whole frontier.
+export async function withSources(config) {
+    let { seeds, follow } = config;
+    for (const entry of config.sources) {
+        const [id = "", ...rest] = entry.split(":");
+        const argument = rest.join(":");
+        const source = allSources().find((candidate) => candidate.id === id);
+        if (!source)
+            throw new ConfigError(`source ${id}: unknown (known: ${allSources()
+                .map((candidate) => candidate.id)
+                .join(", ")})`);
+        let urls;
+        try {
+            urls = await source.urls(argument, AbortSignal.timeout(SOURCE_MS));
+        }
+        catch (error) {
+            throw new ConfigError(`source ${entry}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        seeds = [...seeds, ...urls];
+        follow &&= source.follow !== false;
+        log.debug({ source: id, argument, urls: urls.length, follow }, "source read");
+    }
+    return { ...config, seeds: [...new Set(seeds)], follow };
+}
+// Extractors whose ID is the first key of a fact some rule reads.
+export function extractorsFor(rules) {
+    const roots = new Set(rules.flatMap((rule) => rule.meta.facts.map((fact) => fact.split(".", 1)[0])));
+    const active = allExtractors().filter((extractor) => roots.has(extractor.id));
+    log.debug({ extractors: active.map((extractor) => extractor.id), roots: roots.size }, "extractors chosen");
+    return active;
+}
+// Site extractors whose ID sits under `site.origins.*` or `site.hosts.*` in a fact some rule reads.
+export function siteExtractorsFor(rules) {
+    const read = new Set(rules.flatMap((rule) => rule.meta.facts.flatMap((fact) => {
+        const subject = subjectPath(fact);
+        return subject ? [`${subject.kind}\t${subject.id}`] : [];
+    })));
+    const active = plugins.flatMap((plugin) => (plugin.sites ?? []).filter((extractor) => read.has(`${extractor.per}s\t${extractor.id}`)).map((extractor) => (settings.has(plugin.name) ? { ...extractor, settings: settings.get(plugin.name) } : extractor)));
+    log.debug({ extractors: active.map((extractor) => extractor.id), read: read.size }, "site extractors chosen");
+    return active;
+}
+// The rule, else the site extractor, that reads pages past the seeds; none when `rules` judge only the seeds’ origins and hosts.
+export function pageReader(rules) {
+    const offsite = rules.find((rule) => rule.meta.linked || rule.meta.facts.some((fact) => !fact.startsWith("site.") || fact.startsWith("site.redirects")));
+    return offsite?.meta.id ?? siteExtractorsFor(rules).find((extractor) => extractor.crawled)?.id;
+}
+// IDs of the `per: host` site extractors a `linked` rule reads, which then also run on linked hosts.
+export function linkedSiteExtractors(rules) {
+    const ids = new Set(rules
+        .filter((rule) => rule.meta.linked)
+        .flatMap((rule) => rule.meta.facts.flatMap((fact) => {
+        const subject = subjectPath(fact);
+        return subject?.kind === "hosts" ? [subject.id] : [];
+    })));
+    log.debug({ extractors: [...ids] }, "site extractors judging linked hosts");
+    return ids;
+}
+// Resource extractors whose ID is the second key of a `resources.<id>` fact some rule reads.
+export function resourceExtractorsFor(rules) {
+    const read = new Set(rules.flatMap((rule) => rule.meta.facts.flatMap((fact) => {
+        const [root, id] = fact.split(".", 2);
+        return root === "resources" && id ? [id] : [];
+    })));
+    const active = allResourceExtractors().filter((extractor) => read.has(extractor.id));
+    log.debug({ extractors: active.map((extractor) => extractor.id), read: read.size }, "resource extractors chosen");
+    return active;
+}
+// What keys `extractor` beyond the body: its `inputs` and, rendered, the browser; undefined when one is unknown.
+function inputsOf(extractor, page, live) {
+    const browser = live?.context().browser();
+    const parts = extractor.mode === "browser" ? [browser ? `${browser.browserType().name()} ${browser.version()}` : undefined] : [];
+    const inputs = extractor.inputs ?? [];
+    for (const input of inputs)
+        parts.push(input === "resources" ? live && loadedDigest(live) : [page.http.headers.link ?? ""].flat().join(", "));
+    return parts.includes(undefined) ? undefined : parts.join("\0");
+}
+// Each extractor’s facts under its ID, through `cache`, returning the IDs that added some; one that throws, or needs a `live` page it lacks, adds nothing.
+export async function extract(page, body, active, cache, live, context) {
+    const kind = `${page.http["content-type"]}${page.http.size.truncated ? " truncated" : ""}`;
+    const added = [];
+    for (const extractor of active) {
+        if (!live && extractor.mode === "browser") {
+            log.debug({ url: page.url.href, extractor: extractor.id }, "extractor needs a rendered page");
+            continue;
+        }
+        try {
+            progressStep(page.url.href, extractor.id);
+            const running = performance.now();
+            const inputs = inputsOf(extractor, page, live);
+            if (inputs === undefined)
+                log.debug({ url: page.url.href, extractor: extractor.id, inputs: extractor.inputs }, "extractor inputs unknown, run uncached");
+            const keyed = inputs === undefined ? { ...extractor, cached: false } : extractor;
+            const value = await inSpan(`extract ${extractor.id}`, { "spiderlint.extractor": extractor.id, "url.full": page.url.href }, async () => cache.run(keyed, page.url.href, inputs ? `${kind}\0${inputs}` : kind, body, () => extractor.extract(page, body, live, context)));
+            const ms = Math.round(performance.now() - running);
+            recordExtractor(extractor.id, ms);
+            log.debug({ url: page.url.href, extractor: extractor.id, ms, isEmpty: value === undefined }, "extractor ran");
+            if (value === undefined)
+                continue;
+            page[extractor.id] = value;
+            added.push(extractor.id);
+        }
+        catch (error) {
+            log.warn({ url: page.url.href, extractor: extractor.id, error: error instanceof Error ? error.message : String(error) }, `${extractor.id} checks skipped on`);
+        }
+    }
+    return added;
+}