spiderlint 0.0.0 → 0.143.0

package/dist/config/environment.js

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { parseResolver } from "../crawl/dns.js";
+import { parsePin } from "../crawl/resolve.js";
+import { normalizeVia } from "../agent.js";
+import { ConfigError, PROFILES, ROLES, originOf, proxyOf } from "./index.js";
+export const FETCH_MODES = ["auto", "http", "browser", "adaptive"];
+export const BROWSERS = ["chromium", "firefox", "webkit"];
+export const SCOPES = ["origin", "host", "domain"];
+export const FAIL_ONS = ["error", "warning", "info", "never"];
+const CACHE_MODES = ["use", "off", "refresh", "offline"];
+function isTruthy(name, raw) {
+    const value = raw.trim().toLowerCase();
+    if (["1", "true", "yes", "y", "on"].includes(value))
+        return true;
+    if (["0", "false", "no", "n", "off"].includes(value))
+        return false;
+    throw new ConfigError(`${name}: invalid value ${raw} (expected a boolean)`);
+}
+function list(raw) {
+    return raw.split(/[\s,]+/).filter((entry) => entry.length > 0);
+}
+export function parseInteger(name, raw) {
+    const value = Number(raw);
+    if (!Number.isSafeInteger(value) || value < 0)
+        throw new ConfigError(`${name}: invalid value ${raw} (expected a non-negative integer)`);
+    return value;
+}
+export function pick(name, raw, valid) {
+    if (!valid.includes(raw))
+        throw new ConfigError(`${name}: invalid value ${raw} (expected: ${valid.join("|")})`);
+    return raw;
+}
+// A level or `never`, or the lowest score, 0.1 to 9.9, that fails the run.
+export function parseFailOn(name, raw) {
+    const score = Number(raw);
+    if (raw.trim() === "" || Number.isNaN(score))
+        return pick(name, raw, FAIL_ONS);
+    if (score < 0.1 || score > 9.9)
+        throw new ConfigError(`${name}: invalid score ${raw} (expected 0.1 to 9.9)`);
+    return score;
+}
+// One severity bucket of rule IDs, as `SPIDERLINT_OVERRIDE_ERROR` etc. carry it.
+function overrideBucket(raw, severity) {
+    return raw === undefined ? {} : Object.fromEntries(list(raw).map((id) => [id, severity]));
+}
+// SPIDERLINT_* mirrors each flag (AGENTS.md ## Configuration, ## Rules); only set variables apply.
+export function environmentSettings(environment) {
+    const overrideVariables = [environment.SPIDERLINT_OVERRIDE_ERROR, environment.SPIDERLINT_OVERRIDE_WARNING, environment.SPIDERLINT_OVERRIDE_INFO, environment.SPIDERLINT_OVERRIDE_HINT];
+    return {
+        ...(environment.SPIDERLINT_TARGETS !== undefined && { seeds: list(environment.SPIDERLINT_TARGETS) }),
+        ...(environment.SPIDERLINT_CANONICAL_ORIGIN !== undefined && { canonicalOrigin: originOf("SPIDERLINT_CANONICAL_ORIGIN", environment.SPIDERLINT_CANONICAL_ORIGIN) }),
+        ...(environment.SPIDERLINT_ROLE !== undefined && { role: pick("SPIDERLINT_ROLE", environment.SPIDERLINT_ROLE, ROLES) }),
+        ...(environment.SPIDERLINT_RESOLVER !== undefined && { resolver: parseResolver(environment.SPIDERLINT_RESOLVER) }),
+        ...(environment.SPIDERLINT_RESOLVE !== undefined && { resolve: list(environment.SPIDERLINT_RESOLVE).map((pin) => parsePin(pin)) }),
+        ...(environment.SPIDERLINT_FETCH !== undefined && { fetch: pick("SPIDERLINT_FETCH", environment.SPIDERLINT_FETCH, FETCH_MODES) }),
+        ...(environment.SPIDERLINT_BROWSER !== undefined && { browser: pick("SPIDERLINT_BROWSER", environment.SPIDERLINT_BROWSER, BROWSERS) }),
+        ...(environment.SPIDERLINT_BROWSER_INSTALL !== undefined && { browserInstall: isTruthy("SPIDERLINT_BROWSER_INSTALL", environment.SPIDERLINT_BROWSER_INSTALL) }),
+        ...(environment.SPIDERLINT_SCOPE !== undefined && { scope: pick("SPIDERLINT_SCOPE", environment.SPIDERLINT_SCOPE, SCOPES) }),
+        ...(environment.SPIDERLINT_CONCURRENCY !== undefined && { concurrency: parseInteger("SPIDERLINT_CONCURRENCY", environment.SPIDERLINT_CONCURRENCY) }),
+        ...(environment.SPIDERLINT_RATE !== undefined && { rate: parseInteger("SPIDERLINT_RATE", environment.SPIDERLINT_RATE) }),
+        ...(environment.SPIDERLINT_TIMEOUT !== undefined && { timeout: parseInteger("SPIDERLINT_TIMEOUT", environment.SPIDERLINT_TIMEOUT) }),
+        ...(environment.SPIDERLINT_PROFILE !== undefined && { profile: pick("SPIDERLINT_PROFILE", environment.SPIDERLINT_PROFILE, Object.keys(PROFILES)) }),
+        ...(environment.SPIDERLINT_PROXY !== undefined && { proxy: proxyOf("SPIDERLINT_PROXY", environment.SPIDERLINT_PROXY) }),
+        ...(environment.SPIDERLINT_VIA !== undefined && { via: normalizeVia(environment.SPIDERLINT_VIA) }),
+        ...(environment.SPIDERLINT_MAX_PAGES !== undefined && { maxPages: parseInteger("SPIDERLINT_MAX_PAGES", environment.SPIDERLINT_MAX_PAGES) }),
+        ...(environment.SPIDERLINT_MAX_DEPTH !== undefined && { maxDepth: parseInteger("SPIDERLINT_MAX_DEPTH", environment.SPIDERLINT_MAX_DEPTH) }),
+        ...(environment.SPIDERLINT_MAX_BODY_SIZE !== undefined && { maxBodySize: parseInteger("SPIDERLINT_MAX_BODY_SIZE", environment.SPIDERLINT_MAX_BODY_SIZE) }),
+        ...(environment.SPIDERLINT_KEEPALIVE !== undefined && { keepalive: isTruthy("SPIDERLINT_KEEPALIVE", environment.SPIDERLINT_KEEPALIVE) }),
+        ...(environment.SPIDERLINT_ALLOW_PRIVATE !== undefined && { allowPrivate: isTruthy("SPIDERLINT_ALLOW_PRIVATE", environment.SPIDERLINT_ALLOW_PRIVATE) }),
+        ...(environment.SPIDERLINT_RESOURCES !== undefined && { fetchResources: isTruthy("SPIDERLINT_RESOURCES", environment.SPIDERLINT_RESOURCES) }),
+        ...(environment.SPIDERLINT_RULES !== undefined && { rules: list(environment.SPIDERLINT_RULES) }),
+        ...(environment.SPIDERLINT_SOURCES !== undefined && { sources: list(environment.SPIDERLINT_SOURCES) }),
+        ...(environment.SPIDERLINT_INCLUDE_URLS !== undefined && { includeUrls: list(environment.SPIDERLINT_INCLUDE_URLS) }),
+        ...(environment.SPIDERLINT_EXCLUDE_URLS !== undefined && { excludeUrls: list(environment.SPIDERLINT_EXCLUDE_URLS) }),
+        ...(environment.SPIDERLINT_ROBOTS !== undefined && { robots: isTruthy("SPIDERLINT_ROBOTS", environment.SPIDERLINT_ROBOTS) }),
+        ...(environment.SPIDERLINT_SITEMAP !== undefined && { sitemap: isTruthy("SPIDERLINT_SITEMAP", environment.SPIDERLINT_SITEMAP) }),
+        ...(environment.SPIDERLINT_FOLD !== undefined && { fold: isTruthy("SPIDERLINT_FOLD", environment.SPIDERLINT_FOLD) && { threshold: 0.8, min: 3 } }),
+        ...(environment.SPIDERLINT_FAIL_ON !== undefined && { failOn: parseFailOn("SPIDERLINT_FAIL_ON", environment.SPIDERLINT_FAIL_ON) }),
+        ...(environment.SPIDERLINT_FORMAT !== undefined && { format: environment.SPIDERLINT_FORMAT }),
+        ...(environment.SPIDERLINT_CACHE !== undefined && { cacheMode: pick("SPIDERLINT_CACHE", environment.SPIDERLINT_CACHE, CACHE_MODES) }),
+        ...(environment.SPIDERLINT_EXCLUDE_RULES !== undefined && { excludeRules: list(environment.SPIDERLINT_EXCLUDE_RULES) }),
+        ...(overrideVariables.some((value) => value !== undefined) && {
+            overrides: { ...overrideBucket(environment.SPIDERLINT_OVERRIDE_ERROR, "error"), ...overrideBucket(environment.SPIDERLINT_OVERRIDE_WARNING, "warning"), ...overrideBucket(environment.SPIDERLINT_OVERRIDE_INFO, "info"), ...overrideBucket(environment.SPIDERLINT_OVERRIDE_HINT, "hint") },
+        }),
+    };
+}

package/dist/config/index.js

@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+export const ROLES = ["production", "staging", "development"];
+export function defaults() {
+    return {
+        seeds: [],
+        role: "production",
+        fetch: "auto",
+        browser: "chromium",
+        browserInstall: true,
+        scope: "origin",
+        concurrency: 0,
+        rate: 0,
+        timeout: 60,
+        crawlDeadline: 0,
+        profile: "",
+        proxy: "",
+        via: "",
+        maxPages: 0,
+        maxDepth: 0,
+        maxBodySize: 10_000_000,
+        keepalive: true,
+        fetchResources: true,
+        maxResourcesPerPage: 200,
+        linkExclude: [],
+        includeUrls: [],
+        excludeUrls: [],
+        vendorPaths: true,
+        diversify: true,
+        robots: true,
+        sitemap: true,
+        fold: { threshold: 0.8, min: 3 },
+        failOn: "error",
+        format: "human",
+        excludeRules: [],
+        documentTypes: [],
+        denyRules: [],
+        overrides: {},
+        groups: {},
+        rulesets: {},
+        plugins: [],
+        sources: [],
+        follow: true,
+        cacheMode: "use",
+        cacheTtl: {},
+        cacheFailureTtl: 600,
+        allowPrivate: true,
+        resolver: "system",
+        resolve: [],
+        pluginSettings: {},
+    };
+}
+// Settings a profile lays over the defaults; `adaptive: false` resolves an adaptive group as `auto`.
+export const PROFILES = {
+    tor: { concurrency: 4, timeout: 240, adaptive: false, proxy: "socks5h://127.0.0.1:9050" },
+    i2p: { concurrency: 4, timeout: 240, adaptive: false, proxy: "http://127.0.0.1:4444" },
+};
+// Thrown for anything that maps to exit code 2.
+export class ConfigError extends Error {
+}
+// The origin of an absolute http(s) URL; anything else names `name` in the error.
+export function originOf(name, raw) {
+    const url = URL.canParse(raw) ? new URL(raw) : undefined;
+    if (!url || !["http:", "https:"].includes(url.protocol))
+        throw new ConfigError(`${name}: invalid value ${raw} (expected an absolute http or https URL)`);
+    return url.origin;
+}
+// A seed with https:// prepended when it names no scheme, so example.com and localhost:8080 stay domains.
+export function seedOf(raw) {
+    return raw.includes("://") ? raw : `https://${raw}`;
+}
+// A proxy URL of a scheme every client can use, or empty for none; anything else names `name` in the error.
+export function proxyOf(name, raw) {
+    const url = URL.canParse(raw) ? new URL(raw) : undefined;
+    const isValid = raw === "" || ["http:", "https:", "socks:", "socks4:", "socks4a:", "socks5:", "socks5h:"].includes(url?.protocol ?? "");
+    if (!isValid)
+        throw new ConfigError(`${name}: invalid value ${raw} (expected an http, https, socks4, socks4a, socks5 or socks5h URL)`);
+    return raw;
+}
+// Applies every defined key of `patch` over `base`; undefined keys leave `base` untouched.
+// This is the merge step of the flags > env > file > defaults precedence ladder.
+export function overlay(base, patch) {
+    const merged = { ...base };
+    for (const key of Object.keys(patch)) {
+        if (patch[key] !== undefined)
+            merged[key] = patch[key];
+    }
+    return merged;
+}
+// `patches` over the defaults, lowest first, with the profile they name laid in between.
+export function layered(patches) {
+    const merge = (base) => {
+        let merged = base;
+        for (const patch of patches)
+            merged = overlay(merged, patch);
+        return merged;
+    };
+    const { profile } = merge(defaults());
+    const { adaptive, ...bundle } = PROFILES[profile] ?? {};
+    log.debug({ profile, bundle, adaptive }, "profile applied");
+    return merge(overlay(defaults(), bundle));
+}

package/dist/config/policy.js

@@ -0,0 +1,170 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { parse as parseYaml } from "yaml";
+import { normalizeVia } from "../agent.js";
+import { parseDuration } from "../cache/index.js";
+import { parseResolver } from "../crawl/dns.js";
+import { parsePin } from "../crawl/resolve.js";
+import { log } from "../logger.js";
+import { ConfigError, originOf, proxyOf } from "./index.js";
+import { CORE_KEYS, validateSubtree } from "./schema.js";
+const SUBTREE = "org.spiderlint";
+const LEVELS = ["error", "warning", "info", "hint"];
+const DISCOVER_NAMES = ["projectfile.yaml", "projectfile.toml", "projectfile.json"];
+// [subtree key, Settings field] — kebab-case document keys to the camelCase Config shape.
+// `override` is excluded: its three severity buckets flatten into one field, below.
+const KEYS = [
+    ["targets", "seeds"],
+    ["role", "role"],
+    ["fetch", "fetch"],
+    ["browser", "browser"],
+    ["browser-install", "browserInstall"],
+    ["scope", "scope"],
+    ["concurrency", "concurrency"],
+    ["rate", "rate"],
+    ["timeout", "timeout"],
+    ["profile", "profile"],
+    ["max-pages", "maxPages"],
+    ["max-depth", "maxDepth"],
+    ["max-body-size", "maxBodySize"],
+    ["keepalive", "keepalive"],
+    ["include-urls", "includeUrls"],
+    ["exclude-urls", "excludeUrls"],
+    ["vendor-paths", "vendorPaths"],
+    ["diversify", "diversify"],
+    ["robots", "robots"],
+    ["sitemap", "sitemap"],
+    ["fold", "fold"],
+    ["fail-on", "failOn"],
+    ["format", "format"],
+    ["exclude-rules", "excludeRules"],
+    ["document-types", "documentTypes"],
+    ["rules", "rules"],
+    ["groups", "groups"],
+    ["rulesets", "rulesets"],
+    ["plugins", "plugins"],
+    ["sources", "sources"],
+    ["via", "via"],
+    ["allow-private", "allowPrivate"],
+];
+// error, then warning, then info, then hint — a rule ID named in a later bucket wins (AGENTS.md ## Rules).
+function flattenOverride(bucket) {
+    const overrides = {};
+    for (const level of LEVELS) {
+        const ids = bucket[level] ?? [];
+        for (const id of ids)
+            overrides[id] = level;
+    }
+    return overrides;
+}
+// A duration to seconds; a malformed one names its path.
+function seconds(field, raw) {
+    const parsed = parseDuration(raw);
+    if (parsed === undefined)
+        throw new ConfigError(`org.spiderlint/cache/${field}: invalid duration ${String(raw)} (expected seconds or 45s, 30m, 24h, 7d)`);
+    return parsed;
+}
+// `cache.<bucket>.ttl` durations to seconds.
+function cacheTtl(cache) {
+    const out = {};
+    for (const [bucket, { ttl }] of Object.entries(cache)) {
+        if (ttl !== undefined)
+            out[bucket] = seconds(`${bucket}/ttl`, ttl);
+    }
+    return out;
+}
+// One subtree’s keys as the camelCase Settings shape.
+export function fromSubtree(subtree) {
+    const settings = {};
+    for (const [key, field] of KEYS) {
+        if (subtree[key] !== undefined)
+            settings[field] = subtree[key];
+    }
+    const resources = subtree.resources;
+    if (resources?.fetch !== undefined)
+        settings.fetchResources = resources.fetch;
+    if (resources?.["max-per-page"] !== undefined)
+        settings.maxResourcesPerPage = resources["max-per-page"];
+    const links = subtree.links;
+    if (links?.exclude !== undefined)
+        settings.linkExclude = links.exclude.map((host) => host.toLowerCase());
+    if (subtree["canonical-origin"] !== undefined)
+        settings.canonicalOrigin = originOf("org.spiderlint/canonical-origin", subtree["canonical-origin"]);
+    if (subtree.proxy !== undefined)
+        settings.proxy = proxyOf("org.spiderlint/proxy", subtree.proxy);
+    if (settings.via !== undefined)
+        settings.via = normalizeVia(settings.via);
+    if (subtree.resolver !== undefined)
+        settings.resolver = parseResolver(subtree.resolver);
+    if (subtree.resolve !== undefined)
+        settings.resolve = subtree.resolve.map((pin) => parsePin(pin));
+    if (subtree.cache !== undefined)
+        settings.cacheTtl = cacheTtl(subtree.cache);
+    const failureTtl = subtree.cache?.resources?.["failure-ttl"];
+    if (failureTtl !== undefined)
+        settings.cacheFailureTtl = seconds("resources/failure-ttl", failureTtl);
+    if (subtree.override !== undefined)
+        settings.overrides = flattenOverride(subtree.override);
+    const plugins = Object.entries(subtree).filter(([key]) => !CORE_KEYS.has(key));
+    if (plugins.length > 0)
+        settings.pluginSettings = Object.fromEntries(plugins);
+    return settings;
+}
+// Shared settings plus one patch per `sites.<name>`; a site key replaces the shared one whole.
+function resolve(raw) {
+    const { sites = {}, ...shared } = validateSubtree(raw);
+    if (Object.keys(sites).length > 0 && shared.targets !== undefined)
+        throw new ConfigError("org.spiderlint/targets: with sites, every target belongs to a sites.<name>.targets");
+    log.debug({ sites: Object.keys(sites) }, "sites declared");
+    const settings = fromSubtree(shared);
+    // A site’s plugin key replaces the shared one of the same plugin, not every plugin’s.
+    const patch = (site) => (site.pluginSettings ? { ...site, pluginSettings: { ...settings.pluginSettings, ...site.pluginSettings } } : site);
+    return { settings, sites: Object.fromEntries(Object.entries(sites).map(([name, site]) => [name, patch(fromSubtree(site))])) };
+}
+// First recognised projectfile encoding present in the working directory.
+function discover() {
+    return DISCOVER_NAMES.find((name) => existsSync(name));
+}
+// null when pf-cli is not on PATH; otherwise the subtree (possibly an error).
+function runPfCli(document) {
+    const result = spawnSync("pf-cli", ["get", "--file", document, SUBTREE, "--format", "json", "--quiet"], { encoding: "utf8" });
+    if (result.error)
+        return undefined;
+    // pf-cli answers a path the document lacks with `null` and exit 1.
+    if (result.stdout.trim() === "null" && result.stderr.trim() === "")
+        return { raw: {} };
+    return result.status === 0 ? { raw: (JSON.parse(result.stdout || "null") ?? {}) } : { error: (result.stderr || result.stdout).trim() };
+}
+// No pf-cli: the file itself is parsed and its org.spiderlint key read directly.
+function readPlainYaml(document) {
+    const parsed = parseYaml(readFileSync(document, "utf8"));
+    return parsed?.org?.spiderlint ?? {};
+}
+// Resolves the org.spiderlint subtree: explicit --config, else cwd discovery, else no file.
+// pf-cli reads the document when it is on PATH; an explicit --config file falls back to a
+// direct YAML parse otherwise, so outsiders need no pf-cli install (AGENTS.md ## Configuration).
+export function loadSettings(explicit) {
+    if (explicit !== undefined && !existsSync(explicit))
+        throw new ConfigError(`config file not found: ${explicit}`);
+    const document = explicit ?? discover();
+    if (document === undefined) {
+        log.debug({}, "no projectfile found, using defaults");
+        return { settings: {}, sites: {}, document: undefined };
+    }
+    const pfResult = runPfCli(document);
+    if (pfResult === undefined) {
+        if (explicit === undefined) {
+            log.info({ document }, "pf-cli not found, skipping projectfile config");
+            return { settings: {}, sites: {}, document };
+        }
+        log.debug({ document }, "pf-cli not found, parsing config file directly");
+        return { ...resolve(readPlainYaml(document)), document };
+    }
+    if (pfResult.error !== undefined)
+        throw new ConfigError(`pf-cli: cannot read ${SUBTREE} from ${document}: ${pfResult.error}`);
+    log.debug({ document }, "config subtree loaded via pf-cli");
+    return { ...resolve(pfResult.raw), document };
+}

package/dist/config/schema.js

@@ -0,0 +1,127 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { Ajv2020 } from "ajv/dist/2020.js";
+import { ConfigError, PROFILES, ROLES } from "./index.js";
+const ajv = new Ajv2020({ strictTypes: false, allErrors: true });
+const ttl = { oneOf: [{ type: "string" }, { type: "integer", minimum: 0 }] };
+const fetchMode = { enum: ["auto", "http", "browser", "adaptive"] };
+const severity = { enum: ["error", "warning", "info", "hint", "off"] };
+const score = { type: "number", minimum: 0, maximum: 9.9 };
+const ruleSpec = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        fact: { type: "string" },
+        expect: { type: "object" },
+        when: { type: "object" },
+        unique: { type: "string" },
+        scope: { enum: ["page", "group", "site"] },
+        severity,
+        score: { oneOf: [score, { type: "array", minItems: 2, items: { type: "array", minItems: 2, maxItems: 2, items: { type: "number" } } }] },
+        docs: { type: "string" },
+        fix: { type: "string" },
+        hints: { type: "object", additionalProperties: { type: "string" } },
+        message: { type: "string" },
+        linked: { type: "boolean" },
+    },
+};
+// The org.spiderlint subtree, as documented in AGENTS.md ## Configuration.
+// additionalProperties: false below the top, where an object key is a plugin’s until `configurePlugins` finds none claiming it.
+const site = {
+    type: "object",
+    additionalProperties: { type: "object" },
+    properties: {
+        targets: { type: "array", items: { type: "string" } },
+        "canonical-origin": { type: "string" },
+        role: { enum: ROLES },
+        resolver: { type: "string" },
+        resolve: { type: "array", items: { type: "string" } },
+        rules: { type: "array", items: { type: "string" } },
+        fetch: fetchMode,
+        browser: { enum: ["chromium", "firefox", "webkit"] },
+        "browser-install": { type: "boolean" },
+        scope: { enum: ["origin", "host", "domain"] },
+        concurrency: { type: "integer", minimum: 0 },
+        rate: { type: "integer", minimum: 0 },
+        timeout: { type: "integer", minimum: 1 },
+        profile: { enum: Object.keys(PROFILES) },
+        "max-pages": { type: "integer", minimum: 0 },
+        "max-depth": { type: "integer", minimum: 0 },
+        "max-body-size": { type: "integer", minimum: 0 },
+        keepalive: { type: "boolean" },
+        "allow-private": { type: "boolean" },
+        "include-urls": { type: "array", items: { type: "string" } },
+        "exclude-urls": { type: "array", items: { type: "string" } },
+        "vendor-paths": { type: "boolean" },
+        diversify: { type: "boolean" },
+        resources: {
+            type: "object",
+            additionalProperties: false,
+            properties: { fetch: { type: "boolean" }, "max-per-page": { type: "integer", minimum: 0 } },
+        },
+        links: { type: "object", additionalProperties: false, properties: { exclude: { type: "array", items: { type: "string", minLength: 1 } } } },
+        proxy: { type: "string" },
+        via: { type: "string" },
+        robots: { type: "boolean" },
+        sitemap: { type: "boolean" },
+        fold: { oneOf: [{ const: false }, { type: "object", additionalProperties: false, properties: { threshold: { type: "number", minimum: 0, maximum: 1 }, min: { type: "integer", minimum: 1 } } }] },
+        cache: {
+            type: "object",
+            additionalProperties: false,
+            properties: Object.fromEntries(["pages", "probes", "profiles", "lists", "resources", "robots", "sitemaps", "origins", "dns", "extractors"].map((bucket) => [bucket, { type: "object", additionalProperties: false, properties: { ttl, ...(bucket === "resources" && { "failure-ttl": ttl }) } }])),
+        },
+        "fail-on": { oneOf: [{ enum: ["error", "warning", "info", "never"] }, { type: "number", minimum: 0.1, maximum: 9.9 }] },
+        format: { type: "string" },
+        plugins: { type: "array", items: { type: "string" } },
+        sources: { type: "array", items: { type: "string" } },
+        "exclude-rules": { type: "array", items: { type: "string" } },
+        "document-types": { type: "array", items: { type: "string", pattern: "^[!#$%&'*+.^_`|~0-9A-Za-z-]+/[!#$%&'*+.^_`|~0-9A-Za-z-]+$" } },
+        override: {
+            type: "object",
+            additionalProperties: false,
+            properties: Object.fromEntries(["error", "warning", "info", "hint"].map((level) => [level, { type: "array", items: { type: "string" } }])),
+        },
+        groups: {
+            type: "object",
+            additionalProperties: { type: "object", additionalProperties: false, properties: { match: { type: "array", items: { type: "string" } }, rules: { type: "array", items: { type: "string" } }, fetch: fetchMode, sample: { oneOf: [{ type: "integer", minimum: 1 }, { const: "all" }] } } },
+        },
+        rulesets: {
+            type: "object",
+            additionalProperties: {
+                type: "object",
+                additionalProperties: false,
+                properties: {
+                    description: { type: "string" },
+                    extends: { type: "array", items: { type: "string" } },
+                    when: { type: "object" },
+                    rules: { type: "object", additionalProperties: { oneOf: [severity, score, ruleSpec] } },
+                },
+            },
+        },
+    },
+};
+// Keys the core reads; any other top-level key belongs to a plugin.
+export const CORE_KEYS = new Set([...Object.keys(site.properties), "sites"]);
+// Each `sites.<name>` entry carries the same keys as the subtree, minus `sites`.
+const schema = { ...site, properties: { ...site.properties, sites: { type: "object", additionalProperties: site } } };
+const validate = ajv.compile(schema);
+// Schema paths where a top-level key that is no object can only be a misspelt core key.
+const TOP_LEVEL = new Set(["#/additionalProperties/type", "#/properties/sites/additionalProperties/additionalProperties/type"]);
+// A misspelt or misplaced key names its own path under `prefix`, per AGENTS.md ## Configuration.
+export function describe(error, prefix = "org.spiderlint") {
+    if (error.keyword === "additionalProperties") {
+        const key = error.params.additionalProperty;
+        return `${prefix}${error.instancePath}: unknown key "${key}"`;
+    }
+    const cut = error.instancePath.lastIndexOf("/");
+    return TOP_LEVEL.has(error.schemaPath) ? `${prefix}${error.instancePath.slice(0, cut)}: unknown key "${error.instancePath.slice(cut + 1)}"` : `${prefix}${error.instancePath}: ${error.message}`;
+}
+// Throws ConfigError (exit 2) listing every violation found in the subtree.
+// `value` must already be an object (an absent subtree is the caller's `{}`, not `undefined`).
+export function validateSubtree(value, prefix = "org.spiderlint") {
+    if (!validate(value)) {
+        throw new ConfigError((validate.errors ?? []).map((error) => describe(error, prefix)).join("; "));
+    }
+    return value;
+}

package/dist/crawl/body.js

@@ -0,0 +1,92 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { PassThrough } from "node:stream";
+import zlib from "node:zlib";
+import { log } from "../logger.js";
+// Response fields Crawlee reads off the stream it parses.
+const FIELDS = ["statusCode", "statusMessage", "headers", "httpVersion", "rawHeaders", "url", "request", "complete"];
+// Types whose body becomes facts; any other type is judged by its headers alone.
+const PARSED = /^(?:text\/html|application\/xhtml\+xml|(?:text|application)\/xml|application\/json|application\/[\w.-]+\+(?:xml|json))$/;
+// A missing content type is parsed, as Crawlee then guesses from the URL.
+export function isParsed(contentType) {
+    const type = contentType?.split(";", 1)[0]?.trim().toLowerCase();
+    return !type || PARSED.test(type);
+}
+// `body` under `contentType` in place of `source`’s own, its other response fields kept.
+export function replayed(source, body, contentType) {
+    const stream = new PassThrough();
+    for (const field of FIELDS)
+        Object.assign(stream, { [field]: source[field] });
+    Object.assign(stream, { headers: { ...source.headers, "content-type": contentType } });
+    source.resume();
+    stream.end(body);
+    return stream;
+}
+// The codings a page request advertises, each one `decoder` undoes.
+export const ACCEPT_ENCODING = "gzip, deflate, br, zstd";
+// Each known coding’s decoder, tolerant of a stream cut short.
+const DECODERS = {
+    gzip: () => zlib.createGunzip({ finishFlush: zlib.constants.Z_SYNC_FLUSH }),
+    "x-gzip": () => zlib.createGunzip({ finishFlush: zlib.constants.Z_SYNC_FLUSH }),
+    deflate: () => zlib.createInflate({ finishFlush: zlib.constants.Z_SYNC_FLUSH }),
+    br: () => zlib.createBrotliDecompress({ finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH }),
+    zstd: () => zlib.createZstdDecompress(),
+};
+// A stream undoing one `Content-Encoding`; undefined for identity or a coding it does not know.
+export function decoder(encoding) {
+    return DECODERS[encoding?.trim().toLowerCase() ?? ""]?.();
+}
+// The response got wraps once it read every declared byte, which a parse error on bytes past them leaves intact.
+function completed(source) {
+    const response = source.response;
+    return response?.complete ? response : undefined;
+}
+// At most `max` decoded bytes of `source`, which is destroyed once the cap is reached.
+export function capped(source, max) {
+    const stream = new PassThrough();
+    for (const field of FIELDS)
+        Object.assign(stream, { [field]: source[field] });
+    let isCut = false;
+    const unzip = max === 0 ? undefined : decoder([source.headers?.["content-encoding"]].flat()[0]);
+    const body = unzip ? source.pipe(unzip) : source;
+    const cut = () => {
+        isCut = true;
+        source.destroy();
+        unzip?.destroy();
+        stream.end();
+    };
+    if (max === 0) {
+        cut();
+        return { stream, isTruncated: () => isCut };
+    }
+    let seen = 0;
+    const take = (chunk) => {
+        if (isCut)
+            return;
+        const room = max - seen;
+        seen += chunk.length;
+        stream.write(chunk.subarray(0, room));
+        if (seen >= max)
+            cut();
+    };
+    // A failure after every declared byte arrived ends the body with the bytes got never passed on; any other destroys it.
+    const fail = (error) => {
+        const response = completed(source);
+        log.debug({ error: error.message, isComplete: response !== undefined, seen }, "body stream failed");
+        if (!response)
+            return stream.destroy(error);
+        const rest = response.read() ?? Buffer.alloc(0);
+        if (unzip)
+            return void unzip.end(rest);
+        take(rest);
+        stream.end();
+    };
+    body.on("data", take);
+    body.on("end", () => stream.end());
+    source.on("error", fail);
+    unzip?.on("error", (error) => stream.destroy(error));
+    if (source.errored)
+        fail(source.errored);
+    return { stream, isTruncated: () => isCut };
+}