spiderlint 0.0.0 → 0.143.0

package/dist/crawl/resources.js

@@ -0,0 +1,233 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { progressCount, progressPhase } from "../progress.js";
+import { availableParallelism } from "node:os";
+import { ExtractorCache } from "../cache/extractors.js";
+import { Bucket, OfflineMiss } from "../cache/index.js";
+import { fetchCached, fetchHeadCached } from "../cache/http.js";
+import { attemptsFor, MISMATCH, reason } from "./fetch.js";
+import { cookieFacts, redactHeaders } from "../facts/transport.js";
+import { log } from "../logger.js";
+// The Accept header Chromium sends for an image, so an origin negotiating AVIF or WebP answers as it would a browser.
+const IMAGE_ACCEPT = { accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" };
+// The codings Chromium accepts, so an origin compressing in br or zstd alone answers as it would a browser.
+const ACCEPT_ENCODING = { "accept-encoding": "gzip, deflate, br, zstd" };
+// Pool width: `concurrency` when set, else NUMPROCS when the environment sets it, else the host's parallelism.
+export function width(concurrency = 0) {
+    if (concurrency > 0)
+        return concurrency;
+    const numprocs = Number(process.env.NUMPROCS);
+    return Number.isSafeInteger(numprocs) && numprocs > 0 ? numprocs : availableParallelism();
+}
+// The next body chunk; a read failing short of `Content-Length` names both byte counts.
+async function readChunk(reader, response, bytes) {
+    try {
+        return await reader.read();
+    }
+    catch (error) {
+        const declared = Number(response.headers.get("content-length"));
+        log.debug({ url: response.url, bytes, declared, error: String(error) }, "resource body cut off");
+        throw bytes < declared ? new Error(`${MISMATCH}: body ended at ${bytes} of the ${declared} bytes Content-Length declares`) : error;
+    }
+}
+// Reads at most `max` body bytes, then cancels the rest; `isKept` also returns what it read.
+async function drain(response, max, isKept = false) {
+    let bytes = 0;
+    const chunks = [];
+    const reader = response.body?.getReader();
+    while (reader) {
+        const { done, value } = await readChunk(reader, response, bytes);
+        if (done)
+            break;
+        bytes += value.byteLength;
+        if (isKept)
+            chunks.push(value);
+        if (bytes < max)
+            continue;
+        await reader.cancel();
+        break;
+    }
+    return { bytes, ...(isKept && { body: Buffer.concat(chunks) }) };
+}
+// The media type of a content-type header, lower-cased.
+function mediaType(raw) {
+    return (String(raw ?? "")
+        .split(";", 1)[0]
+        ?.trim()
+        .toLowerCase() ?? "");
+}
+// Extractors that read a 2xx body of `contentType`.
+function readers(extractors, status, contentType) {
+    return status >= 200 && status < 300 ? extractors.filter((extractor) => extractor.types.some((type) => contentType.startsWith(type))) : [];
+}
+// Counts the body, and hands it whole to each extractor that reads its type.
+async function consume(url, response, max, extractors, cache) {
+    const contentType = mediaType(response.headers.get("content-type"));
+    const { bytes, body } = await drain(response, max, readers(extractors, response.status, contentType).length > 0);
+    return extractBody(url, response.status, contentType, bytes, body, max, extractors, cache);
+}
+// Hands a whole body to each extractor that reads its type, through `cache`; a body cut at `max` is read by none.
+async function extractBody(url, status, contentType, bytes, body, max, extractors, cache) {
+    const wanted = readers(extractors, status, contentType);
+    const read = wanted.map((extractor) => extractor.id);
+    if (!body || bytes >= max) {
+        if (wanted.length > 0)
+            log.debug({ url, bytes, max, extractors: read }, "resource body too large to extract");
+        return { bytes, ...(wanted.length > 0 && { read }) };
+    }
+    const started = performance.now();
+    const facts = {};
+    for (const extractor of wanted) {
+        try {
+            const value = await cache.run(extractor, url, contentType, body, () => extractor.extract(url, contentType, body));
+            if (value !== undefined)
+                facts[extractor.id] = value;
+        }
+        catch (error) {
+            log.warn({ url, extractor: extractor.id, error: error instanceof Error ? error.message : String(error) }, `${extractor.id} checks skipped on`);
+        }
+    }
+    const ms = Math.round(performance.now() - started);
+    log.debug({ url, contentType, bytes, extractors: Object.keys(facts), ms }, "resource extracted");
+    return { bytes, read, facts, ms };
+}
+// The failure entry key, beside the URL’s own answer.
+const failureKey = (url) => `failed\t${url}`;
+// A 5xx or no answer at all.
+export function isFailure(status) {
+    return status === 0 || status >= 500;
+}
+// Seconds a `Retry-After` in seconds or as an HTTP date asks for, else `fallback`.
+function retryAfter(raw, fallback) {
+    const value = [raw].flat()[0]?.trim();
+    if (!value)
+        return fallback;
+    const seconds = /^\d+$/.test(value) ? Number(value) : (Date.parse(value) - Date.now()) / 1000;
+    return Number.isFinite(seconds) ? Math.max(0, Math.round(seconds)) : fallback;
+}
+// The stored failure for `url` while it is younger than its `until`, or any under `--offline`.
+async function storedFailure(url, failures) {
+    const entry = await failures.get(failureKey(url));
+    const isFresh = entry !== undefined && (failures.mode === "offline" || Date.now() < Date.parse(entry.value.until));
+    log.debug({ url, status: entry?.value.status, until: entry?.value.until, isFresh }, "resource failure looked up");
+    if (!entry || !isFresh)
+        return undefined;
+    const { status, error } = entry.value;
+    return { status, headers: {}, size: { body: 0 }, timing: {}, ...(error && { error }), cached: true };
+}
+// Keeps a failed `result` for its `Retry-After`, else `ttl` seconds.
+async function storeFailure(url, result, failures, ttl) {
+    const seconds = retryAfter(result.headers["retry-after"], ttl);
+    const failure = { status: result.status, ...(result.error && { error: result.error }), attempts: attemptsFor(result.status), until: new Date(Date.now() + seconds * 1000).toISOString() };
+    log.debug({ url, ...failure, seconds }, "resource failure stored");
+    await failures.set(failureKey(url), failure);
+}
+// A stored answer every extractor now reading its type has already read.
+function isServed(stored, extractors) {
+    const read = new Set(stored.value.read);
+    return readers(extractors, stored.status, mediaType(stored.headers["content-type"])).every((extractor) => read.has(extractor.id));
+}
+// A GET through the stored failure first; a fresh 5xx or unreachable answer is served, a new one stored.
+async function fetchRemembered(url, config, bucket, failures, extractors, cache, sent) {
+    const stored = await storedFailure(url, failures);
+    if (stored)
+        return stored;
+    const result = await fetchOne(url, config.maxBodySize, bucket, extractors, cache, sent);
+    if (isFailure(result.status))
+        await storeFailure(url, result, failures, config.cacheFailureTtl);
+    return result;
+}
+// One HEAD through the stored failure first; a fresh 5xx or unreachable answer is served, a new one stored.
+async function headRemembered(url, bucket, failures, ttl, sent) {
+    const stored = await storedFailure(url, failures);
+    if (stored)
+        return stored;
+    try {
+        const { status, headers, ms, cached, revalidated } = await fetchHeadCached(bucket, url, sent);
+        log.debug({ url, status, cached, revalidated }, "enclosure checked");
+        const contentType = mediaType(headers["content-type"]);
+        const cookies = cookieFacts(headers["set-cookie"], headers.date);
+        const result = { status, headers: redactHeaders(headers), ...(contentType && { "content-type": contentType }), size: { body: 0 }, timing: { total: ms }, ...(cookies.length > 0 && { cookies }), ...(cached && { cached }), ...(revalidated && { revalidated }) };
+        if (isFailure(result.status))
+            await storeFailure(url, result, failures, ttl);
+        return result;
+    }
+    catch (error) {
+        if (error instanceof OfflineMiss)
+            throw error;
+        return { status: 0, headers: {}, size: { body: 0 }, timing: {}, error: reason(error) };
+    }
+}
+// One cached or retried GET with `sent` headers; a final failure is status 0 with its error.
+async function fetchOne(url, max, bucket, extractors, cache, sent) {
+    try {
+        const { status, headers, value, ms, cached, revalidated } = await fetchCached(bucket, url, (response) => consume(url, response, max, extractors, cache), false, (stored) => isServed(stored, extractors), sent);
+        const { bytes, facts } = value;
+        log.debug({ url, status, bytes, cached, revalidated }, "resource fetched");
+        const contentType = mediaType(headers["content-type"]);
+        const hasFacts = facts !== undefined && Object.keys(facts).length > 0;
+        const cookies = cookieFacts(headers["set-cookie"], headers.date);
+        return { status, headers: redactHeaders(headers), ...(contentType && { "content-type": contentType }), size: { body: bytes }, timing: { total: ms - (value.ms ?? 0) }, ...(cookies.length > 0 && { cookies }), ...(cached && { cached }), ...(revalidated && { revalidated }), ...(hasFacts && { facts }) };
+    }
+    catch (error) {
+        if (error instanceof OfflineMiss)
+            throw error;
+        return { status: 0, headers: {}, size: { body: 0 }, timing: {}, error: reason(error) };
+    }
+}
+// A logged answer that stands for a fetch: not a failure, nor a status a fetch would retry.
+function isUsable(logged) {
+    return logged !== undefined && logged.status >= 200 && logged.status < 500 && logged.status !== 429;
+}
+// A result from the browser’s network log, read by the same extractors a fetch would feed.
+async function fromLog(url, logged, max, extractors, cache) {
+    const contentType = mediaType(logged.headers["content-type"]);
+    const { facts } = await extractBody(url, logged.status, contentType, logged.bytes, logged.body, max, extractors, cache);
+    const hasFacts = facts !== undefined && Object.keys(facts).length > 0;
+    const cookies = cookieFacts(logged.headers["set-cookie"], logged.headers.date);
+    log.debug({ url, status: logged.status, bytes: logged.bytes, hasBody: logged.body !== undefined, cookies: cookies.length }, "resource answered from the browser log");
+    return { status: logged.status, headers: redactHeaders(logged.headers), ...(contentType && { "content-type": contentType }), size: { body: logged.bytes }, timing: { ...(logged.ms !== undefined && { total: logged.ms }) }, ...(cookies.length > 0 && { cookies }), logged: true, ...(hasFacts && { facts }) };
+}
+// Hangs each fetched result off every page entry that names its URL.
+export function attachResources(pages, results) {
+    const entries = pages.flatMap((page) => page.resources ?? []);
+    for (const entry of entries) {
+        const { facts, ...http } = results[entry.url] ?? {};
+        entry.http = results[entry.url] && http;
+        Object.assign(entry, facts);
+    }
+}
+// GETs every distinct resource URL the pages name that `logged` lacks, once each, with `extractors` reading the bodies of their types through `cache`.
+export async function fetchResources(pages, config, bucket, extractors = [], logged = new Map(), cache = new ExtractorCache(undefined)) {
+    const entries = pages.flatMap((page) => page.resources ?? []);
+    const urls = [...new Set(entries.map((entry) => entry.url))];
+    const gotten = new Set(entries.filter((entry) => entry.kind !== "enclosure").map((entry) => entry.url));
+    const headed = new Set(entries.filter((entry) => entry.kind === "enclosure" && !gotten.has(entry.url)).map((entry) => entry.url));
+    const images = new Set(entries.filter((entry) => entry.kind === "image").map((entry) => entry.url));
+    log.debug({ resources: urls.length, references: entries.length, images: images.size, headed: headed.size, fetch: config.fetchResources }, "resources found");
+    if (!config.fetchResources || urls.length === 0)
+        return {};
+    progressPhase("resources", urls.length);
+    const results = new Map();
+    const failures = new Bucket(bucket.name, bucket.directory, bucket.ttlSeconds, bucket.mode);
+    const queue = urls.values();
+    const worker = async () => {
+        for (const url of queue) {
+            const answer = logged.get(url);
+            results.set(url, headed.has(url) && !isUsable(answer)
+                ? await headRemembered(url, bucket, failures, config.cacheFailureTtl, ACCEPT_ENCODING)
+                : isUsable(answer)
+                    ? await fromLog(url, answer, config.maxBodySize, extractors, cache)
+                    : await fetchRemembered(url, config, bucket, failures, extractors, cache, { ...ACCEPT_ENCODING, ...(images.has(url) && IMAGE_ACCEPT) }));
+            progressCount(results.size);
+        }
+    };
+    const workers = Array.from({ length: Math.min(width(config.concurrency), urls.length) }, worker);
+    await Promise.all(workers);
+    const all = results.values().toArray();
+    const [failed, cached, revalidated, fromBrowser] = [all.filter((result) => result.status === 0).length, all.filter((result) => result.cached).length, all.filter((result) => result.revalidated).length, all.filter((result) => result.logged).length];
+    const failedCached = all.filter((result) => result.cached && isFailure(result.status)).length;
+    log.debug({ resources: urls.length, failed, cached, failedCached, revalidated, logged: fromBrowser }, "resources fetched");
+    return Object.fromEntries(results);
+}

package/dist/crawl/robots.js

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { RobotsTxtFile } from "crawlee";
+import { fetchCached } from "../cache/http.js";
+import { OfflineMiss } from "../cache/index.js";
+import { log } from "../logger.js";
+import { reason } from "./fetch.js";
+const DISALLOW_ALL = "User-agent: *\nDisallow: /\n";
+// Parsed facts per file; Crawlee’s own parser is private, so the body is parsed a second time.
+const parsed = new WeakMap();
+// `key=value` pairs of a `Content-Signal` line, as written.
+function signalsOf(value) {
+    const pairs = value.split(",").map((pair) => pair.split("=", 2).map((part) => part.trim()));
+    return Object.fromEntries(pairs.filter(([key]) => key).map(([key = "", signal = ""]) => [key.toLowerCase(), signal.toLowerCase()]));
+}
+// RFC 9309 groups, `Sitemap:` lines and Content Signals; consecutive `User-agent` lines share one group, a rule before any is dropped.
+export function parseRobots(url, status, body) {
+    const groups = [];
+    const sitemaps = [];
+    const contentSignals = [];
+    let group;
+    let isOpen = false;
+    for (const raw of body.split(/\r\n|\r|\n/)) {
+        const line = raw.replace(/#.*/, "").trim();
+        const colon = line.indexOf(":");
+        if (colon < 1)
+            continue;
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === "user-agent") {
+            if (!isOpen || !group)
+                groups.push((group = { agents: [], allow: [], disallow: [] }));
+            group.agents.push(value.toLowerCase());
+            isOpen = true;
+            continue;
+        }
+        isOpen = false;
+        if (key === "sitemap")
+            sitemaps.push(value);
+        else if (key === "content-signal")
+            contentSignals.push({ agents: group?.agents ?? [], value, signals: signalsOf(value) });
+        else if (!group)
+            log.debug({ url, key }, "robots.txt rule outside a group dropped");
+        else if (value && (key === "allow" || key === "disallow"))
+            group[key].push(value);
+        else if (key === "crawl-delay" && Number.isFinite(Number(value)))
+            group["crawl-delay"] = Number(value);
+    }
+    log.debug({ url, status, groups: groups.length, sitemaps: sitemaps.length, contentSignals: contentSignals.length }, "robots.txt parsed");
+    return { url, status, groups, sitemaps, "content-signals": contentSignals };
+}
+// The groups naming `agent`, else those naming `*`, as RFC 9309 §2.2.1 picks them.
+export function groupsFor(facts, agent) {
+    const named = facts.groups.filter((group) => group.agents.includes(agent));
+    return named.length > 0 ? named : facts.groups.filter((group) => group.agents.includes("*"));
+}
+// The `Crawl-delay` the spiderlint group, else the `*` group, asks for; 0 when none.
+export function crawlDelayOf(file) {
+    const facts = parsed.get(file);
+    return (facts && groupsFor(facts, "spiderlint").find((group) => group["crawl-delay"] !== undefined)?.["crawl-delay"]) ?? 0;
+}
+// The file’s facts; a file this module did not load has none.
+export function robotsFactsOf(file) {
+    return parsed.get(file);
+}
+// RFC 9309 §2.3.1: a 2xx file applies, a 4xx allows everything, a 5xx or no answer disallows everything.
+async function load(origin, bucket) {
+    const url = `${origin}/robots.txt`;
+    let content;
+    let facts;
+    try {
+        const { status, value, cached, revalidated } = await fetchCached(bucket, url, (response) => response.text(), true);
+        const isApplied = status >= 200 && status < 300;
+        content = isApplied ? value : status >= 400 && status < 500 ? "" : DISALLOW_ALL;
+        facts = parseRobots(url, status, isApplied ? value : "");
+        log.debug({ url, status, cached, revalidated, isDisallowAll: content === DISALLOW_ALL }, "robots.txt read");
+    }
+    catch (error) {
+        if (error instanceof OfflineMiss)
+            throw error;
+        content = DISALLOW_ALL;
+        facts = { ...parseRobots(url, 0, ""), error: reason(error) };
+        log.warn({ url, error: reason(error) }, "robots.txt unreachable, disallowing the origin");
+    }
+    const file = RobotsTxtFile.from(url, content);
+    parsed.set(file, facts);
+    log.debug({ url, delay: crawlDelayOf(file) }, "robots.txt crawl-delay read");
+    return file;
+}
+// One robots.txt per origin for the run, read through `bucket`.
+export function robotsLoader(bucket) {
+    const byOrigin = new Map();
+    return (url) => {
+        const { origin } = new URL(url);
+        if (!byOrigin.has(origin))
+            byOrigin.set(origin, load(origin, bucket));
+        return byOrigin.get(origin);
+    };
+}

package/dist/crawl/route.js

@@ -0,0 +1,92 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { setTimeout as sleep } from "node:timers/promises";
+import { assignGroup, compileGroups } from "../groups/assign.js";
+import { log } from "../logger.js";
+// Pages of an adaptive group rendered and compared with their static HTML before it settles on http.
+const DETECTIONS = 3;
+// Milliseconds a page of an undecided group waits for the verdict before the group settles on the browser.
+const VERDICT_MS = 60_000;
+// The html facts a client-side render can change and a rule or the crawl reads.
+function rendered(html) {
+    const { title, lang, canonical, h1 = [], meta = {}, property = {}, links } = html ?? {};
+    return JSON.stringify({ title, lang, canonical, h1, meta, property, internal: links?.internal.toSorted((a, b) => a.localeCompare(b)) });
+}
+// Routes each URL to the crawler its group needs; an adaptive group renders until its pages read the same without a browser.
+export class Router {
+    #groups;
+    #modes;
+    #agreed = new Map();
+    #taken = new Map();
+    #verdicts = new Map();
+    constructor(groups, modes) {
+        this.#groups = compileGroups(groups);
+        this.#modes = new Map(Object.entries(modes));
+    }
+    // The group a URL falls in before its response is known, so `content-type:` entries never match.
+    #groupOf(href) {
+        const url = new URL(href);
+        return assignGroup({ url: { href, pathname: url.pathname, search: url.search }, http: { headers: {} } }, this.#groups);
+    }
+    // A group’s mode, http for one the config never named.
+    #modeOf(group) {
+        return this.#modes.get(group) ?? "http";
+    }
+    // Resolves once `group` settles.
+    #verdict(group) {
+        const known = this.#verdicts.get(group) ?? Promise.withResolvers();
+        this.#verdicts.set(group, known);
+        return known;
+    }
+    // Fixes `group` on `mode` and wakes every page waiting for its verdict.
+    #settle(group, mode, agreed) {
+        this.#modes.set(group, mode);
+        this.#verdict(group).resolve();
+        log.debug({ group, fetch: mode, agreed }, "adaptive group settled");
+    }
+    // Crawlers the run needs: an adaptive group needs both.
+    get crawlers() {
+        const modes = new Set(this.#modes.values());
+        return ["http", "browser"].filter((mode) => modes.has(mode) || modes.has("adaptive"));
+    }
+    // Each group’s mode, adaptive ones as settled so far.
+    get modes() {
+        return Object.fromEntries(this.#modes);
+    }
+    // The queue `href` joins: the browser’s for a group that renders, else http’s, where an adaptive group is decided per request.
+    queue(href) {
+        return this.#modeOf(this.#groupOf(href)) === "browser" ? "browser" : "http";
+    }
+    // Whether the http crawler hands `href` to the browser: its group renders, or is undecided and has a detection slot; past the slots it waits for the verdict.
+    async handOff(href) {
+        const group = this.#groupOf(href);
+        const taken = this.#taken.get(group) ?? 0;
+        const isSlot = this.#modeOf(group) === "adaptive" && taken < DETECTIONS;
+        if (isSlot)
+            this.#taken.set(group, taken + 1);
+        if (!isSlot && this.#modeOf(group) === "adaptive")
+            await Promise.race([this.#verdict(group).promise, sleep(VERDICT_MS, undefined, { ref: false })]);
+        if (!isSlot && this.#modeOf(group) === "adaptive")
+            this.#settle(group, "browser", this.#agreed.get(group) ?? 0);
+        log.debug({ url: href, group, mode: this.#modeOf(group), taken, isSlot }, "hand-off decided");
+        return isSlot || this.#modeOf(group) !== "http";
+    }
+    // Whether a rendered page is to be compared with its static HTML, its group being still undecided.
+    isDetecting(href) {
+        return this.#modeOf(this.#groupOf(href)) === "adaptive";
+    }
+    // One difference settles the group on the browser, DETECTIONS agreements on http; a page without html facts agrees.
+    detected(href, stat, live) {
+        const group = this.#groupOf(href);
+        if (this.#modeOf(group) !== "adaptive")
+            return;
+        const isSame = rendered(stat) === rendered(live);
+        const agreed = (this.#agreed.get(group) ?? 0) + (isSame ? 1 : 0);
+        this.#agreed.set(group, agreed);
+        const settled = isSame ? (agreed >= DETECTIONS ? "http" : undefined) : "browser";
+        log.debug({ url: href, group, isSame, agreed, settled }, "rendering compared");
+        if (settled)
+            this.#settle(group, settled, agreed);
+    }
+}

package/dist/crawl/scope.js

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { getDomain } from "tldts";
+// Crawlee enqueue strategy carrying the same meaning as each scope.
+export const STRATEGY = {
+    origin: "same-origin",
+    host: "same-hostname",
+    domain: "same-domain",
+};
+// Same-domain compares registrable domains, as Crawlee's strategy does.
+export function isInScope(link, page, scope) {
+    if (scope === "origin")
+        return link.origin === page.origin;
+    return scope === "host" ? link.hostname === page.hostname : (getDomain(link.hostname, { mixedInputs: false }) ?? link.hostname) === (getDomain(page.hostname, { mixedInputs: false }) ?? page.hostname);
+}
+// The one origin every URL shares, or empty when they span several.
+export function singleOrigin(urls) {
+    const origins = new Set([...urls].flatMap((href) => (URL.canParse(href) ? [new URL(href).origin] : [])));
+    return origins.size === 1 ? [...origins][0] : "";
+}
+// `href` under `from` moved onto `to`, path and query kept; any other URL passes through.
+export function onOrigin(href, from, to) {
+    if (!from || from === to || !URL.canParse(href))
+        return href;
+    const url = new URL(href);
+    return url.origin === from ? new URL(`${url.pathname}${url.search}${url.hash}`, to).href : href;
+}
+// A URL under `origin` becomes its path; any other passes through.
+export function relative(href, origin) {
+    return origin && href.startsWith(`${origin}/`) ? href.slice(origin.length) : href;
+}

package/dist/crawl/scratch.js

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import path from "node:path";
+import { Configuration } from "crawlee";
+import { userStateDirectory } from "../cache/index.js";
+import { log } from "../logger.js";
+// Crawlee’s global storage is scratch: held in memory, rooted in the XDG state directory should it spill.
+export function useScratchStorage() {
+    const directory = path.join(userStateDirectory(), "crawlee");
+    const global = Configuration.getGlobalConfig();
+    global.set("persistStorage", false);
+    global.set("storageClientOptions", { localDataDirectory: directory });
+    log.debug({ directory }, "crawlee scratch storage set");
+}

package/dist/crawl/sitemap.js

@@ -0,0 +1,187 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { gunzipSync } from "node:zlib";
+import { load } from "cheerio";
+import { parseSitemap } from "crawlee";
+import { fetchCached } from "../cache/http.js";
+import { OfflineMiss } from "../cache/index.js";
+import { log } from "../logger.js";
+import { reason } from "./fetch.js";
+import { onOrigin } from "./scope.js";
+// sitemaps.org caps one file at 50 MiB uncompressed.
+const MAX_BYTES = 50 * 1024 * 1024;
+const GZIP = Buffer.from([0x1f, 0x8b]);
+// Probed when no seed names a sitemap itself, as Crawlee’s discovery does.
+const CANDIDATES = ["/sitemap.xml", "/sitemap.txt", "/sitemap_index.xml"];
+const SITEMAP_NAME = /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i;
+// The body as base64, so gzip bytes survive the JSON bucket.
+async function base64(response) {
+    return Buffer.from(await response.arrayBuffer()).toString("base64");
+}
+// `bucket` behind a per-run memo, so a probed candidate is not asked again when read.
+function fetchOnce(bucket) {
+    const answers = new Map();
+    return (url) => {
+        const known = answers.get(url);
+        log.debug({ url, isKnown: known !== undefined }, "sitemap fetch looked up");
+        if (known)
+            return known;
+        const answer = fetchCached(bucket, url, base64);
+        answers.set(url, answer);
+        return answer;
+    };
+}
+// Body text, gunzipped when the bytes carry the gzip magic.
+function decode(body) {
+    return (body.subarray(0, 2).equals(GZIP) ? gunzipSync(body, { maxOutputLength: MAX_BYTES }) : body).toString("utf8");
+}
+// A text/plain or `.txt` file lists one URL per line; anything else parses as XML, nested sitemaps as `originSitemapUrl: null`.
+async function* entries(url, contentType, content) {
+    const isText = contentType === "text/plain" || new URL(url).pathname.replace(/\.gz$/, "").endsWith(".txt");
+    if (!isText)
+        return yield* parseSitemap([{ type: "raw", content }], undefined, { maxDepth: 0, emitNestedSitemaps: true });
+    for (const line of content.split(/\r?\n/)) {
+        const loc = line.trim();
+        if (URL.canParse(loc))
+            yield { loc, originSitemapUrl: url };
+    }
+}
+// The image and video files a page’s sitemap entry lists.
+export function mediaOf(page) {
+    return [...(page.sitemap?.images ?? []), ...(page.sitemap?.videos ?? [])];
+}
+// Alternates, images and videos per `<loc>`, parsed only when the file declares one of those namespaces.
+function extensions(content) {
+    const found = new Map();
+    if (!/<(?:xhtml|image|video):/.test(content))
+        return found;
+    const $ = load(content, { xml: true });
+    const texts = (selection) => selection
+        .map((_, element) => $(element).text().trim())
+        .get()
+        .filter((text) => text.length > 0);
+    for (const element of $("url")) {
+        const url = $(element);
+        const alternates = url
+            .children(String.raw `xhtml\:link[rel='alternate'][hreflang][href]`)
+            .map((_, link) => ({ lang: String($(link).attr("hreflang")), href: String($(link).attr("href")) }))
+            .get();
+        const images = texts(url.find(String.raw `image\:image > image\:loc`));
+        const videos = texts(url.find(String.raw `video\:video > video\:content_loc, video\:video > video\:player_loc`));
+        found.set(url.children("loc").text().trim(), { ...(alternates.length > 0 && { alternates }), ...(images.length > 0 && { images }), ...(videos.length > 0 && { videos }) });
+    }
+    log.debug({ entries: found.size }, "sitemap extensions parsed");
+    return found;
+}
+// Parses one fetched body into `file` counts: page entries land in `index`, same-host nested sitemaps in `queue`; `canonical` URLs move onto the file’s origin.
+async function collect(file, contentType, body, index, queue, canonical) {
+    const content = decode(body);
+    const extended = extensions(content);
+    const origin = new URL(file.url).origin;
+    const found = entries(file.url, contentType, content);
+    for await (const listed of found) {
+        const entry = { ...listed, loc: onOrigin(listed.loc, canonical, origin) };
+        if (entry.originSitemapUrl === null) {
+            file.sitemaps += 1;
+            const isSameHost = new URL(entry.loc).hostname === new URL(file.url).hostname;
+            log[isSameHost ? "debug" : "info"]({ sitemap: file.url, nested: entry.loc, isSameHost }, "nested sitemap found");
+            if (isSameHost)
+                queue.push(entry.loc);
+            continue;
+        }
+        file.urls += 1;
+        const { alternates, images, videos } = extended.get(listed.loc) ?? {};
+        index.set(entry.loc, {
+            listed: true,
+            ...(entry.lastmod && { lastmod: entry.lastmod.toISOString() }),
+            ...(entry.changefreq && { changefreq: entry.changefreq }),
+            ...(entry.priority !== undefined && { priority: entry.priority }),
+            ...(alternates && { alternates }),
+            ...(images && { images: images.map((href) => onOrigin(href, canonical, origin)) }),
+            ...(videos && { videos: videos.map((href) => onOrigin(href, canonical, origin)) }),
+        });
+    }
+}
+// Fetches and parses one file into its facts; `error` says which step failed.
+async function readSitemap(url, index, queue, fetch, canonical) {
+    const file = { url, status: 0, urls: 0, sitemaps: 0 };
+    try {
+        const { status, headers, value } = await fetch(url);
+        file.status = status;
+        const contentType = String(headers["content-type"] ?? "")
+            .split(";", 1)[0]
+            ?.trim() || undefined;
+        if (status < 200 || status >= 300)
+            file.error = `answers ${status}`;
+        else if (contentType === "text/html")
+            file.error = `is ${contentType}, not a sitemap`;
+        else
+            await collect(file, contentType, Buffer.from(value, "base64"), index, queue, canonical);
+        if (!file.error && file.urls + file.sitemaps === 0)
+            file.error = `names no URL (${contentType ?? "no content-type"})`;
+    }
+    catch (error) {
+        if (error instanceof OfflineMiss)
+            throw error;
+        file.error = `${file.status > 0 ? "does not parse" : "does not fetch"}: ${reason(error).split("\n", 1)[0]}`;
+    }
+    log[file.error ? "warn" : "debug"](file, "sitemap read");
+    return file;
+}
+// A candidate name counts when it answers 2xx; its body stays in `bucket` for the read that follows, and a miss younger than the bucket TTL is not asked again.
+async function isAnswering(url, bucket, fetch) {
+    const stored = await bucket.get(url);
+    const isFreshMiss = stored !== undefined && (stored.value.status < 200 || stored.value.status >= 300) && bucket.isFresh(stored);
+    log.debug({ url, status: stored?.value.status, isFreshMiss }, "sitemap candidate looked up");
+    if (isFreshMiss)
+        return false;
+    try {
+        const { status } = await fetch(url);
+        log.debug({ url, status }, "sitemap candidate probed");
+        return status >= 200 && status < 300;
+    }
+    catch (error) {
+        if (error instanceof OfflineMiss)
+            throw error;
+        log.debug({ url, error: reason(error) }, "sitemap candidate unreachable");
+        return false;
+    }
+}
+// Per origin: its robots.txt `Sitemap:` lines, `canonical` ones read from this origin, then the seeds naming a sitemap, else the common names that answer.
+async function discover(seeds, robotsFor, bucket, fetch, canonical) {
+    const found = new Set();
+    const origins = new Set(seeds.map((seed) => new URL(seed).origin));
+    for (const origin of origins) {
+        const robots = await robotsFor(origin);
+        const listed = robots.getSitemaps({ enqueueStrategy: "all" }).map((url) => onOrigin(url, canonical, origin));
+        for (const url of listed)
+            found.add(url);
+        const named = seeds.filter((seed) => new URL(seed).origin === origin && SITEMAP_NAME.test(seed));
+        const candidates = named.length > 0 ? [] : CANDIDATES.map((pathname) => new URL(pathname, origin).href);
+        for (const url of named)
+            found.add(url);
+        for (const url of candidates)
+            if (await isAnswering(url, bucket, fetch))
+                found.add(url);
+        log.debug({ origin, canonical, listed, found: found.size, named: named.length, probed: candidates.length }, "sitemaps discovered for origin");
+    }
+    return [...found];
+}
+// robots.txt `Sitemap:` lines plus the common `/sitemap.xml` names, each file and its nested files read once.
+export async function loadSitemap(seeds, robotsFor, bucket, canonical) {
+    const index = new Map();
+    const fetch = fetchOnce(bucket);
+    const queue = await discover(seeds, robotsFor, bucket, fetch, canonical);
+    log.debug({ seeds, files: queue }, queue.length > 0 ? "sitemap discovered" : "no sitemap discovered");
+    const seen = new Set();
+    const files = [];
+    for (const url of queue) {
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        files.push(await readSitemap(url, index, queue, fetch, canonical));
+    }
+    log.debug({ files: files.length, failed: files.filter((file) => file.error).length, urls: index.size }, "sitemap parsed");
+    return { index, files };
+}