spiderlint 0.0.0 → 0.143.1

package/dist/cache/http.js

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import CachePolicy from "http-cache-semantics";
+import { agentHeaders } from "../agent.js";
+import { fetchRetrying } from "../crawl/fetch.js";
+import { redactHeaders } from "../facts/transport.js";
+import { log } from "../logger.js";
+// A private cache, and no heuristic freshness: an origin that says nothing gets the bucket TTL instead.
+const OPTIONS = { shared: false, cacheHeuristic: 0 };
+const REQUEST = (url, headers = {}) => ({ url, method: "GET", headers: { ...headers, ...agentHeaders() } });
+// No `Cache-Control` and no `Expires`: the origin states no freshness.
+function isSilent(headers) {
+    return headers["cache-control"] === undefined && headers.expires === undefined;
+}
+// Policy, headers and value as written to the bucket, with secrets redacted.
+function toStored(policy, status, headers, value, ms) {
+    const object = policy.toObject();
+    return { policy: { ...object, resh: redactHeaders(object.resh) }, status, headers: redactHeaders(headers), value, ms };
+}
+// Fresh by the origin’s headers, or by the bucket TTL when they are silent or `isCapped`; everything is fresh offline.
+function isFresh(bucket, entry, policy, isCapped, request) {
+    if (bucket.mode === "offline")
+        return true;
+    return isSilent(entry.value.headers) ? bucket.isFresh(entry) : policy.satisfiesWithoutRevalidation(request) && (!isCapped || bucket.isFresh(entry));
+}
+// A GET with `sent` headers answered from `bucket` while fresh, revalidated with its validators once stale, stored when RFC 9111 allows; an entry `isUsable` rejects is fetched again in full, except offline.
+export async function fetchCached(bucket, url, consume, isCapped = false, isUsable, sent = {}) {
+    const found = await bucket.get(url);
+    const entry = found && (!isUsable || bucket.mode === "offline" || isUsable(found.value)) ? found : undefined;
+    if (found && !entry)
+        log.debug({ bucket: bucket.name, url, stored: found.stored }, "cache entry lacks what the caller needs");
+    if (!entry)
+        bucket.missed(url);
+    const request = REQUEST(url, sent);
+    const policy = entry && CachePolicy.fromObject(entry.value.policy);
+    if (entry && policy && isFresh(bucket, entry, policy, isCapped, request)) {
+        log.debug({ bucket: bucket.name, url, stored: entry.stored }, "served from cache");
+        return { ...entry.value, cached: true, at: entry.stored };
+    }
+    const conditional = policy ? policy.revalidationHeaders(request) : {};
+    const { response, value, ms } = await fetchRetrying(url, consume, { ...sent, ...conditional });
+    const headers = Object.fromEntries(response.headers);
+    if (entry && policy && response.status === 304) {
+        const { policy: updated, modified } = policy.revalidatedPolicy(request, { status: 304, headers });
+        log.debug({ bucket: bucket.name, url, modified }, "revalidated");
+        if (!modified) {
+            const next = toStored(updated, entry.value.status, { ...entry.value.headers, ...headers }, entry.value.value, ms);
+            await bucket.set(url, next);
+            return { ...next, revalidated: true, at: new Date().toISOString() };
+        }
+        const again = await fetchRetrying(url, consume, sent);
+        return store(bucket, url, request, again.response, again.value, again.ms);
+    }
+    return store(bucket, url, request, response, value, ms);
+}
+// A HEAD with `sent` headers answered from `bucket` while fresh, revalidated once stale, stored when RFC 9111 allows; the value keeps headers only, never a body.
+export async function fetchHeadCached(bucket, url, sent = {}) {
+    const found = await bucket.get(url);
+    const entry = found && found.value.value.head === true ? found : undefined;
+    if (found && !entry)
+        log.debug({ bucket: bucket.name, url, stored: found.stored }, "cache entry holds a body, not a HEAD answer");
+    if (!entry)
+        bucket.missed(url);
+    const request = REQUEST(url, sent);
+    const policy = entry && CachePolicy.fromObject(entry.value.policy);
+    if (entry && policy && isFresh(bucket, entry, policy, false, request)) {
+        log.debug({ bucket: bucket.name, url, stored: entry.stored }, "served from cache");
+        return { ...entry.value, cached: true, at: entry.stored };
+    }
+    const conditional = policy ? policy.revalidationHeaders(request) : {};
+    const { response, ms } = await fetchRetrying(url, async (answer) => {
+        await answer.body?.cancel();
+        return {};
+    }, { ...sent, ...conditional }, "HEAD");
+    const headers = Object.fromEntries(response.headers);
+    if (entry && policy && response.status === 304) {
+        const { policy: updated, modified } = policy.revalidatedPolicy(request, { status: 304, headers });
+        log.debug({ bucket: bucket.name, url, modified }, "revalidated");
+        if (!modified) {
+            const next = toStored(updated, entry.value.status, { ...entry.value.headers, ...headers }, entry.value.value, ms);
+            await bucket.set(url, next);
+            return { ...next, revalidated: true, at: new Date().toISOString() };
+        }
+    }
+    const value = { bytes: 0, head: true };
+    return store(bucket, url, request, response, value, ms);
+}
+// A full response, kept when its policy is storable.
+async function store(bucket, url, request, response, value, ms) {
+    const headers = Object.fromEntries(response.headers);
+    const policy = new CachePolicy(request, { status: response.status, headers }, OPTIONS);
+    const stored = toStored(policy, response.status, headers, value, ms);
+    const isStorable = policy.storable();
+    log.debug({ bucket: bucket.name, url, status: response.status, isStorable }, "response judged for cache");
+    if (isStorable)
+        await bucket.set(url, stored);
+    return { ...stored, at: new Date().toISOString() };
+}

package/dist/cache/index.js

@@ -0,0 +1,101 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import path from "node:path";
+import { log } from "../logger.js";
+// Seconds each bucket stays fresh when the origin says nothing; `pages` 0 lets origin headers alone decide, `extractors` 0 keeps an entry until its body changes.
+export const TTL_DEFAULTS = { pages: 0, probes: 7 * 86_400, profiles: 3600, lists: 7 * 86_400, resources: 86_400, robots: 86_400, sitemaps: 86_400, origins: 86_400, dns: 60, extractors: 0 };
+// Buckets beside the project; the rest hold third-party observations shared by every site on the machine.
+const PROJECT = new Set(["pages", "resources", "sitemaps", "origins", "dns", "extractors"]);
+// `--offline` found nothing stored for a key; the run exits 3.
+export class OfflineMiss extends Error {
+}
+// Write to a unique sibling temp file, then rename over the target.
+export async function writeAtomic(file, content) {
+    const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
+    await writeFile(temporary, content);
+    await rename(temporary, file);
+}
+// `$XDG_CACHE_HOME/spiderlint`, else `~/.cache/spiderlint`.
+export function userCacheDirectory() {
+    return path.join(process.env.XDG_CACHE_HOME || path.join(homedir(), ".cache"), "spiderlint");
+}
+// `$XDG_STATE_HOME/spiderlint`, else `~/.local/state/spiderlint`.
+export function userStateDirectory() {
+    return path.join(process.env.XDG_STATE_HOME || path.join(homedir(), ".local", "state"), "spiderlint");
+}
+// `<user cache>/<host>` for the seeds’ hosts, `+`-joined when they span several; undefined without a seed.
+export function siteDirectory(seeds) {
+    const hosts = [...new Set(seeds.filter((seed) => URL.canParse(seed)).map((seed) => new URL(seed).host))].toSorted((a, b) => a.localeCompare(b));
+    return hosts.length === 0 ? undefined : path.join(userCacheDirectory(), hosts.join("+"));
+}
+// Where `name` lives: the store's `cache/` for project buckets, the user cache otherwise; none without a store.
+export function bucketDirectory(name, store) {
+    if (!PROJECT.has(name))
+        return path.join(userCacheDirectory(), name);
+    return store === undefined ? undefined : path.join(store, "cache", name);
+}
+// One JSON file per key, named by the key's hash; `off` neither reads nor writes, `refresh` only writes.
+export class Bucket {
+    name;
+    directory;
+    ttlSeconds;
+    mode;
+    constructor(name, directory, ttlSeconds, mode) {
+        this.name = name;
+        this.directory = directory;
+        this.ttlSeconds = ttlSeconds;
+        this.mode = mode;
+    }
+    #file(key) {
+        return path.join(this.directory, `${createHash("sha256").update(key).digest("hex")}.json`);
+    }
+    // The stored entry for `key`, however old.
+    async get(key) {
+        if (this.directory === undefined || this.mode === "off" || this.mode === "refresh")
+            return undefined;
+        let entry;
+        try {
+            entry = JSON.parse(await readFile(this.#file(key), "utf8"));
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                log.warn({ bucket: this.name, key, error: error instanceof Error ? error.message : String(error) }, `${this.name} cache entry unreadable and ignored:`);
+            log.debug({ bucket: this.name, key }, "cache miss");
+            return undefined;
+        }
+        if (entry.key !== key)
+            return undefined;
+        log.debug({ bucket: this.name, key, stored: entry.stored }, "cache hit");
+        return entry;
+    }
+    // Younger than the bucket TTL.
+    isFresh(entry) {
+        return Date.now() - Date.parse(entry.stored) < this.ttlSeconds * 1000;
+    }
+    async set(key, value) {
+        if (this.directory === undefined || this.mode === "off" || this.mode === "offline")
+            return;
+        await mkdir(this.directory, { recursive: true });
+        const entry = { key, stored: new Date().toISOString(), value };
+        await writeAtomic(this.#file(key), JSON.stringify(entry));
+        log.debug({ bucket: this.name, key }, "cache stored");
+    }
+    // A miss under `--offline` ends the run.
+    missed(key) {
+        if (this.mode === "offline")
+            throw new OfflineMiss(`--offline: ${this.name} holds nothing for ${key}`);
+    }
+}
+// The bucket `name` for this run, located and timed from the config.
+export function openBucket(name, config, store) {
+    return new Bucket(name, bucketDirectory(name, store), config.cacheTtl[name] ?? TTL_DEFAULTS[name], config.cacheMode);
+}
+// `90`, `45s`, `30m`, `24h`, `7d` to seconds.
+export function parseDuration(raw) {
+    const match = /^(\d+)([smhd]?)$/.exec(String(raw).trim());
+    return match ? Number(match[1]) * { "": 1, s: 1, m: 60, h: 3600, d: 86_400 }[match[2]] : undefined;
+}

package/dist/cache/purge.js

@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { existsSync } from "node:fs";
+import { readFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { lockStore } from "../store/disk.js";
+import { log } from "../logger.js";
+import { bucketDirectory } from "./index.js";
+import { FILE_BUCKETS, files } from "./status.js";
+// Store directories that together make up the `pages` bucket.
+const PAGES = ["datasets", "key_value_stores", "request_queues", "cache/pages", "manifest.json"];
+export const PURGEABLE = new Set(["pages", ...FILE_BUCKETS]);
+// Deletes entries older than `olderThanSeconds` (all when 0) and returns how many went.
+async function purgeFiles(directory, olderThanSeconds) {
+    const cutoff = Date.now() - olderThanSeconds * 1000;
+    const present = await files(directory);
+    const stale = present.filter((file) => file.name.endsWith(".json") && file.mtime.getTime() <= cutoff);
+    await Promise.all(stale.map((file) => rm(path.join(directory, file.name), { force: true })));
+    return stale.length;
+}
+// The store manifest’s start, absent when there is no manifest.
+async function readManifest(root) {
+    try {
+        return JSON.parse(await readFile(path.join(root, "manifest.json"), "utf8"));
+    }
+    catch {
+        return {};
+    }
+}
+// A crawl’s pages are as old as the crawl, so `pages` goes whole once its manifest’s start passes the cutoff.
+async function purgePages(root, olderThanSeconds) {
+    const manifest = await readManifest(root);
+    const isOld = !manifest.started || Date.now() - Date.parse(manifest.started) >= olderThanSeconds * 1000;
+    log.debug({ root, started: manifest.started, olderThanSeconds, isOld }, "pages bucket aged");
+    if (!isOld)
+        return 0;
+    const facts = await files(path.join(root, "datasets/facts"));
+    const pages = facts.filter((file) => file.name.endsWith(".json")).length;
+    await Promise.all(PAGES.map((entry) => rm(path.join(root, entry), { recursive: true, force: true })));
+    return pages;
+}
+// Purges `bucket`, or every bucket, holding the store lock while project buckets are touched.
+export async function purgeCache(root, bucket, olderThanSeconds) {
+    const names = bucket === undefined ? [...PURGEABLE] : [bucket];
+    const hasStore = root !== undefined && existsSync(root);
+    const release = hasStore ? await lockStore(root) : undefined;
+    const purged = {};
+    try {
+        for (const name of names) {
+            const directory = bucketDirectory(name, hasStore ? root : undefined);
+            if (name === "pages")
+                purged[name] = hasStore ? await purgePages(root, olderThanSeconds) : 0;
+            else
+                purged[name] = directory === undefined ? 0 : await purgeFiles(directory, olderThanSeconds);
+            log.info({ bucket: name, purged: purged[name], olderThanSeconds }, `${purged[name]} entries purged from the ${name} cache`);
+        }
+    }
+    finally {
+        await release?.();
+    }
+    return purged;
+}

package/dist/cache/status.js

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { readdir, stat } from "node:fs/promises";
+import path from "node:path";
+import { log } from "../logger.js";
+import { TTL_DEFAULTS, bucketDirectory } from "./index.js";
+// Buckets kept as one JSON file per key: every bucket but the store’s `pages`.
+export const FILE_BUCKETS = Object.keys(TTL_DEFAULTS).filter((name) => name !== "pages");
+// Bucket name, the directories holding it (entries counted in the first), and the extension of one entry; user buckets only without a store.
+function layout(root) {
+    const files = FILE_BUCKETS.filter((name) => bucketDirectory(name, root)).map((name) => [name, [bucketDirectory(name, root)], ".json"]);
+    return root === undefined
+        ? files
+        : [
+            ["pages", [path.join(root, "datasets/facts"), path.join(root, "key_value_stores/bodies"), bucketDirectory("pages", root)], ".json"],
+            ["records", [path.join(root, "key_value_stores/records")], ".json"],
+            ["frontier", [path.join(root, "request_queues/frontier"), path.join(root, "request_queues/frontier-browser")], ".json"],
+            ...files,
+        ];
+}
+// Files directly under `directory`, empty when it does not exist.
+export async function files(directory) {
+    let names;
+    try {
+        names = await readdir(directory);
+    }
+    catch {
+        return [];
+    }
+    return Promise.all(names.map(async (name) => {
+        const { size, mtime } = await stat(path.join(directory, name));
+        return { name, size, mtime };
+    }));
+}
+// Entries, bytes and age range of every bucket present for the store at `root` and in the user cache.
+export async function cacheStatus(root) {
+    const out = [];
+    for (const [bucket, directories, entryExtension] of layout(root)) {
+        const [first = "", ...rest] = directories;
+        const primary = await files(first);
+        const counted = primary.filter((file) => file.name.endsWith(entryExtension) && !file.name.startsWith("__"));
+        const others = await Promise.all(rest.map((directory) => files(directory)));
+        const all = [...counted, ...others.flat()];
+        const times = all.map((file) => file.mtime.getTime()).toSorted((a, b) => a - b);
+        log.debug({ root, bucket, entries: counted.length, files: all.length }, "bucket measured");
+        if (all.length === 0)
+            continue;
+        out.push({ bucket, entries: counted.length, bytes: all.reduce((sum, file) => sum + file.size, 0), oldest: new Date(times[0]).toISOString(), newest: new Date(times.at(-1)).toISOString() });
+    }
+    return out;
+}