spiderlint 0.0.0 → 0.143.0

package/dist/server/worker.js

@@ -0,0 +1,129 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { spawn } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createInterface } from "node:readline";
+import { Worker } from "bullmq";
+import { log } from "../logger.js";
+import { inSpan, recordScan, traceCarrier, withTraceCarrier } from "../telemetry.js";
+import { latestKey, PREFIX, PROGRESS_FD, QUEUE } from "./queue.js";
+const RUNNER = new URL(`scan${path.extname(import.meta.url)}`, import.meta.url).pathname;
+const PROGRESS_MS = 1000;
+const REPORT_MAX = 64 * 1024 * 1024;
+const TAIL = 20;
+// The share of a scan’s time the crawl may use, leaving the rest to lint and the report.
+const CRAWL_SHARE = 0.8;
+// Why a runner exit code failed the scan, as the job’s error says it.
+const EXITS = { 2: "invalid settings", 3: "no page could be fetched", 4: "the scan failed" };
+// Runners in flight, killed together on shutdown.
+const running = new Set();
+// Pipes each line of `stream` to `onLine`.
+function lines(stream, onLine) {
+    createInterface({ input: stream, crlfDelay: Infinity }).on("line", onLine);
+}
+// One JSON log line of a runner, or nothing when it is not JSON.
+function parsed(line) {
+    try {
+        return JSON.parse(line);
+    }
+    catch {
+        return {};
+    }
+}
+// The last error a runner logged, else the meaning of its exit code.
+function reasonOf(code, tail, isTimedOut, scanTimeout) {
+    if (isTimedOut)
+        return `the scan exceeded ${scanTimeout} s`;
+    const logged = tail.map((line) => parsed(line)).findLast((entry) => (entry.level ?? 0) >= 50);
+    return [EXITS[code ?? 4] ?? `the scan exited with ${code}`, logged?.error].filter(Boolean).join(": ");
+}
+// One scan as a span continuing the submitting request’s trace, timed by outcome.
+async function tracedScan(job) {
+    const started = performance.now();
+    try {
+        const result = await withTraceCarrier(job.data.trace, async () => inSpan("scan job", { "spiderlint.job": String(job.id), "url.full": job.data.url, "spiderlint.policy": job.data.policy }, async () => runScan(job)));
+        recordScan((performance.now() - started) / 1000, "done", result.summary.pages);
+        return result;
+    }
+    catch (error) {
+        recordScan((performance.now() - started) / 1000, "failed");
+        throw error;
+    }
+}
+// One scan in a runner child under a temporary working directory, its progress relayed at most once per second.
+async function runScan(job) {
+    const { url, settings, scanTimeout, deny, via } = job.data;
+    const cwd = await mkdtemp(path.join(tmpdir(), "spiderlint-scan-"));
+    const child = spawn(process.execPath, ["--experimental-strip-types", RUNNER], { cwd, stdio: ["pipe", "pipe", "pipe", "pipe"], env: { ...process.env, SPIDERLINT_LOG_FORMAT: "json" } });
+    running.add(child);
+    log.info({ job: job.id, url, pid: child.pid, scanTimeout }, "scan started");
+    let isTimedOut = false;
+    const timer = setTimeout(() => {
+        isTimedOut = true;
+        child.kill("SIGKILL");
+    }, scanTimeout * 1000);
+    const chunks = [];
+    let size = 0;
+    child.stdout?.on("data", (chunk) => {
+        size += chunk.length;
+        if (size <= REPORT_MAX)
+            chunks.push(chunk);
+    });
+    const tail = [];
+    lines(child.stderr, (line) => {
+        tail.push(line);
+        if (tail.length > TAIL)
+            tail.shift();
+        log.debug({ job: job.id, line }, "runner log");
+    });
+    const progress = { sent: 0, held: "" };
+    const store = (line) => void job.updateProgress(JSON.parse(line)).catch((error) => log.warn({ job: job.id, error: String(error) }, "progress not stored"));
+    lines(child.stdio[PROGRESS_FD], (line) => {
+        const now = Date.now();
+        // A line inside the interval is held, so the last one is stored once the runner exits.
+        progress.held = now - progress.sent < PROGRESS_MS ? line : "";
+        if (progress.held)
+            return;
+        progress.sent = now;
+        store(line);
+    });
+    child.stdin?.end(JSON.stringify({ url, settings, deny, via, crawlDeadline: Math.floor(scanTimeout * CRAWL_SHARE), trace: traceCarrier() }));
+    const [code, signal] = await new Promise((resolve) => child.once("close", (exitCode, exitSignal) => resolve([exitCode, exitSignal])));
+    clearTimeout(timer);
+    if (progress.held)
+        store(progress.held);
+    running.delete(child);
+    await rm(cwd, { recursive: true, force: true });
+    log.info({ job: job.id, url, code, signal, bytes: size }, "scan exited");
+    if (code !== 0)
+        throw new Error(reasonOf(code, tail, isTimedOut, scanTimeout));
+    if (size > REPORT_MAX)
+        throw new Error(`the report exceeds ${REPORT_MAX} bytes`);
+    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+}
+// Takes `concurrency` scans at a time off the queue until `close`, which kills the runners in flight; a finished scan becomes its host’s latest for `retention` seconds.
+export function startWorker(redis, concurrency, retention) {
+    const worker = new Worker(QUEUE, tracedScan, { connection: redis, prefix: PREFIX, concurrency });
+    worker.on("failed", (job, error) => log.warn({ job: job?.id, host: job?.data.host, error: error.message }, "scan failed"));
+    worker.on("completed", async (job) => {
+        log.info({ job: job.id, host: job.data.host, findings: job.returnvalue.findings.length, grade: job.returnvalue.summary.rating?.grade }, "scan completed");
+        try {
+            await redis.set(latestKey(job.data.host), job.id, "EX", retention);
+        }
+        catch (error) {
+            log.warn({ job: job.id, error: String(error) }, "latest scan not recorded");
+        }
+    });
+    log.info({ queue: QUEUE, concurrency }, "worker started");
+    return {
+        async close() {
+            log.info({ running: running.size }, "worker stopping");
+            for (const child of running)
+                child.kill("SIGKILL");
+            await worker.close(true);
+        },
+    };
+}

package/dist/store/disk.js

@@ -0,0 +1,215 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile } from "node:fs/promises";
+import { hostname } from "node:os";
+import path from "node:path";
+import { Configuration, Dataset, KeyValueStore, RequestQueue } from "crawlee";
+import lockfile from "proper-lockfile";
+import { VERSION } from "../agent.js";
+import { writeAtomic } from "../cache/index.js";
+import { ConfigError } from "../config/index.js";
+import { log, logRelativeTo } from "../logger.js";
+const RESOURCES = "resources";
+const REPORT = "report";
+const SITE = "site";
+// Crawlee hands a text/plain record back verbatim; a page’s own type would parse JSON into an object.
+const BODY_TYPE = "text/plain; charset=utf-8";
+function key(url) {
+    return createHash("sha256").update(url).digest("hex");
+}
+async function openStorages(config) {
+    return [await Dataset.open("facts", { config }), await KeyValueStore.open("bodies", { config }), await KeyValueStore.open("records", { config }), await RequestQueue.open("frontier", { config }), await RequestQueue.open("frontier-browser", { config })];
+}
+// `lint` or `show-report` found no stored crawl; the run exits 3.
+export class NothingStored extends Error {
+}
+const STALE_MS = 30_000;
+// The lock holder the pid file names, when it is readable.
+async function readHolder(holderPath) {
+    try {
+        return JSON.parse(await readFile(holderPath, "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+// Whether the process that wrote `holder` is gone; one on another host is presumed alive.
+function isGone(holder) {
+    if (!holder || holder.host !== hostname())
+        return false;
+    try {
+        process.kill(holder.pid, 0);
+        return false;
+    }
+    catch (error) {
+        return error.code === "ESRCH";
+    }
+}
+// Holds the store at `directory` for this process; a live second holder is a ConfigError, a dead one’s lock is waited out until stale.
+export async function lockStore(directory) {
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    const lockfilePath = path.join(directory, "manifest.json.lock");
+    const holderPath = `${lockfilePath}.pid`;
+    const options = { lockfilePath, realpath: false, stale: STALE_MS, update: STALE_MS / 3 };
+    let release;
+    try {
+        release = await lockfile.lock(directory, { ...options, retries: 0 });
+    }
+    catch (error) {
+        if (error.code !== "ELOCKED")
+            throw error;
+        const holder = await readHolder(holderPath);
+        const isDead = isGone(holder);
+        log.debug({ directory, holder, isDead }, "store lock held");
+        if (!isDead)
+            throw new ConfigError(`store ${directory} is in use by ${holder ? `process ${holder.pid} on ${holder.host}` : "another process"}`);
+        log.warn({ directory, holder: holder?.pid, staleSeconds: STALE_MS / 1000 }, `store ${directory} locked by process ${holder?.pid}, which is gone; waiting ${STALE_MS / 1000} s for the lock to go stale`);
+        try {
+            release = await lockfile.lock(directory, { ...options, retries: { retries: STALE_MS / 1000 + 5, factor: 1, minTimeout: 1000, maxTimeout: 1000 } });
+        }
+        catch (error_) {
+            if (error_.code === "ELOCKED")
+                throw new ConfigError(`store ${directory} was taken by another process`);
+            throw error_;
+        }
+    }
+    await writeAtomic(holderPath, JSON.stringify({ pid: process.pid, host: hostname() }));
+    return release;
+}
+// The `pages` bucket: facts in a Dataset, bodies and results in KeyValueStores, the frontier in a RequestQueue.
+export class DiskStore {
+    // Locks `directory`; a fresh crawl empties it, a resumed crawl or a re-lint keeps it, and `existing` refuses one never written.
+    static async open(directory, mode) {
+        const isStored = existsSync(path.join(directory, "manifest.json"));
+        log.debug({ directory, existing: mode.existing, isStored }, "store looked up");
+        if (!isStored && mode.existing)
+            throw new NothingStored(`nothing stored in ${directory}; run spiderlint audit or crawl first`);
+        const release = await lockStore(directory);
+        const config = new Configuration({ storageClientOptions: { localDataDirectory: directory }, persistStorage: true, purgeOnStart: false });
+        const previous = await DiskStore.#readManifest(directory);
+        let storages = await openStorages(config);
+        const lastReport = await storages[2].getValue(REPORT);
+        const last = lastReport?.summary;
+        const earlier = new Map();
+        if (mode.fresh) {
+            const stored = await storages[0].map((item) => item);
+            for (const facts of stored)
+                for (const href of [facts.url.href, facts.crawl.requested])
+                    if (href)
+                        earlier.set(href, facts);
+            const crawlerState = await KeyValueStore.open(undefined, { config });
+            const [facts, , records, frontier, rendering] = storages;
+            await Promise.all([facts, records, frontier, rendering, crawlerState].map((storage) => storage.drop()));
+            storages = await openStorages(config);
+        }
+        const { configHash, fresh, seeds = [] } = mode;
+        if (previous && configHash && previous.configHash !== configHash)
+            log.warn({ directory, stored: previous.configHash, current: configHash }, "store was crawled with another configuration");
+        const manifest = fresh || !previous ? { version: VERSION, seeds, configHash: configHash ?? "", started: new Date().toISOString() } : previous;
+        const store = new DiskStore(directory, config, storages, release, manifest, earlier, last ?? undefined);
+        await store.#writeManifest();
+        log.info({ directory, fresh: mode.fresh, started: manifest.started, earlier: earlier.size }, earlier.size > 0 ? `resuming ${earlier.size} pages stored in ${directory}` : `pages stored in ${directory}`);
+        logRelativeTo(manifest.seeds);
+        return store;
+    }
+    static async #readManifest(directory) {
+        try {
+            return JSON.parse(await readFile(path.join(directory, "manifest.json"), "utf8"));
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return undefined;
+            throw new ConfigError(`store ${directory}: unreadable manifest.json (${error instanceof Error ? error.message : String(error)})`);
+        }
+    }
+    #release;
+    directory;
+    config;
+    facts;
+    bodies;
+    records;
+    // One frontier per crawler, so a resumed run hands each its own.
+    frontiers;
+    manifest;
+    earlier;
+    last;
+    constructor(directory, config, storages, release, manifest, earlier, last) {
+        this.earlier = earlier;
+        if (last)
+            this.last = last;
+        this.directory = directory;
+        this.config = config;
+        this.facts = storages[0];
+        this.bodies = storages[1];
+        this.records = storages[2];
+        this.frontiers = { http: storages[3], browser: storages[4] };
+        this.#release = release;
+        this.manifest = manifest;
+    }
+    async #writeManifest() {
+        await writeAtomic(path.join(this.directory, "manifest.json"), `${JSON.stringify(this.manifest, undefined, 2)}\n`);
+    }
+    async add(facts, body) {
+        await this.facts.pushData(facts);
+        await this.bodies.setValue(key(facts.url.href), body, { contentType: BODY_TYPE });
+    }
+    // A page’s stored body, which a fresh crawl keeps for revalidation; one that reads back as neither text nor bytes is a miss.
+    async body(href) {
+        const value = await this.bodies.getValue(key(href));
+        if (value === null || value === undefined || typeof value === "string")
+            return value ?? undefined;
+        if (value instanceof Uint8Array)
+            return Buffer.from(value).toString("utf8");
+        log.warn({ url: href, type: typeof value }, "stored body unreadable, page fetched in full");
+        return undefined;
+    }
+    // Deletes the bodies of pages this crawl no longer has.
+    async pruneBodies(pages) {
+        const keep = new Set(pages.map((page) => key(page.url.href)));
+        const gone = [];
+        await this.bodies.forEachKey((name) => {
+            if (!keep.has(name))
+                gone.push(name);
+        });
+        // eslint-disable-next-line unicorn/no-null -- Crawlee deletes a record when its value is null
+        for (const name of gone)
+            await this.bodies.setValue(name, null);
+        log.debug({ kept: keep.size, pruned: gone.length }, "bodies pruned");
+        return gone.length;
+    }
+    async pages() {
+        return this.facts.map((item) => item);
+    }
+    async saveResources(results) {
+        await this.records.setValue(RESOURCES, results);
+    }
+    async resources() {
+        return (await this.records.getValue(RESOURCES)) ?? {};
+    }
+    async saveSite(site) {
+        await this.records.setValue(SITE, site);
+    }
+    async site() {
+        return (await this.records.getValue(SITE)) ?? { sitemaps: [] };
+    }
+    async saveReport(report) {
+        await this.records.setValue(REPORT, report);
+    }
+    async report() {
+        const report = await this.records.getValue(REPORT);
+        if (!report)
+            throw new ConfigError(`store ${this.directory} holds no report; run spiderlint lint --store ${this.directory} first`);
+        return report;
+    }
+    // Stamps `finished` and releases the lock.
+    async close(isFinished) {
+        if (isFinished)
+            this.manifest.finished = new Date().toISOString();
+        await this.#writeManifest();
+        await this.#release();
+        log.debug({ directory: this.directory, finished: this.manifest.finished }, "store closed");
+    }
+}

package/dist/store/memory.js

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { log } from "../logger.js";
+// Stream-mode store: facts only, nothing persisted; one record per final URL.
+export class MemoryStore {
+    #seen = new Set();
+    pages = [];
+    add(facts) {
+        if (this.#seen.has(facts.url.href)) {
+            log.debug({ url: facts.url.href }, "page already stored");
+            return false;
+        }
+        this.#seen.add(facts.url.href);
+        this.pages.push(facts);
+        return true;
+    }
+}

package/dist/telemetry-environment.js

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+// The OTLP endpoint variables; any one set switches telemetry on.
+export const ENDPOINTS = ["OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"];
+// Whether the environment names an OTLP endpoint and does not disable the SDK.
+export function isTelemetryConfigured(environment = process.env) {
+    return environment.OTEL_SDK_DISABLED !== "true" && ENDPOINTS.some((name) => (environment[name] ?? "") !== "");
+}

package/dist/telemetry.js

@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+import { context, metrics, propagation, SpanStatusCode, trace } from "@opentelemetry/api";
+import { VERSION } from "./agent.js";
+import { log } from "./logger.js";
+import { ENDPOINTS, isTelemetryConfigured } from "./telemetry-environment.js";
+// Milliseconds between metric exports; OTEL_METRIC_EXPORT_INTERVAL overrides it.
+const EXPORT_MS = 15_000;
+const state = {};
+const tracer = trace.getTracer("spiderlint", VERSION);
+// Starts OTLP trace and metric export for `service` when an endpoint is set; unset, nothing is imported and undefined comes back.
+export async function startTelemetry(service) {
+    if (!isTelemetryConfigured()) {
+        log.debug({ service, variables: ENDPOINTS }, "telemetry off, no OTLP endpoint set");
+        return undefined;
+    }
+    const [{ NodeTracerProvider, BatchSpanProcessor }, { MeterProvider, PeriodicExportingMetricReader }, { OTLPTraceExporter }, { OTLPMetricExporter }, { resourceFromAttributes }] = await Promise.all([
+        import("@opentelemetry/sdk-trace-node"),
+        import("@opentelemetry/sdk-metrics"),
+        import("@opentelemetry/exporter-trace-otlp-http"),
+        import("@opentelemetry/exporter-metrics-otlp-http"),
+        import("@opentelemetry/resources"),
+    ]);
+    const name = process.env.OTEL_SERVICE_NAME || service;
+    const resource = resourceFromAttributes({ "service.name": name, "service.version": VERSION });
+    const tracing = new NodeTracerProvider({ resource, spanProcessors: [new BatchSpanProcessor(new OTLPTraceExporter())] });
+    tracing.register();
+    const interval = Number(process.env.OTEL_METRIC_EXPORT_INTERVAL) || EXPORT_MS;
+    const metering = new MeterProvider({ resource, readers: [new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter(), exportIntervalMillis: interval })] });
+    metrics.setGlobalMeterProvider(metering);
+    const meter = metrics.getMeter("spiderlint", VERSION);
+    state.instruments = {
+        requests: meter.createCounter("spiderlint.http.requests", { description: "API requests by route, method and status" }),
+        refusals: meter.createCounter("spiderlint.refusals", { description: "Refused API requests by code" }),
+        scanDuration: meter.createHistogram("spiderlint.scan.duration", { description: "Scan wall time by outcome", unit: "s" }),
+        scanPages: meter.createHistogram("spiderlint.scan.pages", { description: "Pages per finished scan" }),
+        extractorDuration: meter.createHistogram("spiderlint.extractor.duration", { description: "Time per extractor run", unit: "ms" }),
+        queueDepth: meter.createObservableGauge("spiderlint.queue.depth", { description: "Scans by queue state" }),
+    };
+    log.info({ service: name, interval }, "telemetry on, exporting traces and metrics over OTLP");
+    return {
+        async shutdown() {
+            await Promise.allSettled([tracing.shutdown(), metering.shutdown()]);
+            log.debug({ service: name }, "telemetry flushed");
+        },
+    };
+}
+// Runs `work` inside a new active span, recording a throw as its error; a no-op span when telemetry is off.
+export async function inSpan(name, attributes, work) {
+    return tracer.startActiveSpan(name, { attributes }, async (span) => {
+        try {
+            return await work(span);
+        }
+        catch (error) {
+            span.recordException(error instanceof Error ? error : String(error));
+            span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.message : String(error) });
+            throw error;
+        }
+        finally {
+            span.end();
+        }
+    });
+}
+// Renames the active span once its subject is known, as the CLI’s command.
+export function nameSpan(name) {
+    trace.getActiveSpan()?.updateName(name);
+}
+// The active trace context as W3C headers, to carry through the queue and into a scan runner; empty when telemetry is off.
+export function traceCarrier() {
+    const carrier = {};
+    propagation.inject(context.active(), carrier);
+    return carrier;
+}
+// Runs `work` as a continuation of the trace `carrier` names.
+export function withTraceCarrier(carrier, work) {
+    return context.with(propagation.extract(context.active(), carrier ?? {}), work);
+}
+// Counts one API request.
+export function countRequest(route, method, status) {
+    state.instruments?.requests.add(1, { "http.route": route, "http.request.method": method, "http.response.status_code": status });
+}
+// Counts one refusal by its code: invalid bodies, policy refusals, rate windows and client buckets alike.
+export function countRefusal(code) {
+    state.instruments?.refusals.add(1, { code });
+}
+// Records one finished or failed scan.
+export function recordScan(seconds, outcome, pages) {
+    state.instruments?.scanDuration.record(seconds, { outcome });
+    if (pages !== undefined)
+        state.instruments?.scanPages.record(pages);
+}
+// Records one extractor run.
+export function recordExtractor(id, ms) {
+    state.instruments?.extractorDuration.record(ms, { extractor: id });
+}
+// Reports queue depth by state from `read` on every metric export.
+export function observeQueue(read) {
+    state.instruments?.queueDepth.addCallback(async (result) => {
+        const counts = await read();
+        for (const [name, count] of Object.entries(counts))
+            result.observe(count, { state: name });
+    });
+}