siteread 0.0.0-stage → 0.2.0

package/src/ip-ranges.js

@@ -0,0 +1,219 @@
+// The vendors' published crawler IP lists: what tells a real ChatGPT-User from a scanner
+// wearing its User-Agent.
+//
+// Ported from the crawler-verification code Synacl Labs runs on synacl.com (2026-10). There it
+// lived behind a Redis key shared by a cluster; here the store is pluggable (verifier.js).
+//
+// WHY (measured on that site's nginx log, 2026-09-17 → 10-16). A scanner cycles AI user-agents:
+// 217 of 221 ChatGPT-User requests from outside OpenAI's list came from IPs that also got a 444
+// (nginx's "close the connection, send nothing" answer to a scanner path), and DeepSeekBot /
+// CCBot / Grok / Perplexity-User / Claude-SearchBot were 100 % spoofed. Where a vendor publishes
+// a list, "IP is in it" is the test; where none exists, the fallback is "the IP never hit a
+// scanner path" (the 444 rule: createVerifier's `isKnownScannerIp`). On that site the two tests
+// agreed almost perfectly, so the fallback is honest, just weaker.
+//
+// The 444 rule CANNOT clear a crawler that follows a bad link into a scanner path. ClaudeBot
+// (on the 444 rule until 10-17) resolved a GitHub README's relative links against the site,
+// asked for /protocol//blob/master/examples/docker-compose.yml, drew one 444, and all 50 of
+// that IP's reads that day were counted spoofed. Anthropic publishes ONE list for its bots,
+// claude.com/crawling/bots.json, used here for all three Claude agents: on 30 days of log every
+// ClaudeBot IP and every page-reading Claude-User IP was in it, and the four IPs probing
+// /.env-style paths under Claude names were not. (Claude-User from Anthropic's 160.79.104.0/21
+// was NOT in it: that is the MCP connector calling a tool server, not a page read. The list
+// published 2026-10-07 does carry two /28s from that range.)
+//
+// SHAPE. A list is refetched only when its stored copy is > 24 h old. A failed fetch keeps the
+// previous copy; a copy nobody could refresh for 8 days is dropped, so a vendor's moved ranges
+// cannot quietly mark its new IPs as spoofed forever. Nothing here throws: no store, no network
+// or a broken list all end in "no list for that agent", which is the verdict 'unknown'.
+
+import net from 'node:net';
+
+export const REFRESH_AFTER_MS = 24 * 3_600_000;
+export const DROP_AFTER_MS = 8 * 86_400_000;
+export const FETCH_TIMEOUT_MS = 8_000;
+// A list is a few hundred prefixes; these bound what a broken (or hostile) response can cost.
+export const MAX_BODY_CHARS = 2_000_000;
+export const MAX_PREFIXES = 20_000;
+// No vendor publishes anything near this broad. A /0 in a list would verify EVERY address,
+// so a prefix shorter than these is a broken list, not a big one.
+export const MIN_PREFIX = Object.freeze({ ipv4: 8, ipv6: 16 });
+// Two processes sharing a store can disagree on the time by a little. A fetchedAt further in
+// the future than this is not skew: it would keep a list "fresh" forever and defeat the drop.
+const CLOCK_SKEW_MS = 5 * 60_000;
+
+const CLAUDE_BOTS = 'https://claude.com/crawling/bots.json';
+
+// Agent key (agents.js AGENTS / CRAWLERS) → its vendor's published list. Every URL answers
+// { creationTime, prefixes: [{ ipv4Prefix } | { ipv6Prefix }] } (all checked 2026-10-21).
+export const SOURCES = Object.freeze([
+    { agent: 'chatgpt-user',     url: 'https://openai.com/chatgpt-user.json' },
+    { agent: 'oai-searchbot',    url: 'https://openai.com/searchbot.json' },
+    { agent: 'gptbot',           url: 'https://openai.com/gptbot.json' },
+    { agent: 'perplexity-user',  url: 'https://www.perplexity.com/perplexity-user.json' },
+    { agent: 'perplexitybot',    url: 'https://www.perplexity.com/perplexitybot.json' },
+    { agent: 'applebot',         url: 'https://search.developer.apple.com/applebot.json' },
+    // Not claude-code: Claude Code's WebFetch runs on the user's own machine, never from these.
+    { agent: 'claude-user',      url: CLAUDE_BOTS },
+    { agent: 'claude-searchbot', url: CLAUDE_BOTS },
+    { agent: 'claudebot',        url: CLAUDE_BOTS },
+    // Classic search. googlebot.json covers Googlebot and its -Image/-News/-Video variants.
+    { agent: 'googlebot',        url: 'https://developers.google.com/static/search/apis/ipranges/googlebot.json' },
+    { agent: 'bingbot',          url: 'https://www.bing.com/toolbox/bingbot.json' },
+].map(Object.freeze));
+
+/** 'a.b.c.d/n' or 'x::/n' → { type, net, prefix, cidr } (normalised), or null. */
+export function parseCidr(raw) {
+    if (typeof raw !== 'string') return null;
+    const s = raw.trim();
+    const slash = s.indexOf('/');
+    if (slash < 1 || s.length > 64) return null;
+    const addr = s.slice(0, slash);
+    const bits = s.slice(slash + 1);
+    if (!/^\d{1,3}$/.test(bits)) return null;
+    const prefix = Number(bits);
+    const type = net.isIPv4(addr) ? 'ipv4' : net.isIPv6(addr) ? 'ipv6' : null;
+    if (!type) return null;
+    if (prefix < MIN_PREFIX[type] || prefix > (type === 'ipv4' ? 32 : 128)) return null;
+    return { type, net: addr.toLowerCase(), prefix, cidr: `${addr.toLowerCase()}/${prefix}` };
+}
+
+/** The vendors' shared format: { prefixes: [{ ipv4Prefix } | { ipv6Prefix }] } → [cidr]. */
+export function parseList(body) {
+    const items = Array.isArray(body?.prefixes) ? body.prefixes : [];
+    const out = new Set();
+    for (const it of items) {
+        if (out.size >= MAX_PREFIXES) break;
+        const p = parseCidr(it?.ipv4Prefix ?? it?.ipv6Prefix);
+        if (p) out.add(p.cidr);
+    }
+    return [...out];
+}
+
+/** A stored copy's prefixes ([cidr]) → [cidr], re-validated and bounded. */
+export function parsePrefixes(list) {
+    if (!Array.isArray(list)) return [];
+    const out = new Set();
+    for (const c of list.slice(0, MAX_PREFIXES)) {
+        const p = parseCidr(c);
+        if (p) out.add(p.cidr);
+    }
+    return [...out];
+}
+
+// An address as the log writes it → { ip, type }, or null. An IPv4-mapped IPv6 address
+// (::ffff:1.2.3.4, what a dual-stack listener can log) is checked as the IPv4 it is.
+export function normIp(raw) {
+    let s = String(raw ?? '').trim().toLowerCase();
+    const zone = s.indexOf('%');
+    if (zone !== -1) s = s.slice(0, zone);
+    const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(s);
+    if (mapped) s = mapped[1];
+    if (net.isIPv4(s)) return { ip: s, type: 'ipv4' };
+    if (net.isIPv6(s)) return { ip: s, type: 'ipv6' };
+    return null;
+}
+
+/** ms since epoch from a stored fetchedAt, or NaN. */
+export function fetchedAtMs(l) {
+    return typeof l?.fetchedAt === 'string' ? Date.parse(l.fetchedAt) : NaN;
+}
+
+/** True when a copy fetched at `at` is younger than `maxAgeMs` at `now` (and not from the future). */
+export function youngerThan(at, now, maxAgeMs) {
+    return Number.isFinite(at) && now - at < maxAgeMs && at - now <= CLOCK_SKEW_MS;
+}
+
+/** { lists: { agent: { prefixes, fetchedAt } } } from whatever a store returned. */
+export function normDoc(doc) {
+    const lists = doc && typeof doc === 'object' && doc.lists && typeof doc.lists === 'object' && !Array.isArray(doc.lists)
+        ? doc.lists : {};
+    // A copy: the caller mutates it, and an in-memory store may hand out its own object.
+    const out = {};
+    for (const [agent, l] of Object.entries(lists)) {
+        if (l && typeof l === 'object' && Array.isArray(l.prefixes)) {
+            out[agent] = { prefixes: l.prefixes.slice(0, MAX_PREFIXES), fetchedAt: l.fetchedAt };
+        }
+    }
+    return { lists: out };
+}
+
+/**
+ * A matcher over a stored document: { hasList(agent), contains(agent, addr), get(agent) }.
+ * Only the agents in `sources` are loaded, and a copy past the drop age is ignored even if
+ * nobody ran refresh() to delete it.
+ */
+export function buildMatcher(doc, { sources = SOURCES, now = Date.now() } = {}) {
+    const lists = new Map();   // agent → { block, count, fetchedAt }
+    for (const { agent } of sources) {
+        if (lists.has(agent)) continue;
+        const l = doc?.lists?.[agent];
+        if (!l || !Array.isArray(l.prefixes)) continue;
+        if (!youngerThan(fetchedAtMs(l), now, DROP_AFTER_MS)) continue;
+        const block = new net.BlockList();
+        let count = 0;
+        for (const c of l.prefixes.slice(0, MAX_PREFIXES)) {
+            const p = parseCidr(c);
+            if (!p) continue;
+            try { block.addSubnet(p.net, p.prefix, p.type); count++; } catch { /* skip */ }
+        }
+        if (count) lists.set(agent, { block, count, fetchedAt: l.fetchedAt });
+    }
+    return {
+        hasList: (agent) => lists.has(agent),
+        /** `addr` is normIp()'s { ip, type } or a raw address string. */
+        contains(agent, addr) {
+            const l = lists.get(agent);
+            const a = l && (typeof addr === 'object' && addr ? addr : normIp(addr));
+            if (!a) return false;
+            try { return l.block.check(a.ip, a.type); } catch { return false; }
+        },
+        get: (agent) => {
+            const l = lists.get(agent);
+            return l ? { prefixes: l.count, fetchedAt: l.fetchedAt } : null;
+        },
+    };
+}
+
+// The body, read at most MAX_BODY_CHARS deep: a hostile endpoint streaming forever is cut off
+// here (and by the fetch timeout) instead of being buffered whole before the size check.
+async function readCapped(res, max = MAX_BODY_CHARS) {
+    const reader = res.body?.getReader?.();
+    if (!reader) {
+        const text = await res.text();
+        if (text.length > max) throw new Error('response too large');
+        return text;
+    }
+    const dec = new TextDecoder();
+    let text = '';
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        text += dec.decode(value, { stream: true });
+        if (text.length > max) {
+            await reader.cancel().catch(() => {});
+            throw new Error('response too large');
+        }
+    }
+    return text + dec.decode();
+}
+
+/** GET a URL and JSON.parse it, bounded in time and size. Throws on any failure. */
+export async function fetchJson(url, fetchImpl, { userAgent } = {}) {
+    const headers = { accept: 'application/json' };
+    if (userAgent) headers['user-agent'] = userAgent;
+    const res = await fetchImpl(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), headers });
+    if (!res || !res.ok) throw new Error(`HTTP ${res?.status}`);
+    const text = await readCapped(res);
+    // A list URL that answers 200 with an HTML page (a moved list, a CDN challenge) must not
+    // parse as an empty list: JSON.parse is the check. (Content-Type is no help: OpenAI serves
+    // chatgpt-user.json as application/octet-stream.)
+    return JSON.parse(text);
+}
+
+/** One vendor list → [cidr]. Throws when it holds nothing usable. */
+export async function fetchList(url, fetchImpl, opts) {
+    const prefixes = parseList(await fetchJson(url, fetchImpl, opts));
+    if (!prefixes.length) throw new Error('no usable prefixes');
+    return prefixes;
+}

package/src/scanner.js

@@ -0,0 +1,56 @@
+// Scanner/exploit request paths: what a vulnerability scanner asks for and a real reader never
+// does. An IP that asks for one of these is a scanner, whatever User-Agent it wears; that is the
+// "444 rule" (nginx answers these with 444, "close, send nothing") the verifier falls back on
+// when a vendor publishes no IP list (createVerifier's `isKnownScannerIp`).
+//
+// Ported from the edge classifier Synacl Labs runs on synacl.com (2026-10), keeping only the
+// host-blind, product-agnostic rules; that site's own carve-outs are gone. A site with a real
+// path that matches (a /install.sh, a Decap CMS /admin/config.yml) passes `allow`.
+//
+// Classify on the REQUEST PATH ONLY, never the User-Agent: a scanner whose UA was literally
+// "http://<site>/wp-admin/install.php?step=1" made the raw-line version flag itself. Feed it the
+// raw request target (nginx's $request_uri: path + query, NOT url-decoded), so the rules below
+// see exactly what the edge saw.
+
+export const SCANNER_PATH_RE = new RegExp(
+    '(\\/\\.(?!well-known))' +                                        // /.env /.git … but not /.well-known
+    '|(\\.(php[0-9]?|phtml|phps|asp|aspx|jsp|jspx|cgi|env|sql|bak|old|backup|swp|ini|sh)(\\?|$))' + // dead tech ext
+    '|(^\\/+(wp-|xmlrpc|phpmyadmin|boaform|cgi-bin|HNAP1|SDK\\/webLanguage|' +
+    'apply\\.cgi|tmUnblock\\.cgi|GponForm|\\.git|\\.env|actuator(\\/|$)|_ignition|solr\\/|autodiscover))' +
+    // Words that are also ordinary slugs (/mysql-vs-postgres, /backup-strategies, /shell-tips) need
+    // a segment end after them; the scanner forms (/mysql/, /backup.zip, /shell?cd+/tmp) still match.
+    // (The original matched these as bare prefixes: its site had no such pages.) `/vendor/` is not
+    // here at all: static sites serve /vendor/*.js, and the scanner hits below it end in .php or a
+    // dotfile, which the rules above already catch.
+    '|(^\\/+(wordpress|pma|mysql|shell|backups?)(?=[/?.]|$))' +
+    // WP-artifact probes bury these deep in the path (//wp-includes/…/license.txt/…) with a leading
+    // // to dodge the ^/-anchored rule; these tokens never appear in a legit URL: match anywhere.
+    '|(wp-includes|wp-admin|wp-content|wp-login|wlwmanifest|xmlrpc)' +
+    // Percent-encoded dot: /%2eenv, /$(pwd)/%2egit/config, /private/smtp%2ejson. Every rule above
+    // wants a LITERAL '.', and the raw request target is not url-decoded, so these passed as
+    // ordinary traffic until this rule. Path only: [^?]* stops at the first '?' so an encoded
+    // query value can't false-positive.
+    '|(^[^?]*%2e)' +
+    // Secret/config-file scanners: framework and cloud artifacts no site serves on purpose.
+    // docker-compose only as a YAML FILE, so a /docker-compose-tutorial/ post is not a probe (the
+    // ClaudeBot incident's /…/examples/docker-compose.yml still is).
+    '|(appsettings\\.|aws-exports\\.|composer\\.(json|lock)|/_profiler|/phpinfo|launchSettings|' +
+    'serviceAccountKey|service-account-credentials|api-keys\\.json|docker-compose(\\.[\\w-]+)?\\.ya?ml|' +
+    'config\\.ya?ml|aws\\.ya?ml)' +
+    // Dockerfile and credentials.* only as FILES: a route that merely ends in /credentials is a
+    // normal API path on plenty of apps.
+    '|(/Dockerfile(\\.[a-z]+)?(\\?|$))|(/credentials\\.[a-z]+(\\?|$))',
+    'i',
+);
+
+/**
+ * True when a raw request target (path + query, not url-decoded) is a scanner probe.
+ * `allow` (a RegExp tested against the same string) exempts a site's own legit paths.
+ */
+export function isScannerPath(path, { allow } = {}) {
+    if (typeof path !== 'string' || !path) return false;
+    if (!SCANNER_PATH_RE.test(path)) return false;
+    // search() ignores a /g regex's lastIndex, so a shared global `allow` can't flip-flop.
+    if (allow instanceof RegExp && path.search(allow) !== -1) return false;
+    return true;
+}

package/src/verifier.js

@@ -0,0 +1,221 @@
+// A verdict for one request: is the IP that sent this User-Agent really that vendor's crawler?
+//
+//   verified  the agent's vendor publishes an IP list and the IP is in it
+//   spoofed   the vendor publishes a list and the IP is not in it (or the UA is a robots.txt
+//             token nothing real sends, or `isKnownScannerIp` says the IP probed scanner paths)
+//   unknown   no list to check against: the vendor publishes none, or it could not be loaded
+//   n/a       the UA claims to be no known AI agent or search crawler
+//
+// A vendor list always wins over the 444 rule: ClaudeBot once followed a broken README link
+// into a scanner path, and the 444 rule called a real crawler spoofed (ip-ranges.js).
+//
+// Refresh rules are the ported ones (ip-ranges.js): refetch a list when the stored copy is
+// > 24 h old, keep the old copy when a fetch fails, drop a copy nobody could refresh for 8 days.
+// Nothing here throws.
+
+import {
+    SOURCES, REFRESH_AFTER_MS, DROP_AFTER_MS,
+    normIp, normDoc, buildMatcher, fetchJson, fetchList, parsePrefixes, fetchedAtMs, youngerThan,
+} from './ip-ranges.js';
+import { detectAgent, detectCrawler } from './agents.js';
+import { VERSION } from './version.js';
+
+const USER_AGENT = `siteread/${VERSION} (+https://www.npmjs.com/package/siteread)`;
+
+/** The default store: one process, nothing on disk. */
+export function memoryStore(initial = null) {
+    let doc = initial;
+    return {
+        async get() { return doc; },
+        async set(next) { doc = next; },
+    };
+}
+
+/**
+ * @param {object} [opts]
+ * @param {Function} [opts.fetch]  WHATWG fetch (default globalThis.fetch)
+ * @param {{get(): Promise<object|null>, set(doc: object): Promise<void>}} [opts.store]
+ * @param {Array<{agent: string, url: string}>} [opts.sources]
+ * @param {() => number} [opts.now]
+ * @param {{warn?: Function, error?: Function}} [opts.log]
+ * @param {string|null} [opts.feedUrl]  one merged document instead of the vendor URLs (opt-in)
+ * @param {(ip: string) => boolean} [opts.isKnownScannerIp]  the 444 rule, for agents with no list
+ */
+export function createVerifier({
+    fetch: fetchImpl = globalThis.fetch,
+    store = memoryStore(),
+    sources = SOURCES,
+    now = Date.now,
+    log = console,
+    feedUrl = null,
+    isKnownScannerIp = null,
+} = {}) {
+    // One entry per agent (the first wins), in the order given.
+    const srcs = [];
+    const urlOf = new Map();
+    for (const s of sources) {
+        if (!s || typeof s.agent !== 'string' || typeof s.url !== 'string' || urlOf.has(s.agent)) continue;
+        srcs.push({ agent: s.agent, url: s.url });
+        urlOf.set(s.agent, s.url);
+    }
+    let matcher = buildMatcher(null, { sources: srcs, now: now() });
+
+    async function fetchFromVendors(due, t, doc, out) {
+        // One GET per distinct URL: Anthropic's bots.json serves three agents.
+        const urls = [...new Set(due.map(d => d.url))];
+        const results = await Promise.allSettled(urls.map(u => fetchList(u, fetchImpl, { userAgent: USER_AGENT })));
+        const byUrl = new Map(urls.map((u, i) => [u, results[i]]));
+        for (const { agent, url } of due) {
+            const r = byUrl.get(url);
+            if (r.status === 'fulfilled') {
+                doc.lists[agent] = { prefixes: r.value, fetchedAt: new Date(t).toISOString() };
+                out.fetched.push(agent);
+            } else {
+                // Keep whatever copy is stored: a vendor's CDN hiccup must not turn every one
+                // of its real crawlers into "spoofed" until the next refresh.
+                fail(out, agent, r.reason, `${agent} (${url})`);
+            }
+        }
+    }
+
+    // The future hosted feed: { lists: { agent: { prefixes: [cidr], fetchedAt } } }, one GET.
+    // Every prefix is re-validated exactly like a vendor's, and a list keeps the feed's own
+    // fetchedAt (when the FEED fetched it from the vendor), so the 24 h / 8 day rules still
+    // measure the age of the vendor data, not of the feed hop.
+    async function fetchFromFeed(due, t, doc, out) {
+        let body;
+        try {
+            body = await fetchJson(feedUrl, fetchImpl, { userAgent: USER_AGENT });
+        } catch (err) {
+            for (const { agent } of due) fail(out, agent, err, `feed (${feedUrl})`);
+            return;
+        }
+        const lists = body && typeof body.lists === 'object' && body.lists ? body.lists : {};
+        for (const { agent } of due) {
+            const l = Object.hasOwn(lists, agent) ? lists[agent] : null;
+            const prefixes = parsePrefixes(l?.prefixes);
+            // A future fetchedAt is clamped to now: it would otherwise keep the copy fresh forever.
+            const at = Math.min(fetchedAtMs(l), t);
+            const stored = fetchedAtMs(doc.lists[agent]);
+            if (!prefixes.length) { fail(out, agent, new Error('no usable prefixes in feed'), 'feed'); continue; }
+            if (!Number.isFinite(at)) { fail(out, agent, new Error('feed list has no fetchedAt'), 'feed'); continue; }
+            if (Number.isFinite(stored) && at <= stored) { fail(out, agent, new Error('feed copy is not newer'), 'feed'); continue; }
+            doc.lists[agent] = { prefixes, fetchedAt: new Date(at).toISOString() };
+            out.fetched.push(agent);
+        }
+    }
+
+    function fail(out, agent, err, what) {
+        const msg = err?.message || String(err);
+        out.failed.push(agent);
+        out.errors[agent] = msg;
+        log?.warn?.(`[siteread] ${what} failed: ${msg}`);
+    }
+
+    /**
+     * One refresh pass: read the store, fetch what is due, drop what is too old, write back.
+     * Never throws. Returns { fetched, failed, fresh, dropped, errors } (agent keys; errors maps
+     * a failed agent to its reason) and `error` when the store itself failed.
+     * `force` refetches every list whatever its age (a failed fetch still keeps the old copy).
+     */
+    async function refresh({ force = false } = {}) {
+        const t = now();
+        const out = { fetched: [], failed: [], fresh: [], dropped: [], errors: {} };
+        let doc;
+        try {
+            doc = normDoc(await store.get());
+        } catch (err) {
+            log?.error?.(`[siteread] store read failed: ${err.message}`);
+            out.error = err.message;
+            return out;
+        }
+        try {
+            const due = [];
+            for (const s of srcs) {
+                if (!force && youngerThan(fetchedAtMs(doc.lists[s.agent]), t, REFRESH_AFTER_MS)) out.fresh.push(s.agent);
+                else due.push(s);
+            }
+            if (due.length) {
+                if (typeof fetchImpl !== 'function') {
+                    for (const { agent } of due) fail(out, agent, new Error('no fetch available'), agent);
+                } else if (feedUrl) {
+                    await fetchFromFeed(due, t, doc, out);
+                } else {
+                    await fetchFromVendors(due, t, doc, out);
+                }
+            }
+            for (const [agent, l] of Object.entries(doc.lists)) {
+                if (urlOf.has(agent) && youngerThan(fetchedAtMs(l), t, DROP_AFTER_MS)) continue;
+                delete doc.lists[agent];
+                out.dropped.push(agent);
+            }
+            if (out.fetched.length || out.dropped.length) await store.set(doc);
+        } catch (err) {
+            log?.error?.(`[siteread] refresh failed: ${err.message}`);
+            out.error = err.message;
+        }
+        // Whatever made it into `doc` is usable in this process even if the store write failed.
+        matcher = buildMatcher(doc, { sources: srcs, now: t });
+        return out;
+    }
+
+    /** Rebuild the matcher from the store without fetching (a reader beside one refresher). */
+    async function load() {
+        const t = now();
+        try {
+            matcher = buildMatcher(normDoc(await store.get()), { sources: srcs, now: t });
+        } catch (err) {
+            log?.error?.(`[siteread] store read failed: ${err.message}`);
+        }
+        return lists();
+    }
+
+    /** The verdict for one request. Synchronous: uses the lists the last refresh()/load() saw. */
+    function check(ip, ua) {
+        const agent = detectAgent(ua);
+        const e = agent || detectCrawler(ua);
+        if (!e) {
+            return { agent: null, kind: null, assistant: null, purpose: null, verdict: 'n/a', reason: 'not-a-known-agent', source: null };
+        }
+        const base = {
+            agent: e.key,
+            kind: agent ? 'ai' : 'crawler',
+            assistant: e.assistant ?? null,
+            purpose: e.purpose,
+            source: urlOf.get(e.key) ?? null,
+        };
+        if (e.robotsTokenOnly) return { ...base, verdict: 'spoofed', reason: 'robots-token-not-a-user-agent' };
+        const addr = normIp(ip);
+        if (!addr) return { ...base, verdict: 'unknown', reason: 'invalid-ip' };
+        if (matcher.hasList(e.key)) {
+            return matcher.contains(e.key, addr)
+                ? { ...base, verdict: 'verified', reason: 'in-vendor-list' }
+                : { ...base, verdict: 'spoofed', reason: 'not-in-vendor-list' };
+        }
+        // No list. The 444 rule, if the caller supplies it: an IP that probed scanner paths is not
+        // the crawler it claims to be. It can only ever turn 'unknown' into 'spoofed'.
+        if (typeof isKnownScannerIp === 'function') {
+            let hit = false;
+            try { hit = Boolean(isKnownScannerIp(addr.ip)); } catch { hit = false; }
+            if (hit) return { ...base, verdict: 'spoofed', reason: 'known-scanner-ip' };
+        }
+        return { ...base, verdict: 'unknown', reason: base.source ? 'list-unavailable' : 'no-vendor-list' };
+    }
+
+    /** Freshness of every source, in `sources` order. */
+    function lists() {
+        const t = now();
+        return srcs.map(({ agent, url }) => {
+            const l = matcher.get(agent);
+            return {
+                agent,
+                url,
+                prefixes: l?.prefixes ?? 0,
+                fetchedAt: l?.fetchedAt ?? null,
+                fresh: l ? youngerThan(Date.parse(l.fetchedAt), t, REFRESH_AFTER_MS) : false,
+            };
+        });
+    }
+
+    return { refresh, load, check, lists };
+}

package/src/version.js

@@ -0,0 +1,2 @@
+// Kept in step with package.json by test/package.test.js (no JSON import: Node 20 needs no flag for this).
+export const VERSION = '0.2.0';