siteread 0.0.0-stage → 0.2.0

package/src/digest/log.js

@@ -0,0 +1,363 @@
+// An access log (nginx / Apache "combined", plain or .gz) → people, sources and crawlers.
+//
+// Who counts as a person is the rule synacl.com settled on after its first counts were wrong
+// (2026-09-28, stricter than the read a week earlier, so compare only reads made the same way):
+//   a person = an IP that sent no bot User-Agent, never asked for a scanner path or drew a 444,
+//   and loaded at least one asset (CSS/JS/font/image). Link checkers, previewers and most bots
+//   fetch the HTML and stop; a browser always pulls the stylesheet.
+// That rule alone let one fleet through: AWS us-west-2 IPs with a single Linux Chrome UA that
+// FAKED a Google Referer and loaded assets. They were the whole of one day's "Google bump" that
+// Search Console showed as zero clicks. So a person whose first visit claims a search-engine
+// Referer from a cloud provider's address space is set aside and counted on its own line.
+//
+// Crawlers are counted on page fetches (and on sitemap URLs when a sitemap is given), each IP
+// checked against its vendor's list. The raw UA count lies: one week synacl.com's log showed
+// "Googlebot 6.9k", and it was scanners wearing Googlebot's UA against the bare IP. Googlebot
+// also crawls from 192.178.x, not only 66.249.x: a first-octet guess undercounted it by a third.
+
+import { createReadStream } from 'node:fs';
+import { createGunzip } from 'node:zlib';
+import { createInterface } from 'node:readline';
+import { detectAgent, detectCrawler } from '../agents.js';
+import { isScannerPath } from '../scanner.js';
+import { classifySource, refererHost } from './referrer.js';
+
+// ip - user [time] "METHOD target PROTO" status bytes "referer" "ua" (anything after is ignored).
+// nginx escapes a quote inside a field as \x22, Apache as \"; both are handled.
+const LINE_RE = /^(\S+) \S+ \S+ \[([^\]]+)\] "(\S+) (\S+)(?: [^"]*)?" (\d{3}) \S+(?: "((?:[^"\\]|\\.)*)" "((?:[^"\\]|\\.)*)")?/;
+const MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 };
+
+export const ASSET_RE = /\.(css|js|mjs|woff2?|ttf|otf|png|jpe?g|gif|webp|avif|svg|ico)(\?|$)/i;
+// A page: no extension, a trailing slash, or .html. Feeds, robots.txt, sitemaps and JSON are not.
+const PAGE_RE = /^[^?]*(\/|\/[^./?]+|\.html?)(\?|$)/i;
+const NOT_PAGE_RE = /^\/(robots\.txt|sitemap[^/]*\.xml|favicon|\.well-known\/|feed\/?$|rss\/?$|api\/)/i;
+// UAs that are not a browser and not in the vendor tables: libraries, monitors, previewers.
+const GENERIC_BOT_RE = /bot\b|bot\/|crawl|spider|slurp|fetch|scan|curl\/|wget\/|python|go-http|java\/|okhttp|axios|node-fetch|undici|httpclient|libwww|headless|phantom|lighthouse|pingdom|uptime|monitor|preview|validator|feed|^-?$/i;
+// First octets of AWS (3, 13, 15, 16, 18, 44, 52, 54) and GCP (34, 35) address space. Crude on
+// purpose: it only ever applies to a search-referred "person", where a residential reader on
+// these octets is far rarer than a fake. Exported so a site can see what it filters.
+export const CLOUD_FIRST_OCTETS = Object.freeze([3, 13, 15, 16, 18, 34, 35, 44, 52, 54]);
+const CLOUD = new Set(CLOUD_FIRST_OCTETS);
+
+/** One combined-format line → request, or null. */
+export function parseLine(line) {
+    const m = LINE_RE.exec(line);
+    if (!m) return null;
+    const [, ip, time, method, target, status, referer = '-', ua = '-'] = m;
+    const t = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/.exec(time);
+    if (!t || !(t[2] in MONTHS)) return null;
+    const off = (t[7] === '-' ? -1 : 1) * (Number(t[8]) * 60 + Number(t[9]));
+    const ts = Date.UTC(+t[3], MONTHS[t[2]], +t[1], +t[4], +t[5], +t[6]) - off * 60_000;
+    const unesc = s => s.replace(/\\x22/g, '"').replace(/\\"/g, '"');
+    return { ip, ts, method, target, status: Number(status), referer: unesc(referer), ua: unesc(ua) };
+}
+
+/** Day (YYYY-MM-DD) of a timestamp in an IANA zone. */
+export function dayIn(tz) {
+    const f = new Intl.DateTimeFormat('en-CA', { timeZone: tz, year: 'numeric', month: '2-digit', day: '2-digit' });
+    return ts => f.format(ts);
+}
+
+const pathOnly = t => String(t).split('?')[0];
+
+function countBy(map, key, n = 1) { map.set(key, (map.get(key) ?? 0) + n); }
+function top(map, limit) {
+    return [...map].sort((a, b) => b[1] - a[1] || String(a[0]).localeCompare(String(b[0]))).slice(0, limit);
+}
+
+/**
+ * A streaming digest. add(line) for every log line, then finish(verifier) once.
+ * @param {object} o
+ * @param {string} [o.site]         the site's host; guessed from Referers when absent
+ * @param {string} [o.tz]           zone for days (default America/Los_Angeles, like GSC)
+ * @param {string} [o.from] [o.to]  keep only days in [from, to]
+ * @param {Set<string>} [o.sitemap] sitemap paths, to count crawler coverage on
+ * @param {Array<{label: string, re: RegExp}>} [o.goals] matched against "METHOD path status"
+ * @param {RegExp} [o.teamPath]     an IP that got a 2xx on such a path is the site's own team
+ * @param {Set<string>} [o.excludeIps]
+ * @param {RegExp} [o.allowPath]    a site's own paths that look like scanner probes
+ */
+export function createLogDigest({
+    site = null, tz = 'America/Los_Angeles', from = null, to = null, sitemap = null,
+    goals = [], teamPath = null, excludeIps = new Set(), allowPath = null, limit = 15,
+} = {}) {
+    const day = dayIn(tz);
+    const ips = new Map();       // ip → per-IP facts
+    const crawl = new Map();     // `${agent}\t${ip}` → { agent, ip, ua, requests, pages: Map(day → [path]) }
+    const generic = { requests: 0, ips: new Set(), uas: new Map() };
+    const scannerPaths = new Map();
+    const refHosts = new Map();
+    let lines = 0, unparsed = 0, outOfRange = 0, firstDay = null, lastDay = null;
+
+    function ipRec(ip) {
+        let r = ips.get(ip);
+        if (!r) {
+            r = { bot: false, scanner: false, asset: false, browser: false, team: false, views: [], goals: new Set() };
+            ips.set(ip, r);
+        }
+        return r;
+    }
+
+    function add(line) {
+        lines++;
+        const q = parseLine(line);
+        if (!q) { unparsed++; return; }
+        const d = day(q.ts);
+        if ((from && d < from) || (to && d > to)) { outOfRange++; return; }
+        if (!firstDay || d < firstDay) firstDay = d;
+        if (!lastDay || d > lastDay) lastDay = d;
+        const r = ipRec(q.ip);
+        const path = pathOnly(q.target);
+        const ok = q.status === 200 || q.status === 304;
+
+        // The 444 rule: a scanner path or a 444 marks the IP for the whole log, whatever its UA.
+        if (q.status === 444 || isScannerPath(q.target, { allow: allowPath })) {
+            r.scanner = true;
+            countBy(scannerPaths, path);
+        }
+        if (teamPath && q.status >= 200 && q.status < 300 && teamPath.test(path)) r.team = true;
+
+        const named = detectAgent(q.ua) || detectCrawler(q.ua);
+        const isPage = q.method === 'GET' && ok && PAGE_RE.test(q.target) && !NOT_PAGE_RE.test(path) && !isScannerPath(q.target, { allow: allowPath });
+        if (named) {
+            r.bot = true;
+            const k = `${named.key}\t${q.ip}`;
+            let c = crawl.get(k);
+            if (!c) { c = { agent: named.key, ip: q.ip, ua: q.ua, requests: 0, pages: new Map() }; crawl.set(k, c); }
+            c.requests++;
+            if (isPage) {
+                if (!c.pages.has(d)) c.pages.set(d, []);
+                c.pages.get(d).push(path);
+            }
+            return;
+        }
+        if (GENERIC_BOT_RE.test(q.ua) || !/^Mozilla\//.test(q.ua)) {
+            r.bot = true;
+            generic.requests++;
+            generic.ips.add(q.ip);
+            countBy(generic.uas, q.ua.slice(0, 80));
+            return;
+        }
+        r.browser = true;
+        if (q.method === 'GET' && ok && ASSET_RE.test(q.target)) {
+            r.asset = true;
+            // A stylesheet's Referer is the page that loaded it: the site's own host, almost always.
+            const h = refererHost(q.referer);
+            if (h) countBy(refHosts, h.replace(/^www\./, ''));
+        }
+        if (isPage) r.views.push({ ts: q.ts, day: d, path, target: q.target, referer: q.referer });
+        for (const g of goals) if (g.re.test(`${q.method} ${path} ${q.status}`)) r.goals.add(g.label);
+    }
+
+    function finish(verifier) {
+        // The site's own host: given, or the Referer host browsers' asset loads carry most.
+        const siteHost = site ? site.replace(/^www\./, '') : (top(refHosts, 1)[0]?.[0] ?? null);
+
+        // People.
+        const people = [];
+        const set = { bots: 0, scanners: 0, noAssets: 0, team: 0, excluded: 0, cloudFakes: 0 };
+        const cloudFakeEngines = new Map();
+        for (const [ip, r] of ips) {
+            if (!r.views.length && !r.browser) continue;
+            if (r.scanner) { if (r.browser) set.scanners++; continue; }
+            if (r.bot && !r.browser) continue;
+            if (!r.views.length) continue;
+            if (excludeIps.has(ip)) { set.excluded++; continue; }
+            if (r.team) { set.team++; continue; }
+            if (!r.asset) { set.noAssets++; continue; }
+            if (r.bot) { set.bots++; continue; }
+            r.views.sort((a, b) => a.ts - b.ts);
+            const land = r.views[0];
+            const src = classifySource(land.referer, land.target, { site: siteHost });
+            if (src.kind === 'search' && CLOUD.has(Number(ip.split('.')[0]))) { set.cloudFakes++; countBy(cloudFakeEngines, src.source); continue; }
+            people.push({ ip, land, src, views: r.views, goals: r.goals });
+        }
+
+        const perDay = new Map();
+        const bySource = new Map();
+        const pagePeople = new Map();
+        for (const p of people) {
+            const days = new Set(p.views.map(v => v.day));
+            for (const d of days) countBy(perDay, d);
+            const key = p.src.source;
+            let s = bySource.get(key);
+            if (!s) { s = { source: key, kind: p.src.kind, people: 0, landings: new Map(), days: new Map(), goals: new Map() }; bySource.set(key, s); }
+            s.people++;
+            countBy(s.days, p.land.day);
+            countBy(s.landings, p.land.path);
+            for (const g of p.goals) countBy(s.goals, g);
+            for (const path of new Set(p.views.map(v => v.path))) countBy(pagePeople, path);
+        }
+        // Every day of the log, zeros included: an average over only the days someone came is not one.
+        const dayList = [];
+        if (firstDay) {
+            for (let t = Date.parse(`${firstDay}T00:00:00Z`); t <= Date.parse(`${lastDay}T00:00:00Z`); t += 86_400_000) {
+                const date = new Date(t).toISOString().slice(0, 10);
+                dayList.push({ date, people: perDay.get(date) ?? 0 });
+            }
+        }
+
+        // People per 7-day bucket ending on the last day: unique IPs within each bucket.
+        const weeks = [];
+        if (lastDay) {
+            const end = Date.parse(`${lastDay}T00:00:00Z`);
+            const buckets = new Map();
+            const goalBuckets = new Map();
+            const weekOf = d => Math.floor((end - Date.parse(`${d}T00:00:00Z`)) / (7 * 86_400_000));
+            for (const p of people) {
+                for (const v of p.views) {
+                    const w = weekOf(v.day);
+                    if (!buckets.has(w)) buckets.set(w, new Set());
+                    buckets.get(w).add(p.ip);
+                }
+                // A goal belongs to the week of the person's first page view in the log.
+                const w = weekOf(p.land.day);
+                if (!goalBuckets.has(w)) goalBuckets.set(w, new Map());
+                for (const g of p.goals) countBy(goalBuckets.get(w), g);
+            }
+            for (const w of [...buckets.keys()].sort((a, b) => b - a)) {
+                const iso = ms => new Date(ms).toISOString().slice(0, 10);
+                const lo = iso(end - (7 * w + 6) * 86_400_000);
+                const gb = goalBuckets.get(w);
+                weeks.push({
+                    from: lo < firstDay ? firstDay : lo, to: iso(end - 7 * w * 86_400_000), people: buckets.get(w).size,
+                    goals: Object.fromEntries(goals.map(g => [g.label, gb?.get(g.label) ?? 0])),
+                });
+            }
+        }
+
+        const sources = [...bySource.values()]
+            .sort((a, b) => b.people - a.people || a.source.localeCompare(b.source))
+            .map(s => ({
+                source: s.source, kind: s.kind, people: s.people,
+                landings: top(s.landings, 5).map(([path, n]) => ({ path, people: n })),
+                // People by the day of their first page view (the click a search console counts).
+                landingDays: Object.fromEntries([...s.days].sort((a, b) => a[0].localeCompare(b[0]))),
+                goals: Object.fromEntries(top(s.goals, 50)),
+            }));
+        const goalTotals = Object.fromEntries(goals.map(g => [g.label, people.filter(p => p.goals.has(g.label)).length]));
+
+        // Crawlers: one verdict per (agent, IP). isKnownScannerIp is the 444 rule from this log.
+        const isKnownScannerIp = ip => ips.get(ip)?.scanner === true;
+        const agents = new Map();
+        for (const c of crawl.values()) {
+            const v = verifier
+                ? verifier.check(c.ip, c.ua)
+                : { verdict: 'unknown', reason: 'not-checked', kind: detectAgent(c.ua) ? 'ai' : 'crawler' };
+            const verdict = v.verdict === 'unknown' && isKnownScannerIp(c.ip) ? 'spoofed' : v.verdict;
+            c.verdict = verdict;
+            let a = agents.get(c.agent);
+            if (!a) {
+                const e = detectAgent(c.ua) || detectCrawler(c.ua);
+                a = {
+                    agent: c.agent, kind: detectAgent(c.ua) ? 'ai' : 'crawler', purpose: e?.purpose ?? null,
+                    requests: 0, ips: 0,
+                    verified: { ips: 0, requests: 0, pageFetches: 0, sitemapFetches: 0, pages: new Set(), perDay: new Map() },
+                    spoofed: { ips: 0, requests: 0 },
+                    unknown: { ips: 0, requests: 0, pageFetches: 0 },
+                };
+                agents.set(c.agent, a);
+            }
+            a.requests += c.requests;
+            a.ips++;
+            const fetches = [...c.pages.values()].flat();
+            if (verdict === 'verified') {
+                const b = a.verified;
+                b.ips++; b.requests += c.requests; b.pageFetches += fetches.length;
+                for (const [d, ps] of c.pages) {
+                    const counted = sitemap ? ps.filter(p => sitemap.has(p)) : ps;
+                    countBy(b.perDay, d, counted.length);
+                    b.sitemapFetches += sitemap ? counted.length : 0;
+                    for (const p of ps) b.pages.add(p);
+                }
+            } else if (verdict === 'spoofed') {
+                a.spoofed.ips++; a.spoofed.requests += c.requests;
+            } else {
+                a.unknown.ips++; a.unknown.requests += c.requests; a.unknown.pageFetches += fetches.length;
+            }
+        }
+        const crawlers = [...agents.values()]
+            .sort((x, y) => y.verified.pageFetches - x.verified.pageFetches || y.requests - x.requests || x.agent.localeCompare(y.agent))
+            .map(a => {
+                const v = a.verified;
+                const covered = sitemap ? [...v.pages].filter(p => sitemap.has(p)).length : null;
+                const offSitemap = sitemap ? [...v.pages].filter(p => !sitemap.has(p)).sort() : null;
+                return {
+                    agent: a.agent, kind: a.kind, purpose: a.purpose, requests: a.requests, ips: a.ips,
+                    verified: {
+                        ips: v.ips, requests: v.requests, pageFetches: v.pageFetches,
+                        sitemapFetches: sitemap ? v.sitemapFetches : null,
+                        uniquePages: v.pages.size, sitemapCovered: covered,
+                        offSitemapPages: offSitemap ? offSitemap.slice(0, 10) : null,
+                        perWeek: weekly(v.perDay, lastDay, firstDay),
+                    },
+                    spoofed: a.spoofed,
+                    unknown: a.unknown,
+                };
+            });
+
+        let scannerIps = 0;
+        for (const r of ips.values()) if (r.scanner) scannerIps++;
+
+        return {
+            range: { from: firstDay, to: lastDay, tz },
+            site: { host: siteHost, guessed: !site },
+            lines: { total: lines, unparsed, outOfRange },
+            people: {
+                total: people.length,
+                perDay: dayList,
+                avgPerDay: dayList.length ? Math.round((10 * dayList.reduce((s, d) => s + d.people, 0)) / dayList.length) / 10 : 0,
+                weeks,
+                setAside: { ...set, cloudFakeReferers: Object.fromEntries(top(cloudFakeEngines, 10)) },
+            },
+            sources,
+            goals: goalTotals,
+            pages: top(pagePeople, limit).map(([path, n]) => ({ path, people: n })),
+            crawlers,
+            sitemap: sitemap ? sitemapCoverage(sitemap, crawlers, crawl, limit) : null,
+            otherBots: { requests: generic.requests, ips: generic.ips.size, topUserAgents: top(generic.uas, 5).map(([ua, n]) => ({ ua, requests: n })) },
+            scanners: { ips: scannerIps, topPaths: top(scannerPaths, 5).map(([path, n]) => ({ path, requests: n })) },
+        };
+    }
+
+    return { add, finish };
+}
+
+// Sitemap URLs no verified search crawler fetched in this log. On synacl.com one clean guide was
+// never fetched by Googlebot in 21 days and had 0 impressions; it was linked from too few pages.
+function sitemapCoverage(sitemap, crawlers, crawl, limit) {
+    const fetched = new Set();
+    for (const c of crawl.values()) {
+        if (c.verdict !== 'verified' || crawlers.find(x => x.agent === c.agent)?.purpose !== 'search') continue;
+        for (const ps of c.pages.values()) for (const p of ps) fetched.add(p);
+    }
+    const never = [...sitemap].filter(p => !fetched.has(p));
+    return { urls: sitemap.size, fetchedBySearch: sitemap.size - never.length, neverFetched: never.slice(0, limit), neverFetchedCount: never.length };
+}
+
+// Verified fetches per 7-day bucket ending on the last day: sitemap URLs only when a sitemap was given.
+function weekly(perDay, lastDay, firstDay) {
+    if (!lastDay) return [];
+    const end = Date.parse(`${lastDay}T00:00:00Z`);
+    const b = new Map();
+    for (const [d, n] of perDay) {
+        const w = Math.floor((end - Date.parse(`${d}T00:00:00Z`)) / (7 * 86_400_000));
+        countBy(b, w, n);
+    }
+    const out = [];
+    const nWeeks = Math.floor((end - Date.parse(`${firstDay}T00:00:00Z`)) / (7 * 86_400_000));
+    for (let w = nWeeks; w >= 0; w--) {
+        const iso = ms => new Date(ms).toISOString().slice(0, 10);
+        const lo = iso(end - (7 * w + 6) * 86_400_000);
+        out.push({ from: lo < firstDay ? firstDay : lo, to: iso(end - 7 * w * 86_400_000), fetches: b.get(w) ?? 0 });
+    }
+    return out;
+}
+
+/** Feed every line of a log file (plain or .gz) to digest.add. */
+export async function readLog(file, digest) {
+    let stream = createReadStream(file);
+    if (/\.gz$/i.test(file)) stream = stream.pipe(createGunzip());
+    const rl = createInterface({ input: stream, crlfDelay: Infinity });
+    for await (const line of rl) if (line) digest.add(line);
+}

package/src/digest/referrer.js

@@ -0,0 +1,77 @@
+// Where a visit came from: the Referer of its first page view, or the landing URL's utm_source.
+//
+// utm_source wins because the AI assistants mostly send no Referer: ChatGPT tags its links
+// `?utm_source=chatgpt.com`, and on synacl.com that tag was the only trace of what became the
+// channel with the most tool use per visitor (2026-09). ChatGPT traffic never appears in GSC.
+
+// [source, kind, host test]. First match wins, so gemini.google.com sits before google.
+const SOURCES = [
+    ['chatgpt', 'ai', /(^|\.)(chatgpt\.com|chat\.openai\.com)$/],
+    ['perplexity', 'ai', /(^|\.)perplexity\.ai$/],
+    ['claude', 'ai', /(^|\.)claude\.ai$/],
+    ['gemini', 'ai', /^gemini\.google\.com$/],
+    ['copilot', 'ai', /^copilot\.microsoft\.com$/],
+    ['deepseek', 'ai', /(^|\.)deepseek\.com$/],
+    ['google', 'search', /(^|\.)google\.[a-z.]+$/],
+    ['bing', 'search', /(^|\.)bing\.com$/],
+    ['duckduckgo', 'search', /(^|\.)duckduckgo\.com$/],
+    ['yandex', 'search', /(^|\.)yandex\.[a-z.]+$|(^|\.)ya\.ru$/],
+    ['baidu', 'search', /(^|\.)baidu\.com$/],
+    ['yahoo', 'search', /(^|\.)yahoo\.[a-z.]+$/],
+    ['ecosia', 'search', /(^|\.)ecosia\.org$/],
+    ['brave', 'search', /^search\.brave\.com$/],
+    ['naver', 'search', /(^|\.)naver\.com$/],
+    ['reddit', 'social', /(^|\.)reddit\.com$|^redd\.it$/],
+    ['hackernews', 'social', /^news\.ycombinator\.com$/],
+    ['x', 'social', /^(t\.co|x\.com|(www\.|mobile\.)?twitter\.com)$/],
+    ['linkedin', 'social', /(^|\.)linkedin\.com$|^lnkd\.in$/],
+    ['facebook', 'social', /(^|\.)facebook\.com$|^fb\.me$/],
+    ['youtube', 'social', /(^|\.)youtube\.com$|^youtu\.be$/],
+    ['dev.to', 'social', /^dev\.to$/],
+    ['github', 'social', /(^|\.)github\.com$/],
+    ['medium', 'social', /(^|\.)medium\.com$/],
+    ['lobsters', 'social', /^lobste\.rs$/],
+    ['bluesky', 'social', /^bsky\.app$/],
+];
+
+const UTM = new Map([
+    ['chatgpt.com', 'chatgpt'], ['chatgpt', 'chatgpt'], ['openai', 'chatgpt'],
+    ['perplexity', 'perplexity'], ['perplexity.ai', 'perplexity'], ['claude.ai', 'claude'], ['claude', 'claude'],
+    ['copilot.com', 'copilot'], ['gemini', 'gemini'], ['reddit', 'reddit'], ['hn', 'hackernews'],
+    ['news.ycombinator.com', 'hackernews'], ['twitter', 'x'], ['x', 'x'], ['linkedin', 'linkedin'],
+    ['devto', 'dev.to'], ['dev.to', 'dev.to'], ['newsletter', 'newsletter'], ['email', 'email'],
+]);
+const KIND = new Map(SOURCES.map(([s, k]) => [s, k]));
+
+function hostOf(url) {
+    try { return new URL(url).hostname.toLowerCase(); } catch { return null; }
+}
+
+const bare = h => h.replace(/^www\./, '');
+
+/**
+ * { source, kind } for a landing. kind: ai · search · social · referral · internal · direct.
+ * `site` is the site's own host (www. ignored); a Referer from it is `internal`.
+ */
+export function classifySource(referer, target, { site = null } = {}) {
+    const q = String(target ?? '').split('?')[1];
+    if (q) {
+        const utm = new URLSearchParams(q).get('utm_source');
+        if (utm) {
+            const u = utm.toLowerCase();
+            const s = UTM.get(u) ?? u;
+            return { source: s, kind: KIND.get(s) ?? 'campaign' };
+        }
+    }
+    if (!referer || referer === '-') return { source: 'direct', kind: 'direct' };
+    const h = hostOf(referer);
+    if (!h) return { source: 'direct', kind: 'direct' };
+    if (site && bare(h) === bare(site)) return { source: 'internal', kind: 'internal' };
+    for (const [source, kind, re] of SOURCES) if (re.test(h)) return { source, kind };
+    return { source: bare(h), kind: 'referral' };
+}
+
+/** The referer's host, for guessing the site's own host when `site` is not given. */
+export function refererHost(referer) {
+    return referer && referer !== '-' ? hostOf(referer) : null;
+}

package/src/digest/zip.js

@@ -0,0 +1,41 @@
+// The smallest zip reader that opens a Search Console export: walks the central directory and
+// inflates stored (0) or deflated (8) entries with node:zlib. No zip64, no encryption, which
+// no GSC export uses.
+
+import { inflateRawSync } from 'node:zlib';
+
+const EOCD_SIG = 0x06054b50;
+const CEN_SIG = 0x02014b50;
+const LOC_SIG = 0x04034b50;
+
+/** Map of entry name → Buffer for every file in a zip. Throws on anything it can't read. */
+export function readZip(buf) {
+    // The end-of-central-directory record sits in the last 22 + up to 65535 (comment) bytes.
+    let eocd = -1;
+    for (let i = buf.length - 22; i >= Math.max(0, buf.length - 22 - 0xffff); i--) {
+        if (buf.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
+    }
+    if (eocd < 0) throw new Error('not a zip file');
+    const count = buf.readUInt16LE(eocd + 10);
+    let p = buf.readUInt32LE(eocd + 16);
+    const out = new Map();
+    for (let n = 0; n < count; n++) {
+        if (buf.readUInt32LE(p) !== CEN_SIG) throw new Error('broken zip central directory');
+        const method = buf.readUInt16LE(p + 10);
+        const size = buf.readUInt32LE(p + 20);
+        const nameLen = buf.readUInt16LE(p + 28);
+        const extraLen = buf.readUInt16LE(p + 30);
+        const commentLen = buf.readUInt16LE(p + 32);
+        const local = buf.readUInt32LE(p + 42);
+        const name = buf.toString('utf8', p + 46, p + 46 + nameLen);
+        p += 46 + nameLen + extraLen + commentLen;
+        if (name.endsWith('/')) continue;
+        if (buf.readUInt32LE(local) !== LOC_SIG) throw new Error(`broken zip entry ${name}`);
+        const start = local + 30 + buf.readUInt16LE(local + 26) + buf.readUInt16LE(local + 28);
+        const data = buf.subarray(start, start + size);
+        if (method === 0) out.set(name, Buffer.from(data));
+        else if (method === 8) out.set(name, inflateRawSync(data));
+        else throw new Error(`zip entry ${name} uses compression method ${method}`);
+    }
+    return out;
+}

package/src/file-store.js

@@ -0,0 +1,49 @@
+// A store that keeps the lists in one JSON file, for the CLI and for anyone who wants the
+// lists to survive a restart without a database.
+//
+// The default CLI path is under os.tmpdir(), which on Linux is a directory every user shares. A
+// planted lists file would make a spoofed IP "verified", so: the directory must be ours and not
+// group/world-writable, the file must be ours, and writes go through an O_EXCL temp file
+// (never through a symlink someone left at that name) and an atomic rename.
+
+import { mkdir, readFile, writeFile, rename, lstat, unlink } from 'node:fs/promises';
+import { dirname } from 'node:path';
+
+const uid = typeof process.getuid === 'function' ? process.getuid() : null;
+
+function trusted(st) {
+    if (uid === null) return true;   // Windows: no POSIX owner to check
+    return st.uid === uid && (st.mode & 0o022) === 0;
+}
+
+/** { get, set } over one JSON file. A missing, foreign or unparseable file reads as null. */
+export function fileStore(file) {
+    const dir = dirname(file);
+    return {
+        async get() {
+            let text;
+            try {
+                const [d, f] = await Promise.all([lstat(dir), lstat(file)]);
+                if (!d.isDirectory() || !trusted(d) || !f.isFile() || !trusted(f)) return null;
+                text = await readFile(file, 'utf8');
+            } catch (err) {
+                if (err.code === 'ENOENT') return null;
+                throw err;
+            }
+            try { return JSON.parse(text); } catch { return null; }
+        },
+        async set(doc) {
+            await mkdir(dir, { recursive: true, mode: 0o700 });
+            const d = await lstat(dir);
+            if (!d.isDirectory() || !trusted(d)) throw new Error(`refusing to write into ${dir}: not ours, or writable by others`);
+            const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
+            await writeFile(tmp, JSON.stringify(doc), { mode: 0o600, flag: 'wx' });
+            try {
+                await rename(tmp, file);
+            } catch (err) {
+                await unlink(tmp).catch(() => {});
+                throw err;
+            }
+        },
+    };
+}

package/src/index.js

@@ -0,0 +1,16 @@
+// siteread: is that crawler who it says it is? Vendor IP lists + a User-Agent table +
+// scanner-path signatures, with zero dependencies. See README.md.
+
+export { AGENTS, CRAWLERS, detectAgent, detectCrawler, isSearchCrawler } from './agents.js';
+export { SCANNER_PATH_RE, isScannerPath } from './scanner.js';
+export {
+    SOURCES, parseCidr, parseList, normIp,
+    REFRESH_AFTER_MS, DROP_AFTER_MS, MIN_PREFIX, MAX_PREFIXES, MAX_BODY_CHARS,
+} from './ip-ranges.js';
+export { createVerifier, memoryStore } from './verifier.js';
+export { fileStore } from './file-store.js';
+export { VERSION } from './version.js';
+export { digest, parseSitemap, parseGoal } from './digest/index.js';
+export { parseCsv } from './digest/csv.js';
+export { classifySource } from './digest/referrer.js';
+export { parseLine } from './digest/log.js';