siteread 0.0.0-stage → 0.2.0

package/src/agents.js

@@ -0,0 +1,85 @@
+// Who a request CLAIMS to be, from its User-Agent. A claim only: anyone can send any UA, and
+// on the site this was extracted from most AI user-agents outside the vendors' IP lists were a
+// scanner cycling through them (ip-ranges.js has the numbers). verifier.js turns a claim into a
+// verdict.
+//
+// Ported from the AI-traffic classifier Synacl Labs runs on synacl.com (2026-10).
+//
+// purpose: `answer`   fetched live while answering someone (ChatGPT-User, Perplexity-User)
+//          `search`   building the assistant's search index (OAI-SearchBot, PerplexityBot)
+//          `training` a model-training crawl (GPTBot, ClaudeBot, CCBot)
+
+// Ordered: the FIRST match on the User-Agent wins. Not AI (left out on purpose, see CRAWLERS):
+// PetalBot, GoogleOther, Googlebot, bingbot. Copilot's fetches cannot be told apart from Bing's.
+export const AGENTS = Object.freeze([
+    { key: 'chatgpt-user',         re: /ChatGPT-User/i,         assistant: 'chatgpt',     purpose: 'answer' },
+    { key: 'oai-searchbot',        re: /OAI-SearchBot/i,        assistant: 'chatgpt',     purpose: 'search' },
+    { key: 'gptbot',               re: /GPTBot/i,               assistant: 'chatgpt',     purpose: 'training' },
+    // Before claude-user: Claude Code's WebFetch sends `Claude-User (claude-code/2.1.292; …)` from
+    // the user's OWN machine, so it can never be in Anthropic's IP list and stays on the 444 rule.
+    { key: 'claude-code',          re: /^claude-code\/|^Claude-User \(claude-code\//i, assistant: 'claude', purpose: 'answer',
+        note: "Claude Code's WebFetch runs from the user's own machine, so no vendor list can cover it" },
+    { key: 'claude-user',          re: /Claude-User/i,          assistant: 'claude',      purpose: 'answer' },
+    { key: 'claude-searchbot',     re: /Claude-SearchBot/i,     assistant: 'claude',      purpose: 'search' },
+    { key: 'claudebot',            re: /ClaudeBot/i,            assistant: 'claude',      purpose: 'training' },
+    { key: 'perplexity-user',      re: /Perplexity-User/i,      assistant: 'perplexity',  purpose: 'answer' },
+    { key: 'perplexitybot',        re: /PerplexityBot/i,        assistant: 'perplexity',  purpose: 'search' },
+    { key: 'applebot',             re: /Applebot/i,             assistant: 'apple',       purpose: 'search' },
+    { key: 'amazonbot',            re: /Amazonbot/i,            assistant: 'amazon',      purpose: 'search' },
+    { key: 'meta-externalfetcher', re: /meta-externalfetcher/i, assistant: 'meta',        purpose: 'answer' },
+    { key: 'meta-externalagent',   re: /meta-externalagent/i,   assistant: 'meta',        purpose: 'training' },
+    { key: 'duckassistbot',        re: /DuckAssistBot/i,        assistant: 'duckduckgo',  purpose: 'answer' },
+    { key: 'mistralai-user',       re: /MistralAI-User/i,       assistant: 'mistral',     purpose: 'answer' },
+    { key: 'youbot',               re: /YouBot/i,               assistant: 'you',         purpose: 'search' },
+    // Not a real User-Agent (Google-Extended is a robots.txt token): always spoofed in practice.
+    // Google's crawlers obey it, but fetch as Googlebot; nothing real ever sends it as a UA.
+    { key: 'google-extended',      re: /Google-Extended/i,      assistant: 'gemini',      purpose: 'training',
+        robotsTokenOnly: true, note: 'Google-Extended is a robots.txt token, never a real user-agent' },
+    { key: 'ccbot',                re: /CCBot/i,                assistant: 'commoncrawl', purpose: 'training' },
+    { key: 'bytespider',           re: /Bytespider/i,           assistant: 'bytedance',   purpose: 'training' },
+    { key: 'cohere',               re: /cohere-ai/i,            assistant: 'cohere',      purpose: 'training' },
+    { key: 'deepseek',             re: /DeepSeekBot/i,          assistant: 'deepseek',    purpose: 'training' },
+    { key: 'xai',                  re: /xAI-Grok|GrokBot/i,     assistant: 'grok',        purpose: 'training' },
+    { key: 'ai2bot',               re: /AI2Bot/i,               assistant: 'ai2',         purpose: 'training' },
+].map(Object.freeze));
+
+// Classic search-engine and SEO-tool crawlers, the non-AI half of the original CRAWLER_RE. The
+// AI names it also matched (GPTBot, OAI-SearchBot, ChatGPT, ClaudeBot, Applebot, Amazonbot,
+// Bytespider) live in AGENTS, which is checked first. purpose: `search` (an engine's index),
+// `seo` (a backlink/SEO tool's crawl) or `preview` (a link-preview fetch).
+export const CRAWLERS = Object.freeze([
+    { key: 'googlebot',           re: /Googlebot/i,           operator: 'google',     purpose: 'search' },
+    { key: 'bingbot',             re: /bingbot/i,             operator: 'bing',       purpose: 'search' },
+    { key: 'yandexbot',           re: /YandexBot/i,           operator: 'yandex',     purpose: 'search' },
+    { key: 'duckduckbot',         re: /DuckDuckBot/i,         operator: 'duckduckgo', purpose: 'search' },
+    { key: 'slurp',               re: /Slurp/i,               operator: 'yahoo',      purpose: 'search' },
+    { key: 'petalbot',            re: /PetalBot/i,            operator: 'huawei',     purpose: 'search' },
+    { key: 'ahrefsbot',           re: /AhrefsBot/i,           operator: 'ahrefs',     purpose: 'seo' },
+    { key: 'semrushbot',          re: /SemrushBot/i,          operator: 'semrush',    purpose: 'seo' },
+    { key: 'mj12bot',             re: /MJ12bot/i,             operator: 'majestic',   purpose: 'seo' },
+    { key: 'facebookexternalhit', re: /facebookexternalhit/i, operator: 'meta',       purpose: 'preview' },
+].map(Object.freeze));
+
+// One cheap test before the ordered walk: most lines are not an AI agent at all.
+const ANY_AGENT_RE = new RegExp(AGENTS.map(a => a.re.source).join('|'), 'i');
+const ANY_CRAWLER_RE = new RegExp(CRAWLERS.map(c => c.re.source).join('|'), 'i');
+
+/** The AGENTS entry a User-Agent claims to be, or null. */
+export function detectAgent(ua) {
+    if (typeof ua !== 'string' || !ua || !ANY_AGENT_RE.test(ua)) return null;
+    for (const a of AGENTS) if (a.re.test(ua)) return a;
+    return null;
+}
+
+/** The CRAWLERS entry a User-Agent claims to be, or null. Not an AI agent: those match AGENTS. */
+export function detectCrawler(ua) {
+    if (typeof ua !== 'string' || !ua || !ANY_CRAWLER_RE.test(ua)) return null;
+    if (detectAgent(ua)) return null;
+    for (const c of CRAWLERS) if (c.re.test(ua)) return c;
+    return null;
+}
+
+/** True when the UA claims to be a classic search-engine or SEO crawler (never an AI agent). */
+export function isSearchCrawler(ua) {
+    return detectCrawler(ua) !== null;
+}

package/src/digest/csv.js

@@ -0,0 +1,37 @@
+// RFC 4180 CSV, enough for a Search Console export: quoted fields, "" escapes, CRLF or LF,
+// a UTF-8 BOM. No dependencies.
+
+/** Rows of a CSV text as arrays of strings. Blank lines are dropped. */
+export function parseCsv(text) {
+    const s = String(text ?? '').replace(/^/, '');
+    const rows = [];
+    let row = [];
+    let field = '';
+    let quoted = false;
+    for (let i = 0; i < s.length; i++) {
+        const c = s[i];
+        if (quoted) {
+            if (c === '"') {
+                if (s[i + 1] === '"') { field += '"'; i++; } else quoted = false;
+            } else field += c;
+        } else if (c === '"') quoted = true;
+        else if (c === ',') { row.push(field); field = ''; }
+        else if (c === '\n' || c === '\r') {
+            if (c === '\r' && s[i + 1] === '\n') i++;
+            row.push(field); field = '';
+            if (row.some(f => f !== '')) rows.push(row);
+            row = [];
+        } else field += c;
+    }
+    row.push(field);
+    if (row.some(f => f !== '')) rows.push(row);
+    return rows;
+}
+
+/** The CSV as objects keyed by its header row. */
+export function parseCsvObjects(text) {
+    const [head, ...rows] = parseCsv(text);
+    if (!head) return { header: [], rows: [] };
+    const header = head.map(h => h.trim());
+    return { header, rows: rows.map(r => Object.fromEntries(header.map((h, i) => [h, (r[i] ?? '').trim()]))) };
+}

package/src/digest/gsc.js

@@ -0,0 +1,233 @@
+// A Google Search Console Performance export (the "Export → Download CSV" zip, or the folder it
+// unzips to) → the numbers a weekly read needs.
+//
+// The method is the one that held up over six weekly reads of synacl.com (2026-09 → 10):
+//   - Totals come from the daily series (Dates.csv; older exports call it Chart.csv), never by
+//     summing Pages.csv: a page table double-counts an impression that showed two URLs.
+//   - Position is impression-weighted (Σ position·impressions / Σ impressions). A plain mean
+//     lets a page shown once at 90 outweigh one shown 50 times at 5.
+//   - GSC lags ~2-3 days and exports the last days as zeros. Trailing all-zero days are cut and
+//     the cut is reported, or every "last week" reads as a drop.
+//   - Pages are keyed by path: one export listed https://www. and the apex separately.
+//   - Queries.csv only names part of the traffic (anonymised queries are left out). The named
+//     share is reported so nobody reads the query table as the whole story.
+// GSC days are US Pacific days; the log half (log.js) buckets in the same zone by default.
+
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import path from 'node:path';
+import { parseCsvObjects } from './csv.js';
+import { readZip } from './zip.js';
+
+// English export file names → table. A non-English UI exports translated names and headers.
+const FILES = {
+    dates: ['Dates.csv', 'Chart.csv'],
+    queries: ['Queries.csv'],
+    pages: ['Pages.csv'],
+    countries: ['Countries.csv'],
+    devices: ['Devices.csv'],
+    appearance: ['Search appearance.csv'],
+    filters: ['Filters.csv'],
+};
+
+/** { name → text } of the CSVs in an export folder or zip. */
+export function readExport(p) {
+    const st = statSync(p);
+    const files = new Map();
+    if (st.isDirectory()) {
+        for (const f of readdirSync(p)) if (f.toLowerCase().endsWith('.csv')) files.set(f, readFileSync(path.join(p, f), 'utf8'));
+    } else {
+        for (const [name, buf] of readZip(readFileSync(p))) {
+            if (name.toLowerCase().endsWith('.csv')) files.set(path.basename(name), buf.toString('utf8'));
+        }
+    }
+    return files;
+}
+
+function pick(files, names) {
+    for (const n of names) for (const [f, text] of files) if (f.toLowerCase() === n.toLowerCase()) return text;
+    return null;
+}
+
+// "Clicks", or in an export that split search types, the web column ("Web Clicks" or similar).
+function col(header, metric) {
+    if (header.includes(metric)) return metric;
+    const hits = header.filter(h => h.toLowerCase().endsWith(metric.toLowerCase()));
+    return hits.find(h => /^web\b/i.test(h)) ?? hits[0] ?? null;
+}
+
+function num(v) {
+    const n = Number(String(v ?? '').replace(/[%,\s]/g, ''));
+    return Number.isFinite(n) ? n : 0;
+}
+
+function table(text, keyCol) {
+    if (text == null) return null;
+    const { header, rows } = parseCsvObjects(text);
+    const key = header.includes(keyCol) ? keyCol : header[0];
+    const c = { clicks: col(header, 'Clicks'), impressions: col(header, 'Impressions'), position: col(header, 'Position') };
+    if (!c.clicks || !c.impressions) throw new Error(`no Clicks/Impressions columns in the ${keyCol} table (header: ${header.join(', ')}); is the export in English?`);
+    return rows.map(r => ({
+        key: r[key],
+        clicks: num(r[c.clicks]),
+        impressions: num(r[c.impressions]),
+        position: c.position ? num(r[c.position]) : null,
+    }));
+}
+
+export const round = (n, d = 1) => (n == null || !Number.isFinite(n) ? null : Math.round(n * 10 ** d) / 10 ** d);
+
+/** Totals of rows { clicks, impressions, position }: CTR in %, position impression-weighted. */
+export function totals(rows) {
+    let clicks = 0, impressions = 0, wpos = 0, wimp = 0;
+    for (const r of rows) {
+        clicks += r.clicks;
+        impressions += r.impressions;
+        if (r.position) { wpos += r.position * r.impressions; wimp += r.impressions; }
+    }
+    return {
+        impressions,
+        clicks,
+        ctr: impressions ? round((100 * clicks) / impressions, 1) : null,
+        position: wimp ? round(wpos / wimp, 1) : null,
+    };
+}
+
+const DAY = 86_400_000;
+const dayMs = d => Date.parse(`${d}T00:00:00Z`);
+const isoDay = ms => new Date(ms).toISOString().slice(0, 10);
+
+/** 7-day buckets ending on `last`, newest last; the oldest may be partial. */
+export function weekBuckets(days, last) {
+    const end = dayMs(last);
+    const byWeek = new Map();
+    for (const d of days) {
+        const w = Math.floor((end - dayMs(d.date)) / (7 * DAY));
+        if (w < 0) continue;
+        if (!byWeek.has(w)) byWeek.set(w, []);
+        byWeek.get(w).push(d);
+    }
+    return [...byWeek.keys()].sort((a, b) => b - a).map(w => {
+        const rows = byWeek.get(w);
+        const t = totals(rows);
+        return {
+            from: isoDay(end - (7 * w + 6) * DAY),
+            to: isoDay(end - 7 * w * DAY),
+            days: rows.length,
+            ...t,
+            impressionsPerDay: round(t.impressions / rows.length, 1),
+            clicksPerDay: round(t.clicks / rows.length, 2),
+        };
+    });
+}
+
+function pathOf(url) {
+    try { const u = new URL(url); return `${u.pathname}${u.search}`; } catch { return url; }
+}
+
+function topRows(rows, limit, { merge } = {}) {
+    let list = rows;
+    if (merge) {
+        // Same path from two hosts (www + apex): sum, weighting position by impressions.
+        const by = new Map();
+        for (const r of rows) {
+            const k = merge(r.key);
+            if (!by.has(k)) by.set(k, []);
+            by.get(k).push(r);
+        }
+        list = [...by].map(([key, rs]) => ({ key, ...totals(rs) }));
+    } else {
+        list = rows.map(r => ({ ...r, ctr: r.impressions ? round((100 * r.clicks) / r.impressions, 1) : null, position: round(r.position, 1) }));
+    }
+    list.sort((a, b) => b.impressions - a.impressions || b.clicks - a.clicks || String(a.key).localeCompare(String(b.key)));
+    return { count: list.length, rows: limit ? list.slice(0, limit) : list };
+}
+
+function withPrev(cur, prev) {
+    if (!prev) return cur;
+    const old = new Map(prev.rows.map(r => [r.key, r]));
+    return {
+        ...cur,
+        rows: cur.rows.map(r => {
+            const p = old.get(r.key);
+            return { ...r, prev: p ? { impressions: p.impressions, clicks: p.clicks, position: p.position } : null };
+        }),
+    };
+}
+
+/**
+ * Digest one export. `split` (YYYY-MM-DD) adds a before/after comparison around a change, per
+ * day so unequal windows compare. `prev` is an earlier digest's `gsc` object: rows gain `prev`.
+ */
+export function digestGsc(files, { split = null, limit = 25, prev = null } = {}) {
+    const warnings = [];
+    const filters = (() => {
+        const t = pick(files, FILES.filters);
+        if (t == null) return null;
+        return Object.fromEntries(parseCsvObjects(t).rows.map(r => Object.values(r)).filter(v => v.length >= 2).map(([k, v]) => [k, v]));
+    })();
+
+    const datesText = pick(files, FILES.dates);
+    if (datesText == null) throw new Error('no Dates.csv (or Chart.csv) in the export: export from Search Console → Performance → Export → Download CSV');
+    let days = table(datesText, 'Date')
+        .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d.key))
+        .map(d => ({ date: d.key, clicks: d.clicks, impressions: d.impressions, position: d.position }))
+        .sort((a, b) => a.date.localeCompare(b.date));
+    const trimmed = [];
+    while (days.length && days.at(-1).impressions === 0 && days.at(-1).clicks === 0) trimmed.unshift(days.pop().date);
+    if (trimmed.length) warnings.push(`cut ${trimmed.length} trailing day(s) with no data (GSC lag): ${trimmed.join(', ')}`);
+    if (!days.length) throw new Error('the export has no days with impressions');
+
+    const first = days[0].date;
+    const last = days.at(-1).date;
+    const all = totals(days);
+    const weeks = weekBuckets(days, last);
+
+    let windows = null;
+    if (split) {
+        const before = days.filter(d => d.date < split);
+        const after = days.filter(d => d.date >= split);
+        const side = (rows) => {
+            const t = totals(rows);
+            return { from: rows[0]?.date ?? null, to: rows.at(-1)?.date ?? null, days: rows.length, ...t,
+                impressionsPerDay: rows.length ? round(t.impressions / rows.length, 1) : null,
+                clicksPerDay: rows.length ? round(t.clicks / rows.length, 2) : null };
+        };
+        windows = { split, before: side(before), after: side(after) };
+        if (!before.length || !after.length) warnings.push(`--split ${split} leaves one side empty (data runs ${first} → ${last})`);
+    }
+
+    const q = table(pick(files, FILES.queries), 'Top queries');
+    const pg = table(pick(files, FILES.pages), 'Top pages');
+    const co = table(pick(files, FILES.countries), 'Country');
+    const de = table(pick(files, FILES.devices), 'Device');
+    for (const [name, t] of [['Queries', q], ['Pages', pg], ['Countries', co], ['Devices', de]]) {
+        if (!t) warnings.push(`no ${name}.csv in the export`);
+    }
+
+    let namedQueries = null;
+    if (q) {
+        const named = q.reduce((s, r) => s + r.impressions, 0);
+        namedQueries = {
+            impressions: named,
+            shareOfTotal: all.impressions ? round((100 * named) / all.impressions, 0) : null,
+            clicks: q.reduce((s, r) => s + r.clicks, 0),
+        };
+    }
+
+    return {
+        range: { from: first, to: last, days: days.length, trimmedTrailingDays: trimmed },
+        filters,
+        totals: all,
+        perDay: { impressions: round(all.impressions / days.length, 1), clicks: round(all.clicks / days.length, 2) },
+        weeks,
+        windows,
+        daily: days,
+        namedQueries,
+        queries: q ? withPrev(topRows(q, limit), prev?.queries) : null,
+        pages: pg ? withPrev(topRows(pg, limit, { merge: pathOf }), prev?.pages) : null,
+        countries: co ? withPrev(topRows(co, limit), prev?.countries) : null,
+        devices: de ? topRows(de, 0) : null,
+        prevTotals: prev?.totals ? { range: prev.range, ...prev.totals } : null,
+        warnings,
+    };
+}

package/src/digest/index.js

@@ -0,0 +1,76 @@
+// `siteread digest`: a Search Console export and an access log → one JSON document of the
+// numbers a weekly read is written from. Deterministic: the same inputs give the same numbers,
+// so this week's read compares with last week's. Nothing is sent anywhere; the only network
+// traffic is the verifier's GETs of the vendors' public IP lists (none with `offline`).
+
+import { readFileSync } from 'node:fs';
+import { digestGsc, readExport } from './gsc.js';
+import { createLogDigest, readLog } from './log.js';
+import { VERSION } from '../version.js';
+
+// GSC clicks vs people the log saw arrive from Google, over the days both cover. They are
+// different counts (a click without a Referer reads as direct; one person can click twice), so
+// this is a sanity check, never a sum. On synacl.com they tracked within 10 % day by day.
+function crossCheck(gsc, log) {
+    const from = gsc.range.from > log.range.from ? gsc.range.from : log.range.from;
+    const to = gsc.range.to < log.range.to ? gsc.range.to : log.range.to;
+    if (from > to) return null;
+    const gscClicks = gsc.daily.filter(d => d.date >= from && d.date <= to).reduce((s, d) => s + d.clicks, 0);
+    const days = log.sources.find(s => s.source === 'google')?.landingDays ?? {};
+    const googlePeople = Object.entries(days).filter(([d]) => d >= from && d <= to).reduce((s, [, n]) => s + n, 0);
+    return { from, to, gscClicks, googlePeople };
+}
+
+/** Paths of every <loc> in a sitemap (or sitemap index) file. */
+export function parseSitemap(xml) {
+    const out = new Set();
+    for (const m of String(xml).matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)) {
+        const loc = m[1].replace(/&amp;/g, '&');
+        try { out.add(new URL(loc).pathname); } catch { out.add(loc); }
+    }
+    return out;
+}
+
+/** "label=regex" → { label, re }. */
+export function parseGoal(s) {
+    const i = String(s).indexOf('=');
+    if (i < 1) throw new Error(`--goal wants label=regex, got "${s}"`);
+    return { label: s.slice(0, i), re: new RegExp(s.slice(i + 1), 'i') };
+}
+
+/**
+ * @param {object} o
+ * @param {string} [o.gsc]        export zip or folder
+ * @param {string[]} [o.logs]     access log files
+ * @param {object} [o.verifier]   from createVerifier(), already refreshed or loaded
+ * @param {object} [o.prev]       an earlier digest (its JSON), for per-row deltas
+ * Plus the createLogDigest options and `split`, `limit`.
+ */
+export async function digest({ gsc = null, logs = [], verifier = null, prev = null, split = null, limit = 25, sitemapFile = null, now = new Date(), ...logOpts } = {}) {
+    if (!gsc && !logs.length) throw new Error('nothing to read: give a Search Console export, a log, or both');
+    const out = { siteread: VERSION, generatedAt: now.toISOString(), gsc: null, log: null, warnings: [] };
+    if (gsc) {
+        out.gsc = digestGsc(readExport(gsc), { split, limit, prev: prev?.gsc ?? null });
+        out.warnings.push(...out.gsc.warnings.map(w => `gsc: ${w}`));
+        delete out.gsc.warnings;
+    }
+    if (logs.length) {
+        const sitemap = sitemapFile ? parseSitemap(readFileSync(sitemapFile, 'utf8')) : null;
+        if (sitemap && !sitemap.size) out.warnings.push(`log: no <loc> URLs in ${sitemapFile}`);
+        const d = createLogDigest({ ...logOpts, sitemap, limit: Math.min(limit, 25) });
+        for (const f of logs) await readLog(f, d);
+        out.log = d.finish(verifier);
+        if (out.log.lines.unparsed > out.log.lines.total / 2) {
+            out.warnings.push(`log: ${out.log.lines.unparsed} of ${out.log.lines.total} lines are not in combined format; counts are from the rest`);
+        }
+        if (out.log.site.guessed) out.warnings.push(`log: site host guessed as ${out.log.site.host ?? 'unknown'} from Referers; pass --site to set it`);
+        if (out.gsc && out.log.range.from) {
+            const a = out.gsc.range, b = out.log.range;
+            if (b.from < a.from || b.to > a.to) {
+                out.warnings.push(`the log runs ${b.from} → ${b.to}, past the export's ${a.from} → ${a.to}; pass --from ${a.from > b.from ? a.from : b.from} --to ${a.to < b.to ? a.to : b.to} to compare the same days`);
+            }
+            out.crossCheck = crossCheck(out.gsc, out.log);
+        }
+    }
+    return out;
+}