readiness-audit 1.2.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/helpers.mjs CHANGED
@@ -20,6 +20,233 @@ export function spread(list, n) {
20
20
  return list.length <= n ? list : Array.from({ length: n }, (_, i) => list[Math.floor((i * list.length) / n)]);
21
21
  }
22
22
 
23
+ /** Up to three items, then how many more: "a, b, c, +4 more". */
24
+ export function list(xs, sep = ', ') {
25
+ return `${xs.slice(0, 3).join(sep)}${xs.length > 3 ? `, +${xs.length - 3} more` : ''}`;
26
+ }
27
+
28
+ /** A URL for comparison: lowercase host, no trailing slash, no fragment. The query stays. */
29
+ export function norm(u) {
30
+ try { const x = new URL(u); return `${x.protocol}//${x.host.toLowerCase()}${x.pathname.replace(/\/+$/, '') || '/'}${x.search}`; } catch { return u; }
31
+ }
32
+
33
+ /** A URL's or a path's path segments: / is [], /docs/mcp is ['docs', 'mcp']. */
34
+ export function pathSegments(u) {
35
+ try { return new URL(u, 'https://x.invalid').pathname.split('/').filter(Boolean); } catch { return []; }
36
+ }
37
+
38
+ /** How many segments a URL's or a path's path has: / is 0, /about is 1, /docs/mcp is 2. */
39
+ export const pathDepth = (u) => pathSegments(u).length;
40
+
41
+ /** HTML text as plain text: entities decoded, whitespace collapsed. */
42
+ export function decodeText(t) {
43
+ const named = { quot: '"', apos: "'", lt: '<', gt: '>', nbsp: ' ' };
44
+ return String(t ?? '')
45
+ .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
46
+ .replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16)))
47
+ .replace(/&(quot|apos|lt|gt|nbsp);/g, (_, k) => named[k])
48
+ .replace(/&amp;/g, '&')
49
+ .replace(/\s+/g, ' ').trim();
50
+ }
51
+
52
+ /**
53
+ * A page's headings in order, as { level, text }. Navigation, footers, asides and
54
+ * screen-reader-only headings aren't part of the page's structure, so they're left out.
55
+ * A <header> stays: pages often put their title h1 inside <article><header>.
56
+ */
57
+ export function headings(html) {
58
+ const body = String(html ?? '')
59
+ .replace(/<(nav|footer|aside)\b[\s\S]*?<\/\1>/gi, '')
60
+ .replace(/<h([1-6])\b[^>]*class="[^"]*\b(visually-hidden|sr-only)\b[^"]*"[^>]*>[\s\S]*?<\/h\1>/gi, '');
61
+ return [...body.matchAll(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi)].map((m) => ({ level: Number(m[1]), text: decodeText(m[2].replace(/<[^>]+>/g, ' ')) }));
62
+ }
63
+
64
+ /** Where headings skip a level going down (h1 then h3), as "h1 to h3". Going back up is fine. */
65
+ export function headingSkips(hs) {
66
+ const skips = [];
67
+ for (let i = 1; i < hs.length; i++) if (hs[i].level > hs[i - 1].level + 1) skips.push(`h${hs[i - 1].level} to h${hs[i].level}`);
68
+ return skips;
69
+ }
70
+
71
+ /**
72
+ * schema.org types that can earn a Google rich result, by the feature Google's search gallery
73
+ * names (read 2026-10-21; the gallery was updated 2026-10-16). FAQ and HowTo aren't there anymore.
74
+ */
75
+ export const GOOGLE_RICH_RESULTS = {
76
+ Article: ['Article', 'NewsArticle', 'BlogPosting'], Breadcrumb: ['BreadcrumbList'], 'Course list': ['Course'], Dataset: ['Dataset'],
77
+ 'Discussion forum': ['DiscussionForumPosting'], 'Education Q&A': ['Quiz'], 'Employer aggregate rating': ['EmployerAggregateRating'],
78
+ Event: ['Event'], 'Image metadata': ['ImageObject'], 'Job posting': ['JobPosting'], 'Local business': ['LocalBusiness'],
79
+ 'Math solver': ['MathSolver'], Movie: ['Movie'], Organization: ['Organization'], Product: ['Product'], 'Profile page': ['ProfilePage'],
80
+ 'Q&A': ['QAPage'], Recipe: ['Recipe'], 'Review snippet': ['Review', 'AggregateRating'],
81
+ 'Software app': ['SoftwareApplication', 'MobileApplication', 'WebApplication', 'VideoGame'], 'Vacation rental': ['VacationRental'], Video: ['VideoObject'],
82
+ };
83
+ const RICH_FEATURE = new Map(Object.entries(GOOGLE_RICH_RESULTS).flatMap(([feature, types]) => types.map((t) => [t, feature])));
84
+
85
+ /**
86
+ * The pages to run through Google's Rich Results Test. Pages from one template share their markup,
87
+ * so one page tests them all; each template is its own code, so each one with rich-result markup is
88
+ * listed once. Candidates come in order (the homepage, the template pages, then the rest), each
89
+ * { url, types, template } with template the page's template pattern, if any; a page outside a
90
+ * template is kept only when it adds a feature no kept page has.
91
+ * Returns up to `max` { url, features, test }.
92
+ */
93
+ export function richResultPages(pages, max = 8) {
94
+ const seen = new Set();
95
+ const listedTemplates = new Set();
96
+ const out = [];
97
+ for (const p of pages) {
98
+ const features = [...new Set((p.types ?? []).map((t) => RICH_FEATURE.get(t)).filter(Boolean))].sort();
99
+ if (!features.length || listedTemplates.has(p.template)) continue;
100
+ if (!p.template && !features.some((f) => !seen.has(f))) continue;
101
+ if (p.template) listedTemplates.add(p.template);
102
+ features.forEach((f) => seen.add(f));
103
+ out.push({ url: p.url, features, test: `https://search.google.com/test/rich-results?url=${encodeURIComponent(p.url)}` });
104
+ if (out.length === max) break;
105
+ }
106
+ return out;
107
+ }
108
+
109
+ /** A sitemap's entries, urlset or index, as { loc, lastmod } (lastmod undefined when absent). */
110
+ export function sitemapEntries(xml) {
111
+ return [...String(xml ?? '').matchAll(/<(url|sitemap)>([\s\S]*?)<\/\1>/g)].map((m) => ({
112
+ loc: m[2].match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1]?.replace(/&amp;/g, '&'),
113
+ lastmod: m[2].match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1],
114
+ })).filter((e) => e.loc);
115
+ }
116
+
117
+ /**
118
+ * The discovery files the audit reads, by name. --logs counts requests for the same paths,
119
+ * so what the scorecard checks and what the logs measure can't drift apart.
120
+ */
121
+ export const DISCOVERY_FILES = {
122
+ robots: '/robots.txt',
123
+ sitemap: '/sitemap.xml',
124
+ llms: '/llms.txt',
125
+ llmsFull: '/llms-full.txt',
126
+ openapi: '/openapi.json',
127
+ apiIndex: '/api',
128
+ apiCatalog: '/.well-known/api-catalog',
129
+ security: '/.well-known/security.txt',
130
+ authMd: '/auth.md',
131
+ docsMd: '/docs/api.md',
132
+ agentCard: '/.well-known/agent-card.json',
133
+ ard: '/.well-known/ard.json',
134
+ aiCatalog: '/.well-known/ai-catalog.json',
135
+ skills: '/.well-known/agent-skills/index.json',
136
+ prm: '/.well-known/oauth-protected-resource',
137
+ asMeta: '/.well-known/oauth-authorization-server',
138
+ aiPlugin: '/.well-known/ai-plugin.json',
139
+ legacyCard: '/.well-known/mcp.json',
140
+ wkCard: '/.well-known/mcp/server-card.json',
141
+ };
142
+
143
+ /** The AI crawlers and assistant fetchers D15 wants named in robots.txt, as robots.txt user-agent tokens. */
144
+ export const AI_BOTS = ['gptbot', 'oai-searchbot', 'chatgpt-user', 'claudebot', 'claude-user', 'claude-searchbot', 'perplexitybot', 'perplexity-user', 'google-extended', 'applebot-extended'];
145
+
146
+ /**
147
+ * Section headings (h1 to h3) the HTML page has and its markdown copy lacks, compared as
148
+ * lowercase words so formatting doesn't count. Page chrome is left out, as in headings().
149
+ * An empty list when they agree.
150
+ */
151
+ export function missingSections(html, markdown) {
152
+ const words = (t) => decodeText(t).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ' ').trim();
153
+ const htmlHeads = headings(html).filter((h) => h.level <= 3 && words(h.text));
154
+ const mdHeads = new Set([...String(markdown ?? '').matchAll(/^#{1,6}\s+(.+?)\s*#*\s*$/gm)].map((m) => words(m[1].replace(/[*_`[\]]|\]\([^)]*\)/g, ' '))));
155
+ return [...new Set(htmlHeads.filter((h) => !mdHeads.has(words(h.text))).map((h) => h.text))];
156
+ }
157
+
158
+ /** Text that links an API reference or an MCP reference: what S1 wants in a skill, and what declares an API. */
159
+ export const API_LINK = /openapi|\/docs\/api|https?:\/\/[^\s)]+\/api\b/i;
160
+ export const MCP_LINK = /\/docs\/mcp|https?:\/\/[^\s)]+\/mcp\b/i;
161
+
162
+ /** An RFC 8288 Link header as a Map of rel to hrefs. */
163
+ export function linkRels(header) {
164
+ const rels = new Map();
165
+ for (const part of String(header ?? '').split(/,(?=\s*<)/)) {
166
+ const href = part.match(/<([^>]+)>/)?.[1];
167
+ const relAttr = part.match(/rel="?([^";]+)"?/)?.[1];
168
+ if (href && relAttr) for (const r of relAttr.split(/\s+/)) rels.set(r, [...(rels.get(r) ?? []), href]);
169
+ }
170
+ return rels;
171
+ }
172
+
173
+ /**
174
+ * Whether a site has a public API, the way the audit decides it has a hosted MCP endpoint:
175
+ * one answers (/openapi.json is an OpenAPI document, or /api answers JSON), or the site
176
+ * declares one (a service-desc naming an OpenAPI document, an API catalog, an API link in
177
+ * llms.txt). A declared API gets the API checks even when nothing answers: claims get checked.
178
+ * Returns { present, answers, declared } with declared naming where the claims are.
179
+ */
180
+ export function apiPresence({ openapiSpec, apiIndexJson, rels, catalogJson, llmsText }) {
181
+ const answers = Boolean(openapiSpec?.openapi) || Boolean(apiIndexJson);
182
+ const declared = [];
183
+ if ((rels?.get('service-desc') ?? []).some((h) => /openapi/i.test(h))) declared.push('a service-desc Link');
184
+ if (Array.isArray(catalogJson?.linkset) && catalogJson.linkset.length) declared.push('/.well-known/api-catalog');
185
+ if (llmsText && API_LINK.test(llmsText)) declared.push('llms.txt');
186
+ return { present: answers || declared.length > 0, answers, declared };
187
+ }
188
+
189
+ /**
190
+ * The pages the W checks read: every kind of page, not a spread that a big template drowns out.
191
+ * URLs are grouped by template, the first path segment plus the depth (/creatures/a/b and
192
+ * /creatures/c/d share /creatures/*\/*; one-segment pages share /*; / is its own). A group of
193
+ * up to 20 URLs is static pages, all read; a bigger group is a template, three read with spread().
194
+ * Templates come first, then static pages by shortest path, up to `cap` pages.
195
+ * Returns { pages, templates: [{ pattern, count, urls }], static: number read, skipped: static pages not read }.
196
+ */
197
+ export function samplePages(urls, { cap = 40, staticMax = 20, perTemplate = 3 } = {}) {
198
+ const pattern = (u) => {
199
+ const segs = pathSegments(u);
200
+ return segs.length === 0 ? '/' : segs.length === 1 ? '/*' : `/${segs[0]}/${Array(segs.length - 1).fill('*').join('/')}`;
201
+ };
202
+ const groups = new Map();
203
+ for (const u of urls) groups.set(pattern(u), [...(groups.get(pattern(u)) ?? []), u]);
204
+ const templates = [...groups].filter(([, us]) => us.length > staticMax).map(([p, us]) => ({ pattern: p, count: us.length, urls: spread(us, perTemplate) }));
205
+ const statics = [...groups].filter(([, us]) => us.length <= staticMax).flatMap(([, us]) => us)
206
+ .map((u, i) => ({ u, i, d: pathDepth(u), n: u.length })).sort((a, b) => a.d - b.d || a.n - b.n || a.i - b.i).map((x) => x.u);
207
+ const pages = templates.flatMap((t) => t.urls).slice(0, cap);
208
+ const room = Math.max(0, cap - pages.length);
209
+ pages.push(...statics.slice(0, room));
210
+ return { pages, templates, static: Math.min(room, statics.length), skipped: Math.max(0, statics.length - room) };
211
+ }
212
+
213
+ /**
214
+ * Pages whose share card shows another page's text: a title or description
215
+ * that several pages share and that isn't the page's own (its <title> or meta
216
+ * description, or the homepage's when it matches no page's own). X reads twitter:title and twitter:description first and falls
217
+ * back to og:; Facebook and LinkedIn read og:. Each page is
218
+ * { path, title, description, og: { title, description }, twitter: { title, description } }.
219
+ * Returns { x, og }, each { paths, text } with text an example of the shared value.
220
+ */
221
+ export function sharedCardText(pages) {
222
+ const views = {
223
+ x: (p) => [p.twitter?.title ?? p.og?.title, p.twitter?.description ?? p.og?.description],
224
+ og: (p) => [p.og?.title, p.og?.description],
225
+ };
226
+ const out = {};
227
+ for (const [key, view] of Object.entries(views)) {
228
+ const flagged = new Set();
229
+ let text;
230
+ for (const i of [0, 1]) {
231
+ const groups = new Map();
232
+ for (const p of pages) {
233
+ const value = view(p)[i];
234
+ if (value) groups.set(value, [...(groups.get(value) ?? []), p]);
235
+ }
236
+ for (const [value, members] of groups) {
237
+ if (members.length < 2) continue;
238
+ // A shared value belongs to the page it matches; failing that, to the homepage, where a layout's default comes from.
239
+ const own = (p) => value === (i === 0 ? p.title : p.description);
240
+ const owner = members.some(own) ? own : (p) => p.path === '/';
241
+ const borrowed = members.filter((p) => !owner(p));
242
+ if (borrowed.length) { borrowed.forEach((p) => flagged.add(p.path)); text ??= value; }
243
+ }
244
+ }
245
+ out[key] = { paths: [...flagged], text };
246
+ }
247
+ return out;
248
+ }
249
+
23
250
  /**
24
251
  * The site's own links in an llms.txt: markdown links and bare URLs, resolved
25
252
  * against the file's URL, without fragments, kept only when they point at one
@@ -72,13 +299,18 @@ export function formatRecorded(x) {
72
299
  const num = (v, label) => (v === undefined || v === null || v === '' ? null : `${v}${label}`);
73
300
  const traffic = (o) => [num(o.clicks, ' clicks'), num(o.impressions, ' impressions'), num(o.ctr, '% CTR'), o.position != null ? `position ${o.position}` : null].filter(Boolean).join(', ');
74
301
  const parts = [];
302
+ if (x.attempts != null) { // T5
303
+ const run = [x.model, x.max_turns != null ? `${x.max_turns} turns max` : null].filter(Boolean).join(', ');
304
+ parts.push([`${x.first_try ?? '?'} of ${x.attempts} first try`, run || null, num(x.median_calls, ' calls (median)'), num(x.stops, ' stops'), x.cost_usd != null ? `$${x.cost_usd}` : null].filter(Boolean).join('; '));
305
+ }
75
306
  if (x.google && typeof x.google === 'object') {
76
307
  const g = x.google;
77
308
  const index = [num(g.indexed, ' indexed'), num(g.not_indexed, ' not indexed'), num(g.crawled_not_indexed, ' crawled but not indexed')].filter(Boolean).join(', ');
78
309
  parts.push(`Google: ${[traffic(g), index].filter(Boolean).join('; ')}`);
79
310
  }
80
311
  if (x.bing && typeof x.bing === 'object') parts.push(`Bing: ${traffic(x.bing)}`);
81
- const when = [parts.length ? x.period : null, x.date].filter(Boolean).join(', ');
312
+ if (Array.isArray(x.queries) && x.queries.length) parts.push(`${x.queries.length} top quer${x.queries.length === 1 ? 'y' : 'ies'} recorded`);
313
+ const when = [x.attempts == null && parts.length ? x.period : null, x.date].filter(Boolean).join(', ');
82
314
  const text = parts.length ? parts.join('. ') : String(x.result ?? '');
83
315
  return `${text}${when ? ` (${when})` : ''}`.replace(/\|/g, '/');
84
316
  }
package/src/http.mjs ADDED
@@ -0,0 +1,47 @@
1
+ // The audit's one HTTP request, on node:http(s). Internal: the package exports only audit.mjs.
2
+ //
3
+ // Not fetch(): fetch decodes a Brotli body even when the request didn't ask for one, so a
4
+ // hostile site could hand any body to the Brotli decoder (CVE-2026-111497 was an out-of-bounds
5
+ // write there). node:http decodes nothing; this decodes gzip and deflate and refuses the rest.
6
+
7
+ import { request as httpRequest } from 'node:http';
8
+ import { request as httpsRequest } from 'node:https';
9
+ import { gunzipSync, inflateSync, inflateRawSync } from 'node:zlib';
10
+
11
+ export const TIMEOUT_MS = 15000;
12
+
13
+ /**
14
+ * One request, no redirects followed. Headers default to `accept: *\/*` and
15
+ * `accept-encoding: gzip, deflate`; a header set to null isn't sent (M4's no-Accept probe).
16
+ * Resolves, never rejects: { status, headers (a Headers object), text, error? }, status 0 on no answer.
17
+ */
18
+ export function request(url, { method = 'GET', headers = {}, body, userAgent, timeoutMs = TIMEOUT_MS } = {}) {
19
+ const target = new URL(url);
20
+ const send = target.protocol === 'http:' ? httpRequest : httpsRequest;
21
+ const all = { 'user-agent': userAgent, accept: '*/*', 'accept-encoding': 'gzip, deflate', ...headers };
22
+ if (body !== undefined) all['content-length'] = Buffer.byteLength(body);
23
+ const sent = Object.fromEntries(Object.entries(all).filter(([, v]) => v !== null && v !== undefined));
24
+ const none = (error) => ({ status: 0, headers: new Headers(), text: '', error });
25
+ return new Promise((resolve) => {
26
+ const req = send(target, { method, headers: sent, signal: AbortSignal.timeout(timeoutMs) }, (res) => {
27
+ const chunks = [];
28
+ res.on('data', (c) => chunks.push(c));
29
+ res.on('error', (e) => resolve(none(e.message)));
30
+ res.on('end', () => {
31
+ const headers = new Headers(Object.entries(res.headers).flatMap(([k, v]) => [].concat(v).map((x) => [k, String(x)])));
32
+ const raw = Buffer.concat(chunks);
33
+ const encoding = (headers.get('content-encoding') ?? 'identity').trim().toLowerCase();
34
+ const out = { status: res.statusCode, headers, text: '' };
35
+ try {
36
+ if (encoding === 'identity' || encoding === '') out.text = raw.toString('utf8');
37
+ else if (encoding === 'gzip' || encoding === 'x-gzip') out.text = gunzipSync(raw).toString('utf8');
38
+ else if (encoding === 'deflate') { try { out.text = inflateSync(raw).toString('utf8'); } catch { out.text = inflateRawSync(raw).toString('utf8'); } } // zlib-wrapped, or raw as some servers send it
39
+ else out.error = `refused: ${encoding} body`;
40
+ } catch (e) { out.error = `undecodable ${encoding} body: ${e.message}`; }
41
+ resolve(out);
42
+ });
43
+ });
44
+ req.on('error', (e) => resolve(none(e.name === 'AbortError' || e.name === 'TimeoutError' ? `no answer in ${timeoutMs / 1000} seconds` : e.message)));
45
+ req.end(body);
46
+ });
47
+ }
package/src/logs.mjs ADDED
@@ -0,0 +1,120 @@
1
+ // --logs: count requests for the discovery files in an exported access log, by client family.
2
+ // Pure: the CLI feeds it lines. It prints counts and families, never addresses.
3
+
4
+ import { DISCOVERY_FILES, AI_BOTS } from './helpers.mjs';
5
+
6
+ // Client families, checked in order: the first match wins. §2 of STANDARD.md describes each.
7
+ // Directories come before the generic "bot" match, assistants before AI crawlers.
8
+ export const FAMILIES = [
9
+ ['This audit (readiness-audit)', /^readiness-audit\//i],
10
+ ['Agent and MCP directories', /agenstrybot|brickbluebot|agentprobe|mcpharbor|agenttrustbot|taifoonharvester|mcphub-probe|invokerankbot|proofbench|thepluginstorebot|protogrid-probe|glidemcpindex|agent-tools\.cloud/i],
11
+ ['llms.txt directories', /llms\.?txt/i],
12
+ ['Assistants fetching for a person', /chatgpt-user|claude-user|perplexity-user|meta-externalfetcher/i],
13
+ ['AI crawlers', new RegExp([...AI_BOTS, 'meta-webindexer', 'meta-externalagent', 'bytespider', 'ccbot'].map((b) => b.replace(/[-.]/g, '\\$&')).join('|'), 'i')],
14
+ ['Search crawlers', /googlebot|bingbot|applebot|duckduckbot|yandexbot|baiduspider/i],
15
+ ['Scripts', /^(curl|wget|python|node|undici|axios|go-http-client|okhttp|java\/|libwww|httpie|deno|bun\/)/i],
16
+ ['Headless browsers', /headlesschrome|phantomjs|puppeteer|playwright|selenium/i],
17
+ ['Other bots', /bot|crawl|spider|scan|fetch|probe/i],
18
+ ['Browsers', /^mozilla\//i],
19
+ ];
20
+
21
+ export function family(ua) {
22
+ if (!ua || ua === '-') return 'No user agent';
23
+ return FAMILIES.find(([, re]) => re.test(ua))?.[0] ?? 'Unidentified';
24
+ }
25
+
26
+ // Families whose user agent can't show an agent: count them as one only when the same address called the API.
27
+ const UNVERIFIED = new Set(['Browsers', 'Headless browsers', 'Unidentified', 'No user agent']);
28
+
29
+ const COMBINED = /^(\S+) \S+ \S+ \[[^\]]*\] "(\S+) (\S+)[^"]*" (\d{3}) \S+(?: "[^"]*" "([^"]*)")?/;
30
+
31
+ // Field names per host's JSON lines, from each host's docs (read 2026-10-21): Cloudflare Logpush, Vercel log
32
+ // drains in NDJSON (proxy.path before the function's own path), Railway HTTP logs, then generic names.
33
+ const FIELDS = {
34
+ path: ['ClientRequestPath', 'ClientRequestURI', 'proxy.path', 'path', 'requestPath', 'request_path', 'url', 'httpRequest.requestUrl'],
35
+ ua: ['ClientRequestUserAgent', 'proxy.userAgent', 'userAgent', 'user_agent', 'clientUa', 'httpRequest.userAgent'],
36
+ addr: ['ClientIP', 'proxy.clientIp', 'srcIp', 'clientIp', 'client_ip', 'ip', 'remote_addr', 'httpRequest.remoteIp'],
37
+ method: ['ClientRequestMethod', 'proxy.method', 'method', 'httpRequest.requestMethod'],
38
+ status: ['EdgeResponseStatus', 'proxy.statusCode', 'httpStatus', 'status', 'statusCode', 'httpRequest.status'],
39
+ };
40
+ const dig = (o, key) => key.split('.').reduce((x, k) => (x == null ? undefined : x[k]), o);
41
+ const pick = (o, keys) => { for (const k of keys) { const v = dig(o, k); if (v !== undefined && v !== null && v !== '') return Array.isArray(v) ? v[0] : v; } return undefined; };
42
+
43
+ /** One log line as { path, ua, addr, method, status }, or null. Combined log format and JSON lines. */
44
+ export function parseLine(line) {
45
+ const t = line.trim();
46
+ if (!t) return null;
47
+ if (t.startsWith('{')) {
48
+ let o;
49
+ try { o = JSON.parse(t); } catch { return null; }
50
+ let path = pick(o, FIELDS.path);
51
+ if (typeof path !== 'string') return null;
52
+ if (/^https?:\/\//.test(path)) { try { path = new URL(path).pathname; } catch { return null; } }
53
+ return { path: path.split('?')[0], ua: String(pick(o, FIELDS.ua) ?? ''), addr: String(pick(o, FIELDS.addr) ?? ''), method: String(pick(o, FIELDS.method) ?? 'GET').toUpperCase(), status: Number(pick(o, FIELDS.status)) || 0 };
54
+ }
55
+ const m = t.match(COMBINED);
56
+ if (!m) return null;
57
+ return { addr: m[1], method: m[2].toUpperCase(), path: m[3].split('?')[0], status: Number(m[4]), ua: m[5] ?? '' };
58
+ }
59
+
60
+ const FILE_PATHS = new Map(Object.values(DISCOVERY_FILES).map((p) => [p, p]));
61
+ /** Which discovery bucket a path counts toward, or null. */
62
+ export function bucket(path) {
63
+ const p = path.length > 1 ? path.replace(/\/+$/, '') : path;
64
+ if (FILE_PATHS.has(p)) return p;
65
+ if (p === '/mcp' || p.startsWith('/mcp/')) return '/mcp';
66
+ if (p.startsWith('/.well-known/')) return 'other /.well-known paths';
67
+ if (/\.md$/i.test(p)) return 'markdown pages (*.md)';
68
+ return null;
69
+ }
70
+ const isApiCall = (r) => r.path.startsWith('/api/') || (r.path === '/mcp' && r.method === 'POST');
71
+
72
+ /** Tally lines one at a time; result() gives the report. */
73
+ export function logTally() {
74
+ let lines = 0, parsed = 0;
75
+ const hits = new Map(); // bucket -> Map(family -> count)
76
+ const readers = new Map(); // bucket -> Set(addr)
77
+ const apiAddrs = new Set();
78
+ const unverifiedAddrs = new Set();
79
+ return {
80
+ add(line) {
81
+ lines++;
82
+ const r = parseLine(line);
83
+ if (!r) return;
84
+ parsed++;
85
+ const fam = family(r.ua);
86
+ if (isApiCall(r)) { if (r.addr) apiAddrs.add(r.addr); if (UNVERIFIED.has(fam) && r.addr) unverifiedAddrs.add(r.addr); }
87
+ const b = bucket(r.path);
88
+ if (!b) return;
89
+ if (!hits.has(b)) hits.set(b, new Map());
90
+ hits.get(b).set(fam, (hits.get(b).get(fam) ?? 0) + 1);
91
+ if (r.addr) { if (!readers.has(b)) readers.set(b, new Set()); readers.get(b).add(r.addr); }
92
+ },
93
+ result() {
94
+ const paths = [...hits].map(([path, fams]) => {
95
+ const addrs = readers.get(path) ?? new Set();
96
+ return {
97
+ path,
98
+ requests: [...fams.values()].reduce((a, b) => a + b, 0),
99
+ families: Object.fromEntries([...fams].sort((a, b) => b[1] - a[1])),
100
+ addresses: addrs.size,
101
+ also_called_api: [...addrs].filter((a) => apiAddrs.has(a)).length,
102
+ };
103
+ }).sort((a, b) => b.requests - a.requests);
104
+ return { lines, parsed, api_addresses: apiAddrs.size, unverified_api_addresses: unverifiedAddrs.size, paths };
105
+ },
106
+ };
107
+ }
108
+
109
+ /** The report as text. */
110
+ export function formatLogs(rep) {
111
+ const out = [`Read ${rep.parsed} of ${rep.lines} log lines (${rep.lines - rep.parsed} not understood)`, ''];
112
+ if (!rep.paths.length) out.push('No requests for discovery files.');
113
+ for (const p of rep.paths) {
114
+ out.push(`${p.path} ${p.requests} request${p.requests === 1 ? '' : 's'}${p.addresses ? `, ${p.addresses} address${p.addresses === 1 ? '' : 'es'}, ${p.also_called_api} of them also called the API` : ''}`);
115
+ for (const [fam, n] of Object.entries(p.families)) out.push(` ${String(n).padStart(6)} ${fam}`);
116
+ }
117
+ out.push('', `API calls came from ${rep.api_addresses} address${rep.api_addresses === 1 ? '' : 'es'}. ${rep.unverified_api_addresses} of them sent a browser, headless or missing user agent: count those as agents, since they called the API.`);
118
+ out.push('Browsers, headless browsers and unidentified clients above count as agents only where they also called the API (STANDARD.md, T3).');
119
+ return out;
120
+ }