readiness-audit 1.2.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/audit.mjs CHANGED
@@ -4,18 +4,16 @@
4
4
  // The standard and what each ID means: STANDARD.md in
5
5
  // https://github.com/geeks-accelerator/agent-and-search-readiness
6
6
  // Read-only: GETs plus a few POSTs that create nothing (an MCP initialize, server/discover and
7
- // tools/list, an empty POST to /). Node 20+, no dependencies.
7
+ // tools/list, an empty POST to /). Node 26 or later, no dependencies.
8
8
 
9
9
  import { lookup, resolveTxt, Resolver } from 'node:dns/promises';
10
- import { request as httpRequest } from 'node:http';
11
- import { request as httpsRequest } from 'node:https';
12
10
  import { isIP } from 'node:net';
13
- import { SKILL_NAME, sampleSkillEntries, skillNameProblem, cdnCacheNote, repeatsSummary, hasVariantDescription, spread, llmsLinks, baseMapper, formatRecorded } from './helpers.mjs';
11
+ import { request } from './http.mjs';
12
+ import { SKILL_NAME, sampleSkillEntries, skillNameProblem, cdnCacheNote, repeatsSummary, hasVariantDescription, spread, samplePages, sharedCardText, API_LINK, MCP_LINK, linkRels, apiPresence, DISCOVERY_FILES, AI_BOTS, missingSections, richResultPages, list, norm, pathDepth, decodeText, headings, headingSkips, sitemapEntries, llmsLinks, baseMapper, formatRecorded } from './helpers.mjs';
14
13
 
15
- export const VERSION = '1.2.1';
14
+ export const VERSION = '2.1.0';
16
15
  export const REPO = 'https://github.com/geeks-accelerator/agent-and-search-readiness';
17
16
  const UA = `readiness-audit/${VERSION} (+${REPO})`;
18
- const AI_BOTS = ['gptbot', 'oai-searchbot', 'chatgpt-user', 'claudebot', 'claude-user', 'claude-searchbot', 'perplexitybot', 'perplexity-user', 'google-extended', 'applebot-extended'];
19
17
  const AID_PROTOCOLS = ['mcp', 'a2a', 'openapi', 'grpc', 'graphql', 'websocket', 'local', 'zeroconf', 'ucp'];
20
18
  const SKILLS_SCHEMA = 'https://schemas.agentskills.io/discovery/0.2.0/schema.json';
21
19
  const CARD_SCHEMA = 'https://static.modelcontextprotocol.io/schemas/v1/server-card.schema.json';
@@ -24,7 +22,8 @@ const ORDER = ['D1', 'D2', 'D4', 'D6', 'D11', 'D12', 'W1', 'W2', 'W3', 'W4', 'W5
24
22
  // Items the scorecard can't see from outside. Each project checks these in review and tests.
25
23
  export const REVIEW = [
26
24
  ['D3', 'pages read well: server-rendered, unique prose on generated pages (the scorecard checks W1 to W6 on a sample)'],
27
- ['W7', 'internal links: entity names link to their pages, descriptive anchors, no orphan pages'],
25
+ ['W4', 'Google\'s Rich Results Test on one page per kind of markup (listed below: the scorecard checks the markup parses, Google checks its required properties), after deploying a change to a template\'s markup; Search Console\'s rich result reports cover every page (T6)'],
26
+ ['W7', 'internal links: entity names link to their pages, anchors that say what the target is (not the brand alone), no orphan pages; sibling links in context, never sitewide (E6)'],
28
27
  ['W8', 'Core Web Vitals on the main pages (Lighthouse or PageSpeed Insights)'],
29
28
  ['W9', 'listed in the directories your audience uses'],
30
29
  ['D13', 'a page for agents\' builders with a copy-paste prompt'],
@@ -106,7 +105,7 @@ export function makeGuard(domain) {
106
105
  return { siteHosts, isSafe };
107
106
  }
108
107
 
109
- export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = null } = {}) {
108
+ export async function audit(domain, { mcpPath = '/mcp', base = null } = {}) {
110
109
  const BASE = `https://${domain}`;
111
110
  const rand = Math.random().toString(36).slice(2, 10);
112
111
 
@@ -122,38 +121,16 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
122
121
  for (let hop = 0; ; hop++) {
123
122
  const target = toBase(url);
124
123
  if (target === url && !(await isSafe(url))) return { status: 0, headers: new Headers(), text: '', url, type: '', error: 'refused: not a public address' };
125
- let res;
126
- try {
127
- res = await fetch(target, { method: m, body: b, redirect: 'manual', headers: { 'user-agent': UA, ...headers }, signal: AbortSignal.timeout(15000) });
128
- } catch (e) {
129
- return { status: 0, headers: new Headers(), text: '', url, type: '', error: e.message };
130
- }
124
+ const res = await request(target, { method: m, body: b, headers, userAgent: UA });
131
125
  const loc = res.headers.get('location');
132
126
  if (redirect === 'follow' && res.status >= 300 && res.status < 400 && loc && hop < 5) {
133
- try { await res.body?.cancel(); } catch { /* already closed */ }
134
127
  url = fromBase(new URL(loc, target).href);
135
128
  if (res.status !== 307 && res.status !== 308) { m = 'GET'; b = undefined; }
136
129
  continue;
137
130
  }
138
- const text = await res.text().catch(() => '');
139
- return { status: res.status, headers: res.headers, text, url, type: res.headers.get('content-type') ?? '' };
131
+ return { ...res, url, type: res.headers.get('content-type') ?? '' };
140
132
  }
141
133
  }
142
- // fetch() always sends an Accept header, so the "no Accept" MCP probe uses node:https.
143
- function postNoAccept(path, payload) {
144
- const target = new URL(toBase(BASE + path));
145
- const send = target.protocol === 'http:' ? httpRequest : httpsRequest;
146
- return new Promise((resolve) => {
147
- const req = send(target, { method: 'POST', timeout: 15000, headers: { 'user-agent': UA, 'content-type': 'application/json', 'content-length': Buffer.byteLength(payload) } }, (res) => {
148
- let text = '';
149
- res.on('data', (c) => { text += c; });
150
- res.on('end', () => resolve({ status: res.statusCode, type: res.headers['content-type'] ?? '', text }));
151
- });
152
- req.on('error', () => resolve({ status: 0, type: '', text: '' }));
153
- req.on('timeout', () => { req.destroy(); resolve({ status: 0, type: '', text: '' }); });
154
- req.end(payload);
155
- });
156
- }
157
134
  const json = (r) => { try { return JSON.parse(r.text); } catch { return null; } };
158
135
  const isJson = (r) => /json/.test(r.type) && json(r) !== null;
159
136
  const ok = (r) => r.status >= 200 && r.status < 300;
@@ -174,34 +151,37 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
174
151
  const wrong = (id, name, level, detail) => record(id, name, level, 'fail', detail); // present but false or invalid
175
152
  const skip = (id, name, level, detail) => record(id, name, level, 'skip', detail);
176
153
 
154
+ const D = DISCOVERY_FILES;
177
155
  const [home, homeMd, robots, sitemap, llms, llmsFull, openapi, apiIndex, apiCatalog, security, authMd, docsMd,
178
156
  agentCard, ard, aiCatalog, skills, prm, asMeta, aiPlugin, wkMiss, apiMiss, postRoot, legacyCard, wkCard] = await mapLimit([
179
157
  () => get('/'),
180
158
  () => get('/', { headers: { accept: 'text/markdown' } }),
181
- () => get('/robots.txt'),
182
- () => get('/sitemap.xml'),
183
- () => get('/llms.txt'),
184
- () => get('/llms-full.txt'),
185
- () => get('/openapi.json'),
186
- () => get('/api', { headers: { accept: 'application/json' } }),
187
- () => get('/.well-known/api-catalog'),
188
- () => get('/.well-known/security.txt'),
189
- () => get('/auth.md'),
190
- () => get('/docs/api.md'),
191
- () => get('/.well-known/agent-card.json'),
192
- () => get('/.well-known/ard.json'),
193
- () => get('/.well-known/ai-catalog.json'),
194
- () => get('/.well-known/agent-skills/index.json'),
195
- () => get('/.well-known/oauth-protected-resource'),
196
- () => get('/.well-known/oauth-authorization-server'),
197
- () => get('/.well-known/ai-plugin.json'),
159
+ () => get(D.robots),
160
+ () => get(D.sitemap),
161
+ () => get(D.llms),
162
+ () => get(D.llmsFull),
163
+ () => get(D.openapi),
164
+ () => get(D.apiIndex, { headers: { accept: 'application/json' } }),
165
+ () => get(D.apiCatalog),
166
+ () => get(D.security),
167
+ () => get(D.authMd),
168
+ () => get(D.docsMd),
169
+ () => get(D.agentCard),
170
+ () => get(D.ard),
171
+ () => get(D.aiCatalog),
172
+ () => get(D.skills),
173
+ () => get(D.prm),
174
+ () => get(D.asMeta),
175
+ () => get(D.aiPlugin),
198
176
  () => get(`/.well-known/no-such-file-${rand}`),
199
177
  () => get(`/api/no-such-endpoint-${rand}`, { headers: { accept: 'application/json' } }),
200
178
  () => get('/', { method: 'POST', redirect: 'manual' }),
201
- () => get('/.well-known/mcp.json'),
202
- () => get('/.well-known/mcp/server-card.json'),
179
+ () => get(D.legacyCard),
180
+ () => get(D.wkCard),
203
181
  ], 6, (job) => job());
204
182
  const catalogEntries = [...(json(ard)?.entries ?? []), ...(json(aiCatalog)?.entries ?? [])];
183
+ const rels = linkRels(home.headers.get('link'));
184
+ const api = apiPresence({ openapiSpec: json(openapi), apiIndexJson: ok(apiIndex) && isJson(apiIndex), rels, catalogJson: ok(apiCatalog) ? json(apiCatalog) : null, llmsText: ok(llms) ? llms.text : '' });
205
185
 
206
186
  // ---- D1 robots.txt (required) and D15 AI crawlers and Content-Signal (recommended) ----
207
187
  const groups = [];
@@ -232,7 +212,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
232
212
  const star = groups.find((g) => g.agents.includes('*'));
233
213
  const leaks = star ? groups.filter((g) => g !== star).flatMap((g) => [...star.disallow].filter((d) => !g.disallow.has(d)).map((d) => `${g.agents[0]} skips Disallow ${d}`)) : [];
234
214
  if (!ok(robots)) miss(id, name, level, `/robots.txt answered ${robots.status}${cached(robots)}`);
235
- else if (leaks.length) miss(id, name, level, `named groups drop rules from the * group (a crawler obeys only its own group): ${[...new Set(leaks)].slice(0, 3).join('; ')}${cached(robots)}`);
215
+ else if (leaks.length) miss(id, name, level, `named groups drop rules from the * group (a crawler obeys only its own group): ${list([...new Set(leaks)], '; ')}${cached(robots)}`);
236
216
  else if (!sitemapLine) miss(id, name, level, `no Sitemap line${cached(robots)}`);
237
217
  else pass(id, name, level, `${groups.length} group${groups.length === 1 ? '' : 's'} with consistent rules, Sitemap listed`);
238
218
  }
@@ -241,54 +221,68 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
241
221
  const named = AI_BOTS.filter((b) => groups.some((g) => g.agents.includes(b)));
242
222
  const unsignaled = groups.filter((g) => !g.signal);
243
223
  const problems = [];
244
- if (unsignaled.length) problems.push(`${unsignaled.length} of ${groups.length} group${groups.length === 1 ? '' : 's'} ${unsignaled.length === 1 ? 'has' : 'have'} no Content-Signal (${unsignaled.slice(0, 3).map((g) => g.agents[0]).join(', ')}${unsignaled.length > 3 ? ', ...' : ''})`);
224
+ if (unsignaled.length) problems.push(`${unsignaled.length} of ${groups.length} group${groups.length === 1 ? '' : 's'} ${unsignaled.length === 1 ? 'has' : 'have'} no Content-Signal (${list(unsignaled.map((g) => g.agents[0]))})`);
245
225
  if (named.length < 3) problems.push(`names ${named.length} of the main AI crawlers`);
246
226
  if (!ok(robots)) miss(id, name, level, 'no robots.txt');
247
227
  else if (problems.length) miss(id, name, level, problems.join('; ') + cached(robots));
248
228
  else pass(id, name, level, `${named.length} AI crawlers named, Content-Signal in every group`);
249
229
  }
250
230
 
231
+ // ---- The sitemap, read once for D2 and the W checks: an index is expanded to its pages ----
232
+ const sitemapIndex = /<sitemapindex/.test(sitemap.text);
233
+ const sitemapLists = sitemapIndex
234
+ ? await mapLimit(sitemapEntries(sitemap.text).slice(0, 10), 4, async (e) => sitemapEntries((await get(e.loc)).text))
235
+ : [sitemapEntries(sitemap.text)];
236
+ const seenLocs = new Set();
237
+ const entries = sitemapLists.flat().filter((e) => {
238
+ try { if (!siteHosts.has(new URL(e.loc).hostname) || seenLocs.has(e.loc)) return false; } catch { return false; }
239
+ seenLocs.add(e.loc);
240
+ return true;
241
+ });
242
+
251
243
  // ---- D2 sitemap ----
252
244
  {
253
245
  const id = 'D2', name = 'sitemap.xml', level = 'required';
254
- const lastmods = [...sitemap.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map((m) => m[1].slice(0, 10));
246
+ const lastmods = entries.map((e) => e.lastmod?.slice(0, 10)).filter(Boolean);
255
247
  if (!ok(sitemap) || !/<urlset|<sitemapindex/.test(sitemap.text)) miss(id, name, level, `/sitemap.xml answered ${sitemap.status}${cached(sitemap)}`);
248
+ else if (!entries.length) miss(id, name, level, `${sitemapIndex ? 'the sitemap index\'s sitemaps list' : 'the sitemap lists'} no URLs on this site${cached(sitemap)}`);
256
249
  else if (!lastmods.length) miss(id, name, level, `no lastmod dates${cached(sitemap)}`);
257
250
  else if (new Set(lastmods).size === 1 && lastmods.length > 5) miss(id, name, level, `all ${lastmods.length} lastmod dates are ${lastmods[0]}; use each page's real change date${cached(sitemap)}`);
258
251
  else pass(id, name, level, `${lastmods.length} URLs with ${new Set(lastmods).size} distinct lastmod dates`);
259
252
  }
260
253
 
261
254
  // ---- D4 llms.txt ----
255
+ let llmsAnswers = []; // D8 compares one of these pages with its markdown copy
262
256
  {
263
257
  const id = 'D4', name = 'llms.txt and llms-full.txt', level = 'required';
264
258
  // Agents follow the links in llms.txt, so a sample of the site's own links must answer 200.
265
259
  const links = ok(llms) ? llmsLinks(llms.text, `${BASE}/llms.txt`, siteHosts) : [];
266
260
  const sample = spread(links, 10);
267
261
  const answers = await mapLimit(sample, 4, async (u) => ({ u, r: await get(u) }));
262
+ llmsAnswers = answers;
268
263
  // Missing pages, server errors and no answer are broken. A 401, 403 or 405 is an endpoint that
269
264
  // exists but wants a key or another method, which is fine to list.
270
265
  const broken = answers.filter(({ r }) => !r.status || r.status === 404 || r.status === 410 || r.status >= 500).map(({ u, r }) => `${u.replace(BASE, '') || '/'} (${r.status || 'no answer'})`);
266
+ // A retired page redirects; the files that list it should name where it went.
267
+ const redirected = answers.filter(({ u, r }) => r.status >= 200 && r.status < 300 && norm(r.url) !== norm(u)).map(({ u, r }) => `${u.replace(BASE, '') || '/'} → ${r.url.replace(BASE, '') || '/'}`);
268
+ const linkProblems = [];
269
+ if (broken.length) linkProblems.push(`broken links in llms.txt: ${list(broken)} (${broken.length} of ${sample.length} sampled)`);
270
+ if (redirected.length) linkProblems.push(`llms.txt links that redirect (link the destination): ${list(redirected)}`);
271
271
  const sizes = `llms.txt ${(llms.text.length / 1024).toFixed(1)} KB, llms-full.txt ${(llmsFull.text.length / 1024).toFixed(0)} KB`;
272
272
  if (!ok(llms) || !/^\s*# \S/.test(llms.text)) miss(id, name, level, (ok(llms) ? 'llms.txt has no H1 title on its first line' : `/llms.txt answered ${llms.status}`) + cached(llms));
273
273
  else if (!ok(llmsFull)) miss(id, name, level, `/llms-full.txt missing${cached(llmsFull)}`);
274
- else if (broken.length) record(id, name, level, 'warn', `broken links in llms.txt: ${broken.slice(0, 3).join(', ')}${broken.length > 3 ? `, +${broken.length - 3} more` : ''} (${broken.length} of ${sample.length} sampled)${cached(llms)}`);
275
- else pass(id, name, level, `${sizes}${sample.length ? `; ${sample.length} sampled link${sample.length === 1 ? '' : 's'} resolve` : ''}`);
274
+ else if (linkProblems.length) record(id, name, level, 'warn', `${linkProblems.join('; ')}${cached(llms)}`);
275
+ else pass(id, name, level, `${sizes}${sample.length ? `; ${sample.length} sampled link${sample.length === 1 ? '' : 's'} resolve without a redirect` : ''}`);
276
276
  }
277
277
 
278
278
  // ---- D5 Link headers (recommended; a wrong service-desc is a false declaration) ----
279
279
  {
280
280
  const id = 'D5', name = 'Link headers', level = 'recommended';
281
281
  const link = home.headers.get('link') ?? '';
282
- const rels = new Map();
283
- for (const part of link.split(/,(?=\s*<)/)) {
284
- const href = part.match(/<([^>]+)>/)?.[1];
285
- const relAttr = part.match(/rel="?([^";]+)"?/)?.[1];
286
- if (href && relAttr) for (const r of relAttr.split(/\s+/)) rels.set(r, [...(rels.get(r) ?? []), href]);
287
- }
288
282
  const desc = rels.get('service-desc') ?? [];
289
- const missing = ['service-desc', 'describedby', ...(hasApi ? ['api-catalog'] : [])].filter((r) => !rels.has(r));
283
+ const missing = ['describedby', ...(api.present ? ['service-desc', 'api-catalog'] : [])].filter((r) => !rels.has(r)); // with no API, a service-desc would declare one
290
284
  if (!link) miss(id, name, level, 'no Link header on the homepage');
291
- else if (hasApi && desc.length && !desc.some((h) => /openapi/i.test(h))) wrong(id, name, level, `service-desc points at ${desc.join(', ')}; it must name the OpenAPI document`);
285
+ else if (api.present && desc.length && !desc.some((h) => /openapi/i.test(h))) wrong(id, name, level, `service-desc points at ${desc.join(', ')}; it must name the OpenAPI document`);
292
286
  else if (missing.length) miss(id, name, level, `missing rel ${missing.join(', ')} (has ${[...rels.keys()].join(', ')})`);
293
287
  else pass(id, name, level, `rels: ${[...rels.keys()].join(', ')}`);
294
288
  }
@@ -320,10 +314,18 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
320
314
  const id = 'D8', name = 'Markdown for agents', level = 'next';
321
315
  const varyAccept = (r) => /(^|,)\s*accept\s*(,|$)/i.test(r.headers.get('vary') ?? '');
322
316
  const tokens = homeMd.headers.get('x-markdown-tokens');
317
+ // The homepage's markdown may be llms.txt, so compare a page llms.txt links instead, when it serves markdown.
318
+ const copy = { path: null, gone: [] };
319
+ // The first linked page that serves markdown; pages that don't are fine (start with the homepage and docs).
320
+ for (const { u, r } of llmsAnswers.filter(({ u, r }) => ok(r) && /text\/html/.test(r.type) && new URL(u).pathname !== '/')) {
321
+ const md = await get(u, { headers: { accept: 'text/markdown' } });
322
+ if (ok(md) && /text\/markdown/.test(md.type)) { Object.assign(copy, { path: new URL(u).pathname, gone: missingSections(r.text, md.text) }); break; }
323
+ }
323
324
  if (!/text\/markdown/.test(homeMd.type)) miss(id, name, level, `Accept: text/markdown on / returns ${ct(homeMd)}`);
324
325
  else if (!varyAccept(homeMd) || !varyAccept(home)) miss(id, name, level, `Vary: Accept missing on the ${!varyAccept(home) ? 'HTML' : 'markdown'} response`);
326
+ else if (copy.gone.length) miss(id, name, level, `the markdown copy of ${copy.path} lacks ${copy.gone.length} section${copy.gone.length === 1 ? '' : 's'} the page has: ${list(copy.gone.map((h) => `"${h}"`))}`);
325
327
  else if (!tokens) miss(id, name, level, 'markdown and Vary are right; add x-markdown-tokens');
326
- else pass(id, name, level, `markdown with Vary: Accept on both, x-markdown-tokens ${tokens}`);
328
+ else pass(id, name, level, `markdown with Vary: Accept on both, x-markdown-tokens ${tokens}${copy.path ? `; ${copy.path}'s markdown copy has every section` : '; no page llms.txt links serves markdown, so no copy was compared'}`);
327
329
  }
328
330
 
329
331
  // ---- D9 AI catalog (ARD) ----
@@ -360,8 +362,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
360
362
  }
361
363
 
362
364
  // ---- A: API ----
363
- if (!hasApi) {
364
- for (const [id, name, level] of [['A1', 'OpenAPI 3.1', 'required'], ['A2', 'GET /api index', 'required'], ['A6', 'Errors that teach (401 without a key)', 'required'], ['A7', 'JSON catch-all for wrong API paths', 'required'], ['A3', 'API catalog (RFC 9727)', 'recommended'], ['A4', 'auth.md', 'recommended'], ['A5', 'API reference as markdown', 'recommended']]) skip(id, name, level, '--no-api');
365
+ if (!api.present) {
366
+ for (const [id, name, level] of [['A1', 'OpenAPI 3.1', 'required'], ['A2', 'GET /api index', 'required'], ['A6', 'Errors that teach (401 without a key)', 'required'], ['A7', 'JSON catch-all for wrong API paths', 'required'], ['A3', 'API catalog (RFC 9727)', 'recommended'], ['A4', 'auth.md', 'recommended'], ['A5', 'API reference as markdown', 'recommended']]) skip(id, name, level, 'no public API: /openapi.json and /api don\'t answer, and nothing declares one');
365
367
  } else {
366
368
  {
367
369
  const id = 'A1', name = 'OpenAPI 3.1', level = 'required';
@@ -411,8 +413,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
411
413
  const respGaps = new Set(ops.flatMap(({ o }) => Object.values(o.responses ?? {}).flatMap((r) => Object.values(deref(r)?.content ?? {}).flatMap((c) => undescribedProps(c.schema)))));
412
414
  const respNote = respGaps.size ? `; ${respGaps.size} response field${respGaps.size === 1 ? ' has' : 's have'} no description (see N7)` : '';
413
415
  if (!String(spec.openapi).startsWith('3.1')) miss(id, name, level, `OpenAPI ${spec.openapi} (use 3.1)`);
414
- else if (noDesc.length) miss(id, name, level, `${noDesc.length} of ${ops.length} operations ${noDesc.length === 1 ? 'has' : 'have'} no description (${noDesc.slice(0, 3).map((x) => x.key).join(', ')}${noDesc.length > 3 ? ', ...' : ''})${repeatNote}${respNote}`);
415
- else if (fieldGaps.length) miss(id, name, level, `${fieldGaps.length} request field${fieldGaps.length === 1 ? ' has' : 's have'} no description (${fieldGaps.slice(0, 3).join(', ')}${fieldGaps.length > 3 ? ', ...' : ''})${variantNote}${repeatNote}${respNote}`);
416
+ else if (noDesc.length) miss(id, name, level, `${noDesc.length} of ${ops.length} operations ${noDesc.length === 1 ? 'has' : 'have'} no description (${list(noDesc.map((x) => x.key))})${repeatNote}${respNote}`);
417
+ else if (fieldGaps.length) miss(id, name, level, `${fieldGaps.length} request field${fieldGaps.length === 1 ? ' has' : 's have'} no description (${list(fieldGaps)})${variantNote}${repeatNote}${respNote}`);
416
418
  else pass(id, name, level, `OpenAPI ${spec.openapi}; all ${ops.length} operations and their request fields described${repeatNote}${respNote}`);
417
419
  }
418
420
  }
@@ -491,9 +493,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
491
493
  const badName = read.map((f) => [f, skillNameProblem(f)]).filter(([, why]) => why).map(([f, why]) => `${f.label} (${why})`);
492
494
  const badDesc = read.filter((f) => !f.description || f.description.length > 1024).map((f) => f.label);
493
495
  const noMap = read.filter((f) => !/llms\.txt/.test(f.text)).map((f) => f.label);
494
- const noApi = read.filter((f) => !/openapi|\/docs\/(api|mcp)|https?:\/\/[^\s)]+\/(api|mcp)\b/i.test(f.text)).map((f) => f.label);
496
+ const noApi = read.filter((f) => !API_LINK.test(f.text) && !MCP_LINK.test(f.text)).map((f) => f.label);
495
497
  const long = read.filter((f) => f.lines > 500).map((f) => `${f.label} (${f.lines} lines)`);
496
- const list = (xs) => `${xs.slice(0, 3).join(', ')}${xs.length > 3 ? `, +${xs.length - 3} more` : ''}`;
497
498
  const problems = [], warns = [];
498
499
  if (unreachable.length) problems.push(`skill files that don't load: ${list(unreachable)}`);
499
500
  if (badName.length) problems.push(`names that break the spec (a lowercase slug, matching its folder and its index entry): ${list(badName)}`);
@@ -548,7 +549,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
548
549
  const id = 'M4', name = 'Hosted MCP edge', level = 'required';
549
550
  const variants = [['application/json, text/event-stream', probe]];
550
551
  for (const accept of ['application/json', '*/*']) variants.push([accept, await get(mcpPath, { method: 'POST', headers: { 'content-type': 'application/json', accept }, body: INIT })]);
551
- variants.push(['(none)', await postNoAccept(mcpPath, INIT)]);
552
+ variants.push(['(none)', await get(mcpPath, { method: 'POST', headers: { 'content-type': 'application/json', accept: null }, body: INIT })]);
552
553
  const discover = await modern('server/discover', 2);
553
554
  const browser = await get(mcpPath, { headers: { accept: 'text/html' }, redirect: 'manual' });
554
555
  const aliases = await Promise.all(['/sse', '/api/mcp'].map((p) => get(p, { method: 'POST', redirect: 'manual', headers: { 'content-type': 'application/json' }, body: INIT })));
@@ -610,23 +611,20 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
610
611
  }
611
612
 
612
613
  // ---- W: search, on a sample of sitemap pages ----
614
+ let htmlKb = null; // median HTML size of the sampled pages, reported, not scored
615
+ let richTest = []; // the pages to run through Google's Rich Results Test (W4, in review)
613
616
  {
614
- const locs = (xml) => [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1].replace(/&amp;/g, '&'));
615
- let urls = locs(sitemap.text);
616
- if (/<sitemapindex/.test(sitemap.text)) {
617
- const subs = urls.slice(0, 3);
618
- urls = [];
619
- for (const sub of subs) urls.push(...locs((await get(sub)).text));
620
- }
621
- urls = [...new Set(urls)].filter((u) => { try { return siteHosts.has(new URL(u).hostname); } catch { return false; } });
622
- const norm = (u) => { try { const x = new URL(u); return `${x.protocol}//${x.host.toLowerCase()}${x.pathname.replace(/\/+$/, '') || '/'}${x.search}`; } catch { return u; } };
623
- const pick = [];
624
- const n = Math.min(20, urls.length);
625
- for (let i = 0; i < n; i++) pick.push(urls[Math.floor((i * urls.length) / n)]);
626
- const decode = (t) => t.replace(/&amp;/g, '&').replace(/&#39;|&#x27;|&apos;/g, "'").replace(/&quot;/g, '"').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/\s+/g, ' ').trim();
617
+ const urls = entries.map((e) => e.loc);
618
+ const sample = samplePages(urls);
619
+ const pick = sample.pages;
620
+ const templateOf = new Map(sample.templates.flatMap((t) => t.urls.map((u) => [norm(u), t.pattern])));
621
+ const templatePages = new Set(templateOf.keys());
622
+ const sampled = `${pick.length} sampled pages (${sample.static} static, ${sample.templates.length ? `${sample.templates.length} template${sample.templates.length === 1 ? '' : 's'}` : 'no templates'}${sample.skipped ? `; ${sample.skipped} static pages over the cap not read` : ''})`;
627
623
  const attrs = (tag) => Object.fromEntries([...tag.matchAll(/([\w:-]+)\s*=\s*("([^"]*)"|'([^']*)')/g)].map((m) => [m[1].toLowerCase(), m[3] ?? m[4]]));
628
624
  const tags = (html, name) => [...html.matchAll(new RegExp(`<${name}\\b[^>]*>`, 'gi'))].map((m) => attrs(m[0]));
629
625
  const meta = (html, key, val) => tags(html, 'meta').find((a) => (a[key] ?? '').toLowerCase() === val)?.content;
626
+ const text = (v) => (v === undefined ? undefined : decodeText(v));
627
+ const tw = (html, field) => text(meta(html, 'name', `twitter:${field}`) ?? meta(html, 'property', `twitter:${field}`));
630
628
  const jsonLd = (html) => {
631
629
  const ld = [...html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)].map((m) => { try { return JSON.parse(m[1]); } catch { return null; } });
632
630
  const types = [];
@@ -634,27 +632,29 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
634
632
  ld.forEach(walk);
635
633
  return { ld, types };
636
634
  };
637
- const pages = await mapLimit(pick, 4, async (u) => {
638
- const r = await get(u, { headers: { accept: 'text/html' } });
635
+ const parsePage = (u, r) => {
639
636
  const h = r.text;
640
637
  const { ld, types } = jsonLd(h);
641
638
  return {
642
- url: u, status: r.status, finalUrl: r.url, html: /text\/html/.test(r.type),
643
- title: decode(h.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? ''),
644
- description: decode(meta(h, 'name', 'description') ?? ''),
639
+ url: u, status: r.status, finalUrl: r.url, html: /text\/html/.test(r.type), bytes: Buffer.byteLength(h), error: r.error,
640
+ title: decodeText(h.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? ''),
641
+ description: decodeText(meta(h, 'name', 'description') ?? ''),
645
642
  robots: (meta(h, 'name', 'robots') ?? '').toLowerCase(),
646
643
  canonicals: tags(h, 'link').filter((a) => (a.rel ?? '').toLowerCase() === 'canonical').map((a) => a.href),
647
- og: { title: meta(h, 'property', 'og:title'), description: meta(h, 'property', 'og:description'), image: meta(h, 'property', 'og:image'), width: meta(h, 'property', 'og:image:width'), alt: meta(h, 'property', 'og:image:alt') },
644
+ og: { title: text(meta(h, 'property', 'og:title')), description: text(meta(h, 'property', 'og:description')), image: meta(h, 'property', 'og:image'), width: meta(h, 'property', 'og:image:width'), alt: meta(h, 'property', 'og:image:alt') },
648
645
  twitterCard: meta(h, 'name', 'twitter:card'),
649
- h1: (h.match(/<h1[\s>]/gi) ?? []).length,
646
+ twitter: { title: tw(h, 'title'), description: tw(h, 'description') },
647
+ headings: headings(h),
650
648
  lang: /<html[^>]*\slang=["']?[\w-]+/i.test(h),
651
649
  imgNoAlt: tags(h, 'img').filter((a) => a.alt === undefined).length,
652
650
  ldBroken: ld.filter((x) => x === null).length, types,
653
651
  };
654
- });
655
- const ok200 = pages.filter((p) => p.status === 200 && p.html);
652
+ };
653
+ const pages = await mapLimit(pick, 4, async (u) => parsePage(u, await get(u, { headers: { accept: 'text/html' } })));
654
+ // A body the audit refused (an encoding it doesn't decode) is reported once, in W3, not as missing tags everywhere.
655
+ const ok200 = pages.filter((p) => p.status === 200 && p.html && !p.error);
656
+ if (ok200.length) { const sizes = ok200.map((p) => p.bytes).sort((a, b) => a - b); htmlKb = Math.round(sizes[Math.floor(sizes.length / 2)] / 1024); }
656
657
  const short = (u) => u.replace(BASE, '') || '/';
657
- const list = (xs) => `${xs.slice(0, 3).join(', ')}${xs.length > 3 ? `, +${xs.length - 3} more` : ''}`;
658
658
  if (!pick.length) {
659
659
  for (const [id, name] of [['W1', 'Titles and descriptions'], ['W2', 'Canonical URLs'], ['W3', 'Indexable pages only'], ['W4', 'Structured data'], ['W5', 'Link previews'], ['W6', 'Page structure']]) miss(id, name, 'required', 'no URLs in the sitemap to sample');
660
660
  } else {
@@ -677,7 +677,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
677
677
  if (badDescs.length) lengths.push(`descriptions outside 50 to 160 characters: ${list(badDescs)}`);
678
678
  if (problems.length) miss(id, name, level, problems.concat(lengths).join('; '));
679
679
  else if (lengths.length) record(id, name, level, 'warn', lengths.join('; '));
680
- else pass(id, name, level, `${ok200.length} sampled pages, all with their own title and description`);
680
+ else pass(id, name, level, `${sampled}, all with their own title and description`);
681
681
  }
682
682
  // W2 canonical URLs and one canonical host
683
683
  {
@@ -691,9 +691,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
691
691
  let url = start;
692
692
  for (let n = 0; n < 5; n++) {
693
693
  if (new URL(url).origin === origin) return { n, end: url };
694
- if (!(await isSafe(url))) return { n, end: url };
695
- let r;
696
- try { r = await fetch(url, { method: 'GET', redirect: 'manual', headers: { 'user-agent': UA }, signal: AbortSignal.timeout(15000) }); } catch { return n === 0 ? null : { n, end: url }; }
694
+ const r = await get(url, { redirect: 'manual' }); // get() refuses an address that isn't public
695
+ if (!r.status) return n === 0 ? null : { n, end: url };
697
696
  const loc = r.headers.get('location');
698
697
  if (r.status < 300 || r.status >= 400 || !loc) return { n, end: url };
699
698
  url = new URL(loc, url).href;
@@ -714,7 +713,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
714
713
  if (elsewhere.length) problems.push(`sitemap URLs canonical to another URL: ${list(elsewhere)}`);
715
714
  problems.push(...hostProblems);
716
715
  if (problems.length) miss(id, name, level, problems.join('; '));
717
- else pass(id, name, level, `self-referencing canonicals on ${ok200.length} sampled pages; ${baseOrigin ? 'host redirects not checked with --base' : `http and the other host redirect once to ${origin}`}`);
716
+ else pass(id, name, level, `self-referencing canonicals on ${sampled}; ${baseOrigin ? 'host redirects not checked with --base' : `http and the other host redirect once to ${origin}`}`);
718
717
  }
719
718
  // W3 only indexable pages in the sitemap; real 404s
720
719
  {
@@ -722,8 +721,10 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
722
721
  const broken = pages.filter((p) => p.status !== 200).map((p) => `${short(p.url)} (${p.status})`);
723
722
  const moved = pages.filter((p) => p.status === 200 && norm(p.finalUrl) !== norm(p.url)).map((p) => `${short(p.url)} → ${short(p.finalUrl)}`);
724
723
  const noindex = ok200.filter((p) => /noindex/.test(p.robots)).map((p) => short(p.url));
725
- const deep = ok200.map((p) => new URL(p.url).pathname).find((path) => path.split('/').filter(Boolean).length >= 2);
726
- const probes = [`/no-such-page-${rand}`, ...(deep ? [deep.replace(/\/[^/]+\/?$/, `/no-such-item-${rand}`)] : [])];
724
+ // A missing item under each template: a detail route is where a loading boundary or a fallback answers 200.
725
+ // Templates first, then any deeper sampled page, so a small site's /blog/[post] gets a probe too.
726
+ const parents = [...sample.templates.map((t) => t.urls[0]), ...pick].map((u) => new URL(u).pathname).filter((path) => pathDepth(path) >= 2).map((path) => path.replace(/\/[^/]+\/?$/, ''));
727
+ const probes = [`/no-such-page-${rand}`, ...[...new Set(parents)].slice(0, 3).map((parent) => `${parent}/no-such-item-${rand}`)];
727
728
  const soft = [];
728
729
  for (const path of probes) {
729
730
  const r = await get(path, { headers: { accept: 'text/html' } });
@@ -731,11 +732,13 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
731
732
  }
732
733
  const problems = [];
733
734
  if (broken.length) problems.push(`sitemap URLs that don't answer 200: ${list(broken)}`);
735
+ const unread = pages.filter((p) => p.status === 200 && p.error).map((p) => `${short(p.url)} (${p.error})`);
736
+ if (unread.length) problems.push(`pages the audit couldn't read: ${list(unread)}; send gzip or no encoding unless the request asks for another`);
734
737
  if (moved.length) problems.push(`sitemap URLs that redirect: ${list(moved)}`);
735
738
  if (noindex.length) problems.push(`sitemap URLs marked noindex: ${list(noindex)}`);
736
739
  if (soft.length) problems.push(`missing pages answer 200 instead of 404: ${list(soft)}`);
737
740
  if (problems.length) miss(id, name, level, problems.join('; '));
738
- else pass(id, name, level, `${pages.length} sampled sitemap URLs answer 200 with no redirect or noindex; missing pages get a 404`);
741
+ else pass(id, name, level, `${sampled} answer 200 with no redirect or noindex; missing pages get a 404`);
739
742
  }
740
743
  // W4 structured data
741
744
  {
@@ -743,9 +746,13 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
743
746
  const homeTypes = home.status === 200 ? jsonLd(home.text).types : null;
744
747
  const broken = ok200.filter((p) => p.ldBroken).map((p) => short(p.url));
745
748
  const none = ok200.filter((p) => !p.types.length).map((p) => short(p.url));
746
- const deepPages = ok200.filter((p) => new URL(p.url).pathname.split('/').filter(Boolean).length >= 2);
749
+ const deepPages = ok200.filter((p) => templatePages.has(norm(p.url)) && pathDepth(p.url) >= 2);
747
750
  const noCrumbs = deepPages.filter((p) => !p.types.includes('BreadcrumbList')).map((p) => short(p.url));
748
751
  const all = new Set(ok200.flatMap((p) => p.types));
752
+ // One page per kind of markup: the homepage, a page of each template, then the rest.
753
+ const homePage = ok200.find((p) => short(p.url) === '/') ?? (home.status === 200 ? { url: `${BASE}/`, types: homeTypes } : null);
754
+ const isTemplate = (p) => templatePages.has(norm(p.url));
755
+ richTest = richResultPages([homePage, ...ok200.filter(isTemplate).map((p) => ({ ...p, template: templateOf.get(norm(p.url)) })), ...ok200.filter((p) => !isTemplate(p))].filter(Boolean));
749
756
  const notes = [];
750
757
  if (all.has('FAQPage') || all.has('HowTo')) notes.push(`${['FAQPage', 'HowTo'].filter((t) => all.has(t)).join(' and ')} markup earns no rich results in Google anymore (HowTo ended in 2023, FAQ in May 2026); keep it only where the page really is one`);
751
758
  if (all.has('Person')) notes.push('Person markup found: fine on a profile the page labels as an AI agent; don\'t imply a human');
@@ -754,7 +761,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
754
761
  if (homeTypes && !homeTypes.some((t) => ['Organization', 'WebSite'].includes(t))) problems.push('no Organization or WebSite on the homepage');
755
762
  if (none.length) problems.push(`no JSON-LD on ${list(none)}`);
756
763
  if (problems.length) miss(id, name, level, problems.concat(notes).join('; '));
757
- else if (noCrumbs.length > deepPages.length / 2) record(id, name, level, 'warn', [`no BreadcrumbList on ${noCrumbs.length} of ${deepPages.length} deeper pages`, ...notes].join('; '));
764
+ else if (noCrumbs.length > deepPages.length / 2) record(id, name, level, 'warn', [`no BreadcrumbList on ${noCrumbs.length} of ${deepPages.length} sampled template pages`, ...notes].join('; '));
758
765
  else pass(id, name, level, [`types: ${[...all].slice(0, 8).join(', ')}`, ...notes].join('; '));
759
766
  }
760
767
  // W5 link previews
@@ -773,25 +780,37 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
773
780
  if (missingOg.length) problems.push(`missing og:title, og:description or og:image on ${list(missingOg)}`);
774
781
  if (noCard.length) problems.push(`no twitter:card on ${list(noCard)}`);
775
782
  if (badImages.length) problems.push(`share images that don't load: ${list(badImages)}`);
776
- if (problems.length) miss(id, name, level, problems.join('; '));
777
- else if (undeclared.length) record(id, name, level, 'warn', `og:image size or alt text not declared on ${list(undeclared)}`);
778
- else pass(id, name, level, `Open Graph and twitter:card on ${ok200.length} sampled pages; share images load`);
783
+ // The homepage joins the comparison even when the sample missed it: a layout's default card is its text.
784
+ const cardPages = ok200.map((p) => ({ ...p, path: short(p.url) }));
785
+ if (home.status === 200 && !cardPages.some((p) => p.path === '/')) cardPages.push({ ...parsePage(BASE, home), path: '/' });
786
+ const shared = sharedCardText(cardPages);
787
+ const clip = (t) => (t.length > 50 ? `${t.slice(0, 50)}…` : t);
788
+ if (shared.x.paths.length) problems.push(`X shows another page's title or description on ${list(shared.x.paths)} ("${clip(shared.x.text)}"; twitter:title and twitter:description, else og:)`);
789
+ if (shared.og.paths.length) problems.push(`Facebook and LinkedIn show another page's title or description on ${list(shared.og.paths)} ("${clip(shared.og.text)}")`);
790
+ const warns = undeclared.length ? [`og:image size or alt text not declared on ${list(undeclared)}`] : [];
791
+ if (problems.length) miss(id, name, level, problems.concat(warns).join('; '));
792
+ else if (warns.length) record(id, name, level, 'warn', warns.join('; '));
793
+ else pass(id, name, level, `Open Graph and twitter:card on ${sampled}; share images load`);
779
794
  }
780
795
  // W6 page structure
781
796
  {
782
797
  const id = 'W6', name = 'Page structure', level = 'required';
783
- const noH1 = ok200.filter((p) => p.h1 === 0).map((p) => short(p.url));
784
- const manyH1 = ok200.filter((p) => p.h1 > 1).map((p) => `${short(p.url)} (${p.h1})`);
798
+ const h1s = (p) => p.headings.filter((x) => x.level === 1).length;
799
+ const noH1 = ok200.filter((p) => h1s(p) === 0).map((p) => short(p.url));
800
+ const manyH1 = ok200.filter((p) => h1s(p) > 1).map((p) => `${short(p.url)} (${h1s(p)})`);
785
801
  const noLang = ok200.filter((p) => !p.lang).map((p) => short(p.url));
786
802
  const noAlt = ok200.filter((p) => p.imgNoAlt).map((p) => `${short(p.url)} (${p.imgNoAlt})`);
803
+ const skipped = ok200.map((p) => [p, headingSkips(p.headings)]).filter(([, sk]) => sk.length).map(([p, sk]) => `${short(p.url)} (${sk[0]})`);
787
804
  const problems = [], warns = [];
788
805
  if (noH1.length) problems.push(`no h1 on ${list(noH1)}`);
789
806
  if (noLang.length) problems.push(`no html lang on ${list(noLang)}`);
790
807
  if (manyH1.length) warns.push(`several h1s on ${list(manyH1)}`);
791
808
  if (noAlt.length) warns.push(`images without alt on ${list(noAlt)}`);
792
- if (problems.length) miss(id, name, level, problems.concat(warns).join('; '));
793
- else if (warns.length) record(id, name, level, 'warn', warns.join('; '));
794
- else pass(id, name, level, `one h1, html lang and alt text on ${ok200.length} sampled pages`);
809
+ if (skipped.length) warns.push(`heading levels skipped on ${list(skipped)}`);
810
+ const weight = `median HTML ${htmlKb} KB`;
811
+ if (problems.length) miss(id, name, level, problems.concat(warns, weight).join('; '));
812
+ else if (warns.length) record(id, name, level, 'warn', warns.concat(weight).join('; '));
813
+ else pass(id, name, level, `one h1, heading levels in order, html lang and alt text on ${sampled}; ${weight}`);
795
814
  }
796
815
  }
797
816
  }
@@ -847,7 +866,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
847
866
  }
848
867
 
849
868
  results.sort((a, b) => LEVELS.indexOf(a.level) - LEVELS.indexOf(b.level) || ORDER.indexOf(a.id) - ORDER.indexOf(b.id));
850
- return { domain, base: baseOrigin, version: VERSION, checked_at: new Date().toISOString(), hosted_mcp: hosted, results };
869
+ return { domain, base: baseOrigin, version: VERSION, checked_at: new Date().toISOString(), hosted_mcp: hosted, has_api: api.present, html_kb_median: htmlKb, rich_results_test: richTest, results };
851
870
  }
852
871
 
853
872
  export function score(results, level) {
@@ -872,6 +891,7 @@ export function matrix(reports, command, recorded = {}) {
872
891
  `| | **Required passed** | | ${reports.map((rep) => `**${score(rep.results, 'required')}**`).join(' | ')} |`,
873
892
  `| | Recommended passed | | ${reports.map((rep) => score(rep.results, 'recommended')).join(' | ')} |`,
874
893
  `| | Next level passed | | ${reports.map((rep) => score(rep.results, 'next')).join(' | ')} |`,
894
+ `| | Public API | | ${reports.map((rep) => (rep.has_api ? 'yes' : 'no')).join(' | ')} |`,
875
895
  `| | Hosted MCP endpoint | | ${reports.map((rep) => (rep.hosted_mcp ? 'yes' : 'no')).join(' | ')} |`,
876
896
  ...[['T5', 'Agent usability test (recorded)'], ['T6', 'Search numbers (recorded)']].map(([id, label]) =>
877
897
  `| ${id} | ${label} | required | ${reports.map((rep) => formatRecorded(recorded[rep.domain]?.[id])).join(' | ')} |`),
@@ -887,5 +907,10 @@ export function matrix(reports, command, recorded = {}) {
887
907
  out.push('');
888
908
  }
889
909
  out.push('## Checked in review and tests, not scored', '', ...REVIEW.map(([id, what]) => `- **${id}**: ${what}`), '');
910
+ const rich = reports.filter((rep) => rep.rich_results_test?.length);
911
+ if (rich.length) {
912
+ out.push('## Pages for Google\'s Rich Results Test (W4)', '', 'One page per kind of markup. Pages built from one template share it, so one tests them all.', '');
913
+ for (const rep of rich) out.push(`**${rep.domain}**`, '', ...rep.rich_results_test.map((p) => `- [${p.url}](${p.test}): ${p.features.join(', ')}`), '');
914
+ }
890
915
  return out.join('\n');
891
916
  }