readiness-audit 1.2.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/audit.mjs CHANGED
@@ -4,18 +4,16 @@
4
4
  // The standard and what each ID means: STANDARD.md in
5
5
  // https://github.com/geeks-accelerator/agent-and-search-readiness
6
6
  // Read-only: GETs plus a few POSTs that create nothing (an MCP initialize, server/discover and
7
- // tools/list, an empty POST to /). Node 20+, no dependencies.
7
+ // tools/list, an empty POST to /). Node 26 or later, no dependencies.
8
8
 
9
9
  import { lookup, resolveTxt, Resolver } from 'node:dns/promises';
10
- import { request as httpRequest } from 'node:http';
11
- import { request as httpsRequest } from 'node:https';
12
10
  import { isIP } from 'node:net';
13
- import { SKILL_NAME, sampleSkillEntries, skillNameProblem, cdnCacheNote, repeatsSummary, hasVariantDescription, spread, llmsLinks, baseMapper, formatRecorded } from './helpers.mjs';
11
+ import { request } from './http.mjs';
12
+ import { SKILL_NAME, sampleSkillEntries, skillNameProblem, cdnCacheNote, repeatsSummary, hasVariantDescription, spread, samplePages, sharedCardText, API_LINK, MCP_LINK, linkRels, apiPresence, DISCOVERY_FILES, AI_BOTS, missingSections, list, norm, pathDepth, decodeText, headings, headingSkips, sitemapEntries, llmsLinks, baseMapper, formatRecorded } from './helpers.mjs';
14
13
 
15
- export const VERSION = '1.2.0';
14
+ export const VERSION = '2.0.0';
16
15
  export const REPO = 'https://github.com/geeks-accelerator/agent-and-search-readiness';
17
16
  const UA = `readiness-audit/${VERSION} (+${REPO})`;
18
- const AI_BOTS = ['gptbot', 'oai-searchbot', 'chatgpt-user', 'claudebot', 'claude-user', 'claude-searchbot', 'perplexitybot', 'perplexity-user', 'google-extended', 'applebot-extended'];
19
17
  const AID_PROTOCOLS = ['mcp', 'a2a', 'openapi', 'grpc', 'graphql', 'websocket', 'local', 'zeroconf', 'ucp'];
20
18
  const SKILLS_SCHEMA = 'https://schemas.agentskills.io/discovery/0.2.0/schema.json';
21
19
  const CARD_SCHEMA = 'https://static.modelcontextprotocol.io/schemas/v1/server-card.schema.json';
@@ -24,7 +22,7 @@ const ORDER = ['D1', 'D2', 'D4', 'D6', 'D11', 'D12', 'W1', 'W2', 'W3', 'W4', 'W5
24
22
  // Items the scorecard can't see from outside. Each project checks these in review and tests.
25
23
  export const REVIEW = [
26
24
  ['D3', 'pages read well: server-rendered, unique prose on generated pages (the scorecard checks W1 to W6 on a sample)'],
27
- ['W7', 'internal links: entity names link to their pages, descriptive anchors, no orphan pages'],
25
+ ['W7', 'internal links: entity names link to their pages, anchors that say what the target is (not the brand alone), no orphan pages; sibling links in context, never sitewide (E6)'],
28
26
  ['W8', 'Core Web Vitals on the main pages (Lighthouse or PageSpeed Insights)'],
29
27
  ['W9', 'listed in the directories your audience uses'],
30
28
  ['D13', 'a page for agents\' builders with a copy-paste prompt'],
@@ -36,7 +34,7 @@ export const REVIEW = [
36
34
  ['A11', 'public and private stated honestly, one privacy filter'],
37
35
  ['M2', 'stdio package: saved key, register guard, rotate, User-Agent'],
38
36
  ['M3', 'published to npm, the MCP Registry, Smithery and Glama, one version'],
39
- ['S1', 'skills ask before anything public or permanent (the scorecard checks names, descriptions and links on a sample)'],
37
+ ['S1', 'skills: triggers name the job, lasting actions say what they do, and an agent can follow them through the core loop on its own (the scorecard checks names, descriptions and links on a sample)'],
40
38
  ['S3', 'ClawHub owners map'],
41
39
  ['S4', 'plugin bundle with generated manifests'],
42
40
  ['E1-E6', 'first-call value, reasons to return, scheduled-agent performance'],
@@ -106,7 +104,7 @@ export function makeGuard(domain) {
106
104
  return { siteHosts, isSafe };
107
105
  }
108
106
 
109
- export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = null } = {}) {
107
+ export async function audit(domain, { mcpPath = '/mcp', base = null } = {}) {
110
108
  const BASE = `https://${domain}`;
111
109
  const rand = Math.random().toString(36).slice(2, 10);
112
110
 
@@ -122,38 +120,16 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
122
120
  for (let hop = 0; ; hop++) {
123
121
  const target = toBase(url);
124
122
  if (target === url && !(await isSafe(url))) return { status: 0, headers: new Headers(), text: '', url, type: '', error: 'refused: not a public address' };
125
- let res;
126
- try {
127
- res = await fetch(target, { method: m, body: b, redirect: 'manual', headers: { 'user-agent': UA, ...headers }, signal: AbortSignal.timeout(15000) });
128
- } catch (e) {
129
- return { status: 0, headers: new Headers(), text: '', url, type: '', error: e.message };
130
- }
123
+ const res = await request(target, { method: m, body: b, headers, userAgent: UA });
131
124
  const loc = res.headers.get('location');
132
125
  if (redirect === 'follow' && res.status >= 300 && res.status < 400 && loc && hop < 5) {
133
- try { await res.body?.cancel(); } catch { /* already closed */ }
134
126
  url = fromBase(new URL(loc, target).href);
135
127
  if (res.status !== 307 && res.status !== 308) { m = 'GET'; b = undefined; }
136
128
  continue;
137
129
  }
138
- const text = await res.text().catch(() => '');
139
- return { status: res.status, headers: res.headers, text, url, type: res.headers.get('content-type') ?? '' };
130
+ return { ...res, url, type: res.headers.get('content-type') ?? '' };
140
131
  }
141
132
  }
142
- // fetch() always sends an Accept header, so the "no Accept" MCP probe uses node:https.
143
- function postNoAccept(path, payload) {
144
- const target = new URL(toBase(BASE + path));
145
- const send = target.protocol === 'http:' ? httpRequest : httpsRequest;
146
- return new Promise((resolve) => {
147
- const req = send(target, { method: 'POST', timeout: 15000, headers: { 'user-agent': UA, 'content-type': 'application/json', 'content-length': Buffer.byteLength(payload) } }, (res) => {
148
- let text = '';
149
- res.on('data', (c) => { text += c; });
150
- res.on('end', () => resolve({ status: res.statusCode, type: res.headers['content-type'] ?? '', text }));
151
- });
152
- req.on('error', () => resolve({ status: 0, type: '', text: '' }));
153
- req.on('timeout', () => { req.destroy(); resolve({ status: 0, type: '', text: '' }); });
154
- req.end(payload);
155
- });
156
- }
157
133
  const json = (r) => { try { return JSON.parse(r.text); } catch { return null; } };
158
134
  const isJson = (r) => /json/.test(r.type) && json(r) !== null;
159
135
  const ok = (r) => r.status >= 200 && r.status < 300;
@@ -174,34 +150,37 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
174
150
  const wrong = (id, name, level, detail) => record(id, name, level, 'fail', detail); // present but false or invalid
175
151
  const skip = (id, name, level, detail) => record(id, name, level, 'skip', detail);
176
152
 
153
+ const D = DISCOVERY_FILES;
177
154
  const [home, homeMd, robots, sitemap, llms, llmsFull, openapi, apiIndex, apiCatalog, security, authMd, docsMd,
178
155
  agentCard, ard, aiCatalog, skills, prm, asMeta, aiPlugin, wkMiss, apiMiss, postRoot, legacyCard, wkCard] = await mapLimit([
179
156
  () => get('/'),
180
157
  () => get('/', { headers: { accept: 'text/markdown' } }),
181
- () => get('/robots.txt'),
182
- () => get('/sitemap.xml'),
183
- () => get('/llms.txt'),
184
- () => get('/llms-full.txt'),
185
- () => get('/openapi.json'),
186
- () => get('/api', { headers: { accept: 'application/json' } }),
187
- () => get('/.well-known/api-catalog'),
188
- () => get('/.well-known/security.txt'),
189
- () => get('/auth.md'),
190
- () => get('/docs/api.md'),
191
- () => get('/.well-known/agent-card.json'),
192
- () => get('/.well-known/ard.json'),
193
- () => get('/.well-known/ai-catalog.json'),
194
- () => get('/.well-known/agent-skills/index.json'),
195
- () => get('/.well-known/oauth-protected-resource'),
196
- () => get('/.well-known/oauth-authorization-server'),
197
- () => get('/.well-known/ai-plugin.json'),
158
+ () => get(D.robots),
159
+ () => get(D.sitemap),
160
+ () => get(D.llms),
161
+ () => get(D.llmsFull),
162
+ () => get(D.openapi),
163
+ () => get(D.apiIndex, { headers: { accept: 'application/json' } }),
164
+ () => get(D.apiCatalog),
165
+ () => get(D.security),
166
+ () => get(D.authMd),
167
+ () => get(D.docsMd),
168
+ () => get(D.agentCard),
169
+ () => get(D.ard),
170
+ () => get(D.aiCatalog),
171
+ () => get(D.skills),
172
+ () => get(D.prm),
173
+ () => get(D.asMeta),
174
+ () => get(D.aiPlugin),
198
175
  () => get(`/.well-known/no-such-file-${rand}`),
199
176
  () => get(`/api/no-such-endpoint-${rand}`, { headers: { accept: 'application/json' } }),
200
177
  () => get('/', { method: 'POST', redirect: 'manual' }),
201
- () => get('/.well-known/mcp.json'),
202
- () => get('/.well-known/mcp/server-card.json'),
178
+ () => get(D.legacyCard),
179
+ () => get(D.wkCard),
203
180
  ], 6, (job) => job());
204
181
  const catalogEntries = [...(json(ard)?.entries ?? []), ...(json(aiCatalog)?.entries ?? [])];
182
+ const rels = linkRels(home.headers.get('link'));
183
+ const api = apiPresence({ openapiSpec: json(openapi), apiIndexJson: ok(apiIndex) && isJson(apiIndex), rels, catalogJson: ok(apiCatalog) ? json(apiCatalog) : null, llmsText: ok(llms) ? llms.text : '' });
205
184
 
206
185
  // ---- D1 robots.txt (required) and D15 AI crawlers and Content-Signal (recommended) ----
207
186
  const groups = [];
@@ -232,7 +211,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
232
211
  const star = groups.find((g) => g.agents.includes('*'));
233
212
  const leaks = star ? groups.filter((g) => g !== star).flatMap((g) => [...star.disallow].filter((d) => !g.disallow.has(d)).map((d) => `${g.agents[0]} skips Disallow ${d}`)) : [];
234
213
  if (!ok(robots)) miss(id, name, level, `/robots.txt answered ${robots.status}${cached(robots)}`);
235
- else if (leaks.length) miss(id, name, level, `named groups drop rules from the * group (a crawler obeys only its own group): ${[...new Set(leaks)].slice(0, 3).join('; ')}${cached(robots)}`);
214
+ else if (leaks.length) miss(id, name, level, `named groups drop rules from the * group (a crawler obeys only its own group): ${list([...new Set(leaks)], '; ')}${cached(robots)}`);
236
215
  else if (!sitemapLine) miss(id, name, level, `no Sitemap line${cached(robots)}`);
237
216
  else pass(id, name, level, `${groups.length} group${groups.length === 1 ? '' : 's'} with consistent rules, Sitemap listed`);
238
217
  }
@@ -241,54 +220,68 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
241
220
  const named = AI_BOTS.filter((b) => groups.some((g) => g.agents.includes(b)));
242
221
  const unsignaled = groups.filter((g) => !g.signal);
243
222
  const problems = [];
244
- if (unsignaled.length) problems.push(`${unsignaled.length} of ${groups.length} group${groups.length === 1 ? '' : 's'} ${unsignaled.length === 1 ? 'has' : 'have'} no Content-Signal (${unsignaled.slice(0, 3).map((g) => g.agents[0]).join(', ')}${unsignaled.length > 3 ? ', ...' : ''})`);
223
+ if (unsignaled.length) problems.push(`${unsignaled.length} of ${groups.length} group${groups.length === 1 ? '' : 's'} ${unsignaled.length === 1 ? 'has' : 'have'} no Content-Signal (${list(unsignaled.map((g) => g.agents[0]))})`);
245
224
  if (named.length < 3) problems.push(`names ${named.length} of the main AI crawlers`);
246
225
  if (!ok(robots)) miss(id, name, level, 'no robots.txt');
247
226
  else if (problems.length) miss(id, name, level, problems.join('; ') + cached(robots));
248
227
  else pass(id, name, level, `${named.length} AI crawlers named, Content-Signal in every group`);
249
228
  }
250
229
 
230
+ // ---- The sitemap, read once for D2 and the W checks: an index is expanded to its pages ----
231
+ const sitemapIndex = /<sitemapindex/.test(sitemap.text);
232
+ const sitemapLists = sitemapIndex
233
+ ? await mapLimit(sitemapEntries(sitemap.text).slice(0, 10), 4, async (e) => sitemapEntries((await get(e.loc)).text))
234
+ : [sitemapEntries(sitemap.text)];
235
+ const seenLocs = new Set();
236
+ const entries = sitemapLists.flat().filter((e) => {
237
+ try { if (!siteHosts.has(new URL(e.loc).hostname) || seenLocs.has(e.loc)) return false; } catch { return false; }
238
+ seenLocs.add(e.loc);
239
+ return true;
240
+ });
241
+
251
242
  // ---- D2 sitemap ----
252
243
  {
253
244
  const id = 'D2', name = 'sitemap.xml', level = 'required';
254
- const lastmods = [...sitemap.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map((m) => m[1].slice(0, 10));
245
+ const lastmods = entries.map((e) => e.lastmod?.slice(0, 10)).filter(Boolean);
255
246
  if (!ok(sitemap) || !/<urlset|<sitemapindex/.test(sitemap.text)) miss(id, name, level, `/sitemap.xml answered ${sitemap.status}${cached(sitemap)}`);
247
+ else if (!entries.length) miss(id, name, level, `${sitemapIndex ? 'the sitemap index\'s sitemaps list' : 'the sitemap lists'} no URLs on this site${cached(sitemap)}`);
256
248
  else if (!lastmods.length) miss(id, name, level, `no lastmod dates${cached(sitemap)}`);
257
249
  else if (new Set(lastmods).size === 1 && lastmods.length > 5) miss(id, name, level, `all ${lastmods.length} lastmod dates are ${lastmods[0]}; use each page's real change date${cached(sitemap)}`);
258
250
  else pass(id, name, level, `${lastmods.length} URLs with ${new Set(lastmods).size} distinct lastmod dates`);
259
251
  }
260
252
 
261
253
  // ---- D4 llms.txt ----
254
+ let llmsAnswers = []; // D8 compares one of these pages with its markdown copy
262
255
  {
263
256
  const id = 'D4', name = 'llms.txt and llms-full.txt', level = 'required';
264
257
  // Agents follow the links in llms.txt, so a sample of the site's own links must answer 200.
265
258
  const links = ok(llms) ? llmsLinks(llms.text, `${BASE}/llms.txt`, siteHosts) : [];
266
259
  const sample = spread(links, 10);
267
260
  const answers = await mapLimit(sample, 4, async (u) => ({ u, r: await get(u) }));
261
+ llmsAnswers = answers;
268
262
  // Missing pages, server errors and no answer are broken. A 401, 403 or 405 is an endpoint that
269
263
  // exists but wants a key or another method, which is fine to list.
270
264
  const broken = answers.filter(({ r }) => !r.status || r.status === 404 || r.status === 410 || r.status >= 500).map(({ u, r }) => `${u.replace(BASE, '') || '/'} (${r.status || 'no answer'})`);
265
+ // A retired page redirects; the files that list it should name where it went.
266
+ const redirected = answers.filter(({ u, r }) => r.status >= 200 && r.status < 300 && norm(r.url) !== norm(u)).map(({ u, r }) => `${u.replace(BASE, '') || '/'} → ${r.url.replace(BASE, '') || '/'}`);
267
+ const linkProblems = [];
268
+ if (broken.length) linkProblems.push(`broken links in llms.txt: ${list(broken)} (${broken.length} of ${sample.length} sampled)`);
269
+ if (redirected.length) linkProblems.push(`llms.txt links that redirect (link the destination): ${list(redirected)}`);
271
270
  const sizes = `llms.txt ${(llms.text.length / 1024).toFixed(1)} KB, llms-full.txt ${(llmsFull.text.length / 1024).toFixed(0)} KB`;
272
271
  if (!ok(llms) || !/^\s*# \S/.test(llms.text)) miss(id, name, level, (ok(llms) ? 'llms.txt has no H1 title on its first line' : `/llms.txt answered ${llms.status}`) + cached(llms));
273
272
  else if (!ok(llmsFull)) miss(id, name, level, `/llms-full.txt missing${cached(llmsFull)}`);
274
- else if (broken.length) record(id, name, level, 'warn', `broken links in llms.txt: ${broken.slice(0, 3).join(', ')}${broken.length > 3 ? `, +${broken.length - 3} more` : ''} (${broken.length} of ${sample.length} sampled)${cached(llms)}`);
275
- else pass(id, name, level, `${sizes}${sample.length ? `; ${sample.length} sampled link${sample.length === 1 ? '' : 's'} resolve` : ''}`);
273
+ else if (linkProblems.length) record(id, name, level, 'warn', `${linkProblems.join('; ')}${cached(llms)}`);
274
+ else pass(id, name, level, `${sizes}${sample.length ? `; ${sample.length} sampled link${sample.length === 1 ? '' : 's'} resolve without a redirect` : ''}`);
276
275
  }
277
276
 
278
277
  // ---- D5 Link headers (recommended; a wrong service-desc is a false declaration) ----
279
278
  {
280
279
  const id = 'D5', name = 'Link headers', level = 'recommended';
281
280
  const link = home.headers.get('link') ?? '';
282
- const rels = new Map();
283
- for (const part of link.split(/,(?=\s*<)/)) {
284
- const href = part.match(/<([^>]+)>/)?.[1];
285
- const relAttr = part.match(/rel="?([^";]+)"?/)?.[1];
286
- if (href && relAttr) for (const r of relAttr.split(/\s+/)) rels.set(r, [...(rels.get(r) ?? []), href]);
287
- }
288
281
  const desc = rels.get('service-desc') ?? [];
289
- const missing = ['service-desc', 'describedby', ...(hasApi ? ['api-catalog'] : [])].filter((r) => !rels.has(r));
282
+ const missing = ['describedby', ...(api.present ? ['service-desc', 'api-catalog'] : [])].filter((r) => !rels.has(r)); // with no API, a service-desc would declare one
290
283
  if (!link) miss(id, name, level, 'no Link header on the homepage');
291
- else if (hasApi && desc.length && !desc.some((h) => /openapi/i.test(h))) wrong(id, name, level, `service-desc points at ${desc.join(', ')}; it must name the OpenAPI document`);
284
+ else if (api.present && desc.length && !desc.some((h) => /openapi/i.test(h))) wrong(id, name, level, `service-desc points at ${desc.join(', ')}; it must name the OpenAPI document`);
292
285
  else if (missing.length) miss(id, name, level, `missing rel ${missing.join(', ')} (has ${[...rels.keys()].join(', ')})`);
293
286
  else pass(id, name, level, `rels: ${[...rels.keys()].join(', ')}`);
294
287
  }
@@ -320,10 +313,18 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
320
313
  const id = 'D8', name = 'Markdown for agents', level = 'next';
321
314
  const varyAccept = (r) => /(^|,)\s*accept\s*(,|$)/i.test(r.headers.get('vary') ?? '');
322
315
  const tokens = homeMd.headers.get('x-markdown-tokens');
316
+ // The homepage's markdown may be llms.txt, so compare a page llms.txt links instead, when it serves markdown.
317
+ const copy = { path: null, gone: [] };
318
+ // The first linked page that serves markdown; pages that don't are fine (start with the homepage and docs).
319
+ for (const { u, r } of llmsAnswers.filter(({ u, r }) => ok(r) && /text\/html/.test(r.type) && new URL(u).pathname !== '/')) {
320
+ const md = await get(u, { headers: { accept: 'text/markdown' } });
321
+ if (ok(md) && /text\/markdown/.test(md.type)) { Object.assign(copy, { path: new URL(u).pathname, gone: missingSections(r.text, md.text) }); break; }
322
+ }
323
323
  if (!/text\/markdown/.test(homeMd.type)) miss(id, name, level, `Accept: text/markdown on / returns ${ct(homeMd)}`);
324
324
  else if (!varyAccept(homeMd) || !varyAccept(home)) miss(id, name, level, `Vary: Accept missing on the ${!varyAccept(home) ? 'HTML' : 'markdown'} response`);
325
+ else if (copy.gone.length) miss(id, name, level, `the markdown copy of ${copy.path} lacks ${copy.gone.length} section${copy.gone.length === 1 ? '' : 's'} the page has: ${list(copy.gone.map((h) => `"${h}"`))}`);
325
326
  else if (!tokens) miss(id, name, level, 'markdown and Vary are right; add x-markdown-tokens');
326
- else pass(id, name, level, `markdown with Vary: Accept on both, x-markdown-tokens ${tokens}`);
327
+ else pass(id, name, level, `markdown with Vary: Accept on both, x-markdown-tokens ${tokens}${copy.path ? `; ${copy.path}'s markdown copy has every section` : '; no page llms.txt links serves markdown, so no copy was compared'}`);
327
328
  }
328
329
 
329
330
  // ---- D9 AI catalog (ARD) ----
@@ -360,8 +361,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
360
361
  }
361
362
 
362
363
  // ---- A: API ----
363
- if (!hasApi) {
364
- for (const [id, name, level] of [['A1', 'OpenAPI 3.1', 'required'], ['A2', 'GET /api index', 'required'], ['A6', 'Errors that teach (401 without a key)', 'required'], ['A7', 'JSON catch-all for wrong API paths', 'required'], ['A3', 'API catalog (RFC 9727)', 'recommended'], ['A4', 'auth.md', 'recommended'], ['A5', 'API reference as markdown', 'recommended']]) skip(id, name, level, '--no-api');
364
+ if (!api.present) {
365
+ for (const [id, name, level] of [['A1', 'OpenAPI 3.1', 'required'], ['A2', 'GET /api index', 'required'], ['A6', 'Errors that teach (401 without a key)', 'required'], ['A7', 'JSON catch-all for wrong API paths', 'required'], ['A3', 'API catalog (RFC 9727)', 'recommended'], ['A4', 'auth.md', 'recommended'], ['A5', 'API reference as markdown', 'recommended']]) skip(id, name, level, 'no public API: /openapi.json and /api don\'t answer, and nothing declares one');
365
366
  } else {
366
367
  {
367
368
  const id = 'A1', name = 'OpenAPI 3.1', level = 'required';
@@ -411,8 +412,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
411
412
  const respGaps = new Set(ops.flatMap(({ o }) => Object.values(o.responses ?? {}).flatMap((r) => Object.values(deref(r)?.content ?? {}).flatMap((c) => undescribedProps(c.schema)))));
412
413
  const respNote = respGaps.size ? `; ${respGaps.size} response field${respGaps.size === 1 ? ' has' : 's have'} no description (see N7)` : '';
413
414
  if (!String(spec.openapi).startsWith('3.1')) miss(id, name, level, `OpenAPI ${spec.openapi} (use 3.1)`);
414
- else if (noDesc.length) miss(id, name, level, `${noDesc.length} of ${ops.length} operations ${noDesc.length === 1 ? 'has' : 'have'} no description (${noDesc.slice(0, 3).map((x) => x.key).join(', ')}${noDesc.length > 3 ? ', ...' : ''})${repeatNote}${respNote}`);
415
- else if (fieldGaps.length) miss(id, name, level, `${fieldGaps.length} request field${fieldGaps.length === 1 ? ' has' : 's have'} no description (${fieldGaps.slice(0, 3).join(', ')}${fieldGaps.length > 3 ? ', ...' : ''})${variantNote}${repeatNote}${respNote}`);
415
+ else if (noDesc.length) miss(id, name, level, `${noDesc.length} of ${ops.length} operations ${noDesc.length === 1 ? 'has' : 'have'} no description (${list(noDesc.map((x) => x.key))})${repeatNote}${respNote}`);
416
+ else if (fieldGaps.length) miss(id, name, level, `${fieldGaps.length} request field${fieldGaps.length === 1 ? ' has' : 's have'} no description (${list(fieldGaps)})${variantNote}${repeatNote}${respNote}`);
416
417
  else pass(id, name, level, `OpenAPI ${spec.openapi}; all ${ops.length} operations and their request fields described${repeatNote}${respNote}`);
417
418
  }
418
419
  }
@@ -491,9 +492,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
491
492
  const badName = read.map((f) => [f, skillNameProblem(f)]).filter(([, why]) => why).map(([f, why]) => `${f.label} (${why})`);
492
493
  const badDesc = read.filter((f) => !f.description || f.description.length > 1024).map((f) => f.label);
493
494
  const noMap = read.filter((f) => !/llms\.txt/.test(f.text)).map((f) => f.label);
494
- const noApi = read.filter((f) => !/openapi|\/docs\/(api|mcp)|https?:\/\/[^\s)]+\/(api|mcp)\b/i.test(f.text)).map((f) => f.label);
495
+ const noApi = read.filter((f) => !API_LINK.test(f.text) && !MCP_LINK.test(f.text)).map((f) => f.label);
495
496
  const long = read.filter((f) => f.lines > 500).map((f) => `${f.label} (${f.lines} lines)`);
496
- const list = (xs) => `${xs.slice(0, 3).join(', ')}${xs.length > 3 ? `, +${xs.length - 3} more` : ''}`;
497
497
  const problems = [], warns = [];
498
498
  if (unreachable.length) problems.push(`skill files that don't load: ${list(unreachable)}`);
499
499
  if (badName.length) problems.push(`names that break the spec (a lowercase slug, matching its folder and its index entry): ${list(badName)}`);
@@ -548,7 +548,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
548
548
  const id = 'M4', name = 'Hosted MCP edge', level = 'required';
549
549
  const variants = [['application/json, text/event-stream', probe]];
550
550
  for (const accept of ['application/json', '*/*']) variants.push([accept, await get(mcpPath, { method: 'POST', headers: { 'content-type': 'application/json', accept }, body: INIT })]);
551
- variants.push(['(none)', await postNoAccept(mcpPath, INIT)]);
551
+ variants.push(['(none)', await get(mcpPath, { method: 'POST', headers: { 'content-type': 'application/json', accept: null }, body: INIT })]);
552
552
  const discover = await modern('server/discover', 2);
553
553
  const browser = await get(mcpPath, { headers: { accept: 'text/html' }, redirect: 'manual' });
554
554
  const aliases = await Promise.all(['/sse', '/api/mcp'].map((p) => get(p, { method: 'POST', redirect: 'manual', headers: { 'content-type': 'application/json' }, body: INIT })));
@@ -610,23 +610,18 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
610
610
  }
611
611
 
612
612
  // ---- W: search, on a sample of sitemap pages ----
613
+ let htmlKb = null; // median HTML size of the sampled pages, reported, not scored
613
614
  {
614
- const locs = (xml) => [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1].replace(/&amp;/g, '&'));
615
- let urls = locs(sitemap.text);
616
- if (/<sitemapindex/.test(sitemap.text)) {
617
- const subs = urls.slice(0, 3);
618
- urls = [];
619
- for (const sub of subs) urls.push(...locs((await get(sub)).text));
620
- }
621
- urls = [...new Set(urls)].filter((u) => { try { return siteHosts.has(new URL(u).hostname); } catch { return false; } });
622
- const norm = (u) => { try { const x = new URL(u); return `${x.protocol}//${x.host.toLowerCase()}${x.pathname.replace(/\/+$/, '') || '/'}${x.search}`; } catch { return u; } };
623
- const pick = [];
624
- const n = Math.min(20, urls.length);
625
- for (let i = 0; i < n; i++) pick.push(urls[Math.floor((i * urls.length) / n)]);
626
- const decode = (t) => t.replace(/&amp;/g, '&').replace(/&#39;|&#x27;|&apos;/g, "'").replace(/&quot;/g, '"').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/\s+/g, ' ').trim();
615
+ const urls = entries.map((e) => e.loc);
616
+ const sample = samplePages(urls);
617
+ const pick = sample.pages;
618
+ const templatePages = new Set(sample.templates.flatMap((t) => t.urls).map(norm));
619
+ const sampled = `${pick.length} sampled pages (${sample.static} static, ${sample.templates.length ? `${sample.templates.length} template${sample.templates.length === 1 ? '' : 's'}` : 'no templates'}${sample.skipped ? `; ${sample.skipped} static pages over the cap not read` : ''})`;
627
620
  const attrs = (tag) => Object.fromEntries([...tag.matchAll(/([\w:-]+)\s*=\s*("([^"]*)"|'([^']*)')/g)].map((m) => [m[1].toLowerCase(), m[3] ?? m[4]]));
628
621
  const tags = (html, name) => [...html.matchAll(new RegExp(`<${name}\\b[^>]*>`, 'gi'))].map((m) => attrs(m[0]));
629
622
  const meta = (html, key, val) => tags(html, 'meta').find((a) => (a[key] ?? '').toLowerCase() === val)?.content;
623
+ const text = (v) => (v === undefined ? undefined : decodeText(v));
624
+ const tw = (html, field) => text(meta(html, 'name', `twitter:${field}`) ?? meta(html, 'property', `twitter:${field}`));
630
625
  const jsonLd = (html) => {
631
626
  const ld = [...html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)].map((m) => { try { return JSON.parse(m[1]); } catch { return null; } });
632
627
  const types = [];
@@ -634,27 +629,29 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
634
629
  ld.forEach(walk);
635
630
  return { ld, types };
636
631
  };
637
- const pages = await mapLimit(pick, 4, async (u) => {
638
- const r = await get(u, { headers: { accept: 'text/html' } });
632
+ const parsePage = (u, r) => {
639
633
  const h = r.text;
640
634
  const { ld, types } = jsonLd(h);
641
635
  return {
642
- url: u, status: r.status, finalUrl: r.url, html: /text\/html/.test(r.type),
643
- title: decode(h.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? ''),
644
- description: decode(meta(h, 'name', 'description') ?? ''),
636
+ url: u, status: r.status, finalUrl: r.url, html: /text\/html/.test(r.type), bytes: Buffer.byteLength(h), error: r.error,
637
+ title: decodeText(h.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? ''),
638
+ description: decodeText(meta(h, 'name', 'description') ?? ''),
645
639
  robots: (meta(h, 'name', 'robots') ?? '').toLowerCase(),
646
640
  canonicals: tags(h, 'link').filter((a) => (a.rel ?? '').toLowerCase() === 'canonical').map((a) => a.href),
647
- og: { title: meta(h, 'property', 'og:title'), description: meta(h, 'property', 'og:description'), image: meta(h, 'property', 'og:image'), width: meta(h, 'property', 'og:image:width'), alt: meta(h, 'property', 'og:image:alt') },
641
+ og: { title: text(meta(h, 'property', 'og:title')), description: text(meta(h, 'property', 'og:description')), image: meta(h, 'property', 'og:image'), width: meta(h, 'property', 'og:image:width'), alt: meta(h, 'property', 'og:image:alt') },
648
642
  twitterCard: meta(h, 'name', 'twitter:card'),
649
- h1: (h.match(/<h1[\s>]/gi) ?? []).length,
643
+ twitter: { title: tw(h, 'title'), description: tw(h, 'description') },
644
+ headings: headings(h),
650
645
  lang: /<html[^>]*\slang=["']?[\w-]+/i.test(h),
651
646
  imgNoAlt: tags(h, 'img').filter((a) => a.alt === undefined).length,
652
647
  ldBroken: ld.filter((x) => x === null).length, types,
653
648
  };
654
- });
655
- const ok200 = pages.filter((p) => p.status === 200 && p.html);
649
+ };
650
+ const pages = await mapLimit(pick, 4, async (u) => parsePage(u, await get(u, { headers: { accept: 'text/html' } })));
651
+ // A body the audit refused (an encoding it doesn't decode) is reported once, in W3, not as missing tags everywhere.
652
+ const ok200 = pages.filter((p) => p.status === 200 && p.html && !p.error);
653
+ if (ok200.length) { const sizes = ok200.map((p) => p.bytes).sort((a, b) => a - b); htmlKb = Math.round(sizes[Math.floor(sizes.length / 2)] / 1024); }
656
654
  const short = (u) => u.replace(BASE, '') || '/';
657
- const list = (xs) => `${xs.slice(0, 3).join(', ')}${xs.length > 3 ? `, +${xs.length - 3} more` : ''}`;
658
655
  if (!pick.length) {
659
656
  for (const [id, name] of [['W1', 'Titles and descriptions'], ['W2', 'Canonical URLs'], ['W3', 'Indexable pages only'], ['W4', 'Structured data'], ['W5', 'Link previews'], ['W6', 'Page structure']]) miss(id, name, 'required', 'no URLs in the sitemap to sample');
660
657
  } else {
@@ -677,7 +674,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
677
674
  if (badDescs.length) lengths.push(`descriptions outside 50 to 160 characters: ${list(badDescs)}`);
678
675
  if (problems.length) miss(id, name, level, problems.concat(lengths).join('; '));
679
676
  else if (lengths.length) record(id, name, level, 'warn', lengths.join('; '));
680
- else pass(id, name, level, `${ok200.length} sampled pages, all with their own title and description`);
677
+ else pass(id, name, level, `${sampled}, all with their own title and description`);
681
678
  }
682
679
  // W2 canonical URLs and one canonical host
683
680
  {
@@ -691,9 +688,8 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
691
688
  let url = start;
692
689
  for (let n = 0; n < 5; n++) {
693
690
  if (new URL(url).origin === origin) return { n, end: url };
694
- if (!(await isSafe(url))) return { n, end: url };
695
- let r;
696
- try { r = await fetch(url, { method: 'GET', redirect: 'manual', headers: { 'user-agent': UA }, signal: AbortSignal.timeout(15000) }); } catch { return n === 0 ? null : { n, end: url }; }
691
+ const r = await get(url, { redirect: 'manual' }); // get() refuses an address that isn't public
692
+ if (!r.status) return n === 0 ? null : { n, end: url };
697
693
  const loc = r.headers.get('location');
698
694
  if (r.status < 300 || r.status >= 400 || !loc) return { n, end: url };
699
695
  url = new URL(loc, url).href;
@@ -714,7 +710,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
714
710
  if (elsewhere.length) problems.push(`sitemap URLs canonical to another URL: ${list(elsewhere)}`);
715
711
  problems.push(...hostProblems);
716
712
  if (problems.length) miss(id, name, level, problems.join('; '));
717
- else pass(id, name, level, `self-referencing canonicals on ${ok200.length} sampled pages; ${baseOrigin ? 'host redirects not checked with --base' : `http and the other host redirect once to ${origin}`}`);
713
+ else pass(id, name, level, `self-referencing canonicals on ${sampled}; ${baseOrigin ? 'host redirects not checked with --base' : `http and the other host redirect once to ${origin}`}`);
718
714
  }
719
715
  // W3 only indexable pages in the sitemap; real 404s
720
716
  {
@@ -722,8 +718,10 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
722
718
  const broken = pages.filter((p) => p.status !== 200).map((p) => `${short(p.url)} (${p.status})`);
723
719
  const moved = pages.filter((p) => p.status === 200 && norm(p.finalUrl) !== norm(p.url)).map((p) => `${short(p.url)} → ${short(p.finalUrl)}`);
724
720
  const noindex = ok200.filter((p) => /noindex/.test(p.robots)).map((p) => short(p.url));
725
- const deep = ok200.map((p) => new URL(p.url).pathname).find((path) => path.split('/').filter(Boolean).length >= 2);
726
- const probes = [`/no-such-page-${rand}`, ...(deep ? [deep.replace(/\/[^/]+\/?$/, `/no-such-item-${rand}`)] : [])];
721
+ // A missing item under each template: a detail route is where a loading boundary or a fallback answers 200.
722
+ // Templates first, then any deeper sampled page, so a small site's /blog/[post] gets a probe too.
723
+ const parents = [...sample.templates.map((t) => t.urls[0]), ...pick].map((u) => new URL(u).pathname).filter((path) => pathDepth(path) >= 2).map((path) => path.replace(/\/[^/]+\/?$/, ''));
724
+ const probes = [`/no-such-page-${rand}`, ...[...new Set(parents)].slice(0, 3).map((parent) => `${parent}/no-such-item-${rand}`)];
727
725
  const soft = [];
728
726
  for (const path of probes) {
729
727
  const r = await get(path, { headers: { accept: 'text/html' } });
@@ -731,11 +729,13 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
731
729
  }
732
730
  const problems = [];
733
731
  if (broken.length) problems.push(`sitemap URLs that don't answer 200: ${list(broken)}`);
732
+ const unread = pages.filter((p) => p.status === 200 && p.error).map((p) => `${short(p.url)} (${p.error})`);
733
+ if (unread.length) problems.push(`pages the audit couldn't read: ${list(unread)}; send gzip or no encoding unless the request asks for another`);
734
734
  if (moved.length) problems.push(`sitemap URLs that redirect: ${list(moved)}`);
735
735
  if (noindex.length) problems.push(`sitemap URLs marked noindex: ${list(noindex)}`);
736
736
  if (soft.length) problems.push(`missing pages answer 200 instead of 404: ${list(soft)}`);
737
737
  if (problems.length) miss(id, name, level, problems.join('; '));
738
- else pass(id, name, level, `${pages.length} sampled sitemap URLs answer 200 with no redirect or noindex; missing pages get a 404`);
738
+ else pass(id, name, level, `${sampled} answer 200 with no redirect or noindex; missing pages get a 404`);
739
739
  }
740
740
  // W4 structured data
741
741
  {
@@ -743,7 +743,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
743
743
  const homeTypes = home.status === 200 ? jsonLd(home.text).types : null;
744
744
  const broken = ok200.filter((p) => p.ldBroken).map((p) => short(p.url));
745
745
  const none = ok200.filter((p) => !p.types.length).map((p) => short(p.url));
746
- const deepPages = ok200.filter((p) => new URL(p.url).pathname.split('/').filter(Boolean).length >= 2);
746
+ const deepPages = ok200.filter((p) => templatePages.has(norm(p.url)) && pathDepth(p.url) >= 2);
747
747
  const noCrumbs = deepPages.filter((p) => !p.types.includes('BreadcrumbList')).map((p) => short(p.url));
748
748
  const all = new Set(ok200.flatMap((p) => p.types));
749
749
  const notes = [];
@@ -754,7 +754,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
754
754
  if (homeTypes && !homeTypes.some((t) => ['Organization', 'WebSite'].includes(t))) problems.push('no Organization or WebSite on the homepage');
755
755
  if (none.length) problems.push(`no JSON-LD on ${list(none)}`);
756
756
  if (problems.length) miss(id, name, level, problems.concat(notes).join('; '));
757
- else if (noCrumbs.length > deepPages.length / 2) record(id, name, level, 'warn', [`no BreadcrumbList on ${noCrumbs.length} of ${deepPages.length} deeper pages`, ...notes].join('; '));
757
+ else if (noCrumbs.length > deepPages.length / 2) record(id, name, level, 'warn', [`no BreadcrumbList on ${noCrumbs.length} of ${deepPages.length} sampled template pages`, ...notes].join('; '));
758
758
  else pass(id, name, level, [`types: ${[...all].slice(0, 8).join(', ')}`, ...notes].join('; '));
759
759
  }
760
760
  // W5 link previews
@@ -773,25 +773,37 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
773
773
  if (missingOg.length) problems.push(`missing og:title, og:description or og:image on ${list(missingOg)}`);
774
774
  if (noCard.length) problems.push(`no twitter:card on ${list(noCard)}`);
775
775
  if (badImages.length) problems.push(`share images that don't load: ${list(badImages)}`);
776
- if (problems.length) miss(id, name, level, problems.join('; '));
777
- else if (undeclared.length) record(id, name, level, 'warn', `og:image size or alt text not declared on ${list(undeclared)}`);
778
- else pass(id, name, level, `Open Graph and twitter:card on ${ok200.length} sampled pages; share images load`);
776
+ // The homepage joins the comparison even when the sample missed it: a layout's default card is its text.
777
+ const cardPages = ok200.map((p) => ({ ...p, path: short(p.url) }));
778
+ if (home.status === 200 && !cardPages.some((p) => p.path === '/')) cardPages.push({ ...parsePage(BASE, home), path: '/' });
779
+ const shared = sharedCardText(cardPages);
780
+ const clip = (t) => (t.length > 50 ? `${t.slice(0, 50)}…` : t);
781
+ if (shared.x.paths.length) problems.push(`X shows another page's title or description on ${list(shared.x.paths)} ("${clip(shared.x.text)}"; twitter:title and twitter:description, else og:)`);
782
+ if (shared.og.paths.length) problems.push(`Facebook and LinkedIn show another page's title or description on ${list(shared.og.paths)} ("${clip(shared.og.text)}")`);
783
+ const warns = undeclared.length ? [`og:image size or alt text not declared on ${list(undeclared)}`] : [];
784
+ if (problems.length) miss(id, name, level, problems.concat(warns).join('; '));
785
+ else if (warns.length) record(id, name, level, 'warn', warns.join('; '));
786
+ else pass(id, name, level, `Open Graph and twitter:card on ${sampled}; share images load`);
779
787
  }
780
788
  // W6 page structure
781
789
  {
782
790
  const id = 'W6', name = 'Page structure', level = 'required';
783
- const noH1 = ok200.filter((p) => p.h1 === 0).map((p) => short(p.url));
784
- const manyH1 = ok200.filter((p) => p.h1 > 1).map((p) => `${short(p.url)} (${p.h1})`);
791
+ const h1s = (p) => p.headings.filter((x) => x.level === 1).length;
792
+ const noH1 = ok200.filter((p) => h1s(p) === 0).map((p) => short(p.url));
793
+ const manyH1 = ok200.filter((p) => h1s(p) > 1).map((p) => `${short(p.url)} (${h1s(p)})`);
785
794
  const noLang = ok200.filter((p) => !p.lang).map((p) => short(p.url));
786
795
  const noAlt = ok200.filter((p) => p.imgNoAlt).map((p) => `${short(p.url)} (${p.imgNoAlt})`);
796
+ const skipped = ok200.map((p) => [p, headingSkips(p.headings)]).filter(([, sk]) => sk.length).map(([p, sk]) => `${short(p.url)} (${sk[0]})`);
787
797
  const problems = [], warns = [];
788
798
  if (noH1.length) problems.push(`no h1 on ${list(noH1)}`);
789
799
  if (noLang.length) problems.push(`no html lang on ${list(noLang)}`);
790
800
  if (manyH1.length) warns.push(`several h1s on ${list(manyH1)}`);
791
801
  if (noAlt.length) warns.push(`images without alt on ${list(noAlt)}`);
792
- if (problems.length) miss(id, name, level, problems.concat(warns).join('; '));
793
- else if (warns.length) record(id, name, level, 'warn', warns.join('; '));
794
- else pass(id, name, level, `one h1, html lang and alt text on ${ok200.length} sampled pages`);
802
+ if (skipped.length) warns.push(`heading levels skipped on ${list(skipped)}`);
803
+ const weight = `median HTML ${htmlKb} KB`;
804
+ if (problems.length) miss(id, name, level, problems.concat(warns, weight).join('; '));
805
+ else if (warns.length) record(id, name, level, 'warn', warns.concat(weight).join('; '));
806
+ else pass(id, name, level, `one h1, heading levels in order, html lang and alt text on ${sampled}; ${weight}`);
795
807
  }
796
808
  }
797
809
  }
@@ -847,7 +859,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp', base = nu
847
859
  }
848
860
 
849
861
  results.sort((a, b) => LEVELS.indexOf(a.level) - LEVELS.indexOf(b.level) || ORDER.indexOf(a.id) - ORDER.indexOf(b.id));
850
- return { domain, base: baseOrigin, version: VERSION, checked_at: new Date().toISOString(), hosted_mcp: hosted, results };
862
+ return { domain, base: baseOrigin, version: VERSION, checked_at: new Date().toISOString(), hosted_mcp: hosted, has_api: api.present, html_kb_median: htmlKb, results };
851
863
  }
852
864
 
853
865
  export function score(results, level) {
@@ -872,6 +884,7 @@ export function matrix(reports, command, recorded = {}) {
872
884
  `| | **Required passed** | | ${reports.map((rep) => `**${score(rep.results, 'required')}**`).join(' | ')} |`,
873
885
  `| | Recommended passed | | ${reports.map((rep) => score(rep.results, 'recommended')).join(' | ')} |`,
874
886
  `| | Next level passed | | ${reports.map((rep) => score(rep.results, 'next')).join(' | ')} |`,
887
+ `| | Public API | | ${reports.map((rep) => (rep.has_api ? 'yes' : 'no')).join(' | ')} |`,
875
888
  `| | Hosted MCP endpoint | | ${reports.map((rep) => (rep.hosted_mcp ? 'yes' : 'no')).join(' | ')} |`,
876
889
  ...[['T5', 'Agent usability test (recorded)'], ['T6', 'Search numbers (recorded)']].map(([id, label]) =>
877
890
  `| ${id} | ${label} | required | ${reports.map((rep) => formatRecorded(recorded[rep.domain]?.[id])).join(' | ')} |`),