readiness-audit 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/STANDARD.md +2 -0
- package/package.json +1 -1
- package/src/audit.mjs +36 -30
- package/src/helpers.mjs +47 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.0.1 (2026-10-05)
|
|
4
|
+
|
|
5
|
+
Fixes from the first full switch-over (animalhouse.ai, 10 to 19 of 19 required).
|
|
6
|
+
|
|
7
|
+
- **S1 no longer fails skill URLs that don't end in `<name>/SKILL.md`.** It read the folder from the URL's second-to-last segment, so an index pointing at `/skills/<name>` failed every name ("folder is skills"). A name is now checked against its folder only when the URL shows one, and always against its index entry's name.
|
|
8
|
+
- **S1 samples only SKILL.md entries.** It read `archive` entries as text and failed their names and descriptions.
|
|
9
|
+
- **S1 resolves relative skill URLs against the index**, not the site root (RFC 3986).
|
|
10
|
+
- **A CDN-cached copy is named.** When robots.txt, the sitemap, llms.txt or security.txt fails and came from a CDN cache, the reason says how old the copy is, since a recent change may not show yet. Cloudflare caches robots.txt by default.
|
|
11
|
+
- **STANDARD.md:** D1 says to purge robots.txt from the CDN after a change. D8 notes that Next.js 14.2 replaces `Vary` on App Router pages, so `Accept` goes on at the edge.
|
|
12
|
+
|
|
3
13
|
## 1.0.0 (2026-10-05)
|
|
4
14
|
|
|
5
15
|
First public release.
|
package/STANDARD.md
CHANGED
|
@@ -121,6 +121,7 @@ Each item says what it is, why it's at its level, and how you know it's done. Se
|
|
|
121
121
|
- **Keep the rules the same for every crawler.** A crawler obeys only the group that names it, and falls back to `*` only when no group does (RFC 9309 §2.2.1). A `User-agent: GPTBot` group that lacks a `Disallow` from the `*` group lets that bot crawl what you meant to block. One group listing `*` and every named crawler is simplest (achurch.ai). Repeating the full rules in every group also works (botsmatter.live).
|
|
122
122
|
- **Disallow what has side effects or private data.** For read-only API endpoints that live-fetch agents need (public stats, a self-documenting `GET /api/auth/register`), allow the fetch and send `X-Robots-Tag: noindex` on the JSON instead, so search engines skip it without blocking agents (botsmatter, inbed).
|
|
123
123
|
- **Allow your share-card path** (`/api/og/` at animalhouse, `/og/` at botsmatter), and add the `Sitemap:` line.
|
|
124
|
+
- **Purge it from your CDN after a change.** Cloudflare caches robots.txt by default, so crawlers keep the old rules until the cached copy expires. The scorecard says when it read a cached copy.
|
|
124
125
|
- Naming AI crawlers and stating a Content-Signal policy is D15 (recommended).
|
|
125
126
|
- **Done when** the scorecard passes D1.
|
|
126
127
|
|
|
@@ -187,6 +188,7 @@ Each item says what it is, why it's at its level, and how you know it's done. Se
|
|
|
187
188
|
**D8 Markdown for agents**
|
|
188
189
|
- `Accept: text/markdown` and a `.md` URL both return the page as markdown.
|
|
189
190
|
- Send `Vary: Accept` on both representations (RFC 9110 §12.5.5). Otherwise a cache can hand HTML readers the markdown, or the reverse.
|
|
191
|
+
- Next.js 14.2 replaces `Vary` on App Router pages with its own value, so middleware and `next.config` headers can't add `Accept` to the HTML. Add it at the edge (a CDN response header rule), and send it from the markdown route yourself (animalhouse.ai).
|
|
190
192
|
- Make links absolute, and link back to the HTML page as canonical.
|
|
191
193
|
- Add `x-markdown-tokens` so an agent can budget before reading. It's Cloudflare's header, with no spec behind it.
|
|
192
194
|
- Negotiate in front of prerendered pages too, or they ignore it (magnifica).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "readiness-audit",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "Score a live site against the Agent and Search Readiness Standard: discovery files, search, API, MCP and skills, for AI agents and search engines.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/audit.mjs
CHANGED
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
import { lookup, resolveTxt } from 'node:dns/promises';
|
|
10
10
|
import { request as httpsRequest } from 'node:https';
|
|
11
11
|
import { isIP } from 'node:net';
|
|
12
|
+
import { SKILL_NAME, sampleSkillEntries, skillNameProblem, cdnCacheNote } from './helpers.mjs';
|
|
12
13
|
|
|
13
|
-
export const VERSION = '1.0.
|
|
14
|
+
export const VERSION = '1.0.1';
|
|
14
15
|
export const REPO = 'https://github.com/geeks-accelerator/agent-and-search-readiness';
|
|
15
16
|
const UA = `readiness-audit/${VERSION} (+${REPO})`;
|
|
16
17
|
const AI_BOTS = ['gptbot', 'oai-searchbot', 'chatgpt-user', 'claudebot', 'claude-user', 'claude-searchbot', 'perplexitybot', 'perplexity-user', 'google-extended', 'applebot-extended'];
|
|
@@ -136,6 +137,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
136
137
|
const json = (r) => { try { return JSON.parse(r.text); } catch { return null; } };
|
|
137
138
|
const isJson = (r) => /json/.test(r.type) && json(r) !== null;
|
|
138
139
|
const ok = (r) => r.status >= 200 && r.status < 300;
|
|
140
|
+
const cached = (r) => cdnCacheNote(r.headers);
|
|
139
141
|
const ct = (r) => r.type.split(';')[0] || (r.status ? 'no body' : 'no answer');
|
|
140
142
|
const rpc = (r) => {
|
|
141
143
|
if (/event-stream/.test(r.type)) {
|
|
@@ -209,9 +211,9 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
209
211
|
const id = 'D1', name = 'robots.txt', level = 'required';
|
|
210
212
|
const star = groups.find((g) => g.agents.includes('*'));
|
|
211
213
|
const leaks = star ? groups.filter((g) => g !== star).flatMap((g) => [...star.disallow].filter((d) => !g.disallow.has(d)).map((d) => `${g.agents[0]} skips Disallow ${d}`)) : [];
|
|
212
|
-
if (!ok(robots)) miss(id, name, level, `/robots.txt answered ${robots.status}`);
|
|
213
|
-
else if (leaks.length) miss(id, name, level, `named groups drop rules from the * group (a crawler obeys only its own group): ${[...new Set(leaks)].slice(0, 3).join('; ')}`);
|
|
214
|
-
else if (!sitemapLine) miss(id, name, level,
|
|
214
|
+
if (!ok(robots)) miss(id, name, level, `/robots.txt answered ${robots.status}${cached(robots)}`);
|
|
215
|
+
else if (leaks.length) miss(id, name, level, `named groups drop rules from the * group (a crawler obeys only its own group): ${[...new Set(leaks)].slice(0, 3).join('; ')}${cached(robots)}`);
|
|
216
|
+
else if (!sitemapLine) miss(id, name, level, `no Sitemap line${cached(robots)}`);
|
|
215
217
|
else pass(id, name, level, `${groups.length} group${groups.length === 1 ? '' : 's'} with consistent rules, Sitemap listed`);
|
|
216
218
|
}
|
|
217
219
|
{
|
|
@@ -222,7 +224,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
222
224
|
if (unsignaled.length) problems.push(`${unsignaled.length} of ${groups.length} group${groups.length === 1 ? '' : 's'} ${unsignaled.length === 1 ? 'has' : 'have'} no Content-Signal (${unsignaled.slice(0, 3).map((g) => g.agents[0]).join(', ')}${unsignaled.length > 3 ? ', ...' : ''})`);
|
|
223
225
|
if (named.length < 3) problems.push(`names ${named.length} of the main AI crawlers`);
|
|
224
226
|
if (!ok(robots)) miss(id, name, level, 'no robots.txt');
|
|
225
|
-
else if (problems.length) miss(id, name, level, problems.join('; '));
|
|
227
|
+
else if (problems.length) miss(id, name, level, problems.join('; ') + cached(robots));
|
|
226
228
|
else pass(id, name, level, `${named.length} AI crawlers named, Content-Signal in every group`);
|
|
227
229
|
}
|
|
228
230
|
|
|
@@ -230,17 +232,17 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
230
232
|
{
|
|
231
233
|
const id = 'D2', name = 'sitemap.xml', level = 'required';
|
|
232
234
|
const lastmods = [...sitemap.text.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map((m) => m[1].slice(0, 10));
|
|
233
|
-
if (!ok(sitemap) || !/<urlset|<sitemapindex/.test(sitemap.text)) miss(id, name, level, `/sitemap.xml answered ${sitemap.status}`);
|
|
234
|
-
else if (!lastmods.length) miss(id, name, level,
|
|
235
|
-
else if (new Set(lastmods).size === 1 && lastmods.length > 5) miss(id, name, level, `all ${lastmods.length} lastmod dates are ${lastmods[0]}; use each page's real change date`);
|
|
235
|
+
if (!ok(sitemap) || !/<urlset|<sitemapindex/.test(sitemap.text)) miss(id, name, level, `/sitemap.xml answered ${sitemap.status}${cached(sitemap)}`);
|
|
236
|
+
else if (!lastmods.length) miss(id, name, level, `no lastmod dates${cached(sitemap)}`);
|
|
237
|
+
else if (new Set(lastmods).size === 1 && lastmods.length > 5) miss(id, name, level, `all ${lastmods.length} lastmod dates are ${lastmods[0]}; use each page's real change date${cached(sitemap)}`);
|
|
236
238
|
else pass(id, name, level, `${lastmods.length} URLs with ${new Set(lastmods).size} distinct lastmod dates`);
|
|
237
239
|
}
|
|
238
240
|
|
|
239
241
|
// ---- D4 llms.txt ----
|
|
240
242
|
{
|
|
241
243
|
const id = 'D4', name = 'llms.txt and llms-full.txt', level = 'required';
|
|
242
|
-
if (!ok(llms) || !/^\s*# \S/.test(llms.text)) miss(id, name, level, ok(llms) ? 'llms.txt has no H1 title on its first line' : `/llms.txt answered ${llms.status}`);
|
|
243
|
-
else if (!ok(llmsFull)) miss(id, name, level,
|
|
244
|
+
if (!ok(llms) || !/^\s*# \S/.test(llms.text)) miss(id, name, level, (ok(llms) ? 'llms.txt has no H1 title on its first line' : `/llms.txt answered ${llms.status}`) + cached(llms));
|
|
245
|
+
else if (!ok(llmsFull)) miss(id, name, level, `/llms-full.txt missing${cached(llmsFull)}`);
|
|
244
246
|
else pass(id, name, level, `llms.txt ${(llms.text.length / 1024).toFixed(1)} KB, llms-full.txt ${(llmsFull.text.length / 1024).toFixed(0)} KB`);
|
|
245
247
|
}
|
|
246
248
|
|
|
@@ -267,10 +269,10 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
267
269
|
const id = 'D6', name = 'security.txt', level = 'required';
|
|
268
270
|
const expires = [...security.text.matchAll(/^expires:\s*(.+)$/gim)].map((m) => Date.parse(m[1].trim()));
|
|
269
271
|
const days = expires.length === 1 ? (expires[0] - Date.now()) / 86_400_000 : NaN;
|
|
270
|
-
if (!ok(security) || !/text\/plain/.test(security.type)) miss(id, name, level, `/.well-known/security.txt answered ${security.status}`);
|
|
271
|
-
else if (!/^contact:\s*\S/im.test(security.text)) miss(id, name, level,
|
|
272
|
-
else if (expires.length !== 1) miss(id, name, level, `${expires.length} Expires fields (RFC 9116 requires exactly one)`);
|
|
273
|
-
else if (!(days > 0)) miss(id, name, level, `Expires passed ${Math.round(-days)} days ago`);
|
|
272
|
+
if (!ok(security) || !/text\/plain/.test(security.type)) miss(id, name, level, `/.well-known/security.txt answered ${security.status}${cached(security)}`);
|
|
273
|
+
else if (!/^contact:\s*\S/im.test(security.text)) miss(id, name, level, `no Contact field${cached(security)}`);
|
|
274
|
+
else if (expires.length !== 1) miss(id, name, level, `${expires.length} Expires fields (RFC 9116 requires exactly one)${cached(security)}`);
|
|
275
|
+
else if (!(days > 0)) miss(id, name, level, `Expires passed ${Math.round(-days)} days ago${cached(security)}`);
|
|
274
276
|
else pass(id, name, level, `Contact present, Expires in ${Math.round(days)} days${days > 366 ? ' (the RFC recommends under a year)' : ''}`);
|
|
275
277
|
}
|
|
276
278
|
|
|
@@ -425,30 +427,34 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
425
427
|
// ---- S1 skill files, on a sample from the skills index ----
|
|
426
428
|
{
|
|
427
429
|
const id = 'S1', name = 'Skill files', level = 'required';
|
|
428
|
-
const
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
430
|
+
const listed = json(skills)?.skills;
|
|
431
|
+
const sample = sampleSkillEntries(listed);
|
|
432
|
+
if (!ok(skills) || !sample.length) {
|
|
433
|
+
skip(id, name, level, Array.isArray(listed) && listed.length
|
|
434
|
+
? 'the skills index lists no SKILL.md files to sample (archives only); check your SKILL.md files in review'
|
|
435
|
+
: 'no skills index to sample; check your SKILL.md files in review');
|
|
436
|
+
} else {
|
|
437
|
+
// Entry URLs may be relative, and resolve against the index (RFC 3986).
|
|
438
|
+
const indexUrl = `${BASE}/.well-known/agent-skills/index.json`;
|
|
432
439
|
const files = await Promise.all(sample.map(async (e) => {
|
|
433
|
-
const url = new URL(e.url,
|
|
440
|
+
const url = new URL(e.url, indexUrl).href;
|
|
434
441
|
const r = await get(url);
|
|
435
442
|
const fm = r.text.match(/^---\r?\n([\s\S]*?)\r?\n---/)?.[1] ?? '';
|
|
436
443
|
const field = (k) => fm.match(new RegExp(`^${k}:\\s*(?:"([^"]*)"|'([^']*)'|(.*))$`, 'm'))?.slice(1).find((v) => v !== undefined)?.trim() ?? '';
|
|
437
|
-
const
|
|
438
|
-
return { url, status: r.status, text: r.text, name: field('name'), description: field('description'),
|
|
444
|
+
const label = typeof e.name === 'string' && e.name ? e.name : url;
|
|
445
|
+
return { url, status: r.status, text: r.text, name: field('name'), description: field('description'), entryName: e.name, label, lines: r.text.split('\n').length };
|
|
439
446
|
}));
|
|
440
|
-
const
|
|
441
|
-
const unreachable = files.filter((f) => f.status !== 200).map((f) => `${label(f)} (${f.status})`);
|
|
447
|
+
const unreachable = files.filter((f) => f.status !== 200).map((f) => `${f.label} (${f.status})`);
|
|
442
448
|
const read = files.filter((f) => f.status === 200);
|
|
443
|
-
const badName = read.
|
|
444
|
-
const badDesc = read.filter((f) => !f.description || f.description.length > 1024).map(label);
|
|
445
|
-
const noMap = read.filter((f) => !/llms\.txt/.test(f.text)).map(label);
|
|
446
|
-
const noApi = read.filter((f) => !/openapi|\/docs\/(api|mcp)|https?:\/\/[^\s)]+\/(api|mcp)\b/i.test(f.text)).map(label);
|
|
447
|
-
const long = read.filter((f) => f.lines > 500).map((f) => `${label
|
|
449
|
+
const badName = read.map((f) => [f, skillNameProblem(f)]).filter(([, why]) => why).map(([f, why]) => `${f.label} (${why})`);
|
|
450
|
+
const badDesc = read.filter((f) => !f.description || f.description.length > 1024).map((f) => f.label);
|
|
451
|
+
const noMap = read.filter((f) => !/llms\.txt/.test(f.text)).map((f) => f.label);
|
|
452
|
+
const noApi = read.filter((f) => !/openapi|\/docs\/(api|mcp)|https?:\/\/[^\s)]+\/(api|mcp)\b/i.test(f.text)).map((f) => f.label);
|
|
453
|
+
const long = read.filter((f) => f.lines > 500).map((f) => `${f.label} (${f.lines} lines)`);
|
|
448
454
|
const list = (xs) => `${xs.slice(0, 3).join(', ')}${xs.length > 3 ? `, +${xs.length - 3} more` : ''}`;
|
|
449
455
|
const problems = [], warns = [];
|
|
450
456
|
if (unreachable.length) problems.push(`skill files that don't load: ${list(unreachable)}`);
|
|
451
|
-
if (badName.length) problems.push(`names that
|
|
457
|
+
if (badName.length) problems.push(`names that break the spec (a lowercase slug, matching its folder and its index entry): ${list(badName)}`);
|
|
452
458
|
if (badDesc.length) problems.push(`missing or over-long descriptions: ${list(badDesc)}`);
|
|
453
459
|
if (noMap.length) warns.push(`no link to llms.txt: ${list(noMap)}`);
|
|
454
460
|
if (noApi.length) warns.push(`no link to the API or MCP reference: ${list(noApi)}`);
|
|
@@ -470,7 +476,7 @@ export async function audit(domain, { hasApi = true, mcpPath = '/mcp' } = {}) {
|
|
|
470
476
|
if (doc.$schema !== SKILLS_SCHEMA) bad.push(`unrecognized $schema, so clients skip the whole index (${doc.$schema ?? 'none'})`);
|
|
471
477
|
const types = list.filter((s) => !['skill-md', 'archive'].includes(s.type)).length;
|
|
472
478
|
const digests = list.filter((s) => !/^sha256:[0-9a-f]{64}$/.test(s.digest ?? '')).length;
|
|
473
|
-
const names = list.filter((s) =>
|
|
479
|
+
const names = list.filter((s) => !SKILL_NAME.test(s.name ?? '')).length;
|
|
474
480
|
if (types) bad.push(`${types} entries with a type other than skill-md or archive`);
|
|
475
481
|
if (digests) bad.push(`${digests} entries without a sha256: digest`);
|
|
476
482
|
if (names) bad.push(`${names} names that aren't lowercase slugs`);
|
package/src/helpers.mjs
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Pure helpers for the checks in audit.mjs, kept apart so the tests can run
|
|
2
|
+
// them without a network. Internal: the package exports only audit.mjs.
|
|
3
|
+
|
|
4
|
+
/** The Agent Skills spec's name rule: lowercase letters, digits and single hyphens, 1 to 64 characters. */
|
|
5
|
+
export const SKILL_NAME = /^(?!-)(?!.*--)[a-z0-9-]{1,64}(?<!-)$/;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Up to five SKILL.md entries from a skills index, spread across it. Archive
|
|
9
|
+
* entries are skipped: the scorecard reads SKILL.md text, not zip bytes. An
|
|
10
|
+
* entry without a type counts when its URL ends in .md (older indexes).
|
|
11
|
+
*/
|
|
12
|
+
export function sampleSkillEntries(list) {
|
|
13
|
+
const md = (Array.isArray(list) ? list : []).filter((e) => typeof e?.url === 'string'
|
|
14
|
+
&& (e.type === 'skill-md' || (e.type === undefined && /\.md$/i.test(e.url.split(/[?#]/)[0]))));
|
|
15
|
+
return md.length <= 5 ? md : [0, 1, 2, 3, 4].map((i) => md[Math.floor((i * md.length) / 5)]);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const decode = (s) => { try { return decodeURIComponent(s); } catch { return s; } };
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Why a SKILL.md's frontmatter name breaks the Agent Skills spec, or null.
|
|
22
|
+
* The name must be a lowercase slug, match its folder when the URL shows one
|
|
23
|
+
* (.../<name>/SKILL.md), and match the name its index entry gives. A URL in
|
|
24
|
+
* another shape (/skills/<name>) says nothing about the folder.
|
|
25
|
+
*/
|
|
26
|
+
export function skillNameProblem({ name, url, entryName }) {
|
|
27
|
+
if (!SKILL_NAME.test(name ?? '')) return 'not a lowercase slug';
|
|
28
|
+
const path = new URL(url).pathname;
|
|
29
|
+
const folder = /\/SKILL\.md$/i.test(path) ? decode(path.split('/').slice(-2, -1)[0] ?? '') : '';
|
|
30
|
+
if (folder && folder !== name) return `folder is ${folder}`;
|
|
31
|
+
if (typeof entryName === 'string' && entryName && entryName !== name) return `index says ${entryName}`;
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* A note for checks that read a file a CDN may serve from its cache
|
|
37
|
+
* (Cloudflare caches robots.txt by default): that copy can predate the latest
|
|
38
|
+
* deploy. Empty unless the response says it came from a cache.
|
|
39
|
+
*/
|
|
40
|
+
export function cdnCacheNote(headers) {
|
|
41
|
+
const cf = headers.get('cf-cache-status');
|
|
42
|
+
const hit = cf ? /^(hit|stale|updating)$/i.test(cf.trim()) : /\bhit\b/i.test(headers.get('x-cache') ?? '');
|
|
43
|
+
const age = Number(headers.get('age') ?? 0);
|
|
44
|
+
if (!hit || !(age > 0)) return '';
|
|
45
|
+
const old = age < 120 ? `${age} seconds` : `${Math.round(age / 60)} minutes`;
|
|
46
|
+
return ` (this copy came from a CDN cache and is ${old} old, so a recent change may not show yet)`;
|
|
47
|
+
}
|