turva-llms-txt-validator 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD-PARTY-NOTICES.md +40 -0
- package/bin/cli.mjs +31 -7
- package/package.json +4 -3
- package/src/index.mjs +168 -51
package/README.md
CHANGED
|
@@ -8,7 +8,7 @@ Check a website's `llms.txt` from the command line, Node or CI. Get a clear resu
|
|
|
8
8
|
|
|
9
9
|
## Quick start
|
|
10
10
|
|
|
11
|
-
Requires **Node.js
|
|
11
|
+
Requires **Node.js 22 or newer**. Run it without a global install:
|
|
12
12
|
|
|
13
13
|
```sh
|
|
14
14
|
npx --yes turva-llms-txt-validator example.com
|
|
@@ -54,7 +54,7 @@ Use JSON for automation and strict mode when warnings should fail a CI step:
|
|
|
54
54
|
npx --yes turva-llms-txt-validator example.com --json --strict
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
-
Completed validation returns `{ target, summary, checks }`. Each check contains `{ id, status, label, detail }`. With `--json`, input and fetch errors return `{ "error": "..." }` and exit with code `2`. An unknown option, a second domain and a missing domain are input errors too
|
|
57
|
+
Completed validation returns `{ target, summary, checks }`. Each check contains `{ id, status, label, detail }`. With `--json`, input and fetch errors return `{ "error": "...", "target": "..." }` and exit with code `2`. The target is the address given on the command line, with any user name, password and query value masked, and a network error adds `code`, such as `EAI_AGAIN`. An unknown option, a second domain and a missing domain are input errors too. `--help` exits with code `0`, and `--version` or `-V` prints the installed version and exits with code `0`.
|
|
58
58
|
|
|
59
59
|
## What it checks
|
|
60
60
|
|
|
@@ -110,7 +110,7 @@ for (const check of result.checks) {
|
|
|
110
110
|
|
|
111
111
|
## Use in CI
|
|
112
112
|
|
|
113
|
-
Add this step to a GitHub Actions job that already has Node.js
|
|
113
|
+
Add this step to a GitHub Actions job that already has Node.js 22 or newer available. Replace `your-domain.com` with your domain:
|
|
114
114
|
|
|
115
115
|
```yaml
|
|
116
116
|
- name: Validate llms.txt
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Third-party notices
|
|
2
|
+
|
|
3
|
+
This package's own license is MIT, in `LICENSE`. The named character reference
|
|
4
|
+
table in `src/index.mjs` (`NAMED_REFERENCE_DATA`) is generated from data
|
|
5
|
+
published by a third-party package and carries that package's own license
|
|
6
|
+
below. This is a build-time data source, not an installed runtime dependency:
|
|
7
|
+
the table is generated once and copied into `src/index.mjs`, so no reference
|
|
8
|
+
to it appears in `package.json` or in the dependency tree an installer sees.
|
|
9
|
+
|
|
10
|
+
## character-entities
|
|
11
|
+
|
|
12
|
+
- Package: `character-entities`
|
|
13
|
+
- Version used to generate the table: 2.0.2
|
|
14
|
+
- URL: <https://github.com/wooorm/character-entities>
|
|
15
|
+
- License: MIT
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
(The MIT License)
|
|
19
|
+
|
|
20
|
+
Copyright (c) 2015 Titus Wormer <tituswormer@gmail.com>
|
|
21
|
+
|
|
22
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
23
|
+
a copy of this software and associated documentation files (the
|
|
24
|
+
'Software'), to deal in the Software without restriction, including
|
|
25
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
26
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
27
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
28
|
+
the following conditions:
|
|
29
|
+
|
|
30
|
+
The above copyright notice and this permission notice shall be
|
|
31
|
+
included in all copies or substantial portions of the Software.
|
|
32
|
+
|
|
33
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
|
34
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
35
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
36
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
37
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
|
38
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
39
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
40
|
+
```
|
package/bin/cli.mjs
CHANGED
|
@@ -4,15 +4,18 @@
|
|
|
4
4
|
// or bad input. The two v2 discovery checks and the input-path note carry status
|
|
5
5
|
// "info" and move no exit code, by design. Same checks and JSON shape as the hosted validator:
|
|
6
6
|
// curl -H "Accept: application/json" "https://turva.dev/llms-txt-validator?url=example.com"
|
|
7
|
-
import {
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
import { validateHost, stripControls, maskLocation } from "../src/index.mjs";
|
|
8
10
|
|
|
9
11
|
const USAGE = [
|
|
10
12
|
"usage: llms-txt-validate <domain-or-url> [--json] [--strict]",
|
|
11
|
-
" --json
|
|
12
|
-
" --strict
|
|
13
|
-
" --help
|
|
13
|
+
" --json print the result as JSON (same shape as the hosted validator)",
|
|
14
|
+
" --strict exit 1 on warnings too, for CI gates",
|
|
15
|
+
" --help print this help",
|
|
16
|
+
" --version print the installed version and exit"
|
|
14
17
|
];
|
|
15
|
-
const KNOWN = new Set(["--json", "--strict", "--help", "-h"]);
|
|
18
|
+
const KNOWN = new Set(["--json", "--strict", "--help", "-h", "--version", "-V"]);
|
|
16
19
|
|
|
17
20
|
// The arguments are read strictly since 0.3.8 (Tek-496). An unknown flag such as a misspelled
|
|
18
21
|
// --strcit used to be ignored, so a CI step ran without the gate it asked for, and a second
|
|
@@ -36,6 +39,15 @@ if (flags.includes("--help") || flags.includes("-h")) {
|
|
|
36
39
|
for (const line of USAGE) console.log(line);
|
|
37
40
|
process.exit(0);
|
|
38
41
|
}
|
|
42
|
+
if (flags.includes("--version") || flags.includes("-V")) {
|
|
43
|
+
// Read package.json relative to this file, not the caller's cwd, so this works the same
|
|
44
|
+
// whether the CLI runs installed, linked or through npx (V-04, outside audit 2026-09-26).
|
|
45
|
+
// No network read: the version is the one actually installed, not the npm registry's latest.
|
|
46
|
+
const pkgUrl = new URL("../package.json", import.meta.url);
|
|
47
|
+
const pkg = JSON.parse(readFileSync(fileURLToPath(pkgUrl), "utf8"));
|
|
48
|
+
console.log(pkg.version);
|
|
49
|
+
process.exit(0);
|
|
50
|
+
}
|
|
39
51
|
const unknown = flags.find((f) => !KNOWN.has(f));
|
|
40
52
|
if (unknown) fail("unknown option " + JSON.stringify(stripControls(unknown).slice(0, 40)));
|
|
41
53
|
if (targets.length === 0 || targets[0].trim() === "") fail("no domain or URL given");
|
|
@@ -61,7 +73,19 @@ try {
|
|
|
61
73
|
// The hosted validator answers every failure as JSON when JSON was asked for, so a CI
|
|
62
74
|
// step that parses --json output gets {"error": ...} here too instead of an empty stdout
|
|
63
75
|
// and a plain-text stderr (round 16 S3-3). The exit code is unchanged.
|
|
64
|
-
|
|
65
|
-
|
|
76
|
+
//
|
|
77
|
+
// target and code are added as separate fields (V-05, outside audit 2026-09-26): a bare
|
|
78
|
+
// {"error": message} left a CI step unable to tell which target failed or why without
|
|
79
|
+
// parsing free text. code reads a network error's cause first (undici wraps the real
|
|
80
|
+
// errno code, such as EAI_AGAIN or ECONNREFUSED, in err.cause.code) and falls back to
|
|
81
|
+
// err.code for an error that sets it directly; it is omitted, not null, when neither is set.
|
|
82
|
+
// target goes through maskLocation like every other address the tool prints, so a user name,
|
|
83
|
+
// a password or a query value typed into the command line is never echoed back.
|
|
84
|
+
const code = (err && err.cause && err.cause.code) || (err && err.code);
|
|
85
|
+
if (json) {
|
|
86
|
+
const report = { error: message, target: stripControls(maskLocation(targets[0])) };
|
|
87
|
+
if (code) report.code = stripControls(String(code));
|
|
88
|
+
console.log(JSON.stringify(report, null, 2));
|
|
89
|
+
} else console.error("error: " + message);
|
|
66
90
|
process.exit(2);
|
|
67
91
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "turva-llms-txt-validator",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.10",
|
|
4
4
|
"description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks that read pass or info and never move the summary, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.mjs",
|
|
@@ -12,13 +12,14 @@
|
|
|
12
12
|
"src",
|
|
13
13
|
"bin",
|
|
14
14
|
"README.md",
|
|
15
|
-
"LICENSE"
|
|
15
|
+
"LICENSE",
|
|
16
|
+
"THIRD-PARTY-NOTICES.md"
|
|
16
17
|
],
|
|
17
18
|
"scripts": {
|
|
18
19
|
"test": "node --test"
|
|
19
20
|
},
|
|
20
21
|
"engines": {
|
|
21
|
-
"node": ">=
|
|
22
|
+
"node": ">=22"
|
|
22
23
|
},
|
|
23
24
|
"keywords": [
|
|
24
25
|
"llms.txt",
|
package/src/index.mjs
CHANGED
|
@@ -39,7 +39,13 @@ export function cut(s, n) {
|
|
|
39
39
|
export function maskLocation(href, base) {
|
|
40
40
|
let u = null;
|
|
41
41
|
try { u = new URL(href, base); } catch { u = null; }
|
|
42
|
-
|
|
42
|
+
// An opaque URL (no authority, e.g. "u:name@host/") parses without throwing but has no
|
|
43
|
+
// host, so clearing u.username/u.password is a silent no-op: WHATWG has nowhere to put
|
|
44
|
+
// them and the credential-shaped text stays in the opaque path untouched. Route it through
|
|
45
|
+
// the string branch below instead, the same one an unparseable target already uses
|
|
46
|
+
// (outside review 2026-09-28: a bare "u:" scheme let the text after it through as the
|
|
47
|
+
// CLI JSON "target" field, unmasked).
|
|
48
|
+
if (u && u.host) {
|
|
43
49
|
u.username = "";
|
|
44
50
|
u.password = "";
|
|
45
51
|
for (const key of Array.from(u.searchParams.keys())) u.searchParams.set(key, "***");
|
|
@@ -202,20 +208,25 @@ function isEscaped(s, i) {
|
|
|
202
208
|
return n % 2 === 1;
|
|
203
209
|
}
|
|
204
210
|
|
|
205
|
-
// Inline code spans
|
|
206
|
-
// read as the code it is (outside audit 2026-09-26, Tek-496)
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
|
|
212
|
-
|
|
211
|
+
// Inline code spans, marked per character across the whole document, so a link written inside
|
|
212
|
+
// backticks is read as the code it is (outside audit 2026-09-26, Tek-496), including when the
|
|
213
|
+
// span itself crosses a real line ending: a fenceless "`[a](b)\ncontinued`" used to be masked
|
|
214
|
+
// only on its own line, so the second line escaped the mask and its bracket read as a real link
|
|
215
|
+
// (outside review 2026-09-28). A span opens with a run of backticks and closes with the next run
|
|
216
|
+
// of the same length anywhere in the text; a pair whose interior holds a blank line (two line
|
|
217
|
+
// endings in a row) is not a code span in CommonMark, so it is left unmasked and its backticks
|
|
218
|
+
// stay literal. An escaped first backtick is literal and the rest of its run still opens, while a
|
|
219
|
+
// closing run is never escaped, because a backslash inside a code span is literal. Runs are
|
|
220
|
+
// grouped by length and every group is read with one forward pointer, so the scan stays linear on
|
|
221
|
+
// a document of unmatched runs.
|
|
222
|
+
function codeSpanMask(text) {
|
|
223
|
+
const mask = new Uint8Array(text.length);
|
|
213
224
|
const runs = [];
|
|
214
|
-
for (let i = 0; i <
|
|
215
|
-
if (
|
|
225
|
+
for (let i = 0; i < text.length; ) {
|
|
226
|
+
if (text[i] !== "`") { i++; continue; }
|
|
216
227
|
let j = i;
|
|
217
|
-
while (j <
|
|
218
|
-
runs.push({ at: i, len: j - i, escaped: isEscaped(
|
|
228
|
+
while (j < text.length && text[j] === "`") j++;
|
|
229
|
+
runs.push({ at: i, len: j - i, escaped: isEscaped(text, i) });
|
|
219
230
|
i = j;
|
|
220
231
|
}
|
|
221
232
|
const byLen = new Map();
|
|
@@ -231,29 +242,61 @@ function codeSpanMask(line) {
|
|
|
231
242
|
ptr.set(len, p);
|
|
232
243
|
if (p === list.length) continue;
|
|
233
244
|
const close = runs[list[p]];
|
|
234
|
-
|
|
245
|
+
const start = r.escaped ? r.at + 1 : r.at;
|
|
246
|
+
const end = close.at + close.len;
|
|
247
|
+
if (text.slice(start, end).indexOf("\n\n") === -1) mask.fill(1, start, end);
|
|
235
248
|
k = list[p];
|
|
236
249
|
}
|
|
237
250
|
return mask;
|
|
238
251
|
}
|
|
239
252
|
|
|
253
|
+
// A CommonMark link reference label is matched without regard to case or the exact run of
|
|
254
|
+
// whitespace inside it, so "[Guide]" and "[ guide ]" name the same definition.
|
|
255
|
+
function normalizeLinkLabel(s) {
|
|
256
|
+
return s.trim().replace(/\s+/g, " ").toLowerCase();
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Link reference definitions, "[label]: target", read once per document so a reference link
|
|
260
|
+
// such as "[text][label]" can resolve against one written anywhere else in the same file,
|
|
261
|
+
// before or after its use, the way CommonMark allows (outside review 2026-09-28). A title
|
|
262
|
+
// after the target is not read, because nothing here compares it, and only the first
|
|
263
|
+
// definition of a given label counts, per spec. A destination wrapped in angle brackets has
|
|
264
|
+
// them stripped.
|
|
265
|
+
function collectLinkDefinitions(text) {
|
|
266
|
+
const defs = new Map();
|
|
267
|
+
for (const line of text.split("\n")) {
|
|
268
|
+
const m = /^ {0,3}\[([^\]]+)\]:[ \t]*(<[^<>\n]*>|\S+)/.exec(line);
|
|
269
|
+
if (!m) continue;
|
|
270
|
+
const label = normalizeLinkLabel(m[1]);
|
|
271
|
+
if (!label || defs.has(label)) continue;
|
|
272
|
+
let dest = m[2];
|
|
273
|
+
if (dest[0] === "<" && dest[dest.length - 1] === ">") dest = dest.slice(1, -1);
|
|
274
|
+
defs.set(label, dest);
|
|
275
|
+
}
|
|
276
|
+
return defs;
|
|
277
|
+
}
|
|
278
|
+
|
|
240
279
|
// Every markdown link in the prose, scanned once from left to right. A "[" inside a code span,
|
|
241
280
|
// an escaped "\[" and the "](y)", is link content and is
|
|
284
|
+
// skipped whole rather than ending the scan, so the outer hyperlink is still found; any other
|
|
285
|
+
// nested "[" still ends the scan the way it always has, because CommonMark does not allow a
|
|
286
|
+
// link inside a link (outside review 2026-09-28: the previous scan gave up on the whole outer
|
|
287
|
+
// link the moment any nested "[" appeared, image or not). A full or collapsed reference link,
|
|
288
|
+
// "[text][label]" or "[text][]", resolves against collectLinkDefinitions; an unresolved label
|
|
289
|
+
// is read as plain brackets, the way it always was. A resolved target is decoded for character
|
|
290
|
+
// references (":", "&", ...) the same way an HTML attribute value is, so a destination
|
|
291
|
+
// written as "https://host:99999/" is checked as what it resolves to and not as its literal
|
|
292
|
+
// source spelling (outside review 2026-09-28). The scan carries no bound and no backtracking:
|
|
293
|
+
// the furthest failed plain target is remembered as before, and the closing ">", quote and line
|
|
294
|
+
// end are looked up through a cache that only moves forward, so a file that repeats "[a](" or
|
|
295
|
+
// "[a](x \"" stays linear.
|
|
296
|
+
function collectLinks(text, defs) {
|
|
248
297
|
const out = [];
|
|
249
|
-
const code =
|
|
250
|
-
|
|
251
|
-
let at = 0;
|
|
252
|
-
for (const line of text.split("\n")) {
|
|
253
|
-
code.set(codeSpanMask(line), at);
|
|
254
|
-
at += line.length + 1;
|
|
255
|
-
}
|
|
256
|
-
}
|
|
298
|
+
const code = codeSpanMask(text);
|
|
299
|
+
if (!defs) defs = collectLinkDefinitions(text);
|
|
257
300
|
const cache = Object.create(null);
|
|
258
301
|
const nextIdx = (c, from) => {
|
|
259
302
|
const h = cache[c];
|
|
@@ -304,17 +347,53 @@ function collectLinks(text) {
|
|
|
304
347
|
if (!found) failEnd = e;
|
|
305
348
|
return found;
|
|
306
349
|
};
|
|
350
|
+
// An image nested in the link text is skipped whole: k is the index of its own "[", right
|
|
351
|
+
// after an unescaped "!". Its alt text is not itself scanned for nested links or images, the
|
|
352
|
+
// first unescaped "]" closes it, and it is only skipped when a target follows; a malformed
|
|
353
|
+
// image (no closing "]", no "(target)") is left for the caller to treat as an ordinary
|
|
354
|
+
// nested bracket, the way one always was.
|
|
355
|
+
const skipImage = (k) => {
|
|
356
|
+
let m = k + 1;
|
|
357
|
+
while (m < text.length && text[m] !== "]" && text[m] !== "[") m += text[m] === "\\" ? 2 : 1;
|
|
358
|
+
if (m >= text.length || text[m] !== "]" || text[m + 1] !== "(") return null;
|
|
359
|
+
const t = readTarget(m + 2);
|
|
360
|
+
return t ? t.end + 1 : null;
|
|
361
|
+
};
|
|
307
362
|
for (let i = 0; i < text.length; i++) {
|
|
308
363
|
if (text[i] !== "[" || code[i] || isEscaped(text, i)) continue;
|
|
309
364
|
if (i > 0 && text[i - 1] === "!" && !isEscaped(text, i - 1)) continue;
|
|
310
365
|
let j = i + 1;
|
|
311
|
-
|
|
366
|
+
for (;;) {
|
|
367
|
+
while (j < text.length && text[j] !== "]" && text[j] !== "[") j += text[j] === "\\" ? 2 : 1;
|
|
368
|
+
if (j >= text.length || text[j] !== "[") break;
|
|
369
|
+
if (code[j] || text[j - 1] !== "!" || isEscaped(text, j - 1)) break;
|
|
370
|
+
const after = skipImage(j);
|
|
371
|
+
if (after === null) break;
|
|
372
|
+
j = after;
|
|
373
|
+
}
|
|
312
374
|
if (j >= text.length) break;
|
|
313
375
|
if (text[j] === "[") { i = j - 1; continue; }
|
|
314
|
-
if (text[j + 1]
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
376
|
+
if (text[j + 1] === "(") {
|
|
377
|
+
const t = readTarget(j + 2);
|
|
378
|
+
if (t) { out.push({ name: text.slice(i + 1, j), target: decodeAttributeValue(t.target), at: i }); i = t.end; continue; }
|
|
379
|
+
i = j + 1;
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
382
|
+
if (text[j + 1] === "[") {
|
|
383
|
+
let m = j + 2;
|
|
384
|
+
while (m < text.length && text[m] !== "]" && text[m] !== "\n") m += text[m] === "\\" ? 2 : 1;
|
|
385
|
+
if (m < text.length && text[m] === "]") {
|
|
386
|
+
const rawLabel = text.slice(j + 2, m);
|
|
387
|
+
const label = normalizeLinkLabel(rawLabel === "" ? text.slice(i + 1, j) : rawLabel);
|
|
388
|
+
const dest = label && defs.get(label);
|
|
389
|
+
if (dest !== undefined) {
|
|
390
|
+
out.push({ name: text.slice(i + 1, j), target: decodeAttributeValue(dest), at: i });
|
|
391
|
+
i = m;
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
i = j;
|
|
318
397
|
}
|
|
319
398
|
return out;
|
|
320
399
|
}
|
|
@@ -356,16 +435,12 @@ function fenceMask(lines) {
|
|
|
356
435
|
}
|
|
357
436
|
// A markdown list item that carries a link. A bullet or an ordered marker (1. or 1)) followed by
|
|
358
437
|
// a space or a tab opens the item; an ordered list and a tab after the marker were not read as
|
|
359
|
-
// list items until 2026-09-27 (Tek-496).
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
//
|
|
363
|
-
function listItemHasLink(l) {
|
|
364
|
-
|
|
365
|
-
if (!m) return false;
|
|
366
|
-
const sep = l.slice(m[0].length).search(/[\r\u2028\u2029]/);
|
|
367
|
-
const body = sep === -1 ? l.slice(m[0].length) : l.slice(m[0].length, m[0].length + sep);
|
|
368
|
-
return collectLinks(body).length > 0;
|
|
438
|
+
// list items until 2026-09-27 (Tek-496). Whether the line carries a link is decided by the
|
|
439
|
+
// caller against the whole document's own collectLinks result (hasLink), not by re-scanning
|
|
440
|
+
// this one line in isolation: a code span or a reference link target can resolve only across
|
|
441
|
+
// more than one line, and a per-line rescan cannot see that (outside review 2026-09-28, V2).
|
|
442
|
+
function listItemHasLink(l, hasLink) {
|
|
443
|
+
return !!hasLink && /^ {0,3}(?:[-*+]|\d{1,9}[.)])[ \t]/.test(l);
|
|
369
444
|
}
|
|
370
445
|
|
|
371
446
|
// The text of an ATX H1, or null when the line is not one. One to three leading spaces, a
|
|
@@ -423,8 +498,14 @@ export function validateLlmsTxt(f) {
|
|
|
423
498
|
// v3.164.0): a query value or a fragment in a same-host Location used to reach this
|
|
424
499
|
// detail as sent.
|
|
425
500
|
add("http-status", "pass", "File exists at /llms.txt", f.redirectedFrom ? "HTTP 200, followed a redirect from " + maskLocation(f.redirectedFrom) + " to " + maskLocation(f.finalUrl) : "HTTP 200");
|
|
501
|
+
// A leading byte-order mark is stripped the same way the fetch path already strips it: the
|
|
502
|
+
// TextDecoder used there removes a BOM by default, so the direct API took a different
|
|
503
|
+
// document than the fetched one for the exact same bytes until this was measured (outside
|
|
504
|
+
// review 2026-09-28); the h1-title check saw the mark as part of the first line and failed a
|
|
505
|
+
// document that would pass when fetched.
|
|
506
|
+
const text = f.text.charCodeAt(0) === 0xFEFF ? f.text.slice(1) : f.text;
|
|
426
507
|
const ct = (f.contentType || "").toLowerCase();
|
|
427
|
-
const looksHtml = /^\s*(<!doctype|<html|<head|<body)/i.test(
|
|
508
|
+
const looksHtml = /^\s*(<!doctype|<html|<head|<body)/i.test(text);
|
|
428
509
|
if (looksHtml) {
|
|
429
510
|
add("content-type", "fail", "Response is plain text", "the body looks like an HTML page, not an llms.txt file");
|
|
430
511
|
return checks;
|
|
@@ -437,7 +518,7 @@ export function validateLlmsTxt(f) {
|
|
|
437
518
|
} else {
|
|
438
519
|
add("content-type", "warn", "Response is plain text", "content-type is " + (ct.split(";")[0] || "missing") + ", text/plain or text/markdown is the convention");
|
|
439
520
|
}
|
|
440
|
-
const lines =
|
|
521
|
+
const lines = text.split(/\r?\n/);
|
|
441
522
|
const firstIdx = lines.findIndex((l) => l.trim() !== "");
|
|
442
523
|
const firstRaw = firstIdx === -1 ? "" : lines[firstIdx];
|
|
443
524
|
const first = firstRaw.trim();
|
|
@@ -454,7 +535,16 @@ export function validateLlmsTxt(f) {
|
|
|
454
535
|
add("h1-title", "fail", "Starts with an H1 title", "the first non-empty line should be a markdown H1 (# Site name)");
|
|
455
536
|
}
|
|
456
537
|
const afterH1 = lines.slice(firstIdx + 1).find((l) => l.trim() !== "") || "";
|
|
457
|
-
|
|
538
|
+
// Read the same way as the H1 check and the "late" blockquote search below: at most three
|
|
539
|
+
// leading spaces, per CommonMark. A trim-first prefix test used to erase the difference, so
|
|
540
|
+
// a four-space or tab-indented code block right after the title passed as the summary until
|
|
541
|
+
// 2026-09-27 (Tek-496, mirrored from worker.js).
|
|
542
|
+
// A blockquote marker is 0-3 leading spaces plus ">", optionally followed by one space that
|
|
543
|
+
// is then part of the marker and not the content; CommonMark reads ">Summary" and a tab the
|
|
544
|
+
// same way, and four or more leading spaces stay indented code, not a blockquote (outside
|
|
545
|
+
// review 2026-09-28: a literal space was required, so a tab or no space at all warned as
|
|
546
|
+
// missing).
|
|
547
|
+
if (/^ {0,3}>/.test(afterH1)) {
|
|
458
548
|
add("summary", "pass", "Blockquote summary after the title", JSON.stringify(cut(afterH1.trim(), 80)));
|
|
459
549
|
} else {
|
|
460
550
|
// A blockquote further down, before the first H2, is a summary in the wrong place and not a
|
|
@@ -466,7 +556,7 @@ export function validateLlmsTxt(f) {
|
|
|
466
556
|
for (let i = firstIdx + 1; i < lines.length; i++) {
|
|
467
557
|
if (fence[i]) continue;
|
|
468
558
|
if (/^ {0,3}## /.test(lines[i])) break;
|
|
469
|
-
if (/^ {0,3}
|
|
559
|
+
if (/^ {0,3}>/.test(lines[i])) { late = i; break; }
|
|
470
560
|
}
|
|
471
561
|
add("summary", "warn", "Blockquote summary after the title", late === -1
|
|
472
562
|
? "recommended by the format (> one-line summary), not required"
|
|
@@ -477,6 +567,29 @@ export function validateLlmsTxt(f) {
|
|
|
477
567
|
// demanded column zero, so a correct file indented by one to three spaces was reported as
|
|
478
568
|
// having no sections at all while its list under the same indentation counted fine.
|
|
479
569
|
const fenced = fenceMask(lines);
|
|
570
|
+
// Computed once for both the section-list scan below and the links check further down, so a
|
|
571
|
+
// reference link definition written anywhere in the prose resolves the same way for each
|
|
572
|
+
// (outside review 2026-09-28, V2): "[text][label]" used to be recognized in the whole-document
|
|
573
|
+
// links check but not in the per-line section scan, which only ever saw one list item line.
|
|
574
|
+
const proseText = lines.filter((l, i) => !fenced[i]).join("\n");
|
|
575
|
+
const linkDefs = collectLinkDefinitions(proseText);
|
|
576
|
+
const proseLinks = collectLinks(proseText, linkDefs);
|
|
577
|
+
// Which original line each recognized link starts on, so the section scan below can ask "does
|
|
578
|
+
// this line carry a link" against the same whole-document result the links check itself uses,
|
|
579
|
+
// instead of re-finding links one isolated line at a time (outside review 2026-09-28, V2).
|
|
580
|
+
const linkedLineIdx = new Set();
|
|
581
|
+
{
|
|
582
|
+
const owners = [];
|
|
583
|
+
for (let i = 0; i < lines.length; i++) if (!fenced[i]) owners.push(i);
|
|
584
|
+
const starts = [];
|
|
585
|
+
let offset = 0;
|
|
586
|
+
for (const pl of proseText.split("\n")) { starts.push(offset); offset += pl.length + 1; }
|
|
587
|
+
for (const link of proseLinks) {
|
|
588
|
+
let idx = 0;
|
|
589
|
+
for (let k = 0; k < starts.length; k++) { if (starts[k] <= link.at) idx = k; else break; }
|
|
590
|
+
linkedLineIdx.add(owners[idx]);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
480
593
|
const h2Count = lines.filter((l, i) => !fenced[i] && /^ {0,3}## /.test(l)).length;
|
|
481
594
|
// A section counts when it carries a file list. An H2 followed by a paragraph satisfied
|
|
482
595
|
// this check until 2026-08-29, and the format puts each section's links in a list.
|
|
@@ -493,7 +606,7 @@ export function validateLlmsTxt(f) {
|
|
|
493
606
|
const h1 = /^ {0,3}#(?:[ \t]|$)/.test(l);
|
|
494
607
|
if (misplaced === -1 && i > firstIdx && (h1 || (!seenH2 && /^ {0,3}#{3,6}(?:[ \t]|$)/.test(l)))) misplaced = i;
|
|
495
608
|
if (h1) { inSection = false; continue; }
|
|
496
|
-
if (inSection && !counted && listItemHasLink(l)) { sectionsWithList++; counted = true; }
|
|
609
|
+
if (inSection && !counted && listItemHasLink(l, linkedLineIdx.has(i))) { sectionsWithList++; counted = true; }
|
|
497
610
|
}
|
|
498
611
|
}
|
|
499
612
|
if (h2Count > 0 && sectionsWithList > 0) {
|
|
@@ -509,8 +622,9 @@ export function validateLlmsTxt(f) {
|
|
|
509
622
|
s.detail += "; the heading at line " + (misplaced + 1) + " is out of place, because the format has one H1 and no other heading before the first H2";
|
|
510
623
|
}
|
|
511
624
|
// Links are collected from the prose only, for the same reason the headings are: a link
|
|
512
|
-
// shown inside a code fence is an example of a link, not one an agent can follow.
|
|
513
|
-
|
|
625
|
+
// shown inside a code fence is an example of a link, not one an agent can follow. Computed
|
|
626
|
+
// once above as proseLinks, reused here under its usual name.
|
|
627
|
+
const links = proseLinks;
|
|
514
628
|
// An entry an agent can use has a name and a target with a host. An empty name and a
|
|
515
629
|
// bare "https://" both counted as valid absolute links until 2026-08-29.
|
|
516
630
|
const named = links.filter((m) => m.name.trim() !== "");
|
|
@@ -534,7 +648,7 @@ export function validateLlmsTxt(f) {
|
|
|
534
648
|
} else {
|
|
535
649
|
add("size", "warn", "Small enough to be cheap to read", f.bytes + " bytes; consider moving detail to llms-full.txt");
|
|
536
650
|
}
|
|
537
|
-
if (/<[a-z][a-z0-9-]*[\s>]/i.test(
|
|
651
|
+
if (/<[a-z][a-z0-9-]*[\s>]/i.test(text)) {
|
|
538
652
|
add("no-html", "warn", "No HTML markup in the file", "HTML tags found; llms.txt should be plain markdown");
|
|
539
653
|
} else {
|
|
540
654
|
add("no-html", "pass", "No HTML markup in the file", "plain markdown");
|
|
@@ -679,8 +793,11 @@ var C1_REPLACEMENTS = {
|
|
|
679
793
|
|
|
680
794
|
// The full WHATWG list of named character references, 2 231 names: "name!hex" is a name that
|
|
681
795
|
// also decodes without its ";", "name=hex" one that needs it, and several code points are
|
|
682
|
-
// separated by ",". Generated 2026-09-12 from
|
|
683
|
-
//
|
|
796
|
+
// separated by ",". Generated 2026-09-12 from character-entities 2.0.2
|
|
797
|
+
// (https://github.com/wooorm/character-entities), (c) 2015 Titus Wormer, MIT, with every value
|
|
798
|
+
// and every legacy name checked against the decoder of the entities package. The full license
|
|
799
|
+
// text is in THIRD-PARTY-NOTICES.md (E01, outside audit 2026-09-26); this is a build-time data
|
|
800
|
+
// source copied in at generation time, not an installed runtime dependency.
|
|
684
801
|
var NAMED_REFERENCE_DATA =
|
|
685
802
|
"AElig!c6 AMP!26 Aacute!c1 Abreve=102 Acirc!c2 Acy=410 Afr=1d504 Agrave!c0 Alpha=391 Amacr=100 " +
|
|
686
803
|
"And=2a53 Aogon=104 Aopf=1d538 ApplyFunction=2061 Aring!c5 Ascr=1d49c Assign=2254 Atilde!c3 " +
|