turva-llms-txt-validator 0.3.21 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/THIRD-PARTY-NOTICES.md +6 -5
- package/dist/bin/cli.js +103 -0
- package/dist/src/index.d.ts +59 -0
- package/dist/src/index.js +2257 -0
- package/package.json +21 -7
- package/bin/cli.mjs +0 -95
- package/src/index.mjs +0 -1880
|
@@ -0,0 +1,2257 @@
|
|
|
1
|
+
// turva-llms-txt-validator: validate a site's llms.txt structure.
|
|
2
|
+
// The checks mirror the hosted validator at https://turva.dev/llms-txt-validator,
|
|
3
|
+
// which runs this same logic inside the turva.dev Cloudflare Worker
|
|
4
|
+
// (https://github.com/erekola/turva-worker). The hosted validator stays
|
|
5
|
+
// canonical: if the two ever disagree, the hosted one wins and this package
|
|
6
|
+
// gets the fix.
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
// Version comes from package.json so the two can never drift again
|
|
9
|
+
// (0.1.1 and 0.1.3 both shipped with a stale hardcoded VERSION).
|
|
10
|
+
export const VERSION = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8")).version;
|
|
11
|
+
// The hosted validator identifies itself without a version, because its version is the
|
|
12
|
+
// site's and moves with every release; this package carries its own. The two strings
|
|
13
|
+
// differ on purpose and nothing compares them (round 16 S3-6).
|
|
14
|
+
const UA = "turva-llms-txt-validator/" + VERSION + " (+https://turva.dev/llms-txt-validator)";
|
|
15
|
+
// Cut a string to at most n UTF-16 code units without leaving a lone high surrogate at the
|
|
16
|
+
// end. String.prototype.slice counts code units, so a cut that lands inside a surrogate
|
|
17
|
+
// pair leaves half of it, and the character is lost. How the half is written depends on
|
|
18
|
+
// the runtime: Node's UTF-8 encoder replaces it with U+FFFD, and the hosted validator in
|
|
19
|
+
// workerd wrote three bytes that were not valid UTF-8 (round 16 S3-1 and S3-2, measured
|
|
20
|
+
// there 2026-09-03). Mirrors worker.js cut().
|
|
21
|
+
export function cut(s, n) {
|
|
22
|
+
s = String(s).slice(0, n);
|
|
23
|
+
return /[\uD800-\uDBFF]$/.test(s) ? s.slice(0, -1) : s;
|
|
24
|
+
}
|
|
25
|
+
// A redirect target the way a report may show it. The scheme, the host and the path stay readable,
|
|
26
|
+
// and the parts that can carry a secret do not: the user name and the password are removed, every
|
|
27
|
+
// query value becomes ***, and the fragment is dropped. The caller cuts the result to length
|
|
28
|
+
// afterwards, because a cut is not a mask. Up to 0.3.4 a refused redirect to
|
|
29
|
+
// https://user:password@host/ kept both in the check's detail and in the returned location (outside
|
|
30
|
+
// review, 2026-09-22). A target the URL parser refuses is masked by hand: nothing after the first ?
|
|
31
|
+
// or # survives, and when the target has a scheme or starts with two slashes, everything up to the
|
|
32
|
+
// last @ before them is shown as ***. Mirrors worker.js maskLocation().
|
|
33
|
+
export function maskLocation(href, base) {
|
|
34
|
+
let u = null;
|
|
35
|
+
try {
|
|
36
|
+
u = new URL(href, base);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
u = null;
|
|
40
|
+
}
|
|
41
|
+
// An opaque URL (no authority, e.g. "u:name@host/") parses without throwing but has no
|
|
42
|
+
// host, so clearing u.username/u.password is a silent no-op: WHATWG has nowhere to put
|
|
43
|
+
// them and the credential-shaped text stays in the opaque path untouched. Route it through
|
|
44
|
+
// the string branch below instead, the same one an unparseable target already uses
|
|
45
|
+
// (outside review 2026-09-28: a bare "u:" scheme let the text after it through as the
|
|
46
|
+
// CLI JSON "target" field, unmasked).
|
|
47
|
+
if (u && u.host) {
|
|
48
|
+
u.username = "";
|
|
49
|
+
u.password = "";
|
|
50
|
+
// One pass (Tek-560): a Set keeps the first-seen key order and collapses duplicates, and the
|
|
51
|
+
// masked query is built once. Setting each key on u.searchParams re-serializes the whole
|
|
52
|
+
// query every time, which is quadratic in the number of keys. An empty query is left alone,
|
|
53
|
+
// so "?" stays "?" as before.
|
|
54
|
+
const maskedQuery = new URLSearchParams();
|
|
55
|
+
// append, not set: the keys are already unique, and set searches the whole list on every call.
|
|
56
|
+
for (const key of new Set(u.searchParams.keys()))
|
|
57
|
+
maskedQuery.append(key, "***");
|
|
58
|
+
if (maskedQuery.toString() !== "")
|
|
59
|
+
u.search = maskedQuery.toString();
|
|
60
|
+
u.hash = "";
|
|
61
|
+
return u.href;
|
|
62
|
+
}
|
|
63
|
+
let s = String(href).replace(/[\t\n\r]/g, "");
|
|
64
|
+
const stop = s.search(/[?#]/);
|
|
65
|
+
if (stop >= 0)
|
|
66
|
+
s = s.slice(0, stop) + (s[stop] === "?" ? "?***" : "");
|
|
67
|
+
const scheme = /^[a-z][a-z0-9+.-]*:/i.exec(s);
|
|
68
|
+
const start = scheme ? scheme[0].length : 0;
|
|
69
|
+
let from = start;
|
|
70
|
+
while (from < s.length && (s[from] === "/" || s[from] === "\\"))
|
|
71
|
+
from++;
|
|
72
|
+
// Without a scheme and two slashes there is normally no authority, and an @ is part of a
|
|
73
|
+
// path ("path@2x.png" stays unmasked below). But when an @ still appears before the first
|
|
74
|
+
// /, the text ahead of it reads as userinfo even when the parser here does not recognise it
|
|
75
|
+
// as a scheme (an underscore, a leading digit or space, a stray control character), so that
|
|
76
|
+
// shape is masked too instead of let through (VN1, round 3 Tek-542). With no slash at all
|
|
77
|
+
// the same holds when a ":" comes before the last @: "user_name:secret@example.com" is
|
|
78
|
+
// user:password and is masked, while "path@2x.png" has no ":" ahead of its @ and stays as
|
|
79
|
+
// given (outside review V13, D5-1: the slashless shape used to return early and print the
|
|
80
|
+
// password).
|
|
81
|
+
if (!scheme && from - start < 2) {
|
|
82
|
+
const firstSlash = s.indexOf("/");
|
|
83
|
+
const firstAt = s.indexOf("@");
|
|
84
|
+
if (firstAt < 0)
|
|
85
|
+
return s;
|
|
86
|
+
if (firstSlash < 0) {
|
|
87
|
+
if (!s.slice(0, s.lastIndexOf("@")).includes(":"))
|
|
88
|
+
return s;
|
|
89
|
+
}
|
|
90
|
+
else if (firstAt > firstSlash) {
|
|
91
|
+
return s;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
// Nothing tells which @ of a refused target ends the user information, so everything up to the
|
|
95
|
+
// last one is shown as ***. The mask is visible on purpose: three review rounds on 2026-09-22
|
|
96
|
+
// measured every rule that guessed, and each one either returned a password that holds a slash
|
|
97
|
+
// or made the host vanish without a trace.
|
|
98
|
+
const at = s.lastIndexOf("@");
|
|
99
|
+
return at < from ? s : s.slice(0, from) + "***@" + s.slice(at + 1);
|
|
100
|
+
}
|
|
101
|
+
// Unicode bidi controls (U+202A to U+202E, U+2066 to U+2069) have no place in a check's
|
|
102
|
+
// detail: text quoted from the fetched file could otherwise read in a different order here
|
|
103
|
+
// than in the file itself, and the right-to-left override is the classic way to do that
|
|
104
|
+
// (round 19, V6-U1). Mirrors worker.js stripBidi().
|
|
105
|
+
export function stripBidi(s) {
|
|
106
|
+
return String(s).replace(/[\u202A-\u202E\u2066-\u2069]/g, "");
|
|
107
|
+
}
|
|
108
|
+
// C0 and C1 control characters leave a check's detail as well. JSON escapes them, but a program
|
|
109
|
+
// that prints a detail to a terminal passes ESC through, so a published link could carry a
|
|
110
|
+
// sequence that clears the screen (Tek-496).
|
|
111
|
+
// Mirrors worker.js stripControls().
|
|
112
|
+
export function stripControls(s) {
|
|
113
|
+
return String(s).replace(/[\u0000-\u001F\u007F-\u009F]/g, "");
|
|
114
|
+
}
|
|
115
|
+
export function normalizeHostInput(raw) {
|
|
116
|
+
let s = String(raw || "").trim().toLowerCase();
|
|
117
|
+
if (!s)
|
|
118
|
+
return null;
|
|
119
|
+
if (!/^[a-z][a-z0-9+.-]*:\/\//.test(s))
|
|
120
|
+
s = "https://" + s;
|
|
121
|
+
let u;
|
|
122
|
+
try {
|
|
123
|
+
u = new URL(s);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
if (u.protocol !== "https:" && u.protocol !== "http:")
|
|
129
|
+
return null;
|
|
130
|
+
if (u.port && u.port !== "443" && u.port !== "80")
|
|
131
|
+
return null;
|
|
132
|
+
if (u.username || u.password)
|
|
133
|
+
return null;
|
|
134
|
+
// One trailing dot names the same host in absolute form, so "example.com." is read as
|
|
135
|
+
// example.com instead of refused (round 19, K7-P5). Mirrors worker.js normalizeHostInput().
|
|
136
|
+
const typedHost = u.hostname;
|
|
137
|
+
return typedHost.length > 1 && typedHost.endsWith(".") ? typedHost.slice(0, -1) : typedHost;
|
|
138
|
+
}
|
|
139
|
+
// The path of the address as typed, when it is neither the root nor /llms.txt. validateHost keeps its
|
|
140
|
+
// reads pinned to /llms.txt and the home page, because a fixed path is part of the guard against use
|
|
141
|
+
// as a fetch proxy, so a typed /docs/llms.txt used to be dropped without a word while the result
|
|
142
|
+
// described the root file (outside retest 2026-09-26). The result now names the unused path in an
|
|
143
|
+
// info check. The case of the path is kept, and the query and fragment are never returned.
|
|
144
|
+
// Mirrors worker.js enteredPath() (v3.176.0).
|
|
145
|
+
export function enteredPath(raw) {
|
|
146
|
+
let s = String(raw || "").trim();
|
|
147
|
+
if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(s))
|
|
148
|
+
s = "https://" + s;
|
|
149
|
+
let u;
|
|
150
|
+
try {
|
|
151
|
+
u = new URL(s);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return "";
|
|
155
|
+
}
|
|
156
|
+
const path = u.pathname;
|
|
157
|
+
return path === "/" || path === "" || path === "/llms.txt" ? "" : path;
|
|
158
|
+
}
|
|
159
|
+
export function isValidPublicHost(host) {
|
|
160
|
+
if (!host || host.length > 253)
|
|
161
|
+
return false;
|
|
162
|
+
if (host.startsWith("[") || /^\d+\.\d+\.\d+\.\d+$/.test(host))
|
|
163
|
+
return false;
|
|
164
|
+
if (!/^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{1,62}$/.test(host))
|
|
165
|
+
return false;
|
|
166
|
+
const tld = host.split(".").pop();
|
|
167
|
+
// arpa and onion since round 19 (P3): home.arpa is the home-network name of RFC 8375,
|
|
168
|
+
// in-addr.arpa and ip6.arpa name addresses, and an onion name never resolves on the public web.
|
|
169
|
+
// localdomain since 2026-09-27 (Tek-496): localhost.localdomain is the loopback alias that many
|
|
170
|
+
// hosts files carry, and it passed this list although the name never resolves on the public web.
|
|
171
|
+
if (["localhost", "localdomain", "local", "internal", "home", "lan", "corp", "test", "invalid", "arpa", "onion"].includes(tld))
|
|
172
|
+
return false;
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
// opts.path and opts.accept were added for the v2 discovery checks, which need the
|
|
176
|
+
// site's home page as well as its llms.txt. Nothing else moved: the redirect budget,
|
|
177
|
+
// the same-host rule, the credential and port rejections and the 256 KB cap are the
|
|
178
|
+
// guards this function was measured against.
|
|
179
|
+
export async function fetchLlmsTxt(host, opts = {}) {
|
|
180
|
+
const timeoutMs = opts.timeoutMs ?? 8000;
|
|
181
|
+
const cap = opts.cap ?? 262144;
|
|
182
|
+
const reqApex = host.startsWith("www.") ? host.slice(4) : host;
|
|
183
|
+
let url = "https://" + host + (opts.path ?? "/llms.txt");
|
|
184
|
+
let redirectedFrom = null;
|
|
185
|
+
// One budget for the whole redirect chain, not one per hop. Mirrors the hosted validator
|
|
186
|
+
// (turva.dev/llms-txt-validator), which is canonical: with five hops a per-hop signal made
|
|
187
|
+
// the caller's timeout five times longer than the value it passed in.
|
|
188
|
+
const deadline = Date.now() + timeoutMs;
|
|
189
|
+
for (let hop = 0;; hop++) {
|
|
190
|
+
const res = await fetch(url, {
|
|
191
|
+
redirect: "manual",
|
|
192
|
+
signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())),
|
|
193
|
+
headers: { "user-agent": opts.userAgent ?? UA, "accept": opts.accept ?? "text/plain, text/markdown;q=0.9, */*;q=0.1" }
|
|
194
|
+
});
|
|
195
|
+
if (res.status >= 300 && res.status < 400) {
|
|
196
|
+
// Six paths leave this branch, five returns and one continue, and not one of them
|
|
197
|
+
// ever reads the redirect body. undici holds the connection until a body is read or
|
|
198
|
+
// cancelled, so it is released here, once, before the location is even parsed. A
|
|
199
|
+
// failed cancel must not turn a redirect verdict into a throw (2026-09-10).
|
|
200
|
+
try {
|
|
201
|
+
await res.body?.cancel();
|
|
202
|
+
}
|
|
203
|
+
catch { /* the verdict below is the answer */ }
|
|
204
|
+
const loc = res.headers.get("location") || "";
|
|
205
|
+
if (!loc)
|
|
206
|
+
return { redirect: true, reason: "no-location", status: res.status, location: "" };
|
|
207
|
+
if (hop >= 4)
|
|
208
|
+
return { redirect: true, reason: "too-many", status: res.status, location: cut(maskLocation(loc, url), 120) };
|
|
209
|
+
let next;
|
|
210
|
+
try {
|
|
211
|
+
next = new URL(loc, url);
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
return { redirect: true, reason: "bad-location", status: res.status, location: cut(maskLocation(loc, url), 120) };
|
|
215
|
+
}
|
|
216
|
+
const safeTarget = next.protocol === "https:" && !next.port && !next.username && !next.password && isValidPublicHost(next.hostname);
|
|
217
|
+
const twin = (next.hostname.startsWith("www.") ? next.hostname.slice(4) : next.hostname) === reqApex;
|
|
218
|
+
if (!safeTarget)
|
|
219
|
+
return { redirect: true, reason: "unsafe-target", status: res.status, location: cut(maskLocation(next.href), 120) };
|
|
220
|
+
if (!twin)
|
|
221
|
+
return { redirect: true, reason: "off-host", status: res.status, location: cut(maskLocation(next.href), 120) };
|
|
222
|
+
if (!redirectedFrom)
|
|
223
|
+
redirectedFrom = url;
|
|
224
|
+
url = next.href;
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
let bytes = 0, truncated = false;
|
|
228
|
+
const chunks = [];
|
|
229
|
+
if (res.body) {
|
|
230
|
+
const reader = res.body.getReader();
|
|
231
|
+
for (;;) {
|
|
232
|
+
const { done, value } = await reader.read();
|
|
233
|
+
if (done)
|
|
234
|
+
break;
|
|
235
|
+
bytes += value.length;
|
|
236
|
+
if (bytes > cap) {
|
|
237
|
+
truncated = true;
|
|
238
|
+
chunks.push(value.slice(0, value.length - (bytes - cap)));
|
|
239
|
+
bytes = cap;
|
|
240
|
+
await reader.cancel();
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
chunks.push(value);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
const buf = new Uint8Array(bytes);
|
|
247
|
+
let o = 0;
|
|
248
|
+
for (const c of chunks) {
|
|
249
|
+
buf.set(c, o);
|
|
250
|
+
o += c.length;
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
status: res.status,
|
|
254
|
+
contentType: res.headers.get("content-type") || "",
|
|
255
|
+
linkHeader: res.headers.get("link") || "",
|
|
256
|
+
text: new TextDecoder("utf-8").decode(buf),
|
|
257
|
+
bytes,
|
|
258
|
+
truncated,
|
|
259
|
+
redirectedFrom,
|
|
260
|
+
finalUrl: url
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
// A backslash escapes the character after it when an odd number of backslashes precede it.
|
|
265
|
+
function isEscaped(s, i) {
|
|
266
|
+
let n = 0;
|
|
267
|
+
for (let b = i - 1; b >= 0 && s[b] === "\\"; b--)
|
|
268
|
+
n++;
|
|
269
|
+
return n % 2 === 1;
|
|
270
|
+
}
|
|
271
|
+
// Inline code spans, marked per character across the whole document, so a link written inside
|
|
272
|
+
// backticks is read as the code it is (outside audit 2026-09-26, Tek-496), including when the
|
|
273
|
+
// span itself crosses a real line ending: a fenceless "`[a](b)\ncontinued`" used to be masked
|
|
274
|
+
// only on its own line, so the second line escaped the mask and its bracket read as a real link
|
|
275
|
+
// (outside review 2026-09-28). A span opens with a run of backticks and closes with the next run
|
|
276
|
+
// of the same length anywhere in the text; a pair that crosses a block boundary, a blank line
|
|
277
|
+
// (blank or made only of spaces and tabs), a heading line or a new list item, is not a code span
|
|
278
|
+
// in CommonMark, so it is left unmasked and its backticks stay literal (VREG2, round 3 Tek-542:
|
|
279
|
+
// backticks in two different list items used to enclose a real hyperlink between them). An
|
|
280
|
+
// escaped first backtick is literal and the rest of its run still opens, while a closing run is
|
|
281
|
+
// never escaped, because a backslash inside a code span is literal. Runs are grouped by length
|
|
282
|
+
// and every group is read with one forward pointer, so the scan stays linear on a document of
|
|
283
|
+
// unmatched runs.
|
|
284
|
+
function crossesBlockBoundary(text, start, end) {
|
|
285
|
+
const inner = text.slice(start, end);
|
|
286
|
+
if (/\n[ \t]*\n/.test(inner))
|
|
287
|
+
return true;
|
|
288
|
+
const lines = inner.split("\n");
|
|
289
|
+
for (let i = 1; i < lines.length; i++) {
|
|
290
|
+
if (/^ {0,3}(#{1,6}(?:[ \t]|$)|[-*+][ \t]|\d{1,9}[.)][ \t])/.test(lines[i]))
|
|
291
|
+
return true;
|
|
292
|
+
}
|
|
293
|
+
return false;
|
|
294
|
+
}
|
|
295
|
+
function codeSpanMask(text) {
|
|
296
|
+
const mask = new Uint8Array(text.length);
|
|
297
|
+
const runs = [];
|
|
298
|
+
for (let i = 0; i < text.length;) {
|
|
299
|
+
if (text[i] !== "`") {
|
|
300
|
+
i++;
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
let j = i;
|
|
304
|
+
while (j < text.length && text[j] === "`")
|
|
305
|
+
j++;
|
|
306
|
+
runs.push({ at: i, len: j - i, escaped: isEscaped(text, i) });
|
|
307
|
+
i = j;
|
|
308
|
+
}
|
|
309
|
+
const byLen = new Map();
|
|
310
|
+
runs.forEach((r, k) => { if (!byLen.has(r.len))
|
|
311
|
+
byLen.set(r.len, []); byLen.get(r.len).push(k); });
|
|
312
|
+
const ptr = new Map();
|
|
313
|
+
for (let k = 0; k < runs.length; k++) {
|
|
314
|
+
const r = runs[k];
|
|
315
|
+
const len = r.escaped ? r.len - 1 : r.len;
|
|
316
|
+
const list = len > 0 ? byLen.get(len) : undefined;
|
|
317
|
+
if (!list)
|
|
318
|
+
continue;
|
|
319
|
+
let p = ptr.get(len) || 0;
|
|
320
|
+
while (p < list.length && list[p] <= k)
|
|
321
|
+
p++;
|
|
322
|
+
ptr.set(len, p);
|
|
323
|
+
if (p === list.length)
|
|
324
|
+
continue;
|
|
325
|
+
const close = runs[list[p]];
|
|
326
|
+
const start = r.escaped ? r.at + 1 : r.at;
|
|
327
|
+
const end = close.at + close.len;
|
|
328
|
+
if (!crossesBlockBoundary(text, start, end))
|
|
329
|
+
mask.fill(1, start, end);
|
|
330
|
+
k = list[p];
|
|
331
|
+
}
|
|
332
|
+
return mask;
|
|
333
|
+
}
|
|
334
|
+
// A CommonMark link reference label is matched without regard to case or the exact run of
|
|
335
|
+
// whitespace inside it, so "[Guide]" and "[ guide ]" name the same definition.
|
|
336
|
+
function normalizeLinkLabel(s) {
|
|
337
|
+
return s.trim().replace(/\s+/g, " ").toLowerCase();
|
|
338
|
+
}
|
|
339
|
+
// Link reference definitions, "[label]: target", read once per document so a reference link
|
|
340
|
+
// such as "[text][label]" can resolve against one written anywhere else in the same file,
|
|
341
|
+
// before or after its use, the way CommonMark allows (outside review 2026-09-28). A definition
|
|
342
|
+
// only opens where a new block can start: at the top of the document, right after a blank
|
|
343
|
+
// line, right after a heading or list marker line, or right after another definition line; a
|
|
344
|
+
// line that continues open paragraph text is read as that paragraph, not as a fresh
|
|
345
|
+
// definition, and a "definition" written inside an inline code span is the span's own literal
|
|
346
|
+
// text (VREG1, round 3 Tek-542). A title after the target counts only in quotes or
|
|
347
|
+
// parentheses on the same line; anything else trailing the target, such as a bare word, makes
|
|
348
|
+
// the whole line not a definition, matching CommonMark instead of reading everything up to the
|
|
349
|
+
// next space as the title's own value. Only the first definition of a given label counts, per
|
|
350
|
+
// spec. A destination wrapped in angle brackets has them stripped.
|
|
351
|
+
function collectLinkDefinitions(text) {
|
|
352
|
+
const defs = new Map();
|
|
353
|
+
const lines = text.split("\n");
|
|
354
|
+
const code = codeSpanMask(text);
|
|
355
|
+
const starts = [];
|
|
356
|
+
{
|
|
357
|
+
let offset = 0;
|
|
358
|
+
for (const l of lines) {
|
|
359
|
+
starts.push(offset);
|
|
360
|
+
offset += l.length + 1;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
const DEF_START = /^ {0,3}\[[^\]]+\]:/;
|
|
364
|
+
const BLOCK_START = /^ {0,3}(#{1,6}(?:[ \t]|$)|[-*+][ \t]|\d{1,9}[.)][ \t])/;
|
|
365
|
+
for (let i = 0; i < lines.length; i++) {
|
|
366
|
+
const line = lines[i];
|
|
367
|
+
const prev = i > 0 ? lines[i - 1] : "";
|
|
368
|
+
if (prev.trim() !== "" && !DEF_START.test(prev) && !BLOCK_START.test(prev))
|
|
369
|
+
continue;
|
|
370
|
+
const m = /^ {0,3}\[([^\]]+)\]:[ \t]*(<[^<>\n]*>|\S+)([ \t]+(?:"[^"\n]*"|'[^'\n]*'|\([^()\n]*\)))?[ \t]*$/.exec(line);
|
|
371
|
+
if (!m)
|
|
372
|
+
continue;
|
|
373
|
+
if (code[starts[i] + line.indexOf("[")])
|
|
374
|
+
continue;
|
|
375
|
+
const label = normalizeLinkLabel(m[1]);
|
|
376
|
+
if (!label || defs.has(label))
|
|
377
|
+
continue;
|
|
378
|
+
let dest = m[2];
|
|
379
|
+
if (dest[0] === "<" && dest[dest.length - 1] === ">")
|
|
380
|
+
dest = dest.slice(1, -1);
|
|
381
|
+
defs.set(label, dest);
|
|
382
|
+
}
|
|
383
|
+
return defs;
|
|
384
|
+
}
|
|
385
|
+
// Every markdown link in the prose, scanned once from left to right. A "[" inside a code span,
|
|
386
|
+
// an escaped "\[" and the "](y)", is link content and is
|
|
390
|
+
// skipped whole rather than ending the scan, so the outer hyperlink is still found; any other
|
|
391
|
+
// nested "[" still ends the scan the way it always has, because CommonMark does not allow a
|
|
392
|
+
// link inside a link (outside review 2026-09-28: the previous scan gave up on the whole outer
|
|
393
|
+
// link the moment any nested "[" appeared, image or not). A full or collapsed reference link,
|
|
394
|
+
// "[text][label]" or "[text][]", resolves against collectLinkDefinitions; an unresolved label
|
|
395
|
+
// is read as plain brackets, the way it always was. A resolved target is decoded for character
|
|
396
|
+
// references (":", "&", ...) the same way an HTML attribute value is, so a destination
|
|
397
|
+
// written as "https://host:99999/" is checked as what it resolves to and not as its literal
|
|
398
|
+
// source spelling (outside review 2026-09-28). The scan carries no bound and no backtracking:
|
|
399
|
+
// the furthest failed plain target is remembered as before, and the closing ">", quote and line
|
|
400
|
+
// end are looked up through a cache that only moves forward, so a file that repeats "[a](" or
|
|
401
|
+
// "[a](x \"" stays linear.
|
|
402
|
+
function collectLinks(text, defs) {
|
|
403
|
+
const out = [];
|
|
404
|
+
const code = codeSpanMask(text);
|
|
405
|
+
if (!defs)
|
|
406
|
+
defs = collectLinkDefinitions(text);
|
|
407
|
+
const cache = Object.create(null);
|
|
408
|
+
const nextIdx = (c, from) => {
|
|
409
|
+
const h = cache[c];
|
|
410
|
+
if (h && h.from <= from && (h.at === -1 || h.at >= from))
|
|
411
|
+
return h.at;
|
|
412
|
+
const at = text.indexOf(c, from);
|
|
413
|
+
cache[c] = { from, at };
|
|
414
|
+
return at;
|
|
415
|
+
};
|
|
416
|
+
const isBlank = (c) => c === " " || c === "\t";
|
|
417
|
+
// A blank line ends a paragraph, and a link label cannot reach across one (Tek-560). True when
|
|
418
|
+
// the line break at p is followed by a line that holds only spaces and tabs, or by the end.
|
|
419
|
+
const blankAt = (p) => {
|
|
420
|
+
if (text[p] !== "\n")
|
|
421
|
+
return false;
|
|
422
|
+
let q = p + 1;
|
|
423
|
+
while (q < text.length && isBlank(text[q]))
|
|
424
|
+
q++;
|
|
425
|
+
return q >= text.length || text[q] === "\n";
|
|
426
|
+
};
|
|
427
|
+
// Spaces, tabs and at most one line ending may stand around the destination and the title, as
|
|
428
|
+
// CommonMark allows; "[a]( b )" and a ")" on the next line were missed until the review of
|
|
429
|
+
// 2026-09-27. A blank line still ends the link.
|
|
430
|
+
const skipSpace = (p) => {
|
|
431
|
+
while (p < text.length && isBlank(text[p]))
|
|
432
|
+
p++;
|
|
433
|
+
if (text[p] === "\n") {
|
|
434
|
+
p++;
|
|
435
|
+
while (p < text.length && isBlank(text[p]))
|
|
436
|
+
p++;
|
|
437
|
+
}
|
|
438
|
+
return p;
|
|
439
|
+
};
|
|
440
|
+
const afterTarget = (target, p) => {
|
|
441
|
+
const q = skipSpace(p);
|
|
442
|
+
if (text[q] === ")")
|
|
443
|
+
return { target, end: q };
|
|
444
|
+
const quote = text[q];
|
|
445
|
+
if (q > p && (quote === '"' || quote === "'")) {
|
|
446
|
+
const close = nextIdx(quote, q + 1);
|
|
447
|
+
const nl = nextIdx("\n", q + 1);
|
|
448
|
+
if (close === -1 || (nl !== -1 && nl < close))
|
|
449
|
+
return null;
|
|
450
|
+
const r = skipSpace(close + 1);
|
|
451
|
+
if (text[r] === ")")
|
|
452
|
+
return { target, end: r };
|
|
453
|
+
}
|
|
454
|
+
return null;
|
|
455
|
+
};
|
|
456
|
+
let failEnd = -1;
|
|
457
|
+
const readTarget = (k) => {
|
|
458
|
+
k = skipSpace(k);
|
|
459
|
+
if (text[k] === "<") {
|
|
460
|
+
const gt = nextIdx(">", k + 1);
|
|
461
|
+
if (gt === -1 || gt === k + 1)
|
|
462
|
+
return null;
|
|
463
|
+
const lt = nextIdx("<", k + 1);
|
|
464
|
+
const nl = nextIdx("\n", k + 1);
|
|
465
|
+
if ((lt !== -1 && lt < gt) || (nl !== -1 && nl < gt))
|
|
466
|
+
return null;
|
|
467
|
+
return afterTarget(text.slice(k + 1, gt), gt + 1);
|
|
468
|
+
}
|
|
469
|
+
if (k <= failEnd)
|
|
470
|
+
return null;
|
|
471
|
+
let e = k;
|
|
472
|
+
while (e < text.length && text[e] !== ")" && !/\s/.test(text[e]))
|
|
473
|
+
e++;
|
|
474
|
+
if (e === k)
|
|
475
|
+
return null;
|
|
476
|
+
if (text[e] === ")")
|
|
477
|
+
return { target: text.slice(k, e), end: e };
|
|
478
|
+
const found = e < text.length ? afterTarget(text.slice(k, e), e) : null;
|
|
479
|
+
if (!found)
|
|
480
|
+
failEnd = e;
|
|
481
|
+
return found;
|
|
482
|
+
};
|
|
483
|
+
// An image nested in the link text is skipped whole: k is the index of its own "[", right
|
|
484
|
+
// after an unescaped "!". Its alt text is not itself scanned for nested links or images, the
|
|
485
|
+
// first unescaped "]" closes it, and it is only skipped when a target follows; a malformed
|
|
486
|
+
// image (no closing "]", no "(target)") is left for the caller to treat as an ordinary
|
|
487
|
+
// nested bracket, the way one always was.
|
|
488
|
+
const skipImage = (k) => {
|
|
489
|
+
let m = k + 1;
|
|
490
|
+
while (m < text.length && text[m] !== "]" && text[m] !== "[" && !blankAt(m))
|
|
491
|
+
m += text[m] === "\\" ? 2 : 1;
|
|
492
|
+
if (m >= text.length || text[m] !== "]" || text[m + 1] !== "(")
|
|
493
|
+
return null;
|
|
494
|
+
const t = readTarget(m + 2);
|
|
495
|
+
return t ? t.end + 1 : null;
|
|
496
|
+
};
|
|
497
|
+
for (let i = 0; i < text.length; i++) {
|
|
498
|
+
if (text[i] !== "[" || code[i] || isEscaped(text, i))
|
|
499
|
+
continue;
|
|
500
|
+
if (i > 0 && text[i - 1] === "!" && !isEscaped(text, i - 1))
|
|
501
|
+
continue;
|
|
502
|
+
let j = i + 1;
|
|
503
|
+
for (;;) {
|
|
504
|
+
while (j < text.length && text[j] !== "]" && text[j] !== "[" && !blankAt(j))
|
|
505
|
+
j += text[j] === "\\" ? 2 : 1;
|
|
506
|
+
if (j >= text.length || text[j] !== "[")
|
|
507
|
+
break;
|
|
508
|
+
if (code[j] || text[j - 1] !== "!" || isEscaped(text, j - 1))
|
|
509
|
+
break;
|
|
510
|
+
const after = skipImage(j);
|
|
511
|
+
if (after === null)
|
|
512
|
+
break;
|
|
513
|
+
j = after;
|
|
514
|
+
}
|
|
515
|
+
if (j >= text.length)
|
|
516
|
+
break;
|
|
517
|
+
if (text[j] === "\n") {
|
|
518
|
+
i = j;
|
|
519
|
+
continue;
|
|
520
|
+
}
|
|
521
|
+
if (text[j] === "[") {
|
|
522
|
+
i = j - 1;
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
if (text[j + 1] === "(") {
|
|
526
|
+
const t = readTarget(j + 2);
|
|
527
|
+
if (t) {
|
|
528
|
+
out.push({ name: text.slice(i + 1, j), target: decodeDestination(t.target), at: i });
|
|
529
|
+
i = t.end;
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
i = j + 1;
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
if (text[j + 1] === "[") {
|
|
536
|
+
let m = j + 2;
|
|
537
|
+
while (m < text.length && text[m] !== "]" && text[m] !== "\n")
|
|
538
|
+
m += text[m] === "\\" ? 2 : 1;
|
|
539
|
+
if (m < text.length && text[m] === "]") {
|
|
540
|
+
const rawLabel = text.slice(j + 2, m);
|
|
541
|
+
const label = normalizeLinkLabel(rawLabel === "" ? text.slice(i + 1, j) : rawLabel);
|
|
542
|
+
const dest = label && defs.get(label);
|
|
543
|
+
if (dest !== undefined) {
|
|
544
|
+
out.push({ name: text.slice(i + 1, j), target: decodeDestination(dest), at: i });
|
|
545
|
+
i = m;
|
|
546
|
+
continue;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
else {
|
|
551
|
+
// A shortcut reference, "[text]" alone with neither "(" nor "[" right after it, resolves
|
|
552
|
+
// against a definition using the link text itself as the label (VN2, round 3 Tek-542).
|
|
553
|
+
// Checked last, and only here, so it never overrides the full or collapsed reference
|
|
554
|
+
// forms above. Not tried when a ":" follows: that is the definition's own "[label]:"
|
|
555
|
+
// syntax, not a use of the label, and a definition line must not resolve against itself.
|
|
556
|
+
const label = text[j + 1] === ":" ? "" : normalizeLinkLabel(text.slice(i + 1, j));
|
|
557
|
+
const dest = label ? defs.get(label) : undefined;
|
|
558
|
+
if (dest !== undefined) {
|
|
559
|
+
out.push({ name: text.slice(i + 1, j), target: decodeDestination(dest), at: i });
|
|
560
|
+
i = j;
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
i = j;
|
|
565
|
+
}
|
|
566
|
+
return out;
|
|
567
|
+
}
|
|
568
|
+
// CommonMark fenced code blocks, marked line by line. A "## " or a link inside a fence is
|
|
569
|
+
// example text and not the file's own structure, but until 2026-09-10 both counted, so a
|
|
570
|
+
// file whose only section and only link lived inside ``` or ~~~ was reported valid. The
|
|
571
|
+
// scan is one pass over the lines with no backtracking pattern, because the ReDoS repair
|
|
572
|
+
// of 2026-08-24 bought a worse accuracy defect with a bounded quantifier and the rule out
|
|
573
|
+
// of it was to scan by index instead (mds/gotchas.md 2026-08-24 (jatko 12)).
|
|
574
|
+
// Returns one boolean per line: true for a fence line and for everything inside it.
|
|
575
|
+
function fenceMask(lines) {
|
|
576
|
+
const mask = new Array(lines.length).fill(false);
|
|
577
|
+
let fenceChar = "", fenceLen = 0;
|
|
578
|
+
for (let i = 0; i < lines.length; i++) {
|
|
579
|
+
const l = lines[i];
|
|
580
|
+
let p = 0;
|
|
581
|
+
while (p < 4 && l[p] === " ")
|
|
582
|
+
p++;
|
|
583
|
+
if (p > 3) {
|
|
584
|
+
mask[i] = fenceChar !== "";
|
|
585
|
+
continue;
|
|
586
|
+
}
|
|
587
|
+
const c = l[p];
|
|
588
|
+
let run = 0;
|
|
589
|
+
if (c === "`" || c === "~") {
|
|
590
|
+
while (l[p + run] === c)
|
|
591
|
+
run++;
|
|
592
|
+
}
|
|
593
|
+
if (fenceChar === "") {
|
|
594
|
+
// An opening backtick fence may not carry a backtick in its info string; a tilde
|
|
595
|
+
// fence may. Anything shorter than three markers is not a fence at all. A later
|
|
596
|
+
// round will read the consequence as a bug and it is not: in ```js `x` the info
|
|
597
|
+
// string holds a backtick, so that line is prose, and a bare ``` after it OPENS a
|
|
598
|
+
// block instead of closing one. CommonMark reads the same input the same way,
|
|
599
|
+
// measured against the spec 2026-09-10. Leave it.
|
|
600
|
+
if (run >= 3 && (c !== "`" || l.indexOf("`", p + run) === -1)) {
|
|
601
|
+
fenceChar = c;
|
|
602
|
+
fenceLen = run;
|
|
603
|
+
mask[i] = true;
|
|
604
|
+
}
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
607
|
+
mask[i] = true;
|
|
608
|
+
if (c === fenceChar && run >= fenceLen && l.slice(p + run).trim() === "") {
|
|
609
|
+
fenceChar = "";
|
|
610
|
+
fenceLen = 0;
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
return mask;
|
|
614
|
+
}
|
|
615
|
+
// A markdown list item that carries a link. A bullet or an ordered marker (1. or 1)) followed by
|
|
616
|
+
// a space or a tab opens the item; an ordered list and a tab after the marker were not read as
|
|
617
|
+
// list items until 2026-09-27 (Tek-496). Whether the line carries a link is decided by the
|
|
618
|
+
// caller against the whole document's own collectLinks result (hasLink), not by re-scanning
|
|
619
|
+
// this one line in isolation: a code span or a reference link target can resolve only across
|
|
620
|
+
// more than one line, and a per-line rescan cannot see that (outside review 2026-09-28, V2).
|
|
621
|
+
function listItemHasLink(l, hasLink) {
|
|
622
|
+
return !!hasLink && /^ {0,3}(?:[-*+]|\d{1,9}[.)])[ \t]/.test(l);
|
|
623
|
+
}
|
|
624
|
+
// The destination of a link as the URL test should see it. CommonMark lets a backslash escape any
|
|
625
|
+
// ASCII punctuation character, so "https\://example.com" is the destination "https://example.com".
|
|
626
|
+
// Backslash escapes and character references are read in one pass over the source, so an escaped
|
|
627
|
+
// ampersand stays literal text and is not decoded a second time (Tek-560).
|
|
628
|
+
function decodeDestination(s) {
|
|
629
|
+
const escapeAt = /(\\[!-\/:-@\[-`{-~])/;
|
|
630
|
+
return String(s).split(escapeAt).map((part) => (part.length === 2 && part[0] === "\\" && /[!-\/:-@\[-`{-~]/.test(part[1]) ? part[1] : decodeAttributeValue(part))).join("");
|
|
631
|
+
}
|
|
632
|
+
// A setext H1: one text line indented at most three spaces whose next line is a run of "=" (also
|
|
633
|
+
// indented at most three spaces, trailing spaces allowed). Returns true for the pair. A line that
|
|
634
|
+
// starts a block of its own (heading, blockquote, list item, fence) is not paragraph text, so it
|
|
635
|
+
// never becomes a setext title. Only a one-line title is read; a title over several lines is not
|
|
636
|
+
// (Tek-560).
|
|
637
|
+
function isSetextText(line) {
|
|
638
|
+
if (!/^ {0,3}\S/.test(line))
|
|
639
|
+
return false;
|
|
640
|
+
return !/^ {0,3}(?:#|>|[-*+](?:[ \t]|$)|\d{1,9}[.)](?:[ \t]|$)|`{3,}|~{3,})/.test(line);
|
|
641
|
+
}
|
|
642
|
+
function isSetextH1(line, next) {
|
|
643
|
+
if (typeof next !== "string" || !/^ {0,3}=+[ \t]*$/.test(next))
|
|
644
|
+
return false;
|
|
645
|
+
return isSetextText(line);
|
|
646
|
+
}
|
|
647
|
+
// Lines that open a block of their own in CommonMark, so they are never setext text after the
|
|
648
|
+
// title: a thematic break, the start of an HTML block (an autolink such as <https://...> is not
|
|
649
|
+
// one, it stays paragraph text) and a link reference definition. Only the later-H1 reading skips
|
|
650
|
+
// them; the title reading keeps its Tek-560 predicate. QA 2 of Tek-564 measured "---", "<div>" and
|
|
651
|
+
// "[a]: url" over a run of "=" warning as a second H1 although 0.3.18 passed them.
|
|
652
|
+
const HTML_BLOCK_START = /^ {0,3}<(?:[A-Za-z][A-Za-z0-9-]*(?:[\s\/>]|$)|\/[A-Za-z][A-Za-z0-9-]*(?:[\s>]|$)|!--|\?|![A-Za-z]|!\[CDATA\[)/;
|
|
653
|
+
const LINK_REF_DEF = /^ {0,3}\[[^\]]+\]:/;
|
|
654
|
+
// HTML blocks of CommonMark types 1 to 5 end at a marker, not at a blank line. The end markers are plain substrings matched case-insensitively, as CommonMark defines them, and not a tag regexp (CodeQL js/bad-tag-filter): type 2 ends at "-->" only, never at "--!>".
|
|
655
|
+
// Entry is [start, ends]: </pre>, </script>, </style>, </textarea> after <pre> and kin, "-->" after "<!--", "?>" after "<?", "]]>" after "<![CDATA[", ">" after "<!X"; the marker is looked for after the start on the same line.
|
|
656
|
+
function hasRawEnd(s, ends) { const t = s.toLowerCase(); return ends.some((e) => t.includes(e)); }
|
|
657
|
+
const RAW_HTML_BLOCKS = [
|
|
658
|
+
[/^ {0,3}<(?:pre|script|style|textarea)(?:[ \t>]|$)/i, ['</pre>', '</script>', '</style>', '</textarea>']],
|
|
659
|
+
[/^ {0,3}<!--/, ['-->']],
|
|
660
|
+
[/^ {0,3}<\?/, ['?>']],
|
|
661
|
+
[/^ {0,3}<!\[CDATA\[/, [']]>']],
|
|
662
|
+
[/^ {0,3}<![A-Za-z]/, ['>']]
|
|
663
|
+
];
|
|
664
|
+
function startsOtherBlock(line) {
|
|
665
|
+
return /^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$/.test(line)
|
|
666
|
+
|| HTML_BLOCK_START.test(line)
|
|
667
|
+
|| LINK_REF_DEF.test(line);
|
|
668
|
+
}
|
|
669
|
+
// Setext H1 lines after the title (Tek-564). A second H1 written as one text line over a run of "="
|
|
670
|
+
// is out of place exactly like "# Second title" is, and ends the section before it the same way.
|
|
671
|
+
// The text line is read by isSetextH1, the predicate the first-title reading uses. It must start
|
|
672
|
+
// a paragraph: a line that follows other paragraph text, a list item or a blockquote without a
|
|
673
|
+
// blank line is a continuation line, so a title over several lines is not read as a heading, and
|
|
674
|
+
// text that starts with "#" is not an H1 (isSetextText). The underline is returned separately
|
|
675
|
+
// because it is not a heading and not content.
|
|
676
|
+
// An indented code block is not a paragraph (Tek-565, outside review W46): a line indented four
|
|
677
|
+
// columns or more that does not continue a paragraph is code, so the text after it starts a new
|
|
678
|
+
// paragraph and "Title" over "===" there is a second H1. A four-space line right after paragraph
|
|
679
|
+
// text stays a continuation line (the accepted Tek-564 limit). The exception is a list item: its
|
|
680
|
+
// blank lines and indented lines belong to the item as continuation paragraphs, not as code, so
|
|
681
|
+
// while a list item is open (listCtx) an indented line is read as before. The list context ends at
|
|
682
|
+
// a thematic break or an ATX heading, or at a line with no indent after a blank line. When unsure
|
|
683
|
+
// the function stays in the list context, which is the 0.3.19 reading. A block quote, an HTML block
|
|
684
|
+
// and a link reference definition keep the old reading too, until the next blank line, and an HTML
|
|
685
|
+
// block of type 1 to 5 (<pre>, a comment, <?, <!X, CDATA) until its end marker: an indented line
|
|
686
|
+
// there continues the quote paragraph or the HTML block, or follows a "=" line that is itself
|
|
687
|
+
// paragraph text, and is not code.
|
|
688
|
+
function laterSetextH1(lines, fenced) {
|
|
689
|
+
const text = new Array(lines.length).fill(false);
|
|
690
|
+
const under = new Array(lines.length).fill(false);
|
|
691
|
+
let open = false;
|
|
692
|
+
let listCtx = false;
|
|
693
|
+
let blankBefore = false;
|
|
694
|
+
let quoteCtx = false;
|
|
695
|
+
let htmlCtx = false;
|
|
696
|
+
let rawEnd = null;
|
|
697
|
+
let rawDone = false;
|
|
698
|
+
for (let i = 0; i < lines.length; i++) {
|
|
699
|
+
const l = lines[i];
|
|
700
|
+
if (rawDone) {
|
|
701
|
+
htmlCtx = false;
|
|
702
|
+
rawDone = false;
|
|
703
|
+
}
|
|
704
|
+
if (fenced[i]) {
|
|
705
|
+
open = false;
|
|
706
|
+
continue;
|
|
707
|
+
}
|
|
708
|
+
if (l.trim() === "") {
|
|
709
|
+
open = false;
|
|
710
|
+
blankBefore = true;
|
|
711
|
+
quoteCtx = false;
|
|
712
|
+
if (!rawEnd)
|
|
713
|
+
htmlCtx = false;
|
|
714
|
+
continue;
|
|
715
|
+
}
|
|
716
|
+
const indented = /^(?: {4}| {0,3}\t)/.test(l);
|
|
717
|
+
if (/^ ?#{1,6}(?:[ \t]|$)/.test(l) || /^ ?([-*_])(?:[ \t]*\1){2,}[ \t]*$/.test(l))
|
|
718
|
+
listCtx = false;
|
|
719
|
+
else if (/^ {0,3}(?:[-*+]|\d{1,9}[.)])(?:[ \t]|$)/.test(l))
|
|
720
|
+
listCtx = true;
|
|
721
|
+
else if (blankBefore && /^ ?[^ \t]/.test(l))
|
|
722
|
+
listCtx = false;
|
|
723
|
+
blankBefore = false;
|
|
724
|
+
if (/^ {0,3}>/.test(l))
|
|
725
|
+
quoteCtx = true;
|
|
726
|
+
if (rawEnd) {
|
|
727
|
+
if (hasRawEnd(l, rawEnd)) {
|
|
728
|
+
rawEnd = null;
|
|
729
|
+
rawDone = true;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
else {
|
|
733
|
+
if (HTML_BLOCK_START.test(l) || LINK_REF_DEF.test(l))
|
|
734
|
+
htmlCtx = true;
|
|
735
|
+
for (const [start, end] of RAW_HTML_BLOCKS) {
|
|
736
|
+
const m = start.exec(l);
|
|
737
|
+
if (m) {
|
|
738
|
+
if (!hasRawEnd(l.slice(m[0].length), end))
|
|
739
|
+
rawEnd = end;
|
|
740
|
+
break;
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
if (!open && !listCtx && !quoteCtx && !htmlCtx && indented)
|
|
745
|
+
continue;
|
|
746
|
+
if (!open && !fenced[i + 1] && !startsOtherBlock(l) && isSetextH1(l, lines[i + 1])) {
|
|
747
|
+
text[i] = true;
|
|
748
|
+
under[i + 1] = true;
|
|
749
|
+
i++;
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
752
|
+
if (open && /^ {0,3}=+[ \t]*$/.test(l)) {
|
|
753
|
+
open = false;
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
open = !(/^ {0,3}#{1,6}(?:[ \t]|$)/.test(l) || /^ {0,3}(?:-+|([-*_])(?: *\1){2,})[ \t]*$/.test(l));
|
|
757
|
+
}
|
|
758
|
+
return { text, under };
|
|
759
|
+
}
|
|
760
|
+
// Which lines are H2 headings, outside fences. One reader for every place that asks "is this an
|
|
761
|
+
// H2": the section count, the section membership and the place where the late-summary scan stops.
|
|
762
|
+
// An ATX H2 is "##" followed by a space or a tab (a bare "##" is not one, and "###" is deeper). A
|
|
763
|
+
// setext H2 is one line of paragraph text, read by the same predicate as the setext H1 above,
|
|
764
|
+
// that has a blank line or the start of the file before it and a run of hyphens directly after it.
|
|
765
|
+
// A list item above "---", a thematic break and a line inside a fence are not H2 text. The
|
|
766
|
+
// underline is returned separately because it is not content (Tek-562, outside review W30).
|
|
767
|
+
function h2Lines(lines, fenced) {
|
|
768
|
+
const h2 = new Array(lines.length).fill(false);
|
|
769
|
+
const under = new Array(lines.length).fill(false);
|
|
770
|
+
for (let i = 0; i < lines.length; i++) {
|
|
771
|
+
if (fenced[i])
|
|
772
|
+
continue;
|
|
773
|
+
const l = lines[i];
|
|
774
|
+
if (/^ {0,3}##[ \t]/.test(l)) {
|
|
775
|
+
h2[i] = true;
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
const next = lines[i + 1];
|
|
779
|
+
if (typeof next !== "string" || fenced[i + 1] || !/^ {0,3}-+[ \t]*$/.test(next))
|
|
780
|
+
continue;
|
|
781
|
+
if (!isSetextText(l) || /^ {0,3}([-*_])(?: *\1){2,} *$/.test(l))
|
|
782
|
+
continue;
|
|
783
|
+
if (i > 0 && lines[i - 1].trim() !== "")
|
|
784
|
+
continue;
|
|
785
|
+
h2[i] = true;
|
|
786
|
+
under[i + 1] = true;
|
|
787
|
+
}
|
|
788
|
+
return { h2, under };
|
|
789
|
+
}
|
|
790
|
+
// The text of an ATX H1, or null when the line is not one. One to three leading spaces, a
|
|
791
|
+
// single "#", then a space, a tab or the end of the line. A closing run of "#" preceded by a
|
|
792
|
+
// space or a tab is not part of the title, and the title is what remains once both ends are
|
|
793
|
+
// trimmed. Until 2026-09-27 the check demanded exactly one space and a visible character after
|
|
794
|
+
// it, so "# Example" and "#\tExample" failed and "# ###", a heading with no text, passed
|
|
795
|
+
// (Tek-496). The closing run is read by index, not by a pattern, so a long line of
|
|
796
|
+
// spaces and "#" stays linear.
|
|
797
|
+
function h1Title(line) {
|
|
798
|
+
const m = /^ {0,3}#(?=[ \t]|$)/.exec(line);
|
|
799
|
+
if (!m)
|
|
800
|
+
return null;
|
|
801
|
+
const s = line.slice(m[0].length);
|
|
802
|
+
let end = s.length;
|
|
803
|
+
while (end > 0 && (s[end - 1] === " " || s[end - 1] === "\t"))
|
|
804
|
+
end--;
|
|
805
|
+
let h = end;
|
|
806
|
+
while (h > 0 && s[h - 1] === "#")
|
|
807
|
+
h--;
|
|
808
|
+
if (h < end && (h === 0 || s[h - 1] === " " || s[h - 1] === "\t"))
|
|
809
|
+
end = h;
|
|
810
|
+
let start = 0;
|
|
811
|
+
while (start < end && (s[start] === " " || s[start] === "\t"))
|
|
812
|
+
start++;
|
|
813
|
+
while (end > start && (s[end - 1] === " " || s[end - 1] === "\t"))
|
|
814
|
+
end--;
|
|
815
|
+
return s.slice(start, end);
|
|
816
|
+
}
|
|
817
|
+
// An absolute link target an agent can follow: it starts with http:// or https:// and the URL
|
|
818
|
+
// parser reads it with a host. Until 2026-09-27 a pattern accepted anything after the scheme,
|
|
819
|
+
// so https://% and https://example.com:99999/ passed as absolute URLs although no client can
|
|
820
|
+
// open them (Tek-496).
|
|
821
|
+
function isAbsoluteHttpUrl(t) {
|
|
822
|
+
if (!/^https?:\/\//i.test(t))
|
|
823
|
+
return false;
|
|
824
|
+
let u;
|
|
825
|
+
try {
|
|
826
|
+
u = new URL(t);
|
|
827
|
+
}
|
|
828
|
+
catch {
|
|
829
|
+
return false;
|
|
830
|
+
}
|
|
831
|
+
return (u.protocol === "http:" || u.protocol === "https:") && u.hostname !== "";
|
|
832
|
+
}
|
|
833
|
+
function redirectFailDetail(f) {
|
|
834
|
+
if (f.reason === "off-host")
|
|
835
|
+
return "redirects to " + f.location + ", a different host; llms.txt is host-scoped, so validate that host directly";
|
|
836
|
+
if (f.reason === "unsafe-target")
|
|
837
|
+
return "redirects to an unsupported target (" + f.location + "); only https redirects to the same site are followed";
|
|
838
|
+
if (f.reason === "too-many")
|
|
839
|
+
return "too many redirects; the llms.txt is not served at a stable URL";
|
|
840
|
+
return "got a " + f.status + " redirect without a usable Location header";
|
|
841
|
+
}
|
|
842
|
+
export function validateLlmsTxt(f) {
|
|
843
|
+
const checks = [];
|
|
844
|
+
const add = (id, status, label, detail) => checks.push({ id, status, label, detail });
|
|
845
|
+
if (f.redirect) {
|
|
846
|
+
add("http-status", "fail", "File exists at /llms.txt", redirectFailDetail(f));
|
|
847
|
+
return checks;
|
|
848
|
+
}
|
|
849
|
+
if (f.status !== 200) {
|
|
850
|
+
add("http-status", "fail", "File exists at /llms.txt", "expected HTTP 200, got " + f.status);
|
|
851
|
+
return checks;
|
|
852
|
+
}
|
|
853
|
+
// An accepted redirect is shown masked as well (round 19, K7-2, mirrored from worker.js
|
|
854
|
+
// v3.164.0): a query value or a fragment in a same-host Location used to reach this
|
|
855
|
+
// detail as sent.
|
|
856
|
+
add("http-status", "pass", "File exists at /llms.txt", f.redirectedFrom ? "HTTP 200, followed a redirect from " + maskLocation(f.redirectedFrom) + " to " + maskLocation(f.finalUrl) : "HTTP 200");
|
|
857
|
+
// A leading byte-order mark is stripped the same way the fetch path already strips it: the
|
|
858
|
+
// TextDecoder used there removes a BOM by default, so the direct API took a different
|
|
859
|
+
// document than the fetched one for the exact same bytes until this was measured (outside
|
|
860
|
+
// review 2026-09-28); the h1-title check saw the mark as part of the first line and failed a
|
|
861
|
+
// document that would pass when fetched.
|
|
862
|
+
// CR-only and CRLF line endings become LF here (Tek-560): a file written with bare CR was one
|
|
863
|
+
// line for the old split and read as a title with the whole document in it.
|
|
864
|
+
const text = (f.text.charCodeAt(0) === 0xFEFF ? f.text.slice(1) : f.text).replace(/\r\n?/g, "\n");
|
|
865
|
+
const ct = (f.contentType || "").toLowerCase();
|
|
866
|
+
const looksHtml = /^\s*(<!doctype|<html|<head|<body)/i.test(text);
|
|
867
|
+
if (looksHtml) {
|
|
868
|
+
add("content-type", "fail", "Response is plain text", "the body looks like an HTML page, not an llms.txt file");
|
|
869
|
+
return checks;
|
|
870
|
+
}
|
|
871
|
+
// The media type is compared whole, without its parameters. A substring test passed
|
|
872
|
+
// application/x-text/plain and text/markdownish until 2026-09-27 (Tek-496).
|
|
873
|
+
const mediaType = ct.split(";")[0].trim();
|
|
874
|
+
if (mediaType === "text/plain" || mediaType === "text/markdown") {
|
|
875
|
+
add("content-type", "pass", "Response is plain text", mediaType);
|
|
876
|
+
}
|
|
877
|
+
else {
|
|
878
|
+
add("content-type", "warn", "Response is plain text", "content-type is " + (ct.split(";")[0] || "missing") + ", text/plain or text/markdown is the convention");
|
|
879
|
+
}
|
|
880
|
+
const lines = text.split("\n");
|
|
881
|
+
const firstIdx = lines.findIndex((l) => l.trim() !== "");
|
|
882
|
+
const firstRaw = firstIdx === -1 ? "" : lines[firstIdx];
|
|
883
|
+
const first = firstRaw.trim();
|
|
884
|
+
// The line is read as markdown and not trimmed first. Four spaces or a tab make it an
|
|
885
|
+
// indented code block rather than a heading, and trimming erased that difference, so
|
|
886
|
+
// " # Site" passed as the H1 until 2026-08-29. CommonMark allows three spaces.
|
|
887
|
+
// h1Title reads the heading the way CommonMark does, its closing run included (Tek-496).
|
|
888
|
+
const title = h1Title(firstRaw);
|
|
889
|
+
// A setext H1 (a text line, then a line of "=") is an H1 as well (Tek-560). The summary and the
|
|
890
|
+
// later scans start after its underline.
|
|
891
|
+
const setext = title === null && firstIdx !== -1 && isSetextH1(firstRaw, lines[firstIdx + 1]);
|
|
892
|
+
const h1End = setext ? firstIdx + 1 : firstIdx;
|
|
893
|
+
if (title || setext) {
|
|
894
|
+
add("h1-title", "pass", "Starts with an H1 title", JSON.stringify(cut(first, 80)));
|
|
895
|
+
}
|
|
896
|
+
else if (title === "") {
|
|
897
|
+
add("h1-title", "fail", "Starts with an H1 title", "the H1 has no text; the format requires the project name there");
|
|
898
|
+
}
|
|
899
|
+
else {
|
|
900
|
+
add("h1-title", "fail", "Starts with an H1 title", "the first non-empty line should be a markdown H1 (# Site name, or one line of text underlined with =)");
|
|
901
|
+
}
|
|
902
|
+
const afterIdx = lines.findIndex((l, i) => i > h1End && l.trim() !== "");
|
|
903
|
+
const afterH1 = afterIdx === -1 ? "" : lines[afterIdx];
|
|
904
|
+
// Read the same way as the H1 check and the "late" blockquote search below: at most three
|
|
905
|
+
// leading spaces, per CommonMark. A trim-first prefix test used to erase the difference, so
|
|
906
|
+
// a four-space or tab-indented code block right after the title passed as the summary until
|
|
907
|
+
// 2026-09-27 (Tek-496, mirrored from worker.js).
|
|
908
|
+
// A blockquote marker is 0-3 leading spaces plus ">", optionally followed by one space that
|
|
909
|
+
// is then part of the marker and not the content; CommonMark reads ">Summary" and a tab the
|
|
910
|
+
// same way, and four or more leading spaces stay indented code, not a blockquote (outside
|
|
911
|
+
// review 2026-09-28: a literal space was required, so a tab or no space at all warned as
|
|
912
|
+
// missing).
|
|
913
|
+
// The blockquote directly under the title is one block of consecutive ">" lines, and it is the
|
|
914
|
+
// summary when any line of it has text (Tek-560). ">" followed by "> Real summary" passes, and a
|
|
915
|
+
// block whose every line is empty (">" alone, or ">" on several lines) is an empty blockquote and
|
|
916
|
+
// warns. A nested ">>" or "> >" line counts as text, as it did in 0.3.15.
|
|
917
|
+
let summaryLine = "";
|
|
918
|
+
if (/^ {0,3}>/.test(afterH1)) {
|
|
919
|
+
for (let i = afterIdx; i < lines.length && /^ {0,3}>/.test(lines[i]); i++) {
|
|
920
|
+
if (/^ {0,3}>[ \t]*\S/.test(lines[i])) {
|
|
921
|
+
summaryLine = lines[i];
|
|
922
|
+
break;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
if (summaryLine !== "") {
|
|
927
|
+
add("summary", "pass", "Blockquote summary after the title", JSON.stringify(cut(summaryLine.trim(), 80)));
|
|
928
|
+
}
|
|
929
|
+
else {
|
|
930
|
+
// A blockquote further down, before the first H2, is a summary in the wrong place and not a
|
|
931
|
+
// missing one, and the detail says which (outside retest 2026-09-26, mirrored from worker.js
|
|
932
|
+
// v3.176.0). The status is warn either way, because the format places the summary directly
|
|
933
|
+
// after the title.
|
|
934
|
+
const fence = fenceMask(lines);
|
|
935
|
+
const lateH2 = h2Lines(lines, fence).h2;
|
|
936
|
+
let late = -1;
|
|
937
|
+
for (let i = h1End + 1; i < lines.length; i++) {
|
|
938
|
+
if (fence[i])
|
|
939
|
+
continue;
|
|
940
|
+
if (lateH2[i])
|
|
941
|
+
break;
|
|
942
|
+
if (/^ {0,3}>[ \t]*\S/.test(lines[i])) {
|
|
943
|
+
late = i;
|
|
944
|
+
break;
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
add("summary", "warn", "Blockquote summary after the title", late === -1
|
|
948
|
+
? "recommended by the format (> one-line summary), not required"
|
|
949
|
+
: "the blockquote at line " + (late + 1) + " comes after other text, and the format places the summary directly after the title");
|
|
950
|
+
}
|
|
951
|
+
// Headings are read outside fences only, and with the same indentation the H1 check and
|
|
952
|
+
// listItemHasLink have allowed since 2026-08-29. Until 2026-09-10 this one line still
|
|
953
|
+
// demanded column zero, so a correct file indented by one to three spaces was reported as
|
|
954
|
+
// having no sections at all while its list under the same indentation counted fine.
|
|
955
|
+
const fenced = fenceMask(lines);
|
|
956
|
+
// Computed once for both the section-list scan below and the links check further down, so a
|
|
957
|
+
// reference link definition written anywhere in the prose resolves the same way for each
|
|
958
|
+
// (outside review 2026-09-28, V2): "[text][label]" used to be recognized in the whole-document
|
|
959
|
+
// links check but not in the per-line section scan, which only ever saw one list item line.
|
|
960
|
+
// Fenced lines are blanked rather than removed (VREG1, round 3 Tek-542): deleting them and
|
|
961
|
+
// joining the survivors let a link's own target span straight across a fenced code block, as
|
|
962
|
+
// if the fence had never separated the two halves.
|
|
963
|
+
const proseText = lines.map((l, i) => (fenced[i] ? "" : l)).join("\n");
|
|
964
|
+
const linkDefs = collectLinkDefinitions(proseText);
|
|
965
|
+
const proseLinks = collectLinks(proseText, linkDefs);
|
|
966
|
+
// Which original line each recognized link starts on, so the section scan below can ask "does
|
|
967
|
+
// this line carry a link" against the same whole-document result the links check itself uses,
|
|
968
|
+
// instead of re-finding links one isolated line at a time (outside review 2026-09-28, V2).
|
|
969
|
+
// proseLinks is already ordered by position, so the line cursor only moves forward across the
|
|
970
|
+
// whole list instead of restarting the scan from line zero for every link, which used to make
|
|
971
|
+
// a many-link document's mapping step quadratic (VREG3/VN3, round 3 Tek-542). Each proseText
|
|
972
|
+
// line now sits at the same index as its source line, because fenced lines are blanked, not
|
|
973
|
+
// removed, so the index read off the start-offset table names the original line directly.
|
|
974
|
+
const linkedLineIdx = new Set();
|
|
975
|
+
{
|
|
976
|
+
const starts = [];
|
|
977
|
+
let offset = 0;
|
|
978
|
+
for (const pl of proseText.split("\n")) {
|
|
979
|
+
starts.push(offset);
|
|
980
|
+
offset += pl.length + 1;
|
|
981
|
+
}
|
|
982
|
+
let idx = 0;
|
|
983
|
+
for (const link of proseLinks) {
|
|
984
|
+
while (idx + 1 < starts.length && starts[idx + 1] <= link.at)
|
|
985
|
+
idx++;
|
|
986
|
+
linkedLineIdx.add(idx);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
const hm = h2Lines(lines, fenced);
|
|
990
|
+
const sx = laterSetextH1(lines, fenced);
|
|
991
|
+
const h2Count = hm.h2.filter(Boolean).length;
|
|
992
|
+
// A section counts when it carries a file list. An H2 followed by a paragraph satisfied
|
|
993
|
+
// this check until 2026-08-29, and the format puts each section's links in a list.
|
|
994
|
+
// A heading between the title and the first H2, or a second H1 anywhere, is out of place,
|
|
995
|
+
// because the format has one H1 and allows only paragraphs and lists before the file lists.
|
|
996
|
+
// Both passed until 2026-09-27 (Tek-496). The first one is named in the detail.
|
|
997
|
+
let sectionsWithList = 0, misplaced = -1;
|
|
998
|
+
{
|
|
999
|
+
let inSection = false, counted = false, seenH2 = false, itemOpen = false, itemGap = false, itemEmpty = false, itemCol = 0, afterFence = false;
|
|
1000
|
+
// Width of the leading white space of a line in columns, a tab running to the next multiple of 4.
|
|
1001
|
+
const indentWidth = (s) => { let c = 0; for (const ch of s) {
|
|
1002
|
+
if (ch === " ")
|
|
1003
|
+
c++;
|
|
1004
|
+
else if (ch === "\t")
|
|
1005
|
+
c += 4 - (c % 4);
|
|
1006
|
+
else
|
|
1007
|
+
break;
|
|
1008
|
+
} return c; };
|
|
1009
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1010
|
+
const l = lines[i];
|
|
1011
|
+
// A code fence indented less than the item's content column closes the open item, so a link
|
|
1012
|
+
// after it is not that item's link. The content column is where the item's text starts, 2 for
|
|
1013
|
+
// "- item" and 4 for "- item". A fence indented to that column is inside the item and keeps
|
|
1014
|
+
// it open, as in CommonMark, but the line after it then counts only when it is indented to
|
|
1015
|
+
// that column too, because a lazy continuation cannot follow a fence.
|
|
1016
|
+
if (fenced[i]) {
|
|
1017
|
+
if (l.trim() !== "" && indentWidth(l) < itemCol)
|
|
1018
|
+
itemOpen = false;
|
|
1019
|
+
if (itemOpen)
|
|
1020
|
+
afterFence = true;
|
|
1021
|
+
continue;
|
|
1022
|
+
}
|
|
1023
|
+
if (hm.under[i] || sx.under[i])
|
|
1024
|
+
continue;
|
|
1025
|
+
if (hm.h2[i]) {
|
|
1026
|
+
inSection = true;
|
|
1027
|
+
counted = false;
|
|
1028
|
+
seenH2 = true;
|
|
1029
|
+
itemOpen = false;
|
|
1030
|
+
continue;
|
|
1031
|
+
}
|
|
1032
|
+
const h1 = /^ {0,3}#(?:[ \t]|$)/.test(l) || sx.text[i];
|
|
1033
|
+
if (misplaced === -1 && i > firstIdx && (h1 || (!seenH2 && /^ {0,3}#{3,6}(?:[ \t]|$)/.test(l))))
|
|
1034
|
+
misplaced = i;
|
|
1035
|
+
if (h1) {
|
|
1036
|
+
inSection = false;
|
|
1037
|
+
itemOpen = false;
|
|
1038
|
+
continue;
|
|
1039
|
+
}
|
|
1040
|
+
// A link on a line that continues an open list item counts as that item's link (Tek-560):
|
|
1041
|
+
// a marker alone on its line, a link on the next line, or a lazy continuation line. A blank
|
|
1042
|
+
// line keeps the item open only for a line indented to the item's content column after it. A
|
|
1043
|
+
// heading, a blockquote or a thematic break ends the item. An empty marker takes no lazy
|
|
1044
|
+
// continuation (CommonMark): the next line counts only when indented to the content column,
|
|
1045
|
+
// and a blank line right after the empty marker closes the item.
|
|
1046
|
+
if (l.trim() === "") {
|
|
1047
|
+
if (itemEmpty)
|
|
1048
|
+
itemOpen = false;
|
|
1049
|
+
itemGap = itemOpen;
|
|
1050
|
+
continue;
|
|
1051
|
+
}
|
|
1052
|
+
const fromFence = afterFence;
|
|
1053
|
+
afterFence = false;
|
|
1054
|
+
if (/^ {0,3}(?:[-*+]|\d{1,9}[.)])(?:[ \t]|$)/.test(l)) {
|
|
1055
|
+
itemOpen = true;
|
|
1056
|
+
itemGap = false;
|
|
1057
|
+
itemEmpty = /^ {0,3}(?:[-*+]|\d{1,9}[.)])[ \t]*$/.test(l);
|
|
1058
|
+
{
|
|
1059
|
+
// Content column: where the text after the marker starts when one to four spaces
|
|
1060
|
+
// separate them, and the column after the marker when the line is empty or five or
|
|
1061
|
+
// more spaces follow, which makes the rest indented code.
|
|
1062
|
+
const mk = /^( {0,3})([-*+]|\d{1,9}[.)])([ \t]*)/.exec(l);
|
|
1063
|
+
const markerEnd = mk[1].length + mk[2].length;
|
|
1064
|
+
let c = markerEnd;
|
|
1065
|
+
for (const ch of mk[3])
|
|
1066
|
+
c += ch === "\t" ? 4 - (c % 4) : 1;
|
|
1067
|
+
itemCol = (mk[0].length === l.length || c - markerEnd > 4) ? markerEnd + 1 : c;
|
|
1068
|
+
}
|
|
1069
|
+
if (inSection && !counted && listItemHasLink(l, linkedLineIdx.has(i))) {
|
|
1070
|
+
sectionsWithList++;
|
|
1071
|
+
counted = true;
|
|
1072
|
+
}
|
|
1073
|
+
continue;
|
|
1074
|
+
}
|
|
1075
|
+
if (/^ {0,3}(?:#{1,6}(?:[ \t]|$)|>|([-*_])(?: *\1){2,} *$)/.test(l)) {
|
|
1076
|
+
itemOpen = false;
|
|
1077
|
+
continue;
|
|
1078
|
+
}
|
|
1079
|
+
const indented = indentWidth(l) >= itemCol;
|
|
1080
|
+
if (itemOpen && (!itemGap || indented) && (!itemEmpty || indented) && (!fromFence || indented)) {
|
|
1081
|
+
if (inSection && !counted && linkedLineIdx.has(i)) {
|
|
1082
|
+
sectionsWithList++;
|
|
1083
|
+
counted = true;
|
|
1084
|
+
}
|
|
1085
|
+
itemGap = false;
|
|
1086
|
+
itemEmpty = false;
|
|
1087
|
+
}
|
|
1088
|
+
else {
|
|
1089
|
+
itemOpen = false;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
if (h2Count > 0 && sectionsWithList > 0) {
|
|
1094
|
+
add("sections", "pass", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s") + ", " + sectionsWithList + " carrying a file list");
|
|
1095
|
+
}
|
|
1096
|
+
else if (h2Count > 0) {
|
|
1097
|
+
add("sections", "warn", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s") + " but no file list under any of them; the format puts a section's links in a markdown list");
|
|
1098
|
+
}
|
|
1099
|
+
else {
|
|
1100
|
+
add("sections", "warn", "H2 sections group the content", "no H2 sections found; sections are the convention for grouping links");
|
|
1101
|
+
}
|
|
1102
|
+
if (misplaced !== -1) {
|
|
1103
|
+
const s = checks[checks.length - 1];
|
|
1104
|
+
s.status = "warn";
|
|
1105
|
+
s.detail += "; the heading at line " + (misplaced + 1) + " is out of place, because the format has one H1 and no other heading before the first H2";
|
|
1106
|
+
}
|
|
1107
|
+
// Links are collected from the prose only, for the same reason the headings are: a link
|
|
1108
|
+
// shown inside a code fence is an example of a link, not one an agent can follow. Computed
|
|
1109
|
+
// once above as proseLinks, reused here under its usual name.
|
|
1110
|
+
const links = proseLinks;
|
|
1111
|
+
// An entry an agent can use has a name and a target with a host. An empty name and a
|
|
1112
|
+
// bare "https://" both counted as valid absolute links until 2026-08-29.
|
|
1113
|
+
const named = links.filter((m) => m.name.trim() !== "");
|
|
1114
|
+
const unnamed = links.length - named.length;
|
|
1115
|
+
const absolute = named.filter((m) => isAbsoluteHttpUrl(m.target)).length;
|
|
1116
|
+
if (links.length === 0) {
|
|
1117
|
+
add("links", "warn", "Markdown links an agent can follow", "no markdown links found");
|
|
1118
|
+
}
|
|
1119
|
+
else if (unnamed > 0) {
|
|
1120
|
+
add("links", "warn", "Markdown links an agent can follow", links.length + " link" + (links.length === 1 ? "" : "s") + ", " + unnamed + " with an empty link name; an entry needs a name an agent can show");
|
|
1121
|
+
}
|
|
1122
|
+
else if (absolute === named.length) {
|
|
1123
|
+
add("links", "pass", "Markdown links an agent can follow", named.length + " link" + (named.length === 1 ? "" : "s") + ", all absolute URLs");
|
|
1124
|
+
}
|
|
1125
|
+
else {
|
|
1126
|
+
const relativeCount = named.filter((m) => !/^[a-z][a-z0-9+.-]*:/i.test(m.target)).length;
|
|
1127
|
+
const hostless = named.length - absolute - relativeCount;
|
|
1128
|
+
add("links", "warn", "Markdown links an agent can follow", named.length + " links, " + relativeCount + " relative" + (hostless > 0 ? " and " + hostless + " that are not valid http or https URLs" : "") + "; absolute URLs travel better when the file is read out of context");
|
|
1129
|
+
}
|
|
1130
|
+
if (f.truncated) {
|
|
1131
|
+
add("size", "warn", "Small enough to be cheap to read", "over 256 KB, read truncated");
|
|
1132
|
+
}
|
|
1133
|
+
else if (f.bytes <= 51200) {
|
|
1134
|
+
add("size", "pass", "Small enough to be cheap to read", f.bytes + " bytes");
|
|
1135
|
+
}
|
|
1136
|
+
else {
|
|
1137
|
+
add("size", "warn", "Small enough to be cheap to read", f.bytes + " bytes; consider moving detail to llms-full.txt");
|
|
1138
|
+
}
|
|
1139
|
+
// A tag needs its closing ">" (Tek-560): an opening tag with well formed attributes, a closing tag
|
|
1140
|
+
// or a comment start. "a <b + c" is prose and no longer warns. An attribute name is any run of
|
|
1141
|
+
// characters other than white space, quotes, "<", ">", "/" and "=" that holds a Unicode letter,
|
|
1142
|
+
// a Unicode digit or an underscore, so "@click", "*ngIf", ":class" and "ä" count, while a bare
|
|
1143
|
+
// operator such as the "+" in "a <b + c > d" does not. The tag name itself is ASCII. An unquoted
|
|
1144
|
+
// value may not start with a quote, which keeps the quoted and the unquoted form from matching
|
|
1145
|
+
// the same text, so the scan stays linear. A tag still open at the end of the file, as in
|
|
1146
|
+
// '<div class="x"', has no closing ">" and does not warn.
|
|
1147
|
+
if (/<[a-zA-Z][a-zA-Z0-9-]*(?:\s+[^\s"'<>\/=\p{L}\p{N}_]*[\p{L}\p{N}_][^\s"'<>\/=]*(?:\s*=\s*(?:"[^"]*"|'[^']*'|(?!["'])[^\s<>]+))?)*\s*\/?>|<\/[a-zA-Z][a-zA-Z0-9-]*\s*>|<!--/u.test(text)) {
|
|
1148
|
+
add("no-html", "warn", "No HTML markup in the file", "HTML tags found; llms.txt should be plain markdown");
|
|
1149
|
+
}
|
|
1150
|
+
else {
|
|
1151
|
+
add("no-html", "pass", "No HTML markup in the file", "plain markdown");
|
|
1152
|
+
}
|
|
1153
|
+
return checks;
|
|
1154
|
+
}
|
|
1155
|
+
// A "<" starts a tag only when a letter, "!", "/" or "?" follows it. Anything else is
|
|
1156
|
+
// text, and the NEXT "<" can still start a tag. Measured, not assumed: without this rule
|
|
1157
|
+
// "<<style><link rel=describedby ...>" read the link as published while a real parser
|
|
1158
|
+
// treats the first "<" as text, opens style, and publishes nothing.
|
|
1159
|
+
function startsTag(c) {
|
|
1160
|
+
return c !== undefined && (c === "!" || c === "/" || c === "?" || (c >= "a" && c <= "z"));
|
|
1161
|
+
}
|
|
1162
|
+
// One tag read with the tokenizer's own states, from the "<" at lt: the tag name, then before
|
|
1163
|
+
// attribute name, attribute name, after attribute name, before attribute value, the three value
|
|
1164
|
+
// states, after a quoted value and self-closing start tag. Returns the index of the ">" that
|
|
1165
|
+
// ends the tag, or -1 when the input ends first, in which case a parser drops the tag. When attrs
|
|
1166
|
+
// is an object it also receives the attributes: names ASCII lowercased, values with their
|
|
1167
|
+
// character references decoded, and the first declaration of a name wins.
|
|
1168
|
+
// indexOf(">") is wrong here: <link data-x="a>b" rel="describedby"> is one tag for a parser and
|
|
1169
|
+
// two for indexOf, and the relation was lost. The shortcut that replaced it opened a quote after
|
|
1170
|
+
// any "=", which the tokenizer does not do either: in a="x"=">" the second "=" starts an attribute
|
|
1171
|
+
// NAME, and so does an "=" where a name is expected, so the quote after it is part of that name
|
|
1172
|
+
// and the ">" inside it ends the tag. Measured against parse5, 2026-09-12.
|
|
1173
|
+
function readTag(text, lt, attrs) {
|
|
1174
|
+
const n = text.length;
|
|
1175
|
+
let j = lt + 1;
|
|
1176
|
+
if (text[j] === "/")
|
|
1177
|
+
j++;
|
|
1178
|
+
while (j < n && !isTagBoundary(text[j]))
|
|
1179
|
+
j++;
|
|
1180
|
+
let state = "before", name = "", start = 0, quote = "";
|
|
1181
|
+
const add = (value) => {
|
|
1182
|
+
if (attrs && !(name in attrs))
|
|
1183
|
+
attrs[name] = decodeAttributeValue(value);
|
|
1184
|
+
};
|
|
1185
|
+
for (; j < n; j++) {
|
|
1186
|
+
const c = text[j];
|
|
1187
|
+
if (state === "before") {
|
|
1188
|
+
if (HTML_WHITESPACE.includes(c))
|
|
1189
|
+
continue;
|
|
1190
|
+
if (c === "/") {
|
|
1191
|
+
state = "selfClosing";
|
|
1192
|
+
continue;
|
|
1193
|
+
}
|
|
1194
|
+
if (c === ">")
|
|
1195
|
+
return j;
|
|
1196
|
+
// Any other character starts a name, "=" included.
|
|
1197
|
+
start = j;
|
|
1198
|
+
state = "name";
|
|
1199
|
+
}
|
|
1200
|
+
else if (state === "name") {
|
|
1201
|
+
if (HTML_WHITESPACE.includes(c)) {
|
|
1202
|
+
name = asciiLower(text.slice(start, j));
|
|
1203
|
+
state = "afterName";
|
|
1204
|
+
continue;
|
|
1205
|
+
}
|
|
1206
|
+
if (c === "/" || c === ">") {
|
|
1207
|
+
name = asciiLower(text.slice(start, j));
|
|
1208
|
+
add("");
|
|
1209
|
+
if (c === ">")
|
|
1210
|
+
return j;
|
|
1211
|
+
state = "selfClosing";
|
|
1212
|
+
continue;
|
|
1213
|
+
}
|
|
1214
|
+
if (c === "=" && j > start) {
|
|
1215
|
+
name = asciiLower(text.slice(start, j));
|
|
1216
|
+
state = "beforeValue";
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
else if (state === "afterName") {
|
|
1220
|
+
if (HTML_WHITESPACE.includes(c))
|
|
1221
|
+
continue;
|
|
1222
|
+
if (c === "=") {
|
|
1223
|
+
state = "beforeValue";
|
|
1224
|
+
continue;
|
|
1225
|
+
}
|
|
1226
|
+
add("");
|
|
1227
|
+
if (c === "/") {
|
|
1228
|
+
state = "selfClosing";
|
|
1229
|
+
continue;
|
|
1230
|
+
}
|
|
1231
|
+
if (c === ">")
|
|
1232
|
+
return j;
|
|
1233
|
+
start = j;
|
|
1234
|
+
state = "name";
|
|
1235
|
+
}
|
|
1236
|
+
else if (state === "beforeValue") {
|
|
1237
|
+
if (HTML_WHITESPACE.includes(c))
|
|
1238
|
+
continue;
|
|
1239
|
+
if (c === '"' || c === "'") {
|
|
1240
|
+
quote = c;
|
|
1241
|
+
start = j + 1;
|
|
1242
|
+
state = "quoted";
|
|
1243
|
+
continue;
|
|
1244
|
+
}
|
|
1245
|
+
if (c === ">") {
|
|
1246
|
+
add("");
|
|
1247
|
+
return j;
|
|
1248
|
+
}
|
|
1249
|
+
start = j;
|
|
1250
|
+
state = "unquoted";
|
|
1251
|
+
}
|
|
1252
|
+
else if (state === "quoted") {
|
|
1253
|
+
if (c === quote) {
|
|
1254
|
+
add(text.slice(start, j));
|
|
1255
|
+
state = "afterQuoted";
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
else if (state === "unquoted") {
|
|
1259
|
+
if (HTML_WHITESPACE.includes(c)) {
|
|
1260
|
+
add(text.slice(start, j));
|
|
1261
|
+
state = "before";
|
|
1262
|
+
continue;
|
|
1263
|
+
}
|
|
1264
|
+
if (c === ">") {
|
|
1265
|
+
add(text.slice(start, j));
|
|
1266
|
+
return j;
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
else if (state === "afterQuoted") {
|
|
1270
|
+
if (HTML_WHITESPACE.includes(c)) {
|
|
1271
|
+
state = "before";
|
|
1272
|
+
continue;
|
|
1273
|
+
}
|
|
1274
|
+
if (c === "/") {
|
|
1275
|
+
state = "selfClosing";
|
|
1276
|
+
continue;
|
|
1277
|
+
}
|
|
1278
|
+
if (c === ">")
|
|
1279
|
+
return j;
|
|
1280
|
+
// A missing space between attributes: the character starts the next name.
|
|
1281
|
+
start = j;
|
|
1282
|
+
state = "name";
|
|
1283
|
+
}
|
|
1284
|
+
else {
|
|
1285
|
+
// selfClosing: only ">" ends the tag, anything else is read again as before attribute name.
|
|
1286
|
+
if (c === ">")
|
|
1287
|
+
return j;
|
|
1288
|
+
state = "before";
|
|
1289
|
+
j--;
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
return -1;
|
|
1293
|
+
}
|
|
1294
|
+
function tagEnd(text, lt) {
|
|
1295
|
+
return readTag(text, lt, null);
|
|
1296
|
+
}
|
|
1297
|
+
// ASCII only, on purpose. The HTML tokenizer lowercases A to Z and nothing else, and HTML
|
|
1298
|
+
// whitespace is tab, LF, FF, CR and space. toLowerCase, trim and \s are Unicode aware, and each
|
|
1299
|
+
// of them read a page differently from a parser: U+0130 lowercases to two UTF-16 code units, so
|
|
1300
|
+
// every index taken from the lowered copy pointed past its place in the original and a relation
|
|
1301
|
+
// after it was lost; U+212A lowercases to "k"; and NBSP counted as whitespace between head
|
|
1302
|
+
// elements, where a parser starts the body, and between rel tokens. Measured against parse5,
|
|
1303
|
+
// 2026-09-12.
|
|
1304
|
+
function asciiLower(s) {
|
|
1305
|
+
return s.replace(/[A-Z]+/g, (m) => m.toLowerCase());
|
|
1306
|
+
}
|
|
1307
|
+
var HTML_WHITESPACE = " \t\n\r\f";
|
|
1308
|
+
// A MIME type is trimmed of HTTP whitespace, which does not include FF.
|
|
1309
|
+
var HTTP_WHITESPACE = " \t\n\r";
|
|
1310
|
+
function trimChars(s, chars) {
|
|
1311
|
+
let a = 0, b = s.length;
|
|
1312
|
+
while (a < b && chars.includes(s[a]))
|
|
1313
|
+
a++;
|
|
1314
|
+
while (b > a && chars.includes(s[b - 1]))
|
|
1315
|
+
b--;
|
|
1316
|
+
return s.slice(a, b);
|
|
1317
|
+
}
|
|
1318
|
+
function isHtmlWhitespace(s) {
|
|
1319
|
+
for (let k = 0; k < s.length; k++)
|
|
1320
|
+
if (!HTML_WHITESPACE.includes(s[k]))
|
|
1321
|
+
return false;
|
|
1322
|
+
return true;
|
|
1323
|
+
}
|
|
1324
|
+
function isAsciiAlphanumeric(c) {
|
|
1325
|
+
return c !== undefined && ((c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || (c >= "0" && c <= "9"));
|
|
1326
|
+
}
|
|
1327
|
+
// Character references in an attribute value, decoded after the tag has been tokenized and
|
|
1328
|
+
// never before it: decoding the source first would let a decoded quote move where a value
|
|
1329
|
+
// ends. The rules are those of the tokenizer's character reference state inside an attribute
|
|
1330
|
+
// value. The longest named reference wins. A name without its ";" decodes only for the legacy
|
|
1331
|
+
// names that allow it, and not when "=" or an ASCII letter or digit follows, so "?a=1&b=2"
|
|
1332
|
+
// stays as written. A numeric reference takes decimal or hex digits with the ";" optional;
|
|
1333
|
+
// zero, a surrogate and anything past U+10FFFF become U+FFFD, and 0x80 to 0x9F go through
|
|
1334
|
+
// the replacement table below. A NUL in the value becomes U+FFFD as well. Before this,
|
|
1335
|
+
// rel="describedby" read as no relation and href="/llms.txt?a=1&b=2" was reported
|
|
1336
|
+
// with "&" still in it.
|
|
1337
|
+
var C1_REPLACEMENTS = {
|
|
1338
|
+
0x80: 0x20ac, 0x82: 0x201a, 0x83: 0x0192, 0x84: 0x201e, 0x85: 0x2026, 0x86: 0x2020, 0x87: 0x2021,
|
|
1339
|
+
0x88: 0x02c6, 0x89: 0x2030, 0x8a: 0x0160, 0x8b: 0x2039, 0x8c: 0x0152, 0x8e: 0x017d, 0x91: 0x2018,
|
|
1340
|
+
0x92: 0x2019, 0x93: 0x201c, 0x94: 0x201d, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014, 0x98: 0x02dc,
|
|
1341
|
+
0x99: 0x2122, 0x9a: 0x0161, 0x9b: 0x203a, 0x9c: 0x0153, 0x9e: 0x017e, 0x9f: 0x0178,
|
|
1342
|
+
};
|
|
1343
|
+
// The full WHATWG list of named character references, 2 231 names: "name!hex" is a name that
|
|
1344
|
+
// also decodes without its ";", "name=hex" one that needs it, and several code points are
|
|
1345
|
+
// separated by ",". Generated 2026-09-12 from character-entities 2.0.2
|
|
1346
|
+
// (https://github.com/wooorm/character-entities), (c) 2015 Titus Wormer, MIT, with every value
|
|
1347
|
+
// and every legacy name checked against the decoder of the entities package. The full license
|
|
1348
|
+
// text is in THIRD-PARTY-NOTICES.md (E01, outside audit 2026-09-26); this is a build-time data
|
|
1349
|
+
// source copied in at generation time, not an installed runtime dependency.
|
|
1350
|
+
var NAMED_REFERENCE_DATA = "AElig!c6 AMP!26 Aacute!c1 Abreve=102 Acirc!c2 Acy=410 Afr=1d504 Agrave!c0 Alpha=391 Amacr=100 " +
|
|
1351
|
+
"And=2a53 Aogon=104 Aopf=1d538 ApplyFunction=2061 Aring!c5 Ascr=1d49c Assign=2254 Atilde!c3 " +
|
|
1352
|
+
"Auml!c4 Backslash=2216 Barv=2ae7 Barwed=2306 Bcy=411 Because=2235 Bernoullis=212c Beta=392 " +
|
|
1353
|
+
"Bfr=1d505 Bopf=1d539 Breve=2d8 Bscr=212c Bumpeq=224e CHcy=427 COPY!a9 Cacute=106 Cap=22d2 " +
|
|
1354
|
+
"CapitalDifferentialD=2145 Cayleys=212d Ccaron=10c Ccedil!c7 Ccirc=108 Cconint=2230 Cdot=10a " +
|
|
1355
|
+
"Cedilla=b8 CenterDot=b7 Cfr=212d Chi=3a7 CircleDot=2299 CircleMinus=2296 CirclePlus=2295 " +
|
|
1356
|
+
"CircleTimes=2297 ClockwiseContourIntegral=2232 CloseCurlyDoubleQuote=201d CloseCurlyQuote=2019 " +
|
|
1357
|
+
"Colon=2237 Colone=2a74 Congruent=2261 Conint=222f ContourIntegral=222e Copf=2102 Coproduct=2210 " +
|
|
1358
|
+
"CounterClockwiseContourIntegral=2233 Cross=2a2f Cscr=1d49e Cup=22d3 CupCap=224d DD=2145 " +
|
|
1359
|
+
"DDotrahd=2911 DJcy=402 DScy=405 DZcy=40f Dagger=2021 Darr=21a1 Dashv=2ae4 Dcaron=10e Dcy=414 " +
|
|
1360
|
+
"Del=2207 Delta=394 Dfr=1d507 DiacriticalAcute=b4 DiacriticalDot=2d9 DiacriticalDoubleAcute=2dd " +
|
|
1361
|
+
"DiacriticalGrave=60 DiacriticalTilde=2dc Diamond=22c4 DifferentialD=2146 Dopf=1d53b Dot=a8 " +
|
|
1362
|
+
"DotDot=20dc DotEqual=2250 DoubleContourIntegral=222f DoubleDot=a8 DoubleDownArrow=21d3 " +
|
|
1363
|
+
"DoubleLeftArrow=21d0 DoubleLeftRightArrow=21d4 DoubleLeftTee=2ae4 DoubleLongLeftArrow=27f8 " +
|
|
1364
|
+
"DoubleLongLeftRightArrow=27fa DoubleLongRightArrow=27f9 DoubleRightArrow=21d2 " +
|
|
1365
|
+
"DoubleRightTee=22a8 DoubleUpArrow=21d1 DoubleUpDownArrow=21d5 DoubleVerticalBar=2225 " +
|
|
1366
|
+
"DownArrow=2193 DownArrowBar=2913 DownArrowUpArrow=21f5 DownBreve=311 DownLeftRightVector=2950 " +
|
|
1367
|
+
"DownLeftTeeVector=295e DownLeftVector=21bd DownLeftVectorBar=2956 DownRightTeeVector=295f " +
|
|
1368
|
+
"DownRightVector=21c1 DownRightVectorBar=2957 DownTee=22a4 DownTeeArrow=21a7 Downarrow=21d3 " +
|
|
1369
|
+
"Dscr=1d49f Dstrok=110 ENG=14a ETH!d0 Eacute!c9 Ecaron=11a Ecirc!ca Ecy=42d Edot=116 Efr=1d508 " +
|
|
1370
|
+
"Egrave!c8 Element=2208 Emacr=112 EmptySmallSquare=25fb EmptyVerySmallSquare=25ab Eogon=118 " +
|
|
1371
|
+
"Eopf=1d53c Epsilon=395 Equal=2a75 EqualTilde=2242 Equilibrium=21cc Escr=2130 Esim=2a73 Eta=397 " +
|
|
1372
|
+
"Euml!cb Exists=2203 ExponentialE=2147 Fcy=424 Ffr=1d509 FilledSmallSquare=25fc " +
|
|
1373
|
+
"FilledVerySmallSquare=25aa Fopf=1d53d ForAll=2200 Fouriertrf=2131 Fscr=2131 GJcy=403 GT!3e " +
|
|
1374
|
+
"Gamma=393 Gammad=3dc Gbreve=11e Gcedil=122 Gcirc=11c Gcy=413 Gdot=120 Gfr=1d50a Gg=22d9 " +
|
|
1375
|
+
"Gopf=1d53e GreaterEqual=2265 GreaterEqualLess=22db GreaterFullEqual=2267 GreaterGreater=2aa2 " +
|
|
1376
|
+
"GreaterLess=2277 GreaterSlantEqual=2a7e GreaterTilde=2273 Gscr=1d4a2 Gt=226b HARDcy=42a " +
|
|
1377
|
+
"Hacek=2c7 Hat=5e Hcirc=124 Hfr=210c HilbertSpace=210b Hopf=210d HorizontalLine=2500 Hscr=210b " +
|
|
1378
|
+
"Hstrok=126 HumpDownHump=224e HumpEqual=224f IEcy=415 IJlig=132 IOcy=401 Iacute!cd Icirc!ce " +
|
|
1379
|
+
"Icy=418 Idot=130 Ifr=2111 Igrave!cc Im=2111 Imacr=12a ImaginaryI=2148 Implies=21d2 Int=222c " +
|
|
1380
|
+
"Integral=222b Intersection=22c2 InvisibleComma=2063 InvisibleTimes=2062 Iogon=12e Iopf=1d540 " +
|
|
1381
|
+
"Iota=399 Iscr=2110 Itilde=128 Iukcy=406 Iuml!cf Jcirc=134 Jcy=419 Jfr=1d50d Jopf=1d541 " +
|
|
1382
|
+
"Jscr=1d4a5 Jsercy=408 Jukcy=404 KHcy=425 KJcy=40c Kappa=39a Kcedil=136 Kcy=41a Kfr=1d50e " +
|
|
1383
|
+
"Kopf=1d542 Kscr=1d4a6 LJcy=409 LT!3c Lacute=139 Lambda=39b Lang=27ea Laplacetrf=2112 Larr=219e " +
|
|
1384
|
+
"Lcaron=13d Lcedil=13b Lcy=41b LeftAngleBracket=27e8 LeftArrow=2190 LeftArrowBar=21e4 " +
|
|
1385
|
+
"LeftArrowRightArrow=21c6 LeftCeiling=2308 LeftDoubleBracket=27e6 LeftDownTeeVector=2961 " +
|
|
1386
|
+
"LeftDownVector=21c3 LeftDownVectorBar=2959 LeftFloor=230a LeftRightArrow=2194 " +
|
|
1387
|
+
"LeftRightVector=294e LeftTee=22a3 LeftTeeArrow=21a4 LeftTeeVector=295a LeftTriangle=22b2 " +
|
|
1388
|
+
"LeftTriangleBar=29cf LeftTriangleEqual=22b4 LeftUpDownVector=2951 LeftUpTeeVector=2960 " +
|
|
1389
|
+
"LeftUpVector=21bf LeftUpVectorBar=2958 LeftVector=21bc LeftVectorBar=2952 Leftarrow=21d0 " +
|
|
1390
|
+
"Leftrightarrow=21d4 LessEqualGreater=22da LessFullEqual=2266 LessGreater=2276 LessLess=2aa1 " +
|
|
1391
|
+
"LessSlantEqual=2a7d LessTilde=2272 Lfr=1d50f Ll=22d8 Lleftarrow=21da Lmidot=13f " +
|
|
1392
|
+
"LongLeftArrow=27f5 LongLeftRightArrow=27f7 LongRightArrow=27f6 Longleftarrow=27f8 " +
|
|
1393
|
+
"Longleftrightarrow=27fa Longrightarrow=27f9 Lopf=1d543 LowerLeftArrow=2199 LowerRightArrow=2198 " +
|
|
1394
|
+
"Lscr=2112 Lsh=21b0 Lstrok=141 Lt=226a Map=2905 Mcy=41c MediumSpace=205f Mellintrf=2133 Mfr=1d510 " +
|
|
1395
|
+
"MinusPlus=2213 Mopf=1d544 Mscr=2133 Mu=39c NJcy=40a Nacute=143 Ncaron=147 Ncedil=145 Ncy=41d " +
|
|
1396
|
+
"NegativeMediumSpace=200b NegativeThickSpace=200b NegativeThinSpace=200b " +
|
|
1397
|
+
"NegativeVeryThinSpace=200b NestedGreaterGreater=226b NestedLessLess=226a NewLine=a Nfr=1d511 " +
|
|
1398
|
+
"NoBreak=2060 NonBreakingSpace=a0 Nopf=2115 Not=2aec NotCongruent=2262 NotCupCap=226d " +
|
|
1399
|
+
"NotDoubleVerticalBar=2226 NotElement=2209 NotEqual=2260 NotEqualTilde=2242,338 NotExists=2204 " +
|
|
1400
|
+
"NotGreater=226f NotGreaterEqual=2271 NotGreaterFullEqual=2267,338 NotGreaterGreater=226b,338 " +
|
|
1401
|
+
"NotGreaterLess=2279 NotGreaterSlantEqual=2a7e,338 NotGreaterTilde=2275 NotHumpDownHump=224e,338 " +
|
|
1402
|
+
"NotHumpEqual=224f,338 NotLeftTriangle=22ea NotLeftTriangleBar=29cf,338 NotLeftTriangleEqual=22ec " +
|
|
1403
|
+
"NotLess=226e NotLessEqual=2270 NotLessGreater=2278 NotLessLess=226a,338 " +
|
|
1404
|
+
"NotLessSlantEqual=2a7d,338 NotLessTilde=2274 NotNestedGreaterGreater=2aa2,338 " +
|
|
1405
|
+
"NotNestedLessLess=2aa1,338 NotPrecedes=2280 NotPrecedesEqual=2aaf,338 NotPrecedesSlantEqual=22e0 " +
|
|
1406
|
+
"NotReverseElement=220c NotRightTriangle=22eb NotRightTriangleBar=29d0,338 " +
|
|
1407
|
+
"NotRightTriangleEqual=22ed NotSquareSubset=228f,338 NotSquareSubsetEqual=22e2 " +
|
|
1408
|
+
"NotSquareSuperset=2290,338 NotSquareSupersetEqual=22e3 NotSubset=2282,20d2 NotSubsetEqual=2288 " +
|
|
1409
|
+
"NotSucceeds=2281 NotSucceedsEqual=2ab0,338 NotSucceedsSlantEqual=22e1 NotSucceedsTilde=227f,338 " +
|
|
1410
|
+
"NotSuperset=2283,20d2 NotSupersetEqual=2289 NotTilde=2241 NotTildeEqual=2244 " +
|
|
1411
|
+
"NotTildeFullEqual=2247 NotTildeTilde=2249 NotVerticalBar=2224 Nscr=1d4a9 Ntilde!d1 Nu=39d " +
|
|
1412
|
+
"OElig=152 Oacute!d3 Ocirc!d4 Ocy=41e Odblac=150 Ofr=1d512 Ograve!d2 Omacr=14c Omega=3a9 " +
|
|
1413
|
+
"Omicron=39f Oopf=1d546 OpenCurlyDoubleQuote=201c OpenCurlyQuote=2018 Or=2a54 Oscr=1d4aa " +
|
|
1414
|
+
"Oslash!d8 Otilde!d5 Otimes=2a37 Ouml!d6 OverBar=203e OverBrace=23de OverBracket=23b4 " +
|
|
1415
|
+
"OverParenthesis=23dc PartialD=2202 Pcy=41f Pfr=1d513 Phi=3a6 Pi=3a0 PlusMinus=b1 " +
|
|
1416
|
+
"Poincareplane=210c Popf=2119 Pr=2abb Precedes=227a PrecedesEqual=2aaf PrecedesSlantEqual=227c " +
|
|
1417
|
+
"PrecedesTilde=227e Prime=2033 Product=220f Proportion=2237 Proportional=221d Pscr=1d4ab Psi=3a8 " +
|
|
1418
|
+
"QUOT!22 Qfr=1d514 Qopf=211a Qscr=1d4ac RBarr=2910 REG!ae Racute=154 Rang=27eb Rarr=21a0 " +
|
|
1419
|
+
"Rarrtl=2916 Rcaron=158 Rcedil=156 Rcy=420 Re=211c ReverseElement=220b ReverseEquilibrium=21cb " +
|
|
1420
|
+
"ReverseUpEquilibrium=296f Rfr=211c Rho=3a1 RightAngleBracket=27e9 RightArrow=2192 " +
|
|
1421
|
+
"RightArrowBar=21e5 RightArrowLeftArrow=21c4 RightCeiling=2309 RightDoubleBracket=27e7 " +
|
|
1422
|
+
"RightDownTeeVector=295d RightDownVector=21c2 RightDownVectorBar=2955 RightFloor=230b " +
|
|
1423
|
+
"RightTee=22a2 RightTeeArrow=21a6 RightTeeVector=295b RightTriangle=22b3 RightTriangleBar=29d0 " +
|
|
1424
|
+
"RightTriangleEqual=22b5 RightUpDownVector=294f RightUpTeeVector=295c RightUpVector=21be " +
|
|
1425
|
+
"RightUpVectorBar=2954 RightVector=21c0 RightVectorBar=2953 Rightarrow=21d2 Ropf=211d " +
|
|
1426
|
+
"RoundImplies=2970 Rrightarrow=21db Rscr=211b Rsh=21b1 RuleDelayed=29f4 SHCHcy=429 SHcy=428 " +
|
|
1427
|
+
"SOFTcy=42c Sacute=15a Sc=2abc Scaron=160 Scedil=15e Scirc=15c Scy=421 Sfr=1d516 " +
|
|
1428
|
+
"ShortDownArrow=2193 ShortLeftArrow=2190 ShortRightArrow=2192 ShortUpArrow=2191 Sigma=3a3 " +
|
|
1429
|
+
"SmallCircle=2218 Sopf=1d54a Sqrt=221a Square=25a1 SquareIntersection=2293 SquareSubset=228f " +
|
|
1430
|
+
"SquareSubsetEqual=2291 SquareSuperset=2290 SquareSupersetEqual=2292 SquareUnion=2294 Sscr=1d4ae " +
|
|
1431
|
+
"Star=22c6 Sub=22d0 Subset=22d0 SubsetEqual=2286 Succeeds=227b SucceedsEqual=2ab0 " +
|
|
1432
|
+
"SucceedsSlantEqual=227d SucceedsTilde=227f SuchThat=220b Sum=2211 Sup=22d1 Superset=2283 " +
|
|
1433
|
+
"SupersetEqual=2287 Supset=22d1 THORN!de TRADE=2122 TSHcy=40b TScy=426 Tab=9 Tau=3a4 Tcaron=164 " +
|
|
1434
|
+
"Tcedil=162 Tcy=422 Tfr=1d517 Therefore=2234 Theta=398 ThickSpace=205f,200a ThinSpace=2009 " +
|
|
1435
|
+
"Tilde=223c TildeEqual=2243 TildeFullEqual=2245 TildeTilde=2248 Topf=1d54b TripleDot=20db " +
|
|
1436
|
+
"Tscr=1d4af Tstrok=166 Uacute!da Uarr=219f Uarrocir=2949 Ubrcy=40e Ubreve=16c Ucirc!db Ucy=423 " +
|
|
1437
|
+
"Udblac=170 Ufr=1d518 Ugrave!d9 Umacr=16a UnderBar=5f UnderBrace=23df UnderBracket=23b5 " +
|
|
1438
|
+
"UnderParenthesis=23dd Union=22c3 UnionPlus=228e Uogon=172 Uopf=1d54c UpArrow=2191 " +
|
|
1439
|
+
"UpArrowBar=2912 UpArrowDownArrow=21c5 UpDownArrow=2195 UpEquilibrium=296e UpTee=22a5 " +
|
|
1440
|
+
"UpTeeArrow=21a5 Uparrow=21d1 Updownarrow=21d5 UpperLeftArrow=2196 UpperRightArrow=2197 Upsi=3d2 " +
|
|
1441
|
+
"Upsilon=3a5 Uring=16e Uscr=1d4b0 Utilde=168 Uuml!dc VDash=22ab Vbar=2aeb Vcy=412 Vdash=22a9 " +
|
|
1442
|
+
"Vdashl=2ae6 Vee=22c1 Verbar=2016 Vert=2016 VerticalBar=2223 VerticalLine=7c " +
|
|
1443
|
+
"VerticalSeparator=2758 VerticalTilde=2240 VeryThinSpace=200a Vfr=1d519 Vopf=1d54d Vscr=1d4b1 " +
|
|
1444
|
+
"Vvdash=22aa Wcirc=174 Wedge=22c0 Wfr=1d51a Wopf=1d54e Wscr=1d4b2 Xfr=1d51b Xi=39e Xopf=1d54f " +
|
|
1445
|
+
"Xscr=1d4b3 YAcy=42f YIcy=407 YUcy=42e Yacute!dd Ycirc=176 Ycy=42b Yfr=1d51c Yopf=1d550 " +
|
|
1446
|
+
"Yscr=1d4b4 Yuml=178 ZHcy=416 Zacute=179 Zcaron=17d Zcy=417 Zdot=17b ZeroWidthSpace=200b Zeta=396 " +
|
|
1447
|
+
"Zfr=2128 Zopf=2124 Zscr=1d4b5 aacute!e1 abreve=103 ac=223e acE=223e,333 acd=223f acirc!e2 " +
|
|
1448
|
+
"acute!b4 acy=430 aelig!e6 af=2061 afr=1d51e agrave!e0 alefsym=2135 aleph=2135 alpha=3b1 " +
|
|
1449
|
+
"amacr=101 amalg=2a3f amp!26 and=2227 andand=2a55 andd=2a5c andslope=2a58 andv=2a5a ang=2220 " +
|
|
1450
|
+
"ange=29a4 angle=2220 angmsd=2221 angmsdaa=29a8 angmsdab=29a9 angmsdac=29aa angmsdad=29ab " +
|
|
1451
|
+
"angmsdae=29ac angmsdaf=29ad angmsdag=29ae angmsdah=29af angrt=221f angrtvb=22be angrtvbd=299d " +
|
|
1452
|
+
"angsph=2222 angst=c5 angzarr=237c aogon=105 aopf=1d552 ap=2248 apE=2a70 apacir=2a6f ape=224a " +
|
|
1453
|
+
"apid=224b apos=27 approx=2248 approxeq=224a aring!e5 ascr=1d4b6 ast=2a asymp=2248 asympeq=224d " +
|
|
1454
|
+
"atilde!e3 auml!e4 awconint=2233 awint=2a11 bNot=2aed backcong=224c backepsilon=3f6 " +
|
|
1455
|
+
"backprime=2035 backsim=223d backsimeq=22cd barvee=22bd barwed=2305 barwedge=2305 bbrk=23b5 " +
|
|
1456
|
+
"bbrktbrk=23b6 bcong=224c bcy=431 bdquo=201e becaus=2235 because=2235 bemptyv=29b0 bepsi=3f6 " +
|
|
1457
|
+
"bernou=212c beta=3b2 beth=2136 between=226c bfr=1d51f bigcap=22c2 bigcirc=25ef bigcup=22c3 " +
|
|
1458
|
+
"bigodot=2a00 bigoplus=2a01 bigotimes=2a02 bigsqcup=2a06 bigstar=2605 bigtriangledown=25bd " +
|
|
1459
|
+
"bigtriangleup=25b3 biguplus=2a04 bigvee=22c1 bigwedge=22c0 bkarow=290d blacklozenge=29eb " +
|
|
1460
|
+
"blacksquare=25aa blacktriangle=25b4 blacktriangledown=25be blacktriangleleft=25c2 " +
|
|
1461
|
+
"blacktriangleright=25b8 blank=2423 blk12=2592 blk14=2591 blk34=2593 block=2588 bne=3d,20e5 " +
|
|
1462
|
+
"bnequiv=2261,20e5 bnot=2310 bopf=1d553 bot=22a5 bottom=22a5 bowtie=22c8 boxDL=2557 boxDR=2554 " +
|
|
1463
|
+
"boxDl=2556 boxDr=2553 boxH=2550 boxHD=2566 boxHU=2569 boxHd=2564 boxHu=2567 boxUL=255d " +
|
|
1464
|
+
"boxUR=255a boxUl=255c boxUr=2559 boxV=2551 boxVH=256c boxVL=2563 boxVR=2560 boxVh=256b " +
|
|
1465
|
+
"boxVl=2562 boxVr=255f boxbox=29c9 boxdL=2555 boxdR=2552 boxdl=2510 boxdr=250c boxh=2500 " +
|
|
1466
|
+
"boxhD=2565 boxhU=2568 boxhd=252c boxhu=2534 boxminus=229f boxplus=229e boxtimes=22a0 boxuL=255b " +
|
|
1467
|
+
"boxuR=2558 boxul=2518 boxur=2514 boxv=2502 boxvH=256a boxvL=2561 boxvR=255e boxvh=253c " +
|
|
1468
|
+
"boxvl=2524 boxvr=251c bprime=2035 breve=2d8 brvbar!a6 bscr=1d4b7 bsemi=204f bsim=223d bsime=22cd " +
|
|
1469
|
+
"bsol=5c bsolb=29c5 bsolhsub=27c8 bull=2022 bullet=2022 bump=224e bumpE=2aae bumpe=224f " +
|
|
1470
|
+
"bumpeq=224f cacute=107 cap=2229 capand=2a44 capbrcup=2a49 capcap=2a4b capcup=2a47 capdot=2a40 " +
|
|
1471
|
+
"caps=2229,fe00 caret=2041 caron=2c7 ccaps=2a4d ccaron=10d ccedil!e7 ccirc=109 ccups=2a4c " +
|
|
1472
|
+
"ccupssm=2a50 cdot=10b cedil!b8 cemptyv=29b2 cent!a2 centerdot=b7 cfr=1d520 chcy=447 check=2713 " +
|
|
1473
|
+
"checkmark=2713 chi=3c7 cir=25cb cirE=29c3 circ=2c6 circeq=2257 circlearrowleft=21ba " +
|
|
1474
|
+
"circlearrowright=21bb circledR=ae circledS=24c8 circledast=229b circledcirc=229a " +
|
|
1475
|
+
"circleddash=229d cire=2257 cirfnint=2a10 cirmid=2aef cirscir=29c2 clubs=2663 clubsuit=2663 " +
|
|
1476
|
+
"colon=3a colone=2254 coloneq=2254 comma=2c commat=40 comp=2201 compfn=2218 complement=2201 " +
|
|
1477
|
+
"complexes=2102 cong=2245 congdot=2a6d conint=222e copf=1d554 coprod=2210 copy!a9 copysr=2117 " +
|
|
1478
|
+
"crarr=21b5 cross=2717 cscr=1d4b8 csub=2acf csube=2ad1 csup=2ad0 csupe=2ad2 ctdot=22ef " +
|
|
1479
|
+
"cudarrl=2938 cudarrr=2935 cuepr=22de cuesc=22df cularr=21b6 cularrp=293d cup=222a cupbrcap=2a48 " +
|
|
1480
|
+
"cupcap=2a46 cupcup=2a4a cupdot=228d cupor=2a45 cups=222a,fe00 curarr=21b7 curarrm=293c " +
|
|
1481
|
+
"curlyeqprec=22de curlyeqsucc=22df curlyvee=22ce curlywedge=22cf curren!a4 curvearrowleft=21b6 " +
|
|
1482
|
+
"curvearrowright=21b7 cuvee=22ce cuwed=22cf cwconint=2232 cwint=2231 cylcty=232d dArr=21d3 " +
|
|
1483
|
+
"dHar=2965 dagger=2020 daleth=2138 darr=2193 dash=2010 dashv=22a3 dbkarow=290f dblac=2dd " +
|
|
1484
|
+
"dcaron=10f dcy=434 dd=2146 ddagger=2021 ddarr=21ca ddotseq=2a77 deg!b0 delta=3b4 demptyv=29b1 " +
|
|
1485
|
+
"dfisht=297f dfr=1d521 dharl=21c3 dharr=21c2 diam=22c4 diamond=22c4 diamondsuit=2666 diams=2666 " +
|
|
1486
|
+
"die=a8 digamma=3dd disin=22f2 div=f7 divide!f7 divideontimes=22c7 divonx=22c7 djcy=452 " +
|
|
1487
|
+
"dlcorn=231e dlcrop=230d dollar=24 dopf=1d555 dot=2d9 doteq=2250 doteqdot=2251 dotminus=2238 " +
|
|
1488
|
+
"dotplus=2214 dotsquare=22a1 doublebarwedge=2306 downarrow=2193 downdownarrows=21ca " +
|
|
1489
|
+
"downharpoonleft=21c3 downharpoonright=21c2 drbkarow=2910 drcorn=231f drcrop=230c dscr=1d4b9 " +
|
|
1490
|
+
"dscy=455 dsol=29f6 dstrok=111 dtdot=22f1 dtri=25bf dtrif=25be duarr=21f5 duhar=296f dwangle=29a6 " +
|
|
1491
|
+
"dzcy=45f dzigrarr=27ff eDDot=2a77 eDot=2251 eacute!e9 easter=2a6e ecaron=11b ecir=2256 ecirc!ea " +
|
|
1492
|
+
"ecolon=2255 ecy=44d edot=117 ee=2147 efDot=2252 efr=1d522 eg=2a9a egrave!e8 egs=2a96 egsdot=2a98 " +
|
|
1493
|
+
"el=2a99 elinters=23e7 ell=2113 els=2a95 elsdot=2a97 emacr=113 empty=2205 emptyset=2205 " +
|
|
1494
|
+
"emptyv=2205 emsp13=2004 emsp14=2005 emsp=2003 eng=14b ensp=2002 eogon=119 eopf=1d556 epar=22d5 " +
|
|
1495
|
+
"eparsl=29e3 eplus=2a71 epsi=3b5 epsilon=3b5 epsiv=3f5 eqcirc=2256 eqcolon=2255 eqsim=2242 " +
|
|
1496
|
+
"eqslantgtr=2a96 eqslantless=2a95 equals=3d equest=225f equiv=2261 equivDD=2a78 eqvparsl=29e5 " +
|
|
1497
|
+
"erDot=2253 erarr=2971 escr=212f esdot=2250 esim=2242 eta=3b7 eth!f0 euml!eb euro=20ac excl=21 " +
|
|
1498
|
+
"exist=2203 expectation=2130 exponentiale=2147 fallingdotseq=2252 fcy=444 female=2640 ffilig=fb03 " +
|
|
1499
|
+
"fflig=fb00 ffllig=fb04 ffr=1d523 filig=fb01 fjlig=66,6a flat=266d fllig=fb02 fltns=25b1 fnof=192 " +
|
|
1500
|
+
"fopf=1d557 forall=2200 fork=22d4 forkv=2ad9 fpartint=2a0d frac12!bd frac13=2153 frac14!bc " +
|
|
1501
|
+
"frac15=2155 frac16=2159 frac18=215b frac23=2154 frac25=2156 frac34!be frac35=2157 frac38=215c " +
|
|
1502
|
+
"frac45=2158 frac56=215a frac58=215d frac78=215e frasl=2044 frown=2322 fscr=1d4bb gE=2267 " +
|
|
1503
|
+
"gEl=2a8c gacute=1f5 gamma=3b3 gammad=3dd gap=2a86 gbreve=11f gcirc=11d gcy=433 gdot=121 ge=2265 " +
|
|
1504
|
+
"gel=22db geq=2265 geqq=2267 geqslant=2a7e ges=2a7e gescc=2aa9 gesdot=2a80 gesdoto=2a82 " +
|
|
1505
|
+
"gesdotol=2a84 gesl=22db,fe00 gesles=2a94 gfr=1d524 gg=226b ggg=22d9 gimel=2137 gjcy=453 gl=2277 " +
|
|
1506
|
+
"glE=2a92 gla=2aa5 glj=2aa4 gnE=2269 gnap=2a8a gnapprox=2a8a gne=2a88 gneq=2a88 gneqq=2269 " +
|
|
1507
|
+
"gnsim=22e7 gopf=1d558 grave=60 gscr=210a gsim=2273 gsime=2a8e gsiml=2a90 gt!3e gtcc=2aa7 " +
|
|
1508
|
+
"gtcir=2a7a gtdot=22d7 gtlPar=2995 gtquest=2a7c gtrapprox=2a86 gtrarr=2978 gtrdot=22d7 " +
|
|
1509
|
+
"gtreqless=22db gtreqqless=2a8c gtrless=2277 gtrsim=2273 gvertneqq=2269,fe00 gvnE=2269,fe00 " +
|
|
1510
|
+
"hArr=21d4 hairsp=200a half=bd hamilt=210b hardcy=44a harr=2194 harrcir=2948 harrw=21ad hbar=210f " +
|
|
1511
|
+
"hcirc=125 hearts=2665 heartsuit=2665 hellip=2026 hercon=22b9 hfr=1d525 hksearow=2925 " +
|
|
1512
|
+
"hkswarow=2926 hoarr=21ff homtht=223b hookleftarrow=21a9 hookrightarrow=21aa hopf=1d559 " +
|
|
1513
|
+
"horbar=2015 hscr=1d4bd hslash=210f hstrok=127 hybull=2043 hyphen=2010 iacute!ed ic=2063 icirc!ee " +
|
|
1514
|
+
"icy=438 iecy=435 iexcl!a1 iff=21d4 ifr=1d526 igrave!ec ii=2148 iiiint=2a0c iiint=222d " +
|
|
1515
|
+
"iinfin=29dc iiota=2129 ijlig=133 imacr=12b image=2111 imagline=2110 imagpart=2111 imath=131 " +
|
|
1516
|
+
"imof=22b7 imped=1b5 in=2208 incare=2105 infin=221e infintie=29dd inodot=131 int=222b intcal=22ba " +
|
|
1517
|
+
"integers=2124 intercal=22ba intlarhk=2a17 intprod=2a3c iocy=451 iogon=12f iopf=1d55a iota=3b9 " +
|
|
1518
|
+
"iprod=2a3c iquest!bf iscr=1d4be isin=2208 isinE=22f9 isindot=22f5 isins=22f4 isinsv=22f3 " +
|
|
1519
|
+
"isinv=2208 it=2062 itilde=129 iukcy=456 iuml!ef jcirc=135 jcy=439 jfr=1d527 jmath=237 jopf=1d55b " +
|
|
1520
|
+
"jscr=1d4bf jsercy=458 jukcy=454 kappa=3ba kappav=3f0 kcedil=137 kcy=43a kfr=1d528 kgreen=138 " +
|
|
1521
|
+
"khcy=445 kjcy=45c kopf=1d55c kscr=1d4c0 lAarr=21da lArr=21d0 lAtail=291b lBarr=290e lE=2266 " +
|
|
1522
|
+
"lEg=2a8b lHar=2962 lacute=13a laemptyv=29b4 lagran=2112 lambda=3bb lang=27e8 langd=2991 " +
|
|
1523
|
+
"langle=27e8 lap=2a85 laquo!ab larr=2190 larrb=21e4 larrbfs=291f larrfs=291d larrhk=21a9 " +
|
|
1524
|
+
"larrlp=21ab larrpl=2939 larrsim=2973 larrtl=21a2 lat=2aab latail=2919 late=2aad lates=2aad,fe00 " +
|
|
1525
|
+
"lbarr=290c lbbrk=2772 lbrace=7b lbrack=5b lbrke=298b lbrksld=298f lbrkslu=298d lcaron=13e " +
|
|
1526
|
+
"lcedil=13c lceil=2308 lcub=7b lcy=43b ldca=2936 ldquo=201c ldquor=201e ldrdhar=2967 " +
|
|
1527
|
+
"ldrushar=294b ldsh=21b2 le=2264 leftarrow=2190 leftarrowtail=21a2 leftharpoondown=21bd " +
|
|
1528
|
+
"leftharpoonup=21bc leftleftarrows=21c7 leftrightarrow=2194 leftrightarrows=21c6 " +
|
|
1529
|
+
"leftrightharpoons=21cb leftrightsquigarrow=21ad leftthreetimes=22cb leg=22da leq=2264 leqq=2266 " +
|
|
1530
|
+
"leqslant=2a7d les=2a7d lescc=2aa8 lesdot=2a7f lesdoto=2a81 lesdotor=2a83 lesg=22da,fe00 " +
|
|
1531
|
+
"lesges=2a93 lessapprox=2a85 lessdot=22d6 lesseqgtr=22da lesseqqgtr=2a8b lessgtr=2276 " +
|
|
1532
|
+
"lesssim=2272 lfisht=297c lfloor=230a lfr=1d529 lg=2276 lgE=2a91 lhard=21bd lharu=21bc " +
|
|
1533
|
+
"lharul=296a lhblk=2584 ljcy=459 ll=226a llarr=21c7 llcorner=231e llhard=296b lltri=25fa " +
|
|
1534
|
+
"lmidot=140 lmoust=23b0 lmoustache=23b0 lnE=2268 lnap=2a89 lnapprox=2a89 lne=2a87 lneq=2a87 " +
|
|
1535
|
+
"lneqq=2268 lnsim=22e6 loang=27ec loarr=21fd lobrk=27e6 longleftarrow=27f5 " +
|
|
1536
|
+
"longleftrightarrow=27f7 longmapsto=27fc longrightarrow=27f6 looparrowleft=21ab " +
|
|
1537
|
+
"looparrowright=21ac lopar=2985 lopf=1d55d loplus=2a2d lotimes=2a34 lowast=2217 lowbar=5f " +
|
|
1538
|
+
"loz=25ca lozenge=25ca lozf=29eb lpar=28 lparlt=2993 lrarr=21c6 lrcorner=231f lrhar=21cb " +
|
|
1539
|
+
"lrhard=296d lrm=200e lrtri=22bf lsaquo=2039 lscr=1d4c1 lsh=21b0 lsim=2272 lsime=2a8d lsimg=2a8f " +
|
|
1540
|
+
"lsqb=5b lsquo=2018 lsquor=201a lstrok=142 lt!3c ltcc=2aa6 ltcir=2a79 ltdot=22d6 lthree=22cb " +
|
|
1541
|
+
"ltimes=22c9 ltlarr=2976 ltquest=2a7b ltrPar=2996 ltri=25c3 ltrie=22b4 ltrif=25c2 lurdshar=294a " +
|
|
1542
|
+
"luruhar=2966 lvertneqq=2268,fe00 lvnE=2268,fe00 mDDot=223a macr!af male=2642 malt=2720 " +
|
|
1543
|
+
"maltese=2720 map=21a6 mapsto=21a6 mapstodown=21a7 mapstoleft=21a4 mapstoup=21a5 marker=25ae " +
|
|
1544
|
+
"mcomma=2a29 mcy=43c mdash=2014 measuredangle=2221 mfr=1d52a mho=2127 micro!b5 mid=2223 midast=2a " +
|
|
1545
|
+
"midcir=2af0 middot!b7 minus=2212 minusb=229f minusd=2238 minusdu=2a2a mlcp=2adb mldr=2026 " +
|
|
1546
|
+
"mnplus=2213 models=22a7 mopf=1d55e mp=2213 mscr=1d4c2 mstpos=223e mu=3bc multimap=22b8 " +
|
|
1547
|
+
"mumap=22b8 nGg=22d9,338 nGt=226b,20d2 nGtv=226b,338 nLeftarrow=21cd nLeftrightarrow=21ce " +
|
|
1548
|
+
"nLl=22d8,338 nLt=226a,20d2 nLtv=226a,338 nRightarrow=21cf nVDash=22af nVdash=22ae nabla=2207 " +
|
|
1549
|
+
"nacute=144 nang=2220,20d2 nap=2249 napE=2a70,338 napid=224b,338 napos=149 napprox=2249 " +
|
|
1550
|
+
"natur=266e natural=266e naturals=2115 nbsp!a0 nbump=224e,338 nbumpe=224f,338 ncap=2a43 " +
|
|
1551
|
+
"ncaron=148 ncedil=146 ncong=2247 ncongdot=2a6d,338 ncup=2a42 ncy=43d ndash=2013 ne=2260 " +
|
|
1552
|
+
"neArr=21d7 nearhk=2924 nearr=2197 nearrow=2197 nedot=2250,338 nequiv=2262 nesear=2928 " +
|
|
1553
|
+
"nesim=2242,338 nexist=2204 nexists=2204 nfr=1d52b ngE=2267,338 nge=2271 ngeq=2271 ngeqq=2267,338 " +
|
|
1554
|
+
"ngeqslant=2a7e,338 nges=2a7e,338 ngsim=2275 ngt=226f ngtr=226f nhArr=21ce nharr=21ae nhpar=2af2 " +
|
|
1555
|
+
"ni=220b nis=22fc nisd=22fa niv=220b njcy=45a nlArr=21cd nlE=2266,338 nlarr=219a nldr=2025 " +
|
|
1556
|
+
"nle=2270 nleftarrow=219a nleftrightarrow=21ae nleq=2270 nleqq=2266,338 nleqslant=2a7d,338 " +
|
|
1557
|
+
"nles=2a7d,338 nless=226e nlsim=2274 nlt=226e nltri=22ea nltrie=22ec nmid=2224 nopf=1d55f not!ac " +
|
|
1558
|
+
"notin=2209 notinE=22f9,338 notindot=22f5,338 notinva=2209 notinvb=22f7 notinvc=22f6 notni=220c " +
|
|
1559
|
+
"notniva=220c notnivb=22fe notnivc=22fd npar=2226 nparallel=2226 nparsl=2afd,20e5 npart=2202,338 " +
|
|
1560
|
+
"npolint=2a14 npr=2280 nprcue=22e0 npre=2aaf,338 nprec=2280 npreceq=2aaf,338 nrArr=21cf " +
|
|
1561
|
+
"nrarr=219b nrarrc=2933,338 nrarrw=219d,338 nrightarrow=219b nrtri=22eb nrtrie=22ed nsc=2281 " +
|
|
1562
|
+
"nsccue=22e1 nsce=2ab0,338 nscr=1d4c3 nshortmid=2224 nshortparallel=2226 nsim=2241 nsime=2244 " +
|
|
1563
|
+
"nsimeq=2244 nsmid=2224 nspar=2226 nsqsube=22e2 nsqsupe=22e3 nsub=2284 nsubE=2ac5,338 nsube=2288 " +
|
|
1564
|
+
"nsubset=2282,20d2 nsubseteq=2288 nsubseteqq=2ac5,338 nsucc=2281 nsucceq=2ab0,338 nsup=2285 " +
|
|
1565
|
+
"nsupE=2ac6,338 nsupe=2289 nsupset=2283,20d2 nsupseteq=2289 nsupseteqq=2ac6,338 ntgl=2279 " +
|
|
1566
|
+
"ntilde!f1 ntlg=2278 ntriangleleft=22ea ntrianglelefteq=22ec ntriangleright=22eb " +
|
|
1567
|
+
"ntrianglerighteq=22ed nu=3bd num=23 numero=2116 numsp=2007 nvDash=22ad nvHarr=2904 " +
|
|
1568
|
+
"nvap=224d,20d2 nvdash=22ac nvge=2265,20d2 nvgt=3e,20d2 nvinfin=29de nvlArr=2902 nvle=2264,20d2 " +
|
|
1569
|
+
"nvlt=3c,20d2 nvltrie=22b4,20d2 nvrArr=2903 nvrtrie=22b5,20d2 nvsim=223c,20d2 nwArr=21d6 " +
|
|
1570
|
+
"nwarhk=2923 nwarr=2196 nwarrow=2196 nwnear=2927 oS=24c8 oacute!f3 oast=229b ocir=229a ocirc!f4 " +
|
|
1571
|
+
"ocy=43e odash=229d odblac=151 odiv=2a38 odot=2299 odsold=29bc oelig=153 ofcir=29bf ofr=1d52c " +
|
|
1572
|
+
"ogon=2db ograve!f2 ogt=29c1 ohbar=29b5 ohm=3a9 oint=222e olarr=21ba olcir=29be olcross=29bb " +
|
|
1573
|
+
"oline=203e olt=29c0 omacr=14d omega=3c9 omicron=3bf omid=29b6 ominus=2296 oopf=1d560 opar=29b7 " +
|
|
1574
|
+
"operp=29b9 oplus=2295 or=2228 orarr=21bb ord=2a5d order=2134 orderof=2134 ordf!aa ordm!ba " +
|
|
1575
|
+
"origof=22b6 oror=2a56 orslope=2a57 orv=2a5b oscr=2134 oslash!f8 osol=2298 otilde!f5 otimes=2297 " +
|
|
1576
|
+
"otimesas=2a36 ouml!f6 ovbar=233d par=2225 para!b6 parallel=2225 parsim=2af3 parsl=2afd part=2202 " +
|
|
1577
|
+
"pcy=43f percnt=25 period=2e permil=2030 perp=22a5 pertenk=2031 pfr=1d52d phi=3c6 phiv=3d5 " +
|
|
1578
|
+
"phmmat=2133 phone=260e pi=3c0 pitchfork=22d4 piv=3d6 planck=210f planckh=210e plankv=210f " +
|
|
1579
|
+
"plus=2b plusacir=2a23 plusb=229e pluscir=2a22 plusdo=2214 plusdu=2a25 pluse=2a72 plusmn!b1 " +
|
|
1580
|
+
"plussim=2a26 plustwo=2a27 pm=b1 pointint=2a15 popf=1d561 pound!a3 pr=227a prE=2ab3 prap=2ab7 " +
|
|
1581
|
+
"prcue=227c pre=2aaf prec=227a precapprox=2ab7 preccurlyeq=227c preceq=2aaf precnapprox=2ab9 " +
|
|
1582
|
+
"precneqq=2ab5 precnsim=22e8 precsim=227e prime=2032 primes=2119 prnE=2ab5 prnap=2ab9 prnsim=22e8 " +
|
|
1583
|
+
"prod=220f profalar=232e profline=2312 profsurf=2313 prop=221d propto=221d prsim=227e prurel=22b0 " +
|
|
1584
|
+
"pscr=1d4c5 psi=3c8 puncsp=2008 qfr=1d52e qint=2a0c qopf=1d562 qprime=2057 qscr=1d4c6 " +
|
|
1585
|
+
"quaternions=210d quatint=2a16 quest=3f questeq=225f quot!22 rAarr=21db rArr=21d2 rAtail=291c " +
|
|
1586
|
+
"rBarr=290f rHar=2964 race=223d,331 racute=155 radic=221a raemptyv=29b3 rang=27e9 rangd=2992 " +
|
|
1587
|
+
"range=29a5 rangle=27e9 raquo!bb rarr=2192 rarrap=2975 rarrb=21e5 rarrbfs=2920 rarrc=2933 " +
|
|
1588
|
+
"rarrfs=291e rarrhk=21aa rarrlp=21ac rarrpl=2945 rarrsim=2974 rarrtl=21a3 rarrw=219d ratail=291a " +
|
|
1589
|
+
"ratio=2236 rationals=211a rbarr=290d rbbrk=2773 rbrace=7d rbrack=5d rbrke=298c rbrksld=298e " +
|
|
1590
|
+
"rbrkslu=2990 rcaron=159 rcedil=157 rceil=2309 rcub=7d rcy=440 rdca=2937 rdldhar=2969 rdquo=201d " +
|
|
1591
|
+
"rdquor=201d rdsh=21b3 real=211c realine=211b realpart=211c reals=211d rect=25ad reg!ae " +
|
|
1592
|
+
"rfisht=297d rfloor=230b rfr=1d52f rhard=21c1 rharu=21c0 rharul=296c rho=3c1 rhov=3f1 " +
|
|
1593
|
+
"rightarrow=2192 rightarrowtail=21a3 rightharpoondown=21c1 rightharpoonup=21c0 " +
|
|
1594
|
+
"rightleftarrows=21c4 rightleftharpoons=21cc rightrightarrows=21c9 rightsquigarrow=219d " +
|
|
1595
|
+
"rightthreetimes=22cc ring=2da risingdotseq=2253 rlarr=21c4 rlhar=21cc rlm=200f rmoust=23b1 " +
|
|
1596
|
+
"rmoustache=23b1 rnmid=2aee roang=27ed roarr=21fe robrk=27e7 ropar=2986 ropf=1d563 roplus=2a2e " +
|
|
1597
|
+
"rotimes=2a35 rpar=29 rpargt=2994 rppolint=2a12 rrarr=21c9 rsaquo=203a rscr=1d4c7 rsh=21b1 " +
|
|
1598
|
+
"rsqb=5d rsquo=2019 rsquor=2019 rthree=22cc rtimes=22ca rtri=25b9 rtrie=22b5 rtrif=25b8 " +
|
|
1599
|
+
"rtriltri=29ce ruluhar=2968 rx=211e sacute=15b sbquo=201a sc=227b scE=2ab4 scap=2ab8 scaron=161 " +
|
|
1600
|
+
"sccue=227d sce=2ab0 scedil=15f scirc=15d scnE=2ab6 scnap=2aba scnsim=22e9 scpolint=2a13 " +
|
|
1601
|
+
"scsim=227f scy=441 sdot=22c5 sdotb=22a1 sdote=2a66 seArr=21d8 searhk=2925 searr=2198 " +
|
|
1602
|
+
"searrow=2198 sect!a7 semi=3b seswar=2929 setminus=2216 setmn=2216 sext=2736 sfr=1d530 " +
|
|
1603
|
+
"sfrown=2322 sharp=266f shchcy=449 shcy=448 shortmid=2223 shortparallel=2225 shy!ad sigma=3c3 " +
|
|
1604
|
+
"sigmaf=3c2 sigmav=3c2 sim=223c simdot=2a6a sime=2243 simeq=2243 simg=2a9e simgE=2aa0 siml=2a9d " +
|
|
1605
|
+
"simlE=2a9f simne=2246 simplus=2a24 simrarr=2972 slarr=2190 smallsetminus=2216 smashp=2a33 " +
|
|
1606
|
+
"smeparsl=29e4 smid=2223 smile=2323 smt=2aaa smte=2aac smtes=2aac,fe00 softcy=44c sol=2f " +
|
|
1607
|
+
"solb=29c4 solbar=233f sopf=1d564 spades=2660 spadesuit=2660 spar=2225 sqcap=2293 " +
|
|
1608
|
+
"sqcaps=2293,fe00 sqcup=2294 sqcups=2294,fe00 sqsub=228f sqsube=2291 sqsubset=228f " +
|
|
1609
|
+
"sqsubseteq=2291 sqsup=2290 sqsupe=2292 sqsupset=2290 sqsupseteq=2292 squ=25a1 square=25a1 " +
|
|
1610
|
+
"squarf=25aa squf=25aa srarr=2192 sscr=1d4c8 ssetmn=2216 ssmile=2323 sstarf=22c6 star=2606 " +
|
|
1611
|
+
"starf=2605 straightepsilon=3f5 straightphi=3d5 strns=af sub=2282 subE=2ac5 subdot=2abd sube=2286 " +
|
|
1612
|
+
"subedot=2ac3 submult=2ac1 subnE=2acb subne=228a subplus=2abf subrarr=2979 subset=2282 " +
|
|
1613
|
+
"subseteq=2286 subseteqq=2ac5 subsetneq=228a subsetneqq=2acb subsim=2ac7 subsub=2ad5 subsup=2ad3 " +
|
|
1614
|
+
"succ=227b succapprox=2ab8 succcurlyeq=227d succeq=2ab0 succnapprox=2aba succneqq=2ab6 " +
|
|
1615
|
+
"succnsim=22e9 succsim=227f sum=2211 sung=266a sup1!b9 sup2!b2 sup3!b3 sup=2283 supE=2ac6 " +
|
|
1616
|
+
"supdot=2abe supdsub=2ad8 supe=2287 supedot=2ac4 suphsol=27c9 suphsub=2ad7 suplarr=297b " +
|
|
1617
|
+
"supmult=2ac2 supnE=2acc supne=228b supplus=2ac0 supset=2283 supseteq=2287 supseteqq=2ac6 " +
|
|
1618
|
+
"supsetneq=228b supsetneqq=2acc supsim=2ac8 supsub=2ad4 supsup=2ad6 swArr=21d9 swarhk=2926 " +
|
|
1619
|
+
"swarr=2199 swarrow=2199 swnwar=292a szlig!df target=2316 tau=3c4 tbrk=23b4 tcaron=165 tcedil=163 " +
|
|
1620
|
+
"tcy=442 tdot=20db telrec=2315 tfr=1d531 there4=2234 therefore=2234 theta=3b8 thetasym=3d1 " +
|
|
1621
|
+
"thetav=3d1 thickapprox=2248 thicksim=223c thinsp=2009 thkap=2248 thksim=223c thorn!fe tilde=2dc " +
|
|
1622
|
+
"times!d7 timesb=22a0 timesbar=2a31 timesd=2a30 tint=222d toea=2928 top=22a4 topbot=2336 " +
|
|
1623
|
+
"topcir=2af1 topf=1d565 topfork=2ada tosa=2929 tprime=2034 trade=2122 triangle=25b5 " +
|
|
1624
|
+
"triangledown=25bf triangleleft=25c3 trianglelefteq=22b4 triangleq=225c triangleright=25b9 " +
|
|
1625
|
+
"trianglerighteq=22b5 tridot=25ec trie=225c triminus=2a3a triplus=2a39 trisb=29cd tritime=2a3b " +
|
|
1626
|
+
"trpezium=23e2 tscr=1d4c9 tscy=446 tshcy=45b tstrok=167 twixt=226c twoheadleftarrow=219e " +
|
|
1627
|
+
"twoheadrightarrow=21a0 uArr=21d1 uHar=2963 uacute!fa uarr=2191 ubrcy=45e ubreve=16d ucirc!fb " +
|
|
1628
|
+
"ucy=443 udarr=21c5 udblac=171 udhar=296e ufisht=297e ufr=1d532 ugrave!f9 uharl=21bf uharr=21be " +
|
|
1629
|
+
"uhblk=2580 ulcorn=231c ulcorner=231c ulcrop=230f ultri=25f8 umacr=16b uml!a8 uogon=173 " +
|
|
1630
|
+
"uopf=1d566 uparrow=2191 updownarrow=2195 upharpoonleft=21bf upharpoonright=21be uplus=228e " +
|
|
1631
|
+
"upsi=3c5 upsih=3d2 upsilon=3c5 upuparrows=21c8 urcorn=231d urcorner=231d urcrop=230e uring=16f " +
|
|
1632
|
+
"urtri=25f9 uscr=1d4ca utdot=22f0 utilde=169 utri=25b5 utrif=25b4 uuarr=21c8 uuml!fc uwangle=29a7 " +
|
|
1633
|
+
"vArr=21d5 vBar=2ae8 vBarv=2ae9 vDash=22a8 vangrt=299c varepsilon=3f5 varkappa=3f0 " +
|
|
1634
|
+
"varnothing=2205 varphi=3d5 varpi=3d6 varpropto=221d varr=2195 varrho=3f1 varsigma=3c2 " +
|
|
1635
|
+
"varsubsetneq=228a,fe00 varsubsetneqq=2acb,fe00 varsupsetneq=228b,fe00 varsupsetneqq=2acc,fe00 " +
|
|
1636
|
+
"vartheta=3d1 vartriangleleft=22b2 vartriangleright=22b3 vcy=432 vdash=22a2 vee=2228 veebar=22bb " +
|
|
1637
|
+
"veeeq=225a vellip=22ee verbar=7c vert=7c vfr=1d533 vltri=22b2 vnsub=2282,20d2 vnsup=2283,20d2 " +
|
|
1638
|
+
"vopf=1d567 vprop=221d vrtri=22b3 vscr=1d4cb vsubnE=2acb,fe00 vsubne=228a,fe00 vsupnE=2acc,fe00 " +
|
|
1639
|
+
"vsupne=228b,fe00 vzigzag=299a wcirc=175 wedbar=2a5f wedge=2227 wedgeq=2259 weierp=2118 wfr=1d534 " +
|
|
1640
|
+
"wopf=1d568 wp=2118 wr=2240 wreath=2240 wscr=1d4cc xcap=22c2 xcirc=25ef xcup=22c3 xdtri=25bd " +
|
|
1641
|
+
"xfr=1d535 xhArr=27fa xharr=27f7 xi=3be xlArr=27f8 xlarr=27f5 xmap=27fc xnis=22fb xodot=2a00 " +
|
|
1642
|
+
"xopf=1d569 xoplus=2a01 xotime=2a02 xrArr=27f9 xrarr=27f6 xscr=1d4cd xsqcup=2a06 xuplus=2a04 " +
|
|
1643
|
+
"xutri=25b3 xvee=22c1 xwedge=22c0 yacute!fd yacy=44f ycirc=177 ycy=44b yen!a5 yfr=1d536 yicy=457 " +
|
|
1644
|
+
"yopf=1d56a yscr=1d4ce yucy=44e yuml!ff zacute=17a zcaron=17e zcy=437 zdot=17c zeetrf=2128 " +
|
|
1645
|
+
"zeta=3b6 zfr=1d537 zhcy=436 zigrarr=21dd zopf=1d56b zscr=1d4cf zwj=200d zwnj=200c";
|
|
1646
|
+
let namedReferences = null;
|
|
1647
|
+
function namedReference(key) {
|
|
1648
|
+
if (namedReferences === null) {
|
|
1649
|
+
namedReferences = new Map();
|
|
1650
|
+
for (const entry of NAMED_REFERENCE_DATA.split(" ")) {
|
|
1651
|
+
const sep = entry.search(/[!=]/);
|
|
1652
|
+
const name = entry.slice(0, sep);
|
|
1653
|
+
const value = String.fromCodePoint(...entry.slice(sep + 1).split(",").map((h) => parseInt(h, 16)));
|
|
1654
|
+
namedReferences.set(name + ";", value);
|
|
1655
|
+
if (entry[sep] === "!")
|
|
1656
|
+
namedReferences.set(name, value);
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
return namedReferences.get(key);
|
|
1660
|
+
}
|
|
1661
|
+
function decodeAttributeValue(value) {
|
|
1662
|
+
if (!value.includes("&") && !value.includes("\u0000"))
|
|
1663
|
+
return value;
|
|
1664
|
+
const s = value.replace(/\u0000/g, "\uFFFD");
|
|
1665
|
+
const n = s.length;
|
|
1666
|
+
let out = "";
|
|
1667
|
+
let i = 0;
|
|
1668
|
+
for (;;) {
|
|
1669
|
+
const amp = s.indexOf("&", i);
|
|
1670
|
+
if (amp === -1)
|
|
1671
|
+
return out + s.slice(i);
|
|
1672
|
+
out += s.slice(i, amp);
|
|
1673
|
+
i = amp + 1;
|
|
1674
|
+
if (s[i] === "#") {
|
|
1675
|
+
let j = i + 1;
|
|
1676
|
+
const hex = s[j] === "x" || s[j] === "X";
|
|
1677
|
+
if (hex)
|
|
1678
|
+
j++;
|
|
1679
|
+
const digits = j;
|
|
1680
|
+
let code = 0;
|
|
1681
|
+
for (; j < n; j++) {
|
|
1682
|
+
const c = s[j];
|
|
1683
|
+
const d = c >= "0" && c <= "9" ? c.charCodeAt(0) - 48
|
|
1684
|
+
: hex && c >= "a" && c <= "f" ? c.charCodeAt(0) - 87
|
|
1685
|
+
: hex && c >= "A" && c <= "F" ? c.charCodeAt(0) - 55
|
|
1686
|
+
: -1;
|
|
1687
|
+
if (d === -1)
|
|
1688
|
+
break;
|
|
1689
|
+
code = Math.min(code * (hex ? 16 : 10) + d, 0x110000);
|
|
1690
|
+
}
|
|
1691
|
+
if (j === digits) {
|
|
1692
|
+
out += "&";
|
|
1693
|
+
continue;
|
|
1694
|
+
}
|
|
1695
|
+
if (s[j] === ";")
|
|
1696
|
+
j++;
|
|
1697
|
+
if (code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff))
|
|
1698
|
+
code = 0xfffd;
|
|
1699
|
+
else if (C1_REPLACEMENTS[code])
|
|
1700
|
+
code = C1_REPLACEMENTS[code];
|
|
1701
|
+
out += String.fromCodePoint(code);
|
|
1702
|
+
i = j;
|
|
1703
|
+
continue;
|
|
1704
|
+
}
|
|
1705
|
+
// A name is at most 31 letters and digits, so the run is read no further than 32.
|
|
1706
|
+
let end = i;
|
|
1707
|
+
while (end < n && end - i < 32 && isAsciiAlphanumeric(s[end]))
|
|
1708
|
+
end++;
|
|
1709
|
+
let match, matchEnd = -1;
|
|
1710
|
+
if (end > i && s[end] === ";") {
|
|
1711
|
+
match = namedReference(s.slice(i, end) + ";");
|
|
1712
|
+
if (match !== undefined)
|
|
1713
|
+
matchEnd = end + 1;
|
|
1714
|
+
}
|
|
1715
|
+
for (let k = end; matchEnd === -1 && k > i; k--) {
|
|
1716
|
+
match = namedReference(s.slice(i, k));
|
|
1717
|
+
if (match !== undefined)
|
|
1718
|
+
matchEnd = k;
|
|
1719
|
+
}
|
|
1720
|
+
if (matchEnd === -1) {
|
|
1721
|
+
out += "&";
|
|
1722
|
+
continue;
|
|
1723
|
+
}
|
|
1724
|
+
if (s[matchEnd - 1] !== ";" && (s[matchEnd] === "=" || isAsciiAlphanumeric(s[matchEnd]))) {
|
|
1725
|
+
out += "&";
|
|
1726
|
+
continue;
|
|
1727
|
+
}
|
|
1728
|
+
out += match;
|
|
1729
|
+
i = matchEnd;
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
// The head is where a real parser says it ends, not where the text "</head>" happens to
|
|
1733
|
+
// appear. Erik's decision 2026-08-24: the two v2 checks describe what the page's HEAD
|
|
1734
|
+
// points at, so a link element the parser moves into the body is not one of them. That is
|
|
1735
|
+
// the strict reading, and it is what the HTML parsing spec's "in head" insertion mode
|
|
1736
|
+
// does: whitespace, comments, doctype and the head-only elements keep the head open;
|
|
1737
|
+
// the first text node, the first body-level element and </head>, </body>, </html> or
|
|
1738
|
+
// </br> close it; any other end tag in the head is a parse error and is ignored.
|
|
1739
|
+
var HEAD_ELEMENTS = ["base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "template", "title", "noscript"];
|
|
1740
|
+
// Once </head> has been seen the parser is in "after head", and that list is the one above
|
|
1741
|
+
// WITHOUT noscript: a noscript there opens the body instead of staying in the head.
|
|
1742
|
+
var AFTER_HEAD_ELEMENTS = ["base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "template", "title"];
|
|
1743
|
+
// Their content is not markup: script, style, title, noscript (a parser with scripting on
|
|
1744
|
+
// reads it as raw text) and noframes are raw text or RCDATA, and template content is inert.
|
|
1745
|
+
var HEAD_SKIPPED_CONTENT = ["script", "style", "title", "noscript", "noframes", "template"];
|
|
1746
|
+
// The end of a raw text or RCDATA element: the first </name that is followed by optional
|
|
1747
|
+
// whitespace, an optional "/" (a parser closes on </script/> too) and a ">". Returns the
|
|
1748
|
+
// index after it, or -1 when the element never closes, which means the rest of the
|
|
1749
|
+
// document is inside it.
|
|
1750
|
+
function skipSpace(lower, j) {
|
|
1751
|
+
while (j < lower.length && (lower[j] === " " || lower[j] === "\t" || lower[j] === "\n" || lower[j] === "\r" || lower[j] === "\f"))
|
|
1752
|
+
j++;
|
|
1753
|
+
return j;
|
|
1754
|
+
}
|
|
1755
|
+
// True when the character ends a tag name: whitespace, "/", ">" or the end of the input.
|
|
1756
|
+
function isTagBoundary(c) {
|
|
1757
|
+
return c === undefined || c === ">" || c === "/" || c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f";
|
|
1758
|
+
}
|
|
1759
|
+
// The index just after "</name ... >", or -1 when that is not an end tag there.
|
|
1760
|
+
function endTagAt(lower, name, at) {
|
|
1761
|
+
if (!lower.startsWith("</" + name, at))
|
|
1762
|
+
return -1;
|
|
1763
|
+
let j = skipSpace(lower, at + name.length + 2);
|
|
1764
|
+
if (lower[j] === "/")
|
|
1765
|
+
j = skipSpace(lower, j + 1);
|
|
1766
|
+
return lower[j] === ">" ? j + 1 : -1;
|
|
1767
|
+
}
|
|
1768
|
+
function rawTextEnd(lower, name, from) {
|
|
1769
|
+
for (let at = lower.indexOf("</" + name, from); at !== -1; at = lower.indexOf("</" + name, at + name.length + 2)) {
|
|
1770
|
+
const end = endTagAt(lower, name, at);
|
|
1771
|
+
if (end !== -1)
|
|
1772
|
+
return end;
|
|
1773
|
+
}
|
|
1774
|
+
return -1;
|
|
1775
|
+
}
|
|
1776
|
+
// script is not plain raw text: <!-- puts the tokenizer in the escaped state, a nested
|
|
1777
|
+
// <script there puts it in the double escaped state, and in THAT state </script only ends
|
|
1778
|
+
// the escape, not the element. Without this the legacy shape
|
|
1779
|
+
// <script><!-- ... <script>...</script> ... --></script> ended early and the rest of the
|
|
1780
|
+
// script was read as markup, which is the wrong direction. 4 of 240 000 fuzz inputs.
|
|
1781
|
+
function scriptEnd(lower, from) {
|
|
1782
|
+
let i = from, escaped = false, doubleEscaped = false;
|
|
1783
|
+
while (i < lower.length) {
|
|
1784
|
+
if (lower.startsWith("<!--", i)) {
|
|
1785
|
+
escaped = true;
|
|
1786
|
+
i += 4;
|
|
1787
|
+
continue;
|
|
1788
|
+
}
|
|
1789
|
+
if (escaped && lower.startsWith("-->", i)) {
|
|
1790
|
+
escaped = false;
|
|
1791
|
+
doubleEscaped = false;
|
|
1792
|
+
i += 3;
|
|
1793
|
+
continue;
|
|
1794
|
+
}
|
|
1795
|
+
if (escaped && !doubleEscaped && lower.startsWith("<script", i) && isTagBoundary(lower[i + 7])) {
|
|
1796
|
+
doubleEscaped = true;
|
|
1797
|
+
i += 7;
|
|
1798
|
+
continue;
|
|
1799
|
+
}
|
|
1800
|
+
if (lower.startsWith("</script", i)) {
|
|
1801
|
+
if (doubleEscaped) {
|
|
1802
|
+
doubleEscaped = false;
|
|
1803
|
+
i += 8;
|
|
1804
|
+
continue;
|
|
1805
|
+
}
|
|
1806
|
+
const end = endTagAt(lower, "script", i);
|
|
1807
|
+
if (end !== -1)
|
|
1808
|
+
return end;
|
|
1809
|
+
i += 8;
|
|
1810
|
+
continue;
|
|
1811
|
+
}
|
|
1812
|
+
const next = lower.indexOf("<", i + 1);
|
|
1813
|
+
i = next === -1 ? lower.length : next;
|
|
1814
|
+
}
|
|
1815
|
+
return -1;
|
|
1816
|
+
}
|
|
1817
|
+
// The end of a nested template: templates count, so an inner </template> does not close
|
|
1818
|
+
// an outer one. Returns the index after the closing tag, or -1 when it never closes. A comment
|
|
1819
|
+
// or a raw text element inside the template is skipped whole before its markers are counted.
|
|
1820
|
+
// Raw text and RCDATA elements whose content is text for the tokenizer, so a "<template" or a
|
|
1821
|
+
// "</template" inside one is not a tag. script is read by scriptEnd, the others by rawTextEnd.
|
|
1822
|
+
const TEMPLATE_RAW_TEXT = ["script", "style", "title", "noscript", "noframes", "textarea", "xmp", "iframe", "noembed"];
|
|
1823
|
+
function templateEnd(lower, from) {
|
|
1824
|
+
let depth = 1, i = from;
|
|
1825
|
+
// One forward scan: every "<" is looked at once and the cursor never moves back, so the cost
|
|
1826
|
+
// stays linear (restarting a search each round measured 15 686 ms on 1 MB of "</templateX").
|
|
1827
|
+
// Comments and raw text elements are stepped over whole, because the markers inside them are
|
|
1828
|
+
// text for a parser and counted before as if they were tags (Tek-560).
|
|
1829
|
+
for (;;) {
|
|
1830
|
+
const lt = lower.indexOf("<", i);
|
|
1831
|
+
if (lt === -1)
|
|
1832
|
+
return -1;
|
|
1833
|
+
if (lower.startsWith("<!--", lt)) {
|
|
1834
|
+
if (lower.startsWith("<!-->", lt)) {
|
|
1835
|
+
i = lt + 5;
|
|
1836
|
+
continue;
|
|
1837
|
+
}
|
|
1838
|
+
if (lower.startsWith("<!--->", lt)) {
|
|
1839
|
+
i = lt + 6;
|
|
1840
|
+
continue;
|
|
1841
|
+
}
|
|
1842
|
+
const cend = commentEnd(lower, lt + 4);
|
|
1843
|
+
if (cend === -1)
|
|
1844
|
+
return -1;
|
|
1845
|
+
i = cend;
|
|
1846
|
+
continue;
|
|
1847
|
+
}
|
|
1848
|
+
if (lower.startsWith("</template", lt)) {
|
|
1849
|
+
let j = skipSpace(lower, lt + 10);
|
|
1850
|
+
if (lower[j] === "/")
|
|
1851
|
+
j = skipSpace(lower, j + 1);
|
|
1852
|
+
if (lower[j] !== ">") {
|
|
1853
|
+
// Not an end tag: for the tokenizer the rest of the name runs to the next ">", and a
|
|
1854
|
+
// "<" inside it is part of the name rather than a new tag.
|
|
1855
|
+
const bogus = lower.indexOf(">", lt + 10);
|
|
1856
|
+
if (bogus === -1)
|
|
1857
|
+
return -1;
|
|
1858
|
+
i = bogus + 1;
|
|
1859
|
+
continue;
|
|
1860
|
+
}
|
|
1861
|
+
depth--;
|
|
1862
|
+
if (depth === 0)
|
|
1863
|
+
return j + 1;
|
|
1864
|
+
i = j + 1;
|
|
1865
|
+
continue;
|
|
1866
|
+
}
|
|
1867
|
+
if (lower.startsWith("<template", lt) && isTagBoundary(lower[lt + 9])) {
|
|
1868
|
+
depth++;
|
|
1869
|
+
i = lt + 9;
|
|
1870
|
+
continue;
|
|
1871
|
+
}
|
|
1872
|
+
const rawName = TEMPLATE_RAW_TEXT.find((n) => lower.startsWith(n, lt + 1) && isTagBoundary(lower[lt + 1 + n.length]));
|
|
1873
|
+
if (rawName) {
|
|
1874
|
+
const gt = tagEnd(lower, lt);
|
|
1875
|
+
if (gt === -1)
|
|
1876
|
+
return -1;
|
|
1877
|
+
const rend = rawName === "script" ? scriptEnd(lower, gt + 1) : rawTextEnd(lower, rawName, gt + 1);
|
|
1878
|
+
if (rend === -1)
|
|
1879
|
+
return -1;
|
|
1880
|
+
i = rend;
|
|
1881
|
+
continue;
|
|
1882
|
+
}
|
|
1883
|
+
i = lt + 1;
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
// The index after the "-->" or "--!>" that ends a comment whose text starts at from, or -1 when
|
|
1887
|
+
// the comment never ends. One forward pass over the "--" pairs. The earlier code searched for
|
|
1888
|
+
// "--!>" separately from every comment, and on a page that never uses that ending, which is
|
|
1889
|
+
// almost every page, each search ran to the end of the document: 100 KB of short comments
|
|
1890
|
+
// measured 770 ms and each doubling took four times as long (found by a verifier, 2026-09-12).
|
|
1891
|
+
function commentEnd(lower, from) {
|
|
1892
|
+
for (let p = lower.indexOf("--", from); p !== -1; p = lower.indexOf("--", p + 1)) {
|
|
1893
|
+
if (lower[p + 2] === ">")
|
|
1894
|
+
return p + 3;
|
|
1895
|
+
if (lower[p + 2] === "!" && lower[p + 3] === ">")
|
|
1896
|
+
return p + 4;
|
|
1897
|
+
}
|
|
1898
|
+
return -1;
|
|
1899
|
+
}
|
|
1900
|
+
// One left to right scan by index that returns the head, with comments and the content of
|
|
1901
|
+
// the raw text and template elements already removed. Why a scan and not a regex: a
|
|
1902
|
+
// character class that reads a tag's attributes is quadratic on input the TARGET site
|
|
1903
|
+
// controls (CodeQL js/polynomial-redos, alerts #4 and #5, 2026-08-24), and bounding the
|
|
1904
|
+
// class trades that speed bug for a worse correctness bug, because an open tag longer than
|
|
1905
|
+
// the bound stops being recognised and the element's own text is then read as markup. A
|
|
1906
|
+
// scan has no bound and no backtracking, and it visits every character once.
|
|
1907
|
+
//
|
|
1908
|
+
// The shapes, all measured against a real HTML parser (parse5) rather than reasoned about.
|
|
1909
|
+
// An unterminated <!-- comments out the rest of the document. <!--> and <!---> are EMPTY
|
|
1910
|
+
// comments, not unterminated ones, and --!> ends a comment as well. A "<" that no letter,
|
|
1911
|
+
// "!", "/" or "?" follows is text. A ">" inside a quoted attribute value does not end a
|
|
1912
|
+
// tag. </script/> closes a raw text element as well as </script> does.
|
|
1913
|
+
function headOfDocument(html) {
|
|
1914
|
+
// A parser tokenizes an input stream whose CRLF and CR have already become LF, so a value read
|
|
1915
|
+
// from the head carries LF where the page wrote CR (measured against parse5, 2026-09-12).
|
|
1916
|
+
const text = String(html || "").replace(/\r\n?/g, "\n");
|
|
1917
|
+
const lower = asciiLower(text);
|
|
1918
|
+
const out = [];
|
|
1919
|
+
let i = 0, afterHead = false;
|
|
1920
|
+
for (;;) {
|
|
1921
|
+
const lt = lower.indexOf("<", i);
|
|
1922
|
+
const gap = lt === -1 ? text.slice(i) : text.slice(i, lt);
|
|
1923
|
+
if (!isHtmlWhitespace(gap))
|
|
1924
|
+
break;
|
|
1925
|
+
out.push(gap);
|
|
1926
|
+
if (lt === -1)
|
|
1927
|
+
break;
|
|
1928
|
+
if (lower.startsWith("<!--", lt)) {
|
|
1929
|
+
if (lower.startsWith("<!-->", lt)) {
|
|
1930
|
+
i = lt + 5;
|
|
1931
|
+
continue;
|
|
1932
|
+
}
|
|
1933
|
+
if (lower.startsWith("<!--->", lt)) {
|
|
1934
|
+
i = lt + 6;
|
|
1935
|
+
continue;
|
|
1936
|
+
}
|
|
1937
|
+
const end = commentEnd(lower, lt + 4);
|
|
1938
|
+
if (end === -1)
|
|
1939
|
+
break;
|
|
1940
|
+
i = end;
|
|
1941
|
+
continue;
|
|
1942
|
+
}
|
|
1943
|
+
if (!startsTag(lower[lt + 1]))
|
|
1944
|
+
break;
|
|
1945
|
+
// "<!", "<?" and a "</" that no letter follows open a bogus comment or a doctype, and the
|
|
1946
|
+
// tokenizer ends each of them at the first ">", inside quotes or not.
|
|
1947
|
+
if (lower[lt + 1] === "!" || lower[lt + 1] === "?" || (lower[lt + 1] === "/" && !(lower[lt + 2] >= "a" && lower[lt + 2] <= "z"))) {
|
|
1948
|
+
const bogusEnd = lower.indexOf(">", lt + 2);
|
|
1949
|
+
if (bogusEnd === -1)
|
|
1950
|
+
break;
|
|
1951
|
+
i = bogusEnd + 1;
|
|
1952
|
+
continue;
|
|
1953
|
+
}
|
|
1954
|
+
const gt = tagEnd(text, lt);
|
|
1955
|
+
if (gt === -1)
|
|
1956
|
+
break;
|
|
1957
|
+
const head14 = lower.slice(lt, Math.min(lt + 14, gt + 1));
|
|
1958
|
+
if (lower[lt + 1] === "/") {
|
|
1959
|
+
// The name has to END where a tag name ends: </bodyx> is an unknown end tag, which the
|
|
1960
|
+
// head ignores, and not </body>.
|
|
1961
|
+
const endTag = (/^<\/([a-z]+)(?=[\t\n\f\r />]|$)/.exec(head14) || [])[1];
|
|
1962
|
+
// </body>, </html> and </br> start the body. </head> does NOT end the search: after
|
|
1963
|
+
// it a parser still puts base, link, meta, script, style, title and template into the
|
|
1964
|
+
// HEAD element until real body content starts, and every other end tag in the head is
|
|
1965
|
+
// a parse error that is ignored. Measured against parse5: without this, 2 660 of
|
|
1966
|
+
// 200 000 fuzz inputs lost a relation the head really carries.
|
|
1967
|
+
if (endTag === "body" || endTag === "html" || endTag === "br")
|
|
1968
|
+
break;
|
|
1969
|
+
if (endTag === "head")
|
|
1970
|
+
afterHead = true;
|
|
1971
|
+
i = gt + 1;
|
|
1972
|
+
continue;
|
|
1973
|
+
}
|
|
1974
|
+
const name = (/^<([a-z]+)(?=[\t\n\f\r />]|$)/.exec(head14) || [])[1];
|
|
1975
|
+
if (!name)
|
|
1976
|
+
break;
|
|
1977
|
+
if (name === "html" || name === "head") {
|
|
1978
|
+
i = gt + 1;
|
|
1979
|
+
continue;
|
|
1980
|
+
}
|
|
1981
|
+
if (!(afterHead ? AFTER_HEAD_ELEMENTS : HEAD_ELEMENTS).includes(name))
|
|
1982
|
+
break; // <body> and the first body level element
|
|
1983
|
+
if (HEAD_SKIPPED_CONTENT.includes(name)) {
|
|
1984
|
+
const end = name === "template" ? templateEnd(lower, gt + 1)
|
|
1985
|
+
: name === "script" ? scriptEnd(lower, gt + 1)
|
|
1986
|
+
: rawTextEnd(lower, name, gt + 1);
|
|
1987
|
+
if (end === -1)
|
|
1988
|
+
break;
|
|
1989
|
+
i = end;
|
|
1990
|
+
continue;
|
|
1991
|
+
}
|
|
1992
|
+
out.push(text.slice(lt, gt + 1));
|
|
1993
|
+
i = gt + 1;
|
|
1994
|
+
}
|
|
1995
|
+
return out.join("");
|
|
1996
|
+
}
|
|
1997
|
+
// Tags are found by index and not by /<link\b[^>]*>/g, on purpose. That regex is
|
|
1998
|
+
// quadratic on input the TARGET site controls: every "<link" with no ">" after it makes
|
|
1999
|
+
// the character class scan to the end of the document, and "<link" repeated 16 000 times
|
|
2000
|
+
// measured 196 ms where this loop measures under 1 ms. CodeQL reports the same shape as
|
|
2001
|
+
// js/polynomial-redos (alerts #4 and #5 on the package repo, 2026-08-24). This scan
|
|
2002
|
+
// visits every character once: from each "<" it reads to the next ">" and then continues
|
|
2003
|
+
// after it, which is what the regex meant to say.
|
|
2004
|
+
function* htmlTags(text) {
|
|
2005
|
+
const lower = asciiLower(text);
|
|
2006
|
+
let i = 0;
|
|
2007
|
+
for (;;) {
|
|
2008
|
+
const open = lower.indexOf("<", i);
|
|
2009
|
+
if (open === -1)
|
|
2010
|
+
return;
|
|
2011
|
+
if (!startsTag(lower[open + 1])) {
|
|
2012
|
+
i = open + 1;
|
|
2013
|
+
continue;
|
|
2014
|
+
}
|
|
2015
|
+
const close = tagEnd(text, open);
|
|
2016
|
+
if (close === -1)
|
|
2017
|
+
return;
|
|
2018
|
+
yield text.slice(open, close + 1);
|
|
2019
|
+
i = close + 1;
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
function tagAttributes(tag) {
|
|
2023
|
+
const out = {};
|
|
2024
|
+
readTag(tag, 0, out);
|
|
2025
|
+
return out;
|
|
2026
|
+
}
|
|
2027
|
+
// RFC 8288 section 3: a Link header is a comma separated list, each value an angle
|
|
2028
|
+
// bracketed URI reference followed by semicolon separated parameters, and a parameter
|
|
2029
|
+
// value may be a quoted string that contains commas, semicolons and backslash escapes.
|
|
2030
|
+
// Reading rel and type with one regex over the whole value read the CONTENT of
|
|
2031
|
+
// title="note; rel=alternate; type=text/markdown" as parameters of the link.
|
|
2032
|
+
function parseLinkHeader(header) {
|
|
2033
|
+
const s = String(header || "");
|
|
2034
|
+
const space = " \t";
|
|
2035
|
+
const n = s.length;
|
|
2036
|
+
const out = [];
|
|
2037
|
+
let i = 0;
|
|
2038
|
+
while (i < n) {
|
|
2039
|
+
while (i < n && (s[i] === "," || space.includes(s[i])))
|
|
2040
|
+
i++;
|
|
2041
|
+
if (i >= n)
|
|
2042
|
+
break;
|
|
2043
|
+
if (s[i] !== "<") {
|
|
2044
|
+
i = nextLinkValue(s, i);
|
|
2045
|
+
continue;
|
|
2046
|
+
}
|
|
2047
|
+
const gt = s.indexOf(">", i + 1);
|
|
2048
|
+
if (gt === -1)
|
|
2049
|
+
break;
|
|
2050
|
+
const href = s.slice(i + 1, gt).trim();
|
|
2051
|
+
i = gt + 1;
|
|
2052
|
+
const params = {};
|
|
2053
|
+
while (i < n) {
|
|
2054
|
+
while (i < n && space.includes(s[i]))
|
|
2055
|
+
i++;
|
|
2056
|
+
if (i >= n)
|
|
2057
|
+
break;
|
|
2058
|
+
if (s[i] === ",") {
|
|
2059
|
+
i++;
|
|
2060
|
+
break;
|
|
2061
|
+
}
|
|
2062
|
+
if (s[i] !== ";") {
|
|
2063
|
+
i = nextLinkValue(s, i);
|
|
2064
|
+
break;
|
|
2065
|
+
}
|
|
2066
|
+
i++;
|
|
2067
|
+
while (i < n && space.includes(s[i]))
|
|
2068
|
+
i++;
|
|
2069
|
+
const nameStart = i;
|
|
2070
|
+
while (i < n && s[i] !== "=" && s[i] !== ";" && s[i] !== "," && !space.includes(s[i]))
|
|
2071
|
+
i++;
|
|
2072
|
+
const name = asciiLower(s.slice(nameStart, i));
|
|
2073
|
+
while (i < n && space.includes(s[i]))
|
|
2074
|
+
i++;
|
|
2075
|
+
let value = "";
|
|
2076
|
+
if (s[i] === "=") {
|
|
2077
|
+
i++;
|
|
2078
|
+
while (i < n && space.includes(s[i]))
|
|
2079
|
+
i++;
|
|
2080
|
+
if (s[i] === '"') {
|
|
2081
|
+
i++;
|
|
2082
|
+
let buf = "";
|
|
2083
|
+
while (i < n && s[i] !== '"') {
|
|
2084
|
+
if (s[i] === "\\" && i + 1 < n) {
|
|
2085
|
+
buf += s[i + 1];
|
|
2086
|
+
i += 2;
|
|
2087
|
+
continue;
|
|
2088
|
+
}
|
|
2089
|
+
buf += s[i];
|
|
2090
|
+
i++;
|
|
2091
|
+
}
|
|
2092
|
+
if (i < n)
|
|
2093
|
+
i++;
|
|
2094
|
+
value = buf;
|
|
2095
|
+
}
|
|
2096
|
+
else {
|
|
2097
|
+
const valueStart = i;
|
|
2098
|
+
while (i < n && s[i] !== ";" && s[i] !== "," && !space.includes(s[i]))
|
|
2099
|
+
i++;
|
|
2100
|
+
value = s.slice(valueStart, i);
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
if (name && !(name in params))
|
|
2104
|
+
params[name] = value;
|
|
2105
|
+
}
|
|
2106
|
+
out.push({ href: href, params: params });
|
|
2107
|
+
}
|
|
2108
|
+
return out;
|
|
2109
|
+
}
|
|
2110
|
+
// Move to the start of the next comma separated value without stopping inside a quoted
|
|
2111
|
+
// string, so a comma in title="a, b" does not split one value into two.
|
|
2112
|
+
function nextLinkValue(s, from) {
|
|
2113
|
+
let quote = "";
|
|
2114
|
+
for (let j = from; j < s.length; j++) {
|
|
2115
|
+
const c = s[j];
|
|
2116
|
+
if (quote) {
|
|
2117
|
+
if (c === "\\") {
|
|
2118
|
+
j++;
|
|
2119
|
+
continue;
|
|
2120
|
+
}
|
|
2121
|
+
if (c === quote)
|
|
2122
|
+
quote = "";
|
|
2123
|
+
continue;
|
|
2124
|
+
}
|
|
2125
|
+
if (c === '"') {
|
|
2126
|
+
quote = c;
|
|
2127
|
+
continue;
|
|
2128
|
+
}
|
|
2129
|
+
if (c === ",")
|
|
2130
|
+
return j + 1;
|
|
2131
|
+
}
|
|
2132
|
+
return s.length;
|
|
2133
|
+
}
|
|
2134
|
+
// v2 of the llms.txt proposal (August 2026) left the file format alone and added one
|
|
2135
|
+
// thing: a page should say where its markdown version and its llms.txt are, using
|
|
2136
|
+
// rel="alternate" type="text/markdown" and rel="describedby", as HTML link elements or
|
|
2137
|
+
// as a Link response header. That is a property of the site, not of the file, so these
|
|
2138
|
+
// two carry their own status, "info". They are never a warn and never a fail, the
|
|
2139
|
+
// summary line is unchanged, and --strict keeps exiting on warnings only.
|
|
2140
|
+
export function findLinkRelations(html, linkHeader) {
|
|
2141
|
+
const found = { describedby: null, markdown: null };
|
|
2142
|
+
// Only the head, and only what a parser would put there: a commented-out link element,
|
|
2143
|
+
// one inside a script or a template, and one the parser moves into the body are all not
|
|
2144
|
+
// what these two checks are about, and counting them would report a relation the site
|
|
2145
|
+
// does not serve. See headOfDocument above for the rules shape by shape.
|
|
2146
|
+
// The scan reads only the first 65,536 UTF-16 code units of what headOfDocument returns.
|
|
2147
|
+
// The cap applies after the head is collected, so it bounds this tag scan and not the head
|
|
2148
|
+
// parse, and a relation declared past it is not found here. The Link response header is
|
|
2149
|
+
// read separately below.
|
|
2150
|
+
const whole = headOfDocument(html);
|
|
2151
|
+
const head = whole.slice(0, 65536);
|
|
2152
|
+
for (const tag of htmlTags(head)) {
|
|
2153
|
+
// The name has to END at "link": a real parser reads "<link<link" as ONE tag whose
|
|
2154
|
+
// NAME is "link<link", not as a link element, so \b would count a relation the site
|
|
2155
|
+
// does not publish. Measured against parse5, 2026-08-24.
|
|
2156
|
+
if (!/^<link(?=[\t\n\f\r />])/i.test(tag))
|
|
2157
|
+
continue;
|
|
2158
|
+
// Attributes come from ONE quote aware scan, not from three independent regex matches
|
|
2159
|
+
// over the whole tag. Those three could not tell an attribute from the quoted CONTENT
|
|
2160
|
+
// of another attribute, so data-note=" rel='alternate' type='text/markdown'
|
|
2161
|
+
// href='/fake.md'" reported a relation the page does not publish. The earlier
|
|
2162
|
+
// data-rel fix, 2026-08-29, moved the START of the name and does not reach this.
|
|
2163
|
+
// Measured and fixed 2026-09-09.
|
|
2164
|
+
const attrs = tagAttributes(tag);
|
|
2165
|
+
const rel = trimChars(asciiLower(attrs.rel || ""), HTML_WHITESPACE).split(/[\t\n\f\r ]+/);
|
|
2166
|
+
const type = trimChars(asciiLower(attrs.type || ""), HTTP_WHITESPACE);
|
|
2167
|
+
const href = trimChars(attrs.href || "", HTML_WHITESPACE);
|
|
2168
|
+
// text/markdown, not anything that starts with it, and a relation without a target is
|
|
2169
|
+
// not a relation: both passed until 2026-08-29.
|
|
2170
|
+
const isMarkdown = trimChars(type.split(";")[0], HTTP_WHITESPACE) === "text/markdown";
|
|
2171
|
+
if (!found.describedby && href && rel.includes("describedby"))
|
|
2172
|
+
found.describedby = href;
|
|
2173
|
+
if (!found.markdown && href && rel.includes("alternate") && isMarkdown)
|
|
2174
|
+
found.markdown = href;
|
|
2175
|
+
}
|
|
2176
|
+
for (const link of parseLinkHeader(linkHeader)) {
|
|
2177
|
+
const href = link.href;
|
|
2178
|
+
const rel = trimChars(asciiLower(link.params.rel || ""), HTTP_WHITESPACE).split(/[\t\n\r ]+/);
|
|
2179
|
+
const type = trimChars(asciiLower(link.params.type || ""), HTTP_WHITESPACE);
|
|
2180
|
+
const isMarkdownHeader = trimChars(type.split(";")[0], HTTP_WHITESPACE) === "text/markdown";
|
|
2181
|
+
if (!found.describedby && href && rel.includes("describedby"))
|
|
2182
|
+
found.describedby = href;
|
|
2183
|
+
if (!found.markdown && href && rel.includes("alternate") && isMarkdownHeader)
|
|
2184
|
+
found.markdown = href;
|
|
2185
|
+
}
|
|
2186
|
+
// A head longer than the cap was read in part, so a relation missing from that part is
|
|
2187
|
+
// reported as missing from the part read, not from the page (Tek-496).
|
|
2188
|
+
if (whole.length > 65536)
|
|
2189
|
+
found.headCut = true;
|
|
2190
|
+
return found;
|
|
2191
|
+
}
|
|
2192
|
+
export function validateV2Discovery(found, unreadReason) {
|
|
2193
|
+
const checks = [];
|
|
2194
|
+
const add = (id, status, label, detail) => checks.push({ id, status, label, detail });
|
|
2195
|
+
if (!found) {
|
|
2196
|
+
add("v2-describedby", "info", "Home page points to its llms.txt (v2)", unreadReason);
|
|
2197
|
+
add("v2-markdown-alternate", "info", "Home page points to a markdown version (v2)", unreadReason);
|
|
2198
|
+
return checks;
|
|
2199
|
+
}
|
|
2200
|
+
// A target is shown through maskLocation, as a redirect target is, so a user name, a password,
|
|
2201
|
+
// a query value or a fragment in a published link stays out of the report; until 2026-09-27 it
|
|
2202
|
+
// was only cut (Tek-496). A relative target stays relative.
|
|
2203
|
+
const where = found.headCut ? "in the first 65,536 characters of the head, the part that was read, or in the Link header" : "in the head or the Link header";
|
|
2204
|
+
add("v2-describedby", found.describedby ? "pass" : "info", "Home page points to its llms.txt (v2)", found.describedby ? 'rel="describedby" to ' + cut(maskLocation(found.describedby), 120) : 'no rel="describedby" ' + where + '; v2 recommends it so an agent finds the file without guessing');
|
|
2205
|
+
add("v2-markdown-alternate", found.markdown ? "pass" : "info", "Home page points to a markdown version (v2)", found.markdown ? 'rel="alternate" type="text/markdown" to ' + cut(maskLocation(found.markdown), 120) : 'no rel="alternate" type="text/markdown" ' + where + '; v2 recommends it so an agent finds the markdown form without guessing');
|
|
2206
|
+
return checks;
|
|
2207
|
+
}
|
|
2208
|
+
export function summarizeChecks(checks) {
|
|
2209
|
+
if (checks.some((c) => c.status === "fail"))
|
|
2210
|
+
return "not valid";
|
|
2211
|
+
if (checks.some((c) => c.status === "warn"))
|
|
2212
|
+
return "valid with warnings";
|
|
2213
|
+
return "valid";
|
|
2214
|
+
}
|
|
2215
|
+
export async function validateHost(input, opts = {}) {
|
|
2216
|
+
const host = normalizeHostInput(input);
|
|
2217
|
+
if (!host || !isValidPublicHost(host)) {
|
|
2218
|
+
// The input is repeated only when it cannot carry a secret. An entry with an @, a ? or a #
|
|
2219
|
+
// can hold a user name, a password or a query value, and until 0.3.8 the message copied
|
|
2220
|
+
// its first 120 characters into stderr and the --json error (Tek-496). Such an entry is
|
|
2221
|
+
// named by the host the parser reads from it, or not at all, as the hosted form does.
|
|
2222
|
+
const raw = String(input ?? "");
|
|
2223
|
+
const shown = /[@?#]/.test(raw) ? normalizeHostInput(raw) || "" : stripControls(cut(raw, 120));
|
|
2224
|
+
throw new Error(shown ? "not a public domain name: " + shown : "not a public domain name");
|
|
2225
|
+
}
|
|
2226
|
+
// opts comes from the caller, and path and accept exist for this function's own
|
|
2227
|
+
// second read. Forwarding them to the first read would let a caller point the
|
|
2228
|
+
// "llms.txt" result at any path on the host, so both are pinned here.
|
|
2229
|
+
const fetched = await fetchLlmsTxt(host, { ...opts, path: "/llms.txt", accept: undefined });
|
|
2230
|
+
let discovery;
|
|
2231
|
+
try {
|
|
2232
|
+
const home = await fetchLlmsTxt(host, { ...opts, path: "/", accept: "text/html, */*;q=0.1" });
|
|
2233
|
+
discovery = home.redirect
|
|
2234
|
+
? validateV2Discovery(null, "the home page redirects away from this host, so this was not measured")
|
|
2235
|
+
: home.status !== 200
|
|
2236
|
+
? validateV2Discovery(null, "the home page returned HTTP " + home.status + ", so this was not measured")
|
|
2237
|
+
: validateV2Discovery(findLinkRelations(home.text, home.linkHeader));
|
|
2238
|
+
}
|
|
2239
|
+
catch {
|
|
2240
|
+
discovery = validateV2Discovery(null, "the home page could not be read, so this was not measured");
|
|
2241
|
+
}
|
|
2242
|
+
const checks = validateLlmsTxt(fetched).concat(discovery);
|
|
2243
|
+
// A path in the typed address is not used, and the result says so first (mirrored from worker.js
|
|
2244
|
+
// v3.176.0). Information moves no summary and no exit code.
|
|
2245
|
+
const unusedPath = enteredPath(input);
|
|
2246
|
+
// The typed path goes on its own field with the same 120 character cut it always had (VREG,
|
|
2247
|
+
// round 3 Tek-542, mirrored from worker.js decision 19, Tek-526), instead of being built into
|
|
2248
|
+
// the sentence, so a JSON caller reads the raw value from one field instead of parsing it
|
|
2249
|
+
// back out of a message meant for people.
|
|
2250
|
+
if (unusedPath)
|
|
2251
|
+
checks.unshift({ id: "input-path", status: "info", label: "Path in the address you entered", value: cut(unusedPath, 120), detail: "This path is not used, because the validator always reads /llms.txt at the root of the host." });
|
|
2252
|
+
// Text quoted from the fetched file leaves without bidirectional controls (round 19,
|
|
2253
|
+
// V6-U1, mirrored from worker.js v3.164.0): a site's own H1 or summary could otherwise
|
|
2254
|
+
// read in another order here than in the file itself.
|
|
2255
|
+
const stripped = checks.map((c) => (typeof c.detail === "string" ? { ...c, detail: stripControls(stripBidi(c.detail)) } : c));
|
|
2256
|
+
return { target: "https://" + host + "/llms.txt", summary: summarizeChecks(stripped), checks: stripped };
|
|
2257
|
+
}
|