turva-llms-txt-validator 0.3.0 → 0.3.2

package/README.md

@@ -1,6 +1,6 @@
 # turva-llms-txt-validator
 
-Validate a site's llms.txt structure from the command line, from Node or in CI. Eight structure checks, each reported as pass, warn or fail with a one-line detail, and two v2 discovery checks reported as information. No score on purpose: a structure check can say what is there and what is missing, and a number on top of ten checks would look like an agent-readiness score without measuring one.
+Validate a site's llms.txt structure from the command line, from Node or in CI. Eight structure checks, each reported as pass, warn or fail with a one-line detail, and two v2 discovery checks that read pass or info and never move the summary. No score on purpose: a structure check can say what is there and what is missing, and a number on top of ten checks would look like an agent-readiness score without measuring one.
 
 This is the open-source form of the hosted validator at [turva.dev/llms-txt-validator](https://turva.dev/llms-txt-validator), which runs the same logic inside the open [turva.dev Cloudflare Worker](https://github.com/erekola/turva-worker). The hosted validator stays canonical: if the two ever disagree, the hosted one wins and this package gets the fix.
 

package/bin/cli.mjs

@@ -31,6 +31,11 @@ try {
   if (result.summary === "valid with warnings" && flags.has("--strict")) process.exit(1);
   process.exit(0);
 } catch (err) {
-  console.error("error: " + (err && err.message ? err.message : String(err)));
+  const message = err && err.message ? err.message : String(err);
+  // The hosted validator answers every failure as JSON when JSON was asked for, so a CI
+  // step that parses --json output gets {"error": ...} here too instead of an empty stdout
+  // and a plain-text stderr (round 16 S3-3). The exit code is unchanged.
+  if (flags.has("--json")) console.log(JSON.stringify({ error: message }, null, 2));
+  else console.error("error: " + message);
   process.exit(2);
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.3.0",
-  "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks as information, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
+  "version": "0.3.2",
+  "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks that read pass or info and never move the summary, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",
   "exports": "./src/index.mjs",

package/src/index.mjs

@@ -12,8 +12,20 @@ import { readFileSync } from "node:fs";
 export const VERSION = JSON.parse(
   readFileSync(new URL("../package.json", import.meta.url), "utf8")
 ).version;
+// The hosted validator identifies itself without a version, because its version is the
+// site's and moves with every release; this package carries its own. The two strings
+// differ on purpose and nothing compares them (round 16 S3-6).
 const UA = "turva-llms-txt-validator/" + VERSION + " (+https://turva.dev/llms-txt-validator)";
 
+// Cut a string to at most n UTF-16 code units without leaving a lone high surrogate at the
+// end. String.prototype.slice counts code units, so a cut that lands inside a surrogate
+// pair leaves half of it, and that half serialises as bytes that are not valid UTF-8
+// (round 16 S3-1 and S3-2, measured on the hosted validator). Mirrors worker.js cut().
+export function cut(s, n) {
+  s = String(s).slice(0, n);
+  return /[\uD800-\uDBFF]$/.test(s) ? s.slice(0, -1) : s;
+}
+
 export function normalizeHostInput(raw) {
   let s = String(raw || "").trim().toLowerCase();
   if (!s) return null;
@@ -58,13 +70,13 @@ export async function fetchLlmsTxt(host, opts = {}) {
     if (res.status >= 300 && res.status < 400) {
       const loc = res.headers.get("location") || "";
       if (!loc) return { redirect: true, reason: "no-location", status: res.status, location: "" };
-      if (hop >= 4) return { redirect: true, reason: "too-many", status: res.status, location: loc.slice(0, 120) };
+      if (hop >= 4) return { redirect: true, reason: "too-many", status: res.status, location: cut(loc, 120) };
       let next;
-      try { next = new URL(loc, url); } catch { return { redirect: true, reason: "bad-location", status: res.status, location: loc.slice(0, 120) }; }
+      try { next = new URL(loc, url); } catch { return { redirect: true, reason: "bad-location", status: res.status, location: cut(loc, 120) }; }
       const safeTarget = next.protocol === "https:" && !next.port && !next.username && !next.password && isValidPublicHost(next.hostname);
       const twin = (next.hostname.startsWith("www.") ? next.hostname.slice(4) : next.hostname) === reqApex;
-      if (!safeTarget) return { redirect: true, reason: "unsafe-target", status: res.status, location: next.href.slice(0, 120) };
-      if (!twin) return { redirect: true, reason: "off-host", status: res.status, location: next.href.slice(0, 120) };
+      if (!safeTarget) return { redirect: true, reason: "unsafe-target", status: res.status, location: cut(next.href, 120) };
+      if (!twin) return { redirect: true, reason: "off-host", status: res.status, location: cut(next.href, 120) };
       if (!redirectedFrom) redirectedFrom = url;
       url = next.href;
       continue;
@@ -103,6 +115,68 @@ export async function fetchLlmsTxt(host, opts = {}) {
   }
 }
 
+// Every markdown link in the file, scanned once from left to right instead of collected with
+// matchAll(/\[([^\][]*)\]\(([^)\s]{1,2048})\)/g). That bound meant a target longer than 2048
+// characters was not counted as a link at all, and dropping the bound from the pattern would
+// make it quadratic on a file that repeats "[a](" (Erik 2026-08-29). The scan carries no bound
+// and no backtracking: every character is read once and the furthest failed target scan is
+// remembered. matchAll resumes after a whole match, and so does this.
+function collectLinks(text) {
+  const out = [];
+  let failEnd = -1;
+  for (let i = 0; i < text.length; i++) {
+    if (text[i] !== "[") continue;
+    let j = i + 1;
+    while (j < text.length && text[j] !== "]" && text[j] !== "[") j++;
+    if (j >= text.length) break;
+    if (text[j] === "[") { i = j - 1; continue; }
+    if (text[j + 1] !== "(") { i = j; continue; }
+    const k = j + 2;
+    if (k <= failEnd) { i = j + 1; continue; }
+    let e = k;
+    while (e < text.length && text[e] !== ")" && !/\s/.test(text[e])) e++;
+    if (e > k && text[e] === ")") { out.push({ name: text.slice(i + 1, j), target: text.slice(k, e) }); i = e; continue; }
+    if (text[e] !== ")") failEnd = e;
+    i = j + 1;
+  }
+  return out;
+}
+
+// A markdown list item that carries a link, scanned once from left to right instead of
+// matched with /^ {0,3}[-*+] .*\[[^\][]*\]\([^)\s]+\)/. That pattern is quadratic on a line
+// such as "- " followed by "[a](" repeated, because every candidate rescans the target to the
+// end of the line, and the line comes from the audited site (CodeQL js/polynomial-redos,
+// 2026-08-29). Bounding the quantifier would trade the speed bug for a silent accuracy bug,
+// so the scan is by index: every character is read once and the furthest failed target scan
+// is remembered.
+function listItemHasLink(l) {
+  const m = /^ {0,3}[-*+] /.exec(l);
+  if (!m) return false;
+  const isSep = (c) => c === "\r" || c === "\n" || c === "\u2028" || c === "\u2029";
+  let failEnd = -1;
+  for (let i = m[0].length; i < l.length; i++) {
+    // A "." in the old pattern never crosses a line terminator, and split(/\r?\n/) leaves
+    // a bare CR, U+2028 and U+2029 inside a line, so a link behind one was not a match then
+    // and is not one now.
+    if (isSep(l[i])) return false;
+    if (l[i] !== "[") continue;
+    let j = i + 1, sep = false;
+    while (j < l.length && l[j] !== "]" && l[j] !== "[") { if (isSep(l[j])) sep = true; j++; }
+    if (j >= l.length) return false;
+    if (l[j] === "[") { if (sep) return false; i = j - 1; continue; }
+    if (l[j + 1] !== "(") { if (sep) return false; i = j; continue; }
+    const k = j + 2;
+    if (k <= failEnd) { if (sep) return false; i = j + 1; continue; }
+    let e = k;
+    while (e < l.length && l[e] !== ")" && !/\s/.test(l[e])) e++;
+    if (e > k && l[e] === ")") return true;
+    if (sep) return false;
+    if (l[e] !== ")") failEnd = e;
+    i = j + 1;
+  }
+  return false;
+}
+
 function redirectFailDetail(f) {
   if (f.reason === "off-host") return "redirects to " + f.location + ", a different host; llms.txt is host-scoped, so validate that host directly";
   if (f.reason === "unsafe-target") return "redirects to an unsupported target (" + f.location + "); only https redirects to the same site are followed";
@@ -141,13 +215,13 @@ export function validateLlmsTxt(f) {
   // indented code block rather than a heading, and trimming erased that difference, so
   // "    # Site" passed as the H1 until 2026-08-29. CommonMark allows three spaces.
   if (/^ {0,3}# \S/.test(firstRaw)) {
-    add("h1-title", "pass", "Starts with an H1 title", JSON.stringify(first.slice(0, 80)));
+    add("h1-title", "pass", "Starts with an H1 title", JSON.stringify(cut(first, 80)));
   } else {
     add("h1-title", "fail", "Starts with an H1 title", "the first non-empty line should be a markdown H1 (# Site name)");
   }
   const afterH1 = lines.slice(firstIdx + 1).find((l) => l.trim() !== "") || "";
   if (afterH1.trim().startsWith("> ")) {
-    add("summary", "pass", "Blockquote summary after the title", JSON.stringify(afterH1.trim().slice(0, 80)));
+    add("summary", "pass", "Blockquote summary after the title", JSON.stringify(cut(afterH1.trim(), 80)));
   } else {
     add("summary", "warn", "Blockquote summary after the title", "recommended by the format (> one-line summary), not required");
   }
@@ -160,7 +234,7 @@ export function validateLlmsTxt(f) {
     for (const l of lines) {
       if (/^## /.test(l)) { inSection = true; counted = false; continue; }
       if (/^# /.test(l)) { inSection = false; continue; }
-      if (inSection && !counted && /^ {0,3}[-*+] .*\[[^\][]*\]\([^)\s]+\)/.test(l)) { sectionsWithList++; counted = true; }
+      if (inSection && !counted && listItemHasLink(l)) { sectionsWithList++; counted = true; }
     }
   }
   if (h2Count > 0 && sectionsWithList > 0) {
@@ -170,12 +244,12 @@ export function validateLlmsTxt(f) {
   } else {
     add("sections", "warn", "H2 sections group the content", "no H2 sections found; sections are the convention for grouping links");
   }
-  const links = [...f.text.matchAll(/\[([^\][]*)\]\(([^)\s]{1,2048})\)/g)];
+  const links = collectLinks(f.text);
   // An entry an agent can use has a name and a target with a host. An empty name and a
   // bare "https://" both counted as valid absolute links until 2026-08-29.
-  const named = links.filter((m) => m[1].trim() !== "");
+  const named = links.filter((m) => m.name.trim() !== "");
   const unnamed = links.length - named.length;
-  const absolute = named.filter((m) => /^https?:\/\/[^/\s?#]+/.test(m[2])).length;
+  const absolute = named.filter((m) => /^https?:\/\/[^/\s?#]+/.test(m.target)).length;
   if (links.length === 0) {
     add("links", "warn", "Markdown links an agent can follow", "no markdown links found");
   } else if (unnamed > 0) {
@@ -183,7 +257,7 @@ export function validateLlmsTxt(f) {
   } else if (absolute === named.length) {
     add("links", "pass", "Markdown links an agent can follow", named.length + " link" + (named.length === 1 ? "" : "s") + ", all absolute URLs");
   } else {
-    const relativeCount = named.filter((m) => !/^[a-z][a-z0-9+.-]*:/i.test(m[2])).length;
+    const relativeCount = named.filter((m) => !/^[a-z][a-z0-9+.-]*:/i.test(m.target)).length;
     const hostless = named.length - absolute - relativeCount;
     add("links", "warn", "Markdown links an agent can follow", named.length + " links, " + relativeCount + " relative" + (hostless > 0 ? " and " + hostless + " with a scheme but no host" : "") + "; absolute URLs travel better when the file is read out of context");
   }
@@ -473,9 +547,9 @@ export function validateV2Discovery(found, unreadReason) {
     return checks;
   }
   add("v2-describedby", found.describedby ? "pass" : "info", "Home page points to its llms.txt (v2)",
-    found.describedby ? 'rel="describedby" to ' + found.describedby.slice(0, 120) : 'no rel="describedby" in the head or the Link header; v2 recommends it so an agent finds the file without guessing');
+    found.describedby ? 'rel="describedby" to ' + cut(found.describedby, 120) : 'no rel="describedby" in the head or the Link header; v2 recommends it so an agent finds the file without guessing');
   add("v2-markdown-alternate", found.markdown ? "pass" : "info", "Home page points to a markdown version (v2)",
-    found.markdown ? 'rel="alternate" type="text/markdown" to ' + found.markdown.slice(0, 120) : 'no rel="alternate" type="text/markdown" in the head or the Link header; v2 recommends it so an agent finds the markdown form without guessing');
+    found.markdown ? 'rel="alternate" type="text/markdown" to ' + cut(found.markdown, 120) : 'no rel="alternate" type="text/markdown" in the head or the Link header; v2 recommends it so an agent finds the markdown form without guessing');
   return checks;
 }
 
@@ -488,7 +562,7 @@ export function summarizeChecks(checks) {
 export async function validateHost(input, opts = {}) {
   const host = normalizeHostInput(input);
   if (!host || !isValidPublicHost(host)) {
-    throw new Error("not a public domain name: " + String(input).slice(0, 120));
+    throw new Error("not a public domain name: " + cut(input, 120));
   }
   // opts comes from the caller, and path and accept exist for this function's own
   // second read. Forwarding them to the first read would let a caller point the