turva-llms-txt-validator 0.2.1 → 0.3.1

package/README.md

@@ -28,10 +28,10 @@ The target has to be a public domain name: an IP literal, a bracketed IPv6 addre
 |---|-------|------|------|
 | 1 | File exists at /llms.txt, HTTP 200 (a same-site www/apex redirect is followed) | non-200, or an off-site or unsafe redirect | |
 | 2 | Response is plain text | body looks like HTML | content-type is not text/plain or text/markdown |
-| 3 | Starts with an H1 title | first non-empty line is not a markdown H1 | |
+| 3 | Starts with an H1 title | first non-empty line is not a markdown H1, indentation included, since four spaces make it a code block | |
 | 4 | Blockquote summary after the title | | missing one-line summary |
-| 5 | H2 sections group the content | | no H2 sections |
-| 6 | Markdown links an agent can follow | | no links, or relative links |
+| 5 | H2 sections group the content | | no H2 sections, or no section carrying a markdown file list |
+| 6 | Markdown links an agent can follow | | no links, an empty link name, a scheme without a host, or relative links |
 | 7 | Small enough to be cheap to read | | over 50 KB, or read truncated at 256 KB |
 | 8 | No HTML markup in the file | | HTML tags found |
 
@@ -39,8 +39,8 @@ Two more checks read the site's home page for the link relations v2 of the forma
 
 | # | Check | pass | info |
 |---|-------|------|------|
-| 9 | Home page points to its llms.txt (v2) | `rel="describedby"` found in the head or the Link header | not found, or the home page could not be read |
-| 10 | Home page points to a markdown version (v2) | `rel="alternate" type="text/markdown"` found in the head or the Link header | not found, or the home page could not be read |
+| 9 | Home page points to its llms.txt (v2) | `rel="describedby"` with a target, found in the head or the Link header | not found, no target on the relation, or the home page could not be read |
+| 10 | Home page points to a markdown version (v2) | `rel="alternate"` with the media type `text/markdown` and a target, found in the head or the Link header | not found, no target on the relation, a different media type, or the home page could not be read |
 
 ## Node API
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks as information, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -103,6 +103,68 @@ export async function fetchLlmsTxt(host, opts = {}) {
   }
 }
 
+// Every markdown link in the file, scanned once from left to right instead of collected with
+// matchAll(/\[([^\][]*)\]\(([^)\s]{1,2048})\)/g). That bound meant a target longer than 2048
+// characters was not counted as a link at all, and dropping the bound from the pattern would
+// make it quadratic on a file that repeats "[a](" (Erik 2026-08-29). The scan carries no bound
+// and no backtracking: every character is read once and the furthest failed target scan is
+// remembered. matchAll resumes after a whole match, and so does this.
+function collectLinks(text) {
+  const out = [];
+  let failEnd = -1;
+  for (let i = 0; i < text.length; i++) {
+    if (text[i] !== "[") continue;
+    let j = i + 1;
+    while (j < text.length && text[j] !== "]" && text[j] !== "[") j++;
+    if (j >= text.length) break;
+    if (text[j] === "[") { i = j - 1; continue; }
+    if (text[j + 1] !== "(") { i = j; continue; }
+    const k = j + 2;
+    if (k <= failEnd) { i = j + 1; continue; }
+    let e = k;
+    while (e < text.length && text[e] !== ")" && !/\s/.test(text[e])) e++;
+    if (e > k && text[e] === ")") { out.push({ name: text.slice(i + 1, j), target: text.slice(k, e) }); i = e; continue; }
+    if (text[e] !== ")") failEnd = e;
+    i = j + 1;
+  }
+  return out;
+}
+
+// A markdown list item that carries a link, scanned once from left to right instead of
+// matched with /^ {0,3}[-*+] .*\[[^\][]*\]\([^)\s]+\)/. That pattern is quadratic on a line
+// such as "- " followed by "[a](" repeated, because every candidate rescans the target to the
+// end of the line, and the line comes from the audited site (CodeQL js/polynomial-redos,
+// 2026-08-29). Bounding the quantifier would trade the speed bug for a silent accuracy bug,
+// so the scan is by index: every character is read once and the furthest failed target scan
+// is remembered.
+function listItemHasLink(l) {
+  const m = /^ {0,3}[-*+] /.exec(l);
+  if (!m) return false;
+  const isSep = (c) => c === "\r" || c === "\n" || c === "\u2028" || c === "\u2029";
+  let failEnd = -1;
+  for (let i = m[0].length; i < l.length; i++) {
+    // A "." in the old pattern never crosses a line terminator, and split(/\r?\n/) leaves
+    // a bare CR, U+2028 and U+2029 inside a line, so a link behind one was not a match then
+    // and is not one now.
+    if (isSep(l[i])) return false;
+    if (l[i] !== "[") continue;
+    let j = i + 1, sep = false;
+    while (j < l.length && l[j] !== "]" && l[j] !== "[") { if (isSep(l[j])) sep = true; j++; }
+    if (j >= l.length) return false;
+    if (l[j] === "[") { if (sep) return false; i = j - 1; continue; }
+    if (l[j + 1] !== "(") { if (sep) return false; i = j; continue; }
+    const k = j + 2;
+    if (k <= failEnd) { if (sep) return false; i = j + 1; continue; }
+    let e = k;
+    while (e < l.length && l[e] !== ")" && !/\s/.test(l[e])) e++;
+    if (e > k && l[e] === ")") return true;
+    if (sep) return false;
+    if (l[e] !== ")") failEnd = e;
+    i = j + 1;
+  }
+  return false;
+}
+
 function redirectFailDetail(f) {
   if (f.reason === "off-host") return "redirects to " + f.location + ", a different host; llms.txt is host-scoped, so validate that host directly";
   if (f.reason === "unsafe-target") return "redirects to an unsupported target (" + f.location + "); only https redirects to the same site are followed";
@@ -135,8 +197,12 @@ export function validateLlmsTxt(f) {
   }
   const lines = f.text.split(/\r?\n/);
   const firstIdx = lines.findIndex((l) => l.trim() !== "");
-  const first = firstIdx === -1 ? "" : lines[firstIdx].trim();
-  if (/^# \S/.test(first)) {
+  const firstRaw = firstIdx === -1 ? "" : lines[firstIdx];
+  const first = firstRaw.trim();
+  // The line is read as markdown and not trimmed first. Four spaces or a tab make it an
+  // indented code block rather than a heading, and trimming erased that difference, so
+  // "    # Site" passed as the H1 until 2026-08-29. CommonMark allows three spaces.
+  if (/^ {0,3}# \S/.test(firstRaw)) {
     add("h1-title", "pass", "Starts with an H1 title", JSON.stringify(first.slice(0, 80)));
   } else {
     add("h1-title", "fail", "Starts with an H1 title", "the first non-empty line should be a markdown H1 (# Site name)");
@@ -148,19 +214,40 @@ export function validateLlmsTxt(f) {
     add("summary", "warn", "Blockquote summary after the title", "recommended by the format (> one-line summary), not required");
   }
   const h2Count = (f.text.match(/^## /gm) || []).length;
-  if (h2Count > 0) {
-    add("sections", "pass", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s"));
+  // A section counts when it carries a file list. An H2 followed by a paragraph satisfied
+  // this check until 2026-08-29, and the format puts each section's links in a list.
+  let sectionsWithList = 0;
+  {
+    let inSection = false, counted = false;
+    for (const l of lines) {
+      if (/^## /.test(l)) { inSection = true; counted = false; continue; }
+      if (/^# /.test(l)) { inSection = false; continue; }
+      if (inSection && !counted && listItemHasLink(l)) { sectionsWithList++; counted = true; }
+    }
+  }
+  if (h2Count > 0 && sectionsWithList > 0) {
+    add("sections", "pass", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s") + ", " + sectionsWithList + " carrying a file list");
+  } else if (h2Count > 0) {
+    add("sections", "warn", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s") + " but no file list under any of them; the format puts a section's links in a markdown list");
   } else {
     add("sections", "warn", "H2 sections group the content", "no H2 sections found; sections are the convention for grouping links");
   }
-  const links = [...f.text.matchAll(/\[([^\][]*)\]\(([^)\s]{1,2048})\)/g)];
-  const absolute = links.filter((m) => /^https?:\/\//.test(m[2])).length;
+  const links = collectLinks(f.text);
+  // An entry an agent can use has a name and a target with a host. An empty name and a
+  // bare "https://" both counted as valid absolute links until 2026-08-29.
+  const named = links.filter((m) => m.name.trim() !== "");
+  const unnamed = links.length - named.length;
+  const absolute = named.filter((m) => /^https?:\/\/[^/\s?#]+/.test(m.target)).length;
   if (links.length === 0) {
     add("links", "warn", "Markdown links an agent can follow", "no markdown links found");
-  } else if (absolute === links.length) {
-    add("links", "pass", "Markdown links an agent can follow", links.length + " link" + (links.length === 1 ? "" : "s") + ", all absolute URLs");
+  } else if (unnamed > 0) {
+    add("links", "warn", "Markdown links an agent can follow", links.length + " link" + (links.length === 1 ? "" : "s") + ", " + unnamed + " with an empty link name; an entry needs a name an agent can show");
+  } else if (absolute === named.length) {
+    add("links", "pass", "Markdown links an agent can follow", named.length + " link" + (named.length === 1 ? "" : "s") + ", all absolute URLs");
   } else {
-    add("links", "warn", "Markdown links an agent can follow", links.length + " links, " + (links.length - absolute) + " relative; absolute URLs travel better when the file is read out of context");
+    const relativeCount = named.filter((m) => !/^[a-z][a-z0-9+.-]*:/i.test(m.target)).length;
+    const hostless = named.length - absolute - relativeCount;
+    add("links", "warn", "Markdown links an agent can follow", named.length + " links, " + relativeCount + " relative" + (hostless > 0 ? " and " + hostless + " with a scheme but no host" : "") + "; absolute URLs travel better when the file is read out of context");
   }
   if (f.truncated) {
     add("size", "warn", "Small enough to be cheap to read", "over 256 KB, read truncated");
@@ -414,20 +501,27 @@ export function findLinkRelations(html, linkHeader) {
     // NAME is "link<link", not as a link element, so \b would count a relation the site
     // does not publish. Measured against parse5, 2026-08-24.
     if (!/^<link(?=[\s/>])/i.test(tag)) continue;
-    const rel = ((tag.match(/\brel\s*=\s*["']?([^"'>]+)/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
-    const type = ((tag.match(/\btype\s*=\s*["']?([^"'>\s]+)/i) || [])[1] || "").toLowerCase();
-    const href = ((tag.match(/\bhref\s*=\s*"([^"]*)"|\bhref\s*=\s*'([^']*)'|\bhref\s*=\s*([^\s"'>]+)/i) || []).slice(1).find((x) => x !== undefined) || "").trim();
-    if (!found.describedby && rel.includes("describedby")) found.describedby = href || "(link element without href)";
-    if (!found.markdown && rel.includes("alternate") && type.startsWith("text/markdown")) found.markdown = href || "(link element without href)";
+    // The attribute name has to start the token. \b sits between the hyphen and the name,
+    // so data-rel, data-type and data-href were read as the real attributes until
+    // 2026-08-29, and a page could claim a relation it does not publish.
+    const rel = ((tag.match(/(?:^|[\s/])rel\s*=\s*["']?([^"'>]+)/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
+    const type = ((tag.match(/(?:^|[\s/])type\s*=\s*["']?([^"'>\s]+)/i) || [])[1] || "").toLowerCase();
+    const href = ((tag.match(/(?:^|[\s/])href\s*=\s*"([^"]*)"|(?:^|[\s/])href\s*=\s*'([^']*)'|(?:^|[\s/])href\s*=\s*([^\s"'>]+)/i) || []).slice(1).find((x) => x !== undefined) || "").trim();
+    // text/markdown, not anything that starts with it, and a relation without a target is
+    // not a relation: both passed until 2026-08-29.
+    const isMarkdown = type.split(";")[0].trim() === "text/markdown";
+    if (!found.describedby && href && rel.includes("describedby")) found.describedby = href;
+    if (!found.markdown && href && rel.includes("alternate") && isMarkdown) found.markdown = href;
   }
   for (const part of String(linkHeader || "").split(/,(?=\s*<)/)) {
     const lt = part.indexOf("<");
     const gt = lt === -1 ? -1 : part.indexOf(">", lt + 1);
     const href = (gt === -1 ? "" : part.slice(lt + 1, gt)).trim();
-    const rel = ((part.match(/rel\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
-    const type = ((part.match(/type\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim();
-    if (!found.describedby && rel.includes("describedby")) found.describedby = href || "(Link header without a target)";
-    if (!found.markdown && rel.includes("alternate") && type.startsWith("text/markdown")) found.markdown = href || "(Link header without a target)";
+    const rel = ((part.match(/(?:^|[;\s])rel\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
+    const type = ((part.match(/(?:^|[;\s])type\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim();
+    const isMarkdownHeader = type.split(";")[0].trim() === "text/markdown";
+    if (!found.describedby && href && rel.includes("describedby")) found.describedby = href;
+    if (!found.markdown && href && rel.includes("alternate") && isMarkdownHeader) found.markdown = href;
   }
   return found;
 }