turva-llms-txt-validator 0.2.0 → 0.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks as information, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -177,6 +177,224 @@ export function validateLlmsTxt(f) {
   return checks;
 }
 
+// A "<" starts a tag only when a letter, "!", "/" or "?" follows it. Anything else is
+// text, and the NEXT "<" can still start a tag. Measured, not assumed: without this rule
+// "<<style><link rel=describedby ...>" read the link as published while a real parser
+// treats the first "<" as text, opens style, and publishes nothing.
+function startsTag(c) {
+  return c !== undefined && (c === "!" || c === "/" || c === "?" || (c >= "a" && c <= "z"));
+}
+
+// The end of the tag that starts at lt: the first ">" that is NOT inside a quoted
+// attribute value. A quote opens a value only right after "=", which is what the
+// tokenizer does. indexOf(">") is wrong here: <link data-x="a>b" rel="describedby"> is
+// one tag for a parser and two for indexOf, and the relation was lost.
+function tagEnd(text, lt) {
+  let quote = "", afterEq = false;
+  for (let j = lt + 1; j < text.length; j++) {
+    const c = text[j];
+    if (quote) { if (c === quote) quote = ""; continue; }
+    if (c === ">") return j;
+    if (c === "=") { afterEq = true; continue; }
+    if (c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f") continue;
+    if (afterEq && (c === '"' || c === "'")) { quote = c; afterEq = false; continue; }
+    afterEq = false;
+  }
+  return -1;
+}
+
+// The head is where a real parser says it ends, not where the text "</head>" happens to
+// appear. Erik's decision 2026-08-24: the two v2 checks describe what the page's HEAD
+// points at, so a link element the parser moves into the body is not one of them. That is
+// the strict reading, and it is what the HTML parsing spec's "in head" insertion mode
+// does: whitespace, comments, doctype and the head-only elements keep the head open;
+// the first text node, the first body-level element and </head>, </body>, </html> or
+// </br> close it; any other end tag in the head is a parse error and is ignored.
+var HEAD_ELEMENTS = ["base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "template", "title", "noscript"];
+// Once </head> has been seen the parser is in "after head", and that list is the one above
+// WITHOUT noscript: a noscript there opens the body instead of staying in the head.
+var AFTER_HEAD_ELEMENTS = ["base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "template", "title"];
+// Their content is not markup: script, style, title, noscript (a parser with scripting on
+// reads it as raw text) and noframes are raw text or RCDATA, and template content is inert.
+var HEAD_SKIPPED_CONTENT = ["script", "style", "title", "noscript", "noframes", "template"];
+
+// The end of a raw text or RCDATA element: the first </name that is followed by optional
+// whitespace, an optional "/" (a parser closes on </script/> too) and a ">". Returns the
+// index after it, or -1 when the element never closes, which means the rest of the
+// document is inside it.
+function skipSpace(lower, j) {
+  while (j < lower.length && (lower[j] === " " || lower[j] === "\t" || lower[j] === "\n" || lower[j] === "\r" || lower[j] === "\f")) j++;
+  return j;
+}
+
+// True when the character ends a tag name: whitespace, "/", ">" or the end of the input.
+function isTagBoundary(c) {
+  return c === undefined || c === ">" || c === "/" || c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f";
+}
+
+// The index just after "</name ... >", or -1 when that is not an end tag there.
+function endTagAt(lower, name, at) {
+  if (!lower.startsWith("</" + name, at)) return -1;
+  let j = skipSpace(lower, at + name.length + 2);
+  if (lower[j] === "/") j = skipSpace(lower, j + 1);
+  return lower[j] === ">" ? j + 1 : -1;
+}
+
+function rawTextEnd(lower, name, from) {
+  for (let at = lower.indexOf("</" + name, from); at !== -1; at = lower.indexOf("</" + name, at + name.length + 2)) {
+    const end = endTagAt(lower, name, at);
+    if (end !== -1) return end;
+  }
+  return -1;
+}
+
+// script is not plain raw text: <!-- puts the tokenizer in the escaped state, a nested
+// <script there puts it in the double escaped state, and in THAT state </script only ends
+// the escape, not the element. Without this the legacy shape
+// <script><!-- ... <script>...</script> ... --></script> ended early and the rest of the
+// script was read as markup, which is the wrong direction. 4 of 240 000 fuzz inputs.
+function scriptEnd(lower, from) {
+  let i = from, escaped = false, doubleEscaped = false;
+  while (i < lower.length) {
+    if (lower.startsWith("<!--", i)) { escaped = true; i += 4; continue; }
+    if (escaped && lower.startsWith("-->", i)) { escaped = false; doubleEscaped = false; i += 3; continue; }
+    if (escaped && !doubleEscaped && lower.startsWith("<script", i) && isTagBoundary(lower[i + 7])) { doubleEscaped = true; i += 7; continue; }
+    if (lower.startsWith("</script", i)) {
+      if (doubleEscaped) { doubleEscaped = false; i += 8; continue; }
+      const end = endTagAt(lower, "script", i);
+      if (end !== -1) return end;
+      i += 8;
+      continue;
+    }
+    const next = lower.indexOf("<", i + 1);
+    i = next === -1 ? lower.length : next;
+  }
+  return -1;
+}
+
+// The end of a nested template: templates count, so an inner </template> does not close
+// an outer one. Returns the index after the closing tag, or -1 when it never closes.
+function templateEnd(lower, from) {
+  let depth = 1;
+  // Both searches resume from their own previous hit. Restarting either one from a shared
+  // cursor is quadratic: "</templateX" repeated made every round scan to the end of the
+  // input again, and 1 MB of it measured 15 686 ms.
+  let open = lower.indexOf("<template", from);
+  let close = lower.indexOf("</template", from);
+  for (;;) {
+    if (close === -1) return -1;
+    if (open !== -1 && open < close) {
+      if (isTagBoundary(lower[open + 9])) depth++;
+      open = lower.indexOf("<template", open + 9);
+      continue;
+    }
+    let j = close + 10;
+    j = skipSpace(lower, j);
+    if (lower[j] === "/") j = skipSpace(lower, j + 1);
+    if (lower[j] !== ">") {
+      // Not an end tag: for the tokenizer the rest of the name runs to the next ">", and a
+      // "<" inside it is part of the name rather than a new tag.
+      const bogus = lower.indexOf(">", close + 10);
+      if (bogus === -1) return -1;
+      close = lower.indexOf("</template", bogus + 1);
+      if (open !== -1 && open < bogus) open = lower.indexOf("<template", bogus + 1);
+      continue;
+    }
+    depth--;
+    if (depth === 0) return j + 1;
+    close = lower.indexOf("</template", j + 1);
+  }
+}
+
+// One left to right scan by index that returns the head, with comments and the content of
+// the raw text and template elements already removed. Why a scan and not a regex: a
+// character class that reads a tag's attributes is quadratic on input the TARGET site
+// controls (CodeQL js/polynomial-redos, alerts #4 and #5, 2026-08-24), and bounding the
+// class trades that speed bug for a worse correctness bug, because an open tag longer than
+// the bound stops being recognised and the element's own text is then read as markup. A
+// scan has no bound and no backtracking, and it visits every character once.
+//
+// The shapes, all measured against a real HTML parser (parse5) rather than reasoned about.
+// An unterminated <!-- comments out the rest of the document. <!--> and <!---> are EMPTY
+// comments, not unterminated ones, and --!> ends a comment as well. A "<" that no letter,
+// "!", "/" or "?" follows is text. A ">" inside a quoted attribute value does not end a
+// tag. </script/> closes a raw text element as well as </script> does.
+function headOfDocument(html) {
+  const text = String(html || "");
+  const lower = text.toLowerCase();
+  const out = [];
+  let i = 0, afterHead = false;
+  for (;;) {
+    const lt = lower.indexOf("<", i);
+    const gap = lt === -1 ? text.slice(i) : text.slice(i, lt);
+    if (gap.trim() !== "") break;
+    out.push(gap);
+    if (lt === -1) break;
+    if (lower.startsWith("<!--", lt)) {
+      if (lower.startsWith("<!-->", lt)) { i = lt + 5; continue; }
+      if (lower.startsWith("<!--->", lt)) { i = lt + 6; continue; }
+      const dashes = lower.indexOf("-->", lt + 4);
+      const bang = lower.indexOf("--!>", lt + 4);
+      if (dashes === -1 && bang === -1) break;
+      i = (bang === -1 || (dashes !== -1 && dashes <= bang)) ? dashes + 3 : bang + 4;
+      continue;
+    }
+    if (!startsTag(lower[lt + 1])) break;
+    const gt = tagEnd(text, lt);
+    if (gt === -1) break;
+    if (lower[lt + 1] === "!" || lower[lt + 1] === "?") { i = gt + 1; continue; }
+    const head14 = lower.slice(lt, Math.min(lt + 14, gt + 1));
+    const endTag = (/^<\/([a-z]+)/.exec(head14) || [])[1];
+    if (endTag) {
+      // </body>, </html> and </br> start the body. </head> does NOT end the search: after
+      // it a parser still puts base, link, meta, script, style, title and template into the
+      // HEAD element until real body content starts, and every other end tag in the head is
+      // a parse error that is ignored. Measured against parse5: without this, 2 660 of
+      // 200 000 fuzz inputs lost a relation the head really carries.
+      if (endTag === "body" || endTag === "html" || endTag === "br") break;
+      if (endTag === "head") afterHead = true;
+      i = gt + 1;
+      continue;
+    }
+    const name = (/^<([a-z]+)(?=[\s/>]|$)/.exec(head14) || [])[1];
+    if (!name) break;
+    if (name === "html" || name === "head") { i = gt + 1; continue; }
+    if (!(afterHead ? AFTER_HEAD_ELEMENTS : HEAD_ELEMENTS).includes(name)) break;   // <body> and the first body level element
+    if (HEAD_SKIPPED_CONTENT.includes(name)) {
+      const end = name === "template" ? templateEnd(lower, gt + 1)
+        : name === "script" ? scriptEnd(lower, gt + 1)
+        : rawTextEnd(lower, name, gt + 1);
+      if (end === -1) break;
+      i = end;
+      continue;
+    }
+    out.push(text.slice(lt, gt + 1));
+    i = gt + 1;
+  }
+  return out.join("");
+}
+
+// Tags are found by index and not by /<link\b[^>]*>/g, on purpose. That regex is
+// quadratic on input the TARGET site controls: every "<link" with no ">" after it makes
+// the character class scan to the end of the document, and "<link" repeated 16 000 times
+// measured 196 ms where this loop measures under 1 ms. CodeQL reports the same shape as
+// js/polynomial-redos (alerts #4 and #5 on the package repo, 2026-08-24). This scan
+// visits every character once: from each "<" it reads to the next ">" and then continues
+// after it, which is what the regex meant to say.
+function* htmlTags(text) {
+  const lower = text.toLowerCase();
+  let i = 0;
+  for (;;) {
+    const open = lower.indexOf("<", i);
+    if (open === -1) return;
+    if (!startsTag(lower[open + 1])) { i = open + 1; continue; }
+    const close = tagEnd(text, open);
+    if (close === -1) return;
+    yield text.slice(open, close + 1);
+    i = close + 1;
+  }
+}
+
 // v2 of the llms.txt proposal (August 2026) left the file format alone and added one
 // thing: a page should say where its markdown version and its llms.txt are, using
 // rel="alternate" type="text/markdown" and rel="describedby", as HTML link elements or
@@ -185,14 +403,17 @@ export function validateLlmsTxt(f) {
 // summary line is unchanged, and --strict keeps exiting on warnings only.
 export function findLinkRelations(html, linkHeader) {
   const found = { describedby: null, markdown: null };
-  // Comments are stripped first: a commented-out link element is not published,
-  // and counting one would report a relation the site does not serve. With no
-  // </head> the first 64 KB are scanned, body included, so a malformed document
-  // does not silently report nothing found.
-  const text = String(html || "").replace(/<!--[\s\S]*?-->/g, "");
-  const cut = text.toLowerCase().indexOf("</head>");
-  const head = cut === -1 ? text.slice(0, 65536) : text.slice(0, cut);
-  for (const tag of head.match(/<link\b[^>]*>/gi) || []) {
+  // Only the head, and only what a parser would put there: a commented-out link element,
+  // one inside a script or a template, and one the parser moves into the body are all not
+  // what these two checks are about, and counting them would report a relation the site
+  // does not serve. See headOfDocument above for the rules shape by shape. The 64 KB bound
+  // is a cap on work, not a rule: a head longer than that is not a head.
+  const head = headOfDocument(html).slice(0, 65536);
+  for (const tag of htmlTags(head)) {
+    // The name has to END at "link": a real parser reads "<link<link" as ONE tag whose
+    // NAME is "link<link", not as a link element, so \b would count a relation the site
+    // does not publish. Measured against parse5, 2026-08-24.
+    if (!/^<link(?=[\s/>])/i.test(tag)) continue;
     const rel = ((tag.match(/\brel\s*=\s*["']?([^"'>]+)/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
     const type = ((tag.match(/\btype\s*=\s*["']?([^"'>\s]+)/i) || [])[1] || "").toLowerCase();
     const href = ((tag.match(/\bhref\s*=\s*"([^"]*)"|\bhref\s*=\s*'([^']*)'|\bhref\s*=\s*([^\s"'>]+)/i) || []).slice(1).find((x) => x !== undefined) || "").trim();
@@ -200,7 +421,9 @@ export function findLinkRelations(html, linkHeader) {
     if (!found.markdown && rel.includes("alternate") && type.startsWith("text/markdown")) found.markdown = href || "(link element without href)";
   }
   for (const part of String(linkHeader || "").split(/,(?=\s*<)/)) {
-    const href = ((part.match(/<([^>]*)>/) || [])[1] || "").trim();
+    const lt = part.indexOf("<");
+    const gt = lt === -1 ? -1 : part.indexOf(">", lt + 1);
+    const href = (gt === -1 ? "" : part.slice(lt + 1, gt)).trim();
     const rel = ((part.match(/rel\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
     const type = ((part.match(/type\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim();
     if (!found.describedby && rel.includes("describedby")) found.describedby = href || "(Link header without a target)";