turva-llms-txt-validator 0.3.18 → 0.3.20

package/README.md

@@ -137,10 +137,10 @@ For vulnerability reporting and supported versions, see [SECURITY.md](https://gi
 GitHub Actions publishes every version from 0.1.3 on with npm trusted publishing, and each one carries a provenance attestation. To check one, install it in an empty directory and ask npm to verify the signatures. Replace the version with the one you want to check.
 
 ```sh
-mkdir verify-llms && cd verify-llms && npm init -y && npm install turva-llms-txt-validator@0.3.18 --ignore-scripts && npm audit signatures
+mkdir verify-llms && cd verify-llms && npm init -y && npm install turva-llms-txt-validator@0.3.20 --ignore-scripts && npm audit signatures
 ```
 
-`npm audit signatures` checks the registry signature and the provenance attestation of each installed package that has one. The attestation of this package names three things to compare with what you expect: the repository `github.com/erekola/llms-txt-validator`, the workflow file `.github/workflows/publish.yml` and the commit that produced the tarball. The npm version page shows them under Provenance. Tag `v0.3.18` must point at that same commit, and `gh api repos/erekola/llms-txt-validator/commits/v0.3.17 --jq .sha` prints it. A mismatch is a reason not to use that version. Provenance proves where a release was built and from which commit. It does not prove that the code is safe, so reading the source and the dependencies stays your job.
+`npm audit signatures` checks the registry signature and the provenance attestation of each installed package that has one. The attestation of this package names three things to compare with what you expect: the repository `github.com/erekola/llms-txt-validator`, the workflow file `.github/workflows/publish.yml` and the commit that produced the tarball. The npm version page shows them under Provenance. Tag `v0.3.20` must point at that same commit, and `gh api repos/erekola/llms-txt-validator/commits/v0.3.20 --jq .sha` prints it. A mismatch is a reason not to use that version. Provenance proves where a release was built and from which commit. It does not prove that the code is safe, so reading the source and the dependencies stays your job.
 
 ## Hosted version and source
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.3.18",
+  "version": "0.3.20",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks that read pass or info and never move the summary, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -539,6 +539,86 @@ function isSetextH1(line, next) {
   return isSetextText(line);
 }
 
+// Lines that open a block of their own in CommonMark, so they are never setext text after the
+// title: a thematic break, the start of an HTML block (an autolink such as <https://...> is not
+// one, it stays paragraph text) and a link reference definition. Only the later-H1 reading skips
+// them; the title reading keeps its Tek-560 predicate. QA 2 of Tek-564 measured "---", "<div>" and
+// "[a]: url" over a run of "=" warning as a second H1 although 0.3.18 passed them.
+const HTML_BLOCK_START = /^ {0,3}<(?:[A-Za-z][A-Za-z0-9-]*(?:[\s\/>]|$)|\/[A-Za-z][A-Za-z0-9-]*(?:[\s>]|$)|!--|\?|![A-Za-z]|!\[CDATA\[)/;
+const LINK_REF_DEF = /^ {0,3}\[[^\]]+\]:/;
+// HTML blocks of CommonMark types 1 to 5 end at a marker, not at a blank line: <pre>, <script>,
+// <style> and <textarea> at "</...>", "<!--" at "-->", "<?" at "?>", "<!X" at ">" and "<![CDATA[" at
+// "]]>". Each entry is [start, end]; the end marker is looked for after the start on the same line.
+const RAW_HTML_BLOCKS = [
+  [/^ {0,3}<(?:pre|script|style|textarea)(?:[ \t>]|$)/i, /<\/(?:pre|script|style|textarea)>/i],
+  [/^ {0,3}<!--/, /-->/],
+  [/^ {0,3}<\?/, /\?>/],
+  [/^ {0,3}<!\[CDATA\[/, /\]\]>/],
+  [/^ {0,3}<![A-Za-z]/, />/]
+];
+function startsOtherBlock(line) {
+  return /^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$/.test(line)
+    || HTML_BLOCK_START.test(line)
+    || LINK_REF_DEF.test(line);
+}
+
+// Setext H1 lines after the title (Tek-564). A second H1 written as one text line over a run of "="
+// is out of place exactly like "# Second title" is, and ends the section before it the same way.
+// The text line is read by isSetextH1, the predicate the first-title reading uses. It must start
+// a paragraph: a line that follows other paragraph text, a list item or a blockquote without a
+// blank line is a continuation line, so a title over several lines is not read as a heading, and
+// text that starts with "#" is not an H1 (isSetextText). The underline is returned separately
+// because it is not a heading and not content.
+// An indented code block is not a paragraph (Tek-565, outside review W46): a line indented four
+// columns or more that does not continue a paragraph is code, so the text after it starts a new
+// paragraph and "Title" over "===" there is a second H1. A four-space line right after paragraph
+// text stays a continuation line (the accepted Tek-564 limit). The exception is a list item: its
+// blank lines and indented lines belong to the item as continuation paragraphs, not as code, so
+// while a list item is open (listCtx) an indented line is read as before. The list context ends at
+// a thematic break or an ATX heading, or at a line with no indent after a blank line. When unsure
+// the function stays in the list context, which is the 0.3.19 reading. A block quote, an HTML block
+// and a link reference definition keep the old reading too, until the next blank line, and an HTML
+// block of type 1 to 5 (<pre>, a comment, <?, <!X, CDATA) until its end marker: an indented line
+// there continues the quote paragraph or the HTML block, or follows a "=" line that is itself
+// paragraph text, and is not code.
+function laterSetextH1(lines, fenced) {
+  const text = new Array(lines.length).fill(false);
+  const under = new Array(lines.length).fill(false);
+  let open = false;
+  let listCtx = false;
+  let blankBefore = false;
+  let quoteCtx = false;
+  let htmlCtx = false;
+  let rawEnd = null;
+  let rawDone = false;
+  for (let i = 0; i < lines.length; i++) {
+    const l = lines[i];
+    if (rawDone) { htmlCtx = false; rawDone = false; }
+    if (fenced[i]) { open = false; continue; }
+    if (l.trim() === "") { open = false; blankBefore = true; quoteCtx = false; if (!rawEnd) htmlCtx = false; continue; }
+    const indented = /^(?: {4}| {0,3}\t)/.test(l);
+    if (/^ ?#{1,6}(?:[ \t]|$)/.test(l) || /^ ?([-*_])(?:[ \t]*\1){2,}[ \t]*$/.test(l)) listCtx = false;
+    else if (/^ {0,3}(?:[-*+]|\d{1,9}[.)])(?:[ \t]|$)/.test(l)) listCtx = true;
+    else if (blankBefore && /^ ?[^ \t]/.test(l)) listCtx = false;
+    blankBefore = false;
+    if (/^ {0,3}>/.test(l)) quoteCtx = true;
+    if (rawEnd) {
+      if (rawEnd.test(l)) { rawEnd = null; rawDone = true; }
+    } else {
+      if (HTML_BLOCK_START.test(l) || LINK_REF_DEF.test(l)) htmlCtx = true;
+      for (const [start, end] of RAW_HTML_BLOCKS) {
+        const m = start.exec(l);
+        if (m) { if (!end.test(l.slice(m[0].length))) rawEnd = end; break; }
+      }
+    }
+    if (!open && !listCtx && !quoteCtx && !htmlCtx && indented) continue;
+    if (!open && !fenced[i + 1] && !startsOtherBlock(l) && isSetextH1(l, lines[i + 1])) { text[i] = true; under[i + 1] = true; i++; continue; }
+    if (open && /^ {0,3}=+[ \t]*$/.test(l)) { open = false; continue; }
+    open = !(/^ {0,3}#{1,6}(?:[ \t]|$)/.test(l) || /^ {0,3}(?:-+|([-*_])(?: *\1){2,})[ \t]*$/.test(l));
+  }
+  return { text, under };
+}
+
 // Which lines are H2 headings, outside fences. One reader for every place that asks "is this an
 // H2": the section count, the section membership and the place where the late-summary scan stops.
 // An ATX H2 is "##" followed by a space or a tab (a bare "##" is not one, and "###" is deeper). A
@@ -735,6 +815,7 @@ export function validateLlmsTxt(f) {
     }
   }
   const hm = h2Lines(lines, fenced);
+  const sx = laterSetextH1(lines, fenced);
   const h2Count = hm.h2.filter(Boolean).length;
   // A section counts when it carries a file list. An H2 followed by a paragraph satisfied
   // this check until 2026-08-29, and the format puts each section's links in a list.
@@ -758,9 +839,9 @@ export function validateLlmsTxt(f) {
         if (itemOpen) afterFence = true;
         continue;
       }
-      if (hm.under[i]) continue;
+      if (hm.under[i] || sx.under[i]) continue;
       if (hm.h2[i]) { inSection = true; counted = false; seenH2 = true; itemOpen = false; continue; }
-      const h1 = /^ {0,3}#(?:[ \t]|$)/.test(l);
+      const h1 = /^ {0,3}#(?:[ \t]|$)/.test(l) || sx.text[i];
       if (misplaced === -1 && i > firstIdx && (h1 || (!seenH2 && /^ {0,3}#{3,6}(?:[ \t]|$)/.test(l)))) misplaced = i;
       if (h1) { inSection = false; itemOpen = false; continue; }
       // A link on a line that continues an open list item counts as that item's link (Tek-560):