turva-llms-txt-validator 0.2.1 → 0.3.0

package/README.md

@@ -28,10 +28,10 @@ The target has to be a public domain name: an IP literal, a bracketed IPv6 addre
 |---|-------|------|------|
 | 1 | File exists at /llms.txt, HTTP 200 (a same-site www/apex redirect is followed) | non-200, or an off-site or unsafe redirect | |
 | 2 | Response is plain text | body looks like HTML | content-type is not text/plain or text/markdown |
-| 3 | Starts with an H1 title | first non-empty line is not a markdown H1 | |
+| 3 | Starts with an H1 title | first non-empty line is not a markdown H1, indentation included, since four spaces make it a code block | |
 | 4 | Blockquote summary after the title | | missing one-line summary |
-| 5 | H2 sections group the content | | no H2 sections |
-| 6 | Markdown links an agent can follow | | no links, or relative links |
+| 5 | H2 sections group the content | | no H2 sections, or no section carrying a markdown file list |
+| 6 | Markdown links an agent can follow | | no links, an empty link name, a scheme without a host, or relative links |
 | 7 | Small enough to be cheap to read | | over 50 KB, or read truncated at 256 KB |
 | 8 | No HTML markup in the file | | HTML tags found |
 
@@ -39,8 +39,8 @@ Two more checks read the site's home page for the link relations v2 of the forma
 
 | # | Check | pass | info |
 |---|-------|------|------|
-| 9 | Home page points to its llms.txt (v2) | `rel="describedby"` found in the head or the Link header | not found, or the home page could not be read |
-| 10 | Home page points to a markdown version (v2) | `rel="alternate" type="text/markdown"` found in the head or the Link header | not found, or the home page could not be read |
+| 9 | Home page points to its llms.txt (v2) | `rel="describedby"` with a target, found in the head or the Link header | not found, no target on the relation, or the home page could not be read |
+| 10 | Home page points to a markdown version (v2) | `rel="alternate"` with the media type `text/markdown` and a target, found in the head or the Link header | not found, no target on the relation, a different media type, or the home page could not be read |
 
 ## Node API
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks as information, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -135,8 +135,12 @@ export function validateLlmsTxt(f) {
   }
   const lines = f.text.split(/\r?\n/);
   const firstIdx = lines.findIndex((l) => l.trim() !== "");
-  const first = firstIdx === -1 ? "" : lines[firstIdx].trim();
-  if (/^# \S/.test(first)) {
+  const firstRaw = firstIdx === -1 ? "" : lines[firstIdx];
+  const first = firstRaw.trim();
+  // The line is read as markdown and not trimmed first. Four spaces or a tab make it an
+  // indented code block rather than a heading, and trimming erased that difference, so
+  // "    # Site" passed as the H1 until 2026-08-29. CommonMark allows three spaces.
+  if (/^ {0,3}# \S/.test(firstRaw)) {
     add("h1-title", "pass", "Starts with an H1 title", JSON.stringify(first.slice(0, 80)));
   } else {
     add("h1-title", "fail", "Starts with an H1 title", "the first non-empty line should be a markdown H1 (# Site name)");
@@ -148,19 +152,40 @@ export function validateLlmsTxt(f) {
     add("summary", "warn", "Blockquote summary after the title", "recommended by the format (> one-line summary), not required");
   }
   const h2Count = (f.text.match(/^## /gm) || []).length;
-  if (h2Count > 0) {
-    add("sections", "pass", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s"));
+  // A section counts when it carries a file list. An H2 followed by a paragraph satisfied
+  // this check until 2026-08-29, and the format puts each section's links in a list.
+  let sectionsWithList = 0;
+  {
+    let inSection = false, counted = false;
+    for (const l of lines) {
+      if (/^## /.test(l)) { inSection = true; counted = false; continue; }
+      if (/^# /.test(l)) { inSection = false; continue; }
+      if (inSection && !counted && /^ {0,3}[-*+] .*\[[^\][]*\]\([^)\s]+\)/.test(l)) { sectionsWithList++; counted = true; }
+    }
+  }
+  if (h2Count > 0 && sectionsWithList > 0) {
+    add("sections", "pass", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s") + ", " + sectionsWithList + " carrying a file list");
+  } else if (h2Count > 0) {
+    add("sections", "warn", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s") + " but no file list under any of them; the format puts a section's links in a markdown list");
   } else {
     add("sections", "warn", "H2 sections group the content", "no H2 sections found; sections are the convention for grouping links");
   }
   const links = [...f.text.matchAll(/\[([^\][]*)\]\(([^)\s]{1,2048})\)/g)];
-  const absolute = links.filter((m) => /^https?:\/\//.test(m[2])).length;
+  // An entry an agent can use has a name and a target with a host. An empty name and a
+  // bare "https://" both counted as valid absolute links until 2026-08-29.
+  const named = links.filter((m) => m[1].trim() !== "");
+  const unnamed = links.length - named.length;
+  const absolute = named.filter((m) => /^https?:\/\/[^/\s?#]+/.test(m[2])).length;
   if (links.length === 0) {
     add("links", "warn", "Markdown links an agent can follow", "no markdown links found");
-  } else if (absolute === links.length) {
-    add("links", "pass", "Markdown links an agent can follow", links.length + " link" + (links.length === 1 ? "" : "s") + ", all absolute URLs");
+  } else if (unnamed > 0) {
+    add("links", "warn", "Markdown links an agent can follow", links.length + " link" + (links.length === 1 ? "" : "s") + ", " + unnamed + " with an empty link name; an entry needs a name an agent can show");
+  } else if (absolute === named.length) {
+    add("links", "pass", "Markdown links an agent can follow", named.length + " link" + (named.length === 1 ? "" : "s") + ", all absolute URLs");
   } else {
-    add("links", "warn", "Markdown links an agent can follow", links.length + " links, " + (links.length - absolute) + " relative; absolute URLs travel better when the file is read out of context");
+    const relativeCount = named.filter((m) => !/^[a-z][a-z0-9+.-]*:/i.test(m[2])).length;
+    const hostless = named.length - absolute - relativeCount;
+    add("links", "warn", "Markdown links an agent can follow", named.length + " links, " + relativeCount + " relative" + (hostless > 0 ? " and " + hostless + " with a scheme but no host" : "") + "; absolute URLs travel better when the file is read out of context");
   }
   if (f.truncated) {
     add("size", "warn", "Small enough to be cheap to read", "over 256 KB, read truncated");
@@ -414,20 +439,27 @@ export function findLinkRelations(html, linkHeader) {
     // NAME is "link<link", not as a link element, so \b would count a relation the site
     // does not publish. Measured against parse5, 2026-08-24.
     if (!/^<link(?=[\s/>])/i.test(tag)) continue;
-    const rel = ((tag.match(/\brel\s*=\s*["']?([^"'>]+)/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
-    const type = ((tag.match(/\btype\s*=\s*["']?([^"'>\s]+)/i) || [])[1] || "").toLowerCase();
-    const href = ((tag.match(/\bhref\s*=\s*"([^"]*)"|\bhref\s*=\s*'([^']*)'|\bhref\s*=\s*([^\s"'>]+)/i) || []).slice(1).find((x) => x !== undefined) || "").trim();
-    if (!found.describedby && rel.includes("describedby")) found.describedby = href || "(link element without href)";
-    if (!found.markdown && rel.includes("alternate") && type.startsWith("text/markdown")) found.markdown = href || "(link element without href)";
+    // The attribute name has to start the token. \b sits between the hyphen and the name,
+    // so data-rel, data-type and data-href were read as the real attributes until
+    // 2026-08-29, and a page could claim a relation it does not publish.
+    const rel = ((tag.match(/(?:^|[\s/])rel\s*=\s*["']?([^"'>]+)/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
+    const type = ((tag.match(/(?:^|[\s/])type\s*=\s*["']?([^"'>\s]+)/i) || [])[1] || "").toLowerCase();
+    const href = ((tag.match(/(?:^|[\s/])href\s*=\s*"([^"]*)"|(?:^|[\s/])href\s*=\s*'([^']*)'|(?:^|[\s/])href\s*=\s*([^\s"'>]+)/i) || []).slice(1).find((x) => x !== undefined) || "").trim();
+    // text/markdown, not anything that starts with it, and a relation without a target is
+    // not a relation: both passed until 2026-08-29.
+    const isMarkdown = type.split(";")[0].trim() === "text/markdown";
+    if (!found.describedby && href && rel.includes("describedby")) found.describedby = href;
+    if (!found.markdown && href && rel.includes("alternate") && isMarkdown) found.markdown = href;
   }
   for (const part of String(linkHeader || "").split(/,(?=\s*<)/)) {
     const lt = part.indexOf("<");
     const gt = lt === -1 ? -1 : part.indexOf(">", lt + 1);
     const href = (gt === -1 ? "" : part.slice(lt + 1, gt)).trim();
-    const rel = ((part.match(/rel\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
-    const type = ((part.match(/type\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim();
-    if (!found.describedby && rel.includes("describedby")) found.describedby = href || "(Link header without a target)";
-    if (!found.markdown && rel.includes("alternate") && type.startsWith("text/markdown")) found.markdown = href || "(Link header without a target)";
+    const rel = ((part.match(/(?:^|[;\s])rel\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim().split(/\s+/);
+    const type = ((part.match(/(?:^|[;\s])type\s*=\s*"?([^";,]+)"?/i) || [])[1] || "").toLowerCase().trim();
+    const isMarkdownHeader = type.split(";")[0].trim() === "text/markdown";
+    if (!found.describedby && href && rel.includes("describedby")) found.describedby = href;
+    if (!found.markdown && href && rel.includes("alternate") && isMarkdownHeader) found.markdown = href;
   }
   return found;
 }