turva-llms-txt-validator 0.1.5 → 0.1.7

package/README.md

@@ -28,7 +28,7 @@ The target has to be a public domain name: an IP literal, a bracketed IPv6 addre
 |---|-------|------|------|
 | 1 | File exists at /llms.txt, HTTP 200 (a same-site www/apex redirect is followed) | non-200, or an off-site or unsafe redirect | |
 | 2 | Response is plain text | body looks like HTML | content-type is not text/plain or text/markdown |
-| 3 | Starts with a single H1 title | first non-empty line is not a markdown H1 | |
+| 3 | Starts with an H1 title | first non-empty line is not a markdown H1 | |
 | 4 | Blockquote summary after the title | | missing one-line summary |
 | 5 | H2 sections group the content | | no H2 sections |
 | 6 | Markdown links an agent can follow | | no links, or relative links |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Validate a site's llms.txt structure: eight checks, pass/warn/fail, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -41,10 +41,14 @@ export async function fetchLlmsTxt(host, opts = {}) {
   const reqApex = host.startsWith("www.") ? host.slice(4) : host;
   let url = "https://" + host + "/llms.txt";
   let redirectedFrom = null;
+  // One budget for the whole redirect chain, not one per hop. Mirrors the hosted validator
+  // (turva.dev/llms-txt-validator), which is canonical: with five hops a per-hop signal made
+  // the caller's timeout five times longer than the value it passed in.
+  const deadline = Date.now() + timeoutMs;
   for (let hop = 0; ; hop++) {
     const res = await fetch(url, {
       redirect: "manual",
-      signal: AbortSignal.timeout(timeoutMs),
+      signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())),
       headers: { "user-agent": opts.userAgent ?? UA, "accept": "text/plain, text/markdown;q=0.9, */*;q=0.1" }
     });
     if (res.status >= 300 && res.status < 400) {
@@ -144,7 +148,7 @@ export function validateLlmsTxt(f) {
   } else {
     add("sections", "warn", "H2 sections group the content", "no H2 sections found; sections are the convention for grouping links");
   }
-  const links = [...f.text.matchAll(/\[([^\]]*)\]\(([^)\s]+)\)/g)];
+  const links = [...f.text.matchAll(/\[([^\][]*)\]\(([^)\s]+)\)/g)];
   const absolute = links.filter((m) => /^https?:\/\//.test(m[2])).length;
   if (links.length === 0) {
     add("links", "warn", "Markdown links an agent can follow", "no markdown links found");