turva-llms-txt-validator 0.3.5 → 0.3.7

package/README.md

@@ -14,7 +14,7 @@ Requires **Node.js 18.17 or newer**. Run it without a global install:
 npx --yes turva-llms-txt-validator example.com
 ```
 
-Replace `example.com` with the domain you want to check. A domain, HTTP URL or HTTPS URL is accepted. Validation uses the host's `/llms.txt` and home page over HTTPS, regardless of the supplied path.
+Replace `example.com` with the domain you want to check. A domain, HTTP URL or HTTPS URL is accepted. Validation uses the host's `/llms.txt` and home page over HTTPS, regardless of the supplied path. If the address has a path other than `/` or `/llms.txt`, the result names that path first in an `info` line.
 
 For repeated use, install the CLI globally:
 
@@ -65,7 +65,7 @@ Completed validation returns `{ target, summary, checks }`. Each check contains
 | 1 | `/llms.txt` returns HTTP 200 | Non-200 response, rejected or excessive redirects | none |
 | 2 | Response is plain text | Body looks like an HTML page | Content type is neither `text/plain` nor `text/markdown` |
 | 3 | First non-empty line is a Markdown H1 | Missing H1, including a title indented as a code block | none |
-| 4 | Blockquote summary follows the title | none | Missing summary |
+| 4 | Blockquote summary follows the title | none | Missing summary, or a blockquote after other text but before the first H2, named by its line |
 | 5 | H2 sections group the content | none | No H2 sections, or no section contains a Markdown link list |
 | 6 | Markdown links have names and absolute HTTP or HTTPS targets | none | Missing links, empty names, relative targets or unsupported targets |
 | 7 | File is small enough to read cheaply | none | Over 50 KB, or truncated at the 256 KB read limit |
@@ -123,8 +123,8 @@ The default validation requests two documents: `https://<host>/llms.txt` and `ht
 
 - Timeout: eight seconds per document, shared across its redirect chain.
 - Read limit: 256 KB per response.
-- Redirects: up to four hops, over HTTPS, to the same host or its `www`/apex equivalent. Off-site redirects, embedded credentials and unsupported ports are rejected. A rejected target is reported with its credentials and fragment removed and its query values masked.
-- Host checks: IP literals, bracketed IPv6 addresses, localhost and the internal-use TLDs `local`, `internal`, `home`, `lan`, `corp`, `test` and `invalid` are rejected before a request is sent. Redirect targets are checked too.
+- Redirects: up to four hops, over HTTPS, to the same host or its `www`/apex equivalent. Off-site redirects, embedded credentials and unsupported ports are rejected. A redirect target is reported with its credentials and fragment removed and its query values masked, whether the redirect was followed or rejected.
+- Host checks: IP literals, bracketed IPv6 addresses, localhost and the internal-use TLDs `local`, `internal`, `home`, `lan`, `corp`, `test`, `invalid`, `arpa` and `onion` are rejected before a request is sent. One trailing dot on the host you give, as in `example.com.`, is removed first. Redirect targets are checked too.
 
 **DNS resolution is not checked for private addresses.** A syntactically public domain can resolve to a private IP, so if you expose this package through a service that accepts untrusted domains, enforce private-address restrictions at the network layer. Local and CI runs use their own network policy. The hosted validator runs on the Cloudflare edge.
 

package/bin/cli.mjs

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 // CLI for turva-llms-txt-validator. Exit codes: 0 = valid (or valid with
 // warnings), 1 = not valid (or warnings with --strict), 2 = could not fetch
-// or bad input. The two v2 discovery checks carry status "info" and move no
-// exit code, by design. Same checks and JSON shape as the hosted validator:
+// or bad input. The two v2 discovery checks and the input-path note carry status
+// "info" and move no exit code, by design. Same checks and JSON shape as the hosted validator:
 // curl -H "Accept: application/json" "https://turva.dev/llms-txt-validator?url=example.com"
 import { validateHost } from "../src/index.mjs";
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.3.5",
+  "version": "0.3.7",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks that read pass or info and never move the summary, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -63,6 +63,14 @@ export function maskLocation(href, base) {
   return at < from ? s : s.slice(0, from) + "***@" + s.slice(at + 1);
 }
 
+// Unicode bidi controls (U+202A to U+202E, U+2066 to U+2069) have no place in a check's
+// detail: text quoted from the fetched file could otherwise read in a different order here
+// than in the file itself, and the right-to-left override is the classic way to do that
+// (round 19, V6-U1). Mirrors worker.js stripBidi().
+export function stripBidi(s) {
+  return String(s).replace(/[\u202A-\u202E\u2066-\u2069]/g, "");
+}
+
 export function normalizeHostInput(raw) {
   let s = String(raw || "").trim().toLowerCase();
   if (!s) return null;
@@ -72,7 +80,25 @@ export function normalizeHostInput(raw) {
   if (u.protocol !== "https:" && u.protocol !== "http:") return null;
   if (u.port && u.port !== "443" && u.port !== "80") return null;
   if (u.username || u.password) return null;
-  return u.hostname;
+  // One trailing dot names the same host in absolute form, so "example.com." is read as
+  // example.com instead of refused (round 19, K7-P5). Mirrors worker.js normalizeHostInput().
+  const typedHost = u.hostname;
+  return typedHost.length > 1 && typedHost.endsWith(".") ? typedHost.slice(0, -1) : typedHost;
+}
+
+// The path of the address as typed, when it is neither the root nor /llms.txt. validateHost keeps its
+// reads pinned to /llms.txt and the home page, because a fixed path is part of the guard against use
+// as a fetch proxy, so a typed /docs/llms.txt used to be dropped without a word while the result
+// described the root file (outside retest 2026-09-26). The result now names the unused path in an
+// info check. The case of the path is kept, and the query and fragment are never returned.
+// Mirrors worker.js enteredPath() (v3.176.0).
+export function enteredPath(raw) {
+  let s = String(raw || "").trim();
+  if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(s)) s = "https://" + s;
+  let u;
+  try { u = new URL(s); } catch { return ""; }
+  const path = u.pathname;
+  return path === "/" || path === "" || path === "/llms.txt" ? "" : path;
 }
 
 export function isValidPublicHost(host) {
@@ -80,7 +106,9 @@ export function isValidPublicHost(host) {
   if (host.startsWith("[") || /^\d+\.\d+\.\d+\.\d+$/.test(host)) return false;
   if (!/^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{1,62}$/.test(host)) return false;
   const tld = host.split(".").pop();
-  if (["localhost", "local", "internal", "home", "lan", "corp", "test", "invalid"].includes(tld)) return false;
+  // arpa and onion since round 19 (P3): home.arpa is the home-network name of RFC 8375,
+  // in-addr.arpa and ip6.arpa name addresses, and an onion name never resolves on the public web.
+  if (["localhost", "local", "internal", "home", "lan", "corp", "test", "invalid", "arpa", "onion"].includes(tld)) return false;
   return true;
 }
 
@@ -272,7 +300,10 @@ export function validateLlmsTxt(f) {
     add("http-status", "fail", "File exists at /llms.txt", "expected HTTP 200, got " + f.status);
     return checks;
   }
-  add("http-status", "pass", "File exists at /llms.txt", f.redirectedFrom ? "HTTP 200, followed a redirect from " + f.redirectedFrom + " to " + f.finalUrl : "HTTP 200");
+  // An accepted redirect is shown masked as well (round 19, K7-2, mirrored from worker.js
+  // v3.164.0): a query value or a fragment in a same-host Location used to reach this
+  // detail as sent.
+  add("http-status", "pass", "File exists at /llms.txt", f.redirectedFrom ? "HTTP 200, followed a redirect from " + maskLocation(f.redirectedFrom) + " to " + maskLocation(f.finalUrl) : "HTTP 200");
   const ct = (f.contentType || "").toLowerCase();
   const looksHtml = /^\s*(<!doctype|<html|<head|<body)/i.test(f.text);
   if (looksHtml) {
@@ -300,7 +331,20 @@ export function validateLlmsTxt(f) {
   if (afterH1.trim().startsWith("> ")) {
     add("summary", "pass", "Blockquote summary after the title", JSON.stringify(cut(afterH1.trim(), 80)));
   } else {
-    add("summary", "warn", "Blockquote summary after the title", "recommended by the format (> one-line summary), not required");
+    // A blockquote further down, before the first H2, is a summary in the wrong place and not a
+    // missing one, and the detail says which (outside retest 2026-09-26, mirrored from worker.js
+    // v3.176.0). The status is warn either way, because the format places the summary directly
+    // after the title.
+    const fence = fenceMask(lines);
+    let late = -1;
+    for (let i = firstIdx + 1; i < lines.length; i++) {
+      if (fence[i]) continue;
+      if (/^ {0,3}## /.test(lines[i])) break;
+      if (/^ {0,3}> /.test(lines[i])) { late = i; break; }
+    }
+    add("summary", "warn", "Blockquote summary after the title", late === -1
+      ? "recommended by the format (> one-line summary), not required"
+      : "the blockquote at line " + (late + 1) + " comes after other text, and the format places the summary directly after the title");
   }
   // Headings are read outside fences only, and with the same indentation the H1 check and
   // listItemHasLink have allowed since 2026-08-29. Until 2026-09-10 this one line still
@@ -1254,5 +1298,13 @@ export async function validateHost(input, opts = {}) {
     discovery = validateV2Discovery(null, "the home page could not be read, so this was not measured");
   }
   const checks = validateLlmsTxt(fetched).concat(discovery);
-  return { target: "https://" + host + "/llms.txt", summary: summarizeChecks(checks), checks };
+  // A path in the typed address is not used, and the result says so first (mirrored from worker.js
+  // v3.176.0). Information moves no summary and no exit code.
+  const unusedPath = enteredPath(input);
+  if (unusedPath) checks.unshift({ id: "input-path", status: "info", label: "Path in the address you entered", detail: cut(unusedPath, 120) + " is not used, because the validator always reads /llms.txt at the root of the host" });
+  // Text quoted from the fetched file leaves without bidirectional controls (round 19,
+  // V6-U1, mirrored from worker.js v3.164.0): a site's own H1 or summary could otherwise
+  // read in another order here than in the file itself.
+  const stripped = checks.map((c) => (typeof c.detail === "string" ? { ...c, detail: stripBidi(c.detail) } : c));
+  return { target: "https://" + host + "/llms.txt", summary: summarizeChecks(stripped), checks: stripped };
 }