turva-llms-txt-validator 0.3.4 → 0.3.6

package/README.md

@@ -123,8 +123,8 @@ The default validation requests two documents: `https://<host>/llms.txt` and `ht
 
 - Timeout: eight seconds per document, shared across its redirect chain.
 - Read limit: 256 KB per response.
-- Redirects: up to four hops, over HTTPS, to the same host or its `www`/apex equivalent. Off-site redirects, embedded credentials and unsupported ports are rejected.
-- Host checks: IP literals, bracketed IPv6 addresses, localhost and the internal-use TLDs `local`, `internal`, `home`, `lan`, `corp`, `test` and `invalid` are rejected before a request is sent. Redirect targets are checked too.
+- Redirects: up to four hops, over HTTPS, to the same host or its `www`/apex equivalent. Off-site redirects, embedded credentials and unsupported ports are rejected. A redirect target is reported with its credentials and fragment removed and its query values masked, whether the redirect was followed or rejected.
+- Host checks: IP literals, bracketed IPv6 addresses, localhost and the internal-use TLDs `local`, `internal`, `home`, `lan`, `corp`, `test`, `invalid`, `arpa` and `onion` are rejected before a request is sent. One trailing dot on the host you give, as in `example.com.`, is removed first. Redirect targets are checked too.
 
 **DNS resolution is not checked for private addresses.** A syntactically public domain can resolve to a private IP, so if you expose this package through a service that accepts untrusted domains, enforce private-address restrictions at the network layer. Local and CI runs use their own network policy. The hosted validator runs on the Cloudflare edge.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.3.4",
+  "version": "0.3.6",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks that read pass or info and never move the summary, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -28,6 +28,49 @@ export function cut(s, n) {
   return /[\uD800-\uDBFF]$/.test(s) ? s.slice(0, -1) : s;
 }
 
+// A redirect target the way a report may show it. The scheme, the host and the path stay readable,
+// and the parts that can carry a secret do not: the user name and the password are removed, every
+// query value becomes ***, and the fragment is dropped. The caller cuts the result to length
+// afterwards, because a cut is not a mask. Up to 0.3.4 a refused redirect to
+// https://user:password@host/ kept both in the check's detail and in the returned location (outside
+// review, 2026-09-22). A target the URL parser refuses is masked by hand: nothing after the first ?
+// or # survives, and when the target has a scheme or starts with two slashes, everything up to the
+// last @ before them is shown as ***. Mirrors worker.js maskLocation().
+export function maskLocation(href, base) {
+  let u = null;
+  try { u = new URL(href, base); } catch { u = null; }
+  if (u) {
+    u.username = "";
+    u.password = "";
+    for (const key of Array.from(u.searchParams.keys())) u.searchParams.set(key, "***");
+    u.hash = "";
+    return u.href;
+  }
+  let s = String(href).replace(/[\t\n\r]/g, "");
+  const stop = s.search(/[?#]/);
+  if (stop >= 0) s = s.slice(0, stop) + (s[stop] === "?" ? "?***" : "");
+  const scheme = /^[a-z][a-z0-9+.-]*:/i.exec(s);
+  const start = scheme ? scheme[0].length : 0;
+  let from = start;
+  while (from < s.length && (s[from] === "/" || s[from] === "\\")) from++;
+  // Without a scheme and two slashes there is no authority, and an @ is part of a path.
+  if (!scheme && from - start < 2) return s;
+  // Nothing tells which @ of a refused target ends the user information, so everything up to the
+  // last one is shown as ***. The mask is visible on purpose: three review rounds on 2026-09-22
+  // measured every rule that guessed, and each one either returned a password that holds a slash
+  // or made the host vanish without a trace.
+  const at = s.lastIndexOf("@");
+  return at < from ? s : s.slice(0, from) + "***@" + s.slice(at + 1);
+}
+
+// Unicode bidi controls (U+202A to U+202E, U+2066 to U+2069) have no place in a check's
+// detail: text quoted from the fetched file could otherwise read in a different order here
+// than in the file itself, and the right-to-left override is the classic way to do that
+// (round 19, V6-U1). Mirrors worker.js stripBidi().
+export function stripBidi(s) {
+  return String(s).replace(/[\u202A-\u202E\u2066-\u2069]/g, "");
+}
+
 export function normalizeHostInput(raw) {
   let s = String(raw || "").trim().toLowerCase();
   if (!s) return null;
@@ -37,7 +80,10 @@ export function normalizeHostInput(raw) {
   if (u.protocol !== "https:" && u.protocol !== "http:") return null;
   if (u.port && u.port !== "443" && u.port !== "80") return null;
   if (u.username || u.password) return null;
-  return u.hostname;
+  // One trailing dot names the same host in absolute form, so "example.com." is read as
+  // example.com instead of refused (round 19, K7-P5). Mirrors worker.js normalizeHostInput().
+  const typedHost = u.hostname;
+  return typedHost.length > 1 && typedHost.endsWith(".") ? typedHost.slice(0, -1) : typedHost;
 }
 
 export function isValidPublicHost(host) {
@@ -45,7 +91,9 @@ export function isValidPublicHost(host) {
   if (host.startsWith("[") || /^\d+\.\d+\.\d+\.\d+$/.test(host)) return false;
   if (!/^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{1,62}$/.test(host)) return false;
   const tld = host.split(".").pop();
-  if (["localhost", "local", "internal", "home", "lan", "corp", "test", "invalid"].includes(tld)) return false;
+  // arpa and onion since round 19 (P3): home.arpa is the home-network name of RFC 8375,
+  // in-addr.arpa and ip6.arpa name addresses, and an onion name never resolves on the public web.
+  if (["localhost", "local", "internal", "home", "lan", "corp", "test", "invalid", "arpa", "onion"].includes(tld)) return false;
   return true;
 }
 
@@ -77,13 +125,13 @@ export async function fetchLlmsTxt(host, opts = {}) {
       try { await res.body?.cancel(); } catch { /* the verdict below is the answer */ }
       const loc = res.headers.get("location") || "";
       if (!loc) return { redirect: true, reason: "no-location", status: res.status, location: "" };
-      if (hop >= 4) return { redirect: true, reason: "too-many", status: res.status, location: cut(loc, 120) };
+      if (hop >= 4) return { redirect: true, reason: "too-many", status: res.status, location: cut(maskLocation(loc, url), 120) };
       let next;
-      try { next = new URL(loc, url); } catch { return { redirect: true, reason: "bad-location", status: res.status, location: cut(loc, 120) }; }
+      try { next = new URL(loc, url); } catch { return { redirect: true, reason: "bad-location", status: res.status, location: cut(maskLocation(loc, url), 120) }; }
       const safeTarget = next.protocol === "https:" && !next.port && !next.username && !next.password && isValidPublicHost(next.hostname);
       const twin = (next.hostname.startsWith("www.") ? next.hostname.slice(4) : next.hostname) === reqApex;
-      if (!safeTarget) return { redirect: true, reason: "unsafe-target", status: res.status, location: cut(next.href, 120) };
-      if (!twin) return { redirect: true, reason: "off-host", status: res.status, location: cut(next.href, 120) };
+      if (!safeTarget) return { redirect: true, reason: "unsafe-target", status: res.status, location: cut(maskLocation(next.href), 120) };
+      if (!twin) return { redirect: true, reason: "off-host", status: res.status, location: cut(maskLocation(next.href), 120) };
       if (!redirectedFrom) redirectedFrom = url;
       url = next.href;
       continue;
@@ -237,7 +285,10 @@ export function validateLlmsTxt(f) {
     add("http-status", "fail", "File exists at /llms.txt", "expected HTTP 200, got " + f.status);
     return checks;
   }
-  add("http-status", "pass", "File exists at /llms.txt", f.redirectedFrom ? "HTTP 200, followed a redirect from " + f.redirectedFrom + " to " + f.finalUrl : "HTTP 200");
+  // An accepted redirect is shown masked as well (round 19, K7-2, mirrored from worker.js
+  // v3.164.0): a query value or a fragment in a same-host Location used to reach this
+  // detail as sent.
+  add("http-status", "pass", "File exists at /llms.txt", f.redirectedFrom ? "HTTP 200, followed a redirect from " + maskLocation(f.redirectedFrom) + " to " + maskLocation(f.finalUrl) : "HTTP 200");
   const ct = (f.contentType || "").toLowerCase();
   const looksHtml = /^\s*(<!doctype|<html|<head|<body)/i.test(f.text);
   if (looksHtml) {
@@ -1219,5 +1270,9 @@ export async function validateHost(input, opts = {}) {
     discovery = validateV2Discovery(null, "the home page could not be read, so this was not measured");
   }
   const checks = validateLlmsTxt(fetched).concat(discovery);
-  return { target: "https://" + host + "/llms.txt", summary: summarizeChecks(checks), checks };
+  // Text quoted from the fetched file leaves without bidirectional controls (round 19,
+  // V6-U1, mirrored from worker.js v3.164.0): a site's own H1 or summary could otherwise
+  // read in another order here than in the file itself.
+  const stripped = checks.map((c) => (typeof c.detail === "string" ? { ...c, detail: stripBidi(c.detail) } : c));
+  return { target: "https://" + host + "/llms.txt", summary: summarizeChecks(stripped), checks: stripped };
 }