turva-llms-txt-validator 0.3.4 → 0.3.5

package/README.md

@@ -123,7 +123,7 @@ The default validation requests two documents: `https://<host>/llms.txt` and `ht
 
 - Timeout: eight seconds per document, shared across its redirect chain.
 - Read limit: 256 KB per response.
-- Redirects: up to four hops, over HTTPS, to the same host or its `www`/apex equivalent. Off-site redirects, embedded credentials and unsupported ports are rejected.
+- Redirects: up to four hops, over HTTPS, to the same host or its `www`/apex equivalent. Off-site redirects, embedded credentials and unsupported ports are rejected. A rejected target is reported with its credentials and fragment removed and its query values masked.
 - Host checks: IP literals, bracketed IPv6 addresses, localhost and the internal-use TLDs `local`, `internal`, `home`, `lan`, `corp`, `test` and `invalid` are rejected before a request is sent. Redirect targets are checked too.
 
 **DNS resolution is not checked for private addresses.** A syntactically public domain can resolve to a private IP, so if you expose this package through a service that accepts untrusted domains, enforce private-address restrictions at the network layer. Local and CI runs use their own network policy. The hosted validator runs on the Cloudflare edge.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turva-llms-txt-validator",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "description": "Validate a site's llms.txt structure: eight checks as pass/warn/fail plus two llms.txt v2 discovery checks that read pass or info and never move the summary, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
   "type": "module",
   "main": "./src/index.mjs",

package/src/index.mjs

@@ -28,6 +28,41 @@ export function cut(s, n) {
   return /[\uD800-\uDBFF]$/.test(s) ? s.slice(0, -1) : s;
 }
 
+// A redirect target the way a report may show it. The scheme, the host and the path stay readable,
+// and the parts that can carry a secret do not: the user name and the password are removed, every
+// query value becomes ***, and the fragment is dropped. The caller cuts the result to length
+// afterwards, because a cut is not a mask. Up to 0.3.4 a refused redirect to
+// https://user:password@host/ kept both in the check's detail and in the returned location (outside
+// review, 2026-09-22). A target the URL parser refuses is masked by hand: nothing after the first ?
+// or # survives, and when the target has a scheme or starts with two slashes, everything up to the
+// last @ before them is shown as ***. Mirrors worker.js maskLocation().
+export function maskLocation(href, base) {
+  let u = null;
+  try { u = new URL(href, base); } catch { u = null; }
+  if (u) {
+    u.username = "";
+    u.password = "";
+    for (const key of Array.from(u.searchParams.keys())) u.searchParams.set(key, "***");
+    u.hash = "";
+    return u.href;
+  }
+  let s = String(href).replace(/[\t\n\r]/g, "");
+  const stop = s.search(/[?#]/);
+  if (stop >= 0) s = s.slice(0, stop) + (s[stop] === "?" ? "?***" : "");
+  const scheme = /^[a-z][a-z0-9+.-]*:/i.exec(s);
+  const start = scheme ? scheme[0].length : 0;
+  let from = start;
+  while (from < s.length && (s[from] === "/" || s[from] === "\\")) from++;
+  // Without a scheme and two slashes there is no authority, and an @ is part of a path.
+  if (!scheme && from - start < 2) return s;
+  // Nothing tells which @ of a refused target ends the user information, so everything up to the
+  // last one is shown as ***. The mask is visible on purpose: three review rounds on 2026-09-22
+  // measured every rule that guessed, and each one either returned a password that holds a slash
+  // or made the host vanish without a trace.
+  const at = s.lastIndexOf("@");
+  return at < from ? s : s.slice(0, from) + "***@" + s.slice(at + 1);
+}
+
 export function normalizeHostInput(raw) {
   let s = String(raw || "").trim().toLowerCase();
   if (!s) return null;
@@ -77,13 +112,13 @@ export async function fetchLlmsTxt(host, opts = {}) {
       try { await res.body?.cancel(); } catch { /* the verdict below is the answer */ }
       const loc = res.headers.get("location") || "";
       if (!loc) return { redirect: true, reason: "no-location", status: res.status, location: "" };
-      if (hop >= 4) return { redirect: true, reason: "too-many", status: res.status, location: cut(loc, 120) };
+      if (hop >= 4) return { redirect: true, reason: "too-many", status: res.status, location: cut(maskLocation(loc, url), 120) };
       let next;
-      try { next = new URL(loc, url); } catch { return { redirect: true, reason: "bad-location", status: res.status, location: cut(loc, 120) }; }
+      try { next = new URL(loc, url); } catch { return { redirect: true, reason: "bad-location", status: res.status, location: cut(maskLocation(loc, url), 120) }; }
       const safeTarget = next.protocol === "https:" && !next.port && !next.username && !next.password && isValidPublicHost(next.hostname);
       const twin = (next.hostname.startsWith("www.") ? next.hostname.slice(4) : next.hostname) === reqApex;
-      if (!safeTarget) return { redirect: true, reason: "unsafe-target", status: res.status, location: cut(next.href, 120) };
-      if (!twin) return { redirect: true, reason: "off-host", status: res.status, location: cut(next.href, 120) };
+      if (!safeTarget) return { redirect: true, reason: "unsafe-target", status: res.status, location: cut(maskLocation(next.href), 120) };
+      if (!twin) return { redirect: true, reason: "off-host", status: res.status, location: cut(maskLocation(next.href), 120) };
       if (!redirectedFrom) redirectedFrom = url;
       url = next.href;
       continue;