turva-llms-txt-validator 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Erik Rekola
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,64 @@
+# turva-llms-txt-validator
+
+Validate a site's llms.txt structure from the command line, from Node, or in CI. Seven checks, each reported as pass, warn or fail, with a one-line detail. No score on purpose: a structure check can say what is there and what is missing, and a number on top of seven checks would look like an agent-readiness score without measuring one.
+
+This is the open-source form of the hosted validator at [turva.dev/llms-txt-validator](https://turva.dev/llms-txt-validator), which runs the same logic inside the open [turva.dev Cloudflare Worker](https://codeberg.org/erekola/turva-worker). The hosted validator stays canonical: if the two ever disagree, the hosted one wins and this package gets the fix.
+
+## Install
+
+    npm install -g turva-llms-txt-validator
+
+Or run it without installing:
+
+    npx turva-llms-txt-validator example.com
+
+## CLI
+
+    llms-txt-validate example.com
+    llms-txt-validate example.com --json
+    llms-txt-validate example.com --strict
+
+Exit codes: 0 valid (warnings allowed), 1 not valid (with --strict, warnings also exit 1), 2 fetch or input error. Only the target site's /llms.txt file is fetched over https, redirects are not followed (a redirect fails the first check), the fetch times out after 8 seconds and the read is capped at 256 KB. Nothing is stored.
+
+## The seven checks
+
+| # | Check | fail | warn |
+|---|-------|------|------|
+| 1 | File exists at /llms.txt, HTTP 200 | non-200 or a redirect | |
+| 2 | Response is plain text | body looks like HTML | content-type is not text/plain or text/markdown |
+| 3 | Starts with a single H1 title | first non-empty line is not a markdown H1 | |
+| 4 | Blockquote summary after the title | | missing one-line summary |
+| 5 | H2 sections group the content | | no H2 sections |
+| 6 | Markdown links an agent can follow | | no links, or relative links |
+| 7 | Small enough to be cheap to read | | over 50 KB, or read truncated at 256 KB |
+
+An eighth signal, inline HTML in the file, is reported in the same list as a warning.
+
+## Node API
+
+    import { validateHost } from "turva-llms-txt-validator";
+
+    const result = await validateHost("example.com");
+    console.log(result.summary);
+    for (const c of result.checks) console.log(c.status, c.label, c.detail);
+
+The summary is one of valid, valid with warnings, or not valid. The result object is { target, summary, checks }, the same shape the hosted validator returns as JSON:
+
+    curl -H "Accept: application/json" "https://turva.dev/llms-txt-validator?url=example.com"
+
+## CI
+
+GitHub Actions:
+
+    - name: Validate llms.txt
+      run: npx turva-llms-txt-validator your-domain.com --strict
+
+Woodpecker or any other runner: the same npx line works anywhere Node 18.17 or newer is present.
+
+## Why these checks
+
+The llms.txt format is a plain text map of a site for AI agents: an H1 title, a one-line blockquote summary, and H2 sections of markdown links. The checks test exactly that structure and nothing else. What the format is and why it matters is written out at [llms.txt explained](https://turva.dev/guides/llms-txt), and the free tools this package belongs to are collected at [turva.dev/tools](https://turva.dev/tools).
+
+## License
+
+MIT. Source at [codeberg.org/erekola/llms-txt-validator](https://codeberg.org/erekola/llms-txt-validator), mirrored at [github.com/erekola/llms-txt-validator](https://github.com/erekola/llms-txt-validator).

package/bin/cli.mjs

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+// CLI for turva-llms-txt-validator. Exit codes: 0 = valid (or valid with
+// warnings), 1 = not valid (or warnings with --strict), 2 = could not fetch
+// or bad input. Same checks and JSON shape as the hosted validator:
+// curl -H "Accept: application/json" "https://turva.dev/llms-txt-validator?url=example.com"
+import { validateHost } from "../src/index.mjs";
+
+const args = process.argv.slice(2);
+const flags = new Set(args.filter((a) => a.startsWith("--")));
+const target = args.find((a) => !a.startsWith("--"));
+
+if (!target || flags.has("--help")) {
+  console.log("usage: llms-txt-validate <domain-or-url> [--json] [--strict]");
+  console.log("  --json    print the result as JSON (same shape as the hosted validator)");
+  console.log("  --strict  exit 1 on warnings too, for CI gates");
+  process.exit(target ? 0 : 2);
+}
+
+const mark = { pass: "ok  ", warn: "warn", fail: "FAIL" };
+try {
+  const result = await validateHost(target);
+  if (flags.has("--json")) {
+    console.log(JSON.stringify(result, null, 2));
+  } else {
+    console.log(result.target);
+    for (const c of result.checks) console.log("  " + mark[c.status] + "  " + c.label + " (" + c.detail + ")");
+    console.log("result: " + result.summary);
+  }
+  if (result.summary === "not valid") process.exit(1);
+  if (result.summary === "valid with warnings" && flags.has("--strict")) process.exit(1);
+  process.exit(0);
+} catch (err) {
+  console.error("error: " + (err && err.message ? err.message : String(err)));
+  process.exit(2);
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "turva-llms-txt-validator",
+  "version": "0.1.1",
+  "description": "Validate a site's llms.txt structure: seven checks, pass/warn/fail, JSON output. Same checks as the hosted validator at turva.dev/llms-txt-validator.",
+  "type": "module",
+  "main": "./src/index.mjs",
+  "exports": "./src/index.mjs",
+  "bin": {
+    "llms-txt-validate": "bin/cli.mjs"
+  },
+  "files": [
+    "src",
+    "bin",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "node --test"
+  },
+  "engines": {
+    "node": ">=18.17"
+  },
+  "keywords": [
+    "llms.txt",
+    "llms-txt",
+    "validator",
+    "ai-agents",
+    "agent-readiness",
+    "cli"
+  ],
+  "author": "Erik Rekola (https://turva.dev)",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://codeberg.org/erekola/llms-txt-validator"
+  },
+  "homepage": "https://turva.dev/llms-txt-validator",
+  "bugs": {
+    "url": "https://codeberg.org/erekola/llms-txt-validator/issues"
+  }
+}

package/src/index.mjs

@@ -0,0 +1,155 @@
+// turva-llms-txt-validator: validate a site's llms.txt structure.
+// The checks mirror the hosted validator at https://turva.dev/llms-txt-validator,
+// which runs this same logic inside the turva.dev Cloudflare Worker
+// (https://codeberg.org/erekola/turva-worker). The hosted validator stays
+// canonical: if the two ever disagree, the hosted one wins and this package
+// gets the fix.
+
+export const VERSION = "0.1.0";
+const UA = "turva-llms-txt-validator/" + VERSION + " (+https://turva.dev/llms-txt-validator)";
+
+export function normalizeHostInput(raw) {
+  let s = String(raw || "").trim().toLowerCase();
+  if (!s) return null;
+  if (!/^[a-z][a-z0-9+.-]*:\/\//.test(s)) s = "https://" + s;
+  let u;
+  try { u = new URL(s); } catch { return null; }
+  if (u.protocol !== "https:" && u.protocol !== "http:") return null;
+  if (u.port && u.port !== "443" && u.port !== "80") return null;
+  if (u.username || u.password) return null;
+  return u.hostname;
+}
+
+export function isValidPublicHost(host) {
+  if (!host || host.length > 253) return false;
+  if (host.startsWith("[") || /^\d+\.\d+\.\d+\.\d+$/.test(host)) return false;
+  if (!/^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z][a-z0-9-]{1,62}$/.test(host)) return false;
+  const tld = host.split(".").pop();
+  if (["localhost", "local", "internal", "home", "lan", "corp", "test", "invalid"].includes(tld)) return false;
+  return true;
+}
+
+export async function fetchLlmsTxt(host, opts = {}) {
+  const timeoutMs = opts.timeoutMs ?? 8000;
+  const cap = opts.cap ?? 262144;
+  const target = "https://" + host + "/llms.txt";
+  const res = await fetch(target, {
+    redirect: "manual",
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { "user-agent": opts.userAgent ?? UA, "accept": "text/plain, text/markdown;q=0.9, */*;q=0.1" }
+  });
+  if (res.status >= 300 && res.status < 400) {
+    return { redirect: true, status: res.status, location: res.headers.get("location") || "" };
+  }
+  let bytes = 0, truncated = false;
+  const chunks = [];
+  if (res.body) {
+    const reader = res.body.getReader();
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.length;
+      if (bytes > cap) {
+        truncated = true;
+        chunks.push(value.slice(0, value.length - (bytes - cap)));
+        bytes = cap;
+        await reader.cancel();
+        break;
+      }
+      chunks.push(value);
+    }
+  }
+  const buf = new Uint8Array(bytes);
+  let o = 0;
+  for (const c of chunks) { buf.set(c, o); o += c.length; }
+  return {
+    status: res.status,
+    contentType: res.headers.get("content-type") || "",
+    text: new TextDecoder("utf-8").decode(buf),
+    bytes,
+    truncated
+  };
+}
+
+export function validateLlmsTxt(f) {
+  const checks = [];
+  const add = (id, status, label, detail) => checks.push({ id, status, label, detail });
+  if (f.redirect) {
+    add("http-status", "fail", "File exists at /llms.txt", "expected HTTP 200 at /llms.txt, got a " + f.status + " redirect" + (f.location ? " to " + f.location.slice(0, 120) : "") + "; llms.txt should be served directly");
+    return checks;
+  }
+  if (f.status !== 200) {
+    add("http-status", "fail", "File exists at /llms.txt", "expected HTTP 200, got " + f.status);
+    return checks;
+  }
+  add("http-status", "pass", "File exists at /llms.txt", "HTTP 200");
+  const ct = (f.contentType || "").toLowerCase();
+  const looksHtml = /^\s*(<!doctype|<html|<head|<body)/i.test(f.text);
+  if (looksHtml) {
+    add("content-type", "fail", "Response is plain text", "the body looks like an HTML page, not an llms.txt file");
+    return checks;
+  }
+  if (ct.includes("text/plain") || ct.includes("text/markdown")) {
+    add("content-type", "pass", "Response is plain text", ct.split(";")[0]);
+  } else {
+    add("content-type", "warn", "Response is plain text", "content-type is " + (ct.split(";")[0] || "missing") + ", text/plain or text/markdown is the convention");
+  }
+  const lines = f.text.split(/\r?\n/);
+  const firstIdx = lines.findIndex((l) => l.trim() !== "");
+  const first = firstIdx === -1 ? "" : lines[firstIdx].trim();
+  if (/^# \S/.test(first)) {
+    add("h1-title", "pass", "Starts with an H1 title", JSON.stringify(first.slice(0, 80)));
+  } else {
+    add("h1-title", "fail", "Starts with an H1 title", "the first non-empty line should be a markdown H1 (# Site name)");
+  }
+  const afterH1 = lines.slice(firstIdx + 1).find((l) => l.trim() !== "") || "";
+  if (afterH1.trim().startsWith("> ")) {
+    add("summary", "pass", "Blockquote summary after the title", JSON.stringify(afterH1.trim().slice(0, 80)));
+  } else {
+    add("summary", "warn", "Blockquote summary after the title", "recommended by the format (> one-line summary), not required");
+  }
+  const h2Count = (f.text.match(/^## /gm) || []).length;
+  if (h2Count > 0) {
+    add("sections", "pass", "H2 sections group the content", h2Count + " section" + (h2Count === 1 ? "" : "s"));
+  } else {
+    add("sections", "warn", "H2 sections group the content", "no H2 sections found; sections are the convention for grouping links");
+  }
+  const links = [...f.text.matchAll(/\[([^\]]*)\]\(([^)\s]+)\)/g)];
+  const absolute = links.filter((m) => /^https?:\/\//.test(m[2])).length;
+  if (links.length === 0) {
+    add("links", "warn", "Markdown links an agent can follow", "no markdown links found");
+  } else if (absolute === links.length) {
+    add("links", "pass", "Markdown links an agent can follow", links.length + " link" + (links.length === 1 ? "" : "s") + ", all absolute URLs");
+  } else {
+    add("links", "warn", "Markdown links an agent can follow", links.length + " links, " + (links.length - absolute) + " relative; absolute URLs travel better when the file is read out of context");
+  }
+  if (f.truncated) {
+    add("size", "warn", "Small enough to be cheap to read", "over 256 KB, read truncated");
+  } else if (f.bytes <= 51200) {
+    add("size", "pass", "Small enough to be cheap to read", f.bytes + " bytes");
+  } else {
+    add("size", "warn", "Small enough to be cheap to read", f.bytes + " bytes; consider moving detail to llms-full.txt");
+  }
+  if (/<[a-z][a-z0-9-]*[\s>]/i.test(f.text)) {
+    add("no-html", "warn", "No HTML markup in the file", "HTML tags found; llms.txt should be plain markdown");
+  } else {
+    add("no-html", "pass", "No HTML markup in the file", "plain markdown");
+  }
+  return checks;
+}
+
+export function summarizeChecks(checks) {
+  if (checks.some((c) => c.status === "fail")) return "not valid";
+  if (checks.some((c) => c.status === "warn")) return "valid with warnings";
+  return "valid";
+}
+
+export async function validateHost(input, opts = {}) {
+  const host = normalizeHostInput(input);
+  if (!host || !isValidPublicHost(host)) {
+    throw new Error("not a public domain name: " + String(input).slice(0, 120));
+  }
+  const fetched = await fetchLlmsTxt(host, opts);
+  const checks = validateLlmsTxt(fetched);
+  return { target: "https://" + host + "/llms.txt", summary: summarizeChecks(checks), checks };
+}