turva-llms-txt-validator 0.3.21 → 0.4.0

package/README.md

@@ -108,6 +108,8 @@ for (const check of result.checks) {
 
 `validateHost` rejects on invalid input or a failed `/llms.txt` fetch. Structural failures are returned in `checks` with the summary `not valid`.
 
+The package is written in TypeScript and ships its type declarations.
+
 ## Use in CI
 
 Add this step to a GitHub Actions job that already has Node.js 22 or newer available. Replace `your-domain.com` with your domain:
@@ -137,10 +139,10 @@ For vulnerability reporting and supported versions, see [SECURITY.md](https://gi
 GitHub Actions publishes every version from 0.1.3 on with npm trusted publishing, and each one carries a provenance attestation. To check one, install it in an empty directory and ask npm to verify the signatures. Replace the version with the one you want to check.
 
 ```sh
-mkdir verify-llms && cd verify-llms && npm init -y && npm install turva-llms-txt-validator@0.3.21 --ignore-scripts && npm audit signatures
+mkdir verify-llms && cd verify-llms && npm init -y && npm install turva-llms-txt-validator@0.4.0 --ignore-scripts && npm audit signatures
 ```
 
-`npm audit signatures` checks the registry signature and the provenance attestation of each installed package that has one. The attestation of this package names three things to compare with what you expect: the repository `github.com/erekola/llms-txt-validator`, the workflow file `.github/workflows/publish.yml` and the commit that produced the tarball. The npm version page shows them under Provenance. Tag `v0.3.21` must point at that same commit, and `gh api repos/erekola/llms-txt-validator/commits/v0.3.21 --jq .sha` prints it. A mismatch is a reason not to use that version. Provenance proves where a release was built and from which commit. It does not prove that the code is safe, so reading the source and the dependencies stays your job.
+`npm audit signatures` checks the registry signature and the provenance attestation of each installed package that has one. The attestation of this package names three things to compare with what you expect: the repository `github.com/erekola/llms-txt-validator`, the workflow file `.github/workflows/publish.yml` and the commit that produced the tarball. The npm version page shows them under Provenance. Tag `v0.4.0` must point at that same commit, and `gh api repos/erekola/llms-txt-validator/commits/v0.4.0 --jq .sha` prints it. A mismatch is a reason not to use that version. Provenance proves where a release was built and from which commit. It does not prove that the code is safe, so reading the source and the dependencies stays your job.
 
 ## Hosted version and source
 

package/THIRD-PARTY-NOTICES.md

@@ -1,11 +1,12 @@
 # Third-party notices
 
 This package's own license is MIT, in `LICENSE`. The named character reference
-table in `src/index.mjs` (`NAMED_REFERENCE_DATA`) is generated from data
-published by a third-party package and carries that package's own license
-below. This is a build-time data source, not an installed runtime dependency:
-the table is generated once and copied into `src/index.mjs`, so no reference
-to it appears in `package.json` or in the dependency tree an installer sees.
+table `NAMED_REFERENCE_DATA` in `src/index.ts` is generated from data published
+by a third-party package and carries that package's own license below. It is
+compiled into `dist/src/index.js` with the rest of the source. This is a
+build-time data source, not an installed runtime dependency: the table is
+generated once and copied into `src/index.ts`, so no reference to it appears in
+`package.json` or in the dependency tree an installer sees.
 
 ## character-entities
 

package/dist/bin/cli.js

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+// CLI for turva-llms-txt-validator. Exit codes: 0 = valid (or valid with
+// warnings), 1 = not valid (or warnings with --strict), 2 = could not fetch
+// or bad input. The two v2 discovery checks and the input-path note carry status
+// "info" and move no exit code, by design. Same checks as the hosted validator, and the same
+// JSON shape for a completed validation:
+// curl -H "Accept: application/json" "https://turva.dev/llms-txt-validator?url=example.com"
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { validateHost, stripControls, maskLocation } from "../src/index.js";
+const USAGE = [
+    "usage: llms-txt-validate <domain-or-url> [--json] [--strict]",
+    "  --json     print the result as JSON (same shape as the hosted validator for a completed validation)",
+    "  --strict   exit 1 on warnings too, for CI gates",
+    "  --help     print this help",
+    "  --version  print the installed version and exit"
+];
+const KNOWN = new Set(["--json", "--strict", "--help", "-h", "--version", "-V"]);
+// The arguments are read strictly since 0.3.8 (Tek-496). An unknown flag such as a misspelled
+// --strcit used to be ignored, so a CI step ran without the gate it asked for, and a second
+// target was dropped without a word. Both now stop with exit 2, and every input error is JSON
+// when --json was given, the missing target included.
+const args = process.argv.slice(2);
+const json = args.includes("--json");
+const flags = args.filter((a) => a.startsWith("-") && a !== "-");
+const targets = args.filter((a) => !(a.startsWith("-") && a !== "-"));
+function fail(message) {
+    if (json)
+        console.log(JSON.stringify({ error: message }, null, 2));
+    else {
+        console.error("error: " + message);
+        console.error(USAGE[0]);
+    }
+    process.exit(2);
+}
+if (flags.includes("--help") || flags.includes("-h")) {
+    for (const line of USAGE)
+        console.log(line);
+    process.exit(0);
+}
+if (flags.includes("--version") || flags.includes("-V")) {
+    // Read package.json relative to this file, not the caller's cwd, so this works the same
+    // whether the CLI runs installed, linked or through npx (V-04, outside audit 2026-09-26).
+    // No network read: the version is the one actually installed, not the npm registry's latest.
+    const pkgUrl = new URL("../../package.json", import.meta.url);
+    const pkg = JSON.parse(readFileSync(fileURLToPath(pkgUrl), "utf8"));
+    console.log(pkg.version);
+    process.exit(0);
+}
+const unknown = flags.find((f) => !KNOWN.has(f));
+if (unknown)
+    fail("unknown option " + JSON.stringify(stripControls(unknown).slice(0, 40)));
+if (targets.length === 0 || targets[0].trim() === "")
+    fail("no domain or URL given");
+if (targets.length > 1)
+    fail("one domain or URL at a time, got " + targets.length);
+const mark = { pass: "ok  ", warn: "warn", fail: "FAIL", info: "info" };
+try {
+    const result = await validateHost(targets[0]);
+    if (json) {
+        console.log(JSON.stringify(result, null, 2));
+    }
+    else {
+        // Details reach this point without control characters (validateHost strips them), so a
+        // published link cannot write an escape sequence to the terminal. A check's reflected
+        // value, when it has one, is printed ahead of the sentence, the same position the old
+        // concatenated string put it in, so text output reads the same while the JSON field stays
+        // separate (round 3 Tek-542, mirrored from worker.js decision 19, Tek-526).
+        console.log(result.target);
+        for (const c of result.checks)
+            console.log("  " + mark[c.status] + "  " + c.label + " (" + (c.value !== undefined ? c.value + " " : "") + c.detail + ")");
+        console.log("result: " + result.summary);
+    }
+    if (result.summary === "not valid")
+        process.exit(1);
+    if (result.summary === "valid with warnings" && flags.includes("--strict"))
+        process.exit(1);
+    process.exit(0);
+}
+catch (err) {
+    const message = stripControls(err && err.message ? err.message : String(err));
+    // The hosted validator answers every failure as JSON when JSON was asked for, so a CI
+    // step that parses --json output gets {"error": ...} here too instead of an empty stdout
+    // and a plain-text stderr (round 16 S3-3). The exit code is unchanged.
+    //
+    // target and code are added as separate fields (V-05, outside audit 2026-09-26): a bare
+    // {"error": message} left a CI step unable to tell which target failed or why without
+    // parsing free text. code reads a network error's cause first (undici wraps the real
+    // errno code, such as EAI_AGAIN or ECONNREFUSED, in err.cause.code) and falls back to
+    // err.code for an error that sets it directly; it is omitted, not null, when neither is set.
+    // target goes through maskLocation like every other address the tool prints, so a user name,
+    // a password or a query value typed into the command line is never echoed back.
+    const code = (err && err.cause && err.cause.code) || (err && err.code);
+    if (json) {
+        const report = { error: message, target: stripControls(maskLocation(targets[0])) };
+        if (code)
+            report.code = stripControls(String(code));
+        console.log(JSON.stringify(report, null, 2));
+    }
+    else
+        console.error("error: " + message);
+    process.exit(2);
+}

package/dist/src/index.d.ts

@@ -0,0 +1,59 @@
+export type CheckStatus = "pass" | "warn" | "fail" | "info";
+export type LlmsCheck = {
+    id: string;
+    status: CheckStatus;
+    label: string;
+    detail: string;
+    value?: string;
+};
+export type ChecksSummary = "not valid" | "valid with warnings" | "valid";
+export type ValidationResult = {
+    target: string;
+    summary: ChecksSummary;
+    checks: LlmsCheck[];
+};
+export type LinkRelations = {
+    describedby: string | null;
+    markdown: string | null;
+    headCut?: boolean;
+};
+export type FetchedLlmsTxt = {
+    redirect?: undefined;
+    status: number;
+    contentType: string;
+    linkHeader: string;
+    text: string;
+    bytes: number;
+    truncated: boolean;
+    redirectedFrom: string | null;
+    finalUrl: string;
+};
+export type FetchRedirect = {
+    redirect: true;
+    reason: "no-location" | "too-many" | "bad-location" | "unsafe-target" | "off-host";
+    status: number;
+    location: string;
+};
+export type ValidateHostOptions = {
+    timeoutMs?: number;
+    cap?: number;
+    userAgent?: string;
+    path?: string;
+    accept?: string;
+};
+export type FetchOptions = ValidateHostOptions;
+export type LlmsTxtInput = FetchRedirect | (Partial<FetchedLlmsTxt> & Pick<FetchedLlmsTxt, "status" | "contentType" | "text" | "bytes" | "truncated">);
+export declare const VERSION: string;
+export declare function cut(s: string | number, n: number): string;
+export declare function maskLocation(href: string, base?: string): string;
+export declare function stripBidi(s: unknown): string;
+export declare function stripControls(s: unknown): string;
+export declare function normalizeHostInput(raw: unknown): string | null;
+export declare function enteredPath(raw: unknown): string;
+export declare function isValidPublicHost(host: string | null | undefined): boolean;
+export declare function fetchLlmsTxt(host: string, opts?: FetchOptions): Promise<FetchedLlmsTxt | FetchRedirect>;
+export declare function validateLlmsTxt(f: LlmsTxtInput): LlmsCheck[];
+export declare function findLinkRelations(html: string, linkHeader: string): LinkRelations;
+export declare function validateV2Discovery(found: LinkRelations | null, unreadReason?: string): LlmsCheck[];
+export declare function summarizeChecks(checks: LlmsCheck[]): ChecksSummary;
+export declare function validateHost(input: unknown, opts?: ValidateHostOptions): Promise<ValidationResult>;