vagejs 0.1.0 → 0.3.0

package/dist/checks/integrity.d.ts

@@ -0,0 +1,36 @@
+import { type HTMLElement } from "node-html-parser";
+import type { SiteModel } from "../core/site.js";
+/**
+ * Defenses against injected spam (the classic hacked-site problem): links hidden with
+ * CSS, outbound links nobody approved, and common spam vocabulary. These run in
+ * `vage check` on the final HTML, so they cover page content and layouts alike.
+ */
+export declare const LINK_LOCK = "links.lock.json";
+export interface Hidden {
+    reason: string;
+    /** "strong": a CSS/SVG trick with no legitimate use around a link. "soft": hidden by a mechanism sites also use for menus (hidden, closed <details>). */
+    strength: "strong" | "soft";
+}
+/** Why this element (or an ancestor) is hidden from people, if it is. Only inline signals; stylesheets aren't evaluated. */
+export declare function hiddenReason(el: HTMLElement): Hidden | undefined;
+/** A link with nothing a visitor can see or a screen reader can announce. */
+export declare function isEmptyLink(a: HTMLElement): boolean;
+/** href of <a>/<area>, including SVG's xlink:href. */
+export declare function linkTarget(a: HTMLElement): string | undefined;
+/** Normalized host of an external http(s) link ("www." dropped), or undefined for internal/other links. */
+export declare function externalHost(href: string, site: string | undefined): string | undefined;
+/** Lowercase, punycode, no "www.", no trailing dot: one spelling per host. */
+export declare function normalizeHost(host: string): string;
+/** Every external host the built site links to, with the pages that link there. */
+export declare function outboundHosts(model: SiteModel): Map<string, {
+    route: string;
+    source: string;
+}[]>;
+export interface LinkLock {
+    domains: string[];
+}
+export declare function readLinkLock(root: string): LinkLock | undefined;
+export declare function writeLinkLock(root: string, domains: Iterable<string>): void;
+/** Words that rarely appear on legitimate business sites but are staples of injected spam. Warnings only. */
+export declare const SPAM_PATTERNS: [RegExp, string][];
+export declare function spamHits(text: string): string[];

package/dist/checks/integrity.js

@@ -0,0 +1,145 @@
+import fs from "node:fs";
+import path from "node:path";
+import { parse } from "node-html-parser";
+import { VageError } from "../core/errors.js";
+/**
+ * Defenses against injected spam (the classic hacked-site problem): links hidden with
+ * CSS, outbound links nobody approved, and common spam vocabulary. These run in
+ * `vage check` on the final HTML, so they cover page content and layouts alike.
+ */
+export const LINK_LOCK = "links.lock.json";
+// ---------------------------------------------------------------- hidden links
+const HIDING_STYLES = [
+    [/display\s*:\s*none/i, "display:none"],
+    [/visibility\s*:\s*(hidden|collapse)/i, "visibility:hidden"],
+    [/opacity\s*:\s*0(\.0+)?\s*(;|$|!)/i, "opacity:0"],
+    [/font-size\s*:\s*0(\.0+)?(px|em|rem|%)?\s*(;|$|!)/i, "font-size:0"],
+    [/(^|;|\s)(width|height|max-width|max-height)\s*:\s*0(px)?\s*(;|$|!)/i, "zero size"],
+    [/(^|;|\s)(left|top|right|bottom|margin-left|margin-top)\s*:\s*-\s*\d{3,}/i, "positioned off-screen"],
+    [/text-indent\s*:\s*-\s*\d{3,}/i, "text pushed off-screen"],
+    [/clip\s*:\s*rect\(\s*0/i, "clipped away"],
+    [/clip-path\s*:\s*inset\(\s*(50%|100%)/i, "clipped away"],
+];
+/** Why this element (or an ancestor) is hidden from people, if it is. Only inline signals; stylesheets aren't evaluated. */
+export function hiddenReason(el) {
+    for (let node = el; node && node.tagName; node = node.parentNode) {
+        const tag = node.tagName.toLowerCase();
+        const where = node === el ? "" : "inside ";
+        const style = node.getAttribute?.("style");
+        if (style) {
+            const hit = HIDING_STYLES.find(([re]) => re.test(style));
+            if (hit)
+                return { reason: `${where ? "inside an element " : ""}hidden with ${hit[1]}`, strength: "strong" };
+        }
+        if (node.getAttribute?.("aria-hidden") === "true")
+            return { reason: `${where}aria-hidden="true"`, strength: "strong" };
+        if (tag === "svg" && (/^0(px)?$/.test(node.getAttribute("width") ?? "") || /^0(px)?$/.test(node.getAttribute("height") ?? ""))) {
+            return { reason: "inside a zero-size <svg>", strength: "strong" };
+        }
+        if (node.hasAttribute?.("hidden"))
+            return { reason: `inside a hidden <${tag}>`, strength: "soft" };
+        if (tag === "details" && node !== el && !node.hasAttribute("open") && !el.closest?.("summary")) {
+            return { reason: "inside a closed <details>", strength: "soft" };
+        }
+    }
+    return undefined;
+}
+/** A link with nothing a visitor can see or a screen reader can announce. */
+export function isEmptyLink(a) {
+    return !a.text.trim() && !a.getAttribute("aria-label") && !a.querySelector("img[alt]:not([alt=''])");
+}
+/** href of <a>/<area>, including SVG's xlink:href. */
+export function linkTarget(a) {
+    return a.getAttribute("href") ?? a.getAttribute("xlink:href") ?? undefined;
+}
+// ---------------------------------------------------------------- outbound links
+/** Normalized host of an external http(s) link ("www." dropped), or undefined for internal/other links. */
+export function externalHost(href, site) {
+    let url;
+    try {
+        url = new URL(href.trim(), site ?? "https://site.invalid");
+    }
+    catch {
+        return undefined;
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:")
+        return undefined;
+    const host = normalizeHost(url.hostname);
+    const own = site ? normalizeHost(new URL(site).hostname) : "site.invalid";
+    return host === own ? undefined : host;
+}
+/** Lowercase, punycode, no "www.", no trailing dot: one spelling per host. */
+export function normalizeHost(host) {
+    return host.toLowerCase().replace(/\.$/, "").replace(/^www\./, "");
+}
+/** Places in a page that send visitors or search engines elsewhere, beyond <a>: all go through the link lock. */
+const OTHER_TARGETS = [
+    ['link[rel="canonical"]', (el) => el.getAttribute("href")],
+    ['meta[property="og:url"]', (el) => el.getAttribute("content")],
+    ["meta[http-equiv]", (el) => (/^refresh$/i.test(el.getAttribute("http-equiv") ?? "") ? /url\s*=\s*['"]?([^'";]+)/i.exec(el.getAttribute("content") ?? "")?.[1] : undefined)],
+    ["iframe[src]", (el) => el.getAttribute("src")],
+    ["form[action]", (el) => el.getAttribute("action")],
+];
+/** Every external host the built site links to, with the pages that link there. */
+export function outboundHosts(model) {
+    const hosts = new Map();
+    const pages = model.notFound ? [...model.pages, model.notFound] : model.pages;
+    for (const page of pages) {
+        const seen = new Set();
+        const doc = parse(page.html);
+        const targets = [
+            ...doc.querySelectorAll("a, area").map(linkTarget),
+            ...OTHER_TARGETS.flatMap(([selector, read]) => doc.querySelectorAll(selector).map(read)),
+        ];
+        for (const target of targets) {
+            if (!target)
+                continue;
+            const host = externalHost(target, model.site.site);
+            if (!host || seen.has(host))
+                continue;
+            seen.add(host);
+            hosts.set(host, [...(hosts.get(host) ?? []), { route: page.route, source: page.source }]);
+        }
+    }
+    return new Map([...hosts].sort(([a], [b]) => a.localeCompare(b)));
+}
+export function readLinkLock(root) {
+    const file = path.join(root, LINK_LOCK);
+    if (!fs.existsSync(file))
+        return undefined;
+    let data;
+    try {
+        data = JSON.parse(fs.readFileSync(file, "utf8"));
+    }
+    catch (err) {
+        throw new VageError(`${LINK_LOCK} is not valid JSON: ${err.message}`, file);
+    }
+    const domains = data?.domains;
+    if (!Array.isArray(domains) || !domains.every((d) => typeof d === "string")) {
+        throw new VageError(`${LINK_LOCK} must look like { "domains": ["example.com"] }.`, file);
+    }
+    return { domains: domains.map(normalizeHost) };
+}
+export function writeLinkLock(root, domains) {
+    const sorted = [...new Set(domains)].sort();
+    const body = {
+        $comment: "External domains this site may link to. vage check fails on links to any other domain. Only the site owner approves new ones: npx vage links approve <domain>",
+        domains: sorted,
+    };
+    fs.writeFileSync(path.join(root, LINK_LOCK), `${JSON.stringify(body, null, 2)}\n`);
+}
+// ---------------------------------------------------------------- spam vocabulary
+/** Words that rarely appear on legitimate business sites but are staples of injected spam. Warnings only. */
+export const SPAM_PATTERNS = [
+    [/\b(online|best|free|top)\s+casinos?\b|\bcasino\s+(bonus|online|games?|sites?)\b/i, "casino"],
+    [/\b(viagra|cialis|levitra|kamagra)\b/i, "pharmacy spam"],
+    [/\bpayday\s+loans?\b/i, "payday loans"],
+    [/\breplica\s+(watches|handbags|bags|rolex)\b/i, "replica goods"],
+    [/\bescort\s+(girls|agency|agencies)\b/i, "escort ads"],
+    [/\bporn\b|\bxxx\s+(videos?|porn|sex)\b/i, "adult content"],
+    [/\bbuy\s+(cheap\s+)?(followers|backlinks|likes)\b/i, "bought followers or backlinks"],
+    [/\bonline\s+slots\b|\bslot\s+gacor\b|\bjudi\s+online\b/i, "online gambling"],
+];
+export function spamHits(text) {
+    return [...new Set(SPAM_PATTERNS.filter(([re]) => re.test(text)).map(([, label]) => label))];
+}

package/dist/checks/links.d.ts

@@ -0,0 +1,18 @@
+import { readLinkLock, writeLinkLock } from "./integrity.js";
+export interface LinkStatus {
+    /** External domains the site links to now, with how many pages link there, and whether they're approved. */
+    domains: {
+        domain: string;
+        pages: string[];
+        approved: boolean;
+    }[];
+    /** Approved domains the site no longer links to. */
+    unused: string[];
+    hasLock: boolean;
+}
+export declare function linkStatus(root: string): Promise<LinkStatus>;
+/** "Bücher.DE." -> "xn--bcher-kva.de"; throws for anything that isn't a bare host (or IP). */
+export declare function parseHost(input: string): string;
+/** The lock file's domains after approving `add` (and, with `prune`, dropping unused ones). */
+export declare function nextLock(status: LinkStatus, current: string[], add: string[], prune: boolean): string[];
+export { readLinkLock, writeLinkLock };

package/dist/checks/links.js

@@ -0,0 +1,47 @@
+import path from "node:path";
+import { VageError } from "../core/errors.js";
+import { createBuildLoader } from "../core/loader.js";
+import { renderSite } from "../core/site.js";
+import { normalizeHost, outboundHosts, readLinkLock, writeLinkLock } from "./integrity.js";
+export async function linkStatus(root) {
+    root = path.resolve(root);
+    const loader = await createBuildLoader(root);
+    try {
+        const model = await renderSite(root, loader.load);
+        const hosts = outboundHosts(model);
+        const lock = readLinkLock(root);
+        const approved = new Set(lock?.domains ?? []);
+        return {
+            domains: [...hosts].map(([domain, pages]) => ({ domain, pages: pages.map((p) => p.route), approved: approved.has(domain) })),
+            unused: [...approved].filter((d) => !hosts.has(d)).sort(),
+            hasLock: lock !== undefined,
+        };
+    }
+    finally {
+        await loader.close();
+    }
+}
+/** "Bücher.DE." -> "xn--bcher-kva.de"; throws for anything that isn't a bare host (or IP). */
+export function parseHost(input) {
+    const raw = input.trim();
+    let host;
+    try {
+        if (/[/\\?#@\s]/.test(raw) || raw.includes("://"))
+            throw new Error("not a host");
+        host = new URL(`http://${raw}`).hostname;
+    }
+    catch {
+        throw new VageError(`"${input}" is not a domain name. Use a host like example.com (no https://, no path).`);
+    }
+    host = normalizeHost(host);
+    if (!host || (!host.includes(".") && !host.startsWith("[")))
+        throw new VageError(`"${input}" is not a domain name. Use a host like example.com.`);
+    return host;
+}
+/** The lock file's domains after approving `add` (and, with `prune`, dropping unused ones). */
+export function nextLock(status, current, add, prune) {
+    add = add.map(parseHost);
+    const keep = prune ? current.filter((d) => !status.unused.includes(d)) : current;
+    return [...new Set([...keep, ...add])].sort();
+}
+export { readLinkLock, writeLinkLock };

package/dist/checks/verify.d.ts

@@ -0,0 +1,42 @@
+/**
+ * `vage verify`: compares the live site with a fresh local render of the project. Anything
+ * on the live site that isn't in your files (extra links, extra scripts, changed text, extra
+ * pages in sitemap.xml) means the site was changed outside your repository: a hacked host
+ * account, a compromised deploy, or a CDN injecting content.
+ */
+export interface PageFingerprint {
+    /** Where the page sends people and crawlers: links, canonical/og:url, meta refresh, iframes, embeds, forms. */
+    links: string[];
+    /** Scripts (src or hash), inline event handlers (hash) and JSON-LD blocks (hash). */
+    scripts: string[];
+    /** Scripts a CDN injected (Cloudflare email protection, Web Analytics): not content, but they can break your security policy. */
+    cdnScripts: string[];
+    text: string;
+    /** Spam vocabulary in the visible text (read with spaces intact). */
+    spam: string[];
+}
+export interface Difference {
+    route: string;
+    severity: "error" | "warning";
+    kind: "unreachable" | "links-added" | "links-removed" | "scripts-added" | "text-changed" | "sitemap-extra" | "robots-changed" | "cdn-script" | "crawler-blocked" | "robots-blocked" | "noindex-header" | "redirect-missing" | "duplicate-host";
+    detail: string;
+}
+export interface VerifyResult {
+    ok: boolean;
+    url: string;
+    pagesChecked: number;
+    /** Crawlers probed by the crawlability check (0 when skipped or when the site blocks everyone). */
+    crawlersChecked: number;
+    differences: Difference[];
+}
+/** Every link-like target in a text file (Markdown, RSS, llms.txt), normalized for comparison. */
+export declare function textLinks(text: string): Set<string>;
+/** What matters for injected spam: where the page sends people, what runs on it, and its visible text, independent of formatting. */
+export declare function fingerprint(html: string, pageUrl: string): PageFingerprint;
+export declare function compare(route: string, local: PageFingerprint, live: PageFingerprint): Difference[];
+export declare function verifySite(root: string, options?: {
+    url?: string;
+    fetch?: typeof fetch;
+    concurrency?: number;
+    crawlers?: boolean;
+}): Promise<VerifyResult>;

package/dist/checks/verify.js

@@ -0,0 +1,264 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { parse } from "node-html-parser";
+import { VageError } from "../core/errors.js";
+import { createBuildLoader } from "../core/loader.js";
+import { renderSite } from "../core/site.js";
+import { spamHits } from "./integrity.js";
+const GOOGLEBOT = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 16);
+/**
+ * Scripts Cloudflare adds by itself, matched by their exact shape so an injected script
+ * that merely mentions /cdn-cgi/ is still reported as an added script.
+ */
+const BEACON = /^https:\/\/static\.cloudflareinsights\.com\/beacon\.min\.js(\/[\w.]*)?$/;
+const CDN_PATH = /^\/cdn-cgi\/(scripts\/[0-9a-f]+\/cloudflare-static\/email-decode\.min\.js|challenge-platform\/(h\/[a-z]\/)?scripts\/jsd\/main\.js)$/;
+/** Cloudflare's scripts: the Web Analytics beacon, or a /cdn-cgi/ script on the site's own host (never another site's). */
+const isCdnScript = (url, pageHost) => {
+    if (BEACON.test(url))
+        return true;
+    try {
+        const u = new URL(url);
+        return u.host === pageHost && !u.search && CDN_PATH.test(u.pathname);
+    }
+    catch {
+        return false;
+    }
+};
+/** Cloudflare's JS-detection loader: sets window.__CF$cv$params and loads /cdn-cgi/challenge-platform/.../main.js, nothing else. */
+const CDN_INLINE = /^\s*\(function\(\)\{function c\(\)\{var b=a\.contentDocument\|\|a\.contentWindow\.document;if\(b\)\{var d=b\.createElement\('script'\);d\.innerHTML="window\.__CF\$cv\$params=\{r:'[0-9a-f]+',t:'[A-Za-z0-9+\/=]+'\};var a=document\.createElement\('script'\);a\.src='\/cdn-cgi\/challenge-platform\/scripts\/jsd\/main\.js';document\.getElementsByTagName\('head'\)\[0\]\.appendChild\(a\);";b\.getElementsByTagName\('head'\)\[0\]\.appendChild\(d\)\}\}if\(document\.body\)\{var a=document\.createElement\('iframe'\);a\.height=1;a\.width=1;a\.style\.position='absolute';a\.style\.top=0;a\.style\.left=0;a\.style\.border='none';a\.style\.visibility='hidden';document\.body\.appendChild\(a\);if\('loading'!==document\.readyState\)c\(\);else if\(window\.addEventListener\)document\.addEventListener\('DOMContentLoaded',c\);else\{var e=document\.onreadystatechange\|\|function\(\)\{\};document\.onreadystatechange=function\(b\)\{e\(b\);'loading'!==document\.readyState&&\(document\.onreadystatechange=e,c\(\)\)\}\}\}\}\)\(\);\s*$/;
+/** Cloudflare email protection rewrites mailto: links to this (on the site's own host); not a content change. */
+const isCdnLink = (url, pageHost) => {
+    try {
+        const u = new URL(url);
+        return u.host === pageHost && u.pathname === "/cdn-cgi/l/email-protection" && !u.search && /^(#[0-9a-f]*)?$/.test(u.hash);
+    }
+    catch {
+        return false;
+    }
+};
+/** Every link-like target in a text file (Markdown, RSS, llms.txt), normalized for comparison. */
+export function textLinks(text) {
+    const found = new Set();
+    const add = (raw) => found.add(raw.trim().replace(/[.,;:!?]+$/, "").toLowerCase());
+    for (const m of text.matchAll(/\]\(\s*<?([^)\s>]+)/g))
+        add(m[1]); // [text](target)
+    for (const m of text.matchAll(/^\s{0,3}\[[^\]]+\]:\s*<?([^\s>]+)/gm))
+        add(m[1]); // [ref]: target
+    for (const m of text.matchAll(/<((?:[a-z][a-z0-9+.-]*:|\/\/)[^>\s]+)>/gi))
+        add(m[1]); // <autolink>
+    for (const m of text.matchAll(/(?:\b[a-z][a-z0-9+.-]*:)?\/\/[^\s)<>"'\]]+|\bwww\.[^\s)<>"'\]]+/gi))
+        add(m[0]); // bare URLs, //host, www.
+    for (const m of text.matchAll(/<link>([^<]+)<\/link>|<guid[^>]*>([^<]+)<\/guid>/gi))
+        add((m[1] ?? m[2])); // RSS
+    return found;
+}
+const TARGETS = [
+    ["a, area", "", (el) => el.getAttribute("href") ?? el.getAttribute("xlink:href")],
+    ["link[href]", "link:", (el) => el.getAttribute("href")],
+    ['meta[property="og:url"]', "og:url:", (el) => el.getAttribute("content")],
+    ["meta[http-equiv]", "refresh:", (el) => (/^refresh$/i.test(el.getAttribute("http-equiv") ?? "") ? /url\s*=\s*['"]?([^'";]+)/i.exec(el.getAttribute("content") ?? "")?.[1] : undefined)],
+    ["iframe[src], embed[src], frame[src]", "frame:", (el) => el.getAttribute("src")],
+    ["object[data]", "object:", (el) => el.getAttribute("data")],
+    ["form[action]", "form:", (el) => el.getAttribute("action")],
+];
+/** What matters for injected spam: where the page sends people, what runs on it, and its visible text, independent of formatting. */
+export function fingerprint(html, pageUrl) {
+    const doc = parse(html);
+    const pageHost = new URL(pageUrl).host;
+    const abs = (href) => {
+        try {
+            return new URL(href.trim(), pageUrl).href;
+        }
+        catch {
+            return href.trim();
+        }
+    };
+    const links = new Set();
+    for (const [selector, prefix, read] of TARGETS) {
+        for (const el of doc.querySelectorAll(selector)) {
+            const value = read(el);
+            if (!value || value.trim().startsWith("mailto:"))
+                continue;
+            const url = abs(value);
+            if (isCdnLink(url, pageHost))
+                continue;
+            links.add(prefix + url);
+        }
+    }
+    const scripts = new Set();
+    const cdnScripts = new Set();
+    for (const el of doc.querySelectorAll("script")) {
+        const src = el.getAttribute("src");
+        if (src) {
+            const url = abs(src);
+            if (isCdnScript(url, pageHost))
+                cdnScripts.add(url.replace(/\?.*$/, ""));
+            else
+                scripts.add(`src:${url}`);
+        }
+        else if (el.getAttribute("type") === "application/ld+json")
+            scripts.add(`json-ld:${sha(el.text.replace(/\s+/g, ""))}`);
+        else if (CDN_INLINE.test(el.text))
+            cdnScripts.add("Cloudflare's inline bot-detection script (/cdn-cgi/challenge-platform/)");
+        else
+            scripts.add(`inline:${sha(el.text.trim())}`);
+    }
+    for (const el of doc.querySelectorAll("*")) {
+        for (const [name, value] of Object.entries(el.attributes)) {
+            if (/^on/i.test(name))
+                scripts.add(`handler:${el.tagName.toLowerCase()}.${name.toLowerCase()}:${sha(value)}`);
+        }
+    }
+    for (const el of doc.querySelectorAll("script, style, noscript, template"))
+        el.remove();
+    // Whitespace between elements depends on formatting, not content, so it's ignored entirely.
+    const visible = (doc.querySelector("body") ?? doc).text;
+    return { links: [...links].sort(), scripts: [...scripts].sort(), cdnScripts: [...cdnScripts].sort(), text: visible.replace(/\s+/g, ""), spam: spamHits(visible.replace(/\s+/g, " ")) };
+}
+export function compare(route, local, live) {
+    const out = [];
+    const added = live.links.filter((l) => !local.links.includes(l));
+    const removed = local.links.filter((l) => !live.links.includes(l));
+    const scripts = live.scripts.filter((s) => !local.scripts.includes(s));
+    if (added.length)
+        out.push({ route, severity: "error", kind: "links-added", detail: `Links on the live page that aren't in your files: ${added.slice(0, 5).join(", ")}${added.length > 5 ? ` (+${added.length - 5})` : ""}` });
+    if (scripts.length)
+        out.push({ route, severity: "error", kind: "scripts-added", detail: `Scripts on the live page that aren't in your files: ${scripts.slice(0, 5).join(", ")}` });
+    if (removed.length)
+        out.push({ route, severity: "warning", kind: "links-removed", detail: `Links missing from the live page: ${removed.slice(0, 5).join(", ")}${removed.length > 5 ? ` (+${removed.length - 5})` : ""}` });
+    if (local.text !== live.text) {
+        const spam = live.spam.filter((h) => !local.spam.includes(h));
+        out.push(spam.length
+            ? { route, severity: "error", kind: "text-changed", detail: `The live page's text differs from your files and contains spam words (${spam.join(", ")}).` }
+            : { route, severity: "warning", kind: "text-changed", detail: "The live page's text differs from your files (an old deploy, or content changed outside the repository)." });
+    }
+    return out;
+}
+function sitemapUrls(xml) {
+    return [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => m[1].replace(/&amp;/g, "&"));
+}
+/** Browser-like request headers: some CDNs only inject scripts for browsers, and those are what visitors get. */
+const BROWSER = "Mozilla/5.0 (compatible; vagejs-verify) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+export async function verifySite(root, options = {}) {
+    root = path.resolve(root);
+    const doFetch = options.fetch ?? fetch;
+    const loader = await createBuildLoader(root);
+    try {
+        const model = await renderSite(root, loader.load);
+        const origin = (options.url ?? model.site.site)?.replace(/\/+$/, "");
+        if (!origin || !/^https?:\/\/[^/]+$/.test(origin))
+            throw new VageError('Pass the live site\'s origin with --url https://example.com, or set "site" in vage.config.ts.');
+        const get = async (url, userAgent = BROWSER) => {
+            try {
+                const res = await doFetch(url, { redirect: "follow", headers: { "user-agent": userAgent, accept: "text/html,application/xhtml+xml,*/*;q=0.8" }, signal: AbortSignal.timeout(20_000) });
+                return { status: res.status, body: res.ok ? await res.text() : "" };
+            }
+            catch (err) {
+                return { status: 0, body: "", error: err.message };
+            }
+        };
+        const differences = [];
+        const cdnSeen = new Set();
+        const queue = [...model.pages];
+        const worker = async () => {
+            for (let page = queue.shift(); page; page = queue.shift()) {
+                const url = `${origin}${page.route}`;
+                const live = await get(url);
+                if (live.status !== 200) {
+                    differences.push({ route: page.route, severity: "error", kind: "unreachable", detail: live.status ? `HTTP ${live.status}` : "no response" });
+                    continue;
+                }
+                const local = fingerprint(page.html, url);
+                const livePrint = fingerprint(live.body, url);
+                const seen = compare(page.route, local, livePrint);
+                differences.push(...seen);
+                for (const src of livePrint.cdnScripts)
+                    cdnSeen.add(src);
+                // Hacked sites often show spam only to search engines ("cloaking"): look again as Googlebot.
+                // Skipped when Googlebot gets a challenge page (the crawlability check reports that once).
+                const bot = await get(url, GOOGLEBOT);
+                if (bot.status === 200 && !/<title>\s*(Just a moment|Attention Required|Access denied|Checking your browser)/i.test(bot.body)) {
+                    for (const d of compare(page.route, local, fingerprint(bot.body, url))) {
+                        if (!seen.some((x) => x.kind === d.kind && x.detail === d.detail))
+                            differences.push({ ...d, detail: `Only when visited as Googlebot: ${d.detail}` });
+                    }
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.max(1, options.concurrency ?? 4) }, worker));
+        // A planted Sitemap: line or new rules in robots.txt point crawlers at spam.
+        const localRobots = model.files.get("robots.txt");
+        if (localRobots) {
+            const live = await get(`${origin}/robots.txt`);
+            const lines = (t) => t.split(/\r?\n/).map((l) => l.replace(/#.*/, "").trim()).filter(Boolean).join("\n");
+            if (live.status === 200 && lines(live.body) !== lines(localRobots)) {
+                differences.push({ route: "/robots.txt", severity: "error", kind: "robots-changed", detail: "The live robots.txt differs from the one your build writes." });
+            }
+        }
+        // Pages someone added on the server announce themselves in the live sitemap.
+        const localSitemap = model.files.get("sitemap.xml");
+        if (localSitemap) {
+            const live = await get(`${origin}/sitemap.xml`);
+            if (live.status === 200) {
+                const known = new Set(sitemapUrls(localSitemap).map((u) => new URL(u).pathname));
+                const extra = sitemapUrls(live.body).filter((u) => {
+                    try {
+                        return !known.has(new URL(u).pathname);
+                    }
+                    catch {
+                        return true;
+                    }
+                });
+                if (extra.length)
+                    differences.push({ route: "/sitemap.xml", severity: "error", kind: "sitemap-extra", detail: `The live sitemap lists pages that aren't in your files: ${extra.slice(0, 5).join(", ")}${extra.length > 5 ? ` (+${extra.length - 5})` : ""}` });
+            }
+        }
+        for (const src of cdnSeen) {
+            differences.push({
+                route: "/",
+                severity: "warning",
+                kind: "cdn-script",
+                detail: `Your host or CDN adds ${src} to pages. It isn't in your files, and your security policy may block it (browser console errors). Turn the feature off at your host, or allow it in security.cspSources.`,
+            });
+        }
+        // Generated files must match too: the feed, Markdown copies and optimized images.
+        const generated = [...model.files.keys()].filter((name) => name.endsWith("feed.xml") || name === "llms.txt" || name === "llms-full.txt" || (name.endsWith(".md") && !name.includes("/_")));
+        for (const name of generated) {
+            const live = await get(`${origin}/${name}`);
+            const local = model.files.get(name);
+            const norm = (t) => t.replace(/\s+/g, "");
+            if (live.status !== 200)
+                differences.push({ route: `/${name}`, severity: "error", kind: "unreachable", detail: live.status ? `HTTP ${live.status}` : "no response" });
+            else if (norm(live.body) !== norm(local)) {
+                // Any URL in the live file that isn't in the build is treated like an added link.
+                const known = textLinks(local);
+                const added = [...textLinks(live.body)].filter((u) => !known.has(u));
+                const spam = spamHits(live.body).filter((h) => !spamHits(local).includes(h));
+                if (added.length)
+                    differences.push({ route: `/${name}`, severity: "error", kind: "links-added", detail: `Links in the live file that aren't in your build: ${added.slice(0, 5).join(", ")}` });
+                // Generated files are the same on every build, so any difference means the live file was changed
+                // (or is out of date): an error, however the change is written.
+                differences.push({ route: `/${name}`, severity: "error", kind: "text-changed", detail: `The live file differs from your build${spam.length ? ` and contains spam words (${spam.join(", ")})` : ""}. Redeploy, and if you didn't change it, check who can deploy your site.` });
+            }
+        }
+        const images = new Set(model.pages.flatMap((p) => [...p.assets.keys()]));
+        for (const image of images) {
+            const live = await get(`${origin}/${image}`);
+            if (live.status !== 200)
+                differences.push({ route: `/${image}`, severity: "error", kind: "unreachable", detail: `Optimized image missing on the live site (HTTP ${live.status || "no response"}); browsers using srcset would show a broken image.` });
+        }
+        let crawlersChecked = 0;
+        if (options.crawlers !== false) {
+            const { checkCrawlability } = await import("./crawlability.js");
+            const crawl = await checkCrawlability(origin, { fetch: doFetch });
+            differences.push(...crawl.differences);
+            crawlersChecked = crawl.crawlersChecked;
+        }
+        differences.sort((a, b) => (a.severity === b.severity ? a.route.localeCompare(b.route) : a.severity === "error" ? -1 : 1));
+        return { ok: !differences.some((d) => d.severity === "error"), url: origin, pagesChecked: model.pages.length, crawlersChecked, differences };
+    }
+    finally {
+        await loader.close();
+    }
+}