vagejs 0.0.0-stage → 0.2.0

package/dist/checks/links.js

@@ -0,0 +1,47 @@
+import path from "node:path";
+import { VageError } from "../core/errors.js";
+import { createBuildLoader } from "../core/loader.js";
+import { renderSite } from "../core/site.js";
+import { normalizeHost, outboundHosts, readLinkLock, writeLinkLock } from "./integrity.js";
+export async function linkStatus(root) {
+    root = path.resolve(root);
+    const loader = await createBuildLoader(root);
+    try {
+        const model = await renderSite(root, loader.load);
+        const hosts = outboundHosts(model);
+        const lock = readLinkLock(root);
+        const approved = new Set(lock?.domains ?? []);
+        return {
+            domains: [...hosts].map(([domain, pages]) => ({ domain, pages: pages.map((p) => p.route), approved: approved.has(domain) })),
+            unused: [...approved].filter((d) => !hosts.has(d)).sort(),
+            hasLock: lock !== undefined,
+        };
+    }
+    finally {
+        await loader.close();
+    }
+}
+/** "Bücher.DE." -> "xn--bcher-kva.de"; throws for anything that isn't a bare host (or IP). */
+export function parseHost(input) {
+    const raw = input.trim();
+    let host;
+    try {
+        if (/[/\\?#@\s]/.test(raw) || raw.includes("://"))
+            throw new Error("not a host");
+        host = new URL(`http://${raw}`).hostname;
+    }
+    catch {
+        throw new VageError(`"${input}" is not a domain name. Use a host like example.com (no https://, no path).`);
+    }
+    host = normalizeHost(host);
+    if (!host || (!host.includes(".") && !host.startsWith("[")))
+        throw new VageError(`"${input}" is not a domain name. Use a host like example.com.`);
+    return host;
+}
+/** The lock file's domains after approving `add` (and, with `prune`, dropping unused ones). */
+export function nextLock(status, current, add, prune) {
+    add = add.map(parseHost);
+    const keep = prune ? current.filter((d) => !status.unused.includes(d)) : current;
+    return [...new Set([...keep, ...add])].sort();
+}
+export { readLinkLock, writeLinkLock };

package/dist/checks/verify.d.ts

@@ -0,0 +1,35 @@
+/**
+ * `vage verify`: compares the live site with a fresh local render of the project. Anything
+ * on the live site that isn't in your files (extra links, extra scripts, changed text, extra
+ * pages in sitemap.xml) means the site was changed outside your repository: a hacked host
+ * account, a compromised deploy, or a CDN injecting content.
+ */
+export interface PageFingerprint {
+    /** Where the page sends people and crawlers: links, canonical/og:url, meta refresh, iframes, embeds, forms. */
+    links: string[];
+    /** Scripts (src or hash), inline event handlers (hash) and JSON-LD blocks (hash). */
+    scripts: string[];
+    text: string;
+    /** Spam vocabulary in the visible text (read with spaces intact). */
+    spam: string[];
+}
+export interface Difference {
+    route: string;
+    severity: "error" | "warning";
+    kind: "unreachable" | "links-added" | "links-removed" | "scripts-added" | "text-changed" | "sitemap-extra" | "robots-changed";
+    detail: string;
+}
+export interface VerifyResult {
+    ok: boolean;
+    url: string;
+    pagesChecked: number;
+    differences: Difference[];
+}
+/** What matters for injected spam: where the page sends people, what runs on it, and its visible text, independent of formatting. */
+export declare function fingerprint(html: string, pageUrl: string): PageFingerprint;
+export declare function compare(route: string, local: PageFingerprint, live: PageFingerprint): Difference[];
+export declare function verifySite(root: string, options?: {
+    url?: string;
+    fetch?: typeof fetch;
+    concurrency?: number;
+}): Promise<VerifyResult>;

package/dist/checks/verify.js

@@ -0,0 +1,167 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { parse } from "node-html-parser";
+import { VageError } from "../core/errors.js";
+import { createBuildLoader } from "../core/loader.js";
+import { renderSite } from "../core/site.js";
+import { spamHits } from "./integrity.js";
+const GOOGLEBOT = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
+const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 16);
+/** Scripts a CDN adds by itself (Cloudflare email protection, Web Analytics), not content changes. */
+const CDN_SCRIPT = /\/cdn-cgi\/|static\.cloudflareinsights\.com/;
+const TARGETS = [
+    ["a, area", "", (el) => el.getAttribute("href") ?? el.getAttribute("xlink:href")],
+    ["link[href]", "link:", (el) => el.getAttribute("href")],
+    ['meta[property="og:url"]', "og:url:", (el) => el.getAttribute("content")],
+    ["meta[http-equiv]", "refresh:", (el) => (/^refresh$/i.test(el.getAttribute("http-equiv") ?? "") ? /url\s*=\s*['"]?([^'";]+)/i.exec(el.getAttribute("content") ?? "")?.[1] : undefined)],
+    ["iframe[src], embed[src], frame[src]", "frame:", (el) => el.getAttribute("src")],
+    ["object[data]", "object:", (el) => el.getAttribute("data")],
+    ["form[action]", "form:", (el) => el.getAttribute("action")],
+];
+/** What matters for injected spam: where the page sends people, what runs on it, and its visible text, independent of formatting. */
+export function fingerprint(html, pageUrl) {
+    const doc = parse(html);
+    const abs = (href) => {
+        try {
+            return new URL(href.trim(), pageUrl).href;
+        }
+        catch {
+            return href.trim();
+        }
+    };
+    const links = new Set();
+    for (const [selector, prefix, read] of TARGETS) {
+        for (const el of doc.querySelectorAll(selector)) {
+            const value = read(el);
+            if (!value || value.trim().startsWith("mailto:"))
+                continue;
+            const url = abs(value);
+            // Cloudflare's email protection rewrites mailto: links; not a content change.
+            if (/\/cdn-cgi\//.test(url))
+                continue;
+            links.add(prefix + url);
+        }
+    }
+    const scripts = new Set();
+    for (const el of doc.querySelectorAll("script")) {
+        const src = el.getAttribute("src");
+        if (src) {
+            const url = abs(src);
+            if (!CDN_SCRIPT.test(url))
+                scripts.add(`src:${url}`);
+        }
+        else if (el.getAttribute("type") === "application/ld+json")
+            scripts.add(`json-ld:${sha(el.text.replace(/\s+/g, ""))}`);
+        else
+            scripts.add(`inline:${sha(el.text.trim())}`);
+    }
+    for (const el of doc.querySelectorAll("*")) {
+        for (const [name, value] of Object.entries(el.attributes)) {
+            if (/^on/i.test(name))
+                scripts.add(`handler:${el.tagName.toLowerCase()}.${name.toLowerCase()}:${sha(value)}`);
+        }
+    }
+    for (const el of doc.querySelectorAll("script, style, noscript, template"))
+        el.remove();
+    // Whitespace between elements depends on formatting, not content, so it's ignored entirely.
+    const visible = (doc.querySelector("body") ?? doc).text;
+    return { links: [...links].sort(), scripts: [...scripts].sort(), text: visible.replace(/\s+/g, ""), spam: spamHits(visible.replace(/\s+/g, " ")) };
+}
+export function compare(route, local, live) {
+    const out = [];
+    const added = live.links.filter((l) => !local.links.includes(l));
+    const removed = local.links.filter((l) => !live.links.includes(l));
+    const scripts = live.scripts.filter((s) => !local.scripts.includes(s));
+    if (added.length)
+        out.push({ route, severity: "error", kind: "links-added", detail: `Links on the live page that aren't in your files: ${added.slice(0, 5).join(", ")}${added.length > 5 ? ` (+${added.length - 5})` : ""}` });
+    if (scripts.length)
+        out.push({ route, severity: "error", kind: "scripts-added", detail: `Scripts on the live page that aren't in your files: ${scripts.slice(0, 5).join(", ")}` });
+    if (removed.length)
+        out.push({ route, severity: "warning", kind: "links-removed", detail: `Links missing from the live page: ${removed.slice(0, 5).join(", ")}${removed.length > 5 ? ` (+${removed.length - 5})` : ""}` });
+    if (local.text !== live.text) {
+        const spam = live.spam.filter((h) => !local.spam.includes(h));
+        out.push(spam.length
+            ? { route, severity: "error", kind: "text-changed", detail: `The live page's text differs from your files and contains spam words (${spam.join(", ")}).` }
+            : { route, severity: "warning", kind: "text-changed", detail: "The live page's text differs from your files (an old deploy, or content changed outside the repository)." });
+    }
+    return out;
+}
+function sitemapUrls(xml) {
+    return [...xml.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/gi)].map((m) => m[1].replace(/&amp;/g, "&"));
+}
+export async function verifySite(root, options = {}) {
+    root = path.resolve(root);
+    const doFetch = options.fetch ?? fetch;
+    const loader = await createBuildLoader(root);
+    try {
+        const model = await renderSite(root, loader.load);
+        const origin = (options.url ?? model.site.site)?.replace(/\/+$/, "");
+        if (!origin || !/^https?:\/\/[^/]+$/.test(origin))
+            throw new VageError('Pass the live site\'s origin with --url https://example.com, or set "site" in vage.config.ts.');
+        const get = async (url, userAgent = "vagejs-verify") => {
+            try {
+                const res = await doFetch(url, { redirect: "follow", headers: { "user-agent": userAgent }, signal: AbortSignal.timeout(20_000) });
+                return { status: res.status, body: res.ok ? await res.text() : "" };
+            }
+            catch (err) {
+                return { status: 0, body: "", error: err.message };
+            }
+        };
+        const differences = [];
+        const queue = [...model.pages];
+        const worker = async () => {
+            for (let page = queue.shift(); page; page = queue.shift()) {
+                const url = `${origin}${page.route}`;
+                const live = await get(url);
+                if (live.status !== 200) {
+                    differences.push({ route: page.route, severity: "error", kind: "unreachable", detail: live.status ? `HTTP ${live.status}` : "no response" });
+                    continue;
+                }
+                const local = fingerprint(page.html, url);
+                const seen = compare(page.route, local, fingerprint(live.body, url));
+                differences.push(...seen);
+                // Hacked sites often show spam only to search engines ("cloaking"): look again as Googlebot.
+                const bot = await get(url, GOOGLEBOT);
+                if (bot.status === 200) {
+                    for (const d of compare(page.route, local, fingerprint(bot.body, url))) {
+                        if (!seen.some((x) => x.kind === d.kind && x.detail === d.detail))
+                            differences.push({ ...d, detail: `Only when visited as Googlebot: ${d.detail}` });
+                    }
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.max(1, options.concurrency ?? 4) }, worker));
+        // A planted Sitemap: line or new rules in robots.txt point crawlers at spam.
+        const localRobots = model.files.get("robots.txt");
+        if (localRobots) {
+            const live = await get(`${origin}/robots.txt`);
+            const lines = (t) => t.split(/\r?\n/).map((l) => l.replace(/#.*/, "").trim()).filter(Boolean).join("\n");
+            if (live.status === 200 && lines(live.body) !== lines(localRobots)) {
+                differences.push({ route: "/robots.txt", severity: "error", kind: "robots-changed", detail: "The live robots.txt differs from the one your build writes." });
+            }
+        }
+        // Pages someone added on the server announce themselves in the live sitemap.
+        const localSitemap = model.files.get("sitemap.xml");
+        if (localSitemap) {
+            const live = await get(`${origin}/sitemap.xml`);
+            if (live.status === 200) {
+                const known = new Set(sitemapUrls(localSitemap).map((u) => new URL(u).pathname));
+                const extra = sitemapUrls(live.body).filter((u) => {
+                    try {
+                        return !known.has(new URL(u).pathname);
+                    }
+                    catch {
+                        return true;
+                    }
+                });
+                if (extra.length)
+                    differences.push({ route: "/sitemap.xml", severity: "error", kind: "sitemap-extra", detail: `The live sitemap lists pages that aren't in your files: ${extra.slice(0, 5).join(", ")}${extra.length > 5 ? ` (+${extra.length - 5})` : ""}` });
+            }
+        }
+        differences.sort((a, b) => (a.severity === b.severity ? a.route.localeCompare(b.route) : a.severity === "error" ? -1 : 1));
+        return { ok: !differences.some((d) => d.severity === "error"), url: origin, pagesChecked: model.pages.length, differences };
+    }
+    finally {
+        await loader.close();
+    }
+}

package/dist/cli/create.d.ts

@@ -0,0 +1,10 @@
+export interface CreateResult {
+    dir: string;
+    files: string[];
+}
+/**
+ * Copies the starter template into `dir`. Refuses to write into a non-empty
+ * directory. `vageSpec` is the dependency spec written to package.json
+ * (a version range, or e.g. "file:../vagejs-0.1.0.tgz" while unpublished).
+ */
+export declare function createProject(dir: string, vageSpec: string): CreateResult;

package/dist/cli/create.js

@@ -0,0 +1,39 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { VageError } from "../core/errors.js";
+const TEMPLATE_DIR = fileURLToPath(new URL("../../templates/starter", import.meta.url));
+/** npm strips .gitignore from published packages, so templates store it under this name. */
+const RENAMES = { _gitignore: ".gitignore" };
+/**
+ * Copies the starter template into `dir`. Refuses to write into a non-empty
+ * directory. `vageSpec` is the dependency spec written to package.json
+ * (a version range, or e.g. "file:../vagejs-0.1.0.tgz" while unpublished).
+ */
+export function createProject(dir, vageSpec) {
+    dir = path.resolve(dir);
+    if (fs.existsSync(dir) && fs.readdirSync(dir).length > 0) {
+        throw new VageError(`${dir} already exists and is not empty. Choose a new directory name.`);
+    }
+    fs.mkdirSync(dir, { recursive: true });
+    fs.cpSync(TEMPLATE_DIR, dir, { recursive: true });
+    for (const [from, to] of Object.entries(RENAMES)) {
+        if (fs.existsSync(path.join(dir, from)))
+            fs.renameSync(path.join(dir, from), path.join(dir, to));
+    }
+    const pkgPath = path.join(dir, "package.json");
+    const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
+    pkg.name = toPackageName(path.basename(dir));
+    pkg.devDependencies.vagejs = vageSpec;
+    fs.writeFileSync(pkgPath, `${JSON.stringify(pkg, null, 2)}\n`);
+    return { dir, files: listFiles(dir) };
+}
+function toPackageName(name) {
+    return name.toLowerCase().replace(/[^a-z0-9-~._]+/g, "-").replace(/^[-._]+/, "") || "my-website";
+}
+function listFiles(dir, prefix = "") {
+    return fs
+        .readdirSync(dir, { withFileTypes: true })
+        .flatMap((e) => (e.isDirectory() ? listFiles(path.join(dir, e.name), `${prefix}${e.name}/`) : [`${prefix}${e.name}`]))
+        .sort();
+}

package/dist/cli/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};