vagejs 0.2.0 → 0.3.0

package/dist/core/site.js

@@ -1,6 +1,9 @@
 import fs from "node:fs";
 import path from "node:path";
 import { llmsFullTxt, llmsTxt } from "../discovery/llms.js";
+import { feedPath, rssFeed } from "../discovery/feed.js";
+import { gitPageDates } from "../discovery/git-dates.js";
+import { htmlToMarkdown, markdownCopy, markdownCopyPath } from "../discovery/markdown-copy.js";
 import { robotsTxt } from "../discovery/robots.js";
 import { SITEMAP_URL_LIMIT, sitemapXml } from "../discovery/sitemap.js";
 import { headersFile } from "../deploy/headers.js";
@@ -13,6 +16,7 @@ import { VageError } from "./errors.js";
 import { escapeHtml } from "./html.js";
 import { renderPage } from "./render.js";
 import { parseFrontmatter } from "../content/markdown.js";
+import { sharpResize } from "../content/optimize.js";
 import { collectPages } from "./pages.js";
 import { discoverRoutes, findNotFoundPage, normalizeRequestPath } from "./routes.js";
 /** Renders every page and derives discovery files from the same data, without writing anything. */
@@ -42,6 +46,8 @@ export async function renderSite(root, load, options = {}) {
         }
     };
     const pages = await collectPages(root, site, load, { skipInvalid: collect });
+    // Loaded once for the whole site, so a missing sharp is one clear error, not one per page.
+    const resize = site.images.optimize ? await sharpResize(root) : undefined;
     for (const entry of discoverRoutes(pagesDir)) {
         const source = path.relative(root, entry.file);
         await attempt(entry.file, entry.route, async () => {
@@ -51,30 +57,38 @@ export async function renderSite(root, load, options = {}) {
                 model.draftRoutes.push(entry.route);
                 return;
             }
-            const page = await renderPage({ root, file: entry.file, route: entry.route, site, load, analytics: true, pages });
+            const page = await renderPage({ root, file: entry.file, route: entry.route, site, load, analytics: true, pages, resize });
             model.pages.push({ ...page, route: entry.route, source, outFile: entry.outFile });
         });
     }
     const notFound = findNotFoundPage(pagesDir);
     if (notFound) {
         await attempt(notFound, undefined, async () => {
-            const page = await renderPage({ root, file: notFound, route: "/404/", site, load, notFound: true, analytics: true, pages });
+            const page = await renderPage({ root, file: notFound, route: "/404/", site, load, notFound: true, analytics: true, pages, resize });
             model.notFound = { ...page, route: "/404/", source: path.relative(root, notFound), outFile: "404.html" };
         });
     }
     checkPublicDir(root, model.warnings);
+    // Markdown versions of pages, converted from their sanitized HTML (copies and llms-full.txt).
+    const markdown = new Map();
+    if (site.seo.markdownCopies || (site.seo.llmsTxt && site.seo.llmsTxt.full)) {
+        for (const p of listedPages(model))
+            markdown.set(p.route, await htmlToMarkdown(p.content, { route: p.route, origin: site.site }));
+    }
     model.redirects = buildRedirects(site, new Set(model.pages.map((p) => p.route)));
-    addDiscoveryFiles(root, model);
+    addDiscoveryFiles(root, model, markdown);
     return model;
 }
 /** Pages that should appear in sitemap.xml and llms.txt: indexable and canonical to themselves. */
 export function listedPages(model) {
     return model.pages.filter((p) => !p.meta.noindex && p.meta.canonical === absoluteUrl(model.site.site, p.route));
 }
-function addDiscoveryFiles(root, model) {
+function addDiscoveryFiles(root, model, markdown) {
     const { site, files, warnings } = model;
     const listed = listedPages(model);
     const generate = (name, content) => {
+        if (files.has(name))
+            throw new VageError(`Two generated files would be written to ${name}. Rename one of the pages behind it.`);
         if (fs.existsSync(path.join(root, "public", name))) {
             throw new VageError(`public/${name} conflicts with the generated ${name}. Delete it, or turn generation off in vage.config.ts (seo).`);
         }
@@ -94,7 +108,28 @@ function addDiscoveryFiles(root, model) {
         if (listed.length > SITEMAP_URL_LIMIT) {
             throw new VageError(`sitemap.xml would list ${listed.length} URLs; sitemap indexes (over ${SITEMAP_URL_LIMIT}) are not supported yet.`);
         }
-        generate("sitemap.xml", () => sitemapXml(listed.map((p) => ({ url: p.meta.canonical, lastmod: p.frontmatter.dateModified ?? p.frontmatter.datePublished }))));
+        let gitDates;
+        if (site.seo.lastmod === "git") {
+            const result = gitPageDates(root);
+            gitDates = result.dates;
+            if (result.problem)
+                warnings.push(`seo.lastmod is "git", but ${result.problem}; sitemap dates come from frontmatter instead.`);
+        }
+        generate("sitemap.xml", () => sitemapXml(listed.map((p) => ({ url: p.meta.canonical, lastmod: p.frontmatter.dateModified ?? gitDates?.get(p.source) ?? p.frontmatter.datePublished }))));
+    }
+    for (const feed of site.seo.feeds) {
+        if (!site.site) {
+            warnings.push(`seo.feeds: the feed for ${feed.under} needs "site" in vage.config.ts and was skipped.`);
+            continue;
+        }
+        if (!listed.some((p) => p.route.startsWith(feed.under) && p.route !== feed.under)) {
+            warnings.push(`seo.feeds: no published pages under ${feed.under}, so its feed is empty.`);
+        }
+        generate(feedPath(feed.under), () => rssFeed(site, feed, listed, site.site));
+    }
+    if (site.seo.markdownCopies) {
+        for (const p of listed)
+            generate(markdownCopyPath(p.route), () => markdownCopy(p.frontmatter.title, p.frontmatter.description, markdown.get(p.route) ?? ""));
     }
     const indexNow = site.integrations.indexNow;
     if (indexNow)
@@ -114,9 +149,10 @@ function addDiscoveryFiles(root, model) {
         else {
             const pages = listed.map((p) => ({
                 title: p.frontmatter.title,
-                url: absoluteUrl(site.site, p.route) ?? p.route,
+                // With Markdown copies, AI tools are pointed at the plain-text version (llmstxt.org).
+                url: site.seo.markdownCopies ? (absoluteUrl(site.site, `/${markdownCopyPath(p.route)}`) ?? `/${markdownCopyPath(p.route)}`) : (absoluteUrl(site.site, p.route) ?? p.route),
                 description: p.frontmatter.description,
-                body: p.body,
+                body: markdown.get(p.route) ?? "",
             }));
             generate("llms.txt", () => llmsTxt(site.title, site.description, pages));
             if (site.seo.llmsTxt.full)

package/dist/deploy/github-pages.js

@@ -25,6 +25,8 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # full history, so seo.lastmod: "git" can date pages
       - uses: actions/setup-node@v7
         with:
           node-version: 22

package/dist/dev/guard.d.ts

@@ -7,3 +7,5 @@ import type { Plugin } from "vite";
  */
 export declare function devAccessGuard(root: string, allowedFiles: Set<string>): Plugin;
 export declare function allowed(url: string, publicDir: string, allowedFiles: Set<string>): boolean;
+/** A regular file whose real path (every folder resolved) is inside public/: no symlinked files or folders. */
+export declare function insidePublic(file: string, publicDir: string): boolean;

package/dist/dev/guard.js

@@ -45,11 +45,40 @@ export function allowed(url, publicDir, allowedFiles) {
         return true; // rendered pages (our middleware decides 200/404)
     if (allowedFiles.has(pathname))
         return true;
+    // Optimized images (images.optimize), served from the cache by the dev server.
+    if (/^\/_vage\/img\/[\w-]+\.webp$/.test(pathname))
+        return true;
+    // Generated files (seo.feeds, seo.markdownCopies) never exist on disk in dev. If a real file
+    // sits at that path, it's a project source (e.g. /src/pages/about.md): Vite would serve it, so refuse.
+    if (/^\/(?:[\w-]+\/)*(?:feed\.xml|[\w-]+\.md)$/.test(pathname)) {
+        if (fs.existsSync(path.join(path.dirname(publicDir), pathname)))
+            return false;
+        // A public/ file with that name is served by Vite, so apply the same rules as below.
+        const candidate = path.join(publicDir, pathname);
+        if (!fs.existsSync(candidate) && !isSymlink(candidate))
+            return true; // not in public/: generated by vagejs
+        return insidePublic(candidate, publicDir);
+    }
     const file = path.resolve(publicDir, `.${pathname}`);
     if (!file.startsWith(publicDir + path.sep))
         return false;
+    return insidePublic(file, publicDir);
+}
+function isSymlink(file) {
+    try {
+        return fs.lstatSync(file).isSymbolicLink();
+    }
+    catch {
+        return false;
+    }
+}
+/** A regular file whose real path (every folder resolved) is inside public/: no symlinked files or folders. */
+export function insidePublic(file, publicDir) {
     try {
-        return fs.lstatSync(file).isFile(); // lstat: symlinks are not served
+        if (!fs.lstatSync(file).isFile())
+            return false;
+        const realPublic = fs.realpathSync(publicDir);
+        return fs.realpathSync(file).startsWith(realPublic + path.sep);
     }
     catch {
         return false;

package/dist/dev/server.js

@@ -6,6 +6,7 @@ import { VageError } from "../core/errors.js";
 import { escapeHtml } from "../core/html.js";
 import { baseViteConfig, viteLoader } from "../core/loader.js";
 import { renderPage } from "../core/render.js";
+import { cacheDir } from "../content/optimize.js";
 import { collectPages } from "../core/pages.js";
 import { renderSite } from "../core/site.js";
 import { discoverRoutes, findNotFoundPage, normalizeRequestPath } from "../core/routes.js";
@@ -42,12 +43,27 @@ export async function startDevServer(root, options = {}) {
         const urlPath = req.url ?? "/";
         const pathname = urlPath.split(/[?#]/)[0];
         try {
-            if (DISCOVERY_FILES[pathname]) {
+            const optimized = /^\/_vage\/img\/([\w-]+\.webp)$/.exec(pathname);
+            if (optimized) {
+                const file = path.join(cacheDir(root), optimized[1]);
+                if (!fs.existsSync(file))
+                    return send(res, 404, plainPage("404 — Not found", `No optimized image at ${pathname}`));
+                res.setHeader("Content-Type", "image/webp");
+                return res.end(fs.readFileSync(file));
+            }
+            const isFeed = /^\/(?:[\w.-]+\/)*feed\.xml$/.test(pathname) && !pathname.includes("..");
+            const isCopy = /^\/(?:[\w-]+\/)*[\w-]+\.md$/.test(pathname);
+            if (DISCOVERY_FILES[pathname] || isFeed || isCopy) {
                 // Built from the whole site (drafts excluded), exactly as `vage build` would write it.
                 const content = (await renderSite(root, load)).files.get(pathname.slice(1));
-                if (content === undefined)
+                if (content === undefined) {
+                    // Feeds and Markdown copies exist only as generated files; never fall through to Vite,
+                    // which would serve project sources (e.g. /src/pages/about.md).
+                    if (isFeed || isCopy)
+                        return send(res, 404, plainPage("404 — Not found", `No generated file at ${pathname}`));
                     return next();
-                res.setHeader("Content-Type", DISCOVERY_FILES[pathname]);
+                }
+                res.setHeader("Content-Type", DISCOVERY_FILES[pathname] ?? (isCopy ? "text/markdown; charset=utf-8" : "application/rss+xml; charset=utf-8"));
                 return res.end(content);
             }
             // Leave asset requests (any extension other than .html) to Vite.

package/dist/discovery/feed.d.ts

@@ -0,0 +1,21 @@
+import type { SiteConfig } from "../core/config.js";
+import type { SitePage } from "../core/site.js";
+export interface FeedConfig {
+    under: string;
+    title?: string;
+    description?: string;
+    limit: number;
+}
+/** The feed's title: configured, else the site title. Used for the channel and the page links. */
+export declare function feedTitle(site: SiteConfig, feed: {
+    title?: string;
+    under: string;
+}): string;
+/** Where a section's feed is published: "/blog/" -> "blog/feed.xml". */
+export declare function feedPath(under: string): string;
+/**
+ * RSS 2.0 for one section: its listed pages, newest datePublished first (undated last).
+ * Item descriptions are the page descriptions; dates come only from frontmatter, so the
+ * feed is the same on every build.
+ */
+export declare function rssFeed(site: SiteConfig, feed: FeedConfig, pages: SitePage[], origin: string): string;

package/dist/discovery/feed.js

@@ -0,0 +1,68 @@
+import { instant } from "../core/dates.js";
+const XML_ESCAPES = { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&apos;" };
+const xml = (s) => s.replace(/[&<>"']/g, (c) => XML_ESCAPES[c]);
+/** The feed's title: configured, else the site title. Used for the channel and the page links. */
+export function feedTitle(site, feed) {
+    return feed.title ?? site.title ?? feed.under;
+}
+/** Where a section's feed is published: "/blog/" -> "blog/feed.xml". */
+export function feedPath(under) {
+    return `${under.replace(/^\/+/, "")}feed.xml`;
+}
+/** RFC 822 date, as RSS requires: "Thu, 23 Oct 2026 00:00:00 GMT". */
+function rfc822(iso) {
+    return new Date(iso.length === 10 ? `${iso}T00:00:00Z` : iso).toUTCString();
+}
+/**
+ * RSS 2.0 for one section: its listed pages, newest datePublished first (undated last).
+ * Item descriptions are the page descriptions; dates come only from frontmatter, so the
+ * feed is the same on every build.
+ */
+export function rssFeed(site, feed, pages, origin) {
+    const items = pages
+        .filter((p) => p.route.startsWith(feed.under) && p.route !== feed.under)
+        .sort((a, b) => {
+        // Compare as instants: "2026-10-23T23:00:00-05:00" is after "2026-10-24".
+        const ta = instant(a.frontmatter.datePublished);
+        const tb = instant(b.frontmatter.datePublished);
+        if (ta === tb || (Number.isNaN(ta) && Number.isNaN(tb)))
+            return a.frontmatter.title.localeCompare(b.frontmatter.title);
+        return Number.isNaN(ta) ? 1 : Number.isNaN(tb) ? -1 : tb - ta;
+    })
+        .slice(0, feed.limit);
+    const section = pages.find((p) => p.route === feed.under);
+    // Same default as the <link rel="alternate"> title on the pages.
+    const title = feedTitle(site, feed);
+    const description = feed.description ?? section?.frontmatter.description ?? site.description ?? title;
+    const self = `${origin}/${feedPath(feed.under)}`;
+    const newest = items.find((p) => p.frontmatter.datePublished)?.frontmatter.datePublished; // items are sorted newest first
+    const body = items
+        .map((p) => {
+        const link = p.meta.canonical ?? `${origin}${p.route}`;
+        const date = p.frontmatter.datePublished;
+        return [
+            "    <item>",
+            `      <title>${xml(p.frontmatter.title)}</title>`,
+            `      <link>${xml(link)}</link>`,
+            `      <guid isPermaLink="true">${xml(link)}</guid>`,
+            p.frontmatter.description ? `      <description>${xml(p.frontmatter.description)}</description>` : "",
+            date ? `      <pubDate>${rfc822(date)}</pubDate>` : "",
+            "    </item>",
+        ]
+            .filter(Boolean)
+            .join("\n");
+    })
+        .join("\n");
+    return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+  <channel>
+    <title>${xml(title)}</title>
+    <link>${xml(`${origin}${feed.under}`)}</link>
+    <description>${xml(description)}</description>
+    <language>${xml(site.lang)}</language>
+    <atom:link href="${xml(self)}" rel="self" type="application/rss+xml" />${newest ? `\n    <lastBuildDate>${rfc822(newest)}</lastBuildDate>` : ""}
+${body}
+  </channel>
+</rss>
+`;
+}

package/dist/discovery/git-dates.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Last commit date (ISO) of every file under src/pages, from one `git log` call.
+ * Returns undefined with a reason when dates would be wrong or unavailable: not a git
+ * repository, or a shallow clone (CI's default), where every file seems to change in
+ * the latest commit.
+ */
+export declare function gitPageDates(root: string): {
+    dates?: Map<string, string>;
+    problem?: string;
+};

package/dist/discovery/git-dates.js

@@ -0,0 +1,57 @@
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+/**
+ * Last commit date (ISO) of every file under src/pages, from one `git log` call.
+ * Returns undefined with a reason when dates would be wrong or unavailable: not a git
+ * repository, or a shallow clone (CI's default), where every file seems to change in
+ * the latest commit.
+ */
+export function gitPageDates(root) {
+    // core.quotepath=off: print non-ASCII file names as-is (default would quote "caf\303\251.md").
+    const git = (args) => execFileSync("git", ["-c", "core.quotepath=off", ...args], { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 64 * 1024 * 1024 });
+    try {
+        git(["--version"]);
+    }
+    catch {
+        return { problem: "git isn't available" };
+    }
+    try {
+        if (git(["rev-parse", "--is-inside-work-tree"]).trim() !== "true")
+            return { problem: "the project isn't a git repository" };
+    }
+    catch {
+        return { problem: "the project isn't a git repository" };
+    }
+    try {
+        if (git(["rev-parse", "--is-shallow-repository"]).trim() === "true") {
+            return { problem: "this is a shallow git clone, so every page would get the latest commit's date (in GitHub Actions, use actions/checkout with fetch-depth: 0)" };
+        }
+        const top = git(["rev-parse", "--show-toplevel"]).trim();
+        // git prints real paths; compare against the project's real path (e.g. /var -> /private/var on macOS).
+        const real = fs.realpathSync(root);
+        // -z: NUL-separated, so file names with quotes, spaces or newlines come through exactly.
+        const out = git(["log", "-z", "--format=@@%cI", "--name-only", "--no-renames", "--", path.join(root, "src", "pages")]);
+        const dates = new Map();
+        let current = "";
+        for (const raw of out.split("\0")) {
+            const field = raw.replace(/^\n+/, "");
+            if (!field)
+                continue;
+            const header = /^@@(\S+)\n?([\s\S]*)$/.exec(field);
+            const name = header ? header[2] : field;
+            if (header)
+                current = header[1];
+            if (name && current) {
+                const rel = path.relative(real, path.join(top, name));
+                if (!dates.has(rel))
+                    dates.set(rel, current); // newest first: keep the first date seen
+            }
+        }
+        return { dates };
+    }
+    catch (err) {
+        const stderr = String(err.stderr ?? "").trim().split("\n")[0];
+        return { problem: `reading the git history failed (${stderr || err.message.split("\n")[0]})` };
+    }
+}

package/dist/discovery/markdown-copy.d.ts

@@ -0,0 +1,13 @@
+/** "/" -> "index.md", "/docs/cli/" -> "docs/cli.md". */
+export declare function markdownCopyPath(route: string): string;
+/**
+ * The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
+ * exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
+ * links the link lock checked), with links and images made absolute.
+ */
+export declare function htmlToMarkdown(html: string, options: {
+    route: string;
+    origin?: string;
+}): Promise<string>;
+/** The copy: the page's own H1 (or the title), the summary, then the content. */
+export declare function markdownCopy(title: string, description: string | undefined, markdown: string): string;

package/dist/discovery/markdown-copy.js

@@ -0,0 +1,112 @@
+import TurndownService from "turndown";
+/** "/" -> "index.md", "/docs/cli/" -> "docs/cli.md". */
+export function markdownCopyPath(route) {
+    return route === "/" ? "index.md" : `${route.slice(1, -1)}.md`;
+}
+let jsdom;
+let service;
+function converter() {
+    if (service)
+        return service;
+    const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-", emDelimiter: "_" });
+    // Never carry executable or embedded content into the copy, even from trusted HTML.
+    td.remove(["script", "style", "noscript", "template", "iframe", "object", "embed", "form", "button", "select", "textarea"]); // svg and math are removed from the DOM first
+    // "<" in text must stay text: a Markdown viewer that renders HTML would otherwise run it.
+    const baseEscape = td.escape.bind(td);
+    td.escape = (text) => baseEscape(text).replace(/</g, "\\<");
+    td.addRule("checkbox", {
+        filter: (node) => node.nodeName === "INPUT",
+        replacement: (_c, node) => (node.type === "checkbox" ? (node.checked || node.hasAttribute("checked") ? "[x] " : "[ ] ") : ""),
+    });
+    td.addRule("strikethrough", { filter: ["del", "s"], replacement: (content) => (content.trim() ? `~~${content}~~` : "") });
+    td.addRule("preformatted", {
+        filter: (node) => node.nodeName === "PRE" && !node.querySelector("code"),
+        replacement: (_c, node) => {
+            const text = (node.textContent ?? "").replace(/\n$/, "");
+            // The fence must be longer than any backtick run inside, or the text could close it early.
+            const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length));
+            const fence = "`".repeat(Math.max(3, longest + 1));
+            return `\n\n${fence}\n${text}\n${fence}\n\n`;
+        },
+    });
+    // A link nobody can see or click on the page shouldn't appear in the copy.
+    td.addRule("emptyLink", {
+        filter: (node) => node.nodeName === "A" && !(node.textContent ?? "").trim() && !node.querySelector("img[alt]:not([alt=''])"),
+        replacement: () => "",
+    });
+    // Simple GFM tables, built in one pass: cells and rows leave separators that the table
+    // rule turns into pipes, so each level of a nested table is converted only once.
+    const CELL = "\u0001";
+    const ROW = "\u0002";
+    const cell = (s) => s.replace(/[\u0001\u0002]/g, "").replace(/\n+/g, " ").replace(/\|/g, "\\|").trim();
+    td.addRule("tableCell", { filter: ["th", "td"], replacement: (content) => `${cell(content)}${CELL}` });
+    td.addRule("tableRow", { filter: "tr", replacement: (content) => `${ROW}${content}` });
+    td.addRule("table", {
+        filter: "table",
+        replacement: (content) => {
+            const rows = content
+                .split(ROW)
+                .map((r) => r.split(CELL).slice(0, -1).map((c) => c.trim()))
+                .filter((r) => r.length);
+            if (!rows.length)
+                return "";
+            const width = Math.max(...rows.map((r) => r.length));
+            const line = (r) => `| ${Array.from({ length: width }, (_, i) => r[i] ?? "").join(" | ")} |`;
+            return `\n\n${[line(rows[0]), line(Array(width).fill("---")), ...rows.slice(1).map(line)].join("\n")}\n\n`;
+        },
+    });
+    service = td;
+    return td;
+}
+/**
+ * The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
+ * exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
+ * links the link lock checked), with links and images made absolute.
+ */
+export async function htmlToMarkdown(html, options) {
+    jsdom ??= await import("jsdom");
+    const doc = new jsdom.JSDOM("").window.document;
+    const template = doc.createElement("template");
+    template.innerHTML = html;
+    const base = `https://site.invalid${options.route}`;
+    const absolute = (value) => {
+        if (/^([a-z][a-z0-9+.-]*:|\/\/|#)/i.test(value))
+            return value;
+        try {
+            const u = new URL(value, base);
+            return `${options.origin ?? ""}${u.pathname}${u.search}${u.hash}`;
+        }
+        catch {
+            return value;
+        }
+    };
+    // Hidden content isn't on the page for visitors, so it isn't in the copy either.
+    for (const el of template.content.querySelectorAll("[hidden], svg, math"))
+        el.remove();
+    for (const a of template.content.querySelectorAll("a[href]")) {
+        const href = a.getAttribute("href").trim();
+        // Script and data URLs never become links in the copy (trusted HTML can contain them).
+        if (/^(javascript|vbscript|data):/i.test(href.replace(/[\s\u0000-\u001f]/g, "")))
+            a.removeAttribute("href");
+        else
+            a.setAttribute("href", absolute(href));
+    }
+    for (const img of template.content.querySelectorAll("img[src]")) {
+        const src = img.getAttribute("src").trim();
+        if (/^(javascript|vbscript|data):/i.test(src.replace(/[\s\u0000-\u001f]/g, "")))
+            img.remove();
+        else
+            img.setAttribute("src", absolute(src));
+        img.removeAttribute("srcset");
+    }
+    const wrapper = doc.createElement("div");
+    wrapper.append(template.content);
+    return converter().turndown(wrapper).trim();
+}
+/** The copy: the page's own H1 (or the title), the summary, then the content. */
+export function markdownCopy(title, description, markdown) {
+    const own = /^#\s+(.+?)\s*(?:\n+|$)/.exec(markdown);
+    const heading = `# ${own ? own[1] : title}`;
+    const rest = own ? markdown.slice(own[0].length) : markdown;
+    return `${heading}\n\n${description ? `> ${description}\n\n` : ""}${rest.trim()}\n`;
+}

package/dist/guidelines/rule-pack.json

@@ -1,5 +1,5 @@
 {
-    "version": "2026.10.0",
+    "version": "2026.10.1",
     "released": "2026-10-23",
     "sources": [
         {
@@ -266,6 +266,14 @@
             "url": "https://developers.cloudflare.com/pages/configuration/headers/",
             "category": "hosting",
             "cadence": "weekly"
+        },
+        {
+            "id": "perplexity-bots",
+            "provider": "Perplexity",
+            "title": "Perplexity Crawlers",
+            "url": "https://docs.perplexity.ai/guides/bots",
+            "category": "ai-crawlers",
+            "cadence": "weekly"
         }
     ],
     "rules": [
@@ -597,6 +605,90 @@
             "data": {
                 "maxLineLength": 2000
             }
+        },
+        {
+            "id": "crawlers.probes",
+            "summary": "Crawlers `vage verify` impersonates to test that a live site lets search and AI crawlers in. Full user-agent strings only where we could copy them verbatim from the provider's page (W.X.Y.Z = current Chrome version); otherwise the documented token, which is what CDN bot rules match. A user-agent probe can't detect blocks based on the real crawlers' IP addresses.",
+            "behaviorChanging": true,
+            "sources": [
+                "google-crawlers",
+                "bing-guidelines",
+                "openai-bots",
+                "anthropic-crawlers",
+                "perplexity-bots"
+            ],
+            "implementedIn": [
+                "src/checks/crawlability.ts"
+            ],
+            "data": {
+                "crawlers": [
+                    {
+                        "token": "Googlebot",
+                        "provider": "Google",
+                        "purpose": "search",
+                        "userAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+                        "source": "google-crawlers"
+                    },
+                    {
+                        "token": "bingbot",
+                        "provider": "Microsoft",
+                        "purpose": "search",
+                        "source": "bing-guidelines"
+                    },
+                    {
+                        "token": "OAI-SearchBot",
+                        "provider": "OpenAI",
+                        "purpose": "ai-search",
+                        "source": "openai-bots"
+                    },
+                    {
+                        "token": "GPTBot",
+                        "provider": "OpenAI",
+                        "purpose": "ai-training",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
+                        "source": "openai-bots"
+                    },
+                    {
+                        "token": "ChatGPT-User",
+                        "provider": "OpenAI",
+                        "purpose": "ai-user",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
+                        "source": "openai-bots"
+                    },
+                    {
+                        "token": "Claude-SearchBot",
+                        "provider": "Anthropic",
+                        "purpose": "ai-search",
+                        "source": "anthropic-crawlers"
+                    },
+                    {
+                        "token": "ClaudeBot",
+                        "provider": "Anthropic",
+                        "purpose": "ai-training",
+                        "source": "anthropic-crawlers"
+                    },
+                    {
+                        "token": "Claude-User",
+                        "provider": "Anthropic",
+                        "purpose": "ai-user",
+                        "source": "anthropic-crawlers"
+                    },
+                    {
+                        "token": "PerplexityBot",
+                        "provider": "Perplexity",
+                        "purpose": "ai-search",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+                        "source": "perplexity-bots"
+                    },
+                    {
+                        "token": "Perplexity-User",
+                        "provider": "Perplexity",
+                        "purpose": "ai-user",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
+                        "source": "perplexity-bots"
+                    }
+                ]
+            }
         }
     ],
     "changelog": [
@@ -606,6 +698,13 @@
             "summary": "Initial rule pack (unreleased; extended during Phase 8 and the security review with IndexNow, Clarity, Plausible, Cloudflare Pages and CSP provider rules). Facts checked against the cited sources on 2026-10-23 during implementation.",
             "reviewedBy": "Venkatesh A G E",
             "contentHash": "8c7272511f733c01984ce1d962b39e2d521a88b31a5c958a95307a04b82bb3dd"
+        },
+        {
+            "version": "2026.10.1",
+            "date": "2026-10-23",
+            "summary": "Add crawlers.probes (search and AI crawler user agents and tokens for vage verify's crawlability check) and the Perplexity crawler source. Full user-agent strings copied verbatim on 2026-10-23 for Googlebot, GPTBot, ChatGPT-User, PerplexityBot and Perplexity-User; OAI-SearchBot, Anthropic's crawlers and bingbot are recorded by token (Anthropic's page lists only tokens, Bing's page couldn't be read, and OAI-SearchBot's full string couldn't be copied verbatim). No existing rule changed.",
+            "reviewedBy": "Venkatesh A G E",
+            "contentHash": "4d88418eb0b5425da970a1a30218a097783e117ce65376171d9a7f3ab7d9f658"
         }
     ]
 }

package/dist/index.d.ts

@@ -4,7 +4,7 @@ export { escapeHtml, html, raw, type SafeHtml } from "./core/html.js";
 export { z } from "zod";
 export type { Layout, LayoutProps } from "./core/render.js";
 export type { Frontmatter } from "./content/markdown.js";
-export { collection, type CollectionOptions, type PageSummary } from "./core/pages.js";
+export { breadcrumbs, collection, flatten, neighbors, tree, type CollectionOptions, type PageSummary, type TreeNode } from "./core/pages.js";
 export { build, type BuildResult } from "./core/build.js";
 export { startDevServer } from "./dev/server.js";
 export { startPreviewServer } from "./dev/preview.js";

package/dist/index.js

@@ -2,7 +2,7 @@ export { defineConfig } from "./core/config.js";
 export { escapeHtml, html, raw } from "./core/html.js";
 /** Zod, for layout field schemas: `export const frontmatter = z.object({ ... })`. Same instance vagejs validates with. */
 export { z } from "zod";
-export { collection } from "./core/pages.js";
+export { breadcrumbs, collection, flatten, neighbors, tree } from "./core/pages.js";
 export { build } from "./core/build.js";
 export { startDevServer } from "./dev/server.js";
 export { startPreviewServer } from "./dev/preview.js";