vagejs 0.2.0 → 0.3.1

package/dist/core/site.js

@@ -1,9 +1,12 @@
 import fs from "node:fs";
 import path from "node:path";
 import { llmsFullTxt, llmsTxt } from "../discovery/llms.js";
+import { feedPath, rssFeed } from "../discovery/feed.js";
+import { gitPageDates } from "../discovery/git-dates.js";
+import { copySource, htmlToMarkdown, markdownCopy, markdownCopyPath } from "../discovery/markdown-copy.js";
 import { robotsTxt } from "../discovery/robots.js";
 import { SITEMAP_URL_LIMIT, sitemapXml } from "../discovery/sitemap.js";
-import { headersFile } from "../deploy/headers.js";
+import { copyCanonicalsFit, headersFile } from "../deploy/headers.js";
 import { cloudflareRedirects } from "../deploy/redirects.js";
 import { findSecrets } from "../deploy/secrets.js";
 import { rulePack } from "../guidelines/pack.js";
@@ -13,6 +16,7 @@ import { VageError } from "./errors.js";
 import { escapeHtml } from "./html.js";
 import { renderPage } from "./render.js";
 import { parseFrontmatter } from "../content/markdown.js";
+import { sharpResize } from "../content/optimize.js";
 import { collectPages } from "./pages.js";
 import { discoverRoutes, findNotFoundPage, normalizeRequestPath } from "./routes.js";
 /** Renders every page and derives discovery files from the same data, without writing anything. */
@@ -42,6 +46,8 @@ export async function renderSite(root, load, options = {}) {
         }
     };
     const pages = await collectPages(root, site, load, { skipInvalid: collect });
+    // Loaded once for the whole site, so a missing sharp is one clear error, not one per page.
+    const resize = site.images.optimize ? await sharpResize(root) : undefined;
     for (const entry of discoverRoutes(pagesDir)) {
         const source = path.relative(root, entry.file);
         await attempt(entry.file, entry.route, async () => {
@@ -51,30 +57,38 @@ export async function renderSite(root, load, options = {}) {
                 model.draftRoutes.push(entry.route);
                 return;
             }
-            const page = await renderPage({ root, file: entry.file, route: entry.route, site, load, analytics: true, pages });
+            const page = await renderPage({ root, file: entry.file, route: entry.route, site, load, analytics: true, pages, resize });
             model.pages.push({ ...page, route: entry.route, source, outFile: entry.outFile });
         });
     }
     const notFound = findNotFoundPage(pagesDir);
     if (notFound) {
         await attempt(notFound, undefined, async () => {
-            const page = await renderPage({ root, file: notFound, route: "/404/", site, load, notFound: true, analytics: true, pages });
+            const page = await renderPage({ root, file: notFound, route: "/404/", site, load, notFound: true, analytics: true, pages, resize });
             model.notFound = { ...page, route: "/404/", source: path.relative(root, notFound), outFile: "404.html" };
         });
     }
     checkPublicDir(root, model.warnings);
+    // Markdown versions of pages, converted from their sanitized HTML (copies and llms-full.txt).
+    const markdown = new Map();
+    if (site.seo.markdownCopies || (site.seo.llmsTxt && site.seo.llmsTxt.full)) {
+        for (const p of listedPages(model))
+            markdown.set(p.route, await htmlToMarkdown(await copySource(p.html, p.content), { route: p.route, origin: site.site }));
+    }
     model.redirects = buildRedirects(site, new Set(model.pages.map((p) => p.route)));
-    addDiscoveryFiles(root, model);
+    addDiscoveryFiles(root, model, markdown);
     return model;
 }
 /** Pages that should appear in sitemap.xml and llms.txt: indexable and canonical to themselves. */
 export function listedPages(model) {
     return model.pages.filter((p) => !p.meta.noindex && p.meta.canonical === absoluteUrl(model.site.site, p.route));
 }
-function addDiscoveryFiles(root, model) {
+function addDiscoveryFiles(root, model, markdown) {
     const { site, files, warnings } = model;
     const listed = listedPages(model);
     const generate = (name, content) => {
+        if (files.has(name))
+            throw new VageError(`Two generated files would be written to ${name}. Rename one of the pages behind it.`);
         if (fs.existsSync(path.join(root, "public", name))) {
             throw new VageError(`public/${name} conflicts with the generated ${name}. Delete it, or turn generation off in vage.config.ts (seo).`);
         }
@@ -94,7 +108,28 @@ function addDiscoveryFiles(root, model) {
         if (listed.length > SITEMAP_URL_LIMIT) {
             throw new VageError(`sitemap.xml would list ${listed.length} URLs; sitemap indexes (over ${SITEMAP_URL_LIMIT}) are not supported yet.`);
         }
-        generate("sitemap.xml", () => sitemapXml(listed.map((p) => ({ url: p.meta.canonical, lastmod: p.frontmatter.dateModified ?? p.frontmatter.datePublished }))));
+        let gitDates;
+        if (site.seo.lastmod === "git") {
+            const result = gitPageDates(root);
+            gitDates = result.dates;
+            if (result.problem)
+                warnings.push(`seo.lastmod is "git", but ${result.problem}; sitemap dates come from frontmatter instead.`);
+        }
+        generate("sitemap.xml", () => sitemapXml(listed.map((p) => ({ url: p.meta.canonical, lastmod: p.frontmatter.dateModified ?? gitDates?.get(p.source) ?? p.frontmatter.datePublished }))));
+    }
+    for (const feed of site.seo.feeds) {
+        if (!site.site) {
+            warnings.push(`seo.feeds: the feed for ${feed.under} needs "site" in vage.config.ts and was skipped.`);
+            continue;
+        }
+        if (!listed.some((p) => p.route.startsWith(feed.under) && p.route !== feed.under)) {
+            warnings.push(`seo.feeds: no published pages under ${feed.under}, so its feed is empty.`);
+        }
+        generate(feedPath(feed.under), () => rssFeed(site, feed, listed, site.site));
+    }
+    if (site.seo.markdownCopies) {
+        for (const p of listed)
+            generate(markdownCopyPath(p.route), () => markdownCopy(p.frontmatter.title, p.frontmatter.description, markdown.get(p.route) ?? ""));
     }
     const indexNow = site.integrations.indexNow;
     if (indexNow)
@@ -105,7 +140,12 @@ function addDiscoveryFiles(root, model) {
         generate("_redirects", () => cloudflareRedirects(model.redirects));
     if (cloudflare && site.security.headers) {
         const html = [...model.pages, ...(model.notFound ? [model.notFound] : []), ...model.redirects].map((p) => p.html);
-        generate("_headers", () => headersFile(html, site));
+        // Each Markdown copy names its HTML page as canonical, so search engines don't index both.
+        const copies = site.seo.markdownCopies && site.site ? listed.map((p) => ({ path: `/${markdownCopyPath(p.route)}`, canonical: p.meta.canonical })).filter((c) => /^\/[A-Za-z0-9._~\/-]+$/.test(c.path) && /^[^\s<>]+$/.test(c.canonical)) : [];
+        if (!copyCanonicalsFit(copies.length)) {
+            warnings.push(`_headers: ${copies.length} Markdown copies need more header rules than Cloudflare allows, so they're published without a canonical Link header.`);
+        }
+        generate("_headers", () => headersFile(html, site, copies));
     }
     if (site.seo.llmsTxt) {
         if (!site.title) {
@@ -114,9 +154,10 @@ function addDiscoveryFiles(root, model) {
         else {
             const pages = listed.map((p) => ({
                 title: p.frontmatter.title,
-                url: absoluteUrl(site.site, p.route) ?? p.route,
+                // With Markdown copies, AI tools are pointed at the plain-text version (llmstxt.org).
+                url: site.seo.markdownCopies ? (absoluteUrl(site.site, `/${markdownCopyPath(p.route)}`) ?? `/${markdownCopyPath(p.route)}`) : (absoluteUrl(site.site, p.route) ?? p.route),
                 description: p.frontmatter.description,
-                body: p.body,
+                body: markdown.get(p.route) ?? "",
             }));
             generate("llms.txt", () => llmsTxt(site.title, site.description, pages));
             if (site.seo.llmsTxt.full)

package/dist/deploy/github-pages.js

@@ -25,6 +25,8 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # full history, so seo.lastmod: "git" can date pages
       - uses: actions/setup-node@v7
         with:
           node-version: 22

package/dist/deploy/headers.d.ts

@@ -7,5 +7,16 @@ import type { SiteConfig } from "../core/config.js";
  * any https origin so ordinary sites don't break.
  */
 export declare function contentSecurityPolicy(pagesHtml: string[], site: SiteConfig): string;
-/** Cloudflare Pages / Netlify `_headers` for every path. */
-export declare function headersFile(pagesHtml: string[], site: SiteConfig): string;
+/**
+ * Markdown copies can name their HTML page as canonical with one rule each, while the
+ * file stays within Cloudflare's rule limit (the first rule covers every path).
+ */
+export declare const copyCanonicalsFit: (copies: number) => boolean;
+/**
+ * Cloudflare Pages / Netlify `_headers` for every path, plus a canonical Link header on
+ * each Markdown copy (`copies`: its path and its page's URL) when they fit.
+ */
+export declare function headersFile(pagesHtml: string[], site: SiteConfig, copies?: {
+    path: string;
+    canonical: string;
+}[]): string;

package/dist/deploy/headers.js

@@ -3,7 +3,7 @@ import { parse } from "node-html-parser";
 import { VageError } from "../core/errors.js";
 import { ruleData } from "../guidelines/pack.js";
 // https://developers.cloudflare.com/pages/configuration/headers/ (rule pack "cloudflare.headers", "csp.providers")
-const { maxLineLength } = ruleData("cloudflare.headers");
+const { maxLineLength, maxRules } = ruleData("cloudflare.headers");
 const PROVIDERS = ruleData("csp.providers");
 const EXECUTABLE = /^(|module|text\/javascript|application\/javascript)$/i;
 /**
@@ -87,8 +87,16 @@ export function contentSecurityPolicy(pagesHtml, site) {
     ];
     return directives.join("; ");
 }
-/** Cloudflare Pages / Netlify `_headers` for every path. */
-export function headersFile(pagesHtml, site) {
+/**
+ * Markdown copies can name their HTML page as canonical with one rule each, while the
+ * file stays within Cloudflare's rule limit (the first rule covers every path).
+ */
+export const copyCanonicalsFit = (copies) => copies + 1 <= maxRules;
+/**
+ * Cloudflare Pages / Netlify `_headers` for every path, plus a canonical Link header on
+ * each Markdown copy (`copies`: its path and its page's URL) when they fit.
+ */
+export function headersFile(pagesHtml, site, copies = []) {
     const lines = ["# Generated by vagejs (security in vage.config.ts)", "/*", "  X-Content-Type-Options: nosniff", "  Referrer-Policy: strict-origin-when-cross-origin", "  X-Frame-Options: SAMEORIGIN", "  Permissions-Policy: camera=(), microphone=(), geolocation=()"];
     if (site.security.csp !== "off") {
         const name = site.security.csp === "report-only" ? "Content-Security-Policy-Report-Only" : "Content-Security-Policy";
@@ -99,5 +107,9 @@ export function headersFile(pagesHtml, site) {
         throw new VageError(`The generated Content-Security-Policy is ${tooLong.length} characters; _headers lines are limited to ${maxLineLength}. ` +
             "Reduce distinct inline scripts/styles (move them to files in public/), or set security.csp to \"off\".");
     }
+    if (copyCanonicalsFit(copies.length)) {
+        for (const c of copies)
+            lines.push(c.path, `  Link: <${c.canonical}>; rel="canonical"`);
+    }
     return `${lines.join("\n")}\n`;
 }

package/dist/dev/guard.d.ts

@@ -7,3 +7,5 @@ import type { Plugin } from "vite";
  */
 export declare function devAccessGuard(root: string, allowedFiles: Set<string>): Plugin;
 export declare function allowed(url: string, publicDir: string, allowedFiles: Set<string>): boolean;
+/** A regular file whose real path (every folder resolved) is inside public/: no symlinked files or folders. */
+export declare function insidePublic(file: string, publicDir: string): boolean;

package/dist/dev/guard.js

@@ -45,11 +45,40 @@ export function allowed(url, publicDir, allowedFiles) {
         return true; // rendered pages (our middleware decides 200/404)
     if (allowedFiles.has(pathname))
         return true;
+    // Optimized images (images.optimize), served from the cache by the dev server.
+    if (/^\/_vage\/img\/[\w-]+\.webp$/.test(pathname))
+        return true;
+    // Generated files (seo.feeds, seo.markdownCopies) never exist on disk in dev. If a real file
+    // sits at that path, it's a project source (e.g. /src/pages/about.md): Vite would serve it, so refuse.
+    if (/^\/(?:[\w-]+\/)*(?:feed\.xml|[\w-]+\.md)$/.test(pathname)) {
+        if (fs.existsSync(path.join(path.dirname(publicDir), pathname)))
+            return false;
+        // A public/ file with that name is served by Vite, so apply the same rules as below.
+        const candidate = path.join(publicDir, pathname);
+        if (!fs.existsSync(candidate) && !isSymlink(candidate))
+            return true; // not in public/: generated by vagejs
+        return insidePublic(candidate, publicDir);
+    }
     const file = path.resolve(publicDir, `.${pathname}`);
     if (!file.startsWith(publicDir + path.sep))
         return false;
+    return insidePublic(file, publicDir);
+}
+function isSymlink(file) {
+    try {
+        return fs.lstatSync(file).isSymbolicLink();
+    }
+    catch {
+        return false;
+    }
+}
+/** A regular file whose real path (every folder resolved) is inside public/: no symlinked files or folders. */
+export function insidePublic(file, publicDir) {
     try {
-        return fs.lstatSync(file).isFile(); // lstat: symlinks are not served
+        if (!fs.lstatSync(file).isFile())
+            return false;
+        const realPublic = fs.realpathSync(publicDir);
+        return fs.realpathSync(file).startsWith(realPublic + path.sep);
     }
     catch {
         return false;

package/dist/dev/server.js

@@ -6,6 +6,7 @@ import { VageError } from "../core/errors.js";
 import { escapeHtml } from "../core/html.js";
 import { baseViteConfig, viteLoader } from "../core/loader.js";
 import { renderPage } from "../core/render.js";
+import { cacheDir } from "../content/optimize.js";
 import { collectPages } from "../core/pages.js";
 import { renderSite } from "../core/site.js";
 import { discoverRoutes, findNotFoundPage, normalizeRequestPath } from "../core/routes.js";
@@ -42,12 +43,27 @@ export async function startDevServer(root, options = {}) {
         const urlPath = req.url ?? "/";
         const pathname = urlPath.split(/[?#]/)[0];
         try {
-            if (DISCOVERY_FILES[pathname]) {
+            const optimized = /^\/_vage\/img\/([\w-]+\.webp)$/.exec(pathname);
+            if (optimized) {
+                const file = path.join(cacheDir(root), optimized[1]);
+                if (!fs.existsSync(file))
+                    return send(res, 404, plainPage("404 — Not found", `No optimized image at ${pathname}`));
+                res.setHeader("Content-Type", "image/webp");
+                return res.end(fs.readFileSync(file));
+            }
+            const isFeed = /^\/(?:[\w.-]+\/)*feed\.xml$/.test(pathname) && !pathname.includes("..");
+            const isCopy = /^\/(?:[\w-]+\/)*[\w-]+\.md$/.test(pathname);
+            if (DISCOVERY_FILES[pathname] || isFeed || isCopy) {
                 // Built from the whole site (drafts excluded), exactly as `vage build` would write it.
                 const content = (await renderSite(root, load)).files.get(pathname.slice(1));
-                if (content === undefined)
+                if (content === undefined) {
+                    // Feeds and Markdown copies exist only as generated files; never fall through to Vite,
+                    // which would serve project sources (e.g. /src/pages/about.md).
+                    if (isFeed || isCopy)
+                        return send(res, 404, plainPage("404 — Not found", `No generated file at ${pathname}`));
                     return next();
-                res.setHeader("Content-Type", DISCOVERY_FILES[pathname]);
+                }
+                res.setHeader("Content-Type", DISCOVERY_FILES[pathname] ?? (isCopy ? "text/markdown; charset=utf-8" : "application/rss+xml; charset=utf-8"));
                 return res.end(content);
             }
             // Leave asset requests (any extension other than .html) to Vite.

package/dist/discovery/feed.d.ts

@@ -0,0 +1,21 @@
+import type { SiteConfig } from "../core/config.js";
+import type { SitePage } from "../core/site.js";
+export interface FeedConfig {
+    under: string;
+    title?: string;
+    description?: string;
+    limit: number;
+}
+/** The feed's title: configured, else the site title. Used for the channel and the page links. */
+export declare function feedTitle(site: SiteConfig, feed: {
+    title?: string;
+    under: string;
+}): string;
+/** Where a section's feed is published: "/blog/" -> "blog/feed.xml". */
+export declare function feedPath(under: string): string;
+/**
+ * RSS 2.0 for one section: its listed pages, newest datePublished first (undated last).
+ * Item descriptions are the page descriptions; dates come only from frontmatter, so the
+ * feed is the same on every build.
+ */
+export declare function rssFeed(site: SiteConfig, feed: FeedConfig, pages: SitePage[], origin: string): string;

package/dist/discovery/feed.js

@@ -0,0 +1,68 @@
+import { instant } from "../core/dates.js";
+const XML_ESCAPES = { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&apos;" };
+const xml = (s) => s.replace(/[&<>"']/g, (c) => XML_ESCAPES[c]);
+/** The feed's title: configured, else the site title. Used for the channel and the page links. */
+export function feedTitle(site, feed) {
+    return feed.title ?? site.title ?? feed.under;
+}
+/** Where a section's feed is published: "/blog/" -> "blog/feed.xml". */
+export function feedPath(under) {
+    return `${under.replace(/^\/+/, "")}feed.xml`;
+}
+/** RFC 822 date, as RSS requires: "Thu, 23 Oct 2026 00:00:00 GMT". */
+function rfc822(iso) {
+    return new Date(iso.length === 10 ? `${iso}T00:00:00Z` : iso).toUTCString();
+}
+/**
+ * RSS 2.0 for one section: its listed pages, newest datePublished first (undated last).
+ * Item descriptions are the page descriptions; dates come only from frontmatter, so the
+ * feed is the same on every build.
+ */
+export function rssFeed(site, feed, pages, origin) {
+    const items = pages
+        .filter((p) => p.route.startsWith(feed.under) && p.route !== feed.under)
+        .sort((a, b) => {
+        // Compare as instants: "2026-10-23T23:00:00-05:00" is after "2026-10-24".
+        const ta = instant(a.frontmatter.datePublished);
+        const tb = instant(b.frontmatter.datePublished);
+        if (ta === tb || (Number.isNaN(ta) && Number.isNaN(tb)))
+            return a.frontmatter.title.localeCompare(b.frontmatter.title);
+        return Number.isNaN(ta) ? 1 : Number.isNaN(tb) ? -1 : tb - ta;
+    })
+        .slice(0, feed.limit);
+    const section = pages.find((p) => p.route === feed.under);
+    // Same default as the <link rel="alternate"> title on the pages.
+    const title = feedTitle(site, feed);
+    const description = feed.description ?? section?.frontmatter.description ?? site.description ?? title;
+    const self = `${origin}/${feedPath(feed.under)}`;
+    const newest = items.find((p) => p.frontmatter.datePublished)?.frontmatter.datePublished; // items are sorted newest first
+    const body = items
+        .map((p) => {
+        const link = p.meta.canonical ?? `${origin}${p.route}`;
+        const date = p.frontmatter.datePublished;
+        return [
+            "    <item>",
+            `      <title>${xml(p.frontmatter.title)}</title>`,
+            `      <link>${xml(link)}</link>`,
+            `      <guid isPermaLink="true">${xml(link)}</guid>`,
+            p.frontmatter.description ? `      <description>${xml(p.frontmatter.description)}</description>` : "",
+            date ? `      <pubDate>${rfc822(date)}</pubDate>` : "",
+            "    </item>",
+        ]
+            .filter(Boolean)
+            .join("\n");
+    })
+        .join("\n");
+    return `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+  <channel>
+    <title>${xml(title)}</title>
+    <link>${xml(`${origin}${feed.under}`)}</link>
+    <description>${xml(description)}</description>
+    <language>${xml(site.lang)}</language>
+    <atom:link href="${xml(self)}" rel="self" type="application/rss+xml" />${newest ? `\n    <lastBuildDate>${rfc822(newest)}</lastBuildDate>` : ""}
+${body}
+  </channel>
+</rss>
+`;
+}

package/dist/discovery/git-dates.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Last commit date (ISO) of every file under src/pages, from one `git log` call.
+ * Returns undefined with a reason when dates would be wrong or unavailable: not a git
+ * repository, or a shallow clone (CI's default), where every file seems to change in
+ * the latest commit.
+ */
+export declare function gitPageDates(root: string): {
+    dates?: Map<string, string>;
+    problem?: string;
+};

package/dist/discovery/git-dates.js

@@ -0,0 +1,57 @@
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+/**
+ * Last commit date (ISO) of every file under src/pages, from one `git log` call.
+ * Returns undefined with a reason when dates would be wrong or unavailable: not a git
+ * repository, or a shallow clone (CI's default), where every file seems to change in
+ * the latest commit.
+ */
+export function gitPageDates(root) {
+    // core.quotepath=off: print non-ASCII file names as-is (default would quote "caf\303\251.md").
+    const git = (args) => execFileSync("git", ["-c", "core.quotepath=off", ...args], { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 64 * 1024 * 1024 });
+    try {
+        git(["--version"]);
+    }
+    catch {
+        return { problem: "git isn't available" };
+    }
+    try {
+        if (git(["rev-parse", "--is-inside-work-tree"]).trim() !== "true")
+            return { problem: "the project isn't a git repository" };
+    }
+    catch {
+        return { problem: "the project isn't a git repository" };
+    }
+    try {
+        if (git(["rev-parse", "--is-shallow-repository"]).trim() === "true") {
+            return { problem: "this is a shallow git clone, so every page would get the latest commit's date (in GitHub Actions, use actions/checkout with fetch-depth: 0)" };
+        }
+        const top = git(["rev-parse", "--show-toplevel"]).trim();
+        // git prints real paths; compare against the project's real path (e.g. /var -> /private/var on macOS).
+        const real = fs.realpathSync(root);
+        // -z: NUL-separated, so file names with quotes, spaces or newlines come through exactly.
+        const out = git(["log", "-z", "--format=@@%cI", "--name-only", "--no-renames", "--", path.join(root, "src", "pages")]);
+        const dates = new Map();
+        let current = "";
+        for (const raw of out.split("\0")) {
+            const field = raw.replace(/^\n+/, "");
+            if (!field)
+                continue;
+            const header = /^@@(\S+)\n?([\s\S]*)$/.exec(field);
+            const name = header ? header[2] : field;
+            if (header)
+                current = header[1];
+            if (name && current) {
+                const rel = path.relative(real, path.join(top, name));
+                if (!dates.has(rel))
+                    dates.set(rel, current); // newest first: keep the first date seen
+            }
+        }
+        return { dates };
+    }
+    catch (err) {
+        const stderr = String(err.stderr ?? "").trim().split("\n")[0];
+        return { problem: `reading the git history failed (${stderr || err.message.split("\n")[0]})` };
+    }
+}

package/dist/discovery/markdown-copy.d.ts

@@ -0,0 +1,23 @@
+/** "/" -> "index.md", "/docs/cli/" -> "docs/cli.md". */
+export declare function markdownCopyPath(route: string): string;
+/**
+ * The part of a rendered page the copy is made from: the inside of its one `<main>`, so
+ * content a layout builds from frontmatter (FAQ answers, post lists) is included. Pages
+ * without exactly one `<main>` fall back to the Markdown body.
+ */
+export declare function copySource(pageHtml: string, content: string): Promise<string>;
+/**
+ * The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
+ * exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
+ * links the link lock checked), with links and images made absolute.
+ */
+export declare function htmlToMarkdown(html: string, options: {
+    route: string;
+    origin?: string;
+}): Promise<string>;
+/**
+ * The copy: the page's own H1 (or the title), the summary, then the content. The H1 is
+ * moved to the top even when a layout puts something before it, but never taken from
+ * inside a code block.
+ */
+export declare function markdownCopy(title: string, description: string | undefined, markdown: string): string;

package/dist/discovery/markdown-copy.js

@@ -0,0 +1,152 @@
+import TurndownService from "turndown";
+/** "/" -> "index.md", "/docs/cli/" -> "docs/cli.md". */
+export function markdownCopyPath(route) {
+    return route === "/" ? "index.md" : `${route.slice(1, -1)}.md`;
+}
+let jsdom;
+let service;
+function converter() {
+    if (service)
+        return service;
+    const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-", emDelimiter: "_" });
+    // Never carry executable or embedded content into the copy, even from trusted HTML.
+    td.remove(["script", "style", "noscript", "template", "iframe", "object", "embed", "form", "button", "select", "textarea"]); // svg and math are removed from the DOM first
+    // "<" in text must stay text: a Markdown viewer that renders HTML would otherwise run it.
+    const baseEscape = td.escape.bind(td);
+    td.escape = (text) => baseEscape(text).replace(/</g, "\\<");
+    td.addRule("checkbox", {
+        filter: (node) => node.nodeName === "INPUT",
+        replacement: (_c, node) => (node.type === "checkbox" ? (node.checked || node.hasAttribute("checked") ? "[x] " : "[ ] ") : ""),
+    });
+    td.addRule("strikethrough", { filter: ["del", "s"], replacement: (content) => (content.trim() ? `~~${content}~~` : "") });
+    td.addRule("preformatted", {
+        filter: (node) => node.nodeName === "PRE" && !node.querySelector("code"),
+        replacement: (_c, node) => {
+            const text = (node.textContent ?? "").replace(/\n$/, "");
+            // The fence must be longer than any backtick run inside, or the text could close it early.
+            const longest = Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length));
+            const fence = "`".repeat(Math.max(3, longest + 1));
+            return `\n\n${fence}\n${text}\n${fence}\n\n`;
+        },
+    });
+    // A link nobody can see or click on the page shouldn't appear in the copy.
+    td.addRule("emptyLink", {
+        filter: (node) => node.nodeName === "A" && !(node.textContent ?? "").trim() && !node.querySelector("img[alt]:not([alt=''])"),
+        replacement: () => "",
+    });
+    // Simple GFM tables, built in one pass: cells and rows leave separators that the table
+    // rule turns into pipes, so each level of a nested table is converted only once.
+    const CELL = "\u0001";
+    const ROW = "\u0002";
+    const cell = (s) => s.replace(/[\u0001\u0002]/g, "").replace(/\n+/g, " ").replace(/\|/g, "\\|").trim();
+    td.addRule("tableCell", { filter: ["th", "td"], replacement: (content) => `${cell(content)}${CELL}` });
+    td.addRule("tableRow", { filter: "tr", replacement: (content) => `${ROW}${content}` });
+    td.addRule("table", {
+        filter: "table",
+        replacement: (content) => {
+            const rows = content
+                .split(ROW)
+                .map((r) => r.split(CELL).slice(0, -1).map((c) => c.trim()))
+                .filter((r) => r.length);
+            if (!rows.length)
+                return "";
+            const width = Math.max(...rows.map((r) => r.length));
+            const line = (r) => `| ${Array.from({ length: width }, (_, i) => r[i] ?? "").join(" | ")} |`;
+            return `\n\n${[line(rows[0]), line(Array(width).fill("---")), ...rows.slice(1).map(line)].join("\n")}\n\n`;
+        },
+    });
+    service = td;
+    return td;
+}
+/**
+ * The part of a rendered page the copy is made from: the inside of its one `<main>`, so
+ * content a layout builds from frontmatter (FAQ answers, post lists) is included. Pages
+ * without exactly one `<main>` fall back to the Markdown body.
+ */
+export async function copySource(pageHtml, content) {
+    jsdom ??= await import("jsdom");
+    // Parsed as HTML (scripts don't run), so "<main>" text in scripts, comments or attributes doesn't count.
+    const mains = new jsdom.JSDOM(pageHtml).window.document.querySelectorAll("main");
+    return mains.length === 1 ? mains[0].innerHTML : content;
+}
+/**
+ * The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
+ * exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
+ * links the link lock checked), with links and images made absolute.
+ */
+export async function htmlToMarkdown(html, options) {
+    jsdom ??= await import("jsdom");
+    const doc = new jsdom.JSDOM("").window.document;
+    const template = doc.createElement("template");
+    template.innerHTML = html;
+    const base = `https://site.invalid${options.route}`;
+    const absolute = (value) => {
+        if (/^([a-z][a-z0-9+.-]*:|\/\/|#)/i.test(value))
+            return value;
+        try {
+            const u = new URL(value, base);
+            return `${options.origin ?? ""}${u.pathname}${u.search}${u.hash}`;
+        }
+        catch {
+            return value;
+        }
+    };
+    // Hidden content isn't on the page for visitors, so it isn't in the copy either.
+    // Layouts mark repeated extras (share links, calls to action) with data-vage-copy="skip";
+    // aria-hidden parts are decoration that screen readers skip too.
+    for (const el of template.content.querySelectorAll('[hidden], svg, math, [aria-hidden="true"], [data-vage-copy="skip"]'))
+        el.remove();
+    // A heading split over two lines with <br> stays one Markdown heading.
+    for (const br of template.content.querySelectorAll("h1 br, h2 br, h3 br, h4 br, h5 br, h6 br"))
+        br.replaceWith(" ");
+    for (const a of template.content.querySelectorAll("a[href]")) {
+        const href = a.getAttribute("href").trim();
+        // Script and data URLs never become links in the copy (trusted HTML can contain them).
+        if (/^(javascript|vbscript|data):/i.test(href.replace(/[\s\u0000-\u001f]/g, "")))
+            a.removeAttribute("href");
+        else
+            a.setAttribute("href", absolute(href));
+    }
+    for (const img of template.content.querySelectorAll("img[src]")) {
+        const src = img.getAttribute("src").trim();
+        if (/^(javascript|vbscript|data):/i.test(src.replace(/[\s\u0000-\u001f]/g, "")))
+            img.remove();
+        else
+            img.setAttribute("src", absolute(src));
+        img.removeAttribute("srcset");
+    }
+    const wrapper = doc.createElement("div");
+    wrapper.append(template.content);
+    return converter().turndown(wrapper).trim();
+}
+/**
+ * The copy: the page's own H1 (or the title), the summary, then the content. The H1 is
+ * moved to the top even when a layout puts something before it, but never taken from
+ * inside a code block.
+ */
+export function markdownCopy(title, description, markdown) {
+    const lines = markdown.split("\n");
+    let fence = "";
+    let at = -1;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const run = /^(`{3,}|~{3,})/.exec(line)?.[1];
+        if (fence) {
+            if (run && run[0] === fence[0] && run.length >= fence.length && !line.slice(run.length).trim())
+                fence = "";
+        }
+        else if (run) {
+            fence = run;
+        }
+        else if (/^#[ \t]+\S/.test(line)) {
+            at = i;
+            break;
+        }
+    }
+    const heading = `# ${at >= 0 ? lines[at].replace(/^#[ \t]+/, "").trim() : title}`;
+    let next = at + 1;
+    while (at >= 0 && next < lines.length && !lines[next].trim())
+        next++; // and the blank lines after it
+    const rest = at >= 0 ? [...lines.slice(0, at), ...lines.slice(next)].join("\n") : markdown;
+    return `${heading}\n\n${description ? `> ${description}\n\n` : ""}${rest.trim()}\n`;
+}