vagejs 0.3.0 → 0.3.1

package/README.md

@@ -74,7 +74,7 @@ Images in `public/` used as `![Alt text](/images/photo.jpg)` get their `width` a
 
 - `order:` and `navTitle:` in frontmatter, with `tree()`, `breadcrumbs()` and `neighbors()` in layouts, build sidebars, breadcrumbs and previous/next links.
 - `seo: { feeds: [{ under: "/blog/" }] }` writes an RSS feed at `/blog/feed.xml`.
-- `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli.md`) for AI tools, without comments or raw HTML; `seo: { lastmod: "git" }` dates the sitemap from git history (CI needs `fetch-depth: 0`).
+- `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli.md`) for AI tools, made from the page's `<main>`, without comments or raw HTML (layouts leave extras out with `data-vage-copy="skip"`); `seo: { lastmod: "git" }` dates the sitemap from git history (CI needs `fetch-depth: 0`).
 
 ### Write a layout
 

package/dist/cli/info.js

@@ -61,7 +61,7 @@ export function frameworkInfo(cli, version) {
         },
         features: {
             feeds: 'seo.feeds: [{ under: "/blog/" }] writes /blog/feed.xml (RSS 2.0) and links it from that section\'s pages',
-            markdownCopies: "seo.markdownCopies: true publishes /<path>.md copies of listed pages, linked with rel=alternate type=text/markdown",
+            markdownCopies: "seo.markdownCopies: true publishes /<path>.md copies of listed pages, made from each page's <main> and linked with rel=alternate type=text/markdown; layouts mark repeated extras (share links, calls to action) with data-vage-copy=\"skip\". On Cloudflare, _headers gives each copy a canonical Link header to its page",
             lastmod: 'seo.lastmod: "git" takes sitemap <lastmod> from each page\'s last commit (dateModified wins; skipped in shallow clones)',
             images: 'images: { optimize: true } (needs the sharp package) makes resized WebP copies of content images in /_vage/img/ and adds srcset/sizes',
             navigation: "frontmatter order and navTitle; helpers tree(), breadcrumbs(), neighbors(); PageSummary.label",

package/dist/core/config.js

@@ -31,11 +31,11 @@ const robotsSchema = z.strictObject({
     aiTraining: z
         .enum(["allow", "disallow"])
         .optional()
-        .describe("Groups for documented AI training crawlers (GPTBot, ClaudeBot, Google-Extended). Unset: the * group applies."),
+        .describe("Groups for documented AI training crawlers (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended). Google-Extended also covers grounding in Gemini Apps and Vertex AI. Unset: the * group applies."),
     aiSearch: z
         .enum(["allow", "disallow"])
         .optional()
-        .describe("Groups for documented AI search crawlers (OAI-SearchBot, Claude-SearchBot). Unset: the * group applies."),
+        .describe("Groups for documented AI search crawlers (OAI-SearchBot, Claude-SearchBot, PerplexityBot). Unset: the * group applies."),
 });
 const seoSchema = z.strictObject({
     robots: toggle(robotsSchema).describe("Generate robots.txt (true/false or options)."),

package/dist/core/site.js

@@ -3,10 +3,10 @@ import path from "node:path";
 import { llmsFullTxt, llmsTxt } from "../discovery/llms.js";
 import { feedPath, rssFeed } from "../discovery/feed.js";
 import { gitPageDates } from "../discovery/git-dates.js";
-import { htmlToMarkdown, markdownCopy, markdownCopyPath } from "../discovery/markdown-copy.js";
+import { copySource, htmlToMarkdown, markdownCopy, markdownCopyPath } from "../discovery/markdown-copy.js";
 import { robotsTxt } from "../discovery/robots.js";
 import { SITEMAP_URL_LIMIT, sitemapXml } from "../discovery/sitemap.js";
-import { headersFile } from "../deploy/headers.js";
+import { copyCanonicalsFit, headersFile } from "../deploy/headers.js";
 import { cloudflareRedirects } from "../deploy/redirects.js";
 import { findSecrets } from "../deploy/secrets.js";
 import { rulePack } from "../guidelines/pack.js";
@@ -73,7 +73,7 @@ export async function renderSite(root, load, options = {}) {
     const markdown = new Map();
     if (site.seo.markdownCopies || (site.seo.llmsTxt && site.seo.llmsTxt.full)) {
         for (const p of listedPages(model))
-            markdown.set(p.route, await htmlToMarkdown(p.content, { route: p.route, origin: site.site }));
+            markdown.set(p.route, await htmlToMarkdown(await copySource(p.html, p.content), { route: p.route, origin: site.site }));
     }
     model.redirects = buildRedirects(site, new Set(model.pages.map((p) => p.route)));
     addDiscoveryFiles(root, model, markdown);
@@ -140,7 +140,12 @@ function addDiscoveryFiles(root, model, markdown) {
         generate("_redirects", () => cloudflareRedirects(model.redirects));
     if (cloudflare && site.security.headers) {
         const html = [...model.pages, ...(model.notFound ? [model.notFound] : []), ...model.redirects].map((p) => p.html);
-        generate("_headers", () => headersFile(html, site));
+        // Each Markdown copy names its HTML page as canonical, so search engines don't index both.
+        const copies = site.seo.markdownCopies && site.site ? listed.map((p) => ({ path: `/${markdownCopyPath(p.route)}`, canonical: p.meta.canonical })).filter((c) => /^\/[A-Za-z0-9._~\/-]+$/.test(c.path) && /^[^\s<>]+$/.test(c.canonical)) : [];
+        if (!copyCanonicalsFit(copies.length)) {
+            warnings.push(`_headers: ${copies.length} Markdown copies need more header rules than Cloudflare allows, so they're published without a canonical Link header.`);
+        }
+        generate("_headers", () => headersFile(html, site, copies));
     }
     if (site.seo.llmsTxt) {
         if (!site.title) {

package/dist/deploy/headers.d.ts

@@ -7,5 +7,16 @@ import type { SiteConfig } from "../core/config.js";
  * any https origin so ordinary sites don't break.
  */
 export declare function contentSecurityPolicy(pagesHtml: string[], site: SiteConfig): string;
-/** Cloudflare Pages / Netlify `_headers` for every path. */
-export declare function headersFile(pagesHtml: string[], site: SiteConfig): string;
+/**
+ * Markdown copies can name their HTML page as canonical with one rule each, while the
+ * file stays within Cloudflare's rule limit (the first rule covers every path).
+ */
+export declare const copyCanonicalsFit: (copies: number) => boolean;
+/**
+ * Cloudflare Pages / Netlify `_headers` for every path, plus a canonical Link header on
+ * each Markdown copy (`copies`: its path and its page's URL) when they fit.
+ */
+export declare function headersFile(pagesHtml: string[], site: SiteConfig, copies?: {
+    path: string;
+    canonical: string;
+}[]): string;

package/dist/deploy/headers.js

@@ -3,7 +3,7 @@ import { parse } from "node-html-parser";
 import { VageError } from "../core/errors.js";
 import { ruleData } from "../guidelines/pack.js";
 // https://developers.cloudflare.com/pages/configuration/headers/ (rule pack "cloudflare.headers", "csp.providers")
-const { maxLineLength } = ruleData("cloudflare.headers");
+const { maxLineLength, maxRules } = ruleData("cloudflare.headers");
 const PROVIDERS = ruleData("csp.providers");
 const EXECUTABLE = /^(|module|text\/javascript|application\/javascript)$/i;
 /**
@@ -87,8 +87,16 @@ export function contentSecurityPolicy(pagesHtml, site) {
     ];
     return directives.join("; ");
 }
-/** Cloudflare Pages / Netlify `_headers` for every path. */
-export function headersFile(pagesHtml, site) {
+/**
+ * Markdown copies can name their HTML page as canonical with one rule each, while the
+ * file stays within Cloudflare's rule limit (the first rule covers every path).
+ */
+export const copyCanonicalsFit = (copies) => copies + 1 <= maxRules;
+/**
+ * Cloudflare Pages / Netlify `_headers` for every path, plus a canonical Link header on
+ * each Markdown copy (`copies`: its path and its page's URL) when they fit.
+ */
+export function headersFile(pagesHtml, site, copies = []) {
     const lines = ["# Generated by vagejs (security in vage.config.ts)", "/*", "  X-Content-Type-Options: nosniff", "  Referrer-Policy: strict-origin-when-cross-origin", "  X-Frame-Options: SAMEORIGIN", "  Permissions-Policy: camera=(), microphone=(), geolocation=()"];
     if (site.security.csp !== "off") {
         const name = site.security.csp === "report-only" ? "Content-Security-Policy-Report-Only" : "Content-Security-Policy";
@@ -99,5 +107,9 @@ export function headersFile(pagesHtml, site) {
         throw new VageError(`The generated Content-Security-Policy is ${tooLong.length} characters; _headers lines are limited to ${maxLineLength}. ` +
             "Reduce distinct inline scripts/styles (move them to files in public/), or set security.csp to \"off\".");
     }
+    if (copyCanonicalsFit(copies.length)) {
+        for (const c of copies)
+            lines.push(c.path, `  Link: <${c.canonical}>; rel="canonical"`);
+    }
     return `${lines.join("\n")}\n`;
 }

package/dist/discovery/markdown-copy.d.ts

@@ -1,5 +1,11 @@
 /** "/" -> "index.md", "/docs/cli/" -> "docs/cli.md". */
 export declare function markdownCopyPath(route: string): string;
+/**
+ * The part of a rendered page the copy is made from: the inside of its one `<main>`, so
+ * content a layout builds from frontmatter (FAQ answers, post lists) is included. Pages
+ * without exactly one `<main>` fall back to the Markdown body.
+ */
+export declare function copySource(pageHtml: string, content: string): Promise<string>;
 /**
  * The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
  * exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
@@ -9,5 +15,9 @@ export declare function htmlToMarkdown(html: string, options: {
     route: string;
     origin?: string;
 }): Promise<string>;
-/** The copy: the page's own H1 (or the title), the summary, then the content. */
+/**
+ * The copy: the page's own H1 (or the title), the summary, then the content. The H1 is
+ * moved to the top even when a layout puts something before it, but never taken from
+ * inside a code block.
+ */
 export declare function markdownCopy(title: string, description: string | undefined, markdown: string): string;

package/dist/discovery/markdown-copy.js

@@ -58,6 +58,17 @@ function converter() {
     service = td;
     return td;
 }
+/**
+ * The part of a rendered page the copy is made from: the inside of its one `<main>`, so
+ * content a layout builds from frontmatter (FAQ answers, post lists) is included. Pages
+ * without exactly one `<main>` fall back to the Markdown body.
+ */
+export async function copySource(pageHtml, content) {
+    jsdom ??= await import("jsdom");
+    // Parsed as HTML (scripts don't run), so "<main>" text in scripts, comments or attributes doesn't count.
+    const mains = new jsdom.JSDOM(pageHtml).window.document.querySelectorAll("main");
+    return mains.length === 1 ? mains[0].innerHTML : content;
+}
 /**
  * The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
  * exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
@@ -81,8 +92,13 @@ export async function htmlToMarkdown(html, options) {
         }
     };
     // Hidden content isn't on the page for visitors, so it isn't in the copy either.
-    for (const el of template.content.querySelectorAll("[hidden], svg, math"))
+    // Layouts mark repeated extras (share links, calls to action) with data-vage-copy="skip";
+    // aria-hidden parts are decoration that screen readers skip too.
+    for (const el of template.content.querySelectorAll('[hidden], svg, math, [aria-hidden="true"], [data-vage-copy="skip"]'))
         el.remove();
+    // A heading split over two lines with <br> stays one Markdown heading.
+    for (const br of template.content.querySelectorAll("h1 br, h2 br, h3 br, h4 br, h5 br, h6 br"))
+        br.replaceWith(" ");
     for (const a of template.content.querySelectorAll("a[href]")) {
         const href = a.getAttribute("href").trim();
         // Script and data URLs never become links in the copy (trusted HTML can contain them).
@@ -103,10 +119,34 @@ export async function htmlToMarkdown(html, options) {
     wrapper.append(template.content);
     return converter().turndown(wrapper).trim();
 }
-/** The copy: the page's own H1 (or the title), the summary, then the content. */
+/**
+ * The copy: the page's own H1 (or the title), the summary, then the content. The H1 is
+ * moved to the top even when a layout puts something before it, but never taken from
+ * inside a code block.
+ */
 export function markdownCopy(title, description, markdown) {
-    const own = /^#\s+(.+?)\s*(?:\n+|$)/.exec(markdown);
-    const heading = `# ${own ? own[1] : title}`;
-    const rest = own ? markdown.slice(own[0].length) : markdown;
+    const lines = markdown.split("\n");
+    let fence = "";
+    let at = -1;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const run = /^(`{3,}|~{3,})/.exec(line)?.[1];
+        if (fence) {
+            if (run && run[0] === fence[0] && run.length >= fence.length && !line.slice(run.length).trim())
+                fence = "";
+        }
+        else if (run) {
+            fence = run;
+        }
+        else if (/^#[ \t]+\S/.test(line)) {
+            at = i;
+            break;
+        }
+    }
+    const heading = `# ${at >= 0 ? lines[at].replace(/^#[ \t]+/, "").trim() : title}`;
+    let next = at + 1;
+    while (at >= 0 && next < lines.length && !lines[next].trim())
+        next++; // and the blank lines after it
+    const rest = at >= 0 ? [...lines.slice(0, at), ...lines.slice(next)].join("\n") : markdown;
     return `${heading}\n\n${description ? `> ${description}\n\n` : ""}${rest.trim()}\n`;
 }

package/dist/guidelines/rule-pack.json

@@ -1,6 +1,6 @@
 {
-    "version": "2026.10.1",
-    "released": "2026-10-23",
+    "version": "2026.10.2",
+    "released": "2026-10-24",
     "sources": [
         {
             "id": "google-search-essentials",
@@ -274,6 +274,14 @@
             "url": "https://docs.perplexity.ai/guides/bots",
             "category": "ai-crawlers",
             "cadence": "weekly"
+        },
+        {
+            "id": "apple-applebot",
+            "provider": "Apple",
+            "title": "About Applebot (incl. Applebot-Extended)",
+            "url": "https://support.apple.com/en-us/119829",
+            "category": "ai-crawlers",
+            "cadence": "weekly"
         }
     ],
     "rules": [
@@ -284,7 +292,9 @@
             "sources": [
                 "openai-bots",
                 "anthropic-crawlers",
-                "google-crawlers"
+                "google-crawlers",
+                "perplexity-bots",
+                "apple-applebot"
             ],
             "implementedIn": [
                 "src/discovery/crawlers.ts",
@@ -321,6 +331,18 @@
                         "provider": "Google",
                         "purpose": "training",
                         "source": "google-crawlers"
+                    },
+                    {
+                        "token": "PerplexityBot",
+                        "provider": "Perplexity",
+                        "purpose": "search",
+                        "source": "perplexity-bots"
+                    },
+                    {
+                        "token": "Applebot-Extended",
+                        "provider": "Apple",
+                        "purpose": "training",
+                        "source": "apple-applebot"
                     }
                 ]
             }
@@ -603,7 +625,8 @@
                 "src/deploy/headers.ts"
             ],
             "data": {
-                "maxLineLength": 2000
+                "maxLineLength": 2000,
+                "maxRules": 100
             }
         },
         {
@@ -705,6 +728,13 @@
             "summary": "Add crawlers.probes (search and AI crawler user agents and tokens for vage verify's crawlability check) and the Perplexity crawler source. Full user-agent strings copied verbatim on 2026-10-23 for Googlebot, GPTBot, ChatGPT-User, PerplexityBot and Perplexity-User; OAI-SearchBot, Anthropic's crawlers and bingbot are recorded by token (Anthropic's page lists only tokens, Bing's page couldn't be read, and OAI-SearchBot's full string couldn't be copied verbatim). No existing rule changed.",
             "reviewedBy": "Venkatesh A G E",
             "contentHash": "4d88418eb0b5425da970a1a30218a097783e117ce65376171d9a7f3ab7d9f658"
+        },
+        {
+            "version": "2026.10.2",
+            "date": "2026-10-24",
+            "summary": "robots.ai-crawlers: add PerplexityBot (search; Perplexity says it surfaces and links sites in its search results and is not used for foundation-model training) and Applebot-Extended (training; Apple says it does not crawl and only controls use of Applebot-crawled data for training its generative models). Google-Extended stays under training; its source also covers grounding in Gemini Apps and Vertex AI, which the docs now state. Perplexity-User is excluded as a user-initiated fetcher. CCBot and cohere-ai not added: no provider documentation describing them as AI-training robots.txt tokens was found. Checked against the cited sources on 2026-10-24. cloudflare.headers: record the documented limit of 100 rules as data (maxRules), used to add a canonical Link header to each Markdown copy only when the _headers file stays within it.",
+            "reviewedBy": "Venkatesh A G E",
+            "contentHash": "6b1d977ca79843dab0f6b03dc8b3448ca451eafb037bc6af112165fb5c1035c1"
         }
     ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vagejs",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "AI-native, static-first web framework: Markdown sites with built-in SEO, llms.txt, audits, security and safe deploys.",
   "keywords": [
     "static-site-generator",

package/templates/starter/AGENTS.md

@@ -29,7 +29,7 @@ Run `npm run check` after every change and fix all errors before finishing.
 - **Add an image:** put it in `public/images/` and write `![Describe the image](/images/name.jpg)`. Width and height are filled in from the file; write real alt text. For resized WebP copies, the owner can turn on `images: { optimize: true }` (needs `npm install --save-dev sharp`). Only images in page content are optimized, not images a layout writes.
 - **Order a menu or sidebar:** set `order:` (lower first) and optionally a short `navTitle:` in each page's frontmatter (shown as `p.label`), then build the menu in a layout with `tree(pages, "/docs/")`. For previous/next links in the same order use `neighbors(flatten(tree(pages, "/docs/")), page.route)`. For a flat section, `collection(pages, { under: "/docs/", sort: "order", includeIndex: true })` works too. `breadcrumbs(pages, page.route)` gives the trail from the home page.
 - **Add an RSS feed for the blog:** `seo: { feeds: [{ under: "/blog/" }] }` in `vage.config.ts` writes `/blog/feed.xml`, newest `datePublished` first (undated posts last, without a date).
-- **Help AI tools read the site:** `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli/` becomes `/docs/cli.md`), converted from the published HTML page, so it contains exactly what the page shows, and `llms.txt` links to them. `seo: { lastmod: "git" }` dates the sitemap from git history (needs full history in CI: `fetch-depth: 0`).
+- **Help AI tools read the site:** `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli/` becomes `/docs/cli.md`), converted from the page's `<main>`, so it contains exactly what the page shows (including content a layout builds from frontmatter), and `llms.txt` links to them. In layouts, add `data-vage-copy="skip"` to repeated extras such as share links or a call to action to leave them out of the copy. On Cloudflare, each copy gets a canonical `Link` header pointing to its page. `seo: { lastmod: "git" }` dates the sitemap from git history (needs full history in CI: `fetch-depth: 0`).
 - **Rename a page:** move the file, then add `redirects: { "/old-slug/": "/new-slug/" }` to `vage.config.ts`.
 - **Hide a page from search:** `noindex: true` (it stays published). To unpublish, use `draft: true` or delete it.
 - **AI crawler policy:** set `seo.robots.aiTraining` / `aiSearch` only on the owner's instruction.

package/templates/starter/package.json

@@ -11,6 +11,6 @@
     "preview": "vage preview"
   },
   "devDependencies": {
-    "vagejs": "^0.3.0"
+    "vagejs": "^0.3.1"
   }
 }