vagejs 0.2.0 → 0.3.1

package/dist/guidelines/rule-pack.json

@@ -1,6 +1,6 @@
 {
-    "version": "2026.10.0",
-    "released": "2026-10-23",
+    "version": "2026.10.2",
+    "released": "2026-10-24",
     "sources": [
         {
             "id": "google-search-essentials",
@@ -266,6 +266,22 @@
             "url": "https://developers.cloudflare.com/pages/configuration/headers/",
             "category": "hosting",
             "cadence": "weekly"
+        },
+        {
+            "id": "perplexity-bots",
+            "provider": "Perplexity",
+            "title": "Perplexity Crawlers",
+            "url": "https://docs.perplexity.ai/guides/bots",
+            "category": "ai-crawlers",
+            "cadence": "weekly"
+        },
+        {
+            "id": "apple-applebot",
+            "provider": "Apple",
+            "title": "About Applebot (incl. Applebot-Extended)",
+            "url": "https://support.apple.com/en-us/119829",
+            "category": "ai-crawlers",
+            "cadence": "weekly"
         }
     ],
     "rules": [
@@ -276,7 +292,9 @@
             "sources": [
                 "openai-bots",
                 "anthropic-crawlers",
-                "google-crawlers"
+                "google-crawlers",
+                "perplexity-bots",
+                "apple-applebot"
             ],
             "implementedIn": [
                 "src/discovery/crawlers.ts",
@@ -313,6 +331,18 @@
                         "provider": "Google",
                         "purpose": "training",
                         "source": "google-crawlers"
+                    },
+                    {
+                        "token": "PerplexityBot",
+                        "provider": "Perplexity",
+                        "purpose": "search",
+                        "source": "perplexity-bots"
+                    },
+                    {
+                        "token": "Applebot-Extended",
+                        "provider": "Apple",
+                        "purpose": "training",
+                        "source": "apple-applebot"
                     }
                 ]
             }
@@ -595,7 +625,92 @@
                 "src/deploy/headers.ts"
             ],
             "data": {
-                "maxLineLength": 2000
+                "maxLineLength": 2000,
+                "maxRules": 100
+            }
+        },
+        {
+            "id": "crawlers.probes",
+            "summary": "Crawlers `vage verify` impersonates to test that a live site lets search and AI crawlers in. Full user-agent strings only where we could copy them verbatim from the provider's page (W.X.Y.Z = current Chrome version); otherwise the documented token, which is what CDN bot rules match. A user-agent probe can't detect blocks based on the real crawlers' IP addresses.",
+            "behaviorChanging": true,
+            "sources": [
+                "google-crawlers",
+                "bing-guidelines",
+                "openai-bots",
+                "anthropic-crawlers",
+                "perplexity-bots"
+            ],
+            "implementedIn": [
+                "src/checks/crawlability.ts"
+            ],
+            "data": {
+                "crawlers": [
+                    {
+                        "token": "Googlebot",
+                        "provider": "Google",
+                        "purpose": "search",
+                        "userAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+                        "source": "google-crawlers"
+                    },
+                    {
+                        "token": "bingbot",
+                        "provider": "Microsoft",
+                        "purpose": "search",
+                        "source": "bing-guidelines"
+                    },
+                    {
+                        "token": "OAI-SearchBot",
+                        "provider": "OpenAI",
+                        "purpose": "ai-search",
+                        "source": "openai-bots"
+                    },
+                    {
+                        "token": "GPTBot",
+                        "provider": "OpenAI",
+                        "purpose": "ai-training",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
+                        "source": "openai-bots"
+                    },
+                    {
+                        "token": "ChatGPT-User",
+                        "provider": "OpenAI",
+                        "purpose": "ai-user",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
+                        "source": "openai-bots"
+                    },
+                    {
+                        "token": "Claude-SearchBot",
+                        "provider": "Anthropic",
+                        "purpose": "ai-search",
+                        "source": "anthropic-crawlers"
+                    },
+                    {
+                        "token": "ClaudeBot",
+                        "provider": "Anthropic",
+                        "purpose": "ai-training",
+                        "source": "anthropic-crawlers"
+                    },
+                    {
+                        "token": "Claude-User",
+                        "provider": "Anthropic",
+                        "purpose": "ai-user",
+                        "source": "anthropic-crawlers"
+                    },
+                    {
+                        "token": "PerplexityBot",
+                        "provider": "Perplexity",
+                        "purpose": "ai-search",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
+                        "source": "perplexity-bots"
+                    },
+                    {
+                        "token": "Perplexity-User",
+                        "provider": "Perplexity",
+                        "purpose": "ai-user",
+                        "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
+                        "source": "perplexity-bots"
+                    }
+                ]
             }
         }
     ],
@@ -606,6 +721,20 @@
             "summary": "Initial rule pack (unreleased; extended during Phase 8 and the security review with IndexNow, Clarity, Plausible, Cloudflare Pages and CSP provider rules). Facts checked against the cited sources on 2026-10-23 during implementation.",
             "reviewedBy": "Venkatesh A G E",
             "contentHash": "8c7272511f733c01984ce1d962b39e2d521a88b31a5c958a95307a04b82bb3dd"
+        },
+        {
+            "version": "2026.10.1",
+            "date": "2026-10-23",
+            "summary": "Add crawlers.probes (search and AI crawler user agents and tokens for vage verify's crawlability check) and the Perplexity crawler source. Full user-agent strings copied verbatim on 2026-10-23 for Googlebot, GPTBot, ChatGPT-User, PerplexityBot and Perplexity-User; OAI-SearchBot, Anthropic's crawlers and bingbot are recorded by token (Anthropic's page lists only tokens, Bing's page couldn't be read, and OAI-SearchBot's full string couldn't be copied verbatim). No existing rule changed.",
+            "reviewedBy": "Venkatesh A G E",
+            "contentHash": "4d88418eb0b5425da970a1a30218a097783e117ce65376171d9a7f3ab7d9f658"
+        },
+        {
+            "version": "2026.10.2",
+            "date": "2026-10-24",
+            "summary": "robots.ai-crawlers: add PerplexityBot (search; Perplexity says it surfaces and links sites in its search results and is not used for foundation-model training) and Applebot-Extended (training; Apple says it does not crawl and only controls use of Applebot-crawled data for training its generative models). Google-Extended stays under training; its source also covers grounding in Gemini Apps and Vertex AI, which the docs now state. Perplexity-User is excluded as a user-initiated fetcher. CCBot and cohere-ai not added: no provider documentation describing them as AI-training robots.txt tokens was found. Checked against the cited sources on 2026-10-24. cloudflare.headers: record the documented limit of 100 rules as data (maxRules), used to add a canonical Link header to each Markdown copy only when the _headers file stays within it.",
+            "reviewedBy": "Venkatesh A G E",
+            "contentHash": "6b1d977ca79843dab0f6b03dc8b3448ca451eafb037bc6af112165fb5c1035c1"
         }
     ]
 }

package/dist/index.d.ts

@@ -4,7 +4,7 @@ export { escapeHtml, html, raw, type SafeHtml } from "./core/html.js";
 export { z } from "zod";
 export type { Layout, LayoutProps } from "./core/render.js";
 export type { Frontmatter } from "./content/markdown.js";
-export { collection, type CollectionOptions, type PageSummary } from "./core/pages.js";
+export { breadcrumbs, collection, flatten, neighbors, tree, type CollectionOptions, type PageSummary, type TreeNode } from "./core/pages.js";
 export { build, type BuildResult } from "./core/build.js";
 export { startDevServer } from "./dev/server.js";
 export { startPreviewServer } from "./dev/preview.js";

package/dist/index.js

@@ -2,7 +2,7 @@ export { defineConfig } from "./core/config.js";
 export { escapeHtml, html, raw } from "./core/html.js";
 /** Zod, for layout field schemas: `export const frontmatter = z.object({ ... })`. Same instance vagejs validates with. */
 export { z } from "zod";
-export { collection } from "./core/pages.js";
+export { breadcrumbs, collection, flatten, neighbors, tree } from "./core/pages.js";
 export { build } from "./core/build.js";
 export { startDevServer } from "./dev/server.js";
 export { startPreviewServer } from "./dev/preview.js";

package/dist/seo/head.js

@@ -1,5 +1,6 @@
 import { VageError } from "../core/errors.js";
 import { escapeHtml } from "../core/html.js";
+import { feedTitle } from "../discovery/feed.js";
 import { article, jsonLdScript, organization, website } from "./jsonld.js";
 import { absoluteUrl } from "./urls.js";
 export function pageMeta(site, route, frontmatter, notFound = false) {
@@ -38,6 +39,21 @@ export function headTags(site, m) {
     if (image)
         tags.push(meta("property", "og:image", image));
     tags.push(meta("name", "twitter:card", image ? "summary_large_image" : "summary"));
+    // Markdown copy for AI tools (seo.markdownCopies), on pages that are listed.
+    if (site.seo.markdownCopies && m.canonical && m.canonical === absoluteUrl(site.site, m.route)) {
+        const href = m.route === "/" ? "/index.md" : `${m.route.slice(0, -1)}.md`;
+        tags.push({ exists: /<link[^>]+text\/markdown/i, html: `<link rel="alternate" type="text/markdown" href="${escapeHtml(href)}" />` });
+    }
+    // Feed discovery: pages in a section with a feed point readers to it.
+    for (const feed of site.site ? site.seo.feeds : []) {
+        if (!m.route.startsWith(feed.under))
+            continue;
+        const href = `${site.site}${feed.under}feed.xml`;
+        tags.push({
+            exists: /<link[^>]+application\/rss\+xml/i,
+            html: `<link rel="alternate" type="application/rss+xml" title="${escapeHtml(feedTitle(site, feed))}" href="${escapeHtml(href)}" />`,
+        });
+    }
     const schemas = structuredData(site, m, description, image);
     if (schemas.length) {
         tags.push({ exists: /<script[^>]+application\/ld\+json/i, html: jsonLdScript(schemas.length === 1 ? schemas[0] : schemas) });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vagejs",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "AI-native, static-first web framework: Markdown sites with built-in SEO, llms.txt, audits, security and safe deploys.",
   "keywords": [
     "static-site-generator",
@@ -50,6 +50,7 @@
     "jsdom": "^28.1.0",
     "marked": "^18.1.1",
     "node-html-parser": "^9.0.5",
+    "turndown": "^7.2.4",
     "vite": "^8.3.5",
     "yaml": "^2.9.1",
     "zod": "^4.6.7"
@@ -58,7 +59,9 @@
     "@anthropic-ai/sdk": "^1.1.0",
     "@types/jsdom": "^28.0.3",
     "@types/node": "^26.6.5",
+    "@types/turndown": "^5.0.6",
     "lighthouse": "^13.5.0",
+    "sharp": "^0.35.5",
     "typescript": "^7.0.2",
     "vitest": "^5.0.5"
   },

package/templates/starter/.github/workflows/ci.yml

@@ -12,6 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # full history, so seo.lastmod: "git" can date pages
       - uses: actions/setup-node@v7
         with:
           node-version: 22

package/templates/starter/.github/workflows/verify.yml

@@ -19,6 +19,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # full history, so seo.lastmod: "git" can date pages
       - uses: actions/setup-node@v7
         with:
           node-version: 22

package/templates/starter/AGENTS.md

@@ -26,7 +26,10 @@ Run `npm run check` after every change and fix all errors before finishing.
 - **Add a layout:** `src/layouts/<name>.ts`, then `layout: <name>` in frontmatter. Keep `<html lang>`, one `<title>` and `</head>`.
 - **List pages (blog index, "latest posts", a sidebar):** in a layout, `collection(pages, { under: "/blog/", sort: "date" })` from `vagejs`. Never keep a hand-written list of pages.
 - **Give a layout its own frontmatter fields:** `export const frontmatter = z.object({ ... })` in the layout (`import { z } from "vagejs"`). Pages using it are then checked strictly: unknown or mistyped fields fail the build.
-- **Add an image:** put it in `public/images/` and write `![Describe the image](/images/name.jpg)`. Width and height are filled in from the file; write real alt text.
+- **Add an image:** put it in `public/images/` and write `![Describe the image](/images/name.jpg)`. Width and height are filled in from the file; write real alt text. For resized WebP copies, the owner can turn on `images: { optimize: true }` (needs `npm install --save-dev sharp`). Only images in page content are optimized, not images a layout writes.
+- **Order a menu or sidebar:** set `order:` (lower first) and optionally a short `navTitle:` in each page's frontmatter (shown as `p.label`), then build the menu in a layout with `tree(pages, "/docs/")`. For previous/next links in the same order use `neighbors(flatten(tree(pages, "/docs/")), page.route)`. For a flat section, `collection(pages, { under: "/docs/", sort: "order", includeIndex: true })` works too. `breadcrumbs(pages, page.route)` gives the trail from the home page.
+- **Add an RSS feed for the blog:** `seo: { feeds: [{ under: "/blog/" }] }` in `vage.config.ts` writes `/blog/feed.xml`, newest `datePublished` first (undated posts last, without a date).
+- **Help AI tools read the site:** `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli/` becomes `/docs/cli.md`), converted from the page's `<main>`, so it contains exactly what the page shows (including content a layout builds from frontmatter), and `llms.txt` links to them. In layouts, add `data-vage-copy="skip"` to repeated extras such as share links or a call to action to leave them out of the copy. On Cloudflare, each copy gets a canonical `Link` header pointing to its page. `seo: { lastmod: "git" }` dates the sitemap from git history (needs full history in CI: `fetch-depth: 0`).
 - **Rename a page:** move the file, then add `redirects: { "/old-slug/": "/new-slug/" }` to `vage.config.ts`.
 - **Hide a page from search:** `noindex: true` (it stays published). To unpublish, use `draft: true` or delete it.
 - **AI crawler policy:** set `seo.robots.aiTraining` / `aiSearch` only on the owner's instruction.
@@ -61,7 +64,9 @@ image: /images/og.png    # og:image (made absolute using `site`)
 type: article            # og:type + Article JSON-LD (default: website)
 author: Jane Doe         # Article author
 datePublished: 2026-01-31
-dateModified: 2026-02-15 # also used as sitemap <lastmod>
+dateModified: 2026-02-15 # also the sitemap <lastmod> (otherwise datePublished, or git with seo.lastmod: "git")
+order: 2                 # position in menus built with tree() or sort: "order" (lower first)
+navTitle: Pricing        # shorter label for menus and breadcrumbs (PageSummary.label)
 ---
 ```
 
@@ -75,7 +80,7 @@ Hacked sites are usually abused for hidden spam links and doorway pages. vagejs
 - **`links.lock.json` lists the external domains the site may link to.** A link to any other domain fails `npm run check` (`link-unapproved`). Only the owner approves domains: show them `npx vage links` and let them run `npx vage links approve <domain>`. Never run `approve` with `--yes` or `--all` yourself unless the owner told you to in this conversation.
 - Inline `style="..."` is removed from page content (`markdown.inlineStyles` is off), and `vage check` fails on links hidden with CSS (`link-hidden`). Style pages in layouts.
 - `vage check` warns about common spam words (`spam-pattern`). Report them to the owner; don't delete or reword content you didn't write without asking.
-- `npx vage verify` compares the live site with these files and fails on links, scripts or pages that aren't in the repository. `.github/workflows/verify.yml` runs it daily once the owner sets the `VAGE_VERIFY_URL` repository variable.
+- `npx vage verify` compares the live site with these files and fails on links, scripts or pages that aren't in the repository. It also checks that search and AI crawlers can reach the site, that `http` and `www` addresses redirect, and warns about scripts the host or CDN injects. `.github/workflows/verify.yml` runs it daily once the owner sets the `VAGE_VERIFY_URL` repository variable.
 - CI runs `npm audit`, and `.github/dependabot.yml` proposes dependency updates. Review them like any other change.
 
 ## Content safety
@@ -107,13 +112,16 @@ From the published pages and config (never from `public/`; a hand-written `publi
 - `sitemap.xml`: indexable, self-canonical, non-draft pages. Needs `site`.
 - `llms.txt` (+ `llms-full.txt` with `llmsTxt: { full: true }`): same pages. An emerging convention; not a guarantee that AI tools use it.
 - Redirects: `redirects: { "/old/": "/new/" }` become noindex meta-refresh pages (dev server answers 301).
+- With `seo.feeds`: `<section>/feed.xml` (RSS). With `seo.markdownCopies`: `/<path>.md` copies of listed pages. With `images.optimize`: resized images in `/_vage/img/`.
+
 
 ## Config
 
 `vage.config.ts` keys: `site` (origin only, no path), `title`, `description`, `lang`,
-`seo: { robots, sitemap, llmsTxt, structuredData }`, `redirects`, `markdown`, `performance`,
-`deploy: { target, cloudflareWorker, cloudflareProject }`, `security: { headers, csp, cspSources }`,
-`integrations`, `guidelines`. Unknown keys are an error; `npx vage info --json` has the full schema.
+`seo: { robots, sitemap, llmsTxt, structuredData, feeds, markdownCopies, lastmod }`, `redirects`, `markdown`,
+`images: { optimize, widths, quality, sizes }`, `performance`, `deploy: { target, cloudflareWorker, cloudflareProject }`,
+`security: { headers, csp, cspSources, outboundLinks, spamPatterns }`, `integrations`, `guidelines`.
+Keys not in this list are an error; `npx vage info --json` has the full schema.
 
 ## Performance and accessibility
 
@@ -155,7 +163,7 @@ Off unless configured in `vage.config.ts` under `integrations`. Ask the site own
 
 ## Not implemented yet
 
-Bing Webmaster Tools API, Netlify/Vercel adapters, CMS content sources, multi-language sites, base paths, image resizing and WebP/AVIF conversion. Do not use or document them as available. A passing `check` does not guarantee indexing, rankings or AI citations.
+Bing Webmaster Tools API, Netlify/Vercel adapters, CMS content sources, multi-language sites, base paths, AVIF images. Do not use or document them as available. A passing `check` does not guarantee indexing, rankings or AI citations.
 
 ## Favicon
 

package/templates/starter/package.json

@@ -11,6 +11,6 @@
     "preview": "vage preview"
   },
   "devDependencies": {
-    "vagejs": "^0.2.0"
+    "vagejs": "^0.3.1"
   }
 }