vagejs 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/cli/index.js +23 -5
- package/dist/cli/info.js +1 -1
- package/dist/core/config.js +2 -2
- package/dist/core/site.js +9 -4
- package/dist/deploy/headers.d.ts +13 -2
- package/dist/deploy/headers.js +15 -3
- package/dist/dev/listen.d.ts +11 -0
- package/dist/dev/listen.js +26 -0
- package/dist/discovery/markdown-copy.d.ts +11 -1
- package/dist/discovery/markdown-copy.js +45 -5
- package/dist/guidelines/rule-pack.json +34 -4
- package/package.json +5 -5
- package/templates/starter/AGENTS.md +3 -2
- package/templates/starter/package.json +2 -1
package/README.md
CHANGED
|
@@ -29,10 +29,10 @@ Requires **Node.js 22.12 or newer**.
|
|
|
29
29
|
npm create vagejs@latest my-site
|
|
30
30
|
cd my-site
|
|
31
31
|
npm install
|
|
32
|
-
npm
|
|
32
|
+
npm start
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
`npm start` opens the site in your browser. Edit `src/pages/index.md` and the page reloads. (`npm run dev` does the same without opening a browser, and prints the address.)
|
|
36
36
|
|
|
37
37
|
## Using vagejs
|
|
38
38
|
|
|
@@ -74,7 +74,7 @@ Images in `public/` used as `` get their `width` a
|
|
|
74
74
|
|
|
75
75
|
- `order:` and `navTitle:` in frontmatter, with `tree()`, `breadcrumbs()` and `neighbors()` in layouts, build sidebars, breadcrumbs and previous/next links.
|
|
76
76
|
- `seo: { feeds: [{ under: "/blog/" }] }` writes an RSS feed at `/blog/feed.xml`.
|
|
77
|
-
- `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli.md`) for AI tools, without comments or raw HTML; `seo: { lastmod: "git" }` dates the sitemap from git history (CI needs `fetch-depth: 0`).
|
|
77
|
+
- `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli.md`) for AI tools, made from the page's `<main>`, without comments or raw HTML (layouts leave extras out with `data-vage-copy="skip"`); `seo: { lastmod: "git" }` dates the sitemap from git history (CI needs `fetch-depth: 0`).
|
|
78
78
|
|
|
79
79
|
### Write a layout
|
|
80
80
|
|
|
@@ -103,7 +103,8 @@ export default layout;
|
|
|
103
103
|
|
|
104
104
|
| Command | What it does |
|
|
105
105
|
| --- | --- |
|
|
106
|
-
| `npm
|
|
106
|
+
| `npm start` | Dev server with live reload, opened in your browser |
|
|
107
|
+
| `npm run dev` | The same, without opening a browser; prints the address (and a direct `[::1]` or `127.0.0.1` address to use if the browser shows HTTP 431) |
|
|
107
108
|
| `npm run check` | Links, SEO, structured data and basic accessibility; writes nothing |
|
|
108
109
|
| `npm run build` | Static site in `dist/`, plus `robots.txt`, `sitemap.xml`, `llms.txt` |
|
|
109
110
|
| `npm run audit` | Page-weight budgets and automated accessibility (`-- --lighthouse` adds Lighthouse) |
|
package/dist/cli/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import { VageError } from "../core/errors.js";
|
|
|
9
9
|
import { assertBranchName, assertCloudflareProject, assertVageSpec, assertWorkerName } from "../core/validate.js";
|
|
10
10
|
import { startDevServer } from "../dev/server.js";
|
|
11
11
|
import { startPreviewServer } from "../dev/preview.js";
|
|
12
|
+
import { directUrl, printUrls } from "../dev/listen.js";
|
|
12
13
|
import { createInterface } from "node:readline/promises";
|
|
13
14
|
import { createProject } from "./create.js";
|
|
14
15
|
import { frameworkInfo } from "./info.js";
|
|
@@ -30,10 +31,14 @@ cli
|
|
|
30
31
|
.option("--root <dir>", "Project root", { default: "." })
|
|
31
32
|
.option("--port <port>", "Port (fails if taken; default picks a free port from 5173)")
|
|
32
33
|
.option("--host [host]", "Expose on the network")
|
|
34
|
+
.option("--open", "Open the site in your browser (at the direct address, so localhost cookies can't cause HTTP 431)")
|
|
33
35
|
.action((opts) => run(opts, async () => {
|
|
34
36
|
const server = await startDevServer(opts.root, { port: parsePort(opts.port), host: opts.host });
|
|
35
|
-
|
|
36
|
-
server.resolvedUrls?.
|
|
37
|
+
const direct = directUrl(server.httpServer?.address());
|
|
38
|
+
printUrls("vagejs dev server", server.resolvedUrls?.local[0], direct, server.resolvedUrls?.network);
|
|
39
|
+
const target = direct ?? server.resolvedUrls?.local[0] ?? server.resolvedUrls?.network[0];
|
|
40
|
+
if (opts.open && target)
|
|
41
|
+
openBrowser(target);
|
|
37
42
|
}));
|
|
38
43
|
cli
|
|
39
44
|
.command("build", "Build the static site into dist/")
|
|
@@ -128,10 +133,14 @@ cli
|
|
|
128
133
|
.option("--root <dir>", "Project root", { default: "." })
|
|
129
134
|
.option("--port <port>", "Port (fails if taken; default picks a free port from 4173)")
|
|
130
135
|
.option("--host [host]", "Expose on the network")
|
|
136
|
+
.option("--open", "Open the site in your browser")
|
|
131
137
|
.action((opts) => run(opts, async () => {
|
|
132
138
|
const server = await startPreviewServer(opts.root, { port: parsePort(opts.port), host: opts.host });
|
|
133
|
-
|
|
134
|
-
server.resolvedUrls?.
|
|
139
|
+
const direct = directUrl(server.httpServer?.address());
|
|
140
|
+
printUrls("vagejs preview", server.resolvedUrls?.local[0], direct, server.resolvedUrls?.network);
|
|
141
|
+
const target = direct ?? server.resolvedUrls?.local[0] ?? server.resolvedUrls?.network[0];
|
|
142
|
+
if (opts.open && target)
|
|
143
|
+
openBrowser(target);
|
|
135
144
|
}));
|
|
136
145
|
cli
|
|
137
146
|
.command("info", "Describe commands, config and frontmatter (use --json for agents)")
|
|
@@ -493,7 +502,16 @@ try {
|
|
|
493
502
|
}
|
|
494
503
|
}
|
|
495
504
|
catch (err) {
|
|
496
|
-
fail(err, Boolean(cli.options.json));
|
|
505
|
+
fail(usageError(err), Boolean(cli.options.json));
|
|
506
|
+
}
|
|
507
|
+
/** Mistakes in the command line itself (unknown option, extra value) get a clear message, not a stack trace. */
|
|
508
|
+
function usageError(err) {
|
|
509
|
+
if (err?.name !== "CACError")
|
|
510
|
+
return err;
|
|
511
|
+
const name = cli.matchedCommand?.name;
|
|
512
|
+
const help = name ? `vage ${name} --help` : "vage --help";
|
|
513
|
+
const lists = /^Unused args/.test(err.message) ? ' Options that take a list want one comma-separated value, e.g. --urls "/a/,/b/".' : "";
|
|
514
|
+
return new VageError(`${err.message}.${lists} See "${help}".`);
|
|
497
515
|
}
|
|
498
516
|
async function run(opts, fn) {
|
|
499
517
|
try {
|
package/dist/cli/info.js
CHANGED
|
@@ -61,7 +61,7 @@ export function frameworkInfo(cli, version) {
|
|
|
61
61
|
},
|
|
62
62
|
features: {
|
|
63
63
|
feeds: 'seo.feeds: [{ under: "/blog/" }] writes /blog/feed.xml (RSS 2.0) and links it from that section\'s pages',
|
|
64
|
-
markdownCopies: "seo.markdownCopies: true publishes /<path>.md copies of listed pages, linked with rel=alternate type=text/markdown",
|
|
64
|
+
markdownCopies: "seo.markdownCopies: true publishes /<path>.md copies of listed pages, made from each page's <main> and linked with rel=alternate type=text/markdown; layouts mark repeated extras (share links, calls to action) with data-vage-copy=\"skip\". On Cloudflare, _headers gives each copy a canonical Link header to its page",
|
|
65
65
|
lastmod: 'seo.lastmod: "git" takes sitemap <lastmod> from each page\'s last commit (dateModified wins; skipped in shallow clones)',
|
|
66
66
|
images: 'images: { optimize: true } (needs the sharp package) makes resized WebP copies of content images in /_vage/img/ and adds srcset/sizes',
|
|
67
67
|
navigation: "frontmatter order and navTitle; helpers tree(), breadcrumbs(), neighbors(); PageSummary.label",
|
package/dist/core/config.js
CHANGED
|
@@ -31,11 +31,11 @@ const robotsSchema = z.strictObject({
|
|
|
31
31
|
aiTraining: z
|
|
32
32
|
.enum(["allow", "disallow"])
|
|
33
33
|
.optional()
|
|
34
|
-
.describe("Groups for documented AI training crawlers (GPTBot, ClaudeBot, Google-Extended). Unset: the * group applies."),
|
|
34
|
+
.describe("Groups for documented AI training crawlers (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended). Google-Extended also covers grounding in Gemini Apps and Vertex AI. Unset: the * group applies."),
|
|
35
35
|
aiSearch: z
|
|
36
36
|
.enum(["allow", "disallow"])
|
|
37
37
|
.optional()
|
|
38
|
-
.describe("Groups for documented AI search crawlers (OAI-SearchBot, Claude-SearchBot). Unset: the * group applies."),
|
|
38
|
+
.describe("Groups for documented AI search crawlers (OAI-SearchBot, Claude-SearchBot, PerplexityBot). Unset: the * group applies."),
|
|
39
39
|
});
|
|
40
40
|
const seoSchema = z.strictObject({
|
|
41
41
|
robots: toggle(robotsSchema).describe("Generate robots.txt (true/false or options)."),
|
package/dist/core/site.js
CHANGED
|
@@ -3,10 +3,10 @@ import path from "node:path";
|
|
|
3
3
|
import { llmsFullTxt, llmsTxt } from "../discovery/llms.js";
|
|
4
4
|
import { feedPath, rssFeed } from "../discovery/feed.js";
|
|
5
5
|
import { gitPageDates } from "../discovery/git-dates.js";
|
|
6
|
-
import { htmlToMarkdown, markdownCopy, markdownCopyPath } from "../discovery/markdown-copy.js";
|
|
6
|
+
import { copySource, htmlToMarkdown, markdownCopy, markdownCopyPath } from "../discovery/markdown-copy.js";
|
|
7
7
|
import { robotsTxt } from "../discovery/robots.js";
|
|
8
8
|
import { SITEMAP_URL_LIMIT, sitemapXml } from "../discovery/sitemap.js";
|
|
9
|
-
import { headersFile } from "../deploy/headers.js";
|
|
9
|
+
import { copyCanonicalsFit, headersFile } from "../deploy/headers.js";
|
|
10
10
|
import { cloudflareRedirects } from "../deploy/redirects.js";
|
|
11
11
|
import { findSecrets } from "../deploy/secrets.js";
|
|
12
12
|
import { rulePack } from "../guidelines/pack.js";
|
|
@@ -73,7 +73,7 @@ export async function renderSite(root, load, options = {}) {
|
|
|
73
73
|
const markdown = new Map();
|
|
74
74
|
if (site.seo.markdownCopies || (site.seo.llmsTxt && site.seo.llmsTxt.full)) {
|
|
75
75
|
for (const p of listedPages(model))
|
|
76
|
-
markdown.set(p.route, await htmlToMarkdown(p.content, { route: p.route, origin: site.site }));
|
|
76
|
+
markdown.set(p.route, await htmlToMarkdown(await copySource(p.html, p.content), { route: p.route, origin: site.site }));
|
|
77
77
|
}
|
|
78
78
|
model.redirects = buildRedirects(site, new Set(model.pages.map((p) => p.route)));
|
|
79
79
|
addDiscoveryFiles(root, model, markdown);
|
|
@@ -140,7 +140,12 @@ function addDiscoveryFiles(root, model, markdown) {
|
|
|
140
140
|
generate("_redirects", () => cloudflareRedirects(model.redirects));
|
|
141
141
|
if (cloudflare && site.security.headers) {
|
|
142
142
|
const html = [...model.pages, ...(model.notFound ? [model.notFound] : []), ...model.redirects].map((p) => p.html);
|
|
143
|
-
|
|
143
|
+
// Each Markdown copy names its HTML page as canonical, so search engines don't index both.
|
|
144
|
+
const copies = site.seo.markdownCopies && site.site ? listed.map((p) => ({ path: `/${markdownCopyPath(p.route)}`, canonical: p.meta.canonical })).filter((c) => /^\/[A-Za-z0-9._~\/-]+$/.test(c.path) && /^[^\s<>]+$/.test(c.canonical)) : [];
|
|
145
|
+
if (!copyCanonicalsFit(copies.length)) {
|
|
146
|
+
warnings.push(`_headers: ${copies.length} Markdown copies need more header rules than Cloudflare allows, so they're published without a canonical Link header.`);
|
|
147
|
+
}
|
|
148
|
+
generate("_headers", () => headersFile(html, site, copies));
|
|
144
149
|
}
|
|
145
150
|
if (site.seo.llmsTxt) {
|
|
146
151
|
if (!site.title) {
|
package/dist/deploy/headers.d.ts
CHANGED
|
@@ -7,5 +7,16 @@ import type { SiteConfig } from "../core/config.js";
|
|
|
7
7
|
* any https origin so ordinary sites don't break.
|
|
8
8
|
*/
|
|
9
9
|
export declare function contentSecurityPolicy(pagesHtml: string[], site: SiteConfig): string;
|
|
10
|
-
/**
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Markdown copies can name their HTML page as canonical with one rule each, while the
|
|
12
|
+
* file stays within Cloudflare's rule limit (the first rule covers every path).
|
|
13
|
+
*/
|
|
14
|
+
export declare const copyCanonicalsFit: (copies: number) => boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Cloudflare Pages / Netlify `_headers` for every path, plus a canonical Link header on
|
|
17
|
+
* each Markdown copy (`copies`: its path and its page's URL) when they fit.
|
|
18
|
+
*/
|
|
19
|
+
export declare function headersFile(pagesHtml: string[], site: SiteConfig, copies?: {
|
|
20
|
+
path: string;
|
|
21
|
+
canonical: string;
|
|
22
|
+
}[]): string;
|
package/dist/deploy/headers.js
CHANGED
|
@@ -3,7 +3,7 @@ import { parse } from "node-html-parser";
|
|
|
3
3
|
import { VageError } from "../core/errors.js";
|
|
4
4
|
import { ruleData } from "../guidelines/pack.js";
|
|
5
5
|
// https://developers.cloudflare.com/pages/configuration/headers/ (rule pack "cloudflare.headers", "csp.providers")
|
|
6
|
-
const { maxLineLength } = ruleData("cloudflare.headers");
|
|
6
|
+
const { maxLineLength, maxRules } = ruleData("cloudflare.headers");
|
|
7
7
|
const PROVIDERS = ruleData("csp.providers");
|
|
8
8
|
const EXECUTABLE = /^(|module|text\/javascript|application\/javascript)$/i;
|
|
9
9
|
/**
|
|
@@ -87,8 +87,16 @@ export function contentSecurityPolicy(pagesHtml, site) {
|
|
|
87
87
|
];
|
|
88
88
|
return directives.join("; ");
|
|
89
89
|
}
|
|
90
|
-
/**
|
|
91
|
-
|
|
90
|
+
/**
|
|
91
|
+
* Markdown copies can name their HTML page as canonical with one rule each, while the
|
|
92
|
+
* file stays within Cloudflare's rule limit (the first rule covers every path).
|
|
93
|
+
*/
|
|
94
|
+
export const copyCanonicalsFit = (copies) => copies + 1 <= maxRules;
|
|
95
|
+
/**
|
|
96
|
+
* Cloudflare Pages / Netlify `_headers` for every path, plus a canonical Link header on
|
|
97
|
+
* each Markdown copy (`copies`: its path and its page's URL) when they fit.
|
|
98
|
+
*/
|
|
99
|
+
export function headersFile(pagesHtml, site, copies = []) {
|
|
92
100
|
const lines = ["# Generated by vagejs (security in vage.config.ts)", "/*", " X-Content-Type-Options: nosniff", " Referrer-Policy: strict-origin-when-cross-origin", " X-Frame-Options: SAMEORIGIN", " Permissions-Policy: camera=(), microphone=(), geolocation=()"];
|
|
93
101
|
if (site.security.csp !== "off") {
|
|
94
102
|
const name = site.security.csp === "report-only" ? "Content-Security-Policy-Report-Only" : "Content-Security-Policy";
|
|
@@ -99,5 +107,9 @@ export function headersFile(pagesHtml, site) {
|
|
|
99
107
|
throw new VageError(`The generated Content-Security-Policy is ${tooLong.length} characters; _headers lines are limited to ${maxLineLength}. ` +
|
|
100
108
|
"Reduce distinct inline scripts/styles (move them to files in public/), or set security.csp to \"off\".");
|
|
101
109
|
}
|
|
110
|
+
if (copyCanonicalsFit(copies.length)) {
|
|
111
|
+
for (const c of copies)
|
|
112
|
+
lines.push(c.path, ` Link: <${c.canonical}>; rel="canonical"`);
|
|
113
|
+
}
|
|
102
114
|
return `${lines.join("\n")}\n`;
|
|
103
115
|
}
|
package/dist/dev/listen.d.ts
CHANGED
|
@@ -1,6 +1,17 @@
|
|
|
1
|
+
import type { AddressInfo } from "node:net";
|
|
1
2
|
export interface ServeOptions {
|
|
2
3
|
port?: number;
|
|
3
4
|
host?: string | boolean;
|
|
4
5
|
}
|
|
5
6
|
/** Runs a Vite listen() call and turns a port conflict into an actionable message. */
|
|
6
7
|
export declare function listenOrExplain<T>(listen: () => Promise<T>, port: number | undefined): Promise<T>;
|
|
8
|
+
/**
|
|
9
|
+
* The address the server is really listening on, when it's a loopback address, e.g.
|
|
10
|
+
* http://[::1]:5173/ or http://127.0.0.1:5173/. Browsers keep cookies per host name, so
|
|
11
|
+
* this address doesn't carry the cookies other projects left on "localhost" (the usual
|
|
12
|
+
* cause of HTTP 431). A server on every interface (--host) is reached at 127.0.0.1;
|
|
13
|
+
* a specific network address has no direct loopback address.
|
|
14
|
+
*/
|
|
15
|
+
export declare function directUrl(address: AddressInfo | string | null | undefined): string | undefined;
|
|
16
|
+
/** Prints where the server runs: the localhost URL, the direct address, and network URLs. */
|
|
17
|
+
export declare function printUrls(label: string, local: string | undefined, direct: string | undefined, network?: string[]): void;
|
package/dist/dev/listen.js
CHANGED
|
@@ -11,3 +11,29 @@ export async function listenOrExplain(listen, port) {
|
|
|
11
11
|
throw err;
|
|
12
12
|
}
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* The address the server is really listening on, when it's a loopback address, e.g.
|
|
16
|
+
* http://[::1]:5173/ or http://127.0.0.1:5173/. Browsers keep cookies per host name, so
|
|
17
|
+
* this address doesn't carry the cookies other projects left on "localhost" (the usual
|
|
18
|
+
* cause of HTTP 431). A server on every interface (--host) is reached at 127.0.0.1;
|
|
19
|
+
* a specific network address has no direct loopback address.
|
|
20
|
+
*/
|
|
21
|
+
export function directUrl(address) {
|
|
22
|
+
if (!address || typeof address === "string")
|
|
23
|
+
return undefined;
|
|
24
|
+
if (address.address === "::1")
|
|
25
|
+
return `http://[::1]:${address.port}/`;
|
|
26
|
+
if (address.address === "0.0.0.0" || address.address === "::")
|
|
27
|
+
return `http://127.0.0.1:${address.port}/`;
|
|
28
|
+
if (address.address.startsWith("127."))
|
|
29
|
+
return `http://${address.address}:${address.port}/`;
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
/** Prints where the server runs: the localhost URL, the direct address, and network URLs. */
|
|
33
|
+
export function printUrls(label, local, direct, network = []) {
|
|
34
|
+
console.log(`${label} running at ${local ?? direct ?? network[0]}`);
|
|
35
|
+
if (direct && direct !== local)
|
|
36
|
+
console.log(` same server: ${direct} (use this if the browser shows HTTP 431)`);
|
|
37
|
+
for (const url of network)
|
|
38
|
+
console.log(` network: ${url}`);
|
|
39
|
+
}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
/** "/" -> "index.md", "/docs/cli/" -> "docs/cli.md". */
|
|
2
2
|
export declare function markdownCopyPath(route: string): string;
|
|
3
|
+
/**
|
|
4
|
+
* The part of a rendered page the copy is made from: the inside of its one `<main>`, so
|
|
5
|
+
* content a layout builds from frontmatter (FAQ answers, post lists) is included. Pages
|
|
6
|
+
* without exactly one `<main>` fall back to the Markdown body.
|
|
7
|
+
*/
|
|
8
|
+
export declare function copySource(pageHtml: string, content: string): Promise<string>;
|
|
3
9
|
/**
|
|
4
10
|
* The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
|
|
5
11
|
* exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
|
|
@@ -9,5 +15,9 @@ export declare function htmlToMarkdown(html: string, options: {
|
|
|
9
15
|
route: string;
|
|
10
16
|
origin?: string;
|
|
11
17
|
}): Promise<string>;
|
|
12
|
-
/**
|
|
18
|
+
/**
|
|
19
|
+
* The copy: the page's own H1 (or the title), the summary, then the content. The H1 is
|
|
20
|
+
* moved to the top even when a layout puts something before it, but never taken from
|
|
21
|
+
* inside a code block.
|
|
22
|
+
*/
|
|
13
23
|
export declare function markdownCopy(title: string, description: string | undefined, markdown: string): string;
|
|
@@ -58,6 +58,17 @@ function converter() {
|
|
|
58
58
|
service = td;
|
|
59
59
|
return td;
|
|
60
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* The part of a rendered page the copy is made from: the inside of its one `<main>`, so
|
|
63
|
+
* content a layout builds from frontmatter (FAQ answers, post lists) is included. Pages
|
|
64
|
+
* without exactly one `<main>` fall back to the Markdown body.
|
|
65
|
+
*/
|
|
66
|
+
export async function copySource(pageHtml, content) {
|
|
67
|
+
jsdom ??= await import("jsdom");
|
|
68
|
+
// Parsed as HTML (scripts don't run), so "<main>" text in scripts, comments or attributes doesn't count.
|
|
69
|
+
const mains = new jsdom.JSDOM(pageHtml).window.document.querySelectorAll("main");
|
|
70
|
+
return mains.length === 1 ? mains[0].innerHTML : content;
|
|
71
|
+
}
|
|
61
72
|
/**
|
|
62
73
|
* The page as Markdown, converted from its rendered, sanitized HTML: the copy contains
|
|
63
74
|
* exactly what the page shows (no comments, no raw HTML the sanitizer removed, the same
|
|
@@ -81,8 +92,13 @@ export async function htmlToMarkdown(html, options) {
|
|
|
81
92
|
}
|
|
82
93
|
};
|
|
83
94
|
// Hidden content isn't on the page for visitors, so it isn't in the copy either.
|
|
84
|
-
|
|
95
|
+
// Layouts mark repeated extras (share links, calls to action) with data-vage-copy="skip";
|
|
96
|
+
// aria-hidden parts are decoration that screen readers skip too.
|
|
97
|
+
for (const el of template.content.querySelectorAll('[hidden], svg, math, [aria-hidden="true"], [data-vage-copy="skip"]'))
|
|
85
98
|
el.remove();
|
|
99
|
+
// A heading split over two lines with <br> stays one Markdown heading.
|
|
100
|
+
for (const br of template.content.querySelectorAll("h1 br, h2 br, h3 br, h4 br, h5 br, h6 br"))
|
|
101
|
+
br.replaceWith(" ");
|
|
86
102
|
for (const a of template.content.querySelectorAll("a[href]")) {
|
|
87
103
|
const href = a.getAttribute("href").trim();
|
|
88
104
|
// Script and data URLs never become links in the copy (trusted HTML can contain them).
|
|
@@ -103,10 +119,34 @@ export async function htmlToMarkdown(html, options) {
|
|
|
103
119
|
wrapper.append(template.content);
|
|
104
120
|
return converter().turndown(wrapper).trim();
|
|
105
121
|
}
|
|
106
|
-
/**
|
|
122
|
+
/**
|
|
123
|
+
* The copy: the page's own H1 (or the title), the summary, then the content. The H1 is
|
|
124
|
+
* moved to the top even when a layout puts something before it, but never taken from
|
|
125
|
+
* inside a code block.
|
|
126
|
+
*/
|
|
107
127
|
export function markdownCopy(title, description, markdown) {
|
|
108
|
-
const
|
|
109
|
-
|
|
110
|
-
|
|
128
|
+
const lines = markdown.split("\n");
|
|
129
|
+
let fence = "";
|
|
130
|
+
let at = -1;
|
|
131
|
+
for (let i = 0; i < lines.length; i++) {
|
|
132
|
+
const line = lines[i];
|
|
133
|
+
const run = /^(`{3,}|~{3,})/.exec(line)?.[1];
|
|
134
|
+
if (fence) {
|
|
135
|
+
if (run && run[0] === fence[0] && run.length >= fence.length && !line.slice(run.length).trim())
|
|
136
|
+
fence = "";
|
|
137
|
+
}
|
|
138
|
+
else if (run) {
|
|
139
|
+
fence = run;
|
|
140
|
+
}
|
|
141
|
+
else if (/^#[ \t]+\S/.test(line)) {
|
|
142
|
+
at = i;
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
const heading = `# ${at >= 0 ? lines[at].replace(/^#[ \t]+/, "").trim() : title}`;
|
|
147
|
+
let next = at + 1;
|
|
148
|
+
while (at >= 0 && next < lines.length && !lines[next].trim())
|
|
149
|
+
next++; // and the blank lines after it
|
|
150
|
+
const rest = at >= 0 ? [...lines.slice(0, at), ...lines.slice(next)].join("\n") : markdown;
|
|
111
151
|
return `${heading}\n\n${description ? `> ${description}\n\n` : ""}${rest.trim()}\n`;
|
|
112
152
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "2026.10.
|
|
3
|
-
"released": "2026-10-
|
|
2
|
+
"version": "2026.10.2",
|
|
3
|
+
"released": "2026-10-24",
|
|
4
4
|
"sources": [
|
|
5
5
|
{
|
|
6
6
|
"id": "google-search-essentials",
|
|
@@ -274,6 +274,14 @@
|
|
|
274
274
|
"url": "https://docs.perplexity.ai/guides/bots",
|
|
275
275
|
"category": "ai-crawlers",
|
|
276
276
|
"cadence": "weekly"
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
"id": "apple-applebot",
|
|
280
|
+
"provider": "Apple",
|
|
281
|
+
"title": "About Applebot (incl. Applebot-Extended)",
|
|
282
|
+
"url": "https://support.apple.com/en-us/119829",
|
|
283
|
+
"category": "ai-crawlers",
|
|
284
|
+
"cadence": "weekly"
|
|
277
285
|
}
|
|
278
286
|
],
|
|
279
287
|
"rules": [
|
|
@@ -284,7 +292,9 @@
|
|
|
284
292
|
"sources": [
|
|
285
293
|
"openai-bots",
|
|
286
294
|
"anthropic-crawlers",
|
|
287
|
-
"google-crawlers"
|
|
295
|
+
"google-crawlers",
|
|
296
|
+
"perplexity-bots",
|
|
297
|
+
"apple-applebot"
|
|
288
298
|
],
|
|
289
299
|
"implementedIn": [
|
|
290
300
|
"src/discovery/crawlers.ts",
|
|
@@ -321,6 +331,18 @@
|
|
|
321
331
|
"provider": "Google",
|
|
322
332
|
"purpose": "training",
|
|
323
333
|
"source": "google-crawlers"
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
"token": "PerplexityBot",
|
|
337
|
+
"provider": "Perplexity",
|
|
338
|
+
"purpose": "search",
|
|
339
|
+
"source": "perplexity-bots"
|
|
340
|
+
},
|
|
341
|
+
{
|
|
342
|
+
"token": "Applebot-Extended",
|
|
343
|
+
"provider": "Apple",
|
|
344
|
+
"purpose": "training",
|
|
345
|
+
"source": "apple-applebot"
|
|
324
346
|
}
|
|
325
347
|
]
|
|
326
348
|
}
|
|
@@ -603,7 +625,8 @@
|
|
|
603
625
|
"src/deploy/headers.ts"
|
|
604
626
|
],
|
|
605
627
|
"data": {
|
|
606
|
-
"maxLineLength": 2000
|
|
628
|
+
"maxLineLength": 2000,
|
|
629
|
+
"maxRules": 100
|
|
607
630
|
}
|
|
608
631
|
},
|
|
609
632
|
{
|
|
@@ -705,6 +728,13 @@
|
|
|
705
728
|
"summary": "Add crawlers.probes (search and AI crawler user agents and tokens for vage verify's crawlability check) and the Perplexity crawler source. Full user-agent strings copied verbatim on 2026-10-23 for Googlebot, GPTBot, ChatGPT-User, PerplexityBot and Perplexity-User; OAI-SearchBot, Anthropic's crawlers and bingbot are recorded by token (Anthropic's page lists only tokens, Bing's page couldn't be read, and OAI-SearchBot's full string couldn't be copied verbatim). No existing rule changed.",
|
|
706
729
|
"reviewedBy": "Venkatesh A G E",
|
|
707
730
|
"contentHash": "4d88418eb0b5425da970a1a30218a097783e117ce65376171d9a7f3ab7d9f658"
|
|
731
|
+
},
|
|
732
|
+
{
|
|
733
|
+
"version": "2026.10.2",
|
|
734
|
+
"date": "2026-10-24",
|
|
735
|
+
"summary": "robots.ai-crawlers: add PerplexityBot (search; Perplexity says it surfaces and links sites in its search results and is not used for foundation-model training) and Applebot-Extended (training; Apple says it does not crawl and only controls use of Applebot-crawled data for training its generative models). Google-Extended stays under training; its source also covers grounding in Gemini Apps and Vertex AI, which the docs now state. Perplexity-User is excluded as a user-initiated fetcher. CCBot and cohere-ai not added: no provider documentation describing them as AI-training robots.txt tokens was found. Checked against the cited sources on 2026-10-24. cloudflare.headers: record the documented limit of 100 rules as data (maxRules), used to add a canonical Link header to each Markdown copy only when the _headers file stays within it.",
|
|
736
|
+
"reviewedBy": "Venkatesh A G E",
|
|
737
|
+
"contentHash": "6b1d977ca79843dab0f6b03dc8b3448ca451eafb037bc6af112165fb5c1035c1"
|
|
708
738
|
}
|
|
709
739
|
]
|
|
710
740
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vagejs",
|
|
3
|
-
"version": "0.3.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.3.2",
|
|
4
|
+
"description": "A free static site generator for AI website development: Markdown pages with built-in SEO, llms.txt, checks, spam protection and safe deploys.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"static-site-generator",
|
|
7
7
|
"ssg",
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
"seo",
|
|
10
10
|
"llms-txt",
|
|
11
11
|
"ai-agents",
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
"
|
|
12
|
+
"agents-md",
|
|
13
|
+
"claude-code",
|
|
14
|
+
"cli"
|
|
15
15
|
],
|
|
16
16
|
"homepage": "https://vagejs.com",
|
|
17
17
|
"author": "vagejs.com",
|
|
@@ -6,7 +6,8 @@ This website uses **vagejs** (pre-release). Only the features listed here exist
|
|
|
6
6
|
|
|
7
7
|
| Command | What it does |
|
|
8
8
|
| ----------------- | --------------------------------------------------------- |
|
|
9
|
-
| `npm run dev` | Dev server with live reload. Prints the local URL.
|
|
9
|
+
| `npm run dev` | Dev server with live reload. Prints the local URL and the same server's direct address (e.g. `http://[::1]:5173/`): if a browser shows HTTP 431 on localhost (too many cookies from other projects), use the direct address. Use this one yourself; it doesn't open a browser. |
|
|
10
|
+
| `npm start` | For people: the dev server, opened in the default browser at the direct address. |
|
|
10
11
|
| `npm run build` | Static build into `dist/`. Exits non-zero on any error. |
|
|
11
12
|
| `npm run check` | SEO, link, structured-data and basic a11y checks. Writes nothing. Exit 1 on errors (`--strict`: also on warnings). |
|
|
12
13
|
| `npm run audit` | Builds, then checks page-weight budgets and runs axe accessibility rules. `--lighthouse` adds a local Lighthouse run (install `lighthouse`; needs Chrome). `--url <https://...>` audits a deployed page via PageSpeed Insights (+ CrUX field data with `CRUX_API_KEY`). |
|
|
@@ -29,7 +30,7 @@ Run `npm run check` after every change and fix all errors before finishing.
|
|
|
29
30
|
- **Add an image:** put it in `public/images/` and write ``. Width and height are filled in from the file; write real alt text. For resized WebP copies, the owner can turn on `images: { optimize: true }` (needs `npm install --save-dev sharp`). Only images in page content are optimized, not images a layout writes.
|
|
30
31
|
- **Order a menu or sidebar:** set `order:` (lower first) and optionally a short `navTitle:` in each page's frontmatter (shown as `p.label`), then build the menu in a layout with `tree(pages, "/docs/")`. For previous/next links in the same order use `neighbors(flatten(tree(pages, "/docs/")), page.route)`. For a flat section, `collection(pages, { under: "/docs/", sort: "order", includeIndex: true })` works too. `breadcrumbs(pages, page.route)` gives the trail from the home page.
|
|
31
32
|
- **Add an RSS feed for the blog:** `seo: { feeds: [{ under: "/blog/" }] }` in `vage.config.ts` writes `/blog/feed.xml`, newest `datePublished` first (undated posts last, without a date).
|
|
32
|
-
- **Help AI tools read the site:** `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli/` becomes `/docs/cli.md`), converted from the
|
|
33
|
+
- **Help AI tools read the site:** `seo: { markdownCopies: true }` publishes a Markdown copy of every listed page (`/docs/cli/` becomes `/docs/cli.md`), converted from the page's `<main>`, so it contains exactly what the page shows (including content a layout builds from frontmatter), and `llms.txt` links to them. In layouts, add `data-vage-copy="skip"` to repeated extras such as share links or a call to action to leave them out of the copy. On Cloudflare, each copy gets a canonical `Link` header pointing to its page. `seo: { lastmod: "git" }` dates the sitemap from git history (needs full history in CI: `fetch-depth: 0`).
|
|
33
34
|
- **Rename a page:** move the file, then add `redirects: { "/old-slug/": "/new-slug/" }` to `vage.config.ts`.
|
|
34
35
|
- **Hide a page from search:** `noindex: true` (it stays published). To unpublish, use `draft: true` or delete it.
|
|
35
36
|
- **AI crawler policy:** set `seo.robots.aiTraining` / `aiSearch` only on the owner's instruction.
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
"private": true,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
7
|
+
"start": "vage dev --open",
|
|
7
8
|
"dev": "vage dev",
|
|
8
9
|
"build": "vage build",
|
|
9
10
|
"check": "vage check",
|
|
@@ -11,6 +12,6 @@
|
|
|
11
12
|
"preview": "vage preview"
|
|
12
13
|
},
|
|
13
14
|
"devDependencies": {
|
|
14
|
-
"vagejs": "^0.3.
|
|
15
|
+
"vagejs": "^0.3.2"
|
|
15
16
|
}
|
|
16
17
|
}
|