vagejs 0.0.0-stage → 0.2.0

package/dist/discovery/robots.js

@@ -0,0 +1,40 @@
+import { VageError } from "../core/errors.js";
+import { AI_CRAWLERS } from "./crawlers.js";
+/**
+ * robots.txt from explicit config only. Unset AI options add nothing, so the
+ * owner's `*` policy applies. robots.txt is advisory and is not access control.
+ */
+export function robotsTxt(site, options) {
+    const groups = options.rules.map((r) => ({
+        agents: Array.isArray(r.userAgent) ? r.userAgent : [r.userAgent],
+        allow: r.allow,
+        disallow: r.disallow,
+    }));
+    const named = new Set(groups.flatMap((g) => g.agents.map((a) => a.toLowerCase())));
+    for (const [purpose, policy] of [
+        ["training", options.aiTraining],
+        ["search", options.aiSearch],
+    ]) {
+        if (!policy)
+            continue;
+        for (const crawler of AI_CRAWLERS.filter((c) => c.purpose === purpose)) {
+            if (named.has(crawler.token.toLowerCase())) {
+                throw new VageError(`robots: "${crawler.token}" is set both in seo.robots.rules and by seo.robots.ai${purpose === "training" ? "Training" : "Search"}. Keep only one.`);
+            }
+            groups.push({ agents: [crawler.token], allow: policy === "allow" ? ["/"] : [], disallow: policy === "disallow" ? ["/"] : [] });
+        }
+    }
+    if (!named.has("*"))
+        groups.push({ agents: ["*"], allow: [], disallow: [] });
+    const body = groups
+        .map((g) => {
+        const lines = g.agents.map((a) => `User-agent: ${a}`);
+        lines.push(...g.allow.map((p) => `Allow: ${p}`), ...g.disallow.map((p) => `Disallow: ${p}`));
+        if (!g.allow.length && !g.disallow.length)
+            lines.push("Disallow:");
+        return lines.join("\n");
+    })
+        .join("\n\n");
+    const sitemap = site.seo.sitemap && site.site ? `\n\nSitemap: ${site.site}/sitemap.xml` : "";
+    return `${body}${sitemap}\n`;
+}

package/dist/discovery/sitemap.d.ts

@@ -0,0 +1,8 @@
+export interface SitemapEntry {
+    url: string;
+    /** Only set from frontmatter dates; file timestamps are not trustworthy after a git checkout. */
+    lastmod?: string;
+}
+/** A single urlset. The 50,000-URL limit per file is enforced by the caller. */
+export declare function sitemapXml(entries: SitemapEntry[]): string;
+export declare const SITEMAP_URL_LIMIT: number;

package/dist/discovery/sitemap.js

@@ -0,0 +1,11 @@
+import { ruleData } from "../guidelines/pack.js";
+const XML_ESCAPES = { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&apos;" };
+const xml = (s) => s.replace(/[&<>"']/g, (c) => XML_ESCAPES[c]);
+/** A single urlset. The 50,000-URL limit per file is enforced by the caller. */
+export function sitemapXml(entries) {
+    const urls = entries
+        .map((e) => `  <url>\n    <loc>${xml(e.url)}</loc>${e.lastmod ? `\n    <lastmod>${xml(e.lastmod)}</lastmod>` : ""}\n  </url>`)
+        .join("\n");
+    return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`;
+}
+export const SITEMAP_URL_LIMIT = ruleData("sitemap.limits").maxUrls;

package/dist/guidelines/monitor.d.ts

@@ -0,0 +1,55 @@
+import type { RulePack, Source } from "./pack.js";
+/**
+ * Change detection for official sources. Snapshots keep only section headings
+ * and hashes, not the documents' text, so nothing copyrighted is stored; the
+ * live page is the evidence. Reports are inputs to human review, never
+ * automatic rule changes.
+ */
+export interface SourceSnapshot {
+    id: string;
+    url: string;
+    fetchedAt: string;
+    status: number;
+    /** Final URL after redirects. */
+    finalUrl: string;
+    lastModified?: string;
+    contentHash: string;
+    sections: {
+        heading: string;
+        hash: string;
+    }[];
+}
+export interface SourceChange {
+    source: Source;
+    kind: "changed" | "unreachable" | "unreadable" | "moved" | "new";
+    added: string[];
+    removed: string[];
+    modified: string[];
+    detail?: string;
+}
+/** Splits the main content into sections by h1–h3 and hashes each section's normalized text. */
+export declare function sectionize(html: string): {
+    heading: string;
+    text: string;
+}[];
+/** Fetches a source. `texts` (heading -> current text) is for classification only and is never persisted. */
+export declare function fetchSource(source: Source, fetchImpl?: typeof fetch, now?: Date): Promise<{
+    snapshot: SourceSnapshot;
+    texts: Map<string, string>;
+}>;
+export declare function compareSnapshots(source: Source, before: SourceSnapshot | undefined, after: SourceSnapshot): SourceChange | undefined;
+export interface Classification {
+    sourceId: string;
+    section: string;
+    classification: "informative" | "recommended" | "required" | "deprecated" | "unclear";
+    rationale: string;
+    suggestedAction: string;
+}
+/** Markdown change report: what changed, which rules depend on it, and what needs human review. */
+export declare function changeReport(pack: RulePack, changes: SourceChange[], date: string, classifications?: Classification[]): string;
+/**
+ * Fetched headings and LLM output are untrusted: shown as inline code so they
+ * can't become links, images, HTML or @-mentions in the issue.
+ */
+export declare function untrusted(text: string, max?: number): string;
+export declare function affectedRules(pack: RulePack, sourceId: string): import("./pack.js").Rule[];

package/dist/guidelines/monitor.js

@@ -0,0 +1,126 @@
+import { createHash } from "node:crypto";
+import { parse } from "node-html-parser";
+const NOISE = "script, style, noscript, nav, header, footer, aside, form, button, svg, [aria-hidden=true], .devsite-article-meta, .devsite-feedback";
+/** Splits the main content into sections by h1–h3 and hashes each section's normalized text. */
+export function sectionize(html) {
+    const doc = parse(html);
+    const main = doc.querySelector("main article") ?? doc.querySelector("article") ?? doc.querySelector("main") ?? doc.querySelector("[role=main]") ?? doc.querySelector("body") ?? doc;
+    for (const el of main.querySelectorAll(NOISE))
+        el.remove();
+    const sections = [{ heading: "(intro)", text: [] }];
+    const walk = (node) => {
+        for (const child of node.childNodes) {
+            if (child.nodeType === 3) {
+                sections.at(-1).text.push(child.rawText);
+                continue;
+            }
+            const el = child;
+            if (/^h[1-3]$/i.test(el.tagName ?? ""))
+                sections.push({ heading: normalize(el.text), text: [] });
+            else if (el.tagName)
+                walk(el);
+        }
+    };
+    walk(main);
+    return sections
+        .map((s) => ({ heading: s.heading, text: normalize(decode(s.text.join(" "))) }))
+        .filter((s) => s.text || s.heading !== "(intro)");
+}
+/** Fetches a source. `texts` (heading -> current text) is for classification only and is never persisted. */
+export async function fetchSource(source, fetchImpl = fetch, now = new Date()) {
+    const res = await fetchImpl(source.url, {
+        redirect: "follow",
+        // English, so localized variants don't show up as changes.
+        headers: { "user-agent": "vagejs-guidelines-monitor/0.1", accept: "text/html", "accept-language": "en-US,en;q=0.9" },
+        signal: AbortSignal.timeout(60_000),
+    });
+    const html = res.ok ? await res.text() : "";
+    const parts = sectionize(html);
+    const sections = parts.map((s) => ({ heading: s.heading, hash: hash(`${s.heading}\n${s.text}`) }));
+    const snapshot = {
+        id: source.id,
+        url: source.url,
+        fetchedAt: now.toISOString(),
+        status: res.status,
+        finalUrl: res.url || source.url,
+        lastModified: res.headers.get("last-modified") ?? undefined,
+        contentHash: hash(sections.map((s) => s.hash).join("")),
+        sections,
+    };
+    return { snapshot, texts: new Map(parts.map((p) => [p.heading, p.text])) };
+}
+export function compareSnapshots(source, before, after) {
+    const base = { source, added: [], removed: [], modified: [] };
+    if (after.status < 200 || after.status >= 300)
+        return { ...base, kind: "unreachable", detail: `HTTP ${after.status}` };
+    if (!after.sections.length) {
+        return { ...base, kind: "unreadable", detail: "no readable text (probably rendered client-side); check manually or mark the source monitor: manual" };
+    }
+    if (!before)
+        return { ...base, kind: "new", added: after.sections.map((s) => s.heading) };
+    if (new URL(after.finalUrl).pathname !== new URL(before.finalUrl).pathname) {
+        return { ...base, kind: "moved", detail: `${before.finalUrl} → ${after.finalUrl}` };
+    }
+    if (before.contentHash === after.contentHash)
+        return undefined;
+    const old = new Map(before.sections.map((s) => [s.heading, s.hash]));
+    const now = new Map(after.sections.map((s) => [s.heading, s.hash]));
+    return {
+        ...base,
+        kind: "changed",
+        added: [...now.keys()].filter((h) => !old.has(h)),
+        removed: [...old.keys()].filter((h) => !now.has(h)),
+        modified: [...now.keys()].filter((h) => old.has(h) && old.get(h) !== now.get(h)),
+    };
+}
+/** Markdown change report: what changed, which rules depend on it, and what needs human review. */
+export function changeReport(pack, changes, date, classifications = []) {
+    const lines = [`# Guideline source changes — ${date}`, "", `Rule pack in use: ${pack.version}. Snapshots store headings and hashes only; open each link to read the current text.`, ""];
+    const manual = pack.sources.filter((s) => s.monitor === "manual");
+    if (manual.length)
+        lines.push(`Not monitored automatically (check by hand): ${manual.map((s) => `[${s.title}](${s.url})`).join(", ")}.`, "");
+    if (!changes.length)
+        return `${lines.join("\n")}No changes detected.\n`;
+    const review = changes.some((c) => affectedRules(pack, c.source.id).some((r) => r.behaviorChanging));
+    lines.push(review
+        ? "**Human review required:** at least one change affects a rule that changes crawler access, indexing, privacy, security or build output."
+        : "No behavior-changing rules are affected; review is still recommended.", "");
+    for (const c of changes) {
+        lines.push(`## ${c.source.title} (${c.source.provider})`, "", `- Source: ${c.source.url}`, `- Change: ${c.kind}${c.detail ? ` (${c.detail})` : ""}`);
+        for (const [label, list] of [["Added sections", c.added], ["Removed sections", c.removed], ["Modified sections", c.modified]]) {
+            if (list.length)
+                lines.push(`- ${label}: ${list.map((h) => untrusted(h, 120)).join(", ")}`);
+        }
+        const rules = affectedRules(pack, c.source.id);
+        lines.push(`- Affected rules: ${rules.length ? rules.map((r) => `\`${r.id}\`${r.behaviorChanging ? " (changes output)" : ""}`).join(", ") : "none (tracked only)"}`);
+        const drafts = classifications.filter((x) => x.sourceId === c.source.id);
+        if (drafts.length) {
+            lines.push("- Draft classification (LLM, not authoritative):");
+            for (const d of drafts)
+                lines.push(`  - ${untrusted(d.section, 120)}: **${d.classification}**: ${untrusted(d.rationale)} Suggested: ${untrusted(d.suggestedAction)}`);
+        }
+        lines.push("");
+    }
+    lines.push("## Next steps", "", "1. Read each changed source.", "2. If a rule must change, edit `src/guidelines/rule-pack.json`, bump `version`, add a changelog entry (`reviewedBy`, new `contentHash`).", "3. Run `npm test` (golden outputs show the effect) and release.", "4. Then run the monitor with `--update` to accept the new snapshots.", "");
+    return lines.join("\n");
+}
+/**
+ * Fetched headings and LLM output are untrusted: shown as inline code so they
+ * can't become links, images, HTML or @-mentions in the issue.
+ */
+export function untrusted(text, max = 300) {
+    const one = text.replace(/\s+/g, " ").replace(/`/g, "'").trim();
+    return `\`${one.length > max ? `${one.slice(0, max)}…` : one}\``;
+}
+export function affectedRules(pack, sourceId) {
+    return pack.rules.filter((r) => r.sources.includes(sourceId));
+}
+function hash(text) {
+    return createHash("sha256").update(text).digest("hex").slice(0, 16);
+}
+function normalize(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+function decode(text) {
+    return text.replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+}

package/dist/guidelines/pack.d.ts

@@ -0,0 +1,41 @@
+/**
+ * The rule pack: provider facts the framework relies on, each tied to official
+ * sources. Changing it is a reviewed release (see docs/GUIDELINES.md); the
+ * monitor only reports source changes and never edits it.
+ */
+export interface Source {
+    id: string;
+    provider: string;
+    title: string;
+    url: string;
+    category: string;
+    cadence: "daily" | "weekly";
+    /** "manual": the page can't be read automatically (e.g. client-rendered). */
+    monitor?: "manual";
+}
+export interface Rule {
+    id: string;
+    summary: string;
+    behaviorChanging: boolean;
+    sources: string[];
+    implementedIn: string[];
+    data?: Record<string, unknown>;
+}
+export interface ChangelogEntry {
+    version: string;
+    date: string;
+    summary: string;
+    reviewedBy: string;
+    contentHash: string;
+}
+export interface RulePack {
+    version: string;
+    released: string;
+    sources: Source[];
+    rules: Rule[];
+    changelog: ChangelogEntry[];
+}
+export declare const rulePack: RulePack;
+export declare function ruleData<T>(id: string): T;
+/** Hash of sources + rules (not the changelog), recorded in each changelog entry. */
+export declare function packContentHash(pack?: RulePack): string;

package/dist/guidelines/pack.js

@@ -0,0 +1,13 @@
+import { createHash } from "node:crypto";
+import raw from "./rule-pack.json" with { type: "json" };
+export const rulePack = raw;
+export function ruleData(id) {
+    const rule = rulePack.rules.find((r) => r.id === id);
+    if (!rule?.data)
+        throw new Error(`Rule pack ${rulePack.version} has no data for rule "${id}".`);
+    return rule.data;
+}
+/** Hash of sources + rules (not the changelog), recorded in each changelog entry. */
+export function packContentHash(pack = rulePack) {
+    return createHash("sha256").update(JSON.stringify({ sources: pack.sources, rules: pack.rules })).digest("hex");
+}