wiki-formant 0.45.0 → 0.46.0

package/README.md

@@ -400,6 +400,20 @@ Every agent group also allows `AGENT_SURFACE_PATHS` — `/api/mcp`, the three `l
 
 `trackAiBot(request, event, load)` counts an `AI Bot Visit` inside `event.waitUntil`. `load` is a dynamic import because the tracker pulls in a database client, which a static import would load for every request the proxy sees.
 
+## IndexNow
+
+Bing, Yandex, Seznam, Naver and Yep take a POST of changed URLs instead of waiting to recrawl. The `indexnow` bin reads an origin's live sitemap and posts the rows whose `lastModified` falls inside the window, so the editor, the MCP write tools and the sweep scripts that write straight to the database are all covered with no hook in any of them. It runs where the sweeps run, never as a cron on the host: a submission costs the origin one cached sitemap fetch and no function time.
+
+The key is public by design, since an engine checks it by fetching `/<key>.txt`. The file in the app's `public/` is the only copy, and the bin finds it there, so run it from the app's root. It checks that the live origin serves the key before posting anything.
+
+```sh
+npx indexnow https://radix.wiki              # rows modified in the last 24 hours
+npx indexnow https://radix.wiki --hours 6
+npx indexnow https://radix.wiki --all        # every row: a new origin or a new key
+```
+
+Rows with no `lastModified` go only in a full submission. A 403 or 422 from the engine exits 1.
+
 ## Page metadata
 
 `wiki-formant/metadata`'s `pageMetadata` builds a page's canonical, markdown-twin alternate, Open Graph and Twitter card from one input. Next replaces those objects per route segment rather than merging them, and fills a missing twitter card from `openGraph` only when no `twitter` object was inherited — so under a layout that sets its own card, a page that sets one and forgets the other shows the layout's. Two wikis wrote a helper around that, covering different halves.

package/bin/indexnow.mjs

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+// bin/indexnow.mjs – post an origin's recently changed sitemap rows to IndexNow
+// (`wiki-formant/indexnow`). Run from the app's root: the key is the one file in
+// `public/` whose name is its own content plus `.txt`. The live copy is checked
+// before anything is posted, since an engine refuses every URL until it can
+// fetch the key.
+//
+//   npx indexnow https://radix.wiki              # rows modified in the last 24 hours
+//   npx indexnow https://radix.wiki --hours 6
+//   npx indexnow https://radix.wiki --all        # every row: a new origin or a new key
+import { readdirSync, readFileSync } from 'node:fs';
+import { sitemapEntries, changedUrls, submitIndexNow } from '../dist/indexnow.js';
+
+const args = process.argv.slice(2);
+const origin = args.find(a => /^https?:\/\//.test(a))?.replace(/\/$/, '');
+const hours = Number(args.includes('--hours') ? args[args.indexOf('--hours') + 1] : 24);
+
+if (!origin || !(hours > 0)) {
+  console.error('usage: indexnow <origin> [--hours n | --all]');
+  process.exit(1);
+}
+
+let files = [];
+try {
+  files = readdirSync('public');
+} catch {
+  // reported below
+}
+const key = files
+  .map(f => /^([A-Za-z0-9-]{8,128})\.txt$/.exec(f)?.[1])
+  .find(k => k && readFileSync(`public/${k}.txt`, 'utf8').trim() === k);
+if (!key) {
+  console.error('no IndexNow key in ./public: a <key>.txt holding the key');
+  process.exit(1);
+}
+
+const live = await fetch(`${origin}/${key}.txt`).then(r => (r.ok ? r.text() : ''));
+if (live.trim() !== key) {
+  console.error(`${origin}/${key}.txt does not serve the key yet; deploy first`);
+  process.exit(1);
+}
+
+const sitemap = await fetch(`${origin}/sitemap.xml`);
+if (!sitemap.ok) {
+  console.error(`${origin}/sitemap.xml answered ${sitemap.status}`);
+  process.exit(1);
+}
+const entries = sitemapEntries(await sitemap.text());
+const urls = args.includes('--all') ? entries.map(e => e.url) : changedUrls(entries, Date.now() - hours * 3_600_000);
+const statuses = await submitIndexNow(key, urls);
+console.log(JSON.stringify({ origin, submitted: urls.length, statuses }));
+if (!statuses.every(s => s === 200 || s === 202)) process.exit(1);

package/dist/indexnow.d.ts

@@ -0,0 +1,22 @@
+export declare const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
+/** The protocol's cap on the URLs in one POST. */
+export declare const INDEXNOW_BATCH = 10000;
+export interface IndexNowEntry {
+    url: string;
+    lastModified?: string;
+}
+/** The `<loc>` and `<lastmod>` of every `<url>` in a sitemap. */
+export declare function sitemapEntries(xml: string): IndexNowEntry[];
+/**
+ * The URLs modified at or after `since`. An entry with no `lastModified` is
+ * left out: nothing says it changed, and the protocol asks for changed pages
+ * only. A full submission (`--all`) is how those reach an engine.
+ */
+export declare function changedUrls(entries: IndexNowEntry[], since: number): string[];
+/**
+ * POST `urls` in batches of `INDEXNOW_BATCH` and return each batch's status.
+ * 200 is accepted and 202 accepted pending the key check; 403 means the key
+ * file is missing or disagrees, 422 that a URL is off the host. Every URL must
+ * share the first one's host, which a sitemap built from one origin does.
+ */
+export declare function submitIndexNow(key: string, urls: string[], fetchImpl?: typeof fetch): Promise<number[]>;

package/dist/indexnow.js

@@ -0,0 +1,63 @@
+// indexnow.ts — telling the IndexNow engines (Bing, Yandex, Seznam, Naver, Yep)
+// which pages changed, rather than waiting for them to recrawl.
+//
+// The `indexnow` bin reads an origin's live sitemap, keeps the rows whose
+// `lastModified` falls inside the window, and posts them. It runs on the machine
+// that runs the sweeps, not as a cron on the host: a submission costs the origin
+// one cached sitemap fetch and no function time. The sitemap is the one place
+// every writer shows up — the editor, the MCP write tools, and the sweep scripts
+// that write to the database without passing through the app — so reading it
+// covers all of them with no hook in any.
+//
+// The key is not a secret. An engine verifies a submission by fetching
+// `/<key>.txt` from the origin, so the key is public by design, and the file in
+// each app's `public/` is the only copy.
+export const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/indexnow';
+/** The protocol's cap on the URLs in one POST. */
+export const INDEXNOW_BATCH = 10_000;
+const XML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
+const unescapeXml = (s) => s.trim().replace(/&(amp|lt|gt|quot|apos);/g, (_, e) => XML_ENTITIES[e]);
+/** The `<loc>` and `<lastmod>` of every `<url>` in a sitemap. */
+export function sitemapEntries(xml) {
+    const entries = [];
+    for (const [, body = ''] of xml.matchAll(/<url>([\s\S]*?)<\/url>/g)) {
+        const loc = /<loc>([\s\S]*?)<\/loc>/.exec(body)?.[1];
+        if (!loc)
+            continue;
+        const lastmod = /<lastmod>([\s\S]*?)<\/lastmod>/.exec(body)?.[1];
+        entries.push(lastmod ? { url: unescapeXml(loc), lastModified: lastmod.trim() } : { url: unescapeXml(loc) });
+    }
+    return entries;
+}
+/**
+ * The URLs modified at or after `since`. An entry with no `lastModified` is
+ * left out: nothing says it changed, and the protocol asks for changed pages
+ * only. A full submission (`--all`) is how those reach an engine.
+ */
+export function changedUrls(entries, since) {
+    return entries
+        .filter(e => e.lastModified !== undefined && new Date(e.lastModified).getTime() >= since)
+        .map(e => e.url);
+}
+/**
+ * POST `urls` in batches of `INDEXNOW_BATCH` and return each batch's status.
+ * 200 is accepted and 202 accepted pending the key check; 403 means the key
+ * file is missing or disagrees, 422 that a URL is off the host. Every URL must
+ * share the first one's host, which a sitemap built from one origin does.
+ */
+export async function submitIndexNow(key, urls, fetchImpl = fetch) {
+    const first = urls[0];
+    if (!first)
+        return [];
+    const host = new URL(first).host;
+    const statuses = [];
+    for (let i = 0; i < urls.length; i += INDEXNOW_BATCH) {
+        const res = await fetchImpl(INDEXNOW_ENDPOINT, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json; charset=utf-8' },
+            body: JSON.stringify({ host, key, urlList: urls.slice(i, i + INDEXNOW_BATCH) }),
+        });
+        statuses.push(res.status);
+    }
+    return statuses;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-formant",
-  "version": "0.45.0",
+  "version": "0.46.0",
   "description": "The portable half of a wiki: derived taxonomy and facet controls, a version-negotiating MCP transport, markdown twins, block rendering, a rich-text editor engine, conditional-GET plumbing, and the headers and CSP every origin sends.",
   "type": "module",
   "license": "MIT",
@@ -92,6 +92,11 @@
       "import": "./dist/combobox.js",
       "default": "./dist/combobox.js"
     },
+    "./indexnow": {
+      "types": "./dist/indexnow.d.ts",
+      "import": "./dist/indexnow.js",
+      "default": "./dist/indexnow.js"
+    },
     "./crawlers": {
       "types": "./dist/crawlers.d.ts",
       "import": "./dist/crawlers.js",
@@ -240,7 +245,8 @@
     "check-pages": "bin/check-pages.mjs",
     "analytics-digest": "bin/analytics-digest.mjs",
     "passkey-invite": "bin/passkey-invite.mjs",
-    "fts-ddl": "bin/fts-ddl.mjs"
+    "fts-ddl": "bin/fts-ddl.mjs",
+    "indexnow": "bin/indexnow.mjs"
   },
   "files": [
     "dist",