wiki-formant 0.45.0 → 0.47.0

package/README.md

@@ -400,6 +400,20 @@ Every agent group also allows `AGENT_SURFACE_PATHS` — `/api/mcp`, the three `l
 
 `trackAiBot(request, event, load)` counts an `AI Bot Visit` inside `event.waitUntil`. `load` is a dynamic import because the tracker pulls in a database client, which a static import would load for every request the proxy sees.
 
+## IndexNow
+
+Bing, Yandex, Seznam, Naver and Yep take a POST of changed URLs instead of waiting to recrawl. The `indexnow` bin reads an origin's live sitemap and posts the rows whose `lastModified` falls inside the window, so the editor, the MCP write tools and the sweep scripts that write straight to the database are all covered with no hook in any of them. It runs where the sweeps run, never as a cron on the host: a submission costs the origin one cached sitemap fetch and no function time.
+
+The key is public by design, since an engine checks it by fetching `/<key>.txt`. The file in the app's `public/` is the only copy, and the bin finds it there, so run it from the app's root. It checks that the live origin serves the key before posting anything.
+
+```sh
+npx indexnow https://radix.wiki              # rows modified in the last 24 hours
+npx indexnow https://radix.wiki --hours 6
+npx indexnow https://radix.wiki --all        # every row: a new origin or a new key
+```
+
+Rows with no `lastModified` go only in a full submission. A 403 or 422 from the engine exits 1.
+
 ## Page metadata
 
 `wiki-formant/metadata`'s `pageMetadata` builds a page's canonical, markdown-twin alternate, Open Graph and Twitter card from one input. Next replaces those objects per route segment rather than merging them, and fills a missing twitter card from `openGraph` only when no `twitter` object was inherited — so under a layout that sets its own card, a page that sets one and forgets the other shows the layout's. Two wikis wrote a helper around that, covering different halves.
@@ -417,6 +431,13 @@ The schema.org nodes sit beside it, because they state the same facts to a diffe
 <JsonLd data={articleLd({ headline: title, url, image, published, modified, publisher: { '@id': `${SITE_URL}/#organization` }, citation: citationsFromReferences(refs) })} />
 ```
 
+`questionPairs(html)` reads a page's FAQ out of its own headings: every `h2`/`h3` ending in `?`, answered by the text beneath it. It never rewrites a statement heading as a question, so a page earns `FAQPage` by being written as one. `faqPageLd(pairs, url)` is that node, or null under two pairs, and takes an authored list just as well.
+
+```tsx
+const faq = faqPageLd(questionPairs(html), url);
+{faq && <JsonLd data={faq} />}
+```
+
 `sitePageMetadata({ siteName, handle })` binds the three fields Next drops from every page that sets its own card, with `locale` defaulting to `en_US`. `siteGraphLd` is the Organization, WebSite (with its `SearchAction`) and `WebAPI` as one `@graph` for the root layout, every node carrying an `@id`, and `siteRefs(url)` hands out the same ids for a page's `publisher` and `isPartOf` — so no page writes an inline copy of the publisher that can disagree with the layout's.
 
 ```ts

package/bin/indexnow.mjs

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+// bin/indexnow.mjs – post an origin's recently changed sitemap rows to IndexNow
+// (`wiki-formant/indexnow`). Run from the app's root: the key is the one file in
+// `public/` whose name is its own content plus `.txt`. The live copy is checked
+// before anything is posted, since an engine refuses every URL until it can
+// fetch the key.
+//
+//   npx indexnow https://radix.wiki              # rows modified in the last 24 hours
+//   npx indexnow https://radix.wiki --hours 6
+//   npx indexnow https://radix.wiki --all        # every row: a new origin or a new key
+import { readdirSync, readFileSync } from 'node:fs';
+import { sitemapEntries, changedUrls, submitIndexNow } from '../dist/indexnow.js';
+
+const args = process.argv.slice(2);
+const origin = args.find(a => /^https?:\/\//.test(a))?.replace(/\/$/, '');
+const hours = Number(args.includes('--hours') ? args[args.indexOf('--hours') + 1] : 24);
+
+if (!origin || !(hours > 0)) {
+  console.error('usage: indexnow <origin> [--hours n | --all]');
+  process.exit(1);
+}
+
+let files = [];
+try {
+  files = readdirSync('public');
+} catch {
+  // reported below
+}
+const key = files
+  .map(f => /^([A-Za-z0-9-]{8,128})\.txt$/.exec(f)?.[1])
+  .find(k => k && readFileSync(`public/${k}.txt`, 'utf8').trim() === k);
+if (!key) {
+  console.error('no IndexNow key in ./public: a <key>.txt holding the key');
+  process.exit(1);
+}
+
+const live = await fetch(`${origin}/${key}.txt`).then(r => (r.ok ? r.text() : ''));
+if (live.trim() !== key) {
+  console.error(`${origin}/${key}.txt does not serve the key yet; deploy first`);
+  process.exit(1);
+}
+
+const sitemap = await fetch(`${origin}/sitemap.xml`);
+if (!sitemap.ok) {
+  console.error(`${origin}/sitemap.xml answered ${sitemap.status}`);
+  process.exit(1);
+}
+const entries = sitemapEntries(await sitemap.text());
+const urls = args.includes('--all') ? entries.map(e => e.url) : changedUrls(entries, Date.now() - hours * 3_600_000);
+const statuses = await submitIndexNow(key, urls);
+console.log(JSON.stringify({ origin, submitted: urls.length, statuses }));
+if (!statuses.every(s => s === 200 || s === 202)) process.exit(1);

package/dist/indexnow.d.ts

@@ -0,0 +1,22 @@
+export declare const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
+/** The protocol's cap on the URLs in one POST. */
+export declare const INDEXNOW_BATCH = 10000;
+export interface IndexNowEntry {
+    url: string;
+    lastModified?: string;
+}
+/** The `<loc>` and `<lastmod>` of every `<url>` in a sitemap. */
+export declare function sitemapEntries(xml: string): IndexNowEntry[];
+/**
+ * The URLs modified at or after `since`. An entry with no `lastModified` is
+ * left out: nothing says it changed, and the protocol asks for changed pages
+ * only. A full submission (`--all`) is how those reach an engine.
+ */
+export declare function changedUrls(entries: IndexNowEntry[], since: number): string[];
+/**
+ * POST `urls` in batches of `INDEXNOW_BATCH` and return each batch's status.
+ * 200 is accepted and 202 accepted pending the key check; 403 means the key
+ * file is missing or disagrees, 422 that a URL is off the host. Every URL must
+ * share the first one's host, which a sitemap built from one origin does.
+ */
+export declare function submitIndexNow(key: string, urls: string[], fetchImpl?: typeof fetch): Promise<number[]>;

package/dist/indexnow.js

@@ -0,0 +1,63 @@
+// indexnow.ts — telling the IndexNow engines (Bing, Yandex, Seznam, Naver, Yep)
+// which pages changed, rather than waiting for them to recrawl.
+//
+// The `indexnow` bin reads an origin's live sitemap, keeps the rows whose
+// `lastModified` falls inside the window, and posts them. It runs on the machine
+// that runs the sweeps, not as a cron on the host: a submission costs the origin
+// one cached sitemap fetch and no function time. The sitemap is the one place
+// every writer shows up — the editor, the MCP write tools, and the sweep scripts
+// that write to the database without passing through the app — so reading it
+// covers all of them with no hook in any.
+//
+// The key is not a secret. An engine verifies a submission by fetching
+// `/<key>.txt` from the origin, so the key is public by design, and the file in
+// each app's `public/` is the only copy.
+export const INDEXNOW_ENDPOINT = 'https://api.indexnow.org/indexnow';
+/** The protocol's cap on the URLs in one POST. */
+export const INDEXNOW_BATCH = 10_000;
+const XML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
+const unescapeXml = (s) => s.trim().replace(/&(amp|lt|gt|quot|apos);/g, (_, e) => XML_ENTITIES[e]);
+/** The `<loc>` and `<lastmod>` of every `<url>` in a sitemap. */
+export function sitemapEntries(xml) {
+    const entries = [];
+    for (const [, body = ''] of xml.matchAll(/<url>([\s\S]*?)<\/url>/g)) {
+        const loc = /<loc>([\s\S]*?)<\/loc>/.exec(body)?.[1];
+        if (!loc)
+            continue;
+        const lastmod = /<lastmod>([\s\S]*?)<\/lastmod>/.exec(body)?.[1];
+        entries.push(lastmod ? { url: unescapeXml(loc), lastModified: lastmod.trim() } : { url: unescapeXml(loc) });
+    }
+    return entries;
+}
+/**
+ * The URLs modified at or after `since`. An entry with no `lastModified` is
+ * left out: nothing says it changed, and the protocol asks for changed pages
+ * only. A full submission (`--all`) is how those reach an engine.
+ */
+export function changedUrls(entries, since) {
+    return entries
+        .filter(e => e.lastModified !== undefined && new Date(e.lastModified).getTime() >= since)
+        .map(e => e.url);
+}
+/**
+ * POST `urls` in batches of `INDEXNOW_BATCH` and return each batch's status.
+ * 200 is accepted and 202 accepted pending the key check; 403 means the key
+ * file is missing or disagrees, 422 that a URL is off the host. Every URL must
+ * share the first one's host, which a sitemap built from one origin does.
+ */
+export async function submitIndexNow(key, urls, fetchImpl = fetch) {
+    const first = urls[0];
+    if (!first)
+        return [];
+    const host = new URL(first).host;
+    const statuses = [];
+    for (let i = 0; i < urls.length; i += INDEXNOW_BATCH) {
+        const res = await fetchImpl(INDEXNOW_ENDPOINT, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json; charset=utf-8' },
+            body: JSON.stringify({ host, key, urlList: urls.slice(i, i + INDEXNOW_BATCH) }),
+        });
+        statuses.push(res.status);
+    }
+    return statuses;
+}

package/dist/metadata.d.ts

@@ -121,6 +121,27 @@ export declare function collectionLd(o: CollectionLdOptions): LdNode;
  * an authored `&` is not what a crawler reads.
  */
 export declare function citationsFromReferences(items: readonly ReferenceItem[], max?: number): LdNode[];
+export interface QaPair {
+    q: string;
+    a: string;
+}
+/**
+ * The question/answer pairs in a page's HTML: every `h2`/`h3` that ends in a
+ * question mark, with the text up to the next such heading as its answer.
+ *
+ * It never turns a statement heading into a question. Markup that rewrites
+ * "Protocol fees" as "What is Protocol fees?" claims a question the page never
+ * asked, so a page earns an FAQ by being written as one. An answer under 40
+ * characters is a heading with no section, and is skipped.
+ */
+export declare function questionPairs(html: string): QaPair[];
+/**
+ * `FAQPage` over question/answer pairs, or null under two. One question is an
+ * article with a question for a heading, not an FAQ. Pairs come from
+ * `questionPairs` on stored HTML, or from a list the caller authors directly
+ * as plain text, so a home-page FAQ and a wiki article render the same node.
+ */
+export declare function faqPageLd(pairs: readonly QaPair[], url: string): LdNode | null;
 /**
  * Characters a document `<title>` has before a search result truncates it.
  *

package/dist/metadata.js

@@ -19,6 +19,7 @@
 // with no image while their og:image named a card, one listed citations, one
 // collection page had no ItemList at all.
 import { decodeEntities } from './markdown.js';
+import { stripHtml } from './text.js';
 export function pageMetadata(o) {
     const { title, description, url, type = 'website', image, imageSize = { width: 1200, height: 630 }, siteName, locale, handle, markdownTwin, article } = o;
     const images = image ? [{ url: image, ...imageSize, alt: o.imageAlt ?? title }] : undefined;
@@ -107,6 +108,49 @@ export function citationsFromReferences(items, max = 50) {
     })
         .slice(0, max);
 }
+/**
+ * The question/answer pairs in a page's HTML: every `h2`/`h3` that ends in a
+ * question mark, with the text up to the next such heading as its answer.
+ *
+ * It never turns a statement heading into a question. Markup that rewrites
+ * "Protocol fees" as "What is Protocol fees?" claims a question the page never
+ * asked, so a page earns an FAQ by being written as one. An answer under 40
+ * characters is a heading with no section, and is skipped.
+ */
+export function questionPairs(html) {
+    const pairs = [];
+    const sections = html.split(/<h[23][^>]*>/i);
+    for (const section of sections.slice(1)) {
+        const [heading = '', ...rest] = section.split(/<\/h[23]>/i);
+        const q = stripHtml(heading).replace(/\s+/g, ' ');
+        if (!q.endsWith('?'))
+            continue;
+        const a = stripHtml(rest.join('')).replace(/\s+/g, ' ');
+        if (a.length >= 40)
+            pairs.push({ q, a: a.slice(0, 1000) });
+    }
+    return pairs;
+}
+/**
+ * `FAQPage` over question/answer pairs, or null under two. One question is an
+ * article with a question for a heading, not an FAQ. Pairs come from
+ * `questionPairs` on stored HTML, or from a list the caller authors directly
+ * as plain text, so a home-page FAQ and a wiki article render the same node.
+ */
+export function faqPageLd(pairs, url) {
+    if (pairs.length < 2)
+        return null;
+    return {
+        '@context': 'https://schema.org',
+        '@type': 'FAQPage',
+        url,
+        mainEntity: pairs.map(p => ({
+            '@type': 'Question',
+            name: p.q,
+            acceptedAnswer: { '@type': 'Answer', text: p.a },
+        })),
+    };
+}
 /**
  * Characters a document `<title>` has before a search result truncates it.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-formant",
-  "version": "0.45.0",
+  "version": "0.47.0",
   "description": "The portable half of a wiki: derived taxonomy and facet controls, a version-negotiating MCP transport, markdown twins, block rendering, a rich-text editor engine, conditional-GET plumbing, and the headers and CSP every origin sends.",
   "type": "module",
   "license": "MIT",
@@ -92,6 +92,11 @@
       "import": "./dist/combobox.js",
       "default": "./dist/combobox.js"
     },
+    "./indexnow": {
+      "types": "./dist/indexnow.d.ts",
+      "import": "./dist/indexnow.js",
+      "default": "./dist/indexnow.js"
+    },
     "./crawlers": {
       "types": "./dist/crawlers.d.ts",
       "import": "./dist/crawlers.js",
@@ -240,7 +245,8 @@
     "check-pages": "bin/check-pages.mjs",
     "analytics-digest": "bin/analytics-digest.mjs",
     "passkey-invite": "bin/passkey-invite.mjs",
-    "fts-ddl": "bin/fts-ddl.mjs"
+    "fts-ddl": "bin/fts-ddl.mjs",
+    "indexnow": "bin/indexnow.mjs"
   },
   "files": [
     "dist",