wiki-plugin-similarity 0.6.0 → 0.8.0

package/client/similarity.js

@@ -1,4 +1,4 @@
-/* wiki-plugin-similarity - 0.6.0 - Sat, 11 Jul 2026 20:14:13 GMT */
+/* wiki-plugin-similarity - 0.8.0 - Tue, 21 Jul 2026 08:09:09 GMT */
 (()=>{var q={high:.78,medium:.68,low:.58},M=q.medium,_=10,V=t=>{let e=[],n=[],o=[],f=null,b=null,d="search",y=!1,L=!1,E=null,k=null,w=(i,m)=>i===m||i.startsWith(m)&&/^[\s:]/.test(i.slice(m.length)),x=(i,m)=>i.slice(m.length).replace(/^\s*:?\s*/,"").trim();for(let i of t.split(`
 `)){let m=i.trim();if(!m||m.startsWith("#"))continue;let g=m.toUpperCase();if(w(g,"LIVE")){y=!0;continue}if(w(g,"AUTHOR")){!e.length&&d==="search"&&(d="author");continue}if(w(g,"REPORT")){d==="search"&&(d="report");continue}if(w(g,"KEYWORD")){d==="search"&&(d="keyword");continue}if(w(g,"BUILD")){d==="search"&&(d="build");continue}if(w(g,"FORCE")){L=!0;continue}if(w(g,"GHOST")){E=x(m,"GHOST"),d==="search"&&(d="ghost");continue}if(w(g,"BUTTON")){k=x(m,"BUTTON");continue}if(w(g,"ROSTER")){let S=x(m,"ROSTER");S&&n.push(S);continue}if(w(g,"FARM")){let S=x(m,"FARM");S&&o.push(S);continue}if(w(g,"LIST")){!e.length&&d==="search"&&(d="list");continue}if(w(g,"SIMILAR")){let S=x(g,"SIMILAR").toLowerCase();f=q[S]||M,!e.length&&d==="search"&&(d="similar");continue}if(w(g,"THRESHOLD")){let S=x(m,"THRESHOLD");f=q[S.toLowerCase()]??(parseFloat(S)||M);continue}if(w(g,"LIMIT")){b=parseInt(x(m,"LIMIT"))||_;continue}e.push(["PUBLIC","LOCAL","PRIVATE"].includes(g)?g:m)}return{mode:d,specs:e,rosterRefs:n,farms:o,threshold:f??M,limit:b??_,live:y,force:L,ghostUrl:E,label:k,thresholdSet:f!==null}},et=t=>t.includes("*")||t.includes("?"),nt=t=>t==="PUBLIC"||t==="LOCAL"||t==="PRIVATE",st=t=>t.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),P=new Map,it=async(t,e)=>{if(P.has(t))return P.get(t);let n=`${e}/system/indexed-domains.json?pattern=${encodeURIComponent(t)}`,o=await fetch(n);if(!o.ok)throw new Error(`indexed-domains failed: ${o.status}`);let f=await o.json();return P.set(t,f),f},G=async(t,e)=>{t.length||(t=[window.location.hostname]);let n=new Set,o=[];for(let f of t)if(f==="*"||et(f)||nt(f))for(let b of await it(f,e))n.has(b.domain)||(n.add(b.domain),o.push(b));else n.has(f)||(n.add(f),o.push({domain:f,page_count:null}));return o},ot=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,at=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,rt=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,B=async t=>{let e=t.indexOf("/"),n=await fetch(`//${t.slice(0,e)}/${t.slice(e+1)}.json`);if(!n.ok)throw new Error(`roster page ${t} failed: ${n.status}`);return n.json()},K=async(t,e,n)=>{if(n.has(t))return;n.add(t);let o;try{o=await B(t)}catch(f){console.warn(f.message);return}for(let f of o.story||[])if(f.type==="roster")for(let b of(f.text||"").split(/\r?\n/)){let d=b.trim();if(!d)continue;let y=d.match(ot);if(y){e.add(y[0]);continue}let L=d.match(at);if(L){await K(L[1],e,n);continue}let E=d.match(rt);if(E)try{let k=await B(E[1]);for(let w of k.story||[])w.type==="reference"&&w.site&&e.add(w.site)}catch(k){console.warn(k.message)}}},lt=async t=>{let e=new Set,n=new Set;for(let o of t)await K(o,e,n);return[...e]},ct=async(t,e)=>e.length?[...t,...await lt(e)]:t,A=new Map,dt=t=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(t)}`,Z=async t=>{if(A.has(t))return A.get(t);let e=await fetch(dt(t));if(!e.ok)return[];let n=await e.json();return A.set(t,n),n},z=async(t,e)=>{let n=await fetch(`${e}/system/embed.json?text=${encodeURIComponent(t)}`);if(!n.ok)throw new Error(`embed failed: ${n.status}`);return(await n.json()).vector},ft=async(t,e)=>{let o=(await Z(e)).find(f=>f.slug===t);return o?o.vector:null},H=(t,e,{threshold:n,limit:o,excludeSlug:f,excludeDomain:b})=>{let d=[];for(let{domain:y,pages:L}of e)for(let{slug:E,title:k,vector:w}of L){if(E===f&&y===b)continue;let x=0;for(let i=0;i<t.length;i++)x+=t[i]*w[i];x>=n&&d.push({domain:y,slug:E,title:k,score:x})}return d.sort((y,L)=>L.score-y.score),d.slice(0,o)},F=async(t,e)=>{let n=await G(t,e);return(await Promise.all(n.map(async({domain:f})=>({domain:f,pages:await Z(f)})))).filter(f=>f.pages.length>0)},J=t=>`sim-cache-${t}`,mt=t=>{try{let e=JSON.parse(localStorage.getItem(J(t.id))||"null");return e?.text===(t.text||"")?e:null}catch{return null}},O=(t,e)=>{try{localStorage.setItem(J(t.id),JSON.stringify({text:t.text||"",ts:Date.now(),...e}))}catch{}},ut=t=>{let e=Math.floor((Date.now()-t)/1e3);return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`},I=`
   .sim-form { display:flex; gap:6px; margin-bottom:8px; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-plugin-similarity",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "Federated Wiki plugin \u2014 semantic similarity search across wiki domains using prebuilt vector indexes",
   "keywords": [
     "wiki",
@@ -42,8 +42,10 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
+    "minisearch": "^7.2.0"
+  },
+  "optionalDependencies": {
     "@xenova/transformers": "^2.17.2",
-    "minisearch": "^7.2.0",
     "onnxruntime-web": "1.14.0"
   },
   "devDependencies": {

package/pages/about-similarity-plugin

@@ -9,7 +9,7 @@
     {
       "type": "markdown",
       "id": "28edff984cc5334d",
-      "text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword) — interactive semantic search form\n- `SIMILAR: high|medium|low` — ambient: pages similar to this page\n- `REPORT` — server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` — galactic MiniSearch over live site-index.json files\n- `LIST` — table of indexed domains\n- `AUTHOR` — search results as a draggable ghost page\n- `BUILD` / `FORCE` — proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` — open any page-json as a ghost page\n- `ROSTER site/slug` — add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` — ask a peer farm to continue the search (experimental; requires the peer's `search-peers` roster to allow you)\n- `THRESHOLD: 0.72` · `LIMIT: 8` · `LIVE` — tuning"
+      "text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword) — interactive semantic search form\n- `SIMILAR: high|medium|low` — ambient: pages similar to this page\n- `REPORT` — server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` — galactic MiniSearch over live site-index.json files\n- `LIST` — table of indexed domains\n- `AUTHOR` — search results as a draggable ghost page\n- `BUILD` / `FORCE` — proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` — open any page-json as a ghost page\n- `ROSTER site/slug` — add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` — ask a peer farm to continue the search. The peer answers from exactly the sites whose own [[Federated Farm Search]] page grants your origin (`FROM` lines; `FROM *` needs the peer admin's `WIKI_PEER_FEDERATION=open`)\n- `THRESHOLD: 0.72` · `LIMIT: 8` · `LIVE` — tuning"
     },
     {
       "type": "markdown",
@@ -54,12 +54,12 @@
     {
       "type": "markdown",
       "id": "d9a96dfebb9d1692",
-      "text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` — keyword search over off-farm federation sites\n- `POST /system/peer-search.json` — answer another farm's search (guarded: allowlist roster, hop limit, replay dedup, rate limit; restricted sites always excluded)\n\nIndexes are `status/semantic-vectors.json` per domain — built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised — byte-compatible with the index builder."
+      "text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` — keyword search over off-farm federation sites\n- `GET /system/peer-hello.json` — capability probe: version, model, federation status (404 = plugin absent)\n- `POST /system/peer-search.json` — answer another farm's search. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain — built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised — byte-compatible with the index builder."
     },
     {
       "type": "markdown",
       "id": "21eb5354d2adb29e",
-      "text": "**Version:** 0.6.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] — architecture, indexing, freshness."
+      "text": "**Version:** 0.7.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] — architecture, indexing, freshness."
     }
   ],
   "plugin": "similarity",
@@ -163,6 +163,36 @@
         "text": "**Version:** 0.6.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] — architecture, indexing, freshness."
       },
       "date": 1783800718188
+    },
+    {
+      "type": "edit",
+      "id": "28edff984cc5334d",
+      "item": {
+        "type": "markdown",
+        "id": "28edff984cc5334d",
+        "text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword) — interactive semantic search form\n- `SIMILAR: high|medium|low` — ambient: pages similar to this page\n- `REPORT` — server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` — galactic MiniSearch over live site-index.json files\n- `LIST` — table of indexed domains\n- `AUTHOR` — search results as a draggable ghost page\n- `BUILD` / `FORCE` — proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` — open any page-json as a ghost page\n- `ROSTER site/slug` — add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` — ask a peer farm to continue the search. The peer answers from exactly the sites whose own [[Federated Farm Search]] page grants your origin (`FROM` lines; `FROM *` needs the peer admin's `WIKI_PEER_FEDERATION=open`)\n- `THRESHOLD: 0.72` · `LIMIT: 8` · `LIVE` — tuning"
+      },
+      "date": 1784591495272
+    },
+    {
+      "type": "edit",
+      "id": "d9a96dfebb9d1692",
+      "item": {
+        "type": "markdown",
+        "id": "d9a96dfebb9d1692",
+        "text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` — keyword search over off-farm federation sites\n- `GET /system/peer-hello.json` — capability probe: version, model, federation status (404 = plugin absent)\n- `POST /system/peer-search.json` — answer another farm's search. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain — built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised — byte-compatible with the index builder."
+      },
+      "date": 1784591495272
+    },
+    {
+      "type": "edit",
+      "id": "21eb5354d2adb29e",
+      "item": {
+        "type": "markdown",
+        "id": "21eb5354d2adb29e",
+        "text": "**Version:** 0.7.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] — architecture, indexing, freshness."
+      },
+      "date": 1784591495272
     }
   ]
 }

package/server/.galaxy-cache/wiki.ralfbarkow.ch/meta.json

@@ -1 +1 @@
-{"sitemap.json":{"lastModified":"Fri, 10 Jul 2026 05:01:01 GMT","fetchedAt":1783794283569},"site-index.json":{"lastModified":"Fri, 10 Jul 2026 05:01:50 GMT","fetchedAt":1783794283566}}
+{"sitemap.json":{"lastModified":"Fri, 10 Jul 2026 05:01:01 GMT","fetchedAt":1783851444700},"site-index.json":{"lastModified":"Fri, 10 Jul 2026 05:01:50 GMT","fetchedAt":1783851444709}}

package/server/embedder.js

@@ -19,6 +19,21 @@ const MODEL     = process.env.WIKI_EMBED_MODEL || 'Xenova/bge-small-en-v1.5'
 const POOLING   = process.env.WIKI_EMBED_POOLING || 'cls'
 const QUANTIZED = process.env.WIKI_EMBED_QUANTIZED === '1'
 
+// Prefer wiki-plugin-semindex's shared worker-thread embedder when installed:
+// one model instance per process (shared with bulk indexing), inference off
+// the event loop, and similarity's own heavy deps become optional. The
+// sibling path works for npm installs (wiki/node_modules/*) and dev symlinks
+// (~/Code/wiki-plugins/*) alike; the bare require covers other layouts.
+const semindexLib = (() => {
+  const path = require('node:path')
+  const fs   = require('node:fs')
+  try {
+    const sibling = path.resolve(__dirname, '../../wiki-plugin-semindex/server/embed-lib.js')
+    if (fs.existsSync(sibling)) return require(sibling)
+    return require('wiki-plugin-semindex/server/embed-lib.js')
+  } catch { return null }
+})()
+
 // transformers.js statically imports onnxruntime-node, whose native binary
 // needs glibc. On musl containers (node:alpine — the public farm) that import
 // throws "Error loading shared library ld-linux-x86-64.so.2". Probe the
@@ -47,7 +62,13 @@ const getExtractor = () => {
         console.log('[wiki-plugin-similarity] native onnxruntime unavailable — using WASM backend')
       }
       // Dynamic import: @xenova/transformers is ESM-only; this file must stay CJS.
-      const { pipeline, env } = await import('@xenova/transformers')
+      // Since 0.8.0 the heavy deps are optionalDependencies — absent on slim
+      // installs, where semindex delegation or WIKI_EMBED_URL is the path.
+      const { pipeline, env } = await import('@xenova/transformers').catch(() => {
+        throw new Error('no local embedder: @xenova/transformers is not installed. ' +
+          'Install wiki-plugin-semindex, set WIKI_EMBED_URL, or reinstall ' +
+          'wiki-plugin-similarity with optional dependencies.')
+      })
       if (process.env.WIKI_MODEL_CACHE) env.cacheDir = process.env.WIKI_MODEL_CACHE
       if (env.backends?.onnx?.wasm) env.backends.onnx.wasm.numThreads = 1
       return pipeline('feature-extraction', MODEL, { quantized: QUANTIZED })
@@ -59,13 +80,15 @@ const getExtractor = () => {
 
 // Embed one text → plain number[384], unit-normalised.
 const embed = async text => {
+  if (semindexLib) return semindexLib.embed(text)
   const extractor = await getExtractor()
   const out = await extractor(text, { pooling: POOLING, normalize: true })
   return Array.from(out.data)
 }
 
 // Warm the model in the background (call at startup; failures just defer to
-// first request).
-const warm = () => { getExtractor().catch(() => {}) }
+// first request). With semindex delegation, don't warm — its worker spawns on
+// demand and self-terminates when idle; pre-warming would just pin ~0.5GB.
+const warm = () => { if (!semindexLib) getExtractor().catch(() => {}) }
 
-module.exports = { embed, warm }
+module.exports = { embed, warm, viaSemindex: !!semindexLib }

package/server/peer-guard.js

@@ -1,41 +1,139 @@
 // wiki-plugin-similarity — peer-search guards (CommonJS)
 //
-// v1 trust mechanics for the FARM prototype (experimental). A peer request is
-// served only when every guard passes:
-//   - allowlist: the requesting farm appears in the serving site's own
-//     `search-peers` page (a roster item — configuration as a wiki page)
-//   - hops: v1 answers locally only, never re-forwards (hops must be 0)
-//   - dedup: a request-id LRU drops replayed/looped envelopes
-//   - rate: a small per-origin token bucket protects the embedder
+// Two keys must turn before a farm answers a peer's search:
 //
-// The allowlist seam is deliberately narrow: later versions swap the roster
-// check for shared-secret pairs (v2) and trust-graph attestation (v3) without
+//   1. The farm admin's ceiling — WIKI_PEER_FEDERATION env:
+//        off     refuse all peer search (kill switch)
+//        grants  per-site owner grants honored; FROM * is inert (default)
+//        open    FROM * takes effect
+//      An env, not a page: the plugin cannot verify who wrote a page line,
+//      and the environment is the one thing only the admin holds.
+//
+//   2. The site owner's grant — a "Federated Farm Search" page (slug
+//      federated-farm-search, override WIKI_PEER_GRANTS_PAGE) whose CODE
+//      items carry FROM lines:
+//        FROM hitchhikers.earth      a named partner
+//        FROM *.fedwiki.org          a glob family
+//        FROM *                      everyone (needs ceiling = open)
+//      Lines are additive. No page and no legacy search-peers roster means
+//      the site is closed to peers — opt-in, never opt-out.
+//
+// The answer scope for an origin is exactly the union of sites that grant
+// it (per-site consent — one site's grant never opens the whole farm).
+// Restricted (login-to-view) sites are excluded below this layer,
+// unconditionally.
+//
+// Migration: a site with no grants page but a legacy `search-peers` roster
+// page has its roster lines read as FROM lines for that site only.
+// Remove the fallback at 0.8.
+//
+// Rate limiting is keyed by remote IP, never by the asserted origin (origins
+// are free to invent in v1), with a bounded LRU and a global cap.
+//
+// This guard seam stays narrow on purpose: v2 swaps the grant check for
+// shared-secret HMAC verification, v3 for trust-graph attestation, without
 // touching the callers.
 
 const fs   = require('node:fs')
 const path = require('node:path')
 
+const { globMatch } = require('./farm-lib')
+
 const SITE_LINE = /^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/
 
-// ── Allowlist: roster item on the serving site's search-peers page ────────────
-
-const readPeerRoster = (farms, servingDomain, slug = null) => {
-  slug = slug || process.env.WIKI_SEARCH_PEERS_PAGE || 'search-peers'
-  for (const [farm] of farms) {
-    const p = path.join(farm, servingDomain, 'pages', slug)
-    let page
-    try { page = JSON.parse(fs.readFileSync(p, 'utf8')) } catch { continue }
-    const allowed = new Set()
-    for (const item of page.story || []) {
-      if (item.type !== 'roster') continue
-      for (const raw of (item.text || '').split(/\r?\n/)) {
-        const m = raw.trim().match(SITE_LINE)
-        if (m) allowed.add(m[0])
-      }
+const CEILINGS = ['off', 'grants', 'open']
+const ceiling = () => {
+  const c = (process.env.WIKI_PEER_FEDERATION || 'grants').toLowerCase()
+  return CEILINGS.includes(c) ? c : 'grants'
+}
+
+const grantsSlug = () => process.env.WIKI_PEER_GRANTS_PAGE || 'federated-farm-search'
+
+// ── Grants parser ─────────────────────────────────────────────────────────────
+// Input: the text of one or more code items. Output:
+//   { open: bool, origins: Set<string>, globs: string[] }
+// Unknown UPPERCASE lines are ignored (safe while every command is
+// additive-permissive; a future restricting command must be version-gated
+// via the hello route).
+
+const parseGrants = texts => {
+  const grants = { open: false, origins: new Set(), globs: [] }
+  for (const text of texts) {
+    for (const raw of (text || '').split(/\r?\n/)) {
+      const line = raw.trim()
+      if (!line || line.startsWith('#')) continue
+      const m = line.match(/^FROM[\s:]+(.+)$/i)
+      if (!m) continue
+      const value = m[1].trim()
+      if (value === '*') grants.open = true
+      else if (value.includes('*') || value.includes('?')) grants.globs.push(value)
+      else if (value.match(SITE_LINE)) grants.origins.add(value)
     }
-    return allowed
   }
-  return null // no peers page at all
+  return grants
+}
+
+// Does this grant set admit the origin, under the admin ceiling?
+const grantsAdmit = (grants, origin, ceil = ceiling()) => {
+  if (!grants || ceil === 'off') return false
+  if (grants.origins.has(origin)) return true
+  if (grants.globs.some(g => globMatch(g, origin))) return true
+  if (grants.open && ceil === 'open') return true
+  return false
+}
+
+// ── Per-site grant reads, mtime-cached ────────────────────────────────────────
+
+const grantCache = new Map() // "farm/domain" → {mtime, grants|null}
+
+const readCodeItemTexts = page =>
+  (page.story || []).filter(i => i.type === 'code').map(i => i.text || '')
+
+const readRosterSiteLines = page => {
+  const out = new Set()
+  for (const item of page.story || []) {
+    if (item.type !== 'roster') continue
+    for (const raw of (item.text || '').split(/\r?\n/)) {
+      const m = raw.trim().match(SITE_LINE)
+      if (m) out.add(m[0])
+    }
+  }
+  return out
+}
+
+// Grants for one site: the federated-farm-search page's code items, falling
+// back to the legacy search-peers roster (read as FROM lines for this site).
+// Returns null when the site declares nothing (closed).
+const readSiteGrants = (farm, domain) => {
+  const tryPage = (slug, extract) => {
+    const p = path.join(farm, domain, 'pages', slug)
+    let mtime
+    try { mtime = fs.statSync(p).mtimeMs } catch { return undefined } // no page
+    const key = `${farm}/${domain}/${slug}`
+    const hit = grantCache.get(key)
+    if (hit && hit.mtime === mtime) return hit.grants
+    let grants = null
+    try { grants = extract(JSON.parse(fs.readFileSync(p, 'utf8'))) } catch { /* unreadable */ }
+    grantCache.set(key, { mtime, grants })
+    if (grantCache.size > 2000) grantCache.delete(grantCache.keys().next().value)
+    return grants
+  }
+
+  const fromPage = tryPage(grantsSlug(), page => parseGrants(readCodeItemTexts(page)))
+  if (fromPage !== undefined) return fromPage
+  const legacy = tryPage('search-peers', page => {
+    const origins = readRosterSiteLines(page)
+    return origins.size ? { open: false, origins, globs: [] } : null
+  })
+  return legacy !== undefined ? legacy : null
+}
+
+// The union of domains (from [{farm, domain}]) whose own grants admit the
+// origin under the current ceiling.
+const grantingDomains = (domains, origin, ceil = ceiling()) => {
+  if (ceil === 'off') return []
+  return domains.filter(({ farm, domain }) =>
+    grantsAdmit(readSiteGrants(farm, domain), origin, ceil))
 }
 
 // ── Request-id LRU dedup ──────────────────────────────────────────────────────
@@ -51,16 +149,24 @@ const makeDedup = (max = 500) => {
   }
 }
 
-// ── Per-origin token bucket ───────────────────────────────────────────────────
+// ── Rate limiting: per-IP token buckets (bounded LRU) + a global bucket ──────
+// Never key limits by the asserted origin: a spoofer rotating origins would
+// both escape the limiter and grow the map without bound.
 
-const makeBucket = ({ capacity = 30, refillPerSec = 0.5 } = {}) => {
+const makeBucket = ({ capacity = 30, refillPerSec = 0.5, maxKeys = 500 } = {}) => {
   const buckets = new Map()
-  return origin => {
+  return key => {
     const now = Date.now()
-    let b = buckets.get(origin)
-    if (!b) { b = { tokens: capacity, at: now }; buckets.set(origin, b) }
-    b.tokens = Math.min(capacity, b.tokens + ((now - b.at) / 1000) * refillPerSec)
-    b.at = now
+    let b = buckets.get(key)
+    if (!b) {
+      b = { tokens: capacity, at: now }
+    } else {
+      buckets.delete(key) // refresh LRU position
+      b.tokens = Math.min(capacity, b.tokens + ((now - b.at) / 1000) * refillPerSec)
+      b.at = now
+    }
+    buckets.set(key, b)
+    while (buckets.size > maxKeys) buckets.delete(buckets.keys().next().value)
     if (b.tokens < 1) return false
     b.tokens -= 1
     return true
@@ -68,29 +174,33 @@ const makeBucket = ({ capacity = 30, refillPerSec = 0.5 } = {}) => {
 }
 
 // ── Envelope guard ────────────────────────────────────────────────────────────
+// Shape and abuse checks only — the consent decision (which sites grant this
+// origin) is per-site and belongs to the caller via grantingDomains().
 // Returns {ok: true} or {ok: false, code, error}.
 
-const guardEnvelope = (envelope, { allowed, isDuplicate, takeToken }) => {
-  const origin = (envelope.origin || '').trim()
-  if (!origin) {
-    return { ok: false, code: 400, error: 'origin required' }
-  }
-  if (allowed === null) {
-    return { ok: false, code: 403, error: 'this site has no search-peers page — peer search is not enabled here' }
+const guardEnvelope = (envelope, { ip, isDuplicate, takeIpToken, takeGlobalToken }) => {
+  if (ceiling() === 'off') {
+    return { ok: false, code: 403, error: 'peer federation is switched off on this farm' }
   }
-  if (!allowed.has(origin)) {
-    return { ok: false, code: 403, error: `${origin} is not on this site's search-peers roster` }
+  if (!(envelope.origin || '').trim()) {
+    return { ok: false, code: 400, error: 'origin required' }
   }
   if ((envelope.hops || 0) > 0) {
-    return { ok: false, code: 400, error: 'relayed requests are not accepted (v1 answers locally only)' }
+    return { ok: false, code: 400, error: 'relayed requests are not accepted (answers are local only)' }
   }
   if (isDuplicate(envelope.requestId)) {
     return { ok: false, code: 409, error: 'duplicate request-id (replay or loop)' }
   }
-  if (!takeToken(origin)) {
-    return { ok: false, code: 429, error: 'rate limit exceeded for this peer' }
+  if (takeGlobalToken && !takeGlobalToken('global')) {
+    return { ok: false, code: 429, error: 'this farm is at its peer-search rate limit' }
+  }
+  if (!takeIpToken(ip || 'unknown')) {
+    return { ok: false, code: 429, error: 'rate limit exceeded' }
   }
   return { ok: true }
 }
 
-module.exports = { readPeerRoster, makeDedup, makeBucket, guardEnvelope, SITE_LINE }
+module.exports = {
+  ceiling, grantsSlug, parseGrants, grantsAdmit, readSiteGrants, grantingDomains,
+  makeDedup, makeBucket, guardEnvelope, SITE_LINE,
+}

package/server/server.js

@@ -23,6 +23,13 @@
 //     Proxy to the farm indexer (WIKI_INDEXER_URL) when configured; heavy
 //     embedding is the indexer's job (Pi5 on the Hitchhikers farm), never
 //     the wiki server's.
+//   GET  /system/peer-hello.json
+//     Capability probe: plugin version, embedding model, federation status.
+//     404 here means the plugin is absent from the farm.
+//   POST /system/peer-search.json
+//     Federated peer search. Two keys: the admin's WIKI_PEER_FEDERATION
+//     ceiling (off|grants|open) and each site's Federated Farm Search
+//     grants page (FROM lines). Scope = union of granting sites only.
 //
 // Farm root is derived from argv.status ({farm}/{domain}/status); extra farm
 // roots come from WIKI_EXTRA_FARMS (colon-separated absolute paths).
@@ -44,9 +51,12 @@ const embedder     = require('./embedder')
 const { buildReport } = require('./search-report')
 const { searchFarm, keywordReportPage } = require('./farm-search')
 const { searchGalaxy } = require('./galaxy-search')
-const { readPeerRoster, makeDedup, makeBucket, guardEnvelope } = require('./peer-guard')
+const { ceiling, grantingDomains, makeDedup, makeBucket, guardEnvelope } = require('./peer-guard')
 
 const MODEL_META = { model: 'BAAI/bge-small-en-v1.5', dim: 384 }
+const PLUGIN_VERSION = (() => {
+  try { return require('../package.json').version } catch { return 'unknown' }
+})()
 
 // Optional external embedder (proxy) — unset means embed in-process.
 const EMBED_URL   = process.env.WIKI_EMBED_URL || null
@@ -205,16 +215,74 @@ const startServer = ({ argv, app }) => {
   for (const route of ['/system/indexed-domains.json', '/system/semantic-vectors.json',
                        '/system/embed.json', '/system/search-report.json',
                        '/system/farm-search.json', '/system/build-index.json',
-                       '/system/galaxy-search.json', '/system/peer-search.json']) {
+                       '/system/galaxy-search.json', '/system/peer-search.json',
+                       '/system/peer-hello.json']) {
     app.options(route, (req, res) => { cors(res); res.sendStatus(204) })
   }
 
-  // FARM prototype guards (shared across peer requests)
-  const isDuplicate = makeDedup()
-  const takeToken   = makeBucket()
+  // Peer federation guards (shared across peer requests). Rate limits are
+  // keyed by remote IP — never by the asserted origin — plus a global cap.
+  const isDuplicate     = makeDedup()
+  const takeIpToken     = makeBucket()
+  const takeGlobalToken = makeBucket({ capacity: 120, refillPerSec: 2, maxKeys: 1 })
+
+  const requestIp = req =>
+    (req.headers['x-forwarded-for'] || '').split(',')[0].trim() ||
+    req.socket.remoteAddress || 'unknown'
+
+  // GET JSON from a peer (hello probe) — same transport rules as postToPeer.
+  const getFromPeer = (peer, routePath, timeoutMs = 10_000) =>
+    new Promise((resolve, reject) => {
+      const isLocal = peer.endsWith('.localhost') || peer.startsWith('localhost')
+      const mod = isLocal ? http : https
+      const opts = {
+        hostname: peer.split(':')[0],
+        port: peer.includes(':') ? peer.split(':')[1] : (isLocal ? 80 : 443),
+        path: routePath,
+        timeout: timeoutMs,
+      }
+      if (isLocal) opts.lookup = (h, o, cb) => cb(null, [{ address: '127.0.0.1', family: 4 }])
+      const req = mod.get(opts, res => {
+        let data = ''
+        res.on('data', chunk => { data += chunk })
+        res.on('end', () => {
+          try { resolve({ status: res.statusCode, body: JSON.parse(data) }) }
+          catch { resolve({ status: res.statusCode, body: null }) }
+        })
+      })
+      req.on('timeout', () => { req.destroy(new Error('peer timeout')) })
+      req.on('error', reject)
+    })
+
+  // Hello probe cache: peer → {at, ok, body}. TTL 1h, small LRU.
+  const helloCache = new Map()
+  const HELLO_TTL = 60 * 60_000
+
+  const probePeer = async peer => {
+    const hit = helloCache.get(peer)
+    if (hit && Date.now() - hit.at < HELLO_TTL) return hit
+    let entry
+    try {
+      const { status, body } = await getFromPeer(peer, '/system/peer-hello.json')
+      entry = { at: Date.now(), ok: status === 200 && body && body.plugin, body }
+    } catch {
+      entry = { at: Date.now(), ok: false, body: null }
+    }
+    helloCache.set(peer, entry)
+    if (helloCache.size > 200) helloCache.delete(helloCache.keys().next().value)
+    return entry
+  }
 
   const askPeers = async (peers, envelope) => {
     const outcomes = await Promise.all(peers.map(async peer => {
+      const hello = await probePeer(peer)
+      if (!hello.ok) {
+        return { peer, status: 0,
+          body: { error: 'peer does not answer hello — no similarity federation there' } }
+      }
+      if (hello.body.federation && hello.body.federation.enabled === false) {
+        return { peer, status: 0, body: { error: 'peer has federation switched off' } }
+      }
       try {
         const { status, body } = await postToPeer(peer, '/system/peer-search.json', envelope)
         return { peer, status, body }
@@ -365,31 +433,67 @@ const startServer = ({ argv, app }) => {
     }
   })
 
-  // ── POST /system/peer-search.json — FARM prototype (experimental) ─────────
-  // Another farm asks this one to continue a search on its own sites. Guarded
-  // by the serving site's search-peers roster page, a hop limit, request-id
-  // dedup and a per-origin rate limit. Restricted (PRIVATE) sites are always
-  // excluded from peer answers.
+  // ── GET /system/peer-hello.json — capability probe ────────────────────────
+  // "Does this farm run the Similarity plugin, and does it federate?"
+  // A 404 (or HTML) here means the plugin is absent. Static JSON, no embedder
+  // cost; does not advertise any FROM list.
+  app.get('/system/peer-hello.json', (req, res) => {
+    cors(res)
+    res.setHeader('Cache-Control', 'public, max-age=3600')
+    res.json({
+      plugin: 'wiki-plugin-similarity',
+      version: PLUGIN_VERSION,
+      ...MODEL_META,
+      site: req.hostname,
+      federation: {
+        enabled: ceiling() !== 'off',
+        kinds: ['report', 'keyword'],
+        hopsAccepted: 0,
+      },
+    })
+  })
+
+  // ── POST /system/peer-search.json — federated peer search ────────────────
+  // Two keys must turn: the farm admin's WIKI_PEER_FEDERATION ceiling, and
+  // each site's own Federated Farm Search grants page. The answer scope for
+  // an origin is exactly the union of sites that granted it — one site's
+  // grant never opens the whole farm. Restricted (login-to-view) sites are
+  // excluded unconditionally. Abuse guards: hop limit, request-id dedup,
+  // IP-keyed + global rate limits.
   app.post('/system/peer-search.json', async (req, res) => {
     cors(res)
     try {
       const envelope = await readBody(req)
-      const allowed = readPeerRoster(farms, req.hostname)
-      const verdict = guardEnvelope(envelope, { allowed, isDuplicate, takeToken })
+      const verdict = guardEnvelope(envelope,
+        { ip: requestIp(req), isDuplicate, takeIpToken, takeGlobalToken })
       if (!verdict.ok) return res.status(verdict.code).json({ error: verdict.error })
       if (!envelope.query) return res.status(400).json({ error: 'query required' })
 
+      const origin = envelope.origin.trim()
+      const allDomains = listDomains(farms, ['*'], restricted)
+        .filter(d => !restricted.has(d.domain))
+      const granted = grantingDomains(allDomains, origin)
+      if (!granted.length) {
+        return res.status(403).json({
+          error: `no site on this farm grants federated search to ${origin}`,
+          hint: 'a site opts in with FROM lines on its Federated Farm Search page',
+        })
+      }
+      const grantedNames = granted.map(d => d.domain)
+
       const limit = envelope.limit || 10
       let page
       if (envelope.kind === 'keyword') {
-        const outcome = searchFarm(farms, ['*'], restricted, envelope.query, limit, restricted)
-        page = keywordReportPage(envelope.query, outcome, limit, ['*'])
+        const outcome = searchFarm(farms, grantedNames, restricted,
+          envelope.query, limit, restricted)
+        page = keywordReportPage(envelope.query, outcome, limit, grantedNames)
       } else {
-        page = await buildReport(envelope.query, ['*'], limit,
+        page = await buildReport(envelope.query, grantedNames, limit,
           { ...ctx, exclude: restricted })
       }
       const count = page.story.filter(i => i.type === 'reference').length
-      res.json({ page, meta: { ...MODEL_META, farm: req.hostname, count } })
+      res.json({ page, meta: { ...MODEL_META, version: PLUGIN_VERSION,
+        farm: req.hostname, sites: grantedNames.length, count } })
     } catch (e) {
       console.error('[wiki-plugin-similarity] peer-search error:', e.message)
       res.status(500).json({ error: `peer-search failed: ${e.message}` })
@@ -397,26 +501,47 @@ const startServer = ({ argv, app }) => {
   })
 
   // ── GET /system/build-index.json?domains=…&force=… ────────────────────────
-  // Heavy embedding is the farm indexer's job — proxy when configured, else
-  // explain where indexing happens.
+  // Resolution order: WIKI_INDEXER_URL proxy (explicit config wins) → local
+  // wiki-plugin-semindex (in-process enqueue, owner/admin only) → 501.
+  const semindex = (() => {
+    try {
+      const sibling = path.resolve(__dirname, '../../wiki-plugin-semindex/server/indexer.js')
+      if (fs.existsSync(sibling)) return require(sibling)
+      return require('wiki-plugin-semindex/server/indexer.js')
+    } catch { return null }
+  })()
+
   app.get('/system/build-index.json', async (req, res) => {
     cors(res)
-    if (!INDEXER_URL) {
-      return res.status(501).json({
-        error: 'no farm indexer configured on this server',
-        hint: 'Indexing runs on the farm indexer (Pi5 on the Hitchhikers farm); ' +
-              'indexes arrive by sync. Set WIKI_INDEXER_URL to enable proxying.',
-      })
+    if (INDEXER_URL) {
+      try {
+        const qs = new URLSearchParams({
+          domains: req.query.domains || '*',
+          force:   req.query.force || '0',
+        })
+        return res.json(await getJson(`${INDEXER_URL}?${qs}`))
+      } catch (e) {
+        return res.status(502).json({ error: `farm indexer unavailable: ${e.message}` })
+      }
     }
-    try {
-      const qs = new URLSearchParams({
-        domains: req.query.domains || '*',
-        force:   req.query.force || '0',
-      })
-      res.json(await getJson(`${INDEXER_URL}?${qs}`))
-    } catch (e) {
-      res.status(502).json({ error: `farm indexer unavailable: ${e.message}` })
+    if (semindex) {
+      const sh = app.securityhandler
+      const ok = (() => {
+        try { return !!sh && (!!sh.isAdmin?.(req) || !!sh.isAuthorized?.(req)) } catch { return false }
+      })()
+      if (!ok) return res.sendStatus(403)
+      if (semindex.MODE !== 'writer') {
+        return res.status(409).json({ error: `semindex mode is '${semindex.MODE}' — not a writer` })
+      }
+      const queued = semindex.enqueueGlobs(req.query.domains || '*', req.query.force === '1')
+      return res.json({ queued, ...semindex.status() })
     }
+    res.status(501).json({
+      error: 'no indexer available on this server',
+      hint: 'Install wiki-plugin-semindex for in-process indexing, or set ' +
+            'WIKI_INDEXER_URL to proxy to a farm indexer; indexes can also ' +
+            'arrive by sync.',
+    })
   })
 
   console.log('[wiki-plugin-similarity] routes registered')