wiki-plugin-similarity 0.19.0 → 0.20.0

package/client/similarity.js

@@ -1,4 +1,4 @@
-/* wiki-plugin-similarity - 0.19.0 - Fri, 04 Sep 2026 07:52:20 GMT */
+/* wiki-plugin-similarity - 0.20.0 - Fri, 04 Sep 2026 09:22:03 GMT */
 (()=>{var lt={high:.78,medium:.68,low:.58},rt=lt.medium,$t=10,ct=t=>{let e=[],n=[],s=[],i=null,r=null,u="search",g=!1,k=!1,b=!1,M=null,R=null,A=null,U=null,w=(a,d)=>a===d||a.startsWith(d)&&/^[\s:]/.test(a.slice(d.length)),v=(a,d)=>a.slice(d.length).replace(/^\s*:?\s*/,"").trim();for(let a of t.split(`
 `)){let d=a.trim();if(!d||d.startsWith("#"))continue;let y=d.toUpperCase();if(w(y,"LIVE")){g=!0;continue}if(w(y,"SUBJECT")){k=!0;continue}if(w(y,"AUTHOR")){!e.length&&u==="search"&&(u="author");continue}if(w(y,"REPORT")){u==="search"&&(u="report");continue}if(w(y,"KEYWORD")){u==="search"&&(u="keyword");continue}if(w(y,"SITES")){u==="search"&&(u="sites");continue}if(w(y,"BUILD")){u==="search"&&(u="build");continue}if(w(y,"FORCE")){b=!0;continue}if(w(y,"GHOST")){M=v(d,"GHOST"),u==="search"&&(u="ghost");continue}if(w(y,"BUTTON")){R=v(d,"BUTTON");continue}if(w(y,"BATCH")){let C=v(d,"BATCH").toLowerCase();A=C==="off"||C==="no"||C==="0"?0:parseInt(C)||50;continue}if(w(y,"ALGORITHM")){U=v(d,"ALGORITHM").replace(/^\[\[|\]\]$/g,"").trim()||"search-algorithm",u==="search"&&!e.length&&(u="algorithm");continue}if(w(y,"ROSTER")){let C=v(d,"ROSTER");C&&n.push(C);continue}if(w(y,"FARM")){let C=v(d,"FARM");C&&s.push(C);continue}if(w(y,"LIST")){!e.length&&u==="search"&&(u="list");continue}if(w(y,"STATUS")){u==="search"&&(u="status");continue}if(w(y,"SIMILAR")){let C=v(y,"SIMILAR").toLowerCase();i=lt[C]||rt,!e.length&&u==="search"&&(u="similar");continue}if(w(y,"THRESHOLD")){let C=v(d,"THRESHOLD");i=lt[C.toLowerCase()]??(parseFloat(C)||rt);continue}if(w(y,"LIMIT")){r=parseInt(v(d,"LIMIT"))||$t;continue}e.push(["PUBLIC","LOCAL","PRIVATE","GALAXY","SITE"].includes(y)?y:d)}return{mode:u,specs:e,rosterRefs:n,farms:s,threshold:i??rt,limit:r??$t,live:g,subject:k,force:b,ghostUrl:M,label:R,batch:A,algorithm:U,thresholdSet:i!==null}},W=t=>t.includes("*")||t.includes("?"),Z=t=>t==="PUBLIC"||t==="LOCAL"||t==="PRIVATE"||t==="GALAXY";var bt=t=>t.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),dt=new Map,Wt=async(t,e)=>{if(dt.has(t))return dt.get(t);let n=`${e}/system/indexed-domains.json?pattern=${encodeURIComponent(t)}`,s=await fetch(n);if(!s.ok)throw new Error(`indexed-domains failed: ${s.status}`);let i=await s.json();return dt.set(t,i),i},K=async(t,e)=>{t.length||(t=[window.location.hostname]);let n=new Set,s=[];for(let i of t)if(i==="*"||W(i)||Z(i))for(let r of await Wt(i,e))n.has(r.domain)||(n.add(r.domain),s.push(r));else n.has(i)||(n.add(i),s.push({domain:i,page_count:null}));return s},Zt=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,Xt=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,Qt=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,yt=async t=>{let e=t.indexOf("/"),n=await fetch(`//${t.slice(0,e)}/${t.slice(e+1)}.json`);if(!n.ok)throw new Error(`roster page ${t} failed: ${n.status}`);return n.json()},xt=async(t,e,n)=>{if(n.has(t))return;n.add(t);let s;try{s=await yt(t)}catch(i){console.warn(i.message);return}for(let i of s.story||[])if(i.type==="roster")for(let r of(i.text||"").split(/\r?\n/)){let u=r.trim();if(!u)continue;let g=u.match(Zt);if(g){e.add(g[0]);continue}let k=u.match(Xt);if(k){await xt(k[1],e,n);continue}let b=u.match(Qt);if(b)try{let M=await yt(b[1]);for(let R of M.story||[])R.type==="reference"&&R.site&&e.add(R.site)}catch(M){console.warn(M.message)}}},te=async t=>{let e=new Set,n=new Set;for(let s of t)await xt(s,e,n);return[...e]},St=async(t,e)=>{let n=typeof window<"u"&&window.location?window.location.hostname:null,s=t.map(i=>i.toUpperCase()==="SITE"&&n?n:i);return e.length?[...s,...await te(e)]:s};var ut=new Map,ee=t=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(t)}`,vt=async t=>{if(ut.has(t))return ut.get(t);let e=await fetch(ee(t));if(!e.ok)return[];let n=await e.json();return ut.set(t,n),n},X=async(t,e)=>{let n=await fetch(`${e}/system/embed.json`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({text:t})});if(!n.ok)throw new Error(`embed failed: ${n.status}`);return(await n.json()).vector},kt=async(t,e)=>{let s=(await vt(e)).find(i=>i.slug===t);return s?s.vector:null},Q=(t,e,{threshold:n,limit:s,excludeSlug:i,excludeDomain:r})=>{let u=[];for(let{domain:g,pages:k}of e)for(let{slug:b,title:M,vector:R}of k){if(b===i&&g===r)continue;let A=0;for(let U=0;U<t.length;U++)A+=t[U]*R[U];A>=n&&u.push({domain:g,slug:b,title:M,score:A})}return u.sort((g,k)=>k.score-g.score),u.slice(0,s)},tt=async(t,e)=>{let n=await K(t,e);return(await Promise.all(n.map(async({domain:i})=>({domain:i,pages:await vt(i)})))).filter(i=>i.pages.length>0)};var Lt=t=>{let e=t.parents(".page:first"),n=e.prev(".page"),s=n.length?n:e,i=(s.attr("id")||"").split("_rev")[0],r=s.data("site")||window.location.hostname,u=s.data("data")||{},g=u.title||s.find(".title").text().trim()||i,k=[g,...(u.story||[]).filter(b=>b.type==="markdown"||b.type==="paragraph").map(b=>(b.text||"").trim()).filter(Boolean)].join(`
 `).slice(0,2e3);return{slug:i,site:r,title:g,text:k,isSelf:!n.length}},B=t=>`Subject: ${t.title}${t.isSelf?" (this page)":` @ ${t.site}`}`;var Et=t=>`sim-cache-${t}`,jt=t=>{try{let e=JSON.parse(localStorage.getItem(Et(t.id))||"null");return e?.text===(t.text||"")?e:null}catch{return null}},Y=(t,e)=>{try{localStorage.setItem(Et(t.id),JSON.stringify({text:t.text||"",ts:Date.now(),...e}))}catch{}},Ct=t=>{let e=Math.floor((Date.now()-t)/1e3);return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`};var ne=()=>{let t=new Set;try{for(let e of Object.keys(window.wiki?.neighborhood||{}))t.add(e);document.querySelectorAll(".page").forEach(e=>{let n=window.$?window.$(e).data("site"):null;t.add(n||window.location.hostname)})}catch{}return[...t]},Tt=async(t,e)=>{let n=await fetch(t,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e)});if(!n.ok){let s=new Error(`${t.split("/").pop()} failed: ${n.status}`);throw s.status=n.status,s.body=await n.json().catch(()=>({})),s}return n.json()},se=(t,e,n)=>{let s=t.slice(0,n).map(r=>`${r.site} ${r.slug}`).join("|"),i=e.slice(0,n).map(r=>`${r.site} ${r.slug}`).join("|");return s===i},It=async t=>{let{origin:e,query:n,seeded:s={},specs:i,limit:r,batch:u,algorithm:g,render:k}=t,b=q=>t.status&&t.status(q),M={roster:[...t.roster||[],...g&&g.liked||[]],neighborhood:ne(),followed:!0,...t.prefer||{}};b("Ranking sites\u2026");let R=await Tt(`${e}/system/site-rank.json`,{query:n,...s,domains:i,prefer:M,algorithm:g||{},batch:u||50}),A=R.batches||[],w={searched:0,total:R.count||A.reduce((q,F)=>q+F.length,0),batches:A.length,done:0,stopped:!1,converged:!1,running:!0,unindexed:R.unindexed||[]},v=new Map,a=[],d=0,y=()=>{w.stopped=!0};w.stop=y;let C=async q=>{let F={query:n,...s,domains:q,limit:Math.max(r*3,30),flat:!0};t.thresholdSet&&(F.threshold=t.threshold);let m=await Tt(`${e}/system/search-report.json`,F);for(let h of m.results||[]){let l=`${h.site} ${h.slug}`,o=v.get(l);(!o||o.score<h.score)&&v.set(l,h)}let p=[...v.values()].sort((h,l)=>l.score-h.score),c=se(a,p,r);a=p,w.searched+=q.length,w.done+=1,d=c?d+1:0},V=()=>A.slice(w.done),_=async(q=!1)=>{for(w.running=!0,w.stopped=!1;w.done<A.length&&!w.stopped;)if(await C(A[w.done]),k(a,w),!q&&d>=3&&w.done<A.length){w.converged=!0;break}return w.running=!1,k(a,w),a};return w.continueAll=()=>(w.converged=!1,d=-1/0,_(!0)),await _(!1),{results:a,state:w,remaining:V}};var nt="similarity:learned",oe=new Set(["liked","visited","neighbourhood","neighborhood","followed","centroid","fresh"]),et=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,ht=t=>{let e={weights:{},always:[],never:[],batch:null};for(let n of(t||"").split(`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-plugin-similarity",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Federated Wiki plugin — semantic similarity search across wiki domains using prebuilt vector indexes",
   "keywords": [
     "wiki",

package/pages/about-similarity-plugin

@@ -96,10 +96,20 @@
       "id": "b6fe791b56728d2c",
       "text": "0.19.0 searches the federation a batch at a time, nearest sites first \u2014 the [https://search.fedwiki.club/view/incremental-federated-search Incremental Federated Search] design. Underneath is the **Site Index**: one vector per site for every site the galaxy registry knows (built nightly by the galaxy writer into `sites.json`, served at `galaxy-sites.json`, fetched from a peer by farms without a galaxy tree), so a query ranks thousands of sites before it reads a single page vector \u2014 the resource-selection step of federated search. `site-rank.json` returns that order: the sites you are near first (an item's `ROSTER` lines, the lineup's neighbourhood, the registry's followed tier, the reader's own [[Search Algorithm]]), then everything else by centroid similarity, ties to the freshest index. A `REPORT` whose scope reaches `GALAXY` \u2014 or any scope with `BATCH n` \u2014 then searches those sites fifty at a time through `search-report.json {flat: true}`, merging in place and stopping when three consecutive batches leave the top results unchanged, with *Search the remaining sites* behind a button and *Open as page* for the merged result. Sites that rank highly while carrying no page vectors are listed, and logged, so the galaxy scout promotes them: what people search for decides what is embedded next. The cascade for farms with no galaxy of their own: `similarity.json` names `peers` in preference order, the Site Index comes from the first, and batch domains a farm holds no vectors for are asked of that peer and merged back marked by source. **Search Algorithm** is a second plugin-served page \u2014 rosters of sites liked and visited, and an `ALGORITHM` item whose `WEIGHT`, `ALWAYS`, `NEVER` and `BATCH` lines the ranker reads \u2014 that a fork makes yours; the browser learns which sites you are near, open and click (decaying weekly, never leaving the browser), and **Remember** writes the top of that into the page's roster through an ordinary save. Off-farm pages built from items the prose count never sees are no longer dropped as stubs when they carry four or more items \u2014 27% of galaxy pages had under 120 characters of prose, most of them none \u2014 and `domains` may be a string as well as an array."
     },
+    {
+      "type": "markdown",
+      "id": "572d1df31653a684",
+      "text": "0.19.1 makes an explicit-domain batch cheap. A scope of fifty named domains was matched against every directory of a 1,400-site tree with the glob matcher, which builds a matrix per comparison \u2014 a second and a half per batch on the Pi5. A pattern with no wildcard is now compared by equality."
+    },
+    {
+      "type": "markdown",
+      "id": "d3a90f5b71c26e48",
+      "text": "0.20.0 answers **SITES over GALAXY** by the stored site vector first. Asking which site a page belongs on used to read every page vector of every federation site \u2014 104,000 of them to rank a few hundred things. The Site Index has one vector per site, so the field is narrowed to the 250 closest by centroid before any page is read, and only those are read in full for the evidence a placement answer shows. Measured on the Pi5: 971 sites considered, 250 read, 104,425 pages down to 72,910, and 2.7 s down to **0.49 s**, with the top five identical to the full scan.\n\nThe ranking here is centroid **only**. `rankSites` normally carries the reader's preferences \u2014 followed tier, rosters, learned scores \u2014 which is right for choosing what to search first for a person and wrong for placement: with them on, a followed-tier bonus pushed the second-best site out of the field and pulled in one ranked #108. Farm scope is left scanning everything: it is ours, it is bounded, and a diluted centroid should not hide a site you own."
+    },
     {
       "type": "markdown",
       "id": "21eb5354d2adb29e",
-      "text": "**Version:** 0.19.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
+      "text": "**Version:** 0.20.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
     }
   ],
   "plugin": "similarity",
@@ -355,6 +365,27 @@
         "text": "**Version:** 0.19.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
       },
       "date": 1788507728115
+    },
+    {
+      "type": "add",
+      "id": "572d1df31653a684",
+      "item": {
+        "type": "markdown",
+        "id": "572d1df31653a684",
+        "text": "0.19.1 makes an explicit-domain batch cheap. A scope of fifty named domains was matched against every directory of a 1,400-site tree with the glob matcher, which builds a matrix per comparison \u2014 a second and a half per batch on the Pi5. A pattern with no wildcard is now compared by equality."
+      },
+      "after": null,
+      "date": 1788508975320
+    },
+    {
+      "type": "edit",
+      "id": "21eb5354d2adb29e",
+      "item": {
+        "type": "markdown",
+        "id": "21eb5354d2adb29e",
+        "text": "**Version:** 0.19.1\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
+      },
+      "date": 1788508975320
     }
   ]
 }

package/server/farm-lib.js

@@ -82,6 +82,11 @@ const loadRestricted = (publicFarms, opts = {}) => {
   return restricted
 }
 
+// An explicit domain is compared by equality: globMatch builds a p×s matrix
+// per call, and a fifty-domain batch against a 1,400-directory tree was
+// spending a second and a half on it (measured on the Pi5, 2026-09-04).
+const isGlobPattern = p => p.includes('*') || p.includes('?')
+
 const matchesAny = (domain, kind, patterns, restricted) =>
   patterns.some(p => {
     if (p === '*') return kind !== 'galaxy'
@@ -89,7 +94,7 @@ const matchesAny = (domain, kind, patterns, restricted) =>
     if (p === 'LOCAL') return kind === 'local'
     if (p === 'PRIVATE') return kind !== 'galaxy' && restricted.has(domain)
     if (p === 'GALAXY') return kind === 'galaxy'
-    return globMatch(p, domain)
+    return isGlobPattern(p) ? globMatch(p, domain) : p === domain
   })
 
 // ── Domain listing ────────────────────────────────────────────────────────────

package/server/site-report.js

@@ -24,6 +24,8 @@ const path   = require('node:path')
 const crypto = require('node:crypto')
 
 const { listDomains } = require('./farm-lib')
+const { loadSiteIndex } = require('./site-index')
+const { rankSites: rankByCentroid } = require('./site-rank')
 const { loadVectors, dot, seedVector } = require('./search-report')
 
 const SITE_TOP_K          = 5     // neighbourhood size
@@ -33,6 +35,23 @@ const SITE_FLOOR          = 0.50  // sites whose topK falls below → dropped
 const HIT_THRESHOLD       = 0.55  // evidence count, matches search-report's floor
 const TOP_PAGES_SHOWN     = 3
 
+// Resource selection for the federation. Answering "which SITE" by reading
+// every page vector of every site means 104,000 reads to rank a few hundred
+// things — and beyond our own farm those are other people's wikis (see the
+// Polite Index Plan). With the Site Index there is one stored vector per site,
+// so the field can be narrowed before any page is touched. Only the strongest
+// sites are then read in full, because the evidence a placement answer shows —
+// topK, hits, the pages themselves — still needs real pages.
+//
+// Farm scope is left alone: it is ours, it is bounded, and a diluted centroid
+// on a broad person-pod should not hide a site we own.
+const PRESELECT_MIN_DOMAINS = 120  // below this, scanning everything is cheap
+// Wide enough that a site whose pages answer well is not lost because its
+// centroid is diluted: on the pattern-language query the true top five sat at
+// centroid ranks 1, 3, 26, 104 and 108, so a narrow field would have dropped
+// two of them. 250 of 1,385 keeps the answer and still skips three quarters.
+const PRESELECT_KEEP        = 250  // sites read in full afterwards
+
 const makeId = () => crypto.randomBytes(8).toString('hex')
 
 // ── Scoring — one pass per site over vectors already in memory ───────────────
@@ -150,6 +169,35 @@ const buildSiteReport = async (query, specs, limit,
   if (exclude) domains = domains.filter(d => !exclude.has(d.domain))
   const qvec = await seedVector(seedOpts, query, farms, embed)
 
+  // Narrow by stored site vector before reading any pages.
+  const reachesGalaxy = useSpecs.some(s2 => String(s2).toUpperCase() === 'GALAXY')
+  let considered = null
+  if (reachesGalaxy && domains.length > PRESELECT_MIN_DOMAINS) {
+    // farms is [[rootPath, kind], …]; the galaxy tree is the one kind 'galaxy'.
+    const galaxyDir = (farms.find(([, kind]) => kind === 'galaxy') || [])[0] || null
+    const index = loadSiteIndex(galaxyDir)
+    if (index) {
+      // Centroid alone. rankSites also carries the reader's preferences — the
+      // followed tier, rosters, learned scores — which is right for deciding
+      // what to search first for a person, and wrong here: "which site should
+      // this page live on" is not a question about whose wikis you like. Left
+      // personalised, a followed-tier bonus of 0.2 swamped centroid gaps and
+      // pushed the second-best site (0.890) out of the field while pulling in
+      // one ranked #108. Weights zeroed so only topical fit orders the field.
+      const OBJECTIVE = { weights: { liked: 0, visited: 0, neighbourhood: 0,
+                                     followed: 0, centroid: 1, fresh: 0 } }
+      const order = new Map(
+        rankByCentroid(qvec, index, [], {}, OBJECTIVE).map((r, i) => [r.domain, i]))
+      const known = domains.filter(d => order.has(d.domain))
+      // A site the index has not caught up with is not silently dropped; it
+      // keeps its place behind the ranked ones rather than losing its turn.
+      const unknown = domains.filter(d => !order.has(d.domain))
+      known.sort((a, b) => order.get(a.domain) - order.get(b.domain))
+      considered = domains.length
+      domains = known.concat(unknown).slice(0, PRESELECT_KEEP)
+    }
+  }
+
   const ex = seedOpts.excludePage
   let totalPages = 0
   const entries = domains.map(({ farm, domain }) => {
@@ -174,6 +222,8 @@ const buildSiteReport = async (query, specs, limit,
   }))
 
   const stats = { domains: domains.length, pages: totalPages, above_floor: ranked.length }
+  // Say so when most of the field was judged on its site vector alone.
+  if (considered !== null) { stats.considered = considered; stats.preselected = true }
   if (format === 'flat') return { query, scanned: stats, sites }
   return siteReportPage(query, sites, stats, limit, useSpecs)
 }