wiki-plugin-similarity 0.19.1 → 0.20.0

package/client/similarity.js

@@ -1,4 +1,4 @@
-/* wiki-plugin-similarity - 0.19.1 - Fri, 04 Sep 2026 08:03:01 GMT */
+/* wiki-plugin-similarity - 0.20.0 - Fri, 04 Sep 2026 09:22:03 GMT */
 (()=>{var lt={high:.78,medium:.68,low:.58},rt=lt.medium,$t=10,ct=t=>{let e=[],n=[],s=[],i=null,r=null,u="search",g=!1,k=!1,b=!1,M=null,R=null,A=null,U=null,w=(a,d)=>a===d||a.startsWith(d)&&/^[\s:]/.test(a.slice(d.length)),v=(a,d)=>a.slice(d.length).replace(/^\s*:?\s*/,"").trim();for(let a of t.split(`
 `)){let d=a.trim();if(!d||d.startsWith("#"))continue;let y=d.toUpperCase();if(w(y,"LIVE")){g=!0;continue}if(w(y,"SUBJECT")){k=!0;continue}if(w(y,"AUTHOR")){!e.length&&u==="search"&&(u="author");continue}if(w(y,"REPORT")){u==="search"&&(u="report");continue}if(w(y,"KEYWORD")){u==="search"&&(u="keyword");continue}if(w(y,"SITES")){u==="search"&&(u="sites");continue}if(w(y,"BUILD")){u==="search"&&(u="build");continue}if(w(y,"FORCE")){b=!0;continue}if(w(y,"GHOST")){M=v(d,"GHOST"),u==="search"&&(u="ghost");continue}if(w(y,"BUTTON")){R=v(d,"BUTTON");continue}if(w(y,"BATCH")){let C=v(d,"BATCH").toLowerCase();A=C==="off"||C==="no"||C==="0"?0:parseInt(C)||50;continue}if(w(y,"ALGORITHM")){U=v(d,"ALGORITHM").replace(/^\[\[|\]\]$/g,"").trim()||"search-algorithm",u==="search"&&!e.length&&(u="algorithm");continue}if(w(y,"ROSTER")){let C=v(d,"ROSTER");C&&n.push(C);continue}if(w(y,"FARM")){let C=v(d,"FARM");C&&s.push(C);continue}if(w(y,"LIST")){!e.length&&u==="search"&&(u="list");continue}if(w(y,"STATUS")){u==="search"&&(u="status");continue}if(w(y,"SIMILAR")){let C=v(y,"SIMILAR").toLowerCase();i=lt[C]||rt,!e.length&&u==="search"&&(u="similar");continue}if(w(y,"THRESHOLD")){let C=v(d,"THRESHOLD");i=lt[C.toLowerCase()]??(parseFloat(C)||rt);continue}if(w(y,"LIMIT")){r=parseInt(v(d,"LIMIT"))||$t;continue}e.push(["PUBLIC","LOCAL","PRIVATE","GALAXY","SITE"].includes(y)?y:d)}return{mode:u,specs:e,rosterRefs:n,farms:s,threshold:i??rt,limit:r??$t,live:g,subject:k,force:b,ghostUrl:M,label:R,batch:A,algorithm:U,thresholdSet:i!==null}},W=t=>t.includes("*")||t.includes("?"),Z=t=>t==="PUBLIC"||t==="LOCAL"||t==="PRIVATE"||t==="GALAXY";var bt=t=>t.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),dt=new Map,Wt=async(t,e)=>{if(dt.has(t))return dt.get(t);let n=`${e}/system/indexed-domains.json?pattern=${encodeURIComponent(t)}`,s=await fetch(n);if(!s.ok)throw new Error(`indexed-domains failed: ${s.status}`);let i=await s.json();return dt.set(t,i),i},K=async(t,e)=>{t.length||(t=[window.location.hostname]);let n=new Set,s=[];for(let i of t)if(i==="*"||W(i)||Z(i))for(let r of await Wt(i,e))n.has(r.domain)||(n.add(r.domain),s.push(r));else n.has(i)||(n.add(i),s.push({domain:i,page_count:null}));return s},Zt=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,Xt=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,Qt=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,yt=async t=>{let e=t.indexOf("/"),n=await fetch(`//${t.slice(0,e)}/${t.slice(e+1)}.json`);if(!n.ok)throw new Error(`roster page ${t} failed: ${n.status}`);return n.json()},xt=async(t,e,n)=>{if(n.has(t))return;n.add(t);let s;try{s=await yt(t)}catch(i){console.warn(i.message);return}for(let i of s.story||[])if(i.type==="roster")for(let r of(i.text||"").split(/\r?\n/)){let u=r.trim();if(!u)continue;let g=u.match(Zt);if(g){e.add(g[0]);continue}let k=u.match(Xt);if(k){await xt(k[1],e,n);continue}let b=u.match(Qt);if(b)try{let M=await yt(b[1]);for(let R of M.story||[])R.type==="reference"&&R.site&&e.add(R.site)}catch(M){console.warn(M.message)}}},te=async t=>{let e=new Set,n=new Set;for(let s of t)await xt(s,e,n);return[...e]},St=async(t,e)=>{let n=typeof window<"u"&&window.location?window.location.hostname:null,s=t.map(i=>i.toUpperCase()==="SITE"&&n?n:i);return e.length?[...s,...await te(e)]:s};var ut=new Map,ee=t=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(t)}`,vt=async t=>{if(ut.has(t))return ut.get(t);let e=await fetch(ee(t));if(!e.ok)return[];let n=await e.json();return ut.set(t,n),n},X=async(t,e)=>{let n=await fetch(`${e}/system/embed.json`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({text:t})});if(!n.ok)throw new Error(`embed failed: ${n.status}`);return(await n.json()).vector},kt=async(t,e)=>{let s=(await vt(e)).find(i=>i.slug===t);return s?s.vector:null},Q=(t,e,{threshold:n,limit:s,excludeSlug:i,excludeDomain:r})=>{let u=[];for(let{domain:g,pages:k}of e)for(let{slug:b,title:M,vector:R}of k){if(b===i&&g===r)continue;let A=0;for(let U=0;U<t.length;U++)A+=t[U]*R[U];A>=n&&u.push({domain:g,slug:b,title:M,score:A})}return u.sort((g,k)=>k.score-g.score),u.slice(0,s)},tt=async(t,e)=>{let n=await K(t,e);return(await Promise.all(n.map(async({domain:i})=>({domain:i,pages:await vt(i)})))).filter(i=>i.pages.length>0)};var Lt=t=>{let e=t.parents(".page:first"),n=e.prev(".page"),s=n.length?n:e,i=(s.attr("id")||"").split("_rev")[0],r=s.data("site")||window.location.hostname,u=s.data("data")||{},g=u.title||s.find(".title").text().trim()||i,k=[g,...(u.story||[]).filter(b=>b.type==="markdown"||b.type==="paragraph").map(b=>(b.text||"").trim()).filter(Boolean)].join(`
 `).slice(0,2e3);return{slug:i,site:r,title:g,text:k,isSelf:!n.length}},B=t=>`Subject: ${t.title}${t.isSelf?" (this page)":` @ ${t.site}`}`;var Et=t=>`sim-cache-${t}`,jt=t=>{try{let e=JSON.parse(localStorage.getItem(Et(t.id))||"null");return e?.text===(t.text||"")?e:null}catch{return null}},Y=(t,e)=>{try{localStorage.setItem(Et(t.id),JSON.stringify({text:t.text||"",ts:Date.now(),...e}))}catch{}},Ct=t=>{let e=Math.floor((Date.now()-t)/1e3);return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`};var ne=()=>{let t=new Set;try{for(let e of Object.keys(window.wiki?.neighborhood||{}))t.add(e);document.querySelectorAll(".page").forEach(e=>{let n=window.$?window.$(e).data("site"):null;t.add(n||window.location.hostname)})}catch{}return[...t]},Tt=async(t,e)=>{let n=await fetch(t,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e)});if(!n.ok){let s=new Error(`${t.split("/").pop()} failed: ${n.status}`);throw s.status=n.status,s.body=await n.json().catch(()=>({})),s}return n.json()},se=(t,e,n)=>{let s=t.slice(0,n).map(r=>`${r.site} ${r.slug}`).join("|"),i=e.slice(0,n).map(r=>`${r.site} ${r.slug}`).join("|");return s===i},It=async t=>{let{origin:e,query:n,seeded:s={},specs:i,limit:r,batch:u,algorithm:g,render:k}=t,b=q=>t.status&&t.status(q),M={roster:[...t.roster||[],...g&&g.liked||[]],neighborhood:ne(),followed:!0,...t.prefer||{}};b("Ranking sites\u2026");let R=await Tt(`${e}/system/site-rank.json`,{query:n,...s,domains:i,prefer:M,algorithm:g||{},batch:u||50}),A=R.batches||[],w={searched:0,total:R.count||A.reduce((q,F)=>q+F.length,0),batches:A.length,done:0,stopped:!1,converged:!1,running:!0,unindexed:R.unindexed||[]},v=new Map,a=[],d=0,y=()=>{w.stopped=!0};w.stop=y;let C=async q=>{let F={query:n,...s,domains:q,limit:Math.max(r*3,30),flat:!0};t.thresholdSet&&(F.threshold=t.threshold);let m=await Tt(`${e}/system/search-report.json`,F);for(let h of m.results||[]){let l=`${h.site} ${h.slug}`,o=v.get(l);(!o||o.score<h.score)&&v.set(l,h)}let p=[...v.values()].sort((h,l)=>l.score-h.score),c=se(a,p,r);a=p,w.searched+=q.length,w.done+=1,d=c?d+1:0},V=()=>A.slice(w.done),_=async(q=!1)=>{for(w.running=!0,w.stopped=!1;w.done<A.length&&!w.stopped;)if(await C(A[w.done]),k(a,w),!q&&d>=3&&w.done<A.length){w.converged=!0;break}return w.running=!1,k(a,w),a};return w.continueAll=()=>(w.converged=!1,d=-1/0,_(!0)),await _(!1),{results:a,state:w,remaining:V}};var nt="similarity:learned",oe=new Set(["liked","visited","neighbourhood","neighborhood","followed","centroid","fresh"]),et=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,ht=t=>{let e={weights:{},always:[],never:[],batch:null};for(let n of(t||"").split(`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-plugin-similarity",
-  "version": "0.19.1",
+  "version": "0.20.0",
   "description": "Federated Wiki plugin — semantic similarity search across wiki domains using prebuilt vector indexes",
   "keywords": [
     "wiki",

package/pages/about-similarity-plugin

@@ -101,10 +101,15 @@
       "id": "572d1df31653a684",
       "text": "0.19.1 makes an explicit-domain batch cheap. A scope of fifty named domains was matched against every directory of a 1,400-site tree with the glob matcher, which builds a matrix per comparison \u2014 a second and a half per batch on the Pi5. A pattern with no wildcard is now compared by equality."
     },
+    {
+      "type": "markdown",
+      "id": "d3a90f5b71c26e48",
+      "text": "0.20.0 answers **SITES over GALAXY** by the stored site vector first. Asking which site a page belongs on used to read every page vector of every federation site \u2014 104,000 of them to rank a few hundred things. The Site Index has one vector per site, so the field is narrowed to the 250 closest by centroid before any page is read, and only those are read in full for the evidence a placement answer shows. Measured on the Pi5: 971 sites considered, 250 read, 104,425 pages down to 72,910, and 2.7 s down to **0.49 s**, with the top five identical to the full scan.\n\nThe ranking here is centroid **only**. `rankSites` normally carries the reader's preferences \u2014 followed tier, rosters, learned scores \u2014 which is right for choosing what to search first for a person and wrong for placement: with them on, a followed-tier bonus pushed the second-best site out of the field and pulled in one ranked #108. Farm scope is left scanning everything: it is ours, it is bounded, and a diluted centroid should not hide a site you own."
+    },
     {
       "type": "markdown",
       "id": "21eb5354d2adb29e",
-      "text": "**Version:** 0.19.1\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
+      "text": "**Version:** 0.20.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
     }
   ],
   "plugin": "similarity",

package/server/site-report.js

@@ -24,6 +24,8 @@ const path   = require('node:path')
 const crypto = require('node:crypto')
 
 const { listDomains } = require('./farm-lib')
+const { loadSiteIndex } = require('./site-index')
+const { rankSites: rankByCentroid } = require('./site-rank')
 const { loadVectors, dot, seedVector } = require('./search-report')
 
 const SITE_TOP_K          = 5     // neighbourhood size
@@ -33,6 +35,23 @@ const SITE_FLOOR          = 0.50  // sites whose topK falls below → dropped
 const HIT_THRESHOLD       = 0.55  // evidence count, matches search-report's floor
 const TOP_PAGES_SHOWN     = 3
 
+// Resource selection for the federation. Answering "which SITE" by reading
+// every page vector of every site means 104,000 reads to rank a few hundred
+// things — and beyond our own farm those are other people's wikis (see the
+// Polite Index Plan). With the Site Index there is one stored vector per site,
+// so the field can be narrowed before any page is touched. Only the strongest
+// sites are then read in full, because the evidence a placement answer shows —
+// topK, hits, the pages themselves — still needs real pages.
+//
+// Farm scope is left alone: it is ours, it is bounded, and a diluted centroid
+// on a broad person-pod should not hide a site we own.
+const PRESELECT_MIN_DOMAINS = 120  // below this, scanning everything is cheap
+// Wide enough that a site whose pages answer well is not lost because its
+// centroid is diluted: on the pattern-language query the true top five sat at
+// centroid ranks 1, 3, 26, 104 and 108, so a narrow field would have dropped
+// two of them. 250 of 1,385 keeps the answer and still skips three quarters.
+const PRESELECT_KEEP        = 250  // sites read in full afterwards
+
 const makeId = () => crypto.randomBytes(8).toString('hex')
 
 // ── Scoring — one pass per site over vectors already in memory ───────────────
@@ -150,6 +169,35 @@ const buildSiteReport = async (query, specs, limit,
   if (exclude) domains = domains.filter(d => !exclude.has(d.domain))
   const qvec = await seedVector(seedOpts, query, farms, embed)
 
+  // Narrow by stored site vector before reading any pages.
+  const reachesGalaxy = useSpecs.some(s2 => String(s2).toUpperCase() === 'GALAXY')
+  let considered = null
+  if (reachesGalaxy && domains.length > PRESELECT_MIN_DOMAINS) {
+    // farms is [[rootPath, kind], …]; the galaxy tree is the one kind 'galaxy'.
+    const galaxyDir = (farms.find(([, kind]) => kind === 'galaxy') || [])[0] || null
+    const index = loadSiteIndex(galaxyDir)
+    if (index) {
+      // Centroid alone. rankSites also carries the reader's preferences — the
+      // followed tier, rosters, learned scores — which is right for deciding
+      // what to search first for a person, and wrong here: "which site should
+      // this page live on" is not a question about whose wikis you like. Left
+      // personalised, a followed-tier bonus of 0.2 swamped centroid gaps and
+      // pushed the second-best site (0.890) out of the field while pulling in
+      // one ranked #108. Weights zeroed so only topical fit orders the field.
+      const OBJECTIVE = { weights: { liked: 0, visited: 0, neighbourhood: 0,
+                                     followed: 0, centroid: 1, fresh: 0 } }
+      const order = new Map(
+        rankByCentroid(qvec, index, [], {}, OBJECTIVE).map((r, i) => [r.domain, i]))
+      const known = domains.filter(d => order.has(d.domain))
+      // A site the index has not caught up with is not silently dropped; it
+      // keeps its place behind the ranked ones rather than losing its turn.
+      const unknown = domains.filter(d => !order.has(d.domain))
+      known.sort((a, b) => order.get(a.domain) - order.get(b.domain))
+      considered = domains.length
+      domains = known.concat(unknown).slice(0, PRESELECT_KEEP)
+    }
+  }
+
   const ex = seedOpts.excludePage
   let totalPages = 0
   const entries = domains.map(({ farm, domain }) => {
@@ -174,6 +222,8 @@ const buildSiteReport = async (query, specs, limit,
   }))
 
   const stats = { domains: domains.length, pages: totalPages, above_floor: ranked.length }
+  // Say so when most of the field was judged on its site vector alone.
+  if (considered !== null) { stats.considered = considered; stats.preselected = true }
   if (format === 'flat') return { query, scanned: stats, sites }
   return siteReportPage(query, sites, stats, limit, useSpecs)
 }