wiki-plugin-similarity 0.19.0 → 0.19.1

package/client/similarity.js

@@ -1,4 +1,4 @@
-/* wiki-plugin-similarity - 0.19.0 - Fri, 04 Sep 2026 07:52:20 GMT */
+/* wiki-plugin-similarity - 0.19.1 - Fri, 04 Sep 2026 08:03:01 GMT */
 (()=>{var lt={high:.78,medium:.68,low:.58},rt=lt.medium,$t=10,ct=t=>{let e=[],n=[],s=[],i=null,r=null,u="search",g=!1,k=!1,b=!1,M=null,R=null,A=null,U=null,w=(a,d)=>a===d||a.startsWith(d)&&/^[\s:]/.test(a.slice(d.length)),v=(a,d)=>a.slice(d.length).replace(/^\s*:?\s*/,"").trim();for(let a of t.split(`
 `)){let d=a.trim();if(!d||d.startsWith("#"))continue;let y=d.toUpperCase();if(w(y,"LIVE")){g=!0;continue}if(w(y,"SUBJECT")){k=!0;continue}if(w(y,"AUTHOR")){!e.length&&u==="search"&&(u="author");continue}if(w(y,"REPORT")){u==="search"&&(u="report");continue}if(w(y,"KEYWORD")){u==="search"&&(u="keyword");continue}if(w(y,"SITES")){u==="search"&&(u="sites");continue}if(w(y,"BUILD")){u==="search"&&(u="build");continue}if(w(y,"FORCE")){b=!0;continue}if(w(y,"GHOST")){M=v(d,"GHOST"),u==="search"&&(u="ghost");continue}if(w(y,"BUTTON")){R=v(d,"BUTTON");continue}if(w(y,"BATCH")){let C=v(d,"BATCH").toLowerCase();A=C==="off"||C==="no"||C==="0"?0:parseInt(C)||50;continue}if(w(y,"ALGORITHM")){U=v(d,"ALGORITHM").replace(/^\[\[|\]\]$/g,"").trim()||"search-algorithm",u==="search"&&!e.length&&(u="algorithm");continue}if(w(y,"ROSTER")){let C=v(d,"ROSTER");C&&n.push(C);continue}if(w(y,"FARM")){let C=v(d,"FARM");C&&s.push(C);continue}if(w(y,"LIST")){!e.length&&u==="search"&&(u="list");continue}if(w(y,"STATUS")){u==="search"&&(u="status");continue}if(w(y,"SIMILAR")){let C=v(y,"SIMILAR").toLowerCase();i=lt[C]||rt,!e.length&&u==="search"&&(u="similar");continue}if(w(y,"THRESHOLD")){let C=v(d,"THRESHOLD");i=lt[C.toLowerCase()]??(parseFloat(C)||rt);continue}if(w(y,"LIMIT")){r=parseInt(v(d,"LIMIT"))||$t;continue}e.push(["PUBLIC","LOCAL","PRIVATE","GALAXY","SITE"].includes(y)?y:d)}return{mode:u,specs:e,rosterRefs:n,farms:s,threshold:i??rt,limit:r??$t,live:g,subject:k,force:b,ghostUrl:M,label:R,batch:A,algorithm:U,thresholdSet:i!==null}},W=t=>t.includes("*")||t.includes("?"),Z=t=>t==="PUBLIC"||t==="LOCAL"||t==="PRIVATE"||t==="GALAXY";var bt=t=>t.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),dt=new Map,Wt=async(t,e)=>{if(dt.has(t))return dt.get(t);let n=`${e}/system/indexed-domains.json?pattern=${encodeURIComponent(t)}`,s=await fetch(n);if(!s.ok)throw new Error(`indexed-domains failed: ${s.status}`);let i=await s.json();return dt.set(t,i),i},K=async(t,e)=>{t.length||(t=[window.location.hostname]);let n=new Set,s=[];for(let i of t)if(i==="*"||W(i)||Z(i))for(let r of await Wt(i,e))n.has(r.domain)||(n.add(r.domain),s.push(r));else n.has(i)||(n.add(i),s.push({domain:i,page_count:null}));return s},Zt=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,Xt=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,Qt=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,yt=async t=>{let e=t.indexOf("/"),n=await fetch(`//${t.slice(0,e)}/${t.slice(e+1)}.json`);if(!n.ok)throw new Error(`roster page ${t} failed: ${n.status}`);return n.json()},xt=async(t,e,n)=>{if(n.has(t))return;n.add(t);let s;try{s=await yt(t)}catch(i){console.warn(i.message);return}for(let i of s.story||[])if(i.type==="roster")for(let r of(i.text||"").split(/\r?\n/)){let u=r.trim();if(!u)continue;let g=u.match(Zt);if(g){e.add(g[0]);continue}let k=u.match(Xt);if(k){await xt(k[1],e,n);continue}let b=u.match(Qt);if(b)try{let M=await yt(b[1]);for(let R of M.story||[])R.type==="reference"&&R.site&&e.add(R.site)}catch(M){console.warn(M.message)}}},te=async t=>{let e=new Set,n=new Set;for(let s of t)await xt(s,e,n);return[...e]},St=async(t,e)=>{let n=typeof window<"u"&&window.location?window.location.hostname:null,s=t.map(i=>i.toUpperCase()==="SITE"&&n?n:i);return e.length?[...s,...await te(e)]:s};var ut=new Map,ee=t=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(t)}`,vt=async t=>{if(ut.has(t))return ut.get(t);let e=await fetch(ee(t));if(!e.ok)return[];let n=await e.json();return ut.set(t,n),n},X=async(t,e)=>{let n=await fetch(`${e}/system/embed.json`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({text:t})});if(!n.ok)throw new Error(`embed failed: ${n.status}`);return(await n.json()).vector},kt=async(t,e)=>{let s=(await vt(e)).find(i=>i.slug===t);return s?s.vector:null},Q=(t,e,{threshold:n,limit:s,excludeSlug:i,excludeDomain:r})=>{let u=[];for(let{domain:g,pages:k}of e)for(let{slug:b,title:M,vector:R}of k){if(b===i&&g===r)continue;let A=0;for(let U=0;U<t.length;U++)A+=t[U]*R[U];A>=n&&u.push({domain:g,slug:b,title:M,score:A})}return u.sort((g,k)=>k.score-g.score),u.slice(0,s)},tt=async(t,e)=>{let n=await K(t,e);return(await Promise.all(n.map(async({domain:i})=>({domain:i,pages:await vt(i)})))).filter(i=>i.pages.length>0)};var Lt=t=>{let e=t.parents(".page:first"),n=e.prev(".page"),s=n.length?n:e,i=(s.attr("id")||"").split("_rev")[0],r=s.data("site")||window.location.hostname,u=s.data("data")||{},g=u.title||s.find(".title").text().trim()||i,k=[g,...(u.story||[]).filter(b=>b.type==="markdown"||b.type==="paragraph").map(b=>(b.text||"").trim()).filter(Boolean)].join(`
 `).slice(0,2e3);return{slug:i,site:r,title:g,text:k,isSelf:!n.length}},B=t=>`Subject: ${t.title}${t.isSelf?" (this page)":` @ ${t.site}`}`;var Et=t=>`sim-cache-${t}`,jt=t=>{try{let e=JSON.parse(localStorage.getItem(Et(t.id))||"null");return e?.text===(t.text||"")?e:null}catch{return null}},Y=(t,e)=>{try{localStorage.setItem(Et(t.id),JSON.stringify({text:t.text||"",ts:Date.now(),...e}))}catch{}},Ct=t=>{let e=Math.floor((Date.now()-t)/1e3);return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`};var ne=()=>{let t=new Set;try{for(let e of Object.keys(window.wiki?.neighborhood||{}))t.add(e);document.querySelectorAll(".page").forEach(e=>{let n=window.$?window.$(e).data("site"):null;t.add(n||window.location.hostname)})}catch{}return[...t]},Tt=async(t,e)=>{let n=await fetch(t,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e)});if(!n.ok){let s=new Error(`${t.split("/").pop()} failed: ${n.status}`);throw s.status=n.status,s.body=await n.json().catch(()=>({})),s}return n.json()},se=(t,e,n)=>{let s=t.slice(0,n).map(r=>`${r.site} ${r.slug}`).join("|"),i=e.slice(0,n).map(r=>`${r.site} ${r.slug}`).join("|");return s===i},It=async t=>{let{origin:e,query:n,seeded:s={},specs:i,limit:r,batch:u,algorithm:g,render:k}=t,b=q=>t.status&&t.status(q),M={roster:[...t.roster||[],...g&&g.liked||[]],neighborhood:ne(),followed:!0,...t.prefer||{}};b("Ranking sites\u2026");let R=await Tt(`${e}/system/site-rank.json`,{query:n,...s,domains:i,prefer:M,algorithm:g||{},batch:u||50}),A=R.batches||[],w={searched:0,total:R.count||A.reduce((q,F)=>q+F.length,0),batches:A.length,done:0,stopped:!1,converged:!1,running:!0,unindexed:R.unindexed||[]},v=new Map,a=[],d=0,y=()=>{w.stopped=!0};w.stop=y;let C=async q=>{let F={query:n,...s,domains:q,limit:Math.max(r*3,30),flat:!0};t.thresholdSet&&(F.threshold=t.threshold);let m=await Tt(`${e}/system/search-report.json`,F);for(let h of m.results||[]){let l=`${h.site} ${h.slug}`,o=v.get(l);(!o||o.score<h.score)&&v.set(l,h)}let p=[...v.values()].sort((h,l)=>l.score-h.score),c=se(a,p,r);a=p,w.searched+=q.length,w.done+=1,d=c?d+1:0},V=()=>A.slice(w.done),_=async(q=!1)=>{for(w.running=!0,w.stopped=!1;w.done<A.length&&!w.stopped;)if(await C(A[w.done]),k(a,w),!q&&d>=3&&w.done<A.length){w.converged=!0;break}return w.running=!1,k(a,w),a};return w.continueAll=()=>(w.converged=!1,d=-1/0,_(!0)),await _(!1),{results:a,state:w,remaining:V}};var nt="similarity:learned",oe=new Set(["liked","visited","neighbourhood","neighborhood","followed","centroid","fresh"]),et=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,ht=t=>{let e={weights:{},always:[],never:[],batch:null};for(let n of(t||"").split(`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-plugin-similarity",
-  "version": "0.19.0",
+  "version": "0.19.1",
   "description": "Federated Wiki plugin — semantic similarity search across wiki domains using prebuilt vector indexes",
   "keywords": [
     "wiki",

package/pages/about-similarity-plugin

@@ -96,10 +96,15 @@
       "id": "b6fe791b56728d2c",
       "text": "0.19.0 searches the federation a batch at a time, nearest sites first \u2014 the [https://search.fedwiki.club/view/incremental-federated-search Incremental Federated Search] design. Underneath is the **Site Index**: one vector per site for every site the galaxy registry knows (built nightly by the galaxy writer into `sites.json`, served at `galaxy-sites.json`, fetched from a peer by farms without a galaxy tree), so a query ranks thousands of sites before it reads a single page vector \u2014 the resource-selection step of federated search. `site-rank.json` returns that order: the sites you are near first (an item's `ROSTER` lines, the lineup's neighbourhood, the registry's followed tier, the reader's own [[Search Algorithm]]), then everything else by centroid similarity, ties to the freshest index. A `REPORT` whose scope reaches `GALAXY` \u2014 or any scope with `BATCH n` \u2014 then searches those sites fifty at a time through `search-report.json {flat: true}`, merging in place and stopping when three consecutive batches leave the top results unchanged, with *Search the remaining sites* behind a button and *Open as page* for the merged result. Sites that rank highly while carrying no page vectors are listed, and logged, so the galaxy scout promotes them: what people search for decides what is embedded next. The cascade for farms with no galaxy of their own: `similarity.json` names `peers` in preference order, the Site Index comes from the first, and batch domains a farm holds no vectors for are asked of that peer and merged back marked by source. **Search Algorithm** is a second plugin-served page \u2014 rosters of sites liked and visited, and an `ALGORITHM` item whose `WEIGHT`, `ALWAYS`, `NEVER` and `BATCH` lines the ranker reads \u2014 that a fork makes yours; the browser learns which sites you are near, open and click (decaying weekly, never leaving the browser), and **Remember** writes the top of that into the page's roster through an ordinary save. Off-farm pages built from items the prose count never sees are no longer dropped as stubs when they carry four or more items \u2014 27% of galaxy pages had under 120 characters of prose, most of them none \u2014 and `domains` may be a string as well as an array."
     },
+    {
+      "type": "markdown",
+      "id": "572d1df31653a684",
+      "text": "0.19.1 makes an explicit-domain batch cheap. A scope of fifty named domains was matched against every directory of a 1,400-site tree with the glob matcher, which builds a matrix per comparison \u2014 a second and a half per batch on the Pi5. A pattern with no wildcard is now compared by equality."
+    },
     {
       "type": "markdown",
       "id": "21eb5354d2adb29e",
-      "text": "**Version:** 0.19.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
+      "text": "**Version:** 0.19.1\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
     }
   ],
   "plugin": "similarity",
@@ -355,6 +360,27 @@
         "text": "**Version:** 0.19.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
       },
       "date": 1788507728115
+    },
+    {
+      "type": "add",
+      "id": "572d1df31653a684",
+      "item": {
+        "type": "markdown",
+        "id": "572d1df31653a684",
+        "text": "0.19.1 makes an explicit-domain batch cheap. A scope of fifty named domains was matched against every directory of a 1,400-site tree with the glob matcher, which builds a matrix per comparison \u2014 a second and a half per batch on the Pi5. A pattern with no wildcard is now compared by equality."
+      },
+      "after": null,
+      "date": 1788508975320
+    },
+    {
+      "type": "edit",
+      "id": "21eb5354d2adb29e",
+      "item": {
+        "type": "markdown",
+        "id": "21eb5354d2adb29e",
+        "text": "**Version:** 0.19.1\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
+      },
+      "date": 1788508975320
     }
   ]
 }

package/server/farm-lib.js

@@ -82,6 +82,11 @@ const loadRestricted = (publicFarms, opts = {}) => {
   return restricted
 }
 
+// An explicit domain is compared by equality: globMatch builds a p×s matrix
+// per call, and a fifty-domain batch against a 1,400-directory tree was
+// spending a second and a half on it (measured on the Pi5, 2026-09-04).
+const isGlobPattern = p => p.includes('*') || p.includes('?')
+
 const matchesAny = (domain, kind, patterns, restricted) =>
   patterns.some(p => {
     if (p === '*') return kind !== 'galaxy'
@@ -89,7 +94,7 @@ const matchesAny = (domain, kind, patterns, restricted) =>
     if (p === 'LOCAL') return kind === 'local'
     if (p === 'PRIVATE') return kind !== 'galaxy' && restricted.has(domain)
     if (p === 'GALAXY') return kind === 'galaxy'
-    return globMatch(p, domain)
+    return isGlobPattern(p) ? globMatch(p, domain) : p === domain
   })
 
 // ── Domain listing ────────────────────────────────────────────────────────────