wiki-plugin-similarity 0.25.0 → 0.25.1

package/client/similarity.js

@@ -1,4 +1,4 @@
-/* wiki-plugin-similarity - 0.25.0 - Sat, 05 Sep 2026 15:23:44 GMT */
+/* wiki-plugin-similarity - 0.25.1 - Sat, 05 Sep 2026 19:34:16 GMT */
 (()=>{var ue={high:.78,medium:.68,low:.58},he=ue.medium,je=10,me=e=>{let t=[],n=[],s=[],i=null,a=null,l="search",u=!1,E=!1,f=!1,R=null,O=null,D=null,U=null,p=!1,T=(h,m)=>h===m||h.startsWith(m)&&/^[\s:]/.test(h.slice(m.length)),x=(h,m)=>h.slice(m.length).replace(/^\s*:?\s*/,"").trim();for(let h of e.split(`
 `)){let m=h.trim();if(!m||m.startsWith("#"))continue;let b=m.toUpperCase();if(T(b,"LIVE")){u=!0;continue}if(T(b,"AUTO")){p=!0;continue}if(T(b,"SUBJECT")){E=!0;continue}if(T(b,"AUTHOR")){!t.length&&l==="search"&&(l="author");continue}if(T(b,"REPORT")){l==="search"&&(l="report");continue}if(T(b,"KEYWORD")){l==="search"&&(l="keyword");continue}if(T(b,"SITES")){l==="search"&&(l="sites");continue}if(T(b,"BUILD")){l==="search"&&(l="build");continue}if(T(b,"FORCE")){f=!0;continue}if(T(b,"GHOST")){R=x(m,"GHOST"),l==="search"&&(l="ghost");continue}if(T(b,"BUTTON")){O=x(m,"BUTTON");continue}if(T(b,"BATCH")){let C=x(m,"BATCH").toLowerCase();D=C==="off"||C==="no"||C==="0"?0:parseInt(C)||100;continue}if(T(b,"ALGORITHM")){U=x(m,"ALGORITHM").replace(/^\[\[|\]\]$/g,"").trim()||"search-algorithm",l==="search"&&!t.length&&(l="algorithm");continue}if(T(b,"ROSTER")){let C=x(m,"ROSTER");C&&n.push(C);continue}if(T(b,"FARM")){let C=x(m,"FARM");C&&s.push(C);continue}if(T(b,"LIST")){!t.length&&l==="search"&&(l="list");continue}if(T(b,"STATUS")){l==="search"&&(l="status");continue}if(T(b,"DOOR")){l==="search"&&(l="door");continue}if(T(b,"SIMILAR")){let C=x(b,"SIMILAR").toLowerCase();i=ue[C]||he,!t.length&&l==="search"&&(l="similar");continue}if(T(b,"THRESHOLD")){let C=x(m,"THRESHOLD");i=ue[C.toLowerCase()]??(parseFloat(C)||he);continue}if(T(b,"LIMIT")){a=parseInt(x(m,"LIMIT"))||je;continue}t.push(["PUBLIC","LOCAL","PRIVATE","GALAXY","SITE"].includes(b)?b:m)}return{mode:l,specs:t,rosterRefs:n,farms:s,threshold:i??he,limit:a??je,live:u,subject:E,force:f,ghostUrl:R,label:O,batch:D,algorithm:U,auto:p,thresholdSet:i!==null}},se=e=>e.includes("*")||e.includes("?"),oe=e=>e==="PUBLIC"||e==="LOCAL"||e==="PRIVATE"||e==="GALAXY";var Ie=e=>e.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),pe=new Map,ut=async(e,t)=>{if(pe.has(e))return pe.get(e);let n=`${t}/system/indexed-domains.json?pattern=${encodeURIComponent(e)}`,s=await fetch(n);if(!s.ok)throw new Error(`indexed-domains failed: ${s.status}`);let i=await s.json();return pe.set(e,i),i},te=async(e,t)=>{e.length||(e=[window.location.hostname]);let n=new Set,s=[];for(let i of e)if(i==="*"||se(i)||oe(i))for(let a of await ut(i,t))n.has(a.domain)||(n.add(a.domain),s.push(a));else n.has(i)||(n.add(i),s.push({domain:i,page_count:null}));return s},mt=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,pt=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,ft=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,Ce=async e=>{let t=e.indexOf("/"),n=await fetch(`//${e.slice(0,t)}/${e.slice(t+1)}.json`);if(!n.ok)throw new Error(`roster page ${e} failed: ${n.status}`);return n.json()},Te=async(e,t,n)=>{if(n.has(e))return;n.add(e);let s;try{s=await Ce(e)}catch(i){console.warn(i.message);return}for(let i of s.story||[])if(i.type==="roster")for(let a of(i.text||"").split(/\r?\n/)){let l=a.trim();if(!l)continue;let u=l.match(mt);if(u){t.add(u[0]);continue}let E=l.match(pt);if(E){await Te(E[1],t,n);continue}let f=l.match(ft);if(f)try{let R=await Ce(f[1]);for(let O of R.story||[])O.type==="reference"&&O.site&&t.add(O.site)}catch(R){console.warn(R.message)}}},gt=async e=>{let t=new Set,n=new Set;for(let s of e)await Te(s,t,n);return[...t]},Ae=async(e,t)=>{let n=typeof window<"u"&&window.location?window.location.hostname:null,s=e.map(i=>i.toUpperCase()==="SITE"&&n?n:i);return t.length?[...s,...await gt(t)]:s};var fe=new Map,wt=e=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(e)}`,yt=async e=>{if(fe.has(e))return fe.get(e);let t=await fetch(wt(e));if(!t.ok)return[];let n=await t.json();return fe.set(e,n),n},ge=async(e,t)=>{let n=await fetch(`${t}/system/embed.json`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({text:e})});if(!n.ok)throw new Error(`embed failed: ${n.status}`);return(await n.json()).vector};var we=(e,t,{threshold:n,limit:s,excludeSlug:i,excludeDomain:a})=>{let l=[];for(let{domain:u,pages:E}of t)for(let{slug:f,title:R,vector:O}of E){if(f===i&&u===a)continue;let D=0;for(let U=0;U<e.length;U++)D+=e[U]*O[U];D>=n&&l.push({domain:u,slug:f,title:R,score:D})}return l.sort((u,E)=>E.score-u.score),l.slice(0,s)},ye=async(e,t)=>{let n=await te(e,t);return(await Promise.all(n.map(async({domain:i})=>({domain:i,pages:await yt(i)})))).filter(i=>i.pages.length>0)};var $t=new Set(["selected-plugin-pages","search-tool","like-this-page","semantic-search","search-algorithm","personal-search-demo","search-tool-demo"]),bt=e=>(e.attr("id")||"").split("_rev")[0],Re=e=>{let t=e.parents(".page:first"),n=t.prev(".page");for(;n.length&&$t.has(bt(n));)n=n.prev(".page");let s=n.length?n:t,i=(s.attr("id")||"").split("_rev")[0],a=s.data("site")||window.location.hostname,l=s.data("data")||{},u=l.title||s.find(".title").text().trim()||i,E=[u,...(l.story||[]).filter(f=>f.type==="markdown"||f.type==="paragraph").map(f=>(f.text||"").trim()).filter(Boolean)].join(`
 `).slice(0,2e3);return{slug:i,site:a,title:u,text:E,isSelf:!n.length}},J=e=>`Subject: ${e.title}${e.isSelf?" (this page)":` @ ${e.site}`}`;var Oe=e=>`sim-cache-${e}`,Me=e=>{try{let t=JSON.parse(localStorage.getItem(Oe(e.id))||"null");return t?.text===(e.text||"")?t:null}catch{return null}},ee=(e,t)=>{try{localStorage.setItem(Oe(e.id),JSON.stringify({text:e.text||"",ts:Date.now(),...t}))}catch{}},$e=e=>{let t=Math.floor((Date.now()-e)/1e3);return t<60?`${t}s ago`:t<3600?`${Math.floor(t/60)}m ago`:t<86400?`${Math.floor(t/3600)}h ago`:`${Math.floor(t/86400)}d ago`};var F=e=>`${e.site} ${e.slug}`,xt=(e,t,n)=>{let s=new Set(e.map(F));return t.slice(0,n).filter(i=>!s.has(F(i))).length},vt=(e,t,n,s)=>{let i=new Map;e.forEach((R,O)=>{s.has(F(R))&&i.set(O,R)});let a=new Set([...i.values()].map(F)),l=t.filter(R=>!a.has(F(R))),u=[],E=0,f=Math.max(n,i.size?Math.max(...i.keys())+1:0);for(let R=0;R<f;R++)i.has(R)?u.push(i.get(R)):E<l.length&&u.push(l[E++]);return u},St=(e,t)=>{let n=new Map(t.map(s=>[F(s),s]));return e.map(s=>n.get(F(s))||s)},kt=()=>{let e=new Set;try{for(let t of Object.keys(window.wiki?.neighborhood||{}))e.add(t);document.querySelectorAll(".page").forEach(t=>{let n=window.$?window.$(t).data("site"):null;e.add(n||window.location.hostname)})}catch{}return[...e]},De=async(e,t)=>{let n=await fetch(e,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)});if(!n.ok){let s=new Error(`${e.split("/").pop()} failed: ${n.status}`);throw s.status=n.status,s.body=await n.json().catch(()=>({})),s}return n.json()},Lt=(e,t,n)=>{let s=e.slice(0,n).map(F).join("|"),i=t.slice(0,n).map(F).join("|");return s===i},Pe=async e=>{let{origin:t,query:n,seeded:s={},specs:i,limit:a,batch:l,algorithm:u,render:E,resume:f}=e,R=M=>e.status&&e.status(M),O;if(f&&Array.isArray(f.remaining))O={batches:f.remaining,count:f.total,sites:f.sites||[],siteIndex:f.siteIndex||null,unindexed:f.unindexed||[]};else{let M={roster:[...e.roster||[],...u&&u.liked||[]],neighborhood:kt(),followed:!0,...e.prefer||{}};R("Ranking sites\u2026"),O=await De(`${t}/system/site-rank.json`,{query:n,...s,domains:i,prefer:M,algorithm:u||{},batch:l||100})}let D=O.batches||[],U=O.count||D.reduce((M,G)=>M+G.length,0),p={searched:f?.searched||0,total:U,batches:D.length,done:0,stopped:!1,converged:!1,running:!0,unindexed:O.unindexed||[],answered:f?.answered||{local:0},siteIndex:O.siteIndex||null,siteInfo:new Map((O.sites||[]).map(M=>[M.domain,M])),shown:f?.shown||[],pending:0,opened:new Set(f?.opened||[]),resumed:!!f},T=new Map((f?.merged||[]).map(M=>[F(M),M])),x=[...T.values()].sort((M,G)=>G.score-M.score),h=0,m=()=>{p.stopped=!0};p.stop=m;let b=()=>{try{e.save&&e.save(p.snapshot())}catch{}};p.foldIn=()=>{p.shown=vt(p.shown,x,a,p.opened),p.pending=0,E(x,p),b()},p.markOpened=M=>{p.opened.add(M),E(x,p),b()},p.snapshot=()=>({query:n,total:U,searched:p.searched,converged:p.converged,stopped:p.stopped,merged:x.slice(0,150),shown:p.shown,opened:[...p.opened],remaining:D.slice(p.done),answered:p.answered,siteIndex:p.siteIndex,unindexed:p.unindexed,sites:[...p.siteInfo.values()].slice(0,400)});let C=async M=>{let G={query:n,...s,domains:M,limit:Math.max(a*3,30),flat:!0};e.thresholdSet&&(G.threshold=e.threshold);let c=await De(`${t}/system/search-report.json`,G);for(let r of c.results||[]){let o=F(r),g=T.get(o);(!g||g.score<r.score)&&T.set(o,r)}let y=0;for(let r of c.peers||[])p.answered[r.host]=(p.answered[r.host]||0)+(r.domains||0),y+=r.domains||0;p.answered.local+=Math.max(0,(c.stats&&c.stats.domains||0)-y);let d=[...T.values()].sort((r,o)=>o.score-r.score),w=Lt(x,d,a);x=d,p.searched+=M.length,p.done+=1,h=w?h+1:0,p.shown.length?p.shown=St(p.shown,d):p.shown=d.slice(0,a),p.pending=xt(p.shown,d,a)},W=()=>D.slice(p.done),X=async(M=!1)=>{for(p.running=!0,p.stopped=!1;p.done<D.length&&!p.stopped;)if(await C(D[p.done]),E(x,p),b(),!M&&h>=3&&p.done<D.length){p.converged=!0;break}return p.running=!1,E(x,p),b(),x};return p.continueAll=()=>(p.converged=!1,h=-1/0,X(!0)),f&&f.remaining&&!f.remaining.length?(p.running=!1,p.converged=!!f.converged,E(x,p),{results:x,state:p,remaining:W}):(await X(!1),{results:x,state:p,remaining:W})};var ae="similarity:learned",Et=new Set(["liked","visited","neighbourhood","neighborhood","followed","centroid","fresh","reliable","stale"]),ie=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,be=e=>{let t={weights:{},always:[],never:[],batch:null,trust:null};for(let n of(e||"").split(`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-plugin-similarity",
-  "version": "0.25.0",
+  "version": "0.25.1",
   "description": "Federated Wiki plugin — semantic similarity search across wiki domains using prebuilt vector indexes",
   "keywords": [
     "wiki",

package/server/server.js

@@ -82,7 +82,7 @@ const { buildSiteReport } = require('./site-report')
 const { searchFarm, keywordReportPage, findTwins } = require('./farm-search')
 const { searchGalaxy } = require('./galaxy-search')
 const { galaxyRoot, galaxyCacheStats } = require('./galaxy-vectors')
-const { warmUp } = require('./vector-store')
+const { warmUp, countPages } = require('./vector-store')
 const { resolveAuthor } = require('./author-index')
 const { ceiling, grantingDomains, guardEnvelope } = require('./peer-guard')
 const { postToPeer, appendPeerSections, makePeerDesk, setModelMeta } = require('./peer')
@@ -367,17 +367,17 @@ const startServer = ({ argv, app }) => {
       .filter(({ domain }) => visible(req, domain))
       .map(({ farm, domain }) => {
         const file = path.join(farm, domain, 'status', 'semantic-vectors.json')
-        let pageCount = null
-        let built = null
-        try {
-          // When the vectors were last written. One stat on top of a read we
-          // are already paying for, and it is what lets a reader see whether
-          // the index behind an answer is current or months old.
-          built = new Date(fs.statSync(file).mtimeMs).toISOString()
-          const pages = JSON.parse(fs.readFileSync(file, 'utf8'))
-          pageCount = Array.isArray(pages) ? pages.length : null
-        } catch { /* ignore */ }
-        return { domain, page_count: pageCount, built }
+        // Count from the vector store, never by parsing the file here: this
+        // route runs three times when the Search Tool page opens, and parsing
+        // every tree's vectors on the event loop froze the farm (0.25.1).
+        // `built` is when the vectors were last written — what lets a reader
+        // see whether the index behind an answer is current or months old.
+        const counted = countPages(file)
+        return {
+          domain,
+          page_count: counted ? counted.n : null,
+          built: counted ? new Date(counted.mtimeMs).toISOString() : null,
+        }
       })
     if (limit) results = results.slice(0, limit)
     res.json(results)

package/server/vector-store.js

@@ -199,6 +199,33 @@ const siteCentroid = file => {
   return e.centroid
 }
 
+// ── Page count without the parse ────────────────────────────────────────────
+// indexed-domains.json used to read and JSON.parse every vectors file just to
+// count its pages — 1.5 GB across both trees on the Pi5, synchronously, which
+// froze the whole farm for 20-90 s and got it restarted by the healthcheck
+// (2026-09-05). The count is already known three cheaper ways, in order:
+// the resident entry; the head of the disk-cache meta (written as
+// {"n":…,"dim":…}); and, only for a file never seen before, one parse that
+// seeds both so the next call is a stat. Returns {n, mtimeMs} or null.
+const countPages = file => {
+  let stat
+  try { stat = fs.statSync(file) } catch { return null }
+  const hit = cache.get(file)
+  if (hit && hit.mtimeMs === stat.mtimeMs) return { n: hit.n, mtimeMs: stat.mtimeMs }
+  const { meta } = storePaths(file, stat)
+  try {
+    const fd = fs.openSync(meta, 'r')
+    try {
+      const buf = Buffer.alloc(64)
+      const got = fs.readSync(fd, buf, 0, 64, 0)
+      const m = /^\{"n":(\d+)/.exec(buf.toString('utf8', 0, got))
+      if (m) return { n: parseInt(m[1], 10), mtimeMs: stat.mtimeMs }
+    } finally { fs.closeSync(fd) }
+  } catch { /* no disk cache for this version of the file yet */ }
+  const entry = loadEntry(file)
+  return { n: entry ? entry.n : null, mtimeMs: stat.mtimeMs }
+}
+
 // The compatibility surface — an array of pages, each with a `.vector` view.
 const loadVectorsFile = file => {
   const entry = loadEntry(file)
@@ -265,6 +292,6 @@ const resetStore = () => {
 }
 
 module.exports = {
-  loadEntry, loadVectorsFile, loadVectorsCached, siteCentroid, warmUp, storeStats,
+  loadEntry, loadVectorsFile, loadVectorsCached, siteCentroid, warmUp, storeStats, countPages,
   resetStore, buildEntry, storeDir, CACHE_BYTES,
 }