wiki-plugin-similarity 0.7.0 → 0.8.0

package/client/similarity.js

@@ -1,4 +1,4 @@
-/* wiki-plugin-similarity - 0.7.0 - Mon, 20 Jul 2026 23:51:54 GMT */
+/* wiki-plugin-similarity - 0.8.0 - Tue, 21 Jul 2026 08:09:09 GMT */
 (()=>{var q={high:.78,medium:.68,low:.58},M=q.medium,_=10,V=t=>{let e=[],n=[],o=[],f=null,b=null,d="search",y=!1,L=!1,E=null,k=null,w=(i,m)=>i===m||i.startsWith(m)&&/^[\s:]/.test(i.slice(m.length)),x=(i,m)=>i.slice(m.length).replace(/^\s*:?\s*/,"").trim();for(let i of t.split(`
 `)){let m=i.trim();if(!m||m.startsWith("#"))continue;let g=m.toUpperCase();if(w(g,"LIVE")){y=!0;continue}if(w(g,"AUTHOR")){!e.length&&d==="search"&&(d="author");continue}if(w(g,"REPORT")){d==="search"&&(d="report");continue}if(w(g,"KEYWORD")){d==="search"&&(d="keyword");continue}if(w(g,"BUILD")){d==="search"&&(d="build");continue}if(w(g,"FORCE")){L=!0;continue}if(w(g,"GHOST")){E=x(m,"GHOST"),d==="search"&&(d="ghost");continue}if(w(g,"BUTTON")){k=x(m,"BUTTON");continue}if(w(g,"ROSTER")){let S=x(m,"ROSTER");S&&n.push(S);continue}if(w(g,"FARM")){let S=x(m,"FARM");S&&o.push(S);continue}if(w(g,"LIST")){!e.length&&d==="search"&&(d="list");continue}if(w(g,"SIMILAR")){let S=x(g,"SIMILAR").toLowerCase();f=q[S]||M,!e.length&&d==="search"&&(d="similar");continue}if(w(g,"THRESHOLD")){let S=x(m,"THRESHOLD");f=q[S.toLowerCase()]??(parseFloat(S)||M);continue}if(w(g,"LIMIT")){b=parseInt(x(m,"LIMIT"))||_;continue}e.push(["PUBLIC","LOCAL","PRIVATE"].includes(g)?g:m)}return{mode:d,specs:e,rosterRefs:n,farms:o,threshold:f??M,limit:b??_,live:y,force:L,ghostUrl:E,label:k,thresholdSet:f!==null}},et=t=>t.includes("*")||t.includes("?"),nt=t=>t==="PUBLIC"||t==="LOCAL"||t==="PRIVATE",st=t=>t.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),P=new Map,it=async(t,e)=>{if(P.has(t))return P.get(t);let n=`${e}/system/indexed-domains.json?pattern=${encodeURIComponent(t)}`,o=await fetch(n);if(!o.ok)throw new Error(`indexed-domains failed: ${o.status}`);let f=await o.json();return P.set(t,f),f},G=async(t,e)=>{t.length||(t=[window.location.hostname]);let n=new Set,o=[];for(let f of t)if(f==="*"||et(f)||nt(f))for(let b of await it(f,e))n.has(b.domain)||(n.add(b.domain),o.push(b));else n.has(f)||(n.add(f),o.push({domain:f,page_count:null}));return o},ot=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,at=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,rt=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,B=async t=>{let e=t.indexOf("/"),n=await fetch(`//${t.slice(0,e)}/${t.slice(e+1)}.json`);if(!n.ok)throw new Error(`roster page ${t} failed: ${n.status}`);return n.json()},K=async(t,e,n)=>{if(n.has(t))return;n.add(t);let o;try{o=await B(t)}catch(f){console.warn(f.message);return}for(let f of o.story||[])if(f.type==="roster")for(let b of(f.text||"").split(/\r?\n/)){let d=b.trim();if(!d)continue;let y=d.match(ot);if(y){e.add(y[0]);continue}let L=d.match(at);if(L){await K(L[1],e,n);continue}let E=d.match(rt);if(E)try{let k=await B(E[1]);for(let w of k.story||[])w.type==="reference"&&w.site&&e.add(w.site)}catch(k){console.warn(k.message)}}},lt=async t=>{let e=new Set,n=new Set;for(let o of t)await K(o,e,n);return[...e]},ct=async(t,e)=>e.length?[...t,...await lt(e)]:t,A=new Map,dt=t=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(t)}`,Z=async t=>{if(A.has(t))return A.get(t);let e=await fetch(dt(t));if(!e.ok)return[];let n=await e.json();return A.set(t,n),n},z=async(t,e)=>{let n=await fetch(`${e}/system/embed.json?text=${encodeURIComponent(t)}`);if(!n.ok)throw new Error(`embed failed: ${n.status}`);return(await n.json()).vector},ft=async(t,e)=>{let o=(await Z(e)).find(f=>f.slug===t);return o?o.vector:null},H=(t,e,{threshold:n,limit:o,excludeSlug:f,excludeDomain:b})=>{let d=[];for(let{domain:y,pages:L}of e)for(let{slug:E,title:k,vector:w}of L){if(E===f&&y===b)continue;let x=0;for(let i=0;i<t.length;i++)x+=t[i]*w[i];x>=n&&d.push({domain:y,slug:E,title:k,score:x})}return d.sort((y,L)=>L.score-y.score),d.slice(0,o)},F=async(t,e)=>{let n=await G(t,e);return(await Promise.all(n.map(async({domain:f})=>({domain:f,pages:await Z(f)})))).filter(f=>f.pages.length>0)},J=t=>`sim-cache-${t}`,mt=t=>{try{let e=JSON.parse(localStorage.getItem(J(t.id))||"null");return e?.text===(t.text||"")?e:null}catch{return null}},O=(t,e)=>{try{localStorage.setItem(J(t.id),JSON.stringify({text:t.text||"",ts:Date.now(),...e}))}catch{}},ut=t=>{let e=Math.floor((Date.now()-t)/1e3);return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`},I=`
   .sim-form { display:flex; gap:6px; margin-bottom:8px; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-plugin-similarity",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Federated Wiki plugin \u2014 semantic similarity search across wiki domains using prebuilt vector indexes",
   "keywords": [
     "wiki",
@@ -42,8 +42,10 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
+    "minisearch": "^7.2.0"
+  },
+  "optionalDependencies": {
     "@xenova/transformers": "^2.17.2",
-    "minisearch": "^7.2.0",
     "onnxruntime-web": "1.14.0"
   },
   "devDependencies": {

package/server/embedder.js

@@ -19,6 +19,21 @@ const MODEL     = process.env.WIKI_EMBED_MODEL || 'Xenova/bge-small-en-v1.5'
 const POOLING   = process.env.WIKI_EMBED_POOLING || 'cls'
 const QUANTIZED = process.env.WIKI_EMBED_QUANTIZED === '1'
 
+// Prefer wiki-plugin-semindex's shared worker-thread embedder when installed:
+// one model instance per process (shared with bulk indexing), inference off
+// the event loop, and similarity's own heavy deps become optional. The
+// sibling path works for npm installs (wiki/node_modules/*) and dev symlinks
+// (~/Code/wiki-plugins/*) alike; the bare require covers other layouts.
+const semindexLib = (() => {
+  const path = require('node:path')
+  const fs   = require('node:fs')
+  try {
+    const sibling = path.resolve(__dirname, '../../wiki-plugin-semindex/server/embed-lib.js')
+    if (fs.existsSync(sibling)) return require(sibling)
+    return require('wiki-plugin-semindex/server/embed-lib.js')
+  } catch { return null }
+})()
+
 // transformers.js statically imports onnxruntime-node, whose native binary
 // needs glibc. On musl containers (node:alpine — the public farm) that import
 // throws "Error loading shared library ld-linux-x86-64.so.2". Probe the
@@ -47,7 +62,13 @@ const getExtractor = () => {
         console.log('[wiki-plugin-similarity] native onnxruntime unavailable — using WASM backend')
       }
       // Dynamic import: @xenova/transformers is ESM-only; this file must stay CJS.
-      const { pipeline, env } = await import('@xenova/transformers')
+      // Since 0.8.0 the heavy deps are optionalDependencies — absent on slim
+      // installs, where semindex delegation or WIKI_EMBED_URL is the path.
+      const { pipeline, env } = await import('@xenova/transformers').catch(() => {
+        throw new Error('no local embedder: @xenova/transformers is not installed. ' +
+          'Install wiki-plugin-semindex, set WIKI_EMBED_URL, or reinstall ' +
+          'wiki-plugin-similarity with optional dependencies.')
+      })
       if (process.env.WIKI_MODEL_CACHE) env.cacheDir = process.env.WIKI_MODEL_CACHE
       if (env.backends?.onnx?.wasm) env.backends.onnx.wasm.numThreads = 1
       return pipeline('feature-extraction', MODEL, { quantized: QUANTIZED })
@@ -59,13 +80,15 @@ const getExtractor = () => {
 
 // Embed one text → plain number[384], unit-normalised.
 const embed = async text => {
+  if (semindexLib) return semindexLib.embed(text)
   const extractor = await getExtractor()
   const out = await extractor(text, { pooling: POOLING, normalize: true })
   return Array.from(out.data)
 }
 
 // Warm the model in the background (call at startup; failures just defer to
-// first request).
-const warm = () => { getExtractor().catch(() => {}) }
+// first request). With semindex delegation, don't warm — its worker spawns on
+// demand and self-terminates when idle; pre-warming would just pin ~0.5GB.
+const warm = () => { if (!semindexLib) getExtractor().catch(() => {}) }
 
-module.exports = { embed, warm }
+module.exports = { embed, warm, viaSemindex: !!semindexLib }

package/server/server.js

@@ -501,26 +501,47 @@ const startServer = ({ argv, app }) => {
   })
 
   // ── GET /system/build-index.json?domains=…&force=… ────────────────────────
-  // Heavy embedding is the farm indexer's job — proxy when configured, else
-  // explain where indexing happens.
+  // Resolution order: WIKI_INDEXER_URL proxy (explicit config wins) → local
+  // wiki-plugin-semindex (in-process enqueue, owner/admin only) → 501.
+  const semindex = (() => {
+    try {
+      const sibling = path.resolve(__dirname, '../../wiki-plugin-semindex/server/indexer.js')
+      if (fs.existsSync(sibling)) return require(sibling)
+      return require('wiki-plugin-semindex/server/indexer.js')
+    } catch { return null }
+  })()
+
   app.get('/system/build-index.json', async (req, res) => {
     cors(res)
-    if (!INDEXER_URL) {
-      return res.status(501).json({
-        error: 'no farm indexer configured on this server',
-        hint: 'Indexing runs on the farm indexer (Pi5 on the Hitchhikers farm); ' +
-              'indexes arrive by sync. Set WIKI_INDEXER_URL to enable proxying.',
-      })
+    if (INDEXER_URL) {
+      try {
+        const qs = new URLSearchParams({
+          domains: req.query.domains || '*',
+          force:   req.query.force || '0',
+        })
+        return res.json(await getJson(`${INDEXER_URL}?${qs}`))
+      } catch (e) {
+        return res.status(502).json({ error: `farm indexer unavailable: ${e.message}` })
+      }
     }
-    try {
-      const qs = new URLSearchParams({
-        domains: req.query.domains || '*',
-        force:   req.query.force || '0',
-      })
-      res.json(await getJson(`${INDEXER_URL}?${qs}`))
-    } catch (e) {
-      res.status(502).json({ error: `farm indexer unavailable: ${e.message}` })
+    if (semindex) {
+      const sh = app.securityhandler
+      const ok = (() => {
+        try { return !!sh && (!!sh.isAdmin?.(req) || !!sh.isAuthorized?.(req)) } catch { return false }
+      })()
+      if (!ok) return res.sendStatus(403)
+      if (semindex.MODE !== 'writer') {
+        return res.status(409).json({ error: `semindex mode is '${semindex.MODE}' — not a writer` })
+      }
+      const queued = semindex.enqueueGlobs(req.query.domains || '*', req.query.force === '1')
+      return res.json({ queued, ...semindex.status() })
     }
+    res.status(501).json({
+      error: 'no indexer available on this server',
+      hint: 'Install wiki-plugin-semindex for in-process indexing, or set ' +
+            'WIKI_INDEXER_URL to proxy to a farm indexer; indexes can also ' +
+            'arrive by sync.',
+    })
   })
 
   console.log('[wiki-plugin-similarity] routes registered')