wiki-plugin-similarity 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/similarity.js +1 -1
- package/package.json +1 -1
- package/pages/about-similarity-plugin +28 -23
- package/server/server.js +91 -9
package/client/similarity.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/* wiki-plugin-similarity - 0.
|
|
1
|
+
/* wiki-plugin-similarity - 0.14.0 - Thu, 03 Sep 2026 20:55:40 GMT */
|
|
2
2
|
(()=>{var q={high:.78,medium:.68,low:.58},N=q.medium,J=10,_=t=>{let e=[],s=[],i=[],p=null,E=null,f="search",y=!1,L=!1,b=!1,k=null,C=null,S=(a,r)=>a===r||a.startsWith(r)&&/^[\s:]/.test(a.slice(r.length)),w=(a,r)=>a.slice(r.length).replace(/^\s*:?\s*/,"").trim();for(let a of t.split(`
|
|
3
3
|
`)){let r=a.trim();if(!r||r.startsWith("#"))continue;let g=r.toUpperCase();if(S(g,"LIVE")){y=!0;continue}if(S(g,"SUBJECT")){L=!0;continue}if(S(g,"AUTHOR")){!e.length&&f==="search"&&(f="author");continue}if(S(g,"REPORT")){f==="search"&&(f="report");continue}if(S(g,"KEYWORD")){f==="search"&&(f="keyword");continue}if(S(g,"SITES")){f==="search"&&(f="sites");continue}if(S(g,"BUILD")){f==="search"&&(f="build");continue}if(S(g,"FORCE")){b=!0;continue}if(S(g,"GHOST")){k=w(r,"GHOST"),f==="search"&&(f="ghost");continue}if(S(g,"BUTTON")){C=w(r,"BUTTON");continue}if(S(g,"ROSTER")){let v=w(r,"ROSTER");v&&s.push(v);continue}if(S(g,"FARM")){let v=w(r,"FARM");v&&i.push(v);continue}if(S(g,"LIST")){!e.length&&f==="search"&&(f="list");continue}if(S(g,"SIMILAR")){let v=w(g,"SIMILAR").toLowerCase();p=q[v]||N,!e.length&&f==="search"&&(f="similar");continue}if(S(g,"THRESHOLD")){let v=w(r,"THRESHOLD");p=q[v.toLowerCase()]??(parseFloat(v)||N);continue}if(S(g,"LIMIT")){E=parseInt(w(r,"LIMIT"))||J;continue}e.push(["PUBLIC","LOCAL","PRIVATE","GALAXY"].includes(g)?g:r)}return{mode:f,specs:e,rosterRefs:s,farms:i,threshold:p??N,limit:E??J,live:y,subject:L,force:b,ghostUrl:k,label:C,thresholdSet:p!==null}},Y=t=>t.includes("*")||t.includes("?"),W=t=>t==="PUBLIC"||t==="LOCAL"||t==="PRIVATE"||t==="GALAXY";var X=t=>t.toLowerCase().replace(/\s+/g,"-").replace(/[^a-z0-9-]/g,""),B=new Map,pt=async(t,e)=>{if(B.has(t))return B.get(t);let s=`${e}/system/indexed-domains.json?pattern=${encodeURIComponent(t)}`,i=await fetch(s);if(!i.ok)throw new Error(`indexed-domains failed: ${i.status}`);let p=await i.json();return B.set(t,p),p},O=async(t,e)=>{t.length||(t=[window.location.hostname]);let s=new Set,i=[];for(let p of t)if(p==="*"||Y(p)||W(p))for(let E of await pt(p,e))s.has(E.domain)||(s.add(E.domain),i.push(E));else s.has(p)||(s.add(p),i.push({domain:p,page_count:null}));return i},gt=/^([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+|localhost)(:\d+)?$/,ht=/^ROSTER ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,wt=/^REFERENCES ([A-Za-z0-9.\-:]+\/[a-z0-9-]+)$/,Z=async t=>{let e=t.indexOf("/"),s=await fetch(`//${t.slice(0,e)}/${t.slice(e+1)}.json`);if(!s.ok)throw new Error(`roster page ${t} failed: ${s.status}`);return s.json()},Q=async(t,e,s)=>{if(s.has(t))return;s.add(t);let i;try{i=await Z(t)}catch(p){console.warn(p.message);return}for(let p of i.story||[])if(p.type==="roster")for(let E of(p.text||"").split(/\r?\n/)){let f=E.trim();if(!f)continue;let y=f.match(gt);if(y){e.add(y[0]);continue}let L=f.match(ht);if(L){await Q(L[1],e,s);continue}let b=f.match(wt);if(b)try{let k=await Z(b[1]);for(let C of k.story||[])C.type==="reference"&&C.site&&e.add(C.site)}catch(k){console.warn(k.message)}}},yt=async t=>{let e=new Set,s=new Set;for(let i of t)await Q(i,e,s);return[...e]},tt=async(t,e)=>e.length?[...t,...await yt(e)]:t;var G=new Map,$t=t=>`${window.location.origin}/system/semantic-vectors.json?domain=${encodeURIComponent(t)}`,et=async t=>{if(G.has(t))return G.get(t);let e=await fetch($t(t));if(!e.ok)return[];let s=await e.json();return G.set(t,s),s},U=async(t,e)=>{let s=await fetch(`${e}/system/embed.json`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({text:t})});if(!s.ok)throw new Error(`embed failed: ${s.status}`);return(await s.json()).vector},st=async(t,e)=>{let i=(await et(e)).find(p=>p.slug===t);return i?i.vector:null},M=(t,e,{threshold:s,limit:i,excludeSlug:p,excludeDomain:E})=>{let f=[];for(let{domain:y,pages:L}of e)for(let{slug:b,title:k,vector:C}of L){if(b===p&&y===E)continue;let S=0;for(let w=0;w<t.length;w++)S+=t[w]*C[w];S>=s&&f.push({domain:y,slug:b,title:k,score:S})}return f.sort((y,L)=>L.score-y.score),f.slice(0,i)},z=async(t,e)=>{let s=await O(t,e);return(await Promise.all(s.map(async({domain:p})=>({domain:p,pages:await et(p)})))).filter(p=>p.pages.length>0)};var nt=t=>{let e=t.parents(".page:first"),s=e.prev(".page"),i=s.length?s:e,p=(i.attr("id")||"").split("_rev")[0],E=i.data("site")||window.location.hostname,f=i.data("data")||{},y=f.title||i.find(".title").text().trim()||p,L=[y,...(f.story||[]).filter(b=>b.type==="markdown"||b.type==="paragraph").map(b=>(b.text||"").trim()).filter(Boolean)].join(`
|
|
4
4
|
`).slice(0,2e3);return{slug:p,site:E,title:y,text:L,isSelf:!s.length}},T=t=>`Subject: ${t.title}${t.isSelf?" (this page)":` @ ${t.site}`}`;var it=t=>`sim-cache-${t}`,ot=t=>{try{let e=JSON.parse(localStorage.getItem(it(t.id))||"null");return e?.text===(t.text||"")?e:null}catch{return null}},P=(t,e)=>{try{localStorage.setItem(it(t.id),JSON.stringify({text:t.text||"",ts:Date.now(),...e}))}catch{}},at=t=>{let e=Math.floor((Date.now()-t)/1e3);return e<60?`${e}s ago`:e<3600?`${Math.floor(e/60)}m ago`:e<86400?`${Math.floor(e/3600)}h ago`:`${Math.floor(e/86400)}d ago`};var D=`
|
package/package.json
CHANGED
|
@@ -4,12 +4,12 @@
|
|
|
4
4
|
{
|
|
5
5
|
"type": "markdown",
|
|
6
6
|
"id": "aec1dfcd5f2bc42b",
|
|
7
|
-
"text": "The Similarity plugin adds search to a Federated Wiki farm
|
|
7
|
+
"text": "The Similarity plugin adds search to a Federated Wiki farm \u2014 semantic similarity over prebuilt vector indexes, ranked search reports, galactic keyword search over each site's own MiniSearch index, and semantic search across the wider federation. One item type \u2014 **similarity** \u2014 with a line-oriented DSL, plus a server component that serves every search same-origin. Works on any host: embedding runs in-process (native ONNX where available, WASM everywhere else)."
|
|
8
8
|
},
|
|
9
9
|
{
|
|
10
10
|
"type": "markdown",
|
|
11
11
|
"id": "28edff984cc5334d",
|
|
12
|
-
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE / GALAXY):\n\n- (no keyword)
|
|
12
|
+
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE / GALAXY):\n\n- (no keyword) \u2014 interactive semantic search form\n- `SIMILAR: high|medium|low` \u2014 ambient: pages similar to this page\n- `REPORT` \u2014 server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` \u2014 galactic MiniSearch over live site-index.json files\n- `SITES` \u2014 which site should this page go on? Per-domain aggregation of the page-vector scan: 0.7 \u00d7 mean of the top 5 page similarities + 0.3 \u00d7 site-centroid similarity. Forked pages count for every site that carries them\n- `LIST` \u2014 table of indexed domains\n- `AUTHOR` \u2014 search results as a draggable ghost page\n- `BUILD` / `FORCE` \u2014 proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` \u2014 open any page-json as a ghost page\n- `ROSTER site/slug` \u2014 add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` \u2014 ask a peer farm to continue the search. The peer answers from exactly the sites whose own [[Federated Farm Search]] page grants your origin (`FROM` lines; `FROM *` needs the peer admin's `WIKI_PEER_FEDERATION=open`)\n- `GALAXY` \u2014 search the off-farm federation sites indexed into this host's galaxy tree. A scope, not a mode: `GALAXY` alone answers only from the federation (the farm's own pages are never swept in, and `*` never includes galaxy sites); `*` and `GALAXY` together search farm and federation as one\n- `SUBJECT` \u2014 act on the previous page in the lineup, the host a tool page was opened beside\n- `THRESHOLD: 0.72` \u00b7 `LIMIT: 8` \u00b7 `LIVE` \u2014 tuning"
|
|
13
13
|
},
|
|
14
14
|
{
|
|
15
15
|
"type": "markdown",
|
|
@@ -64,12 +64,17 @@
|
|
|
64
64
|
{
|
|
65
65
|
"type": "markdown",
|
|
66
66
|
"id": "d9a96dfebb9d1692",
|
|
67
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`
|
|
67
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json` \u2014 accepts an optional `author`: a display name or account username, resolved against this farm's own site ownership (each site's `owner.json`) and intersected with the requested scope. An ambiguous name answers 409 with the competing usernames rather than guessing\n- `POST /system/site-report.json` \u2014 which site should this page go on? Per-domain aggregation of the page-vector scan: 0.7 \u00d7 mean of the top 5 page similarities + 0.3 \u00d7 site-centroid similarity; forked pages count for every site that carries them\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` \u2014 keyword search over off-farm federation sites\n- `GET /system/galaxy-registry.json` \u2014 the galaxy site registry: every federation site known to this host, with provenance, liveness, page counts and scraped site owner (read-only; written by the galaxy indexer host)\n- `GET /system/peer-hello.json` \u2014 capability probe: version, model, federation status (404 = plugin absent)\n- `GET /system/similarity-health.json` \u2014 embedder supervisor state: child-process / semindex / url, breaker status, recent crashes\n- `POST /system/peer-search.json` \u2014 answer another farm's search. An optional `author` travels as the plain name asked for, never a resolved username: the answering farm resolves it against its OWN records and reports `applied`, `ignored` (no records kept) or `ambiguous`. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder.\n\nSince 0.9.0 all query embedding runs in a supervised child process, never in the wiki server itself: a crash in the ONNX runtime fails that one search, and repeated crashes trip a circuit breaker that answers 503 while cooling down. The worst case for a wiki running this plugin is \"semantic search temporarily unavailable\" \u2014 never a dead wiki.\n\nSince 0.11.0 a third index tier sits beside the farm: the **galaxy** \u2014 a farm-shaped tree of off-farm federation sites, built by a separate indexer and read through a byte-capped vector cache (`WIKI_GALAXY_VECTOR_CACHE_MB`, default 512). Galaxy entries carry their enrichment precomputed, because off-farm pages have no local page JSON to read; the stub filter, fork bundling and scoring then run unchanged, so federation results rank directly against farm results.\n\nSince 0.12.0 the plugin also offers a **Semantic Search** page through the wiki's \u2630 menu, declared by a `pages` entry in `factory.json` and shipped in the package.\n\nSince 0.13.0 restricted sites are hidden from **every** route unless the caller is trusted, not only from peer answers. Restricted = the local farm's `wikiDomains` entries marked `restricted`, the globs in `WIKI_RESTRICTED_DOMAINS` (e.g. `*.private.fish`), and extra farms' config files. Trusted = an owner or admin session, a tool on the same host (loopback, no `X-Forwarded-For`), or a client the reverse proxy vouches for from `WIKI_TRUSTED_NETS` (default: the Tailscale ranges and loopback; the header is believed only when the direct peer is loopback). Cross-origin browser requests are never trusted. Untrusted callers get the public view: `semantic-vectors.json` answers 403 for a restricted domain, and the scans simply do not include it. `similarity-health.json` reports the policy and `callerTrusted`."
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "markdown",
|
|
71
|
+
"id": "b4e1c0d2a7f39e56",
|
|
72
|
+
"text": "0.14.0 decides where the query embedder lives, and says so out loud. `WIKI_EMBED_URL` is now validated: a value that is not an http(s) URL \u2014 the public farm had the literal string `disabled` \u2014 is ignored with a warning instead of being POSTed to, which is what whoever typed it meant. Where the environment is not yours to set, the address can come from `{farmRoot}/similarity.json` (`{\"embedUrl\": \"\u2026\"}`) instead, read mtime-cached so a retarget needs no restart. `/system/similarity-health.json` now reports which `source` won and whether a `localEmbedderAvailable` \u2014 the local path is never exercised while a proxy is set, so that question was otherwise unanswerable from outside. Both HTTP helpers also learned `https`: before this they were http-only on port 80, so an https proxy could never have worked."
|
|
68
73
|
},
|
|
69
74
|
{
|
|
70
75
|
"type": "markdown",
|
|
71
76
|
"id": "21eb5354d2adb29e",
|
|
72
|
-
"text": "**Version:** 0.
|
|
77
|
+
"text": "**Version:** 0.14.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
73
78
|
}
|
|
74
79
|
],
|
|
75
80
|
"plugin": "similarity",
|
|
@@ -82,12 +87,12 @@
|
|
|
82
87
|
{
|
|
83
88
|
"type": "markdown",
|
|
84
89
|
"id": "aec1dfcd5f2bc42b",
|
|
85
|
-
"text": "The Similarity plugin adds search to a Federated Wiki farm
|
|
90
|
+
"text": "The Similarity plugin adds search to a Federated Wiki farm \u2014 semantic similarity over prebuilt vector indexes, ranked search reports, and galactic keyword search over each site's own MiniSearch index. One item type \u2014 **similarity** \u2014 with a line-oriented DSL, plus a server component that serves every search same-origin. Works on any host: embedding runs in-process (native ONNX where available, WASM everywhere else)."
|
|
86
91
|
},
|
|
87
92
|
{
|
|
88
93
|
"type": "markdown",
|
|
89
94
|
"id": "28edff984cc5334d",
|
|
90
|
-
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword)
|
|
95
|
+
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword) \u2014 interactive semantic search form\n- `SIMILAR: high|medium|low` \u2014 ambient: pages similar to this page\n- `REPORT` \u2014 server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` \u2014 galactic MiniSearch over live site-index.json files\n- `LIST` \u2014 table of indexed domains\n- `AUTHOR` \u2014 search results as a draggable ghost page\n- `BUILD` / `FORCE` \u2014 proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` \u2014 open any page-json as a ghost page\n- `THRESHOLD: 0.72` \u00b7 `LIMIT: 8` \u00b7 `LIVE` \u2014 tuning"
|
|
91
96
|
},
|
|
92
97
|
{
|
|
93
98
|
"type": "markdown",
|
|
@@ -132,17 +137,17 @@
|
|
|
132
137
|
{
|
|
133
138
|
"type": "markdown",
|
|
134
139
|
"id": "d9a96dfebb9d1692",
|
|
135
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n\nIndexes are `status/semantic-vectors.json` per domain
|
|
140
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder."
|
|
136
141
|
},
|
|
137
142
|
{
|
|
138
143
|
"type": "markdown",
|
|
139
144
|
"id": "21eb5354d2adb29e",
|
|
140
|
-
"text": "**Version:** 0.5.1\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub]
|
|
145
|
+
"text": "**Version:** 0.5.1\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
141
146
|
}
|
|
142
147
|
]
|
|
143
148
|
},
|
|
144
149
|
"date": 1783782734556,
|
|
145
|
-
"provenance": "Authored by an AI agent
|
|
150
|
+
"provenance": "Authored by an AI agent \u2014 Claude Fable 5 via the [[fedwiki-page]] skill. [https://claude.com/claude-code Claude Code]"
|
|
146
151
|
},
|
|
147
152
|
{
|
|
148
153
|
"type": "edit",
|
|
@@ -150,7 +155,7 @@
|
|
|
150
155
|
"item": {
|
|
151
156
|
"type": "markdown",
|
|
152
157
|
"id": "28edff984cc5334d",
|
|
153
|
-
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword)
|
|
158
|
+
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword) \u2014 interactive semantic search form\n- `SIMILAR: high|medium|low` \u2014 ambient: pages similar to this page\n- `REPORT` \u2014 server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` \u2014 galactic MiniSearch over live site-index.json files\n- `LIST` \u2014 table of indexed domains\n- `AUTHOR` \u2014 search results as a draggable ghost page\n- `BUILD` / `FORCE` \u2014 proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` \u2014 open any page-json as a ghost page\n- `ROSTER site/slug` \u2014 add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` \u2014 ask a peer farm to continue the search (experimental; requires the peer's `search-peers` roster to allow you)\n- `THRESHOLD: 0.72` \u00b7 `LIMIT: 8` \u00b7 `LIVE` \u2014 tuning"
|
|
154
159
|
},
|
|
155
160
|
"date": 1783800718188
|
|
156
161
|
},
|
|
@@ -160,7 +165,7 @@
|
|
|
160
165
|
"item": {
|
|
161
166
|
"type": "markdown",
|
|
162
167
|
"id": "d9a96dfebb9d1692",
|
|
163
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=`
|
|
168
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` \u2014 keyword search over off-farm federation sites\n- `POST /system/peer-search.json` \u2014 answer another farm's search (guarded: allowlist roster, hop limit, replay dedup, rate limit; restricted sites always excluded)\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder."
|
|
164
169
|
},
|
|
165
170
|
"date": 1783800718188
|
|
166
171
|
},
|
|
@@ -170,7 +175,7 @@
|
|
|
170
175
|
"item": {
|
|
171
176
|
"type": "markdown",
|
|
172
177
|
"id": "21eb5354d2adb29e",
|
|
173
|
-
"text": "**Version:** 0.6.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub]
|
|
178
|
+
"text": "**Version:** 0.6.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
174
179
|
},
|
|
175
180
|
"date": 1783800718188
|
|
176
181
|
},
|
|
@@ -180,7 +185,7 @@
|
|
|
180
185
|
"item": {
|
|
181
186
|
"type": "markdown",
|
|
182
187
|
"id": "28edff984cc5334d",
|
|
183
|
-
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword)
|
|
188
|
+
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE):\n\n- (no keyword) \u2014 interactive semantic search form\n- `SIMILAR: high|medium|low` \u2014 ambient: pages similar to this page\n- `REPORT` \u2014 server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` \u2014 galactic MiniSearch over live site-index.json files\n- `LIST` \u2014 table of indexed domains\n- `AUTHOR` \u2014 search results as a draggable ghost page\n- `BUILD` / `FORCE` \u2014 proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` \u2014 open any page-json as a ghost page\n- `ROSTER site/slug` \u2014 add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` \u2014 ask a peer farm to continue the search. The peer answers from exactly the sites whose own [[Federated Farm Search]] page grants your origin (`FROM` lines; `FROM *` needs the peer admin's `WIKI_PEER_FEDERATION=open`)\n- `THRESHOLD: 0.72` \u00b7 `LIMIT: 8` \u00b7 `LIVE` \u2014 tuning"
|
|
184
189
|
},
|
|
185
190
|
"date": 1784591495272
|
|
186
191
|
},
|
|
@@ -190,7 +195,7 @@
|
|
|
190
195
|
"item": {
|
|
191
196
|
"type": "markdown",
|
|
192
197
|
"id": "d9a96dfebb9d1692",
|
|
193
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=`
|
|
198
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` \u2014 keyword search over off-farm federation sites\n- `GET /system/peer-hello.json` \u2014 capability probe: version, model, federation status (404 = plugin absent)\n- `POST /system/peer-search.json` \u2014 answer another farm's search. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder."
|
|
194
199
|
},
|
|
195
200
|
"date": 1784591495272
|
|
196
201
|
},
|
|
@@ -200,7 +205,7 @@
|
|
|
200
205
|
"item": {
|
|
201
206
|
"type": "markdown",
|
|
202
207
|
"id": "21eb5354d2adb29e",
|
|
203
|
-
"text": "**Version:** 0.7.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub]
|
|
208
|
+
"text": "**Version:** 0.7.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
204
209
|
},
|
|
205
210
|
"date": 1784591495272
|
|
206
211
|
},
|
|
@@ -210,7 +215,7 @@
|
|
|
210
215
|
"item": {
|
|
211
216
|
"type": "markdown",
|
|
212
217
|
"id": "d9a96dfebb9d1692",
|
|
213
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=`
|
|
218
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` \u2014 keyword search over off-farm federation sites\n- `GET /system/peer-hello.json` \u2014 capability probe: version, model, federation status (404 = plugin absent)\n- `GET /system/similarity-health.json` \u2014 embedder supervisor state: child-process / semindex / url, breaker status, recent crashes\n- `POST /system/peer-search.json` \u2014 answer another farm's search. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder.\n\nSince 0.9.0 all query embedding runs in a supervised child process, never in the wiki server itself: a crash in the ONNX runtime fails that one search, and repeated crashes trip a circuit breaker that answers 503 while cooling down. The worst case for a wiki running this plugin is \"semantic search temporarily unavailable\" \u2014 never a dead wiki."
|
|
214
219
|
},
|
|
215
220
|
"date": 1784706029671
|
|
216
221
|
},
|
|
@@ -220,7 +225,7 @@
|
|
|
220
225
|
"item": {
|
|
221
226
|
"type": "markdown",
|
|
222
227
|
"id": "21eb5354d2adb29e",
|
|
223
|
-
"text": "**Version:** 0.9.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub]
|
|
228
|
+
"text": "**Version:** 0.9.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
224
229
|
},
|
|
225
230
|
"date": 1784706029671
|
|
226
231
|
},
|
|
@@ -230,7 +235,7 @@
|
|
|
230
235
|
"item": {
|
|
231
236
|
"type": "markdown",
|
|
232
237
|
"id": "aec1dfcd5f2bc42b",
|
|
233
|
-
"text": "The Similarity plugin adds search to a Federated Wiki farm
|
|
238
|
+
"text": "The Similarity plugin adds search to a Federated Wiki farm \u2014 semantic similarity over prebuilt vector indexes, ranked search reports, galactic keyword search over each site's own MiniSearch index, and semantic search across the wider federation. One item type \u2014 **similarity** \u2014 with a line-oriented DSL, plus a server component that serves every search same-origin. Works on any host: embedding runs in-process (native ONNX where available, WASM everywhere else)."
|
|
234
239
|
},
|
|
235
240
|
"date": 1785280898931
|
|
236
241
|
},
|
|
@@ -240,7 +245,7 @@
|
|
|
240
245
|
"item": {
|
|
241
246
|
"type": "markdown",
|
|
242
247
|
"id": "28edff984cc5334d",
|
|
243
|
-
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE / GALAXY):\n\n- (no keyword)
|
|
248
|
+
"text": "## The DSL\n\nThe first ALL-CAPS keyword sets the mode; other lines are domain specs (globs, explicit domains, or the scope keywords PUBLIC / LOCAL / PRIVATE / GALAXY):\n\n- (no keyword) \u2014 interactive semantic search form\n- `SIMILAR: high|medium|low` \u2014 ambient: pages similar to this page\n- `REPORT` \u2014 server-side ranked, stub-filtered, fork-bundled report\n- `KEYWORD` \u2014 galactic MiniSearch over live site-index.json files\n- `SITES` \u2014 which site should this page go on? Per-domain aggregation of the page-vector scan: 0.7 \u00d7 mean of the top 5 page similarities + 0.3 \u00d7 site-centroid similarity. Forked pages count for every site that carries them\n- `LIST` \u2014 table of indexed domains\n- `AUTHOR` \u2014 search results as a draggable ghost page\n- `BUILD` / `FORCE` \u2014 proxy to the farm indexer, where configured\n- `GHOST url` / `BUTTON label` \u2014 open any page-json as a ghost page\n- `ROSTER site/slug` \u2014 add a roster page's sites to the scope (off-farm sites searched via the galaxy cache)\n- `FARM domain` \u2014 ask a peer farm to continue the search. The peer answers from exactly the sites whose own [[Federated Farm Search]] page grants your origin (`FROM` lines; `FROM *` needs the peer admin's `WIKI_PEER_FEDERATION=open`)\n- `GALAXY` \u2014 search the off-farm federation sites indexed into this host's galaxy tree. A scope, not a mode: `GALAXY` alone answers only from the federation (the farm's own pages are never swept in, and `*` never includes galaxy sites); `*` and `GALAXY` together search farm and federation as one\n- `SUBJECT` \u2014 act on the previous page in the lineup, the host a tool page was opened beside\n- `THRESHOLD: 0.72` \u00b7 `LIMIT: 8` \u00b7 `LIVE` \u2014 tuning"
|
|
244
249
|
},
|
|
245
250
|
"date": 1785280898931
|
|
246
251
|
},
|
|
@@ -250,7 +255,7 @@
|
|
|
250
255
|
"item": {
|
|
251
256
|
"type": "markdown",
|
|
252
257
|
"id": "d9a96dfebb9d1692",
|
|
253
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `POST /system/site-report.json`
|
|
258
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`\n- `POST /system/site-report.json` \u2014 which site should this page go on? Per-domain aggregation of the page-vector scan: 0.7 \u00d7 mean of the top 5 page similarities + 0.3 \u00d7 site-centroid similarity; forked pages count for every site that carries them\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` \u2014 keyword search over off-farm federation sites\n- `GET /system/galaxy-registry.json` \u2014 the galaxy site registry: every federation site known to this host, with provenance, liveness, page counts and scraped site owner (read-only; written by the galaxy indexer host)\n- `GET /system/peer-hello.json` \u2014 capability probe: version, model, federation status (404 = plugin absent)\n- `GET /system/similarity-health.json` \u2014 embedder supervisor state: child-process / semindex / url, breaker status, recent crashes\n- `POST /system/peer-search.json` \u2014 answer another farm's search. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder.\n\nSince 0.9.0 all query embedding runs in a supervised child process, never in the wiki server itself: a crash in the ONNX runtime fails that one search, and repeated crashes trip a circuit breaker that answers 503 while cooling down. The worst case for a wiki running this plugin is \"semantic search temporarily unavailable\" \u2014 never a dead wiki.\n\nSince 0.11.0 a third index tier sits beside the farm: the **galaxy** \u2014 a farm-shaped tree of off-farm federation sites, built by a separate indexer and read through a byte-capped vector cache (`WIKI_GALAXY_VECTOR_CACHE_MB`, default 512). Galaxy entries carry their enrichment precomputed, because off-farm pages have no local page JSON to read; the stub filter, fork bundling and scoring then run unchanged, so federation results rank directly against farm results."
|
|
254
259
|
},
|
|
255
260
|
"date": 1785280898931
|
|
256
261
|
},
|
|
@@ -260,7 +265,7 @@
|
|
|
260
265
|
"item": {
|
|
261
266
|
"type": "markdown",
|
|
262
267
|
"id": "21eb5354d2adb29e",
|
|
263
|
-
"text": "**Version:** 0.11.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub]
|
|
268
|
+
"text": "**Version:** 0.11.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
264
269
|
},
|
|
265
270
|
"date": 1785280898931
|
|
266
271
|
},
|
|
@@ -270,7 +275,7 @@
|
|
|
270
275
|
"item": {
|
|
271
276
|
"type": "markdown",
|
|
272
277
|
"id": "d9a96dfebb9d1692",
|
|
273
|
-
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json`
|
|
278
|
+
"text": "## Server Routes\n\nAll same-origin, registered by the plugin's server component:\n\n- `GET /system/indexed-domains.json?pattern=`\n- `GET /system/semantic-vectors.json?domain=`\n- `GET /system/embed.json?text=`\n- `POST /system/search-report.json` \u2014 accepts an optional `author`: a display name or account username, resolved against this farm's own site ownership (each site's `owner.json`) and intersected with the requested scope. An ambiguous name answers 409 with the competing usernames rather than guessing\n- `POST /system/site-report.json` \u2014 which site should this page go on? Per-domain aggregation of the page-vector scan: 0.7 \u00d7 mean of the top 5 page similarities + 0.3 \u00d7 site-centroid similarity; forked pages count for every site that carries them\n- `GET /system/farm-search.json?q=&pattern=&limit=`\n- `GET /system/build-index.json?domains=` (proxied to the farm indexer)\n- `GET /system/galaxy-search.json?q=&sites=` \u2014 keyword search over off-farm federation sites\n- `GET /system/galaxy-registry.json` \u2014 the galaxy site registry: every federation site known to this host, with provenance, liveness, page counts and scraped site owner (read-only; written by the galaxy indexer host)\n- `GET /system/peer-hello.json` \u2014 capability probe: version, model, federation status (404 = plugin absent)\n- `GET /system/similarity-health.json` \u2014 embedder supervisor state: child-process / semindex / url, breaker status, recent crashes\n- `POST /system/peer-search.json` \u2014 answer another farm's search. An optional `author` travels as the plain name asked for, never a resolved username: the answering farm resolves it against its OWN records and reports `applied`, `ignored` (no records kept) or `ambiguous`. Two keys: the admin's `WIKI_PEER_FEDERATION` ceiling (off / grants / open) and each site's Federated Farm Search grants page; scope = union of granting sites; hop limit, replay dedup, IP-keyed + global rate limits; restricted sites always excluded\n\nIndexes are `status/semantic-vectors.json` per domain \u2014 built by the farm indexer, distributed by sync, read from farm disk. Keyword search reads each site's per-edit `status/site-index.json`; no separate index is built. Query embedding is BAAI bge-small-en-v1.5, 384-dim, CLS-pooled, unit-normalised \u2014 byte-compatible with the index builder.\n\nSince 0.9.0 all query embedding runs in a supervised child process, never in the wiki server itself: a crash in the ONNX runtime fails that one search, and repeated crashes trip a circuit breaker that answers 503 while cooling down. The worst case for a wiki running this plugin is \"semantic search temporarily unavailable\" \u2014 never a dead wiki.\n\nSince 0.11.0 a third index tier sits beside the farm: the **galaxy** \u2014 a farm-shaped tree of off-farm federation sites, built by a separate indexer and read through a byte-capped vector cache (`WIKI_GALAXY_VECTOR_CACHE_MB`, default 512). Galaxy entries carry their enrichment precomputed, because off-farm pages have no local page JSON to read; the stub filter, fork bundling and scoring then run unchanged, so federation results rank directly against farm results.\n\nSince 0.12.0 the plugin also offers a **Semantic Search** page through the wiki's \u2630 menu, declared by a `pages` entry in `factory.json` and shipped in the package."
|
|
274
279
|
},
|
|
275
280
|
"date": 1785397304700
|
|
276
281
|
},
|
|
@@ -280,7 +285,7 @@
|
|
|
280
285
|
"item": {
|
|
281
286
|
"type": "markdown",
|
|
282
287
|
"id": "21eb5354d2adb29e",
|
|
283
|
-
"text": "**Version:** 0.12.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub]
|
|
288
|
+
"text": "**Version:** 0.12.0\n\n# Links\n\n- [https://github.com/Hitchhikers-Guide-to-the-Galaxy/wiki-plugin-similarity GitHub repository]\n- [https://www.npmjs.com/package/wiki-plugin-similarity npm package]\n- [https://plugin.fedwiki.club/similarity-plugin.html Plugin documentation]\n- [https://search.fedwiki.club/ Search hub] \u2014 architecture, indexing, freshness."
|
|
284
289
|
},
|
|
285
290
|
"date": 1785397304700
|
|
286
291
|
}
|
package/server/server.js
CHANGED
|
@@ -17,7 +17,11 @@
|
|
|
17
17
|
// Set WIKI_EMBED_URL to proxy to an external embedder instead.
|
|
18
18
|
// GET /system/similarity-health.json
|
|
19
19
|
// Embedder supervisor state (child-process | semindex | url, breaker,
|
|
20
|
-
// recent crashes) — 200 always; diagnosable from outside.
|
|
20
|
+
// recent crashes) — 200 always; diagnosable from outside. Since 0.14.0 a
|
|
21
|
+
// url embedder also reports the source it came from (env | file), and
|
|
22
|
+
// localEmbedderAvailable says whether this box could embed unaided — the
|
|
23
|
+
// local path is never exercised while a proxy is set, so nothing else
|
|
24
|
+
// answers that question from outside.
|
|
21
25
|
// POST /system/search-report.json {query, domains, limit, threshold, live,
|
|
22
26
|
// vector?, seed?, text?, excludePage?}
|
|
23
27
|
// Ranked, stub-filtered, fork-bundled semantic report (page JSON).
|
|
@@ -82,7 +86,30 @@ const PLUGIN_VERSION = (() => {
|
|
|
82
86
|
})()
|
|
83
87
|
|
|
84
88
|
// Optional external embedder (proxy) — unset means embed in-process.
|
|
85
|
-
|
|
89
|
+
//
|
|
90
|
+
// Validated since 0.14.0. The public farm was set to the literal string
|
|
91
|
+
// "disabled", which is truthy, so every query was POSTed to a URL that does
|
|
92
|
+
// not parse and semantic search was dead farm-wide. A value that is not an
|
|
93
|
+
// http(s) URL now means what whoever typed it meant: no proxy.
|
|
94
|
+
const httpUrlOrNull = raw => {
|
|
95
|
+
if (!raw) return null
|
|
96
|
+
try {
|
|
97
|
+
const u = new URL(raw)
|
|
98
|
+
if (u.protocol === 'http:' || u.protocol === 'https:') return raw
|
|
99
|
+
} catch { /* not a URL at all */ }
|
|
100
|
+
return null
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const ENV_EMBED_URL = (() => {
|
|
104
|
+
const raw = process.env.WIKI_EMBED_URL
|
|
105
|
+
if (!raw) return null
|
|
106
|
+
const ok = httpUrlOrNull(raw)
|
|
107
|
+
if (!ok) {
|
|
108
|
+
console.log('[wiki-plugin-similarity] WIKI_EMBED_URL is not an http(s) URL ' +
|
|
109
|
+
`(${JSON.stringify(raw)}) — ignoring it and embedding in-process`)
|
|
110
|
+
}
|
|
111
|
+
return ok
|
|
112
|
+
})()
|
|
86
113
|
// Farm indexer for BUILD requests (HomeLab FastAPI, or unset on the public farm).
|
|
87
114
|
const INDEXER_URL = process.env.WIKI_INDEXER_URL || null
|
|
88
115
|
// Optional additional farm roots, colon-separated absolute paths.
|
|
@@ -96,13 +123,18 @@ const TRUSTED_NETS = parseNets()
|
|
|
96
123
|
|
|
97
124
|
// ── HTTP helpers ──────────────────────────────────────────────────────────────
|
|
98
125
|
|
|
126
|
+
// Pick the agent by protocol. Before 0.14.0 both helpers were http-only with
|
|
127
|
+
// port 80 hard-defaulted, so an https:// embed URL silently failed — the
|
|
128
|
+
// delegation path could never have worked across the public internet.
|
|
129
|
+
const agentFor = u => (u.protocol === 'https:' ? https : http)
|
|
130
|
+
|
|
99
131
|
const postJson = (url, body) =>
|
|
100
132
|
new Promise((resolve, reject) => {
|
|
101
133
|
const payload = JSON.stringify(body)
|
|
102
134
|
const u = new URL(url)
|
|
103
135
|
const opts = {
|
|
104
136
|
hostname: u.hostname,
|
|
105
|
-
port: u.port || 80,
|
|
137
|
+
port: u.port || (u.protocol === 'https:' ? 443 : 80),
|
|
106
138
|
path: u.pathname + u.search,
|
|
107
139
|
method: 'POST',
|
|
108
140
|
headers: {
|
|
@@ -110,7 +142,7 @@ const postJson = (url, body) =>
|
|
|
110
142
|
'Content-Length': Buffer.byteLength(payload),
|
|
111
143
|
},
|
|
112
144
|
}
|
|
113
|
-
const req =
|
|
145
|
+
const req = agentFor(u).request(opts, res => {
|
|
114
146
|
let data = ''
|
|
115
147
|
res.on('data', chunk => { data += chunk })
|
|
116
148
|
res.on('end', () => {
|
|
@@ -126,7 +158,7 @@ const postJson = (url, body) =>
|
|
|
126
158
|
const getJson = url =>
|
|
127
159
|
new Promise((resolve, reject) => {
|
|
128
160
|
const u = new URL(url)
|
|
129
|
-
|
|
161
|
+
agentFor(u).get(u, res => {
|
|
130
162
|
let data = ''
|
|
131
163
|
res.on('data', chunk => { data += chunk })
|
|
132
164
|
res.on('end', () => {
|
|
@@ -150,6 +182,45 @@ const readBody = req =>
|
|
|
150
182
|
})
|
|
151
183
|
|
|
152
184
|
|
|
185
|
+
// ── Embed URL from the farm root ──────────────────────────────────────────────
|
|
186
|
+
// Where the environment is not ours to set — the public farm's stack belongs to
|
|
187
|
+
// its host — the address may instead come from a file in the farm root:
|
|
188
|
+
//
|
|
189
|
+
// {farmRoot}/similarity.json { "embedUrl": "https://host/system/…" }
|
|
190
|
+
//
|
|
191
|
+
// Cached by mtime so a synced edit retargets the embedder with no restart, the
|
|
192
|
+
// same discipline wiki-plugin-farm's keys.js uses for status/api-keys.json.
|
|
193
|
+
// The file is never served over HTTP; writing it means write access to the farm.
|
|
194
|
+
let cfgCache = null // { mtimeMs, embedUrl }
|
|
195
|
+
|
|
196
|
+
const fileEmbedUrl = farmRoot => {
|
|
197
|
+
const file = path.join(farmRoot, 'similarity.json')
|
|
198
|
+
let stat
|
|
199
|
+
try { stat = fs.statSync(file) } catch { cfgCache = null; return null }
|
|
200
|
+
if (cfgCache && cfgCache.mtimeMs === stat.mtimeMs) return cfgCache.embedUrl
|
|
201
|
+
let embedUrl = null
|
|
202
|
+
try {
|
|
203
|
+
const cfg = JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
204
|
+
embedUrl = httpUrlOrNull(cfg.embedUrl)
|
|
205
|
+
if (cfg.embedUrl && !embedUrl) {
|
|
206
|
+
console.log(`caution: ${file}: embedUrl is not an http(s) URL — ignoring it`)
|
|
207
|
+
}
|
|
208
|
+
} catch (e) {
|
|
209
|
+
console.log(`caution: ${file}: ${e.message}`)
|
|
210
|
+
}
|
|
211
|
+
cfgCache = { mtimeMs: stat.mtimeMs, embedUrl }
|
|
212
|
+
return embedUrl
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Can this process turn text into a vector without leaving the box? Reported by
|
|
216
|
+
// the health route even when a proxy is configured, because with a proxy set the
|
|
217
|
+
// local path is never exercised and the logs stay silent about it — which is
|
|
218
|
+
// exactly the question that was unanswerable on the public farm.
|
|
219
|
+
const localEmbedderAvailable = () => {
|
|
220
|
+
if (embedder.viaSemindex) return true
|
|
221
|
+
try { require.resolve('@xenova/transformers'); return true } catch { return false }
|
|
222
|
+
}
|
|
223
|
+
|
|
153
224
|
// ── startServer — called by wiki-server/lib/plugins.js ────────────────────────
|
|
154
225
|
|
|
155
226
|
setModelMeta(MODEL_META)
|
|
@@ -157,6 +228,10 @@ setModelMeta(MODEL_META)
|
|
|
157
228
|
const startServer = ({ argv, app }) => {
|
|
158
229
|
// Farm root: argv.status = {farm}/{thisDomain}/status → go up two levels
|
|
159
230
|
const farmRoot = path.dirname(path.dirname(argv.status))
|
|
231
|
+
|
|
232
|
+
// env wins; then the farm-root file; then embed in-process.
|
|
233
|
+
const embedUrl = () => ENV_EMBED_URL || fileEmbedUrl(farmRoot)
|
|
234
|
+
const embedUrlSource = () => (ENV_EMBED_URL ? 'env' : (fileEmbedUrl(farmRoot) ? 'file' : null))
|
|
160
235
|
// primary farm is 'local'; extra farms (Nextcloud mirror) are 'public'.
|
|
161
236
|
// The galaxy tree (off-farm federation sites, written by the galaxy
|
|
162
237
|
// indexer) joins as kind 'galaxy' when present — never matched by '*',
|
|
@@ -188,10 +263,11 @@ const startServer = ({ argv, app }) => {
|
|
|
188
263
|
`| trusted nets: ${TRUSTED_NETS.join(' ')}`)
|
|
189
264
|
|
|
190
265
|
// Warm the embedding model in the background so the first query is fast.
|
|
191
|
-
if (!
|
|
266
|
+
if (!embedUrl()) embedder.warm()
|
|
192
267
|
|
|
193
268
|
async function embedText(text) {
|
|
194
|
-
|
|
269
|
+
const url = embedUrl()
|
|
270
|
+
if (url) return (await postJson(url, { text })).vector
|
|
195
271
|
return embedder.embed(text)
|
|
196
272
|
}
|
|
197
273
|
|
|
@@ -319,7 +395,11 @@ const startServer = ({ argv, app }) => {
|
|
|
319
395
|
plugin: 'wiki-plugin-similarity',
|
|
320
396
|
version: PLUGIN_VERSION,
|
|
321
397
|
...MODEL_META,
|
|
322
|
-
embedder:
|
|
398
|
+
embedder: (() => {
|
|
399
|
+
const url = embedUrl()
|
|
400
|
+
return url ? { via: 'url', url, source: embedUrlSource() } : embedder.status()
|
|
401
|
+
})(),
|
|
402
|
+
localEmbedderAvailable: localEmbedderAvailable(),
|
|
323
403
|
galaxy: fs.existsSync(galaxyDir)
|
|
324
404
|
? { root: galaxyDir, cache: galaxyCacheStats() }
|
|
325
405
|
: null,
|
|
@@ -636,4 +716,6 @@ const startServer = ({ argv, app }) => {
|
|
|
636
716
|
console.log('[wiki-plugin-similarity] routes registered')
|
|
637
717
|
}
|
|
638
718
|
|
|
639
|
-
|
|
719
|
+
// httpUrlOrNull and fileEmbedUrl are exported for the tests: they decide whether
|
|
720
|
+
// this farm embeds for itself or delegates, which is worth pinning down.
|
|
721
|
+
module.exports = { startServer, httpUrlOrNull, fileEmbedUrl, localEmbedderAvailable }
|