web-search-helper 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. web_search_helper-0.2.0/LICENSE +28 -0
  2. web_search_helper-0.2.0/PKG-INFO +243 -0
  3. web_search_helper-0.2.0/README.md +208 -0
  4. web_search_helper-0.2.0/pyproject.toml +58 -0
  5. web_search_helper-0.2.0/setup.cfg +4 -0
  6. web_search_helper-0.2.0/src/web_search_helper/__init__.py +44 -0
  7. web_search_helper-0.2.0/src/web_search_helper/api.py +195 -0
  8. web_search_helper-0.2.0/src/web_search_helper/assets/demo.html +35 -0
  9. web_search_helper-0.2.0/src/web_search_helper/assets/search.php +506 -0
  10. web_search_helper-0.2.0/src/web_search_helper/assets/web-search.d.ts +126 -0
  11. web_search_helper-0.2.0/src/web_search_helper/assets/web-search.js +670 -0
  12. web_search_helper-0.2.0/src/web_search_helper/assets/writer-plan.json +118 -0
  13. web_search_helper-0.2.0/src/web_search_helper/calibrate.py +140 -0
  14. web_search_helper-0.2.0/src/web_search_helper/cli.py +222 -0
  15. web_search_helper-0.2.0/src/web_search_helper/corpus.py +127 -0
  16. web_search_helper-0.2.0/src/web_search_helper/embed.py +139 -0
  17. web_search_helper-0.2.0/src/web_search_helper/engine.py +290 -0
  18. web_search_helper-0.2.0/src/web_search_helper/index.py +305 -0
  19. web_search_helper-0.2.0/src/web_search_helper/mcp_server.py +53 -0
  20. web_search_helper-0.2.0/src/web_search_helper/site.py +191 -0
  21. web_search_helper-0.2.0/src/web_search_helper/text.py +96 -0
  22. web_search_helper-0.2.0/src/web_search_helper.egg-info/PKG-INFO +243 -0
  23. web_search_helper-0.2.0/src/web_search_helper.egg-info/SOURCES.txt +31 -0
  24. web_search_helper-0.2.0/src/web_search_helper.egg-info/dependency_links.txt +1 -0
  25. web_search_helper-0.2.0/src/web_search_helper.egg-info/entry_points.txt +4 -0
  26. web_search_helper-0.2.0/src/web_search_helper.egg-info/requires.txt +23 -0
  27. web_search_helper-0.2.0/src/web_search_helper.egg-info/top_level.txt +1 -0
  28. web_search_helper-0.2.0/tests/test_doors.py +72 -0
  29. web_search_helper-0.2.0/tests/test_engine.py +91 -0
  30. web_search_helper-0.2.0/tests/test_no_personal_paths.py +293 -0
  31. web_search_helper-0.2.0/tests/test_php_parity.py +155 -0
  32. web_search_helper-0.2.0/tests/test_site.py +117 -0
  33. web_search_helper-0.2.0/tests/test_writer.py +58 -0
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, Warith Harchaoui
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,243 @@
1
+ Metadata-Version: 2.4
2
+ Name: web-search-helper
3
+ Version: 0.2.0
4
+ Summary: Search a private corpus from a website with PHP only: index it on your machine, serve it with one PHP file, embed the question in the browser. The corpus is never published; visitors get short excerpts, or an honest 'I cannot answer'.
5
+ Author-email: Warith Harchaoui <warithmetics@deraison.ai>
6
+ License: BSD-3-Clause
7
+ Project-URL: Homepage, https://github.com/warith-harchaoui/web-search-helper
8
+ Keywords: search,rag,semantic-search,bm25,php,transformers.js,static-site
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: BSD License
11
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
12
+ Classifier: Intended Audience :: Developers
13
+ Requires-Python: >=3.10
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENSE
16
+ Requires-Dist: click<9,>=8.1
17
+ Requires-Dist: numpy>=1.24
18
+ Provides-Extra: embed
19
+ Requires-Dist: sentence-transformers>=3; extra == "embed"
20
+ Provides-Extra: route
21
+ Requires-Dist: ann-router>=0.3; extra == "route"
22
+ Provides-Extra: api
23
+ Requires-Dist: fastapi<1,>=0.111; extra == "api"
24
+ Requires-Dist: uvicorn<1,>=0.30; extra == "api"
25
+ Provides-Extra: mcp
26
+ Requires-Dist: fastapi-mcp<1,>=0.4; extra == "mcp"
27
+ Requires-Dist: mcp<2,>=1.20; extra == "mcp"
28
+ Requires-Dist: fastapi<1,>=0.111; extra == "mcp"
29
+ Requires-Dist: uvicorn<1,>=0.30; extra == "mcp"
30
+ Provides-Extra: dev
31
+ Requires-Dist: pytest>=7; extra == "dev"
32
+ Requires-Dist: ruff>=0.6; extra == "dev"
33
+ Requires-Dist: httpx>=0.27; extra == "dev"
34
+ Dynamic: license-file
35
+
36
+ # web-search-helper
37
+
38
+ [🇫🇷](https://github.com/warith-harchaoui/web-search-helper/blob/main/LISEZMOI.md)&nbsp;&nbsp;|&nbsp;&nbsp;[🇬🇧](https://github.com/warith-harchaoui/web-search-helper/blob/main/README.md)
39
+
40
+ ![License](https://img.shields.io/badge/license-BSD--3--Clause-blue)
41
+ ![Python](https://img.shields.io/badge/python-3.10%E2%80%933.13-blue)
42
+ ![PHP](https://img.shields.io/badge/server-PHP%20only-777BB4)
43
+ [![tests](https://github.com/warith-harchaoui/web-search-helper/actions/workflows/tests.yml/badge.svg)](https://github.com/warith-harchaoui/web-search-helper/actions/workflows/tests.yml)
44
+
45
+ **Search a private corpus from a website, with PHP as the only server.** You
46
+ index the corpus on your machine; one PHP file answers the questions; the page
47
+ embeds each question in the browser. The corpus is never published: a visitor
48
+ receives a few sentences of what answers the question, or an honest "I cannot
49
+ answer" and nothing else.
50
+
51
+ `web-search-helper` belongs to the [**AI Helpers**](https://github.com/warith-harchaoui/ai-helpers)
52
+ suite. It works in French and in English out of the box (stopwords, answer
53
+ composition, interface), one index per language, and in any other language
54
+ with a stopword file. By default no LLM is involved: the answer is made of the
55
+ corpus's own sentences. Optionally, a small language model runs in the
56
+ visitor's browser and writes the answer from the excerpts, citing them; which
57
+ model depends on the visitor's machine and line, as planned by
58
+ [`best-engine-ai-helper`](https://github.com/warith-harchaoui/best-engine-ai-helper).
59
+
60
+ BSD 3-Clause licence. Author: Warith Harchaoui.
61
+
62
+ ## Why
63
+
64
+ A fully client-side search must ship the corpus to the browser, because the
65
+ browser can only search what it has. That is fine for public documentation and
66
+ wrong for a manuscript, a course or an internal knowledge base. Encrypting the
67
+ file does not help: the key has to be in the page too.
68
+
69
+ Most hosting offers PHP and nothing else: no Python, no Node, no vector
70
+ database. web-search-helper fits there.
71
+
72
+ | Where | What runs | Needs |
73
+ |---|---|---|
74
+ | your machine | cut, embed and index the corpus; measure the thresholds | Python ≥ 3.10, `sentence-transformers` |
75
+ | the server | `search.php`: BM25 + cosines + fusion + verdict, returns excerpts | PHP ≥ 7.4 with mbstring |
76
+ | the browser | `web-search.js`: embeds the question (transformers.js), composes an extractive answer, optionally writes one with an in-browser LLM | an ES-module browser; WebGPU for the written answer |
77
+
78
+ ## How it searches
79
+
80
+ - **Words**: BM25, with a conjunctive pass first (when some passages contain
81
+ every word, only they compete) and a timid typo tolerance (`elefant` finds
82
+ `éléphant`, but only because `elefant` is absent from the corpus).
83
+ - **Meaning**: passages are cut into windows the model reads entirely; a
84
+ passage scores the cosine of its best window. Each section also has a
85
+ centroid.
86
+ - **Fusion**: reciprocal rank fusion of the two rankings.
87
+ - **Verdict**: the engine answers only when one of three routes says the corpus
88
+ treats the question: the best window *and* the best section are close in
89
+ meaning, or every word was found with a strong BM25, or (for a one-word
90
+ query) the word appears in enough passages. Otherwise it returns nothing.
91
+ Ranking always returns *something*; only an absolute measure can say "no".
92
+
93
+ The search is exhaustive. `ann-router` would pick HNSW from 1 000 vectors at a
94
+ 10 ms budget, but that budget is numpy's, the corpus sizes here are small, and
95
+ the verdict judges the *absolute* best cosine, which an approximate index can
96
+ miss. `web-search-helper bench` measures the PHP time on your index and fails
97
+ when it goes over budget.
98
+
99
+ ## Protections against scraping
100
+
101
+ 1. Nothing is returned when the corpus does not treat the question.
102
+ 2. A passage comes back as an excerpt: its best window and the better of its
103
+ neighbours, snapped to sentence boundaries. Never the whole passage.
104
+ 3. A quota per address, per hour and per day (salted hash of the address; no
105
+ other trace kept).
106
+ 4. The index file is a PHP script: requested by URL, it answers an empty 404.
107
+ The data folder also gets a `.htaccess` that denies everything, in case.
108
+
109
+ Someone patient with thousands of well-chosen questions can still collect a
110
+ good part of the corpus. These measures make that slow, not impossible.
111
+
112
+ ## Quick start
113
+
114
+ ```bash
115
+ pip install 'web-search-helper[embed]'
116
+
117
+ # 1. Index: a .jsonl (one passage per line), a Markdown file or a folder of them.
118
+ web-search-helper build corpus/ -o build/fr.php --lang fr
119
+
120
+ # 2. Measure the thresholds on questions the corpus answers and questions it does not.
121
+ web-search-helper calibrate build/fr.php --inside inside.txt --outside outside.txt --write
122
+
123
+ # 3. Try a question, as the server will answer it.
124
+ web-search-helper search build/fr.php "Quelle est la parabole de l'éléphant ?" --explain
125
+
126
+ # 4. Copy the endpoint, the client and the index into the site; serve it locally.
127
+ web-search-helper install site/ build/fr.php build/en.php --demo
128
+ web-search-helper serve site/ # http://127.0.0.1:8765/demo.html?lang=fr (or ?lang=en)
129
+
130
+ # 5. Time the PHP engine; after upload, check that nothing leaks.
131
+ web-search-helper bench build/fr.php
132
+ web-search-helper check-online https://example.org/search/ --lang fr
133
+ ```
134
+
135
+ JSONL records: `{"text": "...", "doc": "Book title", "section": "Chapter 2", "url": "page.html#x", "meta": {...}}`.
136
+ `meta` is passed through to the browser untouched.
137
+
138
+ ## In the page
139
+
140
+ ```html
141
+ <script type="module">
142
+ import { WebSearch, composeAnswer } from './web-search.js';
143
+ const ws = new WebSearch({ endpoint: 'search.php', lang: 'fr' });
144
+ const r = await ws.ask(question, { onLexical: (first) => { /* early answer */ } });
145
+ if (!r.sure) showUnknown();
146
+ else show(await composeAnswer(r, { lang: 'fr', embed: ws }), r.hits);
147
+ </script>
148
+ ```
149
+
150
+ `ask` makes two calls: words only first (instant, before the model has
151
+ loaded), then words and meaning. The answer composed by `composeAnswer` is made
152
+ of corpus sentences, verbatim, chosen by maximal marginal relevance; nothing is
153
+ generated. Types are in `web-search.d.ts`. `mount(form, output, options)` wires
154
+ a minimal widget.
155
+
156
+ ## A written answer, in the visitor's browser
157
+
158
+ `mount(form, output, { …, writer: true })` adds a button under a `sure`
159
+ result: a small language model, run by transformers.js on WebGPU in the
160
+ visitor's browser, rewrites the excerpts into an answer that cites them as
161
+ [1], [2]… Nothing leaves the browser and the server does not change.
162
+
163
+ Which model is not decided in this package. `writer-plan.json`, written by
164
+ `best-engine-ai-helper browser plan --usage rag-answer`, is an ordered ladder
165
+ ranked by scores measured on this very task: Gemma 4 E4B (0.722) then Gemma 4
166
+ E2B (0.667), each in q4f16 and q4, with exact files and sizes (3.1 to 5.6 GB).
167
+ LFM2 1.2B was measured too (0.167: most of its answers cite nothing) and left
168
+ out, so a machine that cannot fetch a Gemma keeps the extractive answer rather
169
+ than one it would set aside. `scripts/eval_writer.py` reruns the measurement on
170
+ your own corpus; `scripts/make_writer_plan.py` rewrites the plan from
171
+ best-engine. The page probes the machine (WebGPU,
172
+ shader-f16, device memory, storage, mobile, data saver), checks the browser
173
+ cache, measures the bandwidth on a Range request to the model's own files, and
174
+ takes the first rung it can fetch within `max_download_s` (180 s). Nothing fits:
175
+ no button, and the extractive answer stands. `chooseRung` is tested against
176
+ best-engine's Python `browser.choose` on 2 251 simulated devices.
177
+
178
+ The written answer is checked before it is shown: the model only sees a `sure`
179
+ result, it is told to use the excerpts only and to say when they fall short,
180
+ and every sentence it writes must find most of its content words in the
181
+ excerpts it cites (`checkGrounding`). Otherwise the page sets the written
182
+ answer aside and keeps the extractive one. The excerpts stay listed below it,
183
+ verbatim.
184
+
185
+ ```js
186
+ import { Writer } from './web-search.js';
187
+ const { writer, reason } = await Writer.create('writer-plan.json'); // null when nothing fits
188
+ const answer = writer && await writer.write(result, { question, lang: 'fr', onText: show });
189
+ if (answer && answer.grounded) show(answer.text);
190
+ ```
191
+
192
+ ## The suite's other doors
193
+
194
+ Like every AI Helper, the engine is also reachable over HTTP and MCP, for
195
+ agents, scripts and local testing without PHP (`search.php` stays the
196
+ production server):
197
+
198
+ ```bash
199
+ pip install 'web-search-helper[api,mcp,embed]'
200
+ WSH_DATA_DIR=site/search-data web-search-helper-api # http://127.0.0.1:8040/docs, /gui
201
+ WSH_DATA_DIR=site/search-data web-search-helper-mcp # MCP at http://127.0.0.1:8041/mcp
202
+ ```
203
+
204
+ Routes: `POST /query` speaks `search.php`'s protocol, so `web-search.js` can
205
+ point at it (that is what `/gui` does); `POST /search` takes a plain question
206
+ and embeds it server-side; `GET /indexes`; `POST /calibrate` (writes
207
+ nothing); `POST /check-online`. The MCP tools are `indexes`, `search`,
208
+ `calibrate` and `check_online`. The agent skill lives in
209
+ `skills/web-search-helper/`; `TRIGGERS.md` lists what should reach for this
210
+ helper and what should not, and a test keeps it identical to the skill's copy.
211
+
212
+ ## Bring your own embeddings
213
+
214
+ A site that already embeds its corpus can keep its vectors and only hand the
215
+ engine over: `from_vectors(rows, vectors, lang, model)` builds the index from
216
+ passages (`t`, `c`, `s`, the window ends `o`, optional `u` and `m`) and one
217
+ normalised vector per window; `vectors_from_int8` reads int8 vectors back the
218
+ way a browser would. That is how "Interroger le livre" moved from an
219
+ all-in-the-browser search to `search.php` with the SAME vectors, stopwords and
220
+ thresholds, so that only the place of the computation changed.
221
+
222
+ ## Python
223
+
224
+ ```python
225
+ import web_search_helper as wsh
226
+
227
+ idx = wsh.build(wsh.read_corpus("corpus/"), wsh.SentenceTransformerEmbedder(), "fr")
228
+ wsh.save(idx, "build/fr.php")
229
+ print(wsh.search(idx, "the data wall", explain=True))
230
+ ```
231
+
232
+ `engine.py` is the reference twin of `search.php`; `tests/test_php_parity.py`
233
+ runs both on the same questions and fails if they disagree.
234
+
235
+ ## Limits
236
+
237
+ - The question's vector is computed by transformers.js in the page and the
238
+ index by sentence-transformers here: same weights, cosines equal to about
239
+ 1e-2. Calibration reports its margins; a threshold with less margin than that
240
+ is fragile.
241
+ - One index per language; the page says which language it asks.
242
+ - The first question downloads the model in the browser (about 120 MB for the
243
+ default multilingual MiniLM), then the browser keeps it.
@@ -0,0 +1,208 @@
1
+ # web-search-helper
2
+
3
+ [🇫🇷](https://github.com/warith-harchaoui/web-search-helper/blob/main/LISEZMOI.md)&nbsp;&nbsp;|&nbsp;&nbsp;[🇬🇧](https://github.com/warith-harchaoui/web-search-helper/blob/main/README.md)
4
+
5
+ ![License](https://img.shields.io/badge/license-BSD--3--Clause-blue)
6
+ ![Python](https://img.shields.io/badge/python-3.10%E2%80%933.13-blue)
7
+ ![PHP](https://img.shields.io/badge/server-PHP%20only-777BB4)
8
+ [![tests](https://github.com/warith-harchaoui/web-search-helper/actions/workflows/tests.yml/badge.svg)](https://github.com/warith-harchaoui/web-search-helper/actions/workflows/tests.yml)
9
+
10
+ **Search a private corpus from a website, with PHP as the only server.** You
11
+ index the corpus on your machine; one PHP file answers the questions; the page
12
+ embeds each question in the browser. The corpus is never published: a visitor
13
+ receives a few sentences of what answers the question, or an honest "I cannot
14
+ answer" and nothing else.
15
+
16
+ `web-search-helper` belongs to the [**AI Helpers**](https://github.com/warith-harchaoui/ai-helpers)
17
+ suite. It works in French and in English out of the box (stopwords, answer
18
+ composition, interface), one index per language, and in any other language
19
+ with a stopword file. By default no LLM is involved: the answer is made of the
20
+ corpus's own sentences. Optionally, a small language model runs in the
21
+ visitor's browser and writes the answer from the excerpts, citing them; which
22
+ model depends on the visitor's machine and line, as planned by
23
+ [`best-engine-ai-helper`](https://github.com/warith-harchaoui/best-engine-ai-helper).
24
+
25
+ BSD 3-Clause licence. Author: Warith Harchaoui.
26
+
27
+ ## Why
28
+
29
+ A fully client-side search must ship the corpus to the browser, because the
30
+ browser can only search what it has. That is fine for public documentation and
31
+ wrong for a manuscript, a course or an internal knowledge base. Encrypting the
32
+ file does not help: the key has to be in the page too.
33
+
34
+ Most hosting offers PHP and nothing else: no Python, no Node, no vector
35
+ database. web-search-helper fits there.
36
+
37
+ | Where | What runs | Needs |
38
+ |---|---|---|
39
+ | your machine | cut, embed and index the corpus; measure the thresholds | Python ≥ 3.10, `sentence-transformers` |
40
+ | the server | `search.php`: BM25 + cosines + fusion + verdict, returns excerpts | PHP ≥ 7.4 with mbstring |
41
+ | the browser | `web-search.js`: embeds the question (transformers.js), composes an extractive answer, optionally writes one with an in-browser LLM | an ES-module browser; WebGPU for the written answer |
42
+
43
+ ## How it searches
44
+
45
+ - **Words**: BM25, with a conjunctive pass first (when some passages contain
46
+ every word, only they compete) and a timid typo tolerance (`elefant` finds
47
+ `éléphant`, but only because `elefant` is absent from the corpus).
48
+ - **Meaning**: passages are cut into windows the model reads entirely; a
49
+ passage scores the cosine of its best window. Each section also has a
50
+ centroid.
51
+ - **Fusion**: reciprocal rank fusion of the two rankings.
52
+ - **Verdict**: the engine answers only when one of three routes says the corpus
53
+ treats the question: the best window *and* the best section are close in
54
+ meaning, or every word was found with a strong BM25, or (for a one-word
55
+ query) the word appears in enough passages. Otherwise it returns nothing.
56
+ Ranking always returns *something*; only an absolute measure can say "no".
57
+
58
+ The search is exhaustive. `ann-router` would pick HNSW from 1 000 vectors at a
59
+ 10 ms budget, but that budget is numpy's, the corpus sizes here are small, and
60
+ the verdict judges the *absolute* best cosine, which an approximate index can
61
+ miss. `web-search-helper bench` measures the PHP time on your index and fails
62
+ when it goes over budget.
63
+
64
+ ## Protections against scraping
65
+
66
+ 1. Nothing is returned when the corpus does not treat the question.
67
+ 2. A passage comes back as an excerpt: its best window and the better of its
68
+ neighbours, snapped to sentence boundaries. Never the whole passage.
69
+ 3. A quota per address, per hour and per day (salted hash of the address; no
70
+ other trace kept).
71
+ 4. The index file is a PHP script: requested by URL, it answers an empty 404.
72
+ The data folder also gets a `.htaccess` that denies everything, in case.
73
+
74
+ Someone patient with thousands of well-chosen questions can still collect a
75
+ good part of the corpus. These measures make that slow, not impossible.
76
+
77
+ ## Quick start
78
+
79
+ ```bash
80
+ pip install 'web-search-helper[embed]'
81
+
82
+ # 1. Index: a .jsonl (one passage per line), a Markdown file or a folder of them.
83
+ web-search-helper build corpus/ -o build/fr.php --lang fr
84
+
85
+ # 2. Measure the thresholds on questions the corpus answers and questions it does not.
86
+ web-search-helper calibrate build/fr.php --inside inside.txt --outside outside.txt --write
87
+
88
+ # 3. Try a question, as the server will answer it.
89
+ web-search-helper search build/fr.php "Quelle est la parabole de l'éléphant ?" --explain
90
+
91
+ # 4. Copy the endpoint, the client and the index into the site; serve it locally.
92
+ web-search-helper install site/ build/fr.php build/en.php --demo
93
+ web-search-helper serve site/ # http://127.0.0.1:8765/demo.html?lang=fr (or ?lang=en)
94
+
95
+ # 5. Time the PHP engine; after upload, check that nothing leaks.
96
+ web-search-helper bench build/fr.php
97
+ web-search-helper check-online https://example.org/search/ --lang fr
98
+ ```
99
+
100
+ JSONL records: `{"text": "...", "doc": "Book title", "section": "Chapter 2", "url": "page.html#x", "meta": {...}}`.
101
+ `meta` is passed through to the browser untouched.
102
+
103
+ ## In the page
104
+
105
+ ```html
106
+ <script type="module">
107
+ import { WebSearch, composeAnswer } from './web-search.js';
108
+ const ws = new WebSearch({ endpoint: 'search.php', lang: 'fr' });
109
+ const r = await ws.ask(question, { onLexical: (first) => { /* early answer */ } });
110
+ if (!r.sure) showUnknown();
111
+ else show(await composeAnswer(r, { lang: 'fr', embed: ws }), r.hits);
112
+ </script>
113
+ ```
114
+
115
+ `ask` makes two calls: words only first (instant, before the model has
116
+ loaded), then words and meaning. The answer composed by `composeAnswer` is made
117
+ of corpus sentences, verbatim, chosen by maximal marginal relevance; nothing is
118
+ generated. Types are in `web-search.d.ts`. `mount(form, output, options)` wires
119
+ a minimal widget.
120
+
121
+ ## A written answer, in the visitor's browser
122
+
123
+ `mount(form, output, { …, writer: true })` adds a button under a `sure`
124
+ result: a small language model, run by transformers.js on WebGPU in the
125
+ visitor's browser, rewrites the excerpts into an answer that cites them as
126
+ [1], [2]… Nothing leaves the browser and the server does not change.
127
+
128
+ Which model is not decided in this package. `writer-plan.json`, written by
129
+ `best-engine-ai-helper browser plan --usage rag-answer`, is an ordered ladder
130
+ ranked by scores measured on this very task: Gemma 4 E4B (0.722) then Gemma 4
131
+ E2B (0.667), each in q4f16 and q4, with exact files and sizes (3.1 to 5.6 GB).
132
+ LFM2 1.2B was measured too (0.167: most of its answers cite nothing) and left
133
+ out, so a machine that cannot fetch a Gemma keeps the extractive answer rather
134
+ than one it would set aside. `scripts/eval_writer.py` reruns the measurement on
135
+ your own corpus; `scripts/make_writer_plan.py` rewrites the plan from
136
+ best-engine. The page probes the machine (WebGPU,
137
+ shader-f16, device memory, storage, mobile, data saver), checks the browser
138
+ cache, measures the bandwidth on a Range request to the model's own files, and
139
+ takes the first rung it can fetch within `max_download_s` (180 s). Nothing fits:
140
+ no button, and the extractive answer stands. `chooseRung` is tested against
141
+ best-engine's Python `browser.choose` on 2 251 simulated devices.
142
+
143
+ The written answer is checked before it is shown: the model only sees a `sure`
144
+ result, it is told to use the excerpts only and to say when they fall short,
145
+ and every sentence it writes must find most of its content words in the
146
+ excerpts it cites (`checkGrounding`). Otherwise the page sets the written
147
+ answer aside and keeps the extractive one. The excerpts stay listed below it,
148
+ verbatim.
149
+
150
+ ```js
151
+ import { Writer } from './web-search.js';
152
+ const { writer, reason } = await Writer.create('writer-plan.json'); // null when nothing fits
153
+ const answer = writer && await writer.write(result, { question, lang: 'fr', onText: show });
154
+ if (answer && answer.grounded) show(answer.text);
155
+ ```
156
+
157
+ ## The suite's other doors
158
+
159
+ Like every AI Helper, the engine is also reachable over HTTP and MCP, for
160
+ agents, scripts and local testing without PHP (`search.php` stays the
161
+ production server):
162
+
163
+ ```bash
164
+ pip install 'web-search-helper[api,mcp,embed]'
165
+ WSH_DATA_DIR=site/search-data web-search-helper-api # http://127.0.0.1:8040/docs, /gui
166
+ WSH_DATA_DIR=site/search-data web-search-helper-mcp # MCP at http://127.0.0.1:8041/mcp
167
+ ```
168
+
169
+ Routes: `POST /query` speaks `search.php`'s protocol, so `web-search.js` can
170
+ point at it (that is what `/gui` does); `POST /search` takes a plain question
171
+ and embeds it server-side; `GET /indexes`; `POST /calibrate` (writes
172
+ nothing); `POST /check-online`. The MCP tools are `indexes`, `search`,
173
+ `calibrate` and `check_online`. The agent skill lives in
174
+ `skills/web-search-helper/`; `TRIGGERS.md` lists what should reach for this
175
+ helper and what should not, and a test keeps it identical to the skill's copy.
176
+
177
+ ## Bring your own embeddings
178
+
179
+ A site that already embeds its corpus can keep its vectors and only hand the
180
+ engine over: `from_vectors(rows, vectors, lang, model)` builds the index from
181
+ passages (`t`, `c`, `s`, the window ends `o`, optional `u` and `m`) and one
182
+ normalised vector per window; `vectors_from_int8` reads int8 vectors back the
183
+ way a browser would. That is how "Interroger le livre" moved from an
184
+ all-in-the-browser search to `search.php` with the SAME vectors, stopwords and
185
+ thresholds, so that only the place of the computation changed.
186
+
187
+ ## Python
188
+
189
+ ```python
190
+ import web_search_helper as wsh
191
+
192
+ idx = wsh.build(wsh.read_corpus("corpus/"), wsh.SentenceTransformerEmbedder(), "fr")
193
+ wsh.save(idx, "build/fr.php")
194
+ print(wsh.search(idx, "the data wall", explain=True))
195
+ ```
196
+
197
+ `engine.py` is the reference twin of `search.php`; `tests/test_php_parity.py`
198
+ runs both on the same questions and fails if they disagree.
199
+
200
+ ## Limits
201
+
202
+ - The question's vector is computed by transformers.js in the page and the
203
+ index by sentence-transformers here: same weights, cosines equal to about
204
+ 1e-2. Calibration reports its margins; a threshold with less margin than that
205
+ is fragile.
206
+ - One index per language; the page says which language it asks.
207
+ - The first question downloads the model in the browser (about 120 MB for the
208
+ default multilingual MiniLM), then the browser keeps it.
@@ -0,0 +1,58 @@
1
+ [build-system]
2
+ requires = ["setuptools>=64", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "web-search-helper"
7
+ version = "0.2.0"
8
+ description = "Search a private corpus from a website with PHP only: index it on your machine, serve it with one PHP file, embed the question in the browser. The corpus is never published; visitors get short excerpts, or an honest 'I cannot answer'."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "BSD-3-Clause" }
12
+ authors = [{ name = "Warith Harchaoui", email = "warithmetics@deraison.ai" }]
13
+ keywords = ["search", "rag", "semantic-search", "bm25", "php", "transformers.js", "static-site"]
14
+ classifiers = [
15
+ "Programming Language :: Python :: 3",
16
+ "License :: OSI Approved :: BSD License",
17
+ "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
18
+ "Intended Audience :: Developers",
19
+ ]
20
+ # Core: building with the hash embedder (dry runs, tests) needs only numpy and
21
+ # click. The real model is the [embed] extra, because sentence-transformers
22
+ # pulls torch.
23
+ dependencies = [
24
+ "click>=8.1,<9",
25
+ "numpy>=1.24",
26
+ ]
27
+
28
+ [project.optional-dependencies]
29
+ embed = ["sentence-transformers>=3"]
30
+ # ann-router is only consulted, to print its advice next to the measured
31
+ # PHP timing.
32
+ route = ["ann-router>=0.3"]
33
+ api = ["fastapi>=0.111,<1", "uvicorn>=0.30,<1"]
34
+ # fastapi-mcp 0.4.x breaks on mcp 2 (positional Server() signature), as
35
+ # elsewhere in the suite.
36
+ mcp = ["fastapi-mcp>=0.4,<1", "mcp<2,>=1.20", "fastapi>=0.111,<1", "uvicorn>=0.30,<1"]
37
+ dev = ["pytest>=7", "ruff>=0.6", "httpx>=0.27"]
38
+
39
+ [project.scripts]
40
+ web-search-helper = "web_search_helper.cli:main"
41
+ web-search-helper-api = "web_search_helper.api:main"
42
+ web-search-helper-mcp = "web_search_helper.mcp_server:main"
43
+
44
+ [project.urls]
45
+ Homepage = "https://github.com/warith-harchaoui/web-search-helper"
46
+
47
+ [tool.setuptools.packages.find]
48
+ where = ["src"]
49
+
50
+ [tool.setuptools.package-data]
51
+ web_search_helper = ["assets/*"]
52
+
53
+ [tool.pytest.ini_options]
54
+ testpaths = ["tests", "src"]
55
+ addopts = "-q --doctest-modules"
56
+
57
+ [tool.ruff]
58
+ line-length = 110
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,44 @@
1
+ """web-search-helper — search a private corpus from a website, PHP only.
2
+
3
+ Build the index on your machine; serve it with one PHP file; embed the
4
+ question in the browser. The corpus is never published: the page receives
5
+ short excerpts of what answers the question, or nothing when the corpus does
6
+ not treat it.
7
+
8
+ >>> import web_search_helper as wsh
9
+ >>> callable(wsh.build) and callable(wsh.search)
10
+ True
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ __version__ = "0.2.0"
16
+
17
+ from .calibrate import calibrate
18
+ from .corpus import Passage, read_corpus, read_jsonl
19
+ from .embed import DEFAULT_MODEL, MODELS, HashEmbedder, SentenceTransformerEmbedder
20
+ from .engine import search
21
+ from .index import Index, build, from_vectors, load, save, vectors_from_int8
22
+ from .site import bench, check_online, install, serve
23
+
24
+ __all__ = [
25
+ "DEFAULT_MODEL",
26
+ "MODELS",
27
+ "HashEmbedder",
28
+ "Index",
29
+ "Passage",
30
+ "SentenceTransformerEmbedder",
31
+ "bench",
32
+ "build",
33
+ "calibrate",
34
+ "check_online",
35
+ "from_vectors",
36
+ "install",
37
+ "load",
38
+ "read_corpus",
39
+ "read_jsonl",
40
+ "save",
41
+ "search",
42
+ "serve",
43
+ "vectors_from_int8",
44
+ ]