web-search-helper 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- web_search_helper-0.2.0/LICENSE +28 -0
- web_search_helper-0.2.0/PKG-INFO +243 -0
- web_search_helper-0.2.0/README.md +208 -0
- web_search_helper-0.2.0/pyproject.toml +58 -0
- web_search_helper-0.2.0/setup.cfg +4 -0
- web_search_helper-0.2.0/src/web_search_helper/__init__.py +44 -0
- web_search_helper-0.2.0/src/web_search_helper/api.py +195 -0
- web_search_helper-0.2.0/src/web_search_helper/assets/demo.html +35 -0
- web_search_helper-0.2.0/src/web_search_helper/assets/search.php +506 -0
- web_search_helper-0.2.0/src/web_search_helper/assets/web-search.d.ts +126 -0
- web_search_helper-0.2.0/src/web_search_helper/assets/web-search.js +670 -0
- web_search_helper-0.2.0/src/web_search_helper/assets/writer-plan.json +118 -0
- web_search_helper-0.2.0/src/web_search_helper/calibrate.py +140 -0
- web_search_helper-0.2.0/src/web_search_helper/cli.py +222 -0
- web_search_helper-0.2.0/src/web_search_helper/corpus.py +127 -0
- web_search_helper-0.2.0/src/web_search_helper/embed.py +139 -0
- web_search_helper-0.2.0/src/web_search_helper/engine.py +290 -0
- web_search_helper-0.2.0/src/web_search_helper/index.py +305 -0
- web_search_helper-0.2.0/src/web_search_helper/mcp_server.py +53 -0
- web_search_helper-0.2.0/src/web_search_helper/site.py +191 -0
- web_search_helper-0.2.0/src/web_search_helper/text.py +96 -0
- web_search_helper-0.2.0/src/web_search_helper.egg-info/PKG-INFO +243 -0
- web_search_helper-0.2.0/src/web_search_helper.egg-info/SOURCES.txt +31 -0
- web_search_helper-0.2.0/src/web_search_helper.egg-info/dependency_links.txt +1 -0
- web_search_helper-0.2.0/src/web_search_helper.egg-info/entry_points.txt +4 -0
- web_search_helper-0.2.0/src/web_search_helper.egg-info/requires.txt +23 -0
- web_search_helper-0.2.0/src/web_search_helper.egg-info/top_level.txt +1 -0
- web_search_helper-0.2.0/tests/test_doors.py +72 -0
- web_search_helper-0.2.0/tests/test_engine.py +91 -0
- web_search_helper-0.2.0/tests/test_no_personal_paths.py +293 -0
- web_search_helper-0.2.0/tests/test_php_parity.py +155 -0
- web_search_helper-0.2.0/tests/test_site.py +117 -0
- web_search_helper-0.2.0/tests/test_writer.py +58 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Warith Harchaoui
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: web-search-helper
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Search a private corpus from a website with PHP only: index it on your machine, serve it with one PHP file, embed the question in the browser. The corpus is never published; visitors get short excerpts, or an honest 'I cannot answer'.
|
|
5
|
+
Author-email: Warith Harchaoui <warithmetics@deraison.ai>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/warith-harchaoui/web-search-helper
|
|
8
|
+
Keywords: search,rag,semantic-search,bm25,php,transformers.js,static-site
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: click<9,>=8.1
|
|
17
|
+
Requires-Dist: numpy>=1.24
|
|
18
|
+
Provides-Extra: embed
|
|
19
|
+
Requires-Dist: sentence-transformers>=3; extra == "embed"
|
|
20
|
+
Provides-Extra: route
|
|
21
|
+
Requires-Dist: ann-router>=0.3; extra == "route"
|
|
22
|
+
Provides-Extra: api
|
|
23
|
+
Requires-Dist: fastapi<1,>=0.111; extra == "api"
|
|
24
|
+
Requires-Dist: uvicorn<1,>=0.30; extra == "api"
|
|
25
|
+
Provides-Extra: mcp
|
|
26
|
+
Requires-Dist: fastapi-mcp<1,>=0.4; extra == "mcp"
|
|
27
|
+
Requires-Dist: mcp<2,>=1.20; extra == "mcp"
|
|
28
|
+
Requires-Dist: fastapi<1,>=0.111; extra == "mcp"
|
|
29
|
+
Requires-Dist: uvicorn<1,>=0.30; extra == "mcp"
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
32
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
33
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# web-search-helper
|
|
37
|
+
|
|
38
|
+
[🇫🇷](https://github.com/warith-harchaoui/web-search-helper/blob/main/LISEZMOI.md) | [🇬🇧](https://github.com/warith-harchaoui/web-search-helper/blob/main/README.md)
|
|
39
|
+
|
|
40
|
+

|
|
41
|
+

|
|
42
|
+

|
|
43
|
+
[](https://github.com/warith-harchaoui/web-search-helper/actions/workflows/tests.yml)
|
|
44
|
+
|
|
45
|
+
**Search a private corpus from a website, with PHP as the only server.** You
|
|
46
|
+
index the corpus on your machine; one PHP file answers the questions; the page
|
|
47
|
+
embeds each question in the browser. The corpus is never published: a visitor
|
|
48
|
+
receives a few sentences of what answers the question, or an honest "I cannot
|
|
49
|
+
answer" and nothing else.
|
|
50
|
+
|
|
51
|
+
`web-search-helper` belongs to the [**AI Helpers**](https://github.com/warith-harchaoui/ai-helpers)
|
|
52
|
+
suite. It works in French and in English out of the box (stopwords, answer
|
|
53
|
+
composition, interface), one index per language, and in any other language
|
|
54
|
+
with a stopword file. By default no LLM is involved: the answer is made of the
|
|
55
|
+
corpus's own sentences. Optionally, a small language model runs in the
|
|
56
|
+
visitor's browser and writes the answer from the excerpts, citing them; which
|
|
57
|
+
model depends on the visitor's machine and line, as planned by
|
|
58
|
+
[`best-engine-ai-helper`](https://github.com/warith-harchaoui/best-engine-ai-helper).
|
|
59
|
+
|
|
60
|
+
BSD 3-Clause licence. Author: Warith Harchaoui.
|
|
61
|
+
|
|
62
|
+
## Why
|
|
63
|
+
|
|
64
|
+
A fully client-side search must ship the corpus to the browser, because the
|
|
65
|
+
browser can only search what it has. That is fine for public documentation and
|
|
66
|
+
wrong for a manuscript, a course or an internal knowledge base. Encrypting the
|
|
67
|
+
file does not help: the key has to be in the page too.
|
|
68
|
+
|
|
69
|
+
Most hosting offers PHP and nothing else: no Python, no Node, no vector
|
|
70
|
+
database. web-search-helper fits there.
|
|
71
|
+
|
|
72
|
+
| Where | What runs | Needs |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| your machine | cut, embed and index the corpus; measure the thresholds | Python ≥ 3.10, `sentence-transformers` |
|
|
75
|
+
| the server | `search.php`: BM25 + cosines + fusion + verdict, returns excerpts | PHP ≥ 7.4 with mbstring |
|
|
76
|
+
| the browser | `web-search.js`: embeds the question (transformers.js), composes an extractive answer, optionally writes one with an in-browser LLM | an ES-module browser; WebGPU for the written answer |
|
|
77
|
+
|
|
78
|
+
## How it searches
|
|
79
|
+
|
|
80
|
+
- **Words**: BM25, with a conjunctive pass first (when some passages contain
|
|
81
|
+
every word, only they compete) and a timid typo tolerance (`elefant` finds
|
|
82
|
+
`éléphant`, but only because `elefant` is absent from the corpus).
|
|
83
|
+
- **Meaning**: passages are cut into windows the model reads entirely; a
|
|
84
|
+
passage scores the cosine of its best window. Each section also has a
|
|
85
|
+
centroid.
|
|
86
|
+
- **Fusion**: reciprocal rank fusion of the two rankings.
|
|
87
|
+
- **Verdict**: the engine answers only when one of three routes says the corpus
|
|
88
|
+
treats the question: the best window *and* the best section are close in
|
|
89
|
+
meaning, or every word was found with a strong BM25, or (for a one-word
|
|
90
|
+
query) the word appears in enough passages. Otherwise it returns nothing.
|
|
91
|
+
Ranking always returns *something*; only an absolute measure can say "no".
|
|
92
|
+
|
|
93
|
+
The search is exhaustive. `ann-router` would pick HNSW from 1 000 vectors at a
|
|
94
|
+
10 ms budget, but that budget is numpy's, the corpus sizes here are small, and
|
|
95
|
+
the verdict judges the *absolute* best cosine, which an approximate index can
|
|
96
|
+
miss. `web-search-helper bench` measures the PHP time on your index and fails
|
|
97
|
+
when it goes over budget.
|
|
98
|
+
|
|
99
|
+
## Protections against scraping
|
|
100
|
+
|
|
101
|
+
1. Nothing is returned when the corpus does not treat the question.
|
|
102
|
+
2. A passage comes back as an excerpt: its best window and the better of its
|
|
103
|
+
neighbours, snapped to sentence boundaries. Never the whole passage.
|
|
104
|
+
3. A quota per address, per hour and per day (salted hash of the address; no
|
|
105
|
+
other trace kept).
|
|
106
|
+
4. The index file is a PHP script: requested by URL, it answers an empty 404.
|
|
107
|
+
The data folder also gets a `.htaccess` that denies everything, in case.
|
|
108
|
+
|
|
109
|
+
Someone patient with thousands of well-chosen questions can still collect a
|
|
110
|
+
good part of the corpus. These measures make that slow, not impossible.
|
|
111
|
+
|
|
112
|
+
## Quick start
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
pip install 'web-search-helper[embed]'
|
|
116
|
+
|
|
117
|
+
# 1. Index: a .jsonl (one passage per line), a Markdown file or a folder of them.
|
|
118
|
+
web-search-helper build corpus/ -o build/fr.php --lang fr
|
|
119
|
+
|
|
120
|
+
# 2. Measure the thresholds on questions the corpus answers and questions it does not.
|
|
121
|
+
web-search-helper calibrate build/fr.php --inside inside.txt --outside outside.txt --write
|
|
122
|
+
|
|
123
|
+
# 3. Try a question, as the server will answer it.
|
|
124
|
+
web-search-helper search build/fr.php "Quelle est la parabole de l'éléphant ?" --explain
|
|
125
|
+
|
|
126
|
+
# 4. Copy the endpoint, the client and the index into the site; serve it locally.
|
|
127
|
+
web-search-helper install site/ build/fr.php build/en.php --demo
|
|
128
|
+
web-search-helper serve site/ # http://127.0.0.1:8765/demo.html?lang=fr (or ?lang=en)
|
|
129
|
+
|
|
130
|
+
# 5. Time the PHP engine; after upload, check that nothing leaks.
|
|
131
|
+
web-search-helper bench build/fr.php
|
|
132
|
+
web-search-helper check-online https://example.org/search/ --lang fr
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
JSONL records: `{"text": "...", "doc": "Book title", "section": "Chapter 2", "url": "page.html#x", "meta": {...}}`.
|
|
136
|
+
`meta` is passed through to the browser untouched.
|
|
137
|
+
|
|
138
|
+
## In the page
|
|
139
|
+
|
|
140
|
+
```html
|
|
141
|
+
<script type="module">
|
|
142
|
+
import { WebSearch, composeAnswer } from './web-search.js';
|
|
143
|
+
const ws = new WebSearch({ endpoint: 'search.php', lang: 'fr' });
|
|
144
|
+
const r = await ws.ask(question, { onLexical: (first) => { /* early answer */ } });
|
|
145
|
+
if (!r.sure) showUnknown();
|
|
146
|
+
else show(await composeAnswer(r, { lang: 'fr', embed: ws }), r.hits);
|
|
147
|
+
</script>
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
`ask` makes two calls: words only first (instant, before the model has
|
|
151
|
+
loaded), then words and meaning. The answer composed by `composeAnswer` is made
|
|
152
|
+
of corpus sentences, verbatim, chosen by maximal marginal relevance; nothing is
|
|
153
|
+
generated. Types are in `web-search.d.ts`. `mount(form, output, options)` wires
|
|
154
|
+
a minimal widget.
|
|
155
|
+
|
|
156
|
+
## A written answer, in the visitor's browser
|
|
157
|
+
|
|
158
|
+
`mount(form, output, { …, writer: true })` adds a button under a `sure`
|
|
159
|
+
result: a small language model, run by transformers.js on WebGPU in the
|
|
160
|
+
visitor's browser, rewrites the excerpts into an answer that cites them as
|
|
161
|
+
[1], [2]… Nothing leaves the browser and the server does not change.
|
|
162
|
+
|
|
163
|
+
Which model is not decided in this package. `writer-plan.json`, written by
|
|
164
|
+
`best-engine-ai-helper browser plan --usage rag-answer`, is an ordered ladder
|
|
165
|
+
ranked by scores measured on this very task: Gemma 4 E4B (0.722) then Gemma 4
|
|
166
|
+
E2B (0.667), each in q4f16 and q4, with exact files and sizes (3.1 to 5.6 GB).
|
|
167
|
+
LFM2 1.2B was measured too (0.167: most of its answers cite nothing) and left
|
|
168
|
+
out, so a machine that cannot fetch a Gemma keeps the extractive answer rather
|
|
169
|
+
than one it would set aside. `scripts/eval_writer.py` reruns the measurement on
|
|
170
|
+
your own corpus; `scripts/make_writer_plan.py` rewrites the plan from
|
|
171
|
+
best-engine. The page probes the machine (WebGPU,
|
|
172
|
+
shader-f16, device memory, storage, mobile, data saver), checks the browser
|
|
173
|
+
cache, measures the bandwidth on a Range request to the model's own files, and
|
|
174
|
+
takes the first rung it can fetch within `max_download_s` (180 s). Nothing fits:
|
|
175
|
+
no button, and the extractive answer stands. `chooseRung` is tested against
|
|
176
|
+
best-engine's Python `browser.choose` on 2 251 simulated devices.
|
|
177
|
+
|
|
178
|
+
The written answer is checked before it is shown: the model only sees a `sure`
|
|
179
|
+
result, it is told to use the excerpts only and to say when they fall short,
|
|
180
|
+
and every sentence it writes must find most of its content words in the
|
|
181
|
+
excerpts it cites (`checkGrounding`). Otherwise the page sets the written
|
|
182
|
+
answer aside and keeps the extractive one. The excerpts stay listed below it,
|
|
183
|
+
verbatim.
|
|
184
|
+
|
|
185
|
+
```js
|
|
186
|
+
import { Writer } from './web-search.js';
|
|
187
|
+
const { writer, reason } = await Writer.create('writer-plan.json'); // null when nothing fits
|
|
188
|
+
const answer = writer && await writer.write(result, { question, lang: 'fr', onText: show });
|
|
189
|
+
if (answer && answer.grounded) show(answer.text);
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## The suite's other doors
|
|
193
|
+
|
|
194
|
+
Like every AI Helper, the engine is also reachable over HTTP and MCP, for
|
|
195
|
+
agents, scripts and local testing without PHP (`search.php` stays the
|
|
196
|
+
production server):
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
pip install 'web-search-helper[api,mcp,embed]'
|
|
200
|
+
WSH_DATA_DIR=site/search-data web-search-helper-api # http://127.0.0.1:8040/docs, /gui
|
|
201
|
+
WSH_DATA_DIR=site/search-data web-search-helper-mcp # MCP at http://127.0.0.1:8041/mcp
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Routes: `POST /query` speaks `search.php`'s protocol, so `web-search.js` can
|
|
205
|
+
point at it (that is what `/gui` does); `POST /search` takes a plain question
|
|
206
|
+
and embeds it server-side; `GET /indexes`; `POST /calibrate` (writes
|
|
207
|
+
nothing); `POST /check-online`. The MCP tools are `indexes`, `search`,
|
|
208
|
+
`calibrate` and `check_online`. The agent skill lives in
|
|
209
|
+
`skills/web-search-helper/`; `TRIGGERS.md` lists what should reach for this
|
|
210
|
+
helper and what should not, and a test keeps it identical to the skill's copy.
|
|
211
|
+
|
|
212
|
+
## Bring your own embeddings
|
|
213
|
+
|
|
214
|
+
A site that already embeds its corpus can keep its vectors and only hand the
|
|
215
|
+
engine over: `from_vectors(rows, vectors, lang, model)` builds the index from
|
|
216
|
+
passages (`t`, `c`, `s`, the window ends `o`, optional `u` and `m`) and one
|
|
217
|
+
normalised vector per window; `vectors_from_int8` reads int8 vectors back the
|
|
218
|
+
way a browser would. That is how "Interroger le livre" moved from an
|
|
219
|
+
all-in-the-browser search to `search.php` with the SAME vectors, stopwords and
|
|
220
|
+
thresholds, so that only the place of the computation changed.
|
|
221
|
+
|
|
222
|
+
## Python
|
|
223
|
+
|
|
224
|
+
```python
|
|
225
|
+
import web_search_helper as wsh
|
|
226
|
+
|
|
227
|
+
idx = wsh.build(wsh.read_corpus("corpus/"), wsh.SentenceTransformerEmbedder(), "fr")
|
|
228
|
+
wsh.save(idx, "build/fr.php")
|
|
229
|
+
print(wsh.search(idx, "the data wall", explain=True))
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
`engine.py` is the reference twin of `search.php`; `tests/test_php_parity.py`
|
|
233
|
+
runs both on the same questions and fails if they disagree.
|
|
234
|
+
|
|
235
|
+
## Limits
|
|
236
|
+
|
|
237
|
+
- The question's vector is computed by transformers.js in the page and the
|
|
238
|
+
index by sentence-transformers here: same weights, cosines equal to about
|
|
239
|
+
1e-2. Calibration reports its margins; a threshold with less margin than that
|
|
240
|
+
is fragile.
|
|
241
|
+
- One index per language; the page says which language it asks.
|
|
242
|
+
- The first question downloads the model in the browser (about 120 MB for the
|
|
243
|
+
default multilingual MiniLM), then the browser keeps it.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# web-search-helper
|
|
2
|
+
|
|
3
|
+
[🇫🇷](https://github.com/warith-harchaoui/web-search-helper/blob/main/LISEZMOI.md) | [🇬🇧](https://github.com/warith-harchaoui/web-search-helper/blob/main/README.md)
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+

|
|
8
|
+
[](https://github.com/warith-harchaoui/web-search-helper/actions/workflows/tests.yml)
|
|
9
|
+
|
|
10
|
+
**Search a private corpus from a website, with PHP as the only server.** You
|
|
11
|
+
index the corpus on your machine; one PHP file answers the questions; the page
|
|
12
|
+
embeds each question in the browser. The corpus is never published: a visitor
|
|
13
|
+
receives a few sentences of what answers the question, or an honest "I cannot
|
|
14
|
+
answer" and nothing else.
|
|
15
|
+
|
|
16
|
+
`web-search-helper` belongs to the [**AI Helpers**](https://github.com/warith-harchaoui/ai-helpers)
|
|
17
|
+
suite. It works in French and in English out of the box (stopwords, answer
|
|
18
|
+
composition, interface), one index per language, and in any other language
|
|
19
|
+
with a stopword file. By default no LLM is involved: the answer is made of the
|
|
20
|
+
corpus's own sentences. Optionally, a small language model runs in the
|
|
21
|
+
visitor's browser and writes the answer from the excerpts, citing them; which
|
|
22
|
+
model depends on the visitor's machine and line, as planned by
|
|
23
|
+
[`best-engine-ai-helper`](https://github.com/warith-harchaoui/best-engine-ai-helper).
|
|
24
|
+
|
|
25
|
+
BSD 3-Clause licence. Author: Warith Harchaoui.
|
|
26
|
+
|
|
27
|
+
## Why
|
|
28
|
+
|
|
29
|
+
A fully client-side search must ship the corpus to the browser, because the
|
|
30
|
+
browser can only search what it has. That is fine for public documentation and
|
|
31
|
+
wrong for a manuscript, a course or an internal knowledge base. Encrypting the
|
|
32
|
+
file does not help: the key has to be in the page too.
|
|
33
|
+
|
|
34
|
+
Most hosting offers PHP and nothing else: no Python, no Node, no vector
|
|
35
|
+
database. web-search-helper fits there.
|
|
36
|
+
|
|
37
|
+
| Where | What runs | Needs |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| your machine | cut, embed and index the corpus; measure the thresholds | Python ≥ 3.10, `sentence-transformers` |
|
|
40
|
+
| the server | `search.php`: BM25 + cosines + fusion + verdict, returns excerpts | PHP ≥ 7.4 with mbstring |
|
|
41
|
+
| the browser | `web-search.js`: embeds the question (transformers.js), composes an extractive answer, optionally writes one with an in-browser LLM | an ES-module browser; WebGPU for the written answer |
|
|
42
|
+
|
|
43
|
+
## How it searches
|
|
44
|
+
|
|
45
|
+
- **Words**: BM25, with a conjunctive pass first (when some passages contain
|
|
46
|
+
every word, only they compete) and a timid typo tolerance (`elefant` finds
|
|
47
|
+
`éléphant`, but only because `elefant` is absent from the corpus).
|
|
48
|
+
- **Meaning**: passages are cut into windows the model reads entirely; a
|
|
49
|
+
passage scores the cosine of its best window. Each section also has a
|
|
50
|
+
centroid.
|
|
51
|
+
- **Fusion**: reciprocal rank fusion of the two rankings.
|
|
52
|
+
- **Verdict**: the engine answers only when one of three routes says the corpus
|
|
53
|
+
treats the question: the best window *and* the best section are close in
|
|
54
|
+
meaning, or every word was found with a strong BM25, or (for a one-word
|
|
55
|
+
query) the word appears in enough passages. Otherwise it returns nothing.
|
|
56
|
+
Ranking always returns *something*; only an absolute measure can say "no".
|
|
57
|
+
|
|
58
|
+
The search is exhaustive. `ann-router` would pick HNSW from 1 000 vectors at a
|
|
59
|
+
10 ms budget, but that budget is numpy's, the corpus sizes here are small, and
|
|
60
|
+
the verdict judges the *absolute* best cosine, which an approximate index can
|
|
61
|
+
miss. `web-search-helper bench` measures the PHP time on your index and fails
|
|
62
|
+
when it goes over budget.
|
|
63
|
+
|
|
64
|
+
## Protections against scraping
|
|
65
|
+
|
|
66
|
+
1. Nothing is returned when the corpus does not treat the question.
|
|
67
|
+
2. A passage comes back as an excerpt: its best window and the better of its
|
|
68
|
+
neighbours, snapped to sentence boundaries. Never the whole passage.
|
|
69
|
+
3. A quota per address, per hour and per day (salted hash of the address; no
|
|
70
|
+
other trace kept).
|
|
71
|
+
4. The index file is a PHP script: requested by URL, it answers an empty 404.
|
|
72
|
+
The data folder also gets a `.htaccess` that denies everything, in case.
|
|
73
|
+
|
|
74
|
+
Someone patient with thousands of well-chosen questions can still collect a
|
|
75
|
+
good part of the corpus. These measures make that slow, not impossible.
|
|
76
|
+
|
|
77
|
+
## Quick start
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pip install 'web-search-helper[embed]'
|
|
81
|
+
|
|
82
|
+
# 1. Index: a .jsonl (one passage per line), a Markdown file or a folder of them.
|
|
83
|
+
web-search-helper build corpus/ -o build/fr.php --lang fr
|
|
84
|
+
|
|
85
|
+
# 2. Measure the thresholds on questions the corpus answers and questions it does not.
|
|
86
|
+
web-search-helper calibrate build/fr.php --inside inside.txt --outside outside.txt --write
|
|
87
|
+
|
|
88
|
+
# 3. Try a question, as the server will answer it.
|
|
89
|
+
web-search-helper search build/fr.php "Quelle est la parabole de l'éléphant ?" --explain
|
|
90
|
+
|
|
91
|
+
# 4. Copy the endpoint, the client and the index into the site; serve it locally.
|
|
92
|
+
web-search-helper install site/ build/fr.php build/en.php --demo
|
|
93
|
+
web-search-helper serve site/ # http://127.0.0.1:8765/demo.html?lang=fr (or ?lang=en)
|
|
94
|
+
|
|
95
|
+
# 5. Time the PHP engine; after upload, check that nothing leaks.
|
|
96
|
+
web-search-helper bench build/fr.php
|
|
97
|
+
web-search-helper check-online https://example.org/search/ --lang fr
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
JSONL records: `{"text": "...", "doc": "Book title", "section": "Chapter 2", "url": "page.html#x", "meta": {...}}`.
|
|
101
|
+
`meta` is passed through to the browser untouched.
|
|
102
|
+
|
|
103
|
+
## In the page
|
|
104
|
+
|
|
105
|
+
```html
|
|
106
|
+
<script type="module">
|
|
107
|
+
import { WebSearch, composeAnswer } from './web-search.js';
|
|
108
|
+
const ws = new WebSearch({ endpoint: 'search.php', lang: 'fr' });
|
|
109
|
+
const r = await ws.ask(question, { onLexical: (first) => { /* early answer */ } });
|
|
110
|
+
if (!r.sure) showUnknown();
|
|
111
|
+
else show(await composeAnswer(r, { lang: 'fr', embed: ws }), r.hits);
|
|
112
|
+
</script>
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`ask` makes two calls: words only first (instant, before the model has
|
|
116
|
+
loaded), then words and meaning. The answer composed by `composeAnswer` is made
|
|
117
|
+
of corpus sentences, verbatim, chosen by maximal marginal relevance; nothing is
|
|
118
|
+
generated. Types are in `web-search.d.ts`. `mount(form, output, options)` wires
|
|
119
|
+
a minimal widget.
|
|
120
|
+
|
|
121
|
+
## A written answer, in the visitor's browser
|
|
122
|
+
|
|
123
|
+
`mount(form, output, { …, writer: true })` adds a button under a `sure`
|
|
124
|
+
result: a small language model, run by transformers.js on WebGPU in the
|
|
125
|
+
visitor's browser, rewrites the excerpts into an answer that cites them as
|
|
126
|
+
[1], [2]… Nothing leaves the browser and the server does not change.
|
|
127
|
+
|
|
128
|
+
Which model is not decided in this package. `writer-plan.json`, written by
|
|
129
|
+
`best-engine-ai-helper browser plan --usage rag-answer`, is an ordered ladder
|
|
130
|
+
ranked by scores measured on this very task: Gemma 4 E4B (0.722) then Gemma 4
|
|
131
|
+
E2B (0.667), each in q4f16 and q4, with exact files and sizes (3.1 to 5.6 GB).
|
|
132
|
+
LFM2 1.2B was measured too (0.167: most of its answers cite nothing) and left
|
|
133
|
+
out, so a machine that cannot fetch a Gemma keeps the extractive answer rather
|
|
134
|
+
than one it would set aside. `scripts/eval_writer.py` reruns the measurement on
|
|
135
|
+
your own corpus; `scripts/make_writer_plan.py` rewrites the plan from
|
|
136
|
+
best-engine. The page probes the machine (WebGPU,
|
|
137
|
+
shader-f16, device memory, storage, mobile, data saver), checks the browser
|
|
138
|
+
cache, measures the bandwidth on a Range request to the model's own files, and
|
|
139
|
+
takes the first rung it can fetch within `max_download_s` (180 s). Nothing fits:
|
|
140
|
+
no button, and the extractive answer stands. `chooseRung` is tested against
|
|
141
|
+
best-engine's Python `browser.choose` on 2 251 simulated devices.
|
|
142
|
+
|
|
143
|
+
The written answer is checked before it is shown: the model only sees a `sure`
|
|
144
|
+
result, it is told to use the excerpts only and to say when they fall short,
|
|
145
|
+
and every sentence it writes must find most of its content words in the
|
|
146
|
+
excerpts it cites (`checkGrounding`). Otherwise the page sets the written
|
|
147
|
+
answer aside and keeps the extractive one. The excerpts stay listed below it,
|
|
148
|
+
verbatim.
|
|
149
|
+
|
|
150
|
+
```js
|
|
151
|
+
import { Writer } from './web-search.js';
|
|
152
|
+
const { writer, reason } = await Writer.create('writer-plan.json'); // null when nothing fits
|
|
153
|
+
const answer = writer && await writer.write(result, { question, lang: 'fr', onText: show });
|
|
154
|
+
if (answer && answer.grounded) show(answer.text);
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## The suite's other doors
|
|
158
|
+
|
|
159
|
+
Like every AI Helper, the engine is also reachable over HTTP and MCP, for
|
|
160
|
+
agents, scripts and local testing without PHP (`search.php` stays the
|
|
161
|
+
production server):
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
pip install 'web-search-helper[api,mcp,embed]'
|
|
165
|
+
WSH_DATA_DIR=site/search-data web-search-helper-api # http://127.0.0.1:8040/docs, /gui
|
|
166
|
+
WSH_DATA_DIR=site/search-data web-search-helper-mcp # MCP at http://127.0.0.1:8041/mcp
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Routes: `POST /query` speaks `search.php`'s protocol, so `web-search.js` can
|
|
170
|
+
point at it (that is what `/gui` does); `POST /search` takes a plain question
|
|
171
|
+
and embeds it server-side; `GET /indexes`; `POST /calibrate` (writes
|
|
172
|
+
nothing); `POST /check-online`. The MCP tools are `indexes`, `search`,
|
|
173
|
+
`calibrate` and `check_online`. The agent skill lives in
|
|
174
|
+
`skills/web-search-helper/`; `TRIGGERS.md` lists what should reach for this
|
|
175
|
+
helper and what should not, and a test keeps it identical to the skill's copy.
|
|
176
|
+
|
|
177
|
+
## Bring your own embeddings
|
|
178
|
+
|
|
179
|
+
A site that already embeds its corpus can keep its vectors and only hand the
|
|
180
|
+
engine over: `from_vectors(rows, vectors, lang, model)` builds the index from
|
|
181
|
+
passages (`t`, `c`, `s`, the window ends `o`, optional `u` and `m`) and one
|
|
182
|
+
normalised vector per window; `vectors_from_int8` reads int8 vectors back the
|
|
183
|
+
way a browser would. That is how "Interroger le livre" moved from an
|
|
184
|
+
all-in-the-browser search to `search.php` with the SAME vectors, stopwords and
|
|
185
|
+
thresholds, so that only the place of the computation changed.
|
|
186
|
+
|
|
187
|
+
## Python
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
import web_search_helper as wsh
|
|
191
|
+
|
|
192
|
+
idx = wsh.build(wsh.read_corpus("corpus/"), wsh.SentenceTransformerEmbedder(), "fr")
|
|
193
|
+
wsh.save(idx, "build/fr.php")
|
|
194
|
+
print(wsh.search(idx, "the data wall", explain=True))
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
`engine.py` is the reference twin of `search.php`; `tests/test_php_parity.py`
|
|
198
|
+
runs both on the same questions and fails if they disagree.
|
|
199
|
+
|
|
200
|
+
## Limits
|
|
201
|
+
|
|
202
|
+
- The question's vector is computed by transformers.js in the page and the
|
|
203
|
+
index by sentence-transformers here: same weights, cosines equal to about
|
|
204
|
+
1e-2. Calibration reports its margins; a threshold with less margin than that
|
|
205
|
+
is fragile.
|
|
206
|
+
- One index per language; the page says which language it asks.
|
|
207
|
+
- The first question downloads the model in the browser (about 120 MB for the
|
|
208
|
+
default multilingual MiniLM), then the browser keeps it.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "web-search-helper"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Search a private corpus from a website with PHP only: index it on your machine, serve it with one PHP file, embed the question in the browser. The corpus is never published; visitors get short excerpts, or an honest 'I cannot answer'."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "BSD-3-Clause" }
|
|
12
|
+
authors = [{ name = "Warith Harchaoui", email = "warithmetics@deraison.ai" }]
|
|
13
|
+
keywords = ["search", "rag", "semantic-search", "bm25", "php", "transformers.js", "static-site"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: BSD License",
|
|
17
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
]
|
|
20
|
+
# Core: building with the hash embedder (dry runs, tests) needs only numpy and
|
|
21
|
+
# click. The real model is the [embed] extra, because sentence-transformers
|
|
22
|
+
# pulls torch.
|
|
23
|
+
dependencies = [
|
|
24
|
+
"click>=8.1,<9",
|
|
25
|
+
"numpy>=1.24",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
embed = ["sentence-transformers>=3"]
|
|
30
|
+
# ann-router is only consulted, to print its advice next to the measured
|
|
31
|
+
# PHP timing.
|
|
32
|
+
route = ["ann-router>=0.3"]
|
|
33
|
+
api = ["fastapi>=0.111,<1", "uvicorn>=0.30,<1"]
|
|
34
|
+
# fastapi-mcp 0.4.x breaks on mcp 2 (positional Server() signature), as
|
|
35
|
+
# elsewhere in the suite.
|
|
36
|
+
mcp = ["fastapi-mcp>=0.4,<1", "mcp<2,>=1.20", "fastapi>=0.111,<1", "uvicorn>=0.30,<1"]
|
|
37
|
+
dev = ["pytest>=7", "ruff>=0.6", "httpx>=0.27"]
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
web-search-helper = "web_search_helper.cli:main"
|
|
41
|
+
web-search-helper-api = "web_search_helper.api:main"
|
|
42
|
+
web-search-helper-mcp = "web_search_helper.mcp_server:main"
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://github.com/warith-harchaoui/web-search-helper"
|
|
46
|
+
|
|
47
|
+
[tool.setuptools.packages.find]
|
|
48
|
+
where = ["src"]
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.package-data]
|
|
51
|
+
web_search_helper = ["assets/*"]
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
testpaths = ["tests", "src"]
|
|
55
|
+
addopts = "-q --doctest-modules"
|
|
56
|
+
|
|
57
|
+
[tool.ruff]
|
|
58
|
+
line-length = 110
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""web-search-helper — search a private corpus from a website, PHP only.
|
|
2
|
+
|
|
3
|
+
Build the index on your machine; serve it with one PHP file; embed the
|
|
4
|
+
question in the browser. The corpus is never published: the page receives
|
|
5
|
+
short excerpts of what answers the question, or nothing when the corpus does
|
|
6
|
+
not treat it.
|
|
7
|
+
|
|
8
|
+
>>> import web_search_helper as wsh
|
|
9
|
+
>>> callable(wsh.build) and callable(wsh.search)
|
|
10
|
+
True
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
__version__ = "0.2.0"
|
|
16
|
+
|
|
17
|
+
from .calibrate import calibrate
|
|
18
|
+
from .corpus import Passage, read_corpus, read_jsonl
|
|
19
|
+
from .embed import DEFAULT_MODEL, MODELS, HashEmbedder, SentenceTransformerEmbedder
|
|
20
|
+
from .engine import search
|
|
21
|
+
from .index import Index, build, from_vectors, load, save, vectors_from_int8
|
|
22
|
+
from .site import bench, check_online, install, serve
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"DEFAULT_MODEL",
|
|
26
|
+
"MODELS",
|
|
27
|
+
"HashEmbedder",
|
|
28
|
+
"Index",
|
|
29
|
+
"Passage",
|
|
30
|
+
"SentenceTransformerEmbedder",
|
|
31
|
+
"bench",
|
|
32
|
+
"build",
|
|
33
|
+
"calibrate",
|
|
34
|
+
"check_online",
|
|
35
|
+
"from_vectors",
|
|
36
|
+
"install",
|
|
37
|
+
"load",
|
|
38
|
+
"read_corpus",
|
|
39
|
+
"read_jsonl",
|
|
40
|
+
"save",
|
|
41
|
+
"search",
|
|
42
|
+
"serve",
|
|
43
|
+
"vectors_from_int8",
|
|
44
|
+
]
|