scrydb 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrydb-0.2.0/LICENSE +21 -0
- scrydb-0.2.0/MANIFEST.in +2 -0
- scrydb-0.2.0/PKG-INFO +260 -0
- scrydb-0.2.0/README.md +227 -0
- scrydb-0.2.0/pyproject.toml +49 -0
- scrydb-0.2.0/setup.cfg +4 -0
- scrydb-0.2.0/src/scrydb/__init__.py +12 -0
- scrydb-0.2.0/src/scrydb/cli.py +307 -0
- scrydb-0.2.0/src/scrydb/core.py +1317 -0
- scrydb-0.2.0/src/scrydb.egg-info/PKG-INFO +260 -0
- scrydb-0.2.0/src/scrydb.egg-info/SOURCES.txt +15 -0
- scrydb-0.2.0/src/scrydb.egg-info/dependency_links.txt +1 -0
- scrydb-0.2.0/src/scrydb.egg-info/entry_points.txt +2 -0
- scrydb-0.2.0/src/scrydb.egg-info/requires.txt +16 -0
- scrydb-0.2.0/src/scrydb.egg-info/top_level.txt +1 -0
- scrydb-0.2.0/tests/test_search.py +129 -0
- scrydb-0.2.0/tests/test_vec_extension.py +45 -0
scrydb-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Timo Breuer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
scrydb-0.2.0/MANIFEST.in
ADDED
scrydb-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scrydb
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Lexical, semantic, and hybrid search built on SQLite.
|
|
5
|
+
Author-email: Timo Breuer <timobreuer@acm.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/breuert/scrydb
|
|
8
|
+
Project-URL: Repository, https://github.com/breuert/scrydb
|
|
9
|
+
Project-URL: Issues, https://github.com/breuert/scrydb/issues
|
|
10
|
+
Keywords: sqlite,fts5,sqlite-vec,information-retrieval,search,embeddings,vector-search
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
16
|
+
Classifier: Topic :: Database
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: numpy>=1.22
|
|
21
|
+
Requires-Dist: tqdm>=4.60
|
|
22
|
+
Requires-Dist: sqlite-vec>=0.1.0
|
|
23
|
+
Provides-Extra: dense
|
|
24
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "dense"
|
|
25
|
+
Provides-Extra: eval
|
|
26
|
+
Requires-Dist: pandas>=1.5; extra == "eval"
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
29
|
+
Provides-Extra: all
|
|
30
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "all"
|
|
31
|
+
Requires-Dist: pandas>=1.5; extra == "all"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
<p align="center">
|
|
35
|
+
<img src="./docs/images/logo-light.png" width=250/>
|
|
36
|
+
<h1 align="center">scrydb</h1>
|
|
37
|
+
</p>
|
|
38
|
+
|
|
39
|
+
``scrydb``'s purpose is making lexical, dense, and hybrid search possible with [SQLite](https://sqlite.org/). Hardware requirements are kept low and everything is self-contained in a single file, i.e., the raw documents, their embeddings, and the lexical index are stored in a single SQLite file.
|
|
40
|
+
|
|
41
|
+
- Lexical search is made possible by the [FTS5 extension for SQLite](https://sqlite.org/fts5.html).
|
|
42
|
+
- Semantic search over the entire index is made possible by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a small, dependency-free vector search extension for SQLite. Every embedding can be searched at three precisions: **binary** (1 bit/dim, Hamming distance), **int8** (1 byte/dim, cosine), and **float** (full precision, cosine) — trading index size and speed against ranking quality, and combinable as a two-stage rerank (e.g. binary-first >> float-precision rerank).
|
|
43
|
+
- Hybrid search relies on [Reciprocal Rank Fusion](https://dl.acm.org/doi/10.1145/1571941.1572114) to fuse lexical and semantic search results.
|
|
44
|
+
|
|
45
|
+
The library is compatible with [Sentence Transformers](https://www.sbert.net/index.html). However, it is also possible to store precomputed embeddings for both queries and documents.
|
|
46
|
+
|
|
47
|
+
> [!NOTE]
|
|
48
|
+
> The evaluation protocol, benchmark results, and examples are available at [`scrydb-eval`](https://github.com/breuert/scrydb-eval/), the corresponding data is shared on [Hugging Face](https://huggingface.co/datasets/breuert/scrydb-eval).
|
|
49
|
+
|
|
50
|
+
## Usage examples
|
|
51
|
+
|
|
52
|
+
``scrydb`` can be used interactively as follows:
|
|
53
|
+
```python
|
|
54
|
+
from scrydb import Index, SentenceEmbedding
|
|
55
|
+
|
|
56
|
+
with Index.open("idx.db") as index:
|
|
57
|
+
index.add_model(SentenceEmbedding())
|
|
58
|
+
index.index_documents("corpus.jsonl", id_field="docid", text_field="text")
|
|
59
|
+
results = index.search("some query", mode="hybrid", rerank=True)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Batch search for Information Retrieval benchmarks with precomputed embeddings can be run as follows:
|
|
63
|
+
```python
|
|
64
|
+
import scrydb
|
|
65
|
+
|
|
66
|
+
idx = scrydb.Index.open("./path/to/index.db")
|
|
67
|
+
|
|
68
|
+
idx.index_documents(
|
|
69
|
+
source="./path/to/corpus.jsonl",
|
|
70
|
+
id_field="docid",
|
|
71
|
+
text_field="text",
|
|
72
|
+
embedding_field="embedding",
|
|
73
|
+
store_int8_embeddings=True, # opt in to int8 storage alongside binary/float
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
idx.index_queries(
|
|
77
|
+
source="./path/to/queries.jsonl",
|
|
78
|
+
id_field="qid",
|
|
79
|
+
text_field="text",
|
|
80
|
+
embedding_field="embedding",
|
|
81
|
+
store_int8_embeddings=True,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
idx.batch_search(mode="lexical").write_trec("./path/to/lexical/run")
|
|
85
|
+
idx.batch_search(mode="semantic", precision="binary").write_trec("./path/to/binary/run")
|
|
86
|
+
idx.batch_search(mode="semantic", precision="int8").write_trec("./path/to/int8/run")
|
|
87
|
+
idx.batch_search(mode="semantic", precision="float").write_trec("./path/to/float/run")
|
|
88
|
+
idx.batch_search(mode="semantic", precision="binary", rerank="float").write_trec("./path/to/binary-rerank-float/run")
|
|
89
|
+
idx.batch_search(mode="hybrid").write_trec("./path/to/hybrid/run")
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Search modes, precision, and rerank
|
|
93
|
+
|
|
94
|
+
`search()`/`batch_search()` take three orthogonal knobs:
|
|
95
|
+
|
|
96
|
+
- `mode` — `"lexical"` (BM25 over FTS5), `"semantic"` (vector search), or `"hybrid"` (Reciprocal Rank Fusion of both).
|
|
97
|
+
- `precision` — which vector representation `mode="semantic"`/the semantic side of `mode="hybrid"` ranks with: `"binary"` (default), `"int8"`, or `"float"`.
|
|
98
|
+
- `rerank` — `False` (default), or a second-stage rerank over the top candidates from `mode`, at `"binary"`, `"int8"`, or `"float"` precision (`True` is a synonym for `"float"`).
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
idx.search("some query", mode="lexical") # BM25
|
|
102
|
+
idx.search("some query", mode="lexical", rerank="float") # BM25 >> Float/Cosine
|
|
103
|
+
idx.search("some query", mode="semantic", precision="binary") # Binary/Hamming
|
|
104
|
+
idx.search("some query", mode="semantic", precision="int8") # Int8/Cosine
|
|
105
|
+
idx.search("some query", mode="semantic", precision="binary", rerank="float") # Binary >> Float/Cosine
|
|
106
|
+
idx.search("some query", mode="hybrid", rerank=True) # Hybrid/RRF
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Installing
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pip install scrydb
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Or with [uv](https://docs.astral.sh/uv/) (faster, and manages the virtualenv for you):
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
uv venv && uv pip install scrydb
|
|
119
|
+
# or, inside a uv-managed project:
|
|
120
|
+
uv add scrydb
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
No C compiler or SQLite development headers required: vector search is
|
|
124
|
+
powered by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a pure
|
|
125
|
+
pip dependency that ships prebuilt binaries, so `pip install scrydb` is a
|
|
126
|
+
plain, fast, wheel-only install.
|
|
127
|
+
|
|
128
|
+
`Index.open()` loads the `sqlite-vec` extension automatically. This
|
|
129
|
+
requires a Python build whose `sqlite3` module supports
|
|
130
|
+
`enable_load_extension()` — true for Homebrew, pyenv, and
|
|
131
|
+
[uv](https://docs.astral.sh/uv/)-managed builds on macOS, virtually all
|
|
132
|
+
Linux distro packages, and the official Windows builds, but **not** for
|
|
133
|
+
macOS Pythons that link against Apple's SQLite, which disables extension
|
|
134
|
+
loading. That includes macOS's system Python and the CPython that
|
|
135
|
+
`actions/setup-python` installs on GitHub Actions runners. If you hit a
|
|
136
|
+
`RuntimeError` mentioning `enable_load_extension`, switch to Python from
|
|
137
|
+
Homebrew (`brew install python`), uv (`uv python install <version>`), or
|
|
138
|
+
pyenv (`PYTHON_CONFIGURE_OPTS='--enable-loadable-sqlite-extensions'
|
|
139
|
+
pyenv install <version>`).
|
|
140
|
+
|
|
141
|
+
If you'd rather not load the extension at all, you can still install and
|
|
142
|
+
use scrydb for lexical (BM25) search only — just disable it explicitly:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
Index.open("idx.db", vec_ext_path=None)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### Try the CLI without installing (uvx)
|
|
149
|
+
|
|
150
|
+
[`uvx`](https://docs.astral.sh/uv/guides/tools/) runs the `scrydb` command-line
|
|
151
|
+
tool (`index`/`search`/`batch-search`/`auto` — see the [Docker](#docker)
|
|
152
|
+
section below for the full reference) in a throwaway environment, no venv or
|
|
153
|
+
persistent install needed:
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
uvx scrydb index --documents corpus.jsonl --queries queries.jsonl --db idx.db
|
|
157
|
+
uvx scrydb search "some query" --db idx.db --mode hybrid --rerank float
|
|
158
|
+
uvx scrydb batch-search --db idx.db --mode hybrid --output run.trec
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Since scrydb has no compiled artifacts of its own, `uvx` just downloads the
|
|
162
|
+
wheel and its dependencies (including `sqlite-vec`'s prebuilt binary) into
|
|
163
|
+
its cache — no build step at all.
|
|
164
|
+
|
|
165
|
+
## Docker
|
|
166
|
+
|
|
167
|
+
No local Python install needed: the [`Dockerfile`](./Dockerfile) builds a
|
|
168
|
+
Linux image with scrydb already installed, driven by a bundled `scrydb`
|
|
169
|
+
CLI. Everything it reads and writes -- input JSONL, the SQLite index, TREC
|
|
170
|
+
run files -- lives under `/data`, so bind-mount a host directory there.
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
docker build -t scrydb .
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Drop `documents.jsonl` (and, optionally, `queries.jsonl`) into `./data` and
|
|
177
|
+
run the image with no arguments: it indexes whatever's present into
|
|
178
|
+
`./data/index.db`, then either batch-searches the stored queries into
|
|
179
|
+
`./data/run.trec` or -- if there's nothing to index and no stored queries --
|
|
180
|
+
tells you what it's waiting for.
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
docker run --rm -v "$PWD/data":/data scrydb
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
If `./data/index.db` already exists (say, you built it locally, or a
|
|
187
|
+
previous run produced it), the same command re-uses it: skips indexing
|
|
188
|
+
whatever source files aren't present and searches straight away. To query
|
|
189
|
+
an existing index ad hoc instead of running a full batch, override the
|
|
190
|
+
default command:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
docker run --rm -v "$PWD/data":/data scrydb search "some query" --mode hybrid --rerank float
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Every option is also settable as an `SCRYDB_*` environment variable (handy
|
|
197
|
+
for `docker run -e`), and the JSONL id-field names for documents/queries
|
|
198
|
+
default to `docid`/`qid`-style overrides when they differ from `id`:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
docker run --rm -v "$PWD/data":/data \
|
|
202
|
+
-e SCRYDB_DOC_ID_FIELD=docid -e SCRYDB_QUERY_ID_FIELD=qid \
|
|
203
|
+
-e SCRYDB_MODEL=mixedbread-ai/mxbai-embed-large-v1 \
|
|
204
|
+
-e SCRYDB_MODE=hybrid -e SCRYDB_RERANK=float \
|
|
205
|
+
scrydb
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Run `docker run --rm scrydb --help` (or `... <subcommand> --help`) for the
|
|
209
|
+
full `index`/`search`/`batch-search`/`auto` reference, or see the
|
|
210
|
+
module docstring in [`src/scrydb/cli.py`](./src/scrydb/cli.py).
|
|
211
|
+
|
|
212
|
+
Notes:
|
|
213
|
+
|
|
214
|
+
- **Dense/hybrid search** (`sentence-transformers`) isn't in the image by
|
|
215
|
+
default -- build with `--build-arg EXTRAS=all` (or `dense`) to add it.
|
|
216
|
+
This installs the CPU-only `torch` build so the image doesn't pull in
|
|
217
|
+
multi-gigabyte CUDA packages it can't use.
|
|
218
|
+
- **Multi-platform**: build once for both Intel/AMD and Apple
|
|
219
|
+
Silicon/ARM hosts (each running natively, no QEMU emulation) with
|
|
220
|
+
`docker buildx build --platform linux/amd64,linux/arm64 -t scrydb .`
|
|
221
|
+
- **Plain Python access**: `docker run --rm -it -v "$PWD/data":/data --entrypoint python3 scrydb`
|
|
222
|
+
drops into an interpreter with `scrydb` importable.
|
|
223
|
+
|
|
224
|
+
## How extension discovery works at runtime
|
|
225
|
+
|
|
226
|
+
`Index.open()`/`Index()` default to `vec_ext_path="auto"`, which loads
|
|
227
|
+
`sqlite_vec.loadable_path()` — the copy of the extension bundled inside
|
|
228
|
+
the installed `sqlite-vec` pip package, prebuilt for the current
|
|
229
|
+
platform. Pass an explicit path to load a different build (e.g. a newer
|
|
230
|
+
`vec0` release), or `None` to skip loading it entirely.
|
|
231
|
+
|
|
232
|
+
## Development / editable installs
|
|
233
|
+
|
|
234
|
+
```bash
|
|
235
|
+
git clone <repo>
|
|
236
|
+
cd scrydb
|
|
237
|
+
python -m venv .venv && source .venv/bin/activate
|
|
238
|
+
pip install -e ".[all]"
|
|
239
|
+
pytest
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Or with uv:
|
|
243
|
+
|
|
244
|
+
```bash
|
|
245
|
+
git clone <repo>
|
|
246
|
+
cd scrydb
|
|
247
|
+
uv venv
|
|
248
|
+
uv pip install -e ".[all]"
|
|
249
|
+
uv run pytest
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
`uv run` picks up `.venv` automatically, so there's no `source .venv/bin/activate`
|
|
253
|
+
step — any command after it (`uv run pytest`, `uv run python -m scrydb.cli --help`,
|
|
254
|
+
`uv run python your_script.py`) runs inside the project's venv.
|
|
255
|
+
|
|
256
|
+
## Optional extras
|
|
257
|
+
|
|
258
|
+
- `pip install "scrydb[dense]"` / `uv pip install "scrydb[dense]"` — dense/hybrid search via `sentence-transformers`
|
|
259
|
+
- `pip install "scrydb[eval]"` / `uv pip install "scrydb[eval]"` — `Run.to_dataframe()` via `pandas`
|
|
260
|
+
- `pip install "scrydb[all]"` / `uv pip install "scrydb[all]"` — both
|
scrydb-0.2.0/README.md
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="./docs/images/logo-light.png" width=250/>
|
|
3
|
+
<h1 align="center">scrydb</h1>
|
|
4
|
+
</p>
|
|
5
|
+
|
|
6
|
+
``scrydb``'s purpose is making lexical, dense, and hybrid search possible with [SQLite](https://sqlite.org/). Hardware requirements are kept low and everything is self-contained in a single file, i.e., the raw documents, their embeddings, and the lexical index are stored in a single SQLite file.
|
|
7
|
+
|
|
8
|
+
- Lexical search is made possible by the [FTS5 extension for SQLite](https://sqlite.org/fts5.html).
|
|
9
|
+
- Semantic search over the entire index is made possible by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a small, dependency-free vector search extension for SQLite. Every embedding can be searched at three precisions: **binary** (1 bit/dim, Hamming distance), **int8** (1 byte/dim, cosine), and **float** (full precision, cosine) — trading index size and speed against ranking quality, and combinable as a two-stage rerank (e.g. binary-first >> float-precision rerank).
|
|
10
|
+
- Hybrid search relies on [Reciprocal Rank Fusion](https://dl.acm.org/doi/10.1145/1571941.1572114) to fuse lexical and semantic search results.
|
|
11
|
+
|
|
12
|
+
The library is compatible with [Sentence Transformers](https://www.sbert.net/index.html). However, it is also possible to store precomputed embeddings for both queries and documents.
|
|
13
|
+
|
|
14
|
+
> [!NOTE]
|
|
15
|
+
> The evaluation protocol, benchmark results, and examples are available at [`scrydb-eval`](https://github.com/breuert/scrydb-eval/), the corresponding data is shared on [Hugging Face](https://huggingface.co/datasets/breuert/scrydb-eval).
|
|
16
|
+
|
|
17
|
+
## Usage examples
|
|
18
|
+
|
|
19
|
+
``scrydb`` can be used interactively as follows:
|
|
20
|
+
```python
|
|
21
|
+
from scrydb import Index, SentenceEmbedding
|
|
22
|
+
|
|
23
|
+
with Index.open("idx.db") as index:
|
|
24
|
+
index.add_model(SentenceEmbedding())
|
|
25
|
+
index.index_documents("corpus.jsonl", id_field="docid", text_field="text")
|
|
26
|
+
results = index.search("some query", mode="hybrid", rerank=True)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Batch search for Information Retrieval benchmarks with precomputed embeddings can be run as follows:
|
|
30
|
+
```python
|
|
31
|
+
import scrydb
|
|
32
|
+
|
|
33
|
+
idx = scrydb.Index.open("./path/to/index.db")
|
|
34
|
+
|
|
35
|
+
idx.index_documents(
|
|
36
|
+
source="./path/to/corpus.jsonl",
|
|
37
|
+
id_field="docid",
|
|
38
|
+
text_field="text",
|
|
39
|
+
embedding_field="embedding",
|
|
40
|
+
store_int8_embeddings=True, # opt in to int8 storage alongside binary/float
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
idx.index_queries(
|
|
44
|
+
source="./path/to/queries.jsonl",
|
|
45
|
+
id_field="qid",
|
|
46
|
+
text_field="text",
|
|
47
|
+
embedding_field="embedding",
|
|
48
|
+
store_int8_embeddings=True,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
idx.batch_search(mode="lexical").write_trec("./path/to/lexical/run")
|
|
52
|
+
idx.batch_search(mode="semantic", precision="binary").write_trec("./path/to/binary/run")
|
|
53
|
+
idx.batch_search(mode="semantic", precision="int8").write_trec("./path/to/int8/run")
|
|
54
|
+
idx.batch_search(mode="semantic", precision="float").write_trec("./path/to/float/run")
|
|
55
|
+
idx.batch_search(mode="semantic", precision="binary", rerank="float").write_trec("./path/to/binary-rerank-float/run")
|
|
56
|
+
idx.batch_search(mode="hybrid").write_trec("./path/to/hybrid/run")
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Search modes, precision, and rerank
|
|
60
|
+
|
|
61
|
+
`search()`/`batch_search()` take three orthogonal knobs:
|
|
62
|
+
|
|
63
|
+
- `mode` — `"lexical"` (BM25 over FTS5), `"semantic"` (vector search), or `"hybrid"` (Reciprocal Rank Fusion of both).
|
|
64
|
+
- `precision` — which vector representation `mode="semantic"`/the semantic side of `mode="hybrid"` ranks with: `"binary"` (default), `"int8"`, or `"float"`.
|
|
65
|
+
- `rerank` — `False` (default), or a second-stage rerank over the top candidates from `mode`, at `"binary"`, `"int8"`, or `"float"` precision (`True` is a synonym for `"float"`).
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
idx.search("some query", mode="lexical") # BM25
|
|
69
|
+
idx.search("some query", mode="lexical", rerank="float") # BM25 >> Float/Cosine
|
|
70
|
+
idx.search("some query", mode="semantic", precision="binary") # Binary/Hamming
|
|
71
|
+
idx.search("some query", mode="semantic", precision="int8") # Int8/Cosine
|
|
72
|
+
idx.search("some query", mode="semantic", precision="binary", rerank="float") # Binary >> Float/Cosine
|
|
73
|
+
idx.search("some query", mode="hybrid", rerank=True) # Hybrid/RRF
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Installing
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install scrydb
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Or with [uv](https://docs.astral.sh/uv/) (faster, and manages the virtualenv for you):
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
uv venv && uv pip install scrydb
|
|
86
|
+
# or, inside a uv-managed project:
|
|
87
|
+
uv add scrydb
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
No C compiler or SQLite development headers required: vector search is
|
|
91
|
+
powered by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a pure
|
|
92
|
+
pip dependency that ships prebuilt binaries, so `pip install scrydb` is a
|
|
93
|
+
plain, fast, wheel-only install.
|
|
94
|
+
|
|
95
|
+
`Index.open()` loads the `sqlite-vec` extension automatically. This
|
|
96
|
+
requires a Python build whose `sqlite3` module supports
|
|
97
|
+
`enable_load_extension()` — true for Homebrew, pyenv, and
|
|
98
|
+
[uv](https://docs.astral.sh/uv/)-managed builds on macOS, virtually all
|
|
99
|
+
Linux distro packages, and the official Windows builds, but **not** for
|
|
100
|
+
macOS Pythons that link against Apple's SQLite, which disables extension
|
|
101
|
+
loading. That includes macOS's system Python and the CPython that
|
|
102
|
+
`actions/setup-python` installs on GitHub Actions runners. If you hit a
|
|
103
|
+
`RuntimeError` mentioning `enable_load_extension`, switch to Python from
|
|
104
|
+
Homebrew (`brew install python`), uv (`uv python install <version>`), or
|
|
105
|
+
pyenv (`PYTHON_CONFIGURE_OPTS='--enable-loadable-sqlite-extensions'
|
|
106
|
+
pyenv install <version>`).
|
|
107
|
+
|
|
108
|
+
If you'd rather not load the extension at all, you can still install and
|
|
109
|
+
use scrydb for lexical (BM25) search only — just disable it explicitly:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
Index.open("idx.db", vec_ext_path=None)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Try the CLI without installing (uvx)
|
|
116
|
+
|
|
117
|
+
[`uvx`](https://docs.astral.sh/uv/guides/tools/) runs the `scrydb` command-line
|
|
118
|
+
tool (`index`/`search`/`batch-search`/`auto` — see the [Docker](#docker)
|
|
119
|
+
section below for the full reference) in a throwaway environment, no venv or
|
|
120
|
+
persistent install needed:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
uvx scrydb index --documents corpus.jsonl --queries queries.jsonl --db idx.db
|
|
124
|
+
uvx scrydb search "some query" --db idx.db --mode hybrid --rerank float
|
|
125
|
+
uvx scrydb batch-search --db idx.db --mode hybrid --output run.trec
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Since scrydb has no compiled artifacts of its own, `uvx` just downloads the
|
|
129
|
+
wheel and its dependencies (including `sqlite-vec`'s prebuilt binary) into
|
|
130
|
+
its cache — no build step at all.
|
|
131
|
+
|
|
132
|
+
## Docker
|
|
133
|
+
|
|
134
|
+
No local Python install needed: the [`Dockerfile`](./Dockerfile) builds a
|
|
135
|
+
Linux image with scrydb already installed, driven by a bundled `scrydb`
|
|
136
|
+
CLI. Everything it reads and writes -- input JSONL, the SQLite index, TREC
|
|
137
|
+
run files -- lives under `/data`, so bind-mount a host directory there.
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
docker build -t scrydb .
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Drop `documents.jsonl` (and, optionally, `queries.jsonl`) into `./data` and
|
|
144
|
+
run the image with no arguments: it indexes whatever's present into
|
|
145
|
+
`./data/index.db`, then either batch-searches the stored queries into
|
|
146
|
+
`./data/run.trec` or -- if there's nothing to index and no stored queries --
|
|
147
|
+
tells you what it's waiting for.
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
docker run --rm -v "$PWD/data":/data scrydb
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
If `./data/index.db` already exists (say, you built it locally, or a
|
|
154
|
+
previous run produced it), the same command re-uses it: skips indexing
|
|
155
|
+
whatever source files aren't present and searches straight away. To query
|
|
156
|
+
an existing index ad hoc instead of running a full batch, override the
|
|
157
|
+
default command:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
docker run --rm -v "$PWD/data":/data scrydb search "some query" --mode hybrid --rerank float
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Every option is also settable as an `SCRYDB_*` environment variable (handy
|
|
164
|
+
for `docker run -e`), and the JSONL id-field names for documents/queries
|
|
165
|
+
default to `docid`/`qid`-style overrides when they differ from `id`:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
docker run --rm -v "$PWD/data":/data \
|
|
169
|
+
-e SCRYDB_DOC_ID_FIELD=docid -e SCRYDB_QUERY_ID_FIELD=qid \
|
|
170
|
+
-e SCRYDB_MODEL=mixedbread-ai/mxbai-embed-large-v1 \
|
|
171
|
+
-e SCRYDB_MODE=hybrid -e SCRYDB_RERANK=float \
|
|
172
|
+
scrydb
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Run `docker run --rm scrydb --help` (or `... <subcommand> --help`) for the
|
|
176
|
+
full `index`/`search`/`batch-search`/`auto` reference, or see the
|
|
177
|
+
module docstring in [`src/scrydb/cli.py`](./src/scrydb/cli.py).
|
|
178
|
+
|
|
179
|
+
Notes:
|
|
180
|
+
|
|
181
|
+
- **Dense/hybrid search** (`sentence-transformers`) isn't in the image by
|
|
182
|
+
default -- build with `--build-arg EXTRAS=all` (or `dense`) to add it.
|
|
183
|
+
This installs the CPU-only `torch` build so the image doesn't pull in
|
|
184
|
+
multi-gigabyte CUDA packages it can't use.
|
|
185
|
+
- **Multi-platform**: build once for both Intel/AMD and Apple
|
|
186
|
+
Silicon/ARM hosts (each running natively, no QEMU emulation) with
|
|
187
|
+
`docker buildx build --platform linux/amd64,linux/arm64 -t scrydb .`
|
|
188
|
+
- **Plain Python access**: `docker run --rm -it -v "$PWD/data":/data --entrypoint python3 scrydb`
|
|
189
|
+
drops into an interpreter with `scrydb` importable.
|
|
190
|
+
|
|
191
|
+
## How extension discovery works at runtime
|
|
192
|
+
|
|
193
|
+
`Index.open()`/`Index()` default to `vec_ext_path="auto"`, which loads
|
|
194
|
+
`sqlite_vec.loadable_path()` — the copy of the extension bundled inside
|
|
195
|
+
the installed `sqlite-vec` pip package, prebuilt for the current
|
|
196
|
+
platform. Pass an explicit path to load a different build (e.g. a newer
|
|
197
|
+
`vec0` release), or `None` to skip loading it entirely.
|
|
198
|
+
|
|
199
|
+
## Development / editable installs
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
git clone <repo>
|
|
203
|
+
cd scrydb
|
|
204
|
+
python -m venv .venv && source .venv/bin/activate
|
|
205
|
+
pip install -e ".[all]"
|
|
206
|
+
pytest
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Or with uv:
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
git clone <repo>
|
|
213
|
+
cd scrydb
|
|
214
|
+
uv venv
|
|
215
|
+
uv pip install -e ".[all]"
|
|
216
|
+
uv run pytest
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
`uv run` picks up `.venv` automatically, so there's no `source .venv/bin/activate`
|
|
220
|
+
step — any command after it (`uv run pytest`, `uv run python -m scrydb.cli --help`,
|
|
221
|
+
`uv run python your_script.py`) runs inside the project's venv.
|
|
222
|
+
|
|
223
|
+
## Optional extras
|
|
224
|
+
|
|
225
|
+
- `pip install "scrydb[dense]"` / `uv pip install "scrydb[dense]"` — dense/hybrid search via `sentence-transformers`
|
|
226
|
+
- `pip install "scrydb[eval]"` / `uv pip install "scrydb[eval]"` — `Run.to_dataframe()` via `pandas`
|
|
227
|
+
- `pip install "scrydb[all]"` / `uv pip install "scrydb[all]"` — both
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scrydb"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Lexical, semantic, and hybrid search built on SQLite."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Timo Breuer", email = "timobreuer@acm.org" },
|
|
15
|
+
]
|
|
16
|
+
keywords = ["sqlite", "fts5", "sqlite-vec", "information-retrieval", "search", "embeddings", "vector-search"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Operating System :: POSIX :: Linux",
|
|
20
|
+
"Operating System :: MacOS",
|
|
21
|
+
"Operating System :: Microsoft :: Windows",
|
|
22
|
+
"Topic :: Text Processing :: Indexing",
|
|
23
|
+
"Topic :: Database",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"numpy>=1.22",
|
|
27
|
+
"tqdm>=4.60",
|
|
28
|
+
"sqlite-vec>=0.1.0",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dense = ["sentence-transformers>=3.0"]
|
|
33
|
+
eval = ["pandas>=1.5"]
|
|
34
|
+
test = ["pytest>=7"]
|
|
35
|
+
all = ["sentence-transformers>=3.0", "pandas>=1.5"]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/breuert/scrydb"
|
|
39
|
+
Repository = "https://github.com/breuert/scrydb"
|
|
40
|
+
Issues = "https://github.com/breuert/scrydb/issues"
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
scrydb = "scrydb.cli:main"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.packages.find]
|
|
46
|
+
where = ["src"]
|
|
47
|
+
|
|
48
|
+
[tool.pytest.ini_options]
|
|
49
|
+
testpaths = ["tests"]
|
scrydb-0.2.0/setup.cfg
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""scrydb — lexical, semantic, and hybrid search built on SQLite."""
|
|
2
|
+
|
|
3
|
+
from .core import Index, Run, SearchResult, SentenceEmbedding
|
|
4
|
+
|
|
5
|
+
__all__ = ["Index", "Run", "SearchResult", "SentenceEmbedding"]
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from importlib.metadata import version as _version
|
|
9
|
+
|
|
10
|
+
__version__ = _version("scrydb")
|
|
11
|
+
except Exception: # pragma: no cover - package not installed
|
|
12
|
+
__version__ = "0.0.0"
|