scrydb 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scrydb-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Timo Breuer
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ include README.md
2
+ include LICENSE
scrydb-0.2.0/PKG-INFO ADDED
@@ -0,0 +1,260 @@
1
+ Metadata-Version: 2.4
2
+ Name: scrydb
3
+ Version: 0.2.0
4
+ Summary: Lexical, semantic, and hybrid search built on SQLite.
5
+ Author-email: Timo Breuer <timobreuer@acm.org>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/breuert/scrydb
8
+ Project-URL: Repository, https://github.com/breuert/scrydb
9
+ Project-URL: Issues, https://github.com/breuert/scrydb/issues
10
+ Keywords: sqlite,fts5,sqlite-vec,information-retrieval,search,embeddings,vector-search
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Operating System :: POSIX :: Linux
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Operating System :: Microsoft :: Windows
15
+ Classifier: Topic :: Text Processing :: Indexing
16
+ Classifier: Topic :: Database
17
+ Requires-Python: >=3.9
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: numpy>=1.22
21
+ Requires-Dist: tqdm>=4.60
22
+ Requires-Dist: sqlite-vec>=0.1.0
23
+ Provides-Extra: dense
24
+ Requires-Dist: sentence-transformers>=3.0; extra == "dense"
25
+ Provides-Extra: eval
26
+ Requires-Dist: pandas>=1.5; extra == "eval"
27
+ Provides-Extra: test
28
+ Requires-Dist: pytest>=7; extra == "test"
29
+ Provides-Extra: all
30
+ Requires-Dist: sentence-transformers>=3.0; extra == "all"
31
+ Requires-Dist: pandas>=1.5; extra == "all"
32
+ Dynamic: license-file
33
+
34
+ <p align="center">
35
+ <img src="./docs/images/logo-light.png" width=250/>
36
+ <h1 align="center">scrydb</h1>
37
+ </p>
38
+
39
+ ``scrydb``'s purpose is making lexical, dense, and hybrid search possible with [SQLite](https://sqlite.org/). Hardware requirements are kept low and everything is self-contained in a single file, i.e., the raw documents, their embeddings, and the lexical index are stored in a single SQLite file.
40
+
41
+ - Lexical search is made possible by the [FTS5 extension for SQLite](https://sqlite.org/fts5.html).
42
+ - Semantic search over the entire index is made possible by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a small, dependency-free vector search extension for SQLite. Every embedding can be searched at three precisions: **binary** (1 bit/dim, Hamming distance), **int8** (1 byte/dim, cosine), and **float** (full precision, cosine) — trading index size and speed against ranking quality, and combinable as a two-stage rerank (e.g. binary-first >> float-precision rerank).
43
+ - Hybrid search relies on [Reciprocal Rank Fusion](https://dl.acm.org/doi/10.1145/1571941.1572114) to fuse lexical and semantic search results.
44
+
45
+ The library is compatible with [Sentence Transformers](https://www.sbert.net/index.html). However, it is also possible to store precomputed embeddings for both queries and documents.
46
+
47
+ > [!NOTE]
48
+ > The evaluation protocol, benchmark results, and examples are available at [`scrydb-eval`](https://github.com/breuert/scrydb-eval/), the corresponding data is shared on [Hugging Face](https://huggingface.co/datasets/breuert/scrydb-eval).
49
+
50
+ ## Usage examples
51
+
52
+ ``scrydb`` can be used interactively as follows:
53
+ ```python
54
+ from scrydb import Index, SentenceEmbedding
55
+
56
+ with Index.open("idx.db") as index:
57
+ index.add_model(SentenceEmbedding())
58
+ index.index_documents("corpus.jsonl", id_field="docid", text_field="text")
59
+ results = index.search("some query", mode="hybrid", rerank=True)
60
+ ```
61
+
62
+ Batch search for Information Retrieval benchmarks with precomputed embeddings can be run as follows:
63
+ ```python
64
+ import scrydb
65
+
66
+ idx = scrydb.Index.open("./path/to/index.db")
67
+
68
+ idx.index_documents(
69
+ source="./path/to/corpus.jsonl",
70
+ id_field="docid",
71
+ text_field="text",
72
+ embedding_field="embedding",
73
+ store_int8_embeddings=True, # opt in to int8 storage alongside binary/float
74
+ )
75
+
76
+ idx.index_queries(
77
+ source="./path/to/queries.jsonl",
78
+ id_field="qid",
79
+ text_field="text",
80
+ embedding_field="embedding",
81
+ store_int8_embeddings=True,
82
+ )
83
+
84
+ idx.batch_search(mode="lexical").write_trec("./path/to/lexical/run")
85
+ idx.batch_search(mode="semantic", precision="binary").write_trec("./path/to/binary/run")
86
+ idx.batch_search(mode="semantic", precision="int8").write_trec("./path/to/int8/run")
87
+ idx.batch_search(mode="semantic", precision="float").write_trec("./path/to/float/run")
88
+ idx.batch_search(mode="semantic", precision="binary", rerank="float").write_trec("./path/to/binary-rerank-float/run")
89
+ idx.batch_search(mode="hybrid").write_trec("./path/to/hybrid/run")
90
+ ```
91
+
92
+ ### Search modes, precision, and rerank
93
+
94
+ `search()`/`batch_search()` take three orthogonal knobs:
95
+
96
+ - `mode` — `"lexical"` (BM25 over FTS5), `"semantic"` (vector search), or `"hybrid"` (Reciprocal Rank Fusion of both).
97
+ - `precision` — which vector representation `mode="semantic"`/the semantic side of `mode="hybrid"` ranks with: `"binary"` (default), `"int8"`, or `"float"`.
98
+ - `rerank` — `False` (default), or a second-stage rerank over the top candidates from `mode`, at `"binary"`, `"int8"`, or `"float"` precision (`True` is a synonym for `"float"`).
99
+
100
+ ```python
101
+ idx.search("some query", mode="lexical") # BM25
102
+ idx.search("some query", mode="lexical", rerank="float") # BM25 >> Float/Cosine
103
+ idx.search("some query", mode="semantic", precision="binary") # Binary/Hamming
104
+ idx.search("some query", mode="semantic", precision="int8") # Int8/Cosine
105
+ idx.search("some query", mode="semantic", precision="binary", rerank="float") # Binary >> Float/Cosine
106
+ idx.search("some query", mode="hybrid", rerank=True) # Hybrid/RRF
107
+ ```
108
+
109
+ ## Installing
110
+
111
+ ```bash
112
+ pip install scrydb
113
+ ```
114
+
115
+ Or with [uv](https://docs.astral.sh/uv/) (faster, and manages the virtualenv for you):
116
+
117
+ ```bash
118
+ uv venv && uv pip install scrydb
119
+ # or, inside a uv-managed project:
120
+ uv add scrydb
121
+ ```
122
+
123
+ No C compiler or SQLite development headers required: vector search is
124
+ powered by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a pure
125
+ pip dependency that ships prebuilt binaries, so `pip install scrydb` is a
126
+ plain, fast, wheel-only install.
127
+
128
+ `Index.open()` loads the `sqlite-vec` extension automatically. This
129
+ requires a Python build whose `sqlite3` module supports
130
+ `enable_load_extension()` — true for Homebrew, pyenv, and
131
+ [uv](https://docs.astral.sh/uv/)-managed builds on macOS, virtually all
132
+ Linux distro packages, and the official Windows builds, but **not** for
133
+ macOS Pythons that link against Apple's SQLite, which disables extension
134
+ loading. That includes macOS's system Python and the CPython that
135
+ `actions/setup-python` installs on GitHub Actions runners. If you hit a
136
+ `RuntimeError` mentioning `enable_load_extension`, switch to Python from
137
+ Homebrew (`brew install python`), uv (`uv python install <version>`), or
138
+ pyenv (`PYTHON_CONFIGURE_OPTS='--enable-loadable-sqlite-extensions'
139
+ pyenv install <version>`).
140
+
141
+ If you'd rather not load the extension at all, you can still install and
142
+ use scrydb for lexical (BM25) search only — just disable it explicitly:
143
+
144
+ ```python
145
+ Index.open("idx.db", vec_ext_path=None)
146
+ ```
147
+
148
+ ### Try the CLI without installing (uvx)
149
+
150
+ [`uvx`](https://docs.astral.sh/uv/guides/tools/) runs the `scrydb` command-line
151
+ tool (`index`/`search`/`batch-search`/`auto` — see the [Docker](#docker)
152
+ section below for the full reference) in a throwaway environment, no venv or
153
+ persistent install needed:
154
+
155
+ ```bash
156
+ uvx scrydb index --documents corpus.jsonl --queries queries.jsonl --db idx.db
157
+ uvx scrydb search "some query" --db idx.db --mode hybrid --rerank float
158
+ uvx scrydb batch-search --db idx.db --mode hybrid --output run.trec
159
+ ```
160
+
161
+ Since scrydb has no compiled artifacts of its own, `uvx` just downloads the
162
+ wheel and its dependencies (including `sqlite-vec`'s prebuilt binary) into
163
+ its cache — no build step at all.
164
+
165
+ ## Docker
166
+
167
+ No local Python install needed: the [`Dockerfile`](./Dockerfile) builds a
168
+ Linux image with scrydb already installed, driven by a bundled `scrydb`
169
+ CLI. Everything it reads and writes -- input JSONL, the SQLite index, TREC
170
+ run files -- lives under `/data`, so bind-mount a host directory there.
171
+
172
+ ```bash
173
+ docker build -t scrydb .
174
+ ```
175
+
176
+ Drop `documents.jsonl` (and, optionally, `queries.jsonl`) into `./data` and
177
+ run the image with no arguments: it indexes whatever's present into
178
+ `./data/index.db`, then either batch-searches the stored queries into
179
+ `./data/run.trec` or -- if there's nothing to index and no stored queries --
180
+ tells you what it's waiting for.
181
+
182
+ ```bash
183
+ docker run --rm -v "$PWD/data":/data scrydb
184
+ ```
185
+
186
+ If `./data/index.db` already exists (say, you built it locally, or a
187
+ previous run produced it), the same command re-uses it: skips indexing
188
+ whatever source files aren't present and searches straight away. To query
189
+ an existing index ad hoc instead of running a full batch, override the
190
+ default command:
191
+
192
+ ```bash
193
+ docker run --rm -v "$PWD/data":/data scrydb search "some query" --mode hybrid --rerank float
194
+ ```
195
+
196
+ Every option is also settable as an `SCRYDB_*` environment variable (handy
197
+ for `docker run -e`), and the JSONL id-field names for documents/queries
198
+ default to `docid`/`qid`-style overrides when they differ from `id`:
199
+
200
+ ```bash
201
+ docker run --rm -v "$PWD/data":/data \
202
+ -e SCRYDB_DOC_ID_FIELD=docid -e SCRYDB_QUERY_ID_FIELD=qid \
203
+ -e SCRYDB_MODEL=mixedbread-ai/mxbai-embed-large-v1 \
204
+ -e SCRYDB_MODE=hybrid -e SCRYDB_RERANK=float \
205
+ scrydb
206
+ ```
207
+
208
+ Run `docker run --rm scrydb --help` (or `... <subcommand> --help`) for the
209
+ full `index`/`search`/`batch-search`/`auto` reference, or see the
210
+ module docstring in [`src/scrydb/cli.py`](./src/scrydb/cli.py).
211
+
212
+ Notes:
213
+
214
+ - **Dense/hybrid search** (`sentence-transformers`) isn't in the image by
215
+ default -- build with `--build-arg EXTRAS=all` (or `dense`) to add it.
216
+ This installs the CPU-only `torch` build so the image doesn't pull in
217
+ multi-gigabyte CUDA packages it can't use.
218
+ - **Multi-platform**: build once for both Intel/AMD and Apple
219
+ Silicon/ARM hosts (each running natively, no QEMU emulation) with
220
+ `docker buildx build --platform linux/amd64,linux/arm64 -t scrydb .`
221
+ - **Plain Python access**: `docker run --rm -it -v "$PWD/data":/data --entrypoint python3 scrydb`
222
+ drops into an interpreter with `scrydb` importable.
223
+
224
+ ## How extension discovery works at runtime
225
+
226
+ `Index.open()`/`Index()` default to `vec_ext_path="auto"`, which loads
227
+ `sqlite_vec.loadable_path()` — the copy of the extension bundled inside
228
+ the installed `sqlite-vec` pip package, prebuilt for the current
229
+ platform. Pass an explicit path to load a different build (e.g. a newer
230
+ `vec0` release), or `None` to skip loading it entirely.
231
+
232
+ ## Development / editable installs
233
+
234
+ ```bash
235
+ git clone <repo>
236
+ cd scrydb
237
+ python -m venv .venv && source .venv/bin/activate
238
+ pip install -e ".[all]"
239
+ pytest
240
+ ```
241
+
242
+ Or with uv:
243
+
244
+ ```bash
245
+ git clone <repo>
246
+ cd scrydb
247
+ uv venv
248
+ uv pip install -e ".[all]"
249
+ uv run pytest
250
+ ```
251
+
252
+ `uv run` picks up `.venv` automatically, so there's no `source .venv/bin/activate`
253
+ step — any command after it (`uv run pytest`, `uv run python -m scrydb.cli --help`,
254
+ `uv run python your_script.py`) runs inside the project's venv.
255
+
256
+ ## Optional extras
257
+
258
+ - `pip install "scrydb[dense]"` / `uv pip install "scrydb[dense]"` — dense/hybrid search via `sentence-transformers`
259
+ - `pip install "scrydb[eval]"` / `uv pip install "scrydb[eval]"` — `Run.to_dataframe()` via `pandas`
260
+ - `pip install "scrydb[all]"` / `uv pip install "scrydb[all]"` — both
scrydb-0.2.0/README.md ADDED
@@ -0,0 +1,227 @@
1
+ <p align="center">
2
+ <img src="./docs/images/logo-light.png" width=250/>
3
+ <h1 align="center">scrydb</h1>
4
+ </p>
5
+
6
+ ``scrydb``'s purpose is making lexical, dense, and hybrid search possible with [SQLite](https://sqlite.org/). Hardware requirements are kept low and everything is self-contained in a single file, i.e., the raw documents, their embeddings, and the lexical index are stored in a single SQLite file.
7
+
8
+ - Lexical search is made possible by the [FTS5 extension for SQLite](https://sqlite.org/fts5.html).
9
+ - Semantic search over the entire index is made possible by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a small, dependency-free vector search extension for SQLite. Every embedding can be searched at three precisions: **binary** (1 bit/dim, Hamming distance), **int8** (1 byte/dim, cosine), and **float** (full precision, cosine) — trading index size and speed against ranking quality, and combinable as a two-stage rerank (e.g. binary-first >> float-precision rerank).
10
+ - Hybrid search relies on [Reciprocal Rank Fusion](https://dl.acm.org/doi/10.1145/1571941.1572114) to fuse lexical and semantic search results.
11
+
12
+ The library is compatible with [Sentence Transformers](https://www.sbert.net/index.html). However, it is also possible to store precomputed embeddings for both queries and documents.
13
+
14
+ > [!NOTE]
15
+ > The evaluation protocol, benchmark results, and examples are available at [`scrydb-eval`](https://github.com/breuert/scrydb-eval/), the corresponding data is shared on [Hugging Face](https://huggingface.co/datasets/breuert/scrydb-eval).
16
+
17
+ ## Usage examples
18
+
19
+ ``scrydb`` can be used interactively as follows:
20
+ ```python
21
+ from scrydb import Index, SentenceEmbedding
22
+
23
+ with Index.open("idx.db") as index:
24
+ index.add_model(SentenceEmbedding())
25
+ index.index_documents("corpus.jsonl", id_field="docid", text_field="text")
26
+ results = index.search("some query", mode="hybrid", rerank=True)
27
+ ```
28
+
29
+ Batch search for Information Retrieval benchmarks with precomputed embeddings can be run as follows:
30
+ ```python
31
+ import scrydb
32
+
33
+ idx = scrydb.Index.open("./path/to/index.db")
34
+
35
+ idx.index_documents(
36
+ source="./path/to/corpus.jsonl",
37
+ id_field="docid",
38
+ text_field="text",
39
+ embedding_field="embedding",
40
+ store_int8_embeddings=True, # opt in to int8 storage alongside binary/float
41
+ )
42
+
43
+ idx.index_queries(
44
+ source="./path/to/queries.jsonl",
45
+ id_field="qid",
46
+ text_field="text",
47
+ embedding_field="embedding",
48
+ store_int8_embeddings=True,
49
+ )
50
+
51
+ idx.batch_search(mode="lexical").write_trec("./path/to/lexical/run")
52
+ idx.batch_search(mode="semantic", precision="binary").write_trec("./path/to/binary/run")
53
+ idx.batch_search(mode="semantic", precision="int8").write_trec("./path/to/int8/run")
54
+ idx.batch_search(mode="semantic", precision="float").write_trec("./path/to/float/run")
55
+ idx.batch_search(mode="semantic", precision="binary", rerank="float").write_trec("./path/to/binary-rerank-float/run")
56
+ idx.batch_search(mode="hybrid").write_trec("./path/to/hybrid/run")
57
+ ```
58
+
59
+ ### Search modes, precision, and rerank
60
+
61
+ `search()`/`batch_search()` take three orthogonal knobs:
62
+
63
+ - `mode` — `"lexical"` (BM25 over FTS5), `"semantic"` (vector search), or `"hybrid"` (Reciprocal Rank Fusion of both).
64
+ - `precision` — which vector representation `mode="semantic"`/the semantic side of `mode="hybrid"` ranks with: `"binary"` (default), `"int8"`, or `"float"`.
65
+ - `rerank` — `False` (default), or a second-stage rerank over the top candidates from `mode`, at `"binary"`, `"int8"`, or `"float"` precision (`True` is a synonym for `"float"`).
66
+
67
+ ```python
68
+ idx.search("some query", mode="lexical") # BM25
69
+ idx.search("some query", mode="lexical", rerank="float") # BM25 >> Float/Cosine
70
+ idx.search("some query", mode="semantic", precision="binary") # Binary/Hamming
71
+ idx.search("some query", mode="semantic", precision="int8") # Int8/Cosine
72
+ idx.search("some query", mode="semantic", precision="binary", rerank="float") # Binary >> Float/Cosine
73
+ idx.search("some query", mode="hybrid", rerank=True) # Hybrid/RRF
74
+ ```
75
+
76
+ ## Installing
77
+
78
+ ```bash
79
+ pip install scrydb
80
+ ```
81
+
82
+ Or with [uv](https://docs.astral.sh/uv/) (faster, and manages the virtualenv for you):
83
+
84
+ ```bash
85
+ uv venv && uv pip install scrydb
86
+ # or, inside a uv-managed project:
87
+ uv add scrydb
88
+ ```
89
+
90
+ No C compiler or SQLite development headers required: vector search is
91
+ powered by [`sqlite-vec`](https://github.com/asg017/sqlite-vec), a pure
92
+ pip dependency that ships prebuilt binaries, so `pip install scrydb` is a
93
+ plain, fast, wheel-only install.
94
+
95
+ `Index.open()` loads the `sqlite-vec` extension automatically. This
96
+ requires a Python build whose `sqlite3` module supports
97
+ `enable_load_extension()` — true for Homebrew, pyenv, and
98
+ [uv](https://docs.astral.sh/uv/)-managed builds on macOS, virtually all
99
+ Linux distro packages, and the official Windows builds, but **not** for
100
+ macOS Pythons that link against Apple's SQLite, which disables extension
101
+ loading. That includes macOS's system Python and the CPython that
102
+ `actions/setup-python` installs on GitHub Actions runners. If you hit a
103
+ `RuntimeError` mentioning `enable_load_extension`, switch to Python from
104
+ Homebrew (`brew install python`), uv (`uv python install <version>`), or
105
+ pyenv (`PYTHON_CONFIGURE_OPTS='--enable-loadable-sqlite-extensions'
106
+ pyenv install <version>`).
107
+
108
+ If you'd rather not load the extension at all, you can still install and
109
+ use scrydb for lexical (BM25) search only — just disable it explicitly:
110
+
111
+ ```python
112
+ Index.open("idx.db", vec_ext_path=None)
113
+ ```
114
+
115
+ ### Try the CLI without installing (uvx)
116
+
117
+ [`uvx`](https://docs.astral.sh/uv/guides/tools/) runs the `scrydb` command-line
118
+ tool (`index`/`search`/`batch-search`/`auto` — see the [Docker](#docker)
119
+ section below for the full reference) in a throwaway environment, no venv or
120
+ persistent install needed:
121
+
122
+ ```bash
123
+ uvx scrydb index --documents corpus.jsonl --queries queries.jsonl --db idx.db
124
+ uvx scrydb search "some query" --db idx.db --mode hybrid --rerank float
125
+ uvx scrydb batch-search --db idx.db --mode hybrid --output run.trec
126
+ ```
127
+
128
+ Since scrydb has no compiled artifacts of its own, `uvx` just downloads the
129
+ wheel and its dependencies (including `sqlite-vec`'s prebuilt binary) into
130
+ its cache — no build step at all.
131
+
132
+ ## Docker
133
+
134
+ No local Python install needed: the [`Dockerfile`](./Dockerfile) builds a
135
+ Linux image with scrydb already installed, driven by a bundled `scrydb`
136
+ CLI. Everything it reads and writes -- input JSONL, the SQLite index, TREC
137
+ run files -- lives under `/data`, so bind-mount a host directory there.
138
+
139
+ ```bash
140
+ docker build -t scrydb .
141
+ ```
142
+
143
+ Drop `documents.jsonl` (and, optionally, `queries.jsonl`) into `./data` and
144
+ run the image with no arguments: it indexes whatever's present into
145
+ `./data/index.db`, then either batch-searches the stored queries into
146
+ `./data/run.trec` or -- if there's nothing to index and no stored queries --
147
+ tells you what it's waiting for.
148
+
149
+ ```bash
150
+ docker run --rm -v "$PWD/data":/data scrydb
151
+ ```
152
+
153
+ If `./data/index.db` already exists (say, you built it locally, or a
154
+ previous run produced it), the same command re-uses it: skips indexing
155
+ whatever source files aren't present and searches straight away. To query
156
+ an existing index ad hoc instead of running a full batch, override the
157
+ default command:
158
+
159
+ ```bash
160
+ docker run --rm -v "$PWD/data":/data scrydb search "some query" --mode hybrid --rerank float
161
+ ```
162
+
163
+ Every option is also settable as an `SCRYDB_*` environment variable (handy
164
+ for `docker run -e`), and the JSONL id-field names for documents/queries
165
+ default to `docid`/`qid`-style overrides when they differ from `id`:
166
+
167
+ ```bash
168
+ docker run --rm -v "$PWD/data":/data \
169
+ -e SCRYDB_DOC_ID_FIELD=docid -e SCRYDB_QUERY_ID_FIELD=qid \
170
+ -e SCRYDB_MODEL=mixedbread-ai/mxbai-embed-large-v1 \
171
+ -e SCRYDB_MODE=hybrid -e SCRYDB_RERANK=float \
172
+ scrydb
173
+ ```
174
+
175
+ Run `docker run --rm scrydb --help` (or `... <subcommand> --help`) for the
176
+ full `index`/`search`/`batch-search`/`auto` reference, or see the
177
+ module docstring in [`src/scrydb/cli.py`](./src/scrydb/cli.py).
178
+
179
+ Notes:
180
+
181
+ - **Dense/hybrid search** (`sentence-transformers`) isn't in the image by
182
+ default -- build with `--build-arg EXTRAS=all` (or `dense`) to add it.
183
+ This installs the CPU-only `torch` build so the image doesn't pull in
184
+ multi-gigabyte CUDA packages it can't use.
185
+ - **Multi-platform**: build once for both Intel/AMD and Apple
186
+ Silicon/ARM hosts (each running natively, no QEMU emulation) with
187
+ `docker buildx build --platform linux/amd64,linux/arm64 -t scrydb .`
188
+ - **Plain Python access**: `docker run --rm -it -v "$PWD/data":/data --entrypoint python3 scrydb`
189
+ drops into an interpreter with `scrydb` importable.
190
+
191
+ ## How extension discovery works at runtime
192
+
193
+ `Index.open()`/`Index()` default to `vec_ext_path="auto"`, which loads
194
+ `sqlite_vec.loadable_path()` — the copy of the extension bundled inside
195
+ the installed `sqlite-vec` pip package, prebuilt for the current
196
+ platform. Pass an explicit path to load a different build (e.g. a newer
197
+ `vec0` release), or `None` to skip loading it entirely.
198
+
199
+ ## Development / editable installs
200
+
201
+ ```bash
202
+ git clone <repo>
203
+ cd scrydb
204
+ python -m venv .venv && source .venv/bin/activate
205
+ pip install -e ".[all]"
206
+ pytest
207
+ ```
208
+
209
+ Or with uv:
210
+
211
+ ```bash
212
+ git clone <repo>
213
+ cd scrydb
214
+ uv venv
215
+ uv pip install -e ".[all]"
216
+ uv run pytest
217
+ ```
218
+
219
+ `uv run` picks up `.venv` automatically, so there's no `source .venv/bin/activate`
220
+ step — any command after it (`uv run pytest`, `uv run python -m scrydb.cli --help`,
221
+ `uv run python your_script.py`) runs inside the project's venv.
222
+
223
+ ## Optional extras
224
+
225
+ - `pip install "scrydb[dense]"` / `uv pip install "scrydb[dense]"` — dense/hybrid search via `sentence-transformers`
226
+ - `pip install "scrydb[eval]"` / `uv pip install "scrydb[eval]"` — `Run.to_dataframe()` via `pandas`
227
+ - `pip install "scrydb[all]"` / `uv pip install "scrydb[all]"` — both
@@ -0,0 +1,49 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "scrydb"
7
+ version = "0.2.0"
8
+ description = "Lexical, semantic, and hybrid search built on SQLite."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [
14
+ { name = "Timo Breuer", email = "timobreuer@acm.org" },
15
+ ]
16
+ keywords = ["sqlite", "fts5", "sqlite-vec", "information-retrieval", "search", "embeddings", "vector-search"]
17
+ classifiers = [
18
+ "Programming Language :: Python :: 3",
19
+ "Operating System :: POSIX :: Linux",
20
+ "Operating System :: MacOS",
21
+ "Operating System :: Microsoft :: Windows",
22
+ "Topic :: Text Processing :: Indexing",
23
+ "Topic :: Database",
24
+ ]
25
+ dependencies = [
26
+ "numpy>=1.22",
27
+ "tqdm>=4.60",
28
+ "sqlite-vec>=0.1.0",
29
+ ]
30
+
31
+ [project.optional-dependencies]
32
+ dense = ["sentence-transformers>=3.0"]
33
+ eval = ["pandas>=1.5"]
34
+ test = ["pytest>=7"]
35
+ all = ["sentence-transformers>=3.0", "pandas>=1.5"]
36
+
37
+ [project.urls]
38
+ Homepage = "https://github.com/breuert/scrydb"
39
+ Repository = "https://github.com/breuert/scrydb"
40
+ Issues = "https://github.com/breuert/scrydb/issues"
41
+
42
+ [project.scripts]
43
+ scrydb = "scrydb.cli:main"
44
+
45
+ [tool.setuptools.packages.find]
46
+ where = ["src"]
47
+
48
+ [tool.pytest.ini_options]
49
+ testpaths = ["tests"]
scrydb-0.2.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,12 @@
1
+ """scrydb — lexical, semantic, and hybrid search built on SQLite."""
2
+
3
+ from .core import Index, Run, SearchResult, SentenceEmbedding
4
+
5
+ __all__ = ["Index", "Run", "SearchResult", "SentenceEmbedding"]
6
+
7
+ try:
8
+ from importlib.metadata import version as _version
9
+
10
+ __version__ = _version("scrydb")
11
+ except Exception: # pragma: no cover - package not installed
12
+ __version__ = "0.0.0"