sqlite-hybrid-search 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlite_hybrid_search-0.1.0/.github/workflows/release.yml +95 -0
- sqlite_hybrid_search-0.1.0/BENCHMARKS.md +263 -0
- sqlite_hybrid_search-0.1.0/CMakeLists.txt +146 -0
- sqlite_hybrid_search-0.1.0/LICENSE +21 -0
- sqlite_hybrid_search-0.1.0/PKG-INFO +298 -0
- sqlite_hybrid_search-0.1.0/README.md +271 -0
- sqlite_hybrid_search-0.1.0/benchmarks/CMakeLists.txt +8 -0
- sqlite_hybrid_search-0.1.0/benchmarks/corpus.py +101 -0
- sqlite_hybrid_search-0.1.0/benchmarks/harness.py +191 -0
- sqlite_hybrid_search-0.1.0/benchmarks/metrics.py +55 -0
- sqlite_hybrid_search-0.1.0/benchmarks/run_benchmarks.cpp +194 -0
- sqlite_hybrid_search-0.1.0/benchmarks/run_eval.py +139 -0
- sqlite_hybrid_search-0.1.0/benchmarks/test_eval.py +209 -0
- sqlite_hybrid_search-0.1.0/bindings/CMakeLists.txt +28 -0
- sqlite_hybrid_search-0.1.0/bindings/python_bindings.cpp +121 -0
- sqlite_hybrid_search-0.1.0/cmake/patch_usearch_candidates_iterator_bug.cmake +52 -0
- sqlite_hybrid_search-0.1.0/core/CMakeLists.txt +75 -0
- sqlite_hybrid_search-0.1.0/core/include/retrieval_engine/chunking.hpp +34 -0
- sqlite_hybrid_search-0.1.0/core/include/retrieval_engine/retrieval_engine.hpp +256 -0
- sqlite_hybrid_search-0.1.0/core/src/chunking.cpp +76 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/chunk_repository.cpp +258 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/chunk_repository.hpp +90 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/chunk_store.cpp +230 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/chunk_store.hpp +108 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/dense_index.cpp +121 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/dense_index.hpp +61 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/embedding_model_loader.cpp +125 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/embedding_model_loader.hpp +30 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/fts5_query.cpp +38 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/fts5_query.hpp +32 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/mock_text_embedder.cpp +80 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/mock_text_embedder.hpp +36 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/onnx_text_embedder.cpp +132 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/onnx_text_embedder.hpp +70 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/recency_decay.cpp +40 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/recency_decay.hpp +32 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/rrf_fusion.cpp +67 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/rrf_fusion.hpp +46 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/sqlite_util.cpp +40 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/sqlite_util.hpp +66 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/text_embedder.hpp +43 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/time_util.hpp +18 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/usearch_util.hpp +23 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/wordpiece_tokenizer.cpp +215 -0
- sqlite_hybrid_search-0.1.0/core/src/detail/wordpiece_tokenizer.hpp +61 -0
- sqlite_hybrid_search-0.1.0/core/src/retrieval_engine.cpp +175 -0
- sqlite_hybrid_search-0.1.0/core/tests/CMakeLists.txt +62 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_builtin_embedder.cpp +241 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_chunking_and_dense_retrieval.cpp +247 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_hybrid_retrieval.cpp +241 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_infra_sanity.cpp +59 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_onnx_embedder.cpp +141 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_persistence.cpp +202 -0
- sqlite_hybrid_search-0.1.0/core/tests/test_recency_decay_and_temporal_reranking.cpp +203 -0
- sqlite_hybrid_search-0.1.0/docs/DECISIONS.md +333 -0
- sqlite_hybrid_search-0.1.0/pyproject.toml +89 -0
- sqlite_hybrid_search-0.1.0/python/sqlite_hybrid_search/__init__.py +205 -0
- sqlite_hybrid_search-0.1.0/python/sqlite_hybrid_search/cli.py +279 -0
- sqlite_hybrid_search-0.1.0/tests/test_bindings_and_cli.py +147 -0
- sqlite_hybrid_search-0.1.0/tests/test_builtin_embedder.py +233 -0
- sqlite_hybrid_search-0.1.0/tests/test_memory_search.py +143 -0
- sqlite_hybrid_search-0.1.0/tests/test_persistence.py +48 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Builds platform wheels + an sdist and, on a published GitHub Release,
|
|
4
|
+
# uploads them to PyPI via Trusted Publishing (OIDC -- no API token).
|
|
5
|
+
#
|
|
6
|
+
# PyPI setup (one time): project -> Publishing -> add a pending publisher
|
|
7
|
+
# Owner: ashray-00
|
|
8
|
+
# Repository: sqlite-hybrid-search
|
|
9
|
+
# Workflow name: release.yml
|
|
10
|
+
# Environment name: pypi
|
|
11
|
+
# Then create the `pypi` environment under repo Settings -> Environments.
|
|
12
|
+
|
|
13
|
+
on:
|
|
14
|
+
release:
|
|
15
|
+
types: [published]
|
|
16
|
+
workflow_dispatch: {} # manual runs build artifacts but do not publish
|
|
17
|
+
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
wheels:
|
|
23
|
+
name: wheels (${{ matrix.os }})
|
|
24
|
+
runs-on: ${{ matrix.os }}
|
|
25
|
+
strategy:
|
|
26
|
+
fail-fast: false
|
|
27
|
+
matrix:
|
|
28
|
+
# x86_64 Linux + Apple-Silicon macOS. GitHub retired the macos-13
|
|
29
|
+
# (Intel) hosted runners; Intel-Mac users build from the sdist.
|
|
30
|
+
os: [ubuntu-latest, macos-14]
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
|
|
34
|
+
- name: Build wheels
|
|
35
|
+
uses: pypa/cibuildwheel@v2.21.3 # bump to the latest tag periodically
|
|
36
|
+
env:
|
|
37
|
+
# cp39..cp313, 64-bit only; skip PyPy and musl for a first release.
|
|
38
|
+
CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*"
|
|
39
|
+
CIBW_SKIP: "*-musllinux_* *_i686 *-win32 pp*"
|
|
40
|
+
CIBW_ARCHS_MACOS: "arm64"
|
|
41
|
+
# The built-in ONNX Runtime backend is a source-build opt-in; PyPI
|
|
42
|
+
# wheels ship the dependency-free mock backend only.
|
|
43
|
+
CIBW_ENVIRONMENT_LINUX: 'CMAKE_ARGS="-DRETRIEVAL_ENGINE_WITH_ONNX=OFF"'
|
|
44
|
+
# Link the macOS SDK's system SQLite (present on every Mac, part of
|
|
45
|
+
# the OS) instead of a Homebrew build: delocate keeps system libs
|
|
46
|
+
# as plain dynamic deps, so the wheel stays clean and its
|
|
47
|
+
# deployment target is not dragged up to the runner's macOS.
|
|
48
|
+
CIBW_ENVIRONMENT_MACOS: >-
|
|
49
|
+
CMAKE_ARGS="-DRETRIEVAL_ENGINE_WITH_ONNX=OFF
|
|
50
|
+
-DSQLite3_INCLUDE_DIR=$(xcrun --show-sdk-path)/usr/include
|
|
51
|
+
-DSQLite3_LIBRARY=$(xcrun --show-sdk-path)/usr/lib/libsqlite3.tbd"
|
|
52
|
+
CIBW_BEFORE_ALL_LINUX: "yum install -y sqlite-devel || dnf install -y sqlite-devel"
|
|
53
|
+
# Smoke-test every wheel after auditwheel/delocate repairs it.
|
|
54
|
+
CIBW_TEST_COMMAND: >-
|
|
55
|
+
python -c "import sqlite_hybrid_search, tempfile, os;
|
|
56
|
+
e = sqlite_hybrid_search.Engine(os.path.join(tempfile.mkdtemp(), 'x.sqlite3'), dim=3);
|
|
57
|
+
e.add(documents=[{'id': 'a', 'text': 'hi'}], embeddings=[[1, 0, 0]]);
|
|
58
|
+
assert e.search([1, 0, 0], 1)[0]['document_id'] == 'a'"
|
|
59
|
+
|
|
60
|
+
- uses: actions/upload-artifact@v4
|
|
61
|
+
with:
|
|
62
|
+
name: wheels-${{ matrix.os }}
|
|
63
|
+
path: wheelhouse/*.whl
|
|
64
|
+
|
|
65
|
+
sdist:
|
|
66
|
+
name: sdist
|
|
67
|
+
runs-on: ubuntu-latest
|
|
68
|
+
steps:
|
|
69
|
+
- uses: actions/checkout@v4
|
|
70
|
+
- name: Install SQLite headers + library
|
|
71
|
+
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
|
|
72
|
+
- run: pipx run build --sdist
|
|
73
|
+
- name: Verify the sdist builds from source
|
|
74
|
+
run: pip install dist/*.tar.gz
|
|
75
|
+
- uses: actions/upload-artifact@v4
|
|
76
|
+
with:
|
|
77
|
+
name: sdist
|
|
78
|
+
path: dist/*.tar.gz
|
|
79
|
+
|
|
80
|
+
publish:
|
|
81
|
+
name: publish to PyPI
|
|
82
|
+
needs: [wheels, sdist]
|
|
83
|
+
runs-on: ubuntu-latest
|
|
84
|
+
if: github.event_name == 'release'
|
|
85
|
+
environment: pypi # must match the PyPI trusted-publisher config
|
|
86
|
+
permissions:
|
|
87
|
+
id-token: write # OIDC token for Trusted Publishing
|
|
88
|
+
steps:
|
|
89
|
+
- uses: actions/download-artifact@v4
|
|
90
|
+
with:
|
|
91
|
+
path: dist
|
|
92
|
+
merge-multiple: true
|
|
93
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
94
|
+
# with:
|
|
95
|
+
# repository-url: https://test.pypi.org/legacy/ # uncomment for TestPyPI
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
# Benchmarks
|
|
2
|
+
|
|
3
|
+
Reproducible measurement of the four retrieval paths this engine exposes,
|
|
4
|
+
across dataset sizes, on one machine. Generated by `benchmarks/run_eval.py`;
|
|
5
|
+
raw numbers in [`benchmarks/results.json`](benchmarks/results.json).
|
|
6
|
+
|
|
7
|
+
## What was measured
|
|
8
|
+
|
|
9
|
+
| Approach | Call | What it is |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| `dense` | `search_dense()` | usearch HNSW cosine ANN only |
|
|
12
|
+
| `sparse` | `search_sparse()` | SQLite FTS5 BM25 only |
|
|
13
|
+
| `hybrid` | `search_hybrid()` | Reciprocal Rank Fusion (k=60) of dense + sparse |
|
|
14
|
+
| `hybrid_decay` | `search_memory()` | `hybrid` re-scored by exponential recency decay (λ = 0.1) |
|
|
15
|
+
|
|
16
|
+
- **Retrieval quality:** Recall@10, nDCG@10, MRR@10 over 100 labelled queries.
|
|
17
|
+
- **Latency:** warm-cache p50 / p95 / p99 and throughput; cold-cache first-query
|
|
18
|
+
latency; index **load**-on-open time (from the `.usearch` sidecar).
|
|
19
|
+
- **Footprint:** SQLite + sidecar size on disk, process peak RSS, RSS growth
|
|
20
|
+
during indexing, ingestion throughput.
|
|
21
|
+
|
|
22
|
+
### Environment
|
|
23
|
+
|
|
24
|
+
Numbers below were measured on `macOS 26.6 arm64` (Apple Silicon), Python 3.12,
|
|
25
|
+
engine dim 64, single thread. Full C++/Python test suites green (`ctest`: 41,
|
|
26
|
+
`pytest`: 32). The harness is OS-independent; absolute timings will differ on
|
|
27
|
+
other hardware.
|
|
28
|
+
|
|
29
|
+
### Reproduce
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
cmake -B build && cmake --build build # Linux
|
|
33
|
+
# macOS: cmake -B build -DCMAKE_PREFIX_PATH=/opt/homebrew && cmake --build build
|
|
34
|
+
.venv/bin/python benchmarks/run_eval.py # 1k / 10k / 100k, ~1.5 min
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### Honest caveat on the corpus
|
|
38
|
+
|
|
39
|
+
The corpus is **synthetic** (`benchmarks/corpus.py`): every relevant document
|
|
40
|
+
carries a unique rare entity token, and embeddings are a 64-dim hashed
|
|
41
|
+
bag-of-words, not a trained model. So:
|
|
42
|
+
|
|
43
|
+
- **Sparse/BM25 numbers are a best case** — the rare entity token is a perfect
|
|
44
|
+
exact-match signal, which is exactly the situation BM25 is built for.
|
|
45
|
+
- **Dense numbers are a worst case** — a 64-dim hashing-trick vector collides
|
|
46
|
+
badly as the corpus grows; a real embedding model (nomic, BGE-M3, MiniLM via
|
|
47
|
+
the built-in ONNX path) would not degrade like the `dense` column does here.
|
|
48
|
+
- **Latency and memory are model-independent** and transfer directly.
|
|
49
|
+
|
|
50
|
+
Read the quality section as *"how the fusion behaves when the two signals
|
|
51
|
+
disagree"*, not as an absolute quality score for dense retrieval.
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Retrieval quality
|
|
56
|
+
|
|
57
|
+
**Recall@10** — fraction of relevant docs found in the top 10:
|
|
58
|
+
|
|
59
|
+
| Approach | 1k | 10k | 100k |
|
|
60
|
+
|---|---|---|---|
|
|
61
|
+
| dense | 0.987 | 0.783 | 0.493 |
|
|
62
|
+
| sparse | 1.000 | 1.000 | 1.000 |
|
|
63
|
+
| **hybrid** | **1.000** | **1.000** | **1.000** |
|
|
64
|
+
| hybrid_decay | 1.000 | 1.000 | 1.000 |
|
|
65
|
+
|
|
66
|
+
**nDCG@10** — ranking quality:
|
|
67
|
+
|
|
68
|
+
| Approach | 1k | 10k | 100k |
|
|
69
|
+
|---|---|---|---|
|
|
70
|
+
| dense | 0.944 | 0.752 | 0.507 |
|
|
71
|
+
| sparse | 1.000 | 1.000 | 1.000 |
|
|
72
|
+
| hybrid | 0.992 | 0.969 | 0.951 |
|
|
73
|
+
| **hybrid_decay** | **0.998** | **0.998** | **0.997** |
|
|
74
|
+
|
|
75
|
+
**MRR@10:**
|
|
76
|
+
|
|
77
|
+
| Approach | 1k | 10k | 100k |
|
|
78
|
+
|---|---|---|---|
|
|
79
|
+
| dense | 0.978 | 0.870 | 0.673 |
|
|
80
|
+
| sparse | 1.000 | 1.000 | 1.000 |
|
|
81
|
+
| hybrid | 1.000 | 0.995 | 0.990 |
|
|
82
|
+
| hybrid_decay | 1.000 | 1.000 | 0.995 |
|
|
83
|
+
|
|
84
|
+
**Reading it:**
|
|
85
|
+
|
|
86
|
+
- **Hybrid recovers everything dense loses.** As the toy dense vectors collapse
|
|
87
|
+
(Recall 0.99 → 0.49), `hybrid` stays at **1.000** recall — RRF lets the sparse
|
|
88
|
+
side carry the query. This is the headline result: *fusion is strictly safer
|
|
89
|
+
than dense-only.*
|
|
90
|
+
- **When one signal is already perfect, fusing in a noisy one costs a little
|
|
91
|
+
ranking quality.** `sparse` nDCG is 1.000; `hybrid` nDCG is 0.951 at 100k,
|
|
92
|
+
because RRF interleaves dense's mistaken candidates. Fusion trades a few
|
|
93
|
+
points of top-rank precision for robustness.
|
|
94
|
+
- **Recency decay buys most of that back.** `hybrid_decay` nDCG is ~0.997 across
|
|
95
|
+
all sizes: the decay term breaks ties in favour of the (correctly recent)
|
|
96
|
+
relevant docs, pushing them back above dense's false positives. This is the
|
|
97
|
+
agent-memory case working as intended.
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Latency
|
|
102
|
+
|
|
103
|
+
**Warm-cache latency (ms), 100 queries × repeated passes:**
|
|
104
|
+
|
|
105
|
+
| Approach | 1k p50 | 1k p95 | 1k p99 | 10k p50 | 10k p95 | 10k p99 | 100k p50 | 100k p95 | 100k p99 |
|
|
106
|
+
|---|---|---|---|---|---|---|---|---|---|
|
|
107
|
+
| dense | 0.090 | 0.094 | 0.097 | 0.103 | 0.112 | 0.115 | 0.173 | 0.199 | 0.211 |
|
|
108
|
+
| sparse | 0.436 | 0.457 | 0.478 | 4.605 | 5.013 | 5.159 | 49.34 | 49.77 | 49.89 |
|
|
109
|
+
| hybrid | 0.524 | 0.541 | 0.558 | 4.765 | 5.059 | 5.297 | 49.44 | 50.13 | 50.50 |
|
|
110
|
+
| hybrid_decay | 0.729 | 0.790 | 0.818 | 6.517 | 7.350 | 7.630 | 70.06 | 83.28 | 85.93 |
|
|
111
|
+
|
|
112
|
+
**Throughput (queries/sec, single thread):**
|
|
113
|
+
|
|
114
|
+
| Approach | 1k | 10k | 100k |
|
|
115
|
+
|---|---|---|---|
|
|
116
|
+
| dense | 11048 | 9625 | 5705 |
|
|
117
|
+
| sparse | 2286 | 215 | 20 |
|
|
118
|
+
| hybrid | 1905 | 208 | 20 |
|
|
119
|
+
| hybrid_decay | 1371 | 155 | 14 |
|
|
120
|
+
|
|
121
|
+
- **Dense is sub-millisecond and near flat with scale** (0.09 → 0.17 ms p50 from
|
|
122
|
+
1k to 100k) — usearch HNSW is doing its job.
|
|
123
|
+
- **Sparse/hybrid latency is dominated by FTS5** and grows roughly linearly with
|
|
124
|
+
corpus size: ~0.4 ms → ~49 ms p50. The OR-of-terms BM25 query scans large
|
|
125
|
+
posting lists for common tokens. This is the main query-time bottleneck.
|
|
126
|
+
Fusion now feeds BM25's top `min(k, 200)` rows into RRF — a ceiling that
|
|
127
|
+
bounds pathologically large `k` without touching ordinary retrieval, so at
|
|
128
|
+
the benchmark's `k = 10` sparse/hybrid latency is unchanged.
|
|
129
|
+
- **`hybrid_decay` adds a re-scoring pass** over the fused candidate set:
|
|
130
|
+
+40% p50 over `hybrid` at 100k (49.4 → 70.1 ms), and a wider p95/p99 tail.
|
|
131
|
+
|
|
132
|
+
**Index load on open.** The usearch graph is persisted to a `<db>.usearch`
|
|
133
|
+
sidecar and memory-loaded on the next open instead of being rebuilt from
|
|
134
|
+
SQLite. Startup is now effectively instant at every size:
|
|
135
|
+
|
|
136
|
+
| Dataset | Load from sidecar | (was: rebuild from SQLite) |
|
|
137
|
+
|---|---|---|
|
|
138
|
+
| 1k | 0.7 ms | 40 ms |
|
|
139
|
+
| 10k | 3.2 ms | 721 ms |
|
|
140
|
+
| 100k | **24.5 ms** | **14.6 s** |
|
|
141
|
+
|
|
142
|
+
The first open of a brand-new database still rebuilds (there is no sidecar
|
|
143
|
+
yet) and writes the sidecar; every open after that takes the load path. A
|
|
144
|
+
sidecar that is missing, truncated, or out of sync with the chunk table is
|
|
145
|
+
rejected in microseconds and the engine falls back to a rebuild.
|
|
146
|
+
|
|
147
|
+
**Cold vs warm query.** With the graph loaded from the sidecar, the first
|
|
148
|
+
query after an open is no slower than a warm one (100k `hybrid`: 49.3 ms cold
|
|
149
|
+
vs 49.4 ms warm p50; `dense`: 0.32 ms cold vs 0.17 ms warm).
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## Memory & storage footprint
|
|
154
|
+
|
|
155
|
+
| Dataset | SQLite | `.usearch` sidecar | Ingest throughput | RSS growth during indexing | Process peak RSS |
|
|
156
|
+
|---|---|---|---|---|---|
|
|
157
|
+
| 1k | 0.57 MB | ~0.5 MB | 21695 docs/s | 3.5 MB | 35 MB |
|
|
158
|
+
| 10k | 5.21 MB | ~4 MB | 12826 docs/s | 11.1 MB | 79 MB |
|
|
159
|
+
| 100k | 51.98 MB | 40.5 MB | 6505 docs/s | 52.5 MB | 430 MB |
|
|
160
|
+
|
|
161
|
+
- **On-disk scales linearly.** SQLite is ~520 bytes/doc (chunk text + metadata +
|
|
162
|
+
FTS5 index + the vector blob kept for a rebuild); the sidecar adds the
|
|
163
|
+
serialised HNSW graph (~400 bytes/doc at dim 64). At 100k that is ~92 MB
|
|
164
|
+
total.
|
|
165
|
+
- **Ingestion throughput falls ~3.3× from 1k to 100k** — dual-writing SQLite,
|
|
166
|
+
FTS5, and usearch, with HNSW insertion getting more expensive as the graph
|
|
167
|
+
grows. The harness ingests in 5k-row batches and the sidecar is reserialised
|
|
168
|
+
after each one (20 writes at 100k); measured throughput (6505 docs/s) is
|
|
169
|
+
within run-to-run noise of the pre-persistence number (6482), but a workload
|
|
170
|
+
of many tiny batches would pay more, since each save rewrites the whole graph.
|
|
171
|
+
- **Peak RSS at 100k (430 MB) is inflated by the Python driver** holding the
|
|
172
|
+
whole synthetic corpus (100k × 64 floats as Python lists) in memory at once.
|
|
173
|
+
The native cross-check below is the honest engine-only number.
|
|
174
|
+
|
|
175
|
+
### Native cross-check (`benchmarks/run_benchmarks.cpp`, no Python in the loop)
|
|
176
|
+
|
|
177
|
+
20 000 random unit vectors, 200 queries:
|
|
178
|
+
|
|
179
|
+
| | value |
|
|
180
|
+
|---|---|
|
|
181
|
+
| SQLite on disk | 8.58 MB |
|
|
182
|
+
| Peak RSS (engine only) | 37.7 MB |
|
|
183
|
+
| Ingest throughput | 783 docs/s |
|
|
184
|
+
| dense p50 / p95 / p99 | 0.671 / 0.736 / 0.793 ms |
|
|
185
|
+
| hybrid p50 / p95 / p99 | 0.938 / 0.999 / 1.048 ms |
|
|
186
|
+
|
|
187
|
+
The engine's own peak RSS for 20k docs is **~38 MB**, versus the 79–430 MB the
|
|
188
|
+
Python-driven run reports — most of that difference is the driver, not the
|
|
189
|
+
engine. Ingest throughput here (783 docs/s) is a *worst case*: uniformly random
|
|
190
|
+
64-dim vectors are the hardest input for HNSW (every candidate is roughly
|
|
191
|
+
equidistant), whereas the structured corpus in the Python run indexes 8–28×
|
|
192
|
+
faster. Real embeddings sit between the two.
|
|
193
|
+
|
|
194
|
+
---
|
|
195
|
+
|
|
196
|
+
## Honest tradeoffs & writeup
|
|
197
|
+
|
|
198
|
+
### Where this engine wins
|
|
199
|
+
|
|
200
|
+
- **Exact entity / keyword matches.** The FTS5 BM25 side nails rare tokens
|
|
201
|
+
(names, IDs, error codes) — 1.000 recall at every size here — and RRF makes
|
|
202
|
+
`hybrid` inherit that for free. A pure vector store cannot do this without a
|
|
203
|
+
separate keyword index bolted on.
|
|
204
|
+
- **Fusion is strictly safer than dense-only.** `hybrid` never lost recall
|
|
205
|
+
relative to `dense` in any configuration; when the dense signal degraded, RRF
|
|
206
|
+
fell back on sparse automatically. You do not have to tune which retriever to
|
|
207
|
+
trust per query.
|
|
208
|
+
- **Recency-biased agent memory is built in.** `hybrid_decay` held nDCG ≈ 0.997
|
|
209
|
+
across all sizes by preferring recent relevant memories — the Mem0/Zep-style
|
|
210
|
+
capability, in-process, no service, one `decay_lambda` parameter.
|
|
211
|
+
- **Zero-dependency deployment.** One process, one SQLite file plus a
|
|
212
|
+
`.usearch` sidecar, an in-memory usearch index. No server to run, no
|
|
213
|
+
container, nothing listening on a port. Engine-only peak RSS is ~38 MB for
|
|
214
|
+
20k docs.
|
|
215
|
+
- **Instant startup via the disk-backed index sidecar.** The HNSW graph is
|
|
216
|
+
serialised to `<db>.usearch` on write and memory-loaded on the next open:
|
|
217
|
+
**24.5 ms at 100k**, versus 14.6 s to rebuild it from SQLite. Cold start is
|
|
218
|
+
no longer a function of corpus size.
|
|
219
|
+
- **Dense retrieval is fast and scale-stable.** Sub-0.2 ms p50 at 100k, ~5700
|
|
220
|
+
q/s single-threaded.
|
|
221
|
+
- **SQLite stays authoritative.** The sidecar is a cache: if it is missing,
|
|
222
|
+
truncated, or its vector count disagrees with the chunk table, the engine
|
|
223
|
+
discards it and rebuilds — a corrupt index is never data loss.
|
|
224
|
+
|
|
225
|
+
### Where it loses or bottlenecks
|
|
226
|
+
|
|
227
|
+
- **Sparse/hybrid query latency grows with corpus size** — ~49 ms p50 at 100k,
|
|
228
|
+
~20 q/s. FTS5's BM25 over an OR of common terms is the cost. Dense stays
|
|
229
|
+
sub-millisecond; the fusion is only as fast as its slower half.
|
|
230
|
+
- **`hybrid_decay` has a fat tail.** The extra recency re-scoring pass pushes
|
|
231
|
+
100k p99 to ~87 ms and throughput to ~14 q/s.
|
|
232
|
+
- **Dual-write ingestion overhead, plus sidecar reserialisation.** Every
|
|
233
|
+
document hits SQLite, FTS5, and usearch; throughput drops ~3.3× from 1k to
|
|
234
|
+
100k, and HNSW insertion cost is sensitive to how clustered the embeddings
|
|
235
|
+
are (783 → 22k docs/s depending on input). Each `add_documents` call also
|
|
236
|
+
rewrites the whole `.usearch` sidecar — negligible for bulk ingest,
|
|
237
|
+
meaningful for a stream of one-document writes.
|
|
238
|
+
- **Single-threaded, not concurrency-safe.** `RetrievalEngine` must be confined
|
|
239
|
+
to one thread or externally locked; there is no query parallelism yet.
|
|
240
|
+
- **Small embedding dimensions collide.** dim 64 is fine for a demo; production
|
|
241
|
+
needs 384–1024, which raises the memory and index-build numbers accordingly.
|
|
242
|
+
|
|
243
|
+
### DX comparison
|
|
244
|
+
|
|
245
|
+
| | This engine | sqlite-vec + glue | ChromaDB | Qdrant / Weaviate / Milvus |
|
|
246
|
+
|---|---|---|---|---|
|
|
247
|
+
| Deployment | in-process library, 1 file | in-process (SQLite ext) | embedded lib **or** server | separate server / cluster |
|
|
248
|
+
| Process to run | none | none | none (embedded) / one (server) | one+ (plus its deps) |
|
|
249
|
+
| Hybrid dense+sparse | built in (RRF) | DIY: wire up FTS5 + your own fusion | dense only (some keyword filtering) | built in (server-side) |
|
|
250
|
+
| Recency / memory semantics | built in (`search_memory`) | DIY | DIY | DIY (metadata + custom scoring) |
|
|
251
|
+
| Rerank / score trail | `search_explained()` | DIY | limited | varies |
|
|
252
|
+
| Persistence model | SQLite authoritative + `.usearch` sidecar | SQLite table | DuckDB/parquet | own storage engine |
|
|
253
|
+
| Index persistence | ✅ sidecar file, auto-managed | ✅ (rows in SQLite) | ✅ | ✅ |
|
|
254
|
+
| Ops surface | none | none | small | real (scaling, backups, upgrades) |
|
|
255
|
+
| Language | C++ core + Python | C + your language | Python-first | any (client libs) |
|
|
256
|
+
| Best fit | desktop / CLI / edge agent, local-first, privacy | you already live in SQLite and want vectors | Python RAG prototypes | multi-tenant, large-scale, networked |
|
|
257
|
+
|
|
258
|
+
**When to reach for something else:** if you need horizontal scale, multi-writer
|
|
259
|
+
concurrency, or sub-10 ms keyword search over millions of docs, a dedicated
|
|
260
|
+
vector DB is the right tool. If you want dense + BM25 + RRF + recency-aware
|
|
261
|
+
memory in one embeddable library with nothing to operate and instant startup,
|
|
262
|
+
that gap is what this engine fills — and the `hybrid` column above (1.000
|
|
263
|
+
recall, no server) is the argument for it.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.24)
|
|
2
|
+
project(sqlite_hybrid_search LANGUAGES CXX)
|
|
3
|
+
|
|
4
|
+
set(CMAKE_CXX_STANDARD 17)
|
|
5
|
+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
6
|
+
set(CMAKE_CXX_EXTENSIONS OFF)
|
|
7
|
+
|
|
8
|
+
# The static core library gets linked into the nanobind Python extension (a
|
|
9
|
+
# shared object), so all of its objects must be position-independent. ELF
|
|
10
|
+
# linkers reject non-PIC objects in a shared library; Mach-O does not, which
|
|
11
|
+
# is why this was invisible on macOS.
|
|
12
|
+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
|
13
|
+
|
|
14
|
+
option(RETRIEVAL_ENGINE_BUILD_TESTS "Build the C++ unit tests" ON)
|
|
15
|
+
# OFF by default so the plain C++ dev/test build (`cmake -B build &&
|
|
16
|
+
# cmake --build build`) is completely unaffected; turned ON by
|
|
17
|
+
# pyproject.toml's scikit-build-core config when building the Python
|
|
18
|
+
# extension (`pip install -e .`).
|
|
19
|
+
option(RETRIEVAL_ENGINE_BUILD_PYTHON_BINDINGS "Build the nanobind Python extension module" OFF)
|
|
20
|
+
|
|
21
|
+
include(FetchContent)
|
|
22
|
+
|
|
23
|
+
# --- SQLite3 --------------------------------------------------------------
|
|
24
|
+
# Homebrew's sqlite formula is keg-only (macOS ships its own, older sqlite),
|
|
25
|
+
# so it is NOT linked into /opt/homebrew/include or /opt/homebrew/lib by
|
|
26
|
+
# default. Help CMake find the Homebrew ARM64 build explicitly, per
|
|
27
|
+
# CLAUDE.md's M1 Architecture Compatibility directive.
|
|
28
|
+
find_program(HOMEBREW_EXECUTABLE brew)
|
|
29
|
+
if(HOMEBREW_EXECUTABLE)
|
|
30
|
+
execute_process(
|
|
31
|
+
COMMAND ${HOMEBREW_EXECUTABLE} --prefix sqlite
|
|
32
|
+
OUTPUT_VARIABLE HOMEBREW_SQLITE_PREFIX
|
|
33
|
+
OUTPUT_STRIP_TRAILING_WHITESPACE
|
|
34
|
+
ERROR_QUIET
|
|
35
|
+
)
|
|
36
|
+
# `brew --prefix sqlite` prints a path even when the formula is not
|
|
37
|
+
# installed, so only trust it if the library is actually there.
|
|
38
|
+
if(HOMEBREW_SQLITE_PREFIX AND EXISTS "${HOMEBREW_SQLITE_PREFIX}/lib")
|
|
39
|
+
list(APPEND CMAKE_PREFIX_PATH "${HOMEBREW_SQLITE_PREFIX}")
|
|
40
|
+
endif()
|
|
41
|
+
endif()
|
|
42
|
+
|
|
43
|
+
find_package(SQLite3 REQUIRED)
|
|
44
|
+
|
|
45
|
+
# --- usearch (header-only ANN index) ---------------------------------------
|
|
46
|
+
# Fetched via FetchContent rather than assuming a system/Homebrew install
|
|
47
|
+
# (usearch is not packaged in Homebrew).
|
|
48
|
+
FetchContent_Declare(
|
|
49
|
+
usearch
|
|
50
|
+
GIT_REPOSITORY https://github.com/unum-cloud/usearch.git
|
|
51
|
+
GIT_TAG v2.9.2
|
|
52
|
+
GIT_SHALLOW TRUE
|
|
53
|
+
# Only the `fp16` submodule: usearch's headers include <fp16/fp16.h> on
|
|
54
|
+
# targets without native _Float16 (x86_64 without AVX512 -- ARM and
|
|
55
|
+
# AVX512 use the compiler's own half type). `simsimd`/`stringzilla` are
|
|
56
|
+
# not referenced by the header subset this project uses.
|
|
57
|
+
GIT_SUBMODULES "fp16"
|
|
58
|
+
)
|
|
59
|
+
FetchContent_GetProperties(usearch)
|
|
60
|
+
if(NOT usearch_POPULATED)
|
|
61
|
+
# usearch's own CMakeLists.txt builds C/Python/etc. bindings we don't
|
|
62
|
+
# need -- we only want its header-only C++ API, so populate the sources
|
|
63
|
+
# without add_subdirectory()'ing it. CMP0169=OLD keeps the classic
|
|
64
|
+
# Populate() signature usable for that (superseded by MakeAvailable(),
|
|
65
|
+
# which always add_subdirectory()s when a CMakeLists.txt is present).
|
|
66
|
+
cmake_policy(SET CMP0169 OLD)
|
|
67
|
+
FetchContent_Populate(usearch)
|
|
68
|
+
|
|
69
|
+
# v2.9.2 has a genuine upstream bug that fails to compile under
|
|
70
|
+
# AppleClang: see cmake/patch_usearch_candidates_iterator_bug.cmake.
|
|
71
|
+
set(USEARCH_SOURCE_DIR "${usearch_SOURCE_DIR}")
|
|
72
|
+
include(cmake/patch_usearch_candidates_iterator_bug.cmake)
|
|
73
|
+
endif()
|
|
74
|
+
|
|
75
|
+
add_library(usearch INTERFACE)
|
|
76
|
+
add_library(usearch::usearch ALIAS usearch)
|
|
77
|
+
target_include_directories(usearch SYSTEM INTERFACE "${usearch_SOURCE_DIR}/include")
|
|
78
|
+
if(EXISTS "${usearch_SOURCE_DIR}/fp16/include")
|
|
79
|
+
target_include_directories(usearch SYSTEM INTERFACE "${usearch_SOURCE_DIR}/fp16/include")
|
|
80
|
+
endif()
|
|
81
|
+
|
|
82
|
+
# --- ONNX Runtime (built-in embedding backend) ---------------------------
|
|
83
|
+
# The real embed()/add_text()/search_text() backend runs a local ONNX
|
|
84
|
+
# sentence-embedding model (all-MiniLM-L6-v2 by default). ONNX Runtime is
|
|
85
|
+
# NOT header-only and NOT fetchable -- it must be installed
|
|
86
|
+
# (`brew install onnxruntime` on this macOS/ARM64 setup). The option
|
|
87
|
+
# defaults ON but degrades gracefully: if the library isn't found the
|
|
88
|
+
# engine still builds with only the dependency-free mock backend, and any
|
|
89
|
+
# attempt to load a .onnx model reports that clearly at runtime.
|
|
90
|
+
option(RETRIEVAL_ENGINE_WITH_ONNX "Build the ONNX Runtime embedding backend" ON)
|
|
91
|
+
|
|
92
|
+
if(RETRIEVAL_ENGINE_WITH_ONNX)
|
|
93
|
+
find_path(ONNXRUNTIME_INCLUDE_DIR onnxruntime_cxx_api.h
|
|
94
|
+
PATHS /opt/homebrew/include /opt/homebrew/include/onnxruntime /usr/local/include
|
|
95
|
+
/usr/local/include/onnxruntime /usr/include /usr/include/onnxruntime)
|
|
96
|
+
find_library(ONNXRUNTIME_LIBRARY NAMES onnxruntime
|
|
97
|
+
PATHS /opt/homebrew/lib /usr/local/lib /usr/lib)
|
|
98
|
+
|
|
99
|
+
if(ONNXRUNTIME_INCLUDE_DIR AND ONNXRUNTIME_LIBRARY)
|
|
100
|
+
add_library(onnxruntime::onnxruntime UNKNOWN IMPORTED)
|
|
101
|
+
set_target_properties(onnxruntime::onnxruntime PROPERTIES
|
|
102
|
+
IMPORTED_LOCATION "${ONNXRUNTIME_LIBRARY}"
|
|
103
|
+
INTERFACE_INCLUDE_DIRECTORIES "${ONNXRUNTIME_INCLUDE_DIR}")
|
|
104
|
+
message(STATUS "ONNX Runtime embedding backend: ON (${ONNXRUNTIME_LIBRARY})")
|
|
105
|
+
else()
|
|
106
|
+
message(WARNING
|
|
107
|
+
"ONNX Runtime not found (looked for onnxruntime_cxx_api.h and libonnxruntime); "
|
|
108
|
+
"building with the mock embedding backend only. To enable the real backend, install "
|
|
109
|
+
"it (macOS: `brew install onnxruntime`; Linux: extract an onnxruntime-linux-* release "
|
|
110
|
+
"into /usr/local, or pass -DONNXRUNTIME_INCLUDE_DIR= and -DONNXRUNTIME_LIBRARY=) and "
|
|
111
|
+
"reconfigure. See README.md.")
|
|
112
|
+
set(RETRIEVAL_ENGINE_WITH_ONNX OFF)
|
|
113
|
+
endif()
|
|
114
|
+
endif()
|
|
115
|
+
|
|
116
|
+
# --- GoogleTest (test framework) -------------------------------------------
|
|
117
|
+
if(RETRIEVAL_ENGINE_BUILD_TESTS)
|
|
118
|
+
FetchContent_Declare(
|
|
119
|
+
googletest
|
|
120
|
+
GIT_REPOSITORY https://github.com/google/googletest.git
|
|
121
|
+
GIT_TAG v1.18.0
|
|
122
|
+
GIT_SHALLOW TRUE
|
|
123
|
+
)
|
|
124
|
+
# Match the parent project's runtime library on Windows; irrelevant on
|
|
125
|
+
# macOS but harmless to set.
|
|
126
|
+
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
|
|
127
|
+
FetchContent_MakeAvailable(googletest)
|
|
128
|
+
|
|
129
|
+
enable_testing()
|
|
130
|
+
include(GoogleTest)
|
|
131
|
+
endif()
|
|
132
|
+
|
|
133
|
+
add_subdirectory(core)
|
|
134
|
+
|
|
135
|
+
# Native benchmark harness (benchmarks/). ON alongside the C++ tests by
|
|
136
|
+
# default; the Python packaging build turns both OFF (see pyproject.toml).
|
|
137
|
+
option(RETRIEVAL_ENGINE_BUILD_BENCHMARKS
|
|
138
|
+
"Build the native retrieval micro-benchmark (benchmarks/run_benchmarks)"
|
|
139
|
+
${RETRIEVAL_ENGINE_BUILD_TESTS})
|
|
140
|
+
if(RETRIEVAL_ENGINE_BUILD_BENCHMARKS)
|
|
141
|
+
add_subdirectory(benchmarks)
|
|
142
|
+
endif()
|
|
143
|
+
|
|
144
|
+
if(RETRIEVAL_ENGINE_BUILD_PYTHON_BINDINGS)
|
|
145
|
+
add_subdirectory(bindings)
|
|
146
|
+
endif()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashray Adhikari
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|