sqlite-hybrid-search 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. sqlite_hybrid_search-0.1.0/.github/workflows/release.yml +95 -0
  2. sqlite_hybrid_search-0.1.0/BENCHMARKS.md +263 -0
  3. sqlite_hybrid_search-0.1.0/CMakeLists.txt +146 -0
  4. sqlite_hybrid_search-0.1.0/LICENSE +21 -0
  5. sqlite_hybrid_search-0.1.0/PKG-INFO +298 -0
  6. sqlite_hybrid_search-0.1.0/README.md +271 -0
  7. sqlite_hybrid_search-0.1.0/benchmarks/CMakeLists.txt +8 -0
  8. sqlite_hybrid_search-0.1.0/benchmarks/corpus.py +101 -0
  9. sqlite_hybrid_search-0.1.0/benchmarks/harness.py +191 -0
  10. sqlite_hybrid_search-0.1.0/benchmarks/metrics.py +55 -0
  11. sqlite_hybrid_search-0.1.0/benchmarks/run_benchmarks.cpp +194 -0
  12. sqlite_hybrid_search-0.1.0/benchmarks/run_eval.py +139 -0
  13. sqlite_hybrid_search-0.1.0/benchmarks/test_eval.py +209 -0
  14. sqlite_hybrid_search-0.1.0/bindings/CMakeLists.txt +28 -0
  15. sqlite_hybrid_search-0.1.0/bindings/python_bindings.cpp +121 -0
  16. sqlite_hybrid_search-0.1.0/cmake/patch_usearch_candidates_iterator_bug.cmake +52 -0
  17. sqlite_hybrid_search-0.1.0/core/CMakeLists.txt +75 -0
  18. sqlite_hybrid_search-0.1.0/core/include/retrieval_engine/chunking.hpp +34 -0
  19. sqlite_hybrid_search-0.1.0/core/include/retrieval_engine/retrieval_engine.hpp +256 -0
  20. sqlite_hybrid_search-0.1.0/core/src/chunking.cpp +76 -0
  21. sqlite_hybrid_search-0.1.0/core/src/detail/chunk_repository.cpp +258 -0
  22. sqlite_hybrid_search-0.1.0/core/src/detail/chunk_repository.hpp +90 -0
  23. sqlite_hybrid_search-0.1.0/core/src/detail/chunk_store.cpp +230 -0
  24. sqlite_hybrid_search-0.1.0/core/src/detail/chunk_store.hpp +108 -0
  25. sqlite_hybrid_search-0.1.0/core/src/detail/dense_index.cpp +121 -0
  26. sqlite_hybrid_search-0.1.0/core/src/detail/dense_index.hpp +61 -0
  27. sqlite_hybrid_search-0.1.0/core/src/detail/embedding_model_loader.cpp +125 -0
  28. sqlite_hybrid_search-0.1.0/core/src/detail/embedding_model_loader.hpp +30 -0
  29. sqlite_hybrid_search-0.1.0/core/src/detail/fts5_query.cpp +38 -0
  30. sqlite_hybrid_search-0.1.0/core/src/detail/fts5_query.hpp +32 -0
  31. sqlite_hybrid_search-0.1.0/core/src/detail/mock_text_embedder.cpp +80 -0
  32. sqlite_hybrid_search-0.1.0/core/src/detail/mock_text_embedder.hpp +36 -0
  33. sqlite_hybrid_search-0.1.0/core/src/detail/onnx_text_embedder.cpp +132 -0
  34. sqlite_hybrid_search-0.1.0/core/src/detail/onnx_text_embedder.hpp +70 -0
  35. sqlite_hybrid_search-0.1.0/core/src/detail/recency_decay.cpp +40 -0
  36. sqlite_hybrid_search-0.1.0/core/src/detail/recency_decay.hpp +32 -0
  37. sqlite_hybrid_search-0.1.0/core/src/detail/rrf_fusion.cpp +67 -0
  38. sqlite_hybrid_search-0.1.0/core/src/detail/rrf_fusion.hpp +46 -0
  39. sqlite_hybrid_search-0.1.0/core/src/detail/sqlite_util.cpp +40 -0
  40. sqlite_hybrid_search-0.1.0/core/src/detail/sqlite_util.hpp +66 -0
  41. sqlite_hybrid_search-0.1.0/core/src/detail/text_embedder.hpp +43 -0
  42. sqlite_hybrid_search-0.1.0/core/src/detail/time_util.hpp +18 -0
  43. sqlite_hybrid_search-0.1.0/core/src/detail/usearch_util.hpp +23 -0
  44. sqlite_hybrid_search-0.1.0/core/src/detail/wordpiece_tokenizer.cpp +215 -0
  45. sqlite_hybrid_search-0.1.0/core/src/detail/wordpiece_tokenizer.hpp +61 -0
  46. sqlite_hybrid_search-0.1.0/core/src/retrieval_engine.cpp +175 -0
  47. sqlite_hybrid_search-0.1.0/core/tests/CMakeLists.txt +62 -0
  48. sqlite_hybrid_search-0.1.0/core/tests/test_builtin_embedder.cpp +241 -0
  49. sqlite_hybrid_search-0.1.0/core/tests/test_chunking_and_dense_retrieval.cpp +247 -0
  50. sqlite_hybrid_search-0.1.0/core/tests/test_hybrid_retrieval.cpp +241 -0
  51. sqlite_hybrid_search-0.1.0/core/tests/test_infra_sanity.cpp +59 -0
  52. sqlite_hybrid_search-0.1.0/core/tests/test_onnx_embedder.cpp +141 -0
  53. sqlite_hybrid_search-0.1.0/core/tests/test_persistence.cpp +202 -0
  54. sqlite_hybrid_search-0.1.0/core/tests/test_recency_decay_and_temporal_reranking.cpp +203 -0
  55. sqlite_hybrid_search-0.1.0/docs/DECISIONS.md +333 -0
  56. sqlite_hybrid_search-0.1.0/pyproject.toml +89 -0
  57. sqlite_hybrid_search-0.1.0/python/sqlite_hybrid_search/__init__.py +205 -0
  58. sqlite_hybrid_search-0.1.0/python/sqlite_hybrid_search/cli.py +279 -0
  59. sqlite_hybrid_search-0.1.0/tests/test_bindings_and_cli.py +147 -0
  60. sqlite_hybrid_search-0.1.0/tests/test_builtin_embedder.py +233 -0
  61. sqlite_hybrid_search-0.1.0/tests/test_memory_search.py +143 -0
  62. sqlite_hybrid_search-0.1.0/tests/test_persistence.py +48 -0
@@ -0,0 +1,95 @@
1
+ name: Release
2
+
3
+ # Builds platform wheels + an sdist and, on a published GitHub Release,
4
+ # uploads them to PyPI via Trusted Publishing (OIDC -- no API token).
5
+ #
6
+ # PyPI setup (one time): project -> Publishing -> add a pending publisher
7
+ # Owner: ashray-00
8
+ # Repository: sqlite-hybrid-search
9
+ # Workflow name: release.yml
10
+ # Environment name: pypi
11
+ # Then create the `pypi` environment under repo Settings -> Environments.
12
+
13
+ on:
14
+ release:
15
+ types: [published]
16
+ workflow_dispatch: {} # manual runs build artifacts but do not publish
17
+
18
+ permissions:
19
+ contents: read
20
+
21
+ jobs:
22
+ wheels:
23
+ name: wheels (${{ matrix.os }})
24
+ runs-on: ${{ matrix.os }}
25
+ strategy:
26
+ fail-fast: false
27
+ matrix:
28
+ # x86_64 Linux + Apple-Silicon macOS. GitHub retired the macos-13
29
+ # (Intel) hosted runners; Intel-Mac users build from the sdist.
30
+ os: [ubuntu-latest, macos-14]
31
+ steps:
32
+ - uses: actions/checkout@v4
33
+
34
+ - name: Build wheels
35
+ uses: pypa/cibuildwheel@v2.21.3 # bump to the latest tag periodically
36
+ env:
37
+ # cp39..cp313, 64-bit only; skip PyPy and musl for a first release.
38
+ CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*"
39
+ CIBW_SKIP: "*-musllinux_* *_i686 *-win32 pp*"
40
+ CIBW_ARCHS_MACOS: "arm64"
41
+ # The built-in ONNX Runtime backend is a source-build opt-in; PyPI
42
+ # wheels ship the dependency-free mock backend only.
43
+ CIBW_ENVIRONMENT_LINUX: 'CMAKE_ARGS="-DRETRIEVAL_ENGINE_WITH_ONNX=OFF"'
44
+ # Link the macOS SDK's system SQLite (present on every Mac, part of
45
+ # the OS) instead of a Homebrew build: delocate keeps system libs
46
+ # as plain dynamic deps, so the wheel stays clean and its
47
+ # deployment target is not dragged up to the runner's macOS.
48
+ CIBW_ENVIRONMENT_MACOS: >-
49
+ CMAKE_ARGS="-DRETRIEVAL_ENGINE_WITH_ONNX=OFF
50
+ -DSQLite3_INCLUDE_DIR=$(xcrun --show-sdk-path)/usr/include
51
+ -DSQLite3_LIBRARY=$(xcrun --show-sdk-path)/usr/lib/libsqlite3.tbd"
52
+ CIBW_BEFORE_ALL_LINUX: "yum install -y sqlite-devel || dnf install -y sqlite-devel"
53
+ # Smoke-test every wheel after auditwheel/delocate repairs it.
54
+ CIBW_TEST_COMMAND: >-
55
+ python -c "import sqlite_hybrid_search, tempfile, os;
56
+ e = sqlite_hybrid_search.Engine(os.path.join(tempfile.mkdtemp(), 'x.sqlite3'), dim=3);
57
+ e.add(documents=[{'id': 'a', 'text': 'hi'}], embeddings=[[1, 0, 0]]);
58
+ assert e.search([1, 0, 0], 1)[0]['document_id'] == 'a'"
59
+
60
+ - uses: actions/upload-artifact@v4
61
+ with:
62
+ name: wheels-${{ matrix.os }}
63
+ path: wheelhouse/*.whl
64
+
65
+ sdist:
66
+ name: sdist
67
+ runs-on: ubuntu-latest
68
+ steps:
69
+ - uses: actions/checkout@v4
70
+ - name: Install SQLite headers + library
71
+ run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
72
+ - run: pipx run build --sdist
73
+ - name: Verify the sdist builds from source
74
+ run: pip install dist/*.tar.gz
75
+ - uses: actions/upload-artifact@v4
76
+ with:
77
+ name: sdist
78
+ path: dist/*.tar.gz
79
+
80
+ publish:
81
+ name: publish to PyPI
82
+ needs: [wheels, sdist]
83
+ runs-on: ubuntu-latest
84
+ if: github.event_name == 'release'
85
+ environment: pypi # must match the PyPI trusted-publisher config
86
+ permissions:
87
+ id-token: write # OIDC token for Trusted Publishing
88
+ steps:
89
+ - uses: actions/download-artifact@v4
90
+ with:
91
+ path: dist
92
+ merge-multiple: true
93
+ - uses: pypa/gh-action-pypi-publish@release/v1
94
+ # with:
95
+ # repository-url: https://test.pypi.org/legacy/ # uncomment for TestPyPI
@@ -0,0 +1,263 @@
1
+ # Benchmarks
2
+
3
+ Reproducible measurement of the four retrieval paths this engine exposes,
4
+ across dataset sizes, on one machine. Generated by `benchmarks/run_eval.py`;
5
+ raw numbers in [`benchmarks/results.json`](benchmarks/results.json).
6
+
7
+ ## What was measured
8
+
9
+ | Approach | Call | What it is |
10
+ |---|---|---|
11
+ | `dense` | `search_dense()` | usearch HNSW cosine ANN only |
12
+ | `sparse` | `search_sparse()` | SQLite FTS5 BM25 only |
13
+ | `hybrid` | `search_hybrid()` | Reciprocal Rank Fusion (k=60) of dense + sparse |
14
+ | `hybrid_decay` | `search_memory()` | `hybrid` re-scored by exponential recency decay (λ = 0.1) |
15
+
16
+ - **Retrieval quality:** Recall@10, nDCG@10, MRR@10 over 100 labelled queries.
17
+ - **Latency:** warm-cache p50 / p95 / p99 and throughput; cold-cache first-query
18
+ latency; index **load**-on-open time (from the `.usearch` sidecar).
19
+ - **Footprint:** SQLite + sidecar size on disk, process peak RSS, RSS growth
20
+ during indexing, ingestion throughput.
21
+
22
+ ### Environment
23
+
24
+ Numbers below were measured on `macOS 26.6 arm64` (Apple Silicon), Python 3.12,
25
+ engine dim 64, single thread. Full C++/Python test suites green (`ctest`: 41,
26
+ `pytest`: 32). The harness is OS-independent; absolute timings will differ on
27
+ other hardware.
28
+
29
+ ### Reproduce
30
+
31
+ ```
32
+ cmake -B build && cmake --build build # Linux
33
+ # macOS: cmake -B build -DCMAKE_PREFIX_PATH=/opt/homebrew && cmake --build build
34
+ .venv/bin/python benchmarks/run_eval.py # 1k / 10k / 100k, ~1.5 min
35
+ ```
36
+
37
+ ### Honest caveat on the corpus
38
+
39
+ The corpus is **synthetic** (`benchmarks/corpus.py`): every relevant document
40
+ carries a unique rare entity token, and embeddings are a 64-dim hashed
41
+ bag-of-words, not a trained model. So:
42
+
43
+ - **Sparse/BM25 numbers are a best case** — the rare entity token is a perfect
44
+ exact-match signal, which is exactly the situation BM25 is built for.
45
+ - **Dense numbers are a worst case** — a 64-dim hashing-trick vector collides
46
+ badly as the corpus grows; a real embedding model (nomic, BGE-M3, MiniLM via
47
+ the built-in ONNX path) would not degrade like the `dense` column does here.
48
+ - **Latency and memory are model-independent** and transfer directly.
49
+
50
+ Read the quality section as *"how the fusion behaves when the two signals
51
+ disagree"*, not as an absolute quality score for dense retrieval.
52
+
53
+ ---
54
+
55
+ ## Retrieval quality
56
+
57
+ **Recall@10** — fraction of relevant docs found in the top 10:
58
+
59
+ | Approach | 1k | 10k | 100k |
60
+ |---|---|---|---|
61
+ | dense | 0.987 | 0.783 | 0.493 |
62
+ | sparse | 1.000 | 1.000 | 1.000 |
63
+ | **hybrid** | **1.000** | **1.000** | **1.000** |
64
+ | hybrid_decay | 1.000 | 1.000 | 1.000 |
65
+
66
+ **nDCG@10** — ranking quality:
67
+
68
+ | Approach | 1k | 10k | 100k |
69
+ |---|---|---|---|
70
+ | dense | 0.944 | 0.752 | 0.507 |
71
+ | sparse | 1.000 | 1.000 | 1.000 |
72
+ | hybrid | 0.992 | 0.969 | 0.951 |
73
+ | **hybrid_decay** | **0.998** | **0.998** | **0.997** |
74
+
75
+ **MRR@10:**
76
+
77
+ | Approach | 1k | 10k | 100k |
78
+ |---|---|---|---|
79
+ | dense | 0.978 | 0.870 | 0.673 |
80
+ | sparse | 1.000 | 1.000 | 1.000 |
81
+ | hybrid | 1.000 | 0.995 | 0.990 |
82
+ | hybrid_decay | 1.000 | 1.000 | 0.995 |
83
+
84
+ **Reading it:**
85
+
86
+ - **Hybrid recovers everything dense loses.** As the toy dense vectors collapse
87
+ (Recall 0.99 → 0.49), `hybrid` stays at **1.000** recall — RRF lets the sparse
88
+ side carry the query. This is the headline result: *fusion is strictly safer
89
+ than dense-only.*
90
+ - **When one signal is already perfect, fusing in a noisy one costs a little
91
+ ranking quality.** `sparse` nDCG is 1.000; `hybrid` nDCG is 0.951 at 100k,
92
+ because RRF interleaves dense's mistaken candidates. Fusion trades a few
93
+ points of top-rank precision for robustness.
94
+ - **Recency decay buys most of that back.** `hybrid_decay` nDCG is ~0.997 across
95
+ all sizes: the decay term breaks ties in favour of the (correctly recent)
96
+ relevant docs, pushing them back above dense's false positives. This is the
97
+ agent-memory case working as intended.
98
+
99
+ ---
100
+
101
+ ## Latency
102
+
103
+ **Warm-cache latency (ms), 100 queries × repeated passes:**
104
+
105
+ | Approach | 1k p50 | 1k p95 | 1k p99 | 10k p50 | 10k p95 | 10k p99 | 100k p50 | 100k p95 | 100k p99 |
106
+ |---|---|---|---|---|---|---|---|---|---|
107
+ | dense | 0.090 | 0.094 | 0.097 | 0.103 | 0.112 | 0.115 | 0.173 | 0.199 | 0.211 |
108
+ | sparse | 0.436 | 0.457 | 0.478 | 4.605 | 5.013 | 5.159 | 49.34 | 49.77 | 49.89 |
109
+ | hybrid | 0.524 | 0.541 | 0.558 | 4.765 | 5.059 | 5.297 | 49.44 | 50.13 | 50.50 |
110
+ | hybrid_decay | 0.729 | 0.790 | 0.818 | 6.517 | 7.350 | 7.630 | 70.06 | 83.28 | 85.93 |
111
+
112
+ **Throughput (queries/sec, single thread):**
113
+
114
+ | Approach | 1k | 10k | 100k |
115
+ |---|---|---|---|
116
+ | dense | 11048 | 9625 | 5705 |
117
+ | sparse | 2286 | 215 | 20 |
118
+ | hybrid | 1905 | 208 | 20 |
119
+ | hybrid_decay | 1371 | 155 | 14 |
120
+
121
+ - **Dense is sub-millisecond and near flat with scale** (0.09 → 0.17 ms p50 from
122
+ 1k to 100k) — usearch HNSW is doing its job.
123
+ - **Sparse/hybrid latency is dominated by FTS5** and grows roughly linearly with
124
+ corpus size: ~0.4 ms → ~49 ms p50. The OR-of-terms BM25 query scans large
125
+ posting lists for common tokens. This is the main query-time bottleneck.
126
+ Fusion now feeds BM25's top `min(k, 200)` rows into RRF — a ceiling that
127
+ bounds pathologically large `k` without touching ordinary retrieval, so at
128
+ the benchmark's `k = 10` sparse/hybrid latency is unchanged.
129
+ - **`hybrid_decay` adds a re-scoring pass** over the fused candidate set:
130
+ +40% p50 over `hybrid` at 100k (49.4 → 70.1 ms), and a wider p95/p99 tail.
131
+
132
+ **Index load on open.** The usearch graph is persisted to a `<db>.usearch`
133
+ sidecar and memory-loaded on the next open instead of being rebuilt from
134
+ SQLite. Startup is now effectively instant at every size:
135
+
136
+ | Dataset | Load from sidecar | (was: rebuild from SQLite) |
137
+ |---|---|---|
138
+ | 1k | 0.7 ms | 40 ms |
139
+ | 10k | 3.2 ms | 721 ms |
140
+ | 100k | **24.5 ms** | **14.6 s** |
141
+
142
+ The first open of a brand-new database still rebuilds (there is no sidecar
143
+ yet) and writes the sidecar; every open after that takes the load path. A
144
+ sidecar that is missing, truncated, or out of sync with the chunk table is
145
+ rejected in microseconds and the engine falls back to a rebuild.
146
+
147
+ **Cold vs warm query.** With the graph loaded from the sidecar, the first
148
+ query after an open is no slower than a warm one (100k `hybrid`: 49.3 ms cold
149
+ vs 49.4 ms warm p50; `dense`: 0.32 ms cold vs 0.17 ms warm).
150
+
151
+ ---
152
+
153
+ ## Memory & storage footprint
154
+
155
+ | Dataset | SQLite | `.usearch` sidecar | Ingest throughput | RSS growth during indexing | Process peak RSS |
156
+ |---|---|---|---|---|---|
157
+ | 1k | 0.57 MB | ~0.5 MB | 21695 docs/s | 3.5 MB | 35 MB |
158
+ | 10k | 5.21 MB | ~4 MB | 12826 docs/s | 11.1 MB | 79 MB |
159
+ | 100k | 51.98 MB | 40.5 MB | 6505 docs/s | 52.5 MB | 430 MB |
160
+
161
+ - **On-disk scales linearly.** SQLite is ~520 bytes/doc (chunk text + metadata +
162
+ FTS5 index + the vector blob kept for a rebuild); the sidecar adds the
163
+ serialised HNSW graph (~400 bytes/doc at dim 64). At 100k that is ~92 MB
164
+ total.
165
+ - **Ingestion throughput falls ~3.3× from 1k to 100k** — dual-writing SQLite,
166
+ FTS5, and usearch, with HNSW insertion getting more expensive as the graph
167
+ grows. The harness ingests in 5k-row batches and the sidecar is reserialised
168
+ after each one (20 writes at 100k); measured throughput (6505 docs/s) is
169
+ within run-to-run noise of the pre-persistence number (6482), but a workload
170
+ of many tiny batches would pay more, since each save rewrites the whole graph.
171
+ - **Peak RSS at 100k (430 MB) is inflated by the Python driver** holding the
172
+ whole synthetic corpus (100k × 64 floats as Python lists) in memory at once.
173
+ The native cross-check below is the honest engine-only number.
174
+
175
+ ### Native cross-check (`benchmarks/run_benchmarks.cpp`, no Python in the loop)
176
+
177
+ 20 000 random unit vectors, 200 queries:
178
+
179
+ | | value |
180
+ |---|---|
181
+ | SQLite on disk | 8.58 MB |
182
+ | Peak RSS (engine only) | 37.7 MB |
183
+ | Ingest throughput | 783 docs/s |
184
+ | dense p50 / p95 / p99 | 0.671 / 0.736 / 0.793 ms |
185
+ | hybrid p50 / p95 / p99 | 0.938 / 0.999 / 1.048 ms |
186
+
187
+ The engine's own peak RSS for 20k docs is **~38 MB**, versus the 79–430 MB the
188
+ Python-driven run reports — most of that difference is the driver, not the
189
+ engine. Ingest throughput here (783 docs/s) is a *worst case*: uniformly random
190
+ 64-dim vectors are the hardest input for HNSW (every candidate is roughly
191
+ equidistant), whereas the structured corpus in the Python run indexes 8–28×
192
+ faster. Real embeddings sit between the two.
193
+
194
+ ---
195
+
196
+ ## Honest tradeoffs & writeup
197
+
198
+ ### Where this engine wins
199
+
200
+ - **Exact entity / keyword matches.** The FTS5 BM25 side nails rare tokens
201
+ (names, IDs, error codes) — 1.000 recall at every size here — and RRF makes
202
+ `hybrid` inherit that for free. A pure vector store cannot do this without a
203
+ separate keyword index bolted on.
204
+ - **Fusion is strictly safer than dense-only.** `hybrid` never lost recall
205
+ relative to `dense` in any configuration; when the dense signal degraded, RRF
206
+ fell back on sparse automatically. You do not have to tune which retriever to
207
+ trust per query.
208
+ - **Recency-biased agent memory is built in.** `hybrid_decay` held nDCG ≈ 0.997
209
+ across all sizes by preferring recent relevant memories — the Mem0/Zep-style
210
+ capability, in-process, no service, one `decay_lambda` parameter.
211
+ - **Zero-dependency deployment.** One process, one SQLite file plus a
212
+ `.usearch` sidecar, an in-memory usearch index. No server to run, no
213
+ container, nothing listening on a port. Engine-only peak RSS is ~38 MB for
214
+ 20k docs.
215
+ - **Instant startup via the disk-backed index sidecar.** The HNSW graph is
216
+ serialised to `<db>.usearch` on write and memory-loaded on the next open:
217
+ **24.5 ms at 100k**, versus 14.6 s to rebuild it from SQLite. Cold start is
218
+ no longer a function of corpus size.
219
+ - **Dense retrieval is fast and scale-stable.** Sub-0.2 ms p50 at 100k, ~5700
220
+ q/s single-threaded.
221
+ - **SQLite stays authoritative.** The sidecar is a cache: if it is missing,
222
+ truncated, or its vector count disagrees with the chunk table, the engine
223
+ discards it and rebuilds — a corrupt index is never data loss.
224
+
225
+ ### Where it loses or bottlenecks
226
+
227
+ - **Sparse/hybrid query latency grows with corpus size** — ~49 ms p50 at 100k,
228
+ ~20 q/s. FTS5's BM25 over an OR of common terms is the cost. Dense stays
229
+ sub-millisecond; the fusion is only as fast as its slower half.
230
+ - **`hybrid_decay` has a fat tail.** The extra recency re-scoring pass pushes
231
+ 100k p99 to ~87 ms and throughput to ~14 q/s.
232
+ - **Dual-write ingestion overhead, plus sidecar reserialisation.** Every
233
+ document hits SQLite, FTS5, and usearch; throughput drops ~3.3× from 1k to
234
+ 100k, and HNSW insertion cost is sensitive to how clustered the embeddings
235
+ are (783 → 22k docs/s depending on input). Each `add_documents` call also
236
+ rewrites the whole `.usearch` sidecar — negligible for bulk ingest,
237
+ meaningful for a stream of one-document writes.
238
+ - **Single-threaded, not concurrency-safe.** `RetrievalEngine` must be confined
239
+ to one thread or externally locked; there is no query parallelism yet.
240
+ - **Small embedding dimensions collide.** dim 64 is fine for a demo; production
241
+ needs 384–1024, which raises the memory and index-build numbers accordingly.
242
+
243
+ ### DX comparison
244
+
245
+ | | This engine | sqlite-vec + glue | ChromaDB | Qdrant / Weaviate / Milvus |
246
+ |---|---|---|---|---|
247
+ | Deployment | in-process library, 1 file | in-process (SQLite ext) | embedded lib **or** server | separate server / cluster |
248
+ | Process to run | none | none | none (embedded) / one (server) | one+ (plus its deps) |
249
+ | Hybrid dense+sparse | built in (RRF) | DIY: wire up FTS5 + your own fusion | dense only (some keyword filtering) | built in (server-side) |
250
+ | Recency / memory semantics | built in (`search_memory`) | DIY | DIY | DIY (metadata + custom scoring) |
251
+ | Rerank / score trail | `search_explained()` | DIY | limited | varies |
252
+ | Persistence model | SQLite authoritative + `.usearch` sidecar | SQLite table | DuckDB/parquet | own storage engine |
253
+ | Index persistence | ✅ sidecar file, auto-managed | ✅ (rows in SQLite) | ✅ | ✅ |
254
+ | Ops surface | none | none | small | real (scaling, backups, upgrades) |
255
+ | Language | C++ core + Python | C + your language | Python-first | any (client libs) |
256
+ | Best fit | desktop / CLI / edge agent, local-first, privacy | you already live in SQLite and want vectors | Python RAG prototypes | multi-tenant, large-scale, networked |
257
+
258
+ **When to reach for something else:** if you need horizontal scale, multi-writer
259
+ concurrency, or sub-10 ms keyword search over millions of docs, a dedicated
260
+ vector DB is the right tool. If you want dense + BM25 + RRF + recency-aware
261
+ memory in one embeddable library with nothing to operate and instant startup,
262
+ that gap is what this engine fills — and the `hybrid` column above (1.000
263
+ recall, no server) is the argument for it.
@@ -0,0 +1,146 @@
1
+ cmake_minimum_required(VERSION 3.24)
2
+ project(sqlite_hybrid_search LANGUAGES CXX)
3
+
4
+ set(CMAKE_CXX_STANDARD 17)
5
+ set(CMAKE_CXX_STANDARD_REQUIRED ON)
6
+ set(CMAKE_CXX_EXTENSIONS OFF)
7
+
8
+ # The static core library gets linked into the nanobind Python extension (a
9
+ # shared object), so all of its objects must be position-independent. ELF
10
+ # linkers reject non-PIC objects in a shared library; Mach-O does not, which
11
+ # is why this was invisible on macOS.
12
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
13
+
14
+ option(RETRIEVAL_ENGINE_BUILD_TESTS "Build the C++ unit tests" ON)
15
+ # OFF by default so the plain C++ dev/test build (`cmake -B build &&
16
+ # cmake --build build`) is completely unaffected; turned ON by
17
+ # pyproject.toml's scikit-build-core config when building the Python
18
+ # extension (`pip install -e .`).
19
+ option(RETRIEVAL_ENGINE_BUILD_PYTHON_BINDINGS "Build the nanobind Python extension module" OFF)
20
+
21
+ include(FetchContent)
22
+
23
+ # --- SQLite3 --------------------------------------------------------------
24
+ # Homebrew's sqlite formula is keg-only (macOS ships its own, older sqlite),
25
+ # so it is NOT linked into /opt/homebrew/include or /opt/homebrew/lib by
26
+ # default. Help CMake find the Homebrew ARM64 build explicitly, per
27
+ # CLAUDE.md's M1 Architecture Compatibility directive.
28
+ find_program(HOMEBREW_EXECUTABLE brew)
29
+ if(HOMEBREW_EXECUTABLE)
30
+ execute_process(
31
+ COMMAND ${HOMEBREW_EXECUTABLE} --prefix sqlite
32
+ OUTPUT_VARIABLE HOMEBREW_SQLITE_PREFIX
33
+ OUTPUT_STRIP_TRAILING_WHITESPACE
34
+ ERROR_QUIET
35
+ )
36
+ # `brew --prefix sqlite` prints a path even when the formula is not
37
+ # installed, so only trust it if the library is actually there.
38
+ if(HOMEBREW_SQLITE_PREFIX AND EXISTS "${HOMEBREW_SQLITE_PREFIX}/lib")
39
+ list(APPEND CMAKE_PREFIX_PATH "${HOMEBREW_SQLITE_PREFIX}")
40
+ endif()
41
+ endif()
42
+
43
+ find_package(SQLite3 REQUIRED)
44
+
45
+ # --- usearch (header-only ANN index) ---------------------------------------
46
+ # Fetched via FetchContent rather than assuming a system/Homebrew install
47
+ # (usearch is not packaged in Homebrew).
48
+ FetchContent_Declare(
49
+ usearch
50
+ GIT_REPOSITORY https://github.com/unum-cloud/usearch.git
51
+ GIT_TAG v2.9.2
52
+ GIT_SHALLOW TRUE
53
+ # Only the `fp16` submodule: usearch's headers include <fp16/fp16.h> on
54
+ # targets without native _Float16 (x86_64 without AVX512 -- ARM and
55
+ # AVX512 use the compiler's own half type). `simsimd`/`stringzilla` are
56
+ # not referenced by the header subset this project uses.
57
+ GIT_SUBMODULES "fp16"
58
+ )
59
+ FetchContent_GetProperties(usearch)
60
+ if(NOT usearch_POPULATED)
61
+ # usearch's own CMakeLists.txt builds C/Python/etc. bindings we don't
62
+ # need -- we only want its header-only C++ API, so populate the sources
63
+ # without add_subdirectory()'ing it. CMP0169=OLD keeps the classic
64
+ # Populate() signature usable for that (superseded by MakeAvailable(),
65
+ # which always add_subdirectory()s when a CMakeLists.txt is present).
66
+ cmake_policy(SET CMP0169 OLD)
67
+ FetchContent_Populate(usearch)
68
+
69
+ # v2.9.2 has a genuine upstream bug that fails to compile under
70
+ # AppleClang: see cmake/patch_usearch_candidates_iterator_bug.cmake.
71
+ set(USEARCH_SOURCE_DIR "${usearch_SOURCE_DIR}")
72
+ include(cmake/patch_usearch_candidates_iterator_bug.cmake)
73
+ endif()
74
+
75
+ add_library(usearch INTERFACE)
76
+ add_library(usearch::usearch ALIAS usearch)
77
+ target_include_directories(usearch SYSTEM INTERFACE "${usearch_SOURCE_DIR}/include")
78
+ if(EXISTS "${usearch_SOURCE_DIR}/fp16/include")
79
+ target_include_directories(usearch SYSTEM INTERFACE "${usearch_SOURCE_DIR}/fp16/include")
80
+ endif()
81
+
82
+ # --- ONNX Runtime (built-in embedding backend) ---------------------------
83
+ # The real embed()/add_text()/search_text() backend runs a local ONNX
84
+ # sentence-embedding model (all-MiniLM-L6-v2 by default). ONNX Runtime is
85
+ # NOT header-only and NOT fetchable -- it must be installed
86
+ # (`brew install onnxruntime` on this macOS/ARM64 setup). The option
87
+ # defaults ON but degrades gracefully: if the library isn't found the
88
+ # engine still builds with only the dependency-free mock backend, and any
89
+ # attempt to load a .onnx model reports that clearly at runtime.
90
+ option(RETRIEVAL_ENGINE_WITH_ONNX "Build the ONNX Runtime embedding backend" ON)
91
+
92
+ if(RETRIEVAL_ENGINE_WITH_ONNX)
93
+ find_path(ONNXRUNTIME_INCLUDE_DIR onnxruntime_cxx_api.h
94
+ PATHS /opt/homebrew/include /opt/homebrew/include/onnxruntime /usr/local/include
95
+ /usr/local/include/onnxruntime /usr/include /usr/include/onnxruntime)
96
+ find_library(ONNXRUNTIME_LIBRARY NAMES onnxruntime
97
+ PATHS /opt/homebrew/lib /usr/local/lib /usr/lib)
98
+
99
+ if(ONNXRUNTIME_INCLUDE_DIR AND ONNXRUNTIME_LIBRARY)
100
+ add_library(onnxruntime::onnxruntime UNKNOWN IMPORTED)
101
+ set_target_properties(onnxruntime::onnxruntime PROPERTIES
102
+ IMPORTED_LOCATION "${ONNXRUNTIME_LIBRARY}"
103
+ INTERFACE_INCLUDE_DIRECTORIES "${ONNXRUNTIME_INCLUDE_DIR}")
104
+ message(STATUS "ONNX Runtime embedding backend: ON (${ONNXRUNTIME_LIBRARY})")
105
+ else()
106
+ message(WARNING
107
+ "ONNX Runtime not found (looked for onnxruntime_cxx_api.h and libonnxruntime); "
108
+ "building with the mock embedding backend only. To enable the real backend, install "
109
+ "it (macOS: `brew install onnxruntime`; Linux: extract an onnxruntime-linux-* release "
110
+ "into /usr/local, or pass -DONNXRUNTIME_INCLUDE_DIR= and -DONNXRUNTIME_LIBRARY=) and "
111
+ "reconfigure. See README.md.")
112
+ set(RETRIEVAL_ENGINE_WITH_ONNX OFF)
113
+ endif()
114
+ endif()
115
+
116
+ # --- GoogleTest (test framework) -------------------------------------------
117
+ if(RETRIEVAL_ENGINE_BUILD_TESTS)
118
+ FetchContent_Declare(
119
+ googletest
120
+ GIT_REPOSITORY https://github.com/google/googletest.git
121
+ GIT_TAG v1.18.0
122
+ GIT_SHALLOW TRUE
123
+ )
124
+ # Match the parent project's runtime library on Windows; irrelevant on
125
+ # macOS but harmless to set.
126
+ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
127
+ FetchContent_MakeAvailable(googletest)
128
+
129
+ enable_testing()
130
+ include(GoogleTest)
131
+ endif()
132
+
133
+ add_subdirectory(core)
134
+
135
+ # Native benchmark harness (benchmarks/). ON alongside the C++ tests by
136
+ # default; the Python packaging build turns both OFF (see pyproject.toml).
137
+ option(RETRIEVAL_ENGINE_BUILD_BENCHMARKS
138
+ "Build the native retrieval micro-benchmark (benchmarks/run_benchmarks)"
139
+ ${RETRIEVAL_ENGINE_BUILD_TESTS})
140
+ if(RETRIEVAL_ENGINE_BUILD_BENCHMARKS)
141
+ add_subdirectory(benchmarks)
142
+ endif()
143
+
144
+ if(RETRIEVAL_ENGINE_BUILD_PYTHON_BINDINGS)
145
+ add_subdirectory(bindings)
146
+ endif()
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ashray Adhikari
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.