lodedb 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lodedb-0.3.0 → lodedb-0.4.0}/PKG-INFO +75 -13
- {lodedb-0.3.0 → lodedb-0.4.0}/README.md +68 -12
- {lodedb-0.3.0 → lodedb-0.4.0}/pyproject.toml +20 -5
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/__init__.py +5 -1
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/core.py +329 -33
- lodedb-0.4.0/src/lodedb/engine/embedding_backends.py +570 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/gpu_turbovec.py +4 -1
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/route_profiles.py +18 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/turbovec_index.py +15 -7
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/wal_store.py +5 -5
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/graph/knowledge_graph.py +17 -4
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/__init__.py +9 -1
- lodedb-0.4.0/src/lodedb/local/backends.py +302 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/benchmark.py +40 -39
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/cli.py +37 -8
- lodedb-0.4.0/src/lodedb/local/collection.py +411 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/db.py +303 -17
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/doctor.py +74 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/__init__.py +6 -0
- lodedb-0.4.0/src/lodedb/local/integrations/privategpt.py +196 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/mcp_server.py +3 -0
- lodedb-0.4.0/src/lodedb/local/onnx_artifacts.py +381 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/presets.py +61 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/server.py +10 -6
- lodedb-0.3.0/src/lodedb/engine/embedding_backends.py +0 -152
- lodedb-0.3.0/src/lodedb/local/backends.py +0 -158
- {lodedb-0.3.0 → lodedb-0.4.0}/LICENSE +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/NOTICE +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/__main__.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/config.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/__init__.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_atomic_io.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_commit_manifest.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_filelock.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_filter_plan.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_lexical.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_predicate.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/document_text_store.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/index.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/lexical_index_store.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/mps_turbovec.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/route_registry.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/runtime_policy.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/state_journal_store.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/turbovec_delta_store.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/turbovec_resident.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/graph/__init__.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/graph/_store.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/langchain.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/llama_index.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/llama_index_graph.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/mem0.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/mcp_install.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/py.typed +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/Cargo.lock +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/Cargo.toml +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/Cargo.toml +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/README.md +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/build.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/examples/dump_state.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/examples/kernel_xtest.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/codebook.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/encode.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/error.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/id_map.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/io.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/lib.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/pack.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/rotation.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/search.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/codebook.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/concurrent_search.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/distortion.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/encode.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/encoded_rows.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/filtering.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/id_map.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/input_validation.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/io_versioning.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/kernel_correctness.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/lazy_init.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/reconstruction.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/rotation.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/state_sequences.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/swap_remove.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/tqplus_calibration.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/Cargo.toml +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/README.md +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/build.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/__init__.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/_dedup.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/_persist.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/agno.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/haystack.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/langchain.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/llama_index.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/src/lib.rs +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_agno.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_dedup.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_filtering.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_haystack.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_id_map.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_index.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_langchain.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_llama_index.py +0 -0
- {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_security.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lodedb
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
@@ -8,21 +8,27 @@ Classifier: Topic :: Database :: Database Engines/Servers
|
|
|
8
8
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
9
9
|
Requires-Dist: numpy>=2.0.0
|
|
10
10
|
Requires-Dist: typer>=0.12.0
|
|
11
|
+
Requires-Dist: onnxruntime>=1.20.0
|
|
12
|
+
Requires-Dist: transformers>=4.40.0
|
|
11
13
|
Requires-Dist: sentence-transformers>=3.0.0
|
|
12
14
|
Requires-Dist: pyyaml>=6.0.0
|
|
13
15
|
Requires-Dist: pytest>=8.3.0 ; extra == 'dev'
|
|
14
16
|
Requires-Dist: ruff==0.15.17 ; extra == 'dev'
|
|
15
17
|
Requires-Dist: cupy-cuda12x>=13.0.0 ; platform_system == 'Linux' and extra == 'gpu'
|
|
18
|
+
Requires-Dist: pillow>=10.0.0 ; extra == 'image'
|
|
16
19
|
Requires-Dist: langchain-core>=0.3.0 ; extra == 'langchain'
|
|
17
20
|
Requires-Dist: llama-index-core>=0.11.0 ; extra == 'llama-index'
|
|
18
21
|
Requires-Dist: mcp>=1.0.0 ; extra == 'mcp'
|
|
19
22
|
Requires-Dist: mem0ai>=2.0.0 ; extra == 'mem0'
|
|
23
|
+
Requires-Dist: optimum[exporters]>=1.23.0,<2.0.0 ; extra == 'onnx-export'
|
|
20
24
|
Provides-Extra: dev
|
|
21
25
|
Provides-Extra: gpu
|
|
26
|
+
Provides-Extra: image
|
|
22
27
|
Provides-Extra: langchain
|
|
23
28
|
Provides-Extra: llama-index
|
|
24
29
|
Provides-Extra: mcp
|
|
25
30
|
Provides-Extra: mem0
|
|
31
|
+
Provides-Extra: onnx-export
|
|
26
32
|
License-File: LICENSE
|
|
27
33
|
License-File: NOTICE
|
|
28
34
|
Summary: Local-first, privacy-by-default embedded vector database. Your data never leaves your machine.
|
|
@@ -52,9 +58,20 @@ LodeDB instead of its default store. Over 17.5k documents, per framework default
|
|
|
52
58
|
| vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
|
|
53
59
|
|---|---|---|---|
|
|
54
60
|
| On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
|
|
55
|
-
| Single-query p50 (CPU) | **~
|
|
56
|
-
| Batched retrieval, 64 (GPU) | **~
|
|
57
|
-
| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.
|
|
61
|
+
| Single-query p50 (CPU) | **~510× faster** (0.57 vs 294 ms) | **~610× faster** (0.57 vs 349 ms) | **~48× faster** (0.64 vs 31 ms) |
|
|
62
|
+
| Batched retrieval, 64 (GPU) | **~1,900×** (6,602 vs ~4 qps) | **~2,200×** (6,414 vs ~3 qps) | **~160×** (4,649 vs 29 qps) |
|
|
63
|
+
| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.6 vs 0.5 ms (both sub-ms) |
|
|
64
|
+
|
|
65
|
+
LodeDB matches sqlite-vec/qdrant's per-add latency with a significantly more compact footprint:
|
|
66
|
+
|
|
67
|
+
| **embedded stores** | **durable add p50** | **single-query p50** | **batch-64/query** | **memory footprint** |
|
|
68
|
+
| --- | ---: | ---: | ---: | ---: |
|
|
69
|
+
| sqlite-vec | **0.4 ms** | 25.3 ms | 25.1 ms | 101 MB |
|
|
70
|
+
| qdrant | **0.5 ms** | 27.3 ms | 27.0 ms | 85 MB |
|
|
71
|
+
| **LodeDB** | **0.6 ms** | **0.49 ms** | **0.11 ms** | **29 MB** |
|
|
72
|
+
| pgvector | 2.3 ms | 30.4 ms | 30.6 ms | 50 MB |
|
|
73
|
+
| lancedb | 3.2 ms | 10.3 ms | 10.0 ms | 37 MB |
|
|
74
|
+
| chroma | 6.5 ms | 3.0 ms | 3.1 ms | 151 MB |
|
|
58
75
|
|
|
59
76
|
[Full benchmark, all backends
|
|
60
77
|
(FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
|
|
@@ -76,10 +93,16 @@ incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
|
|
|
76
93
|
[How it works](#concurrency--durability).
|
|
77
94
|
- **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
|
|
78
95
|
(counts, bytes, latency), never raw payloads.
|
|
79
|
-
- **Local embeddings**:
|
|
80
|
-
-
|
|
81
|
-
|
|
82
|
-
|
|
96
|
+
- **Local embeddings**: ONNX Runtime by default (lower per-query latency), with a PyTorch
|
|
97
|
+
`sentence-transformers` fallback; runs on CPU, CUDA, or MPS. Pick with `embedding_runtime=`.
|
|
98
|
+
- **Multimodal**: index images and text in one shared CLIP space (`model="clip"`) for
|
|
99
|
+
cross-modal search, or bring your own vectors from any model.
|
|
100
|
+
[How it works](#multimodal--bring-your-own-vectors).
|
|
101
|
+
- **Batteries included**: a `lodedb` CLI, a loopback/private-network dev server, an
|
|
102
|
+
[MCP server](#use-as-an-mcp-server), LangChain, LlamaIndex, and mem0 adapters
|
|
103
|
+
(`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`), and a one-line
|
|
104
|
+
[PrivateGPT](https://github.com/zylon-ai/private-gpt) vector-store provider built on the
|
|
105
|
+
LlamaIndex adapter.
|
|
83
106
|
|
|
84
107
|
> 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
|
|
85
108
|
> available for commercial support, managed and at-scale serving, and on-prem / BYOC
|
|
@@ -97,7 +120,9 @@ the install with `lodedb doctor`. Optional extras:
|
|
|
97
120
|
|
|
98
121
|
```bash
|
|
99
122
|
pip install "lodedb[gpu]" # GPU-resident scan (Linux/CUDA)
|
|
123
|
+
pip install "lodedb[image]" # image + text (CLIP) embedding (model="clip")
|
|
100
124
|
pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
|
|
125
|
+
pip install "lodedb[onnx-export]" # export ONNX for a custom model (Optimum); presets need no export
|
|
101
126
|
```
|
|
102
127
|
|
|
103
128
|
Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
|
|
@@ -155,7 +180,7 @@ Run with `uv run` (e.g. `uv run lodedb doctor`).
|
|
|
155
180
|
```python
|
|
156
181
|
from lodedb import LodeDB
|
|
157
182
|
|
|
158
|
-
db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality)
|
|
183
|
+
db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality) | "clip" (image+text)
|
|
159
184
|
|
|
160
185
|
fox = db.add("the quick brown fox jumps", metadata={"topic": "animals"})
|
|
161
186
|
db.add("a lazy dog sleeps all day", metadata={"topic": "animals"})
|
|
@@ -179,8 +204,8 @@ db.persist() # durable .tvim/.tvd/.jsd snapshot; replays on reopen
|
|
|
179
204
|
|
|
180
205
|
Reopen with `LodeDB(path="./data")`; no migration step. Original text is kept in a
|
|
181
206
|
`.tvtext` sidecar for `db.get`; pass `store_text=False` to keep none. Presets are `minilm`
|
|
182
|
-
(384-dim)
|
|
183
|
-
[`examples/`](examples/).
|
|
207
|
+
(384-dim), `bge` (768-dim), and `clip` (512-dim, image+text), with weights pulled from Hugging
|
|
208
|
+
Face on first use. More in [`examples/`](examples/).
|
|
184
209
|
|
|
185
210
|
Need to read a store another process is writing to? Open it read-only. It takes no writer
|
|
186
211
|
lock, so it never blocks on (or is blocked by) the writer:
|
|
@@ -259,6 +284,39 @@ reopened.search("E1234", k=5, mode="hybrid") # works after reopen, rebuilt from
|
|
|
259
284
|
```
|
|
260
285
|
</details>
|
|
261
286
|
|
|
287
|
+
## Multimodal & bring-your-own vectors
|
|
288
|
+
|
|
289
|
+
The storage and scan are modality-agnostic: TurboVec stores any normalized float32
|
|
290
|
+
vector, so an image, audio, or video embedding is indexed and scanned exactly like a
|
|
291
|
+
text one. There are two ways to use that.
|
|
292
|
+
|
|
293
|
+
Bring your own vectors. Open a vector-only index at your dimension and pass the
|
|
294
|
+
embeddings you already computed with any model (CLIP, SigLIP, ImageBind, an audio or
|
|
295
|
+
video encoder, a hosted API). No embedding model is bundled on this path:
|
|
296
|
+
|
|
297
|
+
```python
|
|
298
|
+
db = LodeDB.open_vector_store("./media", vector_dim=512)
|
|
299
|
+
db.add_vectors(image_vector, id="img-001", metadata={"path": "photos/img-001.jpg"})
|
|
300
|
+
db.search_by_vector(query_vector, k=10)
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
Or use the built-in `clip` preset for image and text in one shared space, so a text
|
|
304
|
+
query retrieves images and an image query retrieves images and text. It rides the base
|
|
305
|
+
sentence-transformers stack and adds only Pillow (`pip install 'lodedb[image]'`):
|
|
306
|
+
|
|
307
|
+
```python
|
|
308
|
+
db = LodeDB("./gallery", model="clip") # downloads clip-ViT-B-32 on first use
|
|
309
|
+
db.add_image("photos/beach.jpg", metadata={"path": "photos/beach.jpg"})
|
|
310
|
+
db.search("a beach at sunset", k=5) # text -> image, cross-modal
|
|
311
|
+
db.search_by_image("photos/beach.jpg", k=5) # image -> image
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
The raw image is never stored; keep it on disk and put its path in `metadata`. Keep one
|
|
315
|
+
embedding model per index (scores are only comparable within one space); the model
|
|
316
|
+
identity is pinned and re-enforced on reopen. To hold several encoders side by side, use
|
|
317
|
+
`LodeCollection` named spaces, and pass `embedder=` to drive an index with your own
|
|
318
|
+
model. See [`docs/multimodal.md`](docs/multimodal.md).
|
|
319
|
+
|
|
262
320
|
## GPU-resident index
|
|
263
321
|
|
|
264
322
|
With the `[gpu]` extra on a CUDA host, LodeDB reconstructs the compact index into an fp16
|
|
@@ -342,7 +400,7 @@ and end-to-end single-query latency is 5.7 ms p50.
|
|
|
342
400
|
lodedb doctor # capability report: embedding / GPU / TurboVec backend
|
|
343
401
|
lodedb index ... # build / add to an on-disk index
|
|
344
402
|
lodedb query ... # search
|
|
345
|
-
lodedb serve #
|
|
403
|
+
lodedb serve # dev server (127.0.0.1 by default; private LAN only, no auth)
|
|
346
404
|
lodedb mcp # stdio MCP server for agent memory
|
|
347
405
|
lodedb benchmark # local, metrics-only benchmark
|
|
348
406
|
```
|
|
@@ -453,7 +511,11 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star
|
|
|
453
511
|
generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
|
|
454
512
|
`commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
|
|
455
513
|
publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
|
|
456
|
-
out-of-process readers must see each write the instant it commits.
|
|
514
|
+
out-of-process readers must see each write the instant it commits. Note `<key>.wal` is
|
|
515
|
+
**payload-bearing before a checkpoint** (raw text under `store_text=True`, otherwise embedding
|
|
516
|
+
deltas plus, with `index_text=True`, lexical tokens), so treat it as sensitively as the data it
|
|
517
|
+
indexes; `persist()`/`close()` checkpoint and truncate it, and `generation` mode keeps no WAL.
|
|
518
|
+
See the [payload boundary](docs/architecture.md#persistence--payload-boundary) docs.
|
|
457
519
|
- **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
|
|
458
520
|
|
|
459
521
|
## Limitations
|
|
@@ -14,9 +14,20 @@ LodeDB instead of its default store. Over 17.5k documents, per framework default
|
|
|
14
14
|
| vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
|
|
15
15
|
|---|---|---|---|
|
|
16
16
|
| On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
|
|
17
|
-
| Single-query p50 (CPU) | **~
|
|
18
|
-
| Batched retrieval, 64 (GPU) | **~
|
|
19
|
-
| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.
|
|
17
|
+
| Single-query p50 (CPU) | **~510× faster** (0.57 vs 294 ms) | **~610× faster** (0.57 vs 349 ms) | **~48× faster** (0.64 vs 31 ms) |
|
|
18
|
+
| Batched retrieval, 64 (GPU) | **~1,900×** (6,602 vs ~4 qps) | **~2,200×** (6,414 vs ~3 qps) | **~160×** (4,649 vs 29 qps) |
|
|
19
|
+
| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.6 vs 0.5 ms (both sub-ms) |
|
|
20
|
+
|
|
21
|
+
LodeDB matches sqlite-vec/qdrant's per-add latency with a significantly more compact footprint:
|
|
22
|
+
|
|
23
|
+
| **embedded stores** | **durable add p50** | **single-query p50** | **batch-64/query** | **memory footprint** |
|
|
24
|
+
| --- | ---: | ---: | ---: | ---: |
|
|
25
|
+
| sqlite-vec | **0.4 ms** | 25.3 ms | 25.1 ms | 101 MB |
|
|
26
|
+
| qdrant | **0.5 ms** | 27.3 ms | 27.0 ms | 85 MB |
|
|
27
|
+
| **LodeDB** | **0.6 ms** | **0.49 ms** | **0.11 ms** | **29 MB** |
|
|
28
|
+
| pgvector | 2.3 ms | 30.4 ms | 30.6 ms | 50 MB |
|
|
29
|
+
| lancedb | 3.2 ms | 10.3 ms | 10.0 ms | 37 MB |
|
|
30
|
+
| chroma | 6.5 ms | 3.0 ms | 3.1 ms | 151 MB |
|
|
20
31
|
|
|
21
32
|
[Full benchmark, all backends
|
|
22
33
|
(FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
|
|
@@ -38,10 +49,16 @@ incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
|
|
|
38
49
|
[How it works](#concurrency--durability).
|
|
39
50
|
- **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
|
|
40
51
|
(counts, bytes, latency), never raw payloads.
|
|
41
|
-
- **Local embeddings**:
|
|
42
|
-
-
|
|
43
|
-
|
|
44
|
-
|
|
52
|
+
- **Local embeddings**: ONNX Runtime by default (lower per-query latency), with a PyTorch
|
|
53
|
+
`sentence-transformers` fallback; runs on CPU, CUDA, or MPS. Pick with `embedding_runtime=`.
|
|
54
|
+
- **Multimodal**: index images and text in one shared CLIP space (`model="clip"`) for
|
|
55
|
+
cross-modal search, or bring your own vectors from any model.
|
|
56
|
+
[How it works](#multimodal--bring-your-own-vectors).
|
|
57
|
+
- **Batteries included**: a `lodedb` CLI, a loopback/private-network dev server, an
|
|
58
|
+
[MCP server](#use-as-an-mcp-server), LangChain, LlamaIndex, and mem0 adapters
|
|
59
|
+
(`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`), and a one-line
|
|
60
|
+
[PrivateGPT](https://github.com/zylon-ai/private-gpt) vector-store provider built on the
|
|
61
|
+
LlamaIndex adapter.
|
|
45
62
|
|
|
46
63
|
> 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
|
|
47
64
|
> available for commercial support, managed and at-scale serving, and on-prem / BYOC
|
|
@@ -59,7 +76,9 @@ the install with `lodedb doctor`. Optional extras:
|
|
|
59
76
|
|
|
60
77
|
```bash
|
|
61
78
|
pip install "lodedb[gpu]" # GPU-resident scan (Linux/CUDA)
|
|
79
|
+
pip install "lodedb[image]" # image + text (CLIP) embedding (model="clip")
|
|
62
80
|
pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
|
|
81
|
+
pip install "lodedb[onnx-export]" # export ONNX for a custom model (Optimum); presets need no export
|
|
63
82
|
```
|
|
64
83
|
|
|
65
84
|
Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
|
|
@@ -117,7 +136,7 @@ Run with `uv run` (e.g. `uv run lodedb doctor`).
|
|
|
117
136
|
```python
|
|
118
137
|
from lodedb import LodeDB
|
|
119
138
|
|
|
120
|
-
db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality)
|
|
139
|
+
db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality) | "clip" (image+text)
|
|
121
140
|
|
|
122
141
|
fox = db.add("the quick brown fox jumps", metadata={"topic": "animals"})
|
|
123
142
|
db.add("a lazy dog sleeps all day", metadata={"topic": "animals"})
|
|
@@ -141,8 +160,8 @@ db.persist() # durable .tvim/.tvd/.jsd snapshot; replays on reopen
|
|
|
141
160
|
|
|
142
161
|
Reopen with `LodeDB(path="./data")`; no migration step. Original text is kept in a
|
|
143
162
|
`.tvtext` sidecar for `db.get`; pass `store_text=False` to keep none. Presets are `minilm`
|
|
144
|
-
(384-dim)
|
|
145
|
-
[`examples/`](examples/).
|
|
163
|
+
(384-dim), `bge` (768-dim), and `clip` (512-dim, image+text), with weights pulled from Hugging
|
|
164
|
+
Face on first use. More in [`examples/`](examples/).
|
|
146
165
|
|
|
147
166
|
Need to read a store another process is writing to? Open it read-only. It takes no writer
|
|
148
167
|
lock, so it never blocks on (or is blocked by) the writer:
|
|
@@ -221,6 +240,39 @@ reopened.search("E1234", k=5, mode="hybrid") # works after reopen, rebuilt from
|
|
|
221
240
|
```
|
|
222
241
|
</details>
|
|
223
242
|
|
|
243
|
+
## Multimodal & bring-your-own vectors
|
|
244
|
+
|
|
245
|
+
The storage and scan are modality-agnostic: TurboVec stores any normalized float32
|
|
246
|
+
vector, so an image, audio, or video embedding is indexed and scanned exactly like a
|
|
247
|
+
text one. There are two ways to use that.
|
|
248
|
+
|
|
249
|
+
Bring your own vectors. Open a vector-only index at your dimension and pass the
|
|
250
|
+
embeddings you already computed with any model (CLIP, SigLIP, ImageBind, an audio or
|
|
251
|
+
video encoder, a hosted API). No embedding model is bundled on this path:
|
|
252
|
+
|
|
253
|
+
```python
|
|
254
|
+
db = LodeDB.open_vector_store("./media", vector_dim=512)
|
|
255
|
+
db.add_vectors(image_vector, id="img-001", metadata={"path": "photos/img-001.jpg"})
|
|
256
|
+
db.search_by_vector(query_vector, k=10)
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Or use the built-in `clip` preset for image and text in one shared space, so a text
|
|
260
|
+
query retrieves images and an image query retrieves images and text. It rides the base
|
|
261
|
+
sentence-transformers stack and adds only Pillow (`pip install 'lodedb[image]'`):
|
|
262
|
+
|
|
263
|
+
```python
|
|
264
|
+
db = LodeDB("./gallery", model="clip") # downloads clip-ViT-B-32 on first use
|
|
265
|
+
db.add_image("photos/beach.jpg", metadata={"path": "photos/beach.jpg"})
|
|
266
|
+
db.search("a beach at sunset", k=5) # text -> image, cross-modal
|
|
267
|
+
db.search_by_image("photos/beach.jpg", k=5) # image -> image
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
The raw image is never stored; keep it on disk and put its path in `metadata`. Keep one
|
|
271
|
+
embedding model per index (scores are only comparable within one space); the model
|
|
272
|
+
identity is pinned and re-enforced on reopen. To hold several encoders side by side, use
|
|
273
|
+
`LodeCollection` named spaces, and pass `embedder=` to drive an index with your own
|
|
274
|
+
model. See [`docs/multimodal.md`](docs/multimodal.md).
|
|
275
|
+
|
|
224
276
|
## GPU-resident index
|
|
225
277
|
|
|
226
278
|
With the `[gpu]` extra on a CUDA host, LodeDB reconstructs the compact index into an fp16
|
|
@@ -304,7 +356,7 @@ and end-to-end single-query latency is 5.7 ms p50.
|
|
|
304
356
|
lodedb doctor # capability report: embedding / GPU / TurboVec backend
|
|
305
357
|
lodedb index ... # build / add to an on-disk index
|
|
306
358
|
lodedb query ... # search
|
|
307
|
-
lodedb serve #
|
|
359
|
+
lodedb serve # dev server (127.0.0.1 by default; private LAN only, no auth)
|
|
308
360
|
lodedb mcp # stdio MCP server for agent memory
|
|
309
361
|
lodedb benchmark # local, metrics-only benchmark
|
|
310
362
|
```
|
|
@@ -415,7 +467,11 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star
|
|
|
415
467
|
generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
|
|
416
468
|
`commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
|
|
417
469
|
publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
|
|
418
|
-
out-of-process readers must see each write the instant it commits.
|
|
470
|
+
out-of-process readers must see each write the instant it commits. Note `<key>.wal` is
|
|
471
|
+
**payload-bearing before a checkpoint** (raw text under `store_text=True`, otherwise embedding
|
|
472
|
+
deltas plus, with `index_text=True`, lexical tokens), so treat it as sensitively as the data it
|
|
473
|
+
indexes; `persist()`/`close()` checkpoint and truncate it, and `generation` mode keeps no WAL.
|
|
474
|
+
See the [payload boundary](docs/architecture.md#persistence--payload-boundary) docs.
|
|
419
475
|
- **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
|
|
420
476
|
|
|
421
477
|
## Limitations
|
|
@@ -5,7 +5,7 @@ build-backend = "maturin"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "lodedb"
|
|
7
7
|
# Static (maturin reads it here); keep in sync with src/lodedb/__init__.py __version__.
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.4.0"
|
|
9
9
|
description = "Local-first, privacy-by-default embedded vector database. Your data never leaves your machine."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.11"
|
|
@@ -33,13 +33,19 @@ classifiers = [
|
|
|
33
33
|
# Lean runtime set (see docs/architecture.md). The compact TurboVec CPU core +
|
|
34
34
|
# .tvim/.tvd persistence is the bundled `lodedb._turbovec` extension (compiled
|
|
35
35
|
# into the wheel, see [tool.maturin]), not a PyPI dependency. numpy = embedding/
|
|
36
|
-
# scan I/O; typer = CLI;
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
36
|
+
# scan I/O; typer = CLI; pyyaml = loads the bundled route registry.
|
|
37
|
+
# Embedding runtime: onnxruntime is the default (faster feature extraction; the
|
|
38
|
+
# preset models ship a prebuilt ONNX on the Hub, fetched on first use) with
|
|
39
|
+
# transformers for tokenization; sentence-transformers (which pulls torch) is the
|
|
40
|
+
# fallback when ONNX cannot be set up. Exporting ONNX for a model without a prebuilt
|
|
41
|
+
# graph is the opt-in [onnx-export] extra (Optimum), not part of the base install.
|
|
42
|
+
# Heavy research deps (faiss/modal/mteb/datasets/matplotlib/sklearn) are
|
|
43
|
+
# deliberately NOT here and are guarded out by tests/test_import_boundary.py.
|
|
40
44
|
dependencies = [
|
|
41
45
|
"numpy>=2.0.0",
|
|
42
46
|
"typer>=0.12.0",
|
|
47
|
+
"onnxruntime>=1.20.0",
|
|
48
|
+
"transformers>=4.40.0",
|
|
43
49
|
"sentence-transformers>=3.0.0",
|
|
44
50
|
"pyyaml>=6.0.0",
|
|
45
51
|
]
|
|
@@ -48,6 +54,10 @@ dependencies = [
|
|
|
48
54
|
# Opt-in GPU-resident exact scan on CUDA hosts (gpu_turbovec); absent CUDA, LodeDB uses
|
|
49
55
|
# the CPU scan. cupy wheels are Linux/CUDA-only, so this resolves to nothing elsewhere.
|
|
50
56
|
gpu = ["cupy-cuda12x>=13.0.0; platform_system == 'Linux'"]
|
|
57
|
+
# Export an ONNX graph for a model that does not ship a prebuilt one on the Hub
|
|
58
|
+
# (the built-in minilm/bge presets do, so this is only needed for custom models).
|
|
59
|
+
# Optimum runs the export in a subprocess, so it never imports into the lodedb process.
|
|
60
|
+
onnx-export = ["optimum[exporters]>=1.23.0,<2.0.0"]
|
|
51
61
|
# Optional stdio MCP server so coding agents can use LodeDB as local memory.
|
|
52
62
|
mcp = ["mcp>=1.0.0"]
|
|
53
63
|
# Optional LangChain VectorStore adapter.
|
|
@@ -56,6 +66,11 @@ langchain = ["langchain-core>=0.3.0"]
|
|
|
56
66
|
llama-index = ["llama-index-core>=0.11.0"]
|
|
57
67
|
# Optional mem0 vector-store adapter.
|
|
58
68
|
mem0 = ["mem0ai>=2.0.0"]
|
|
69
|
+
# Optional image + text (CLIP) embedding via the "clip" preset / add_image. The CLIP
|
|
70
|
+
# model runs on the existing sentence-transformers + torch stack (already in the base
|
|
71
|
+
# deps), so this adds only Pillow for decoding image files. Both are imported lazily,
|
|
72
|
+
# so a plain `import lodedb` pulls neither (guarded by tests/test_import_boundary.py).
|
|
73
|
+
image = ["pillow>=10.0.0"]
|
|
59
74
|
# ruff is pinned (not floored) so `ruff format`/`ruff check` are reproducible across
|
|
60
75
|
# contributors and CI — the formatter's output can change between ruff releases.
|
|
61
76
|
dev = ["pytest>=8.3.0", "ruff==0.15.17"]
|
|
@@ -11,6 +11,8 @@ from __future__ import annotations
|
|
|
11
11
|
from lodedb.local import (
|
|
12
12
|
LOCAL_MODEL_PRESETS,
|
|
13
13
|
ConcurrentWriterError,
|
|
14
|
+
ImageEmbeddingUnsupportedError,
|
|
15
|
+
LodeCollection,
|
|
14
16
|
LodeDB,
|
|
15
17
|
LodeSearchHit,
|
|
16
18
|
ReadOnlyError,
|
|
@@ -19,11 +21,13 @@ from lodedb.local import (
|
|
|
19
21
|
from lodedb.local.cli import app, main
|
|
20
22
|
|
|
21
23
|
# Keep in sync with `version` in pyproject.toml (the release workflow asserts they match).
|
|
22
|
-
__version__ = "0.
|
|
24
|
+
__version__ = "0.4.0"
|
|
23
25
|
|
|
24
26
|
__all__ = [
|
|
25
27
|
"LOCAL_MODEL_PRESETS",
|
|
26
28
|
"ConcurrentWriterError",
|
|
29
|
+
"ImageEmbeddingUnsupportedError",
|
|
30
|
+
"LodeCollection",
|
|
27
31
|
"LodeDB",
|
|
28
32
|
"LodeSearchHit",
|
|
29
33
|
"ReadOnlyError",
|