lodedb 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. {lodedb-0.3.0 → lodedb-0.4.0}/PKG-INFO +75 -13
  2. {lodedb-0.3.0 → lodedb-0.4.0}/README.md +68 -12
  3. {lodedb-0.3.0 → lodedb-0.4.0}/pyproject.toml +20 -5
  4. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/__init__.py +5 -1
  5. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/core.py +329 -33
  6. lodedb-0.4.0/src/lodedb/engine/embedding_backends.py +570 -0
  7. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/gpu_turbovec.py +4 -1
  8. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/route_profiles.py +18 -0
  9. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/turbovec_index.py +15 -7
  10. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/wal_store.py +5 -5
  11. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/graph/knowledge_graph.py +17 -4
  12. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/__init__.py +9 -1
  13. lodedb-0.4.0/src/lodedb/local/backends.py +302 -0
  14. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/benchmark.py +40 -39
  15. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/cli.py +37 -8
  16. lodedb-0.4.0/src/lodedb/local/collection.py +411 -0
  17. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/db.py +303 -17
  18. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/doctor.py +74 -0
  19. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/__init__.py +6 -0
  20. lodedb-0.4.0/src/lodedb/local/integrations/privategpt.py +196 -0
  21. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/mcp_server.py +3 -0
  22. lodedb-0.4.0/src/lodedb/local/onnx_artifacts.py +381 -0
  23. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/presets.py +61 -0
  24. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/server.py +10 -6
  25. lodedb-0.3.0/src/lodedb/engine/embedding_backends.py +0 -152
  26. lodedb-0.3.0/src/lodedb/local/backends.py +0 -158
  27. {lodedb-0.3.0 → lodedb-0.4.0}/LICENSE +0 -0
  28. {lodedb-0.3.0 → lodedb-0.4.0}/NOTICE +0 -0
  29. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/__main__.py +0 -0
  30. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/config.py +0 -0
  31. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/__init__.py +0 -0
  32. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_atomic_io.py +0 -0
  33. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_commit_manifest.py +0 -0
  34. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_filelock.py +0 -0
  35. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_filter_plan.py +0 -0
  36. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_lexical.py +0 -0
  37. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/_predicate.py +0 -0
  38. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/document_text_store.py +0 -0
  39. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/index.py +0 -0
  40. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/lexical_index_store.py +0 -0
  41. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/mps_turbovec.py +0 -0
  42. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/route_registry.py +0 -0
  43. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/runtime_policy.py +0 -0
  44. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/state_journal_store.py +0 -0
  45. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/turbovec_delta_store.py +0 -0
  46. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/engine/turbovec_resident.py +0 -0
  47. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/graph/__init__.py +0 -0
  48. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/graph/_store.py +0 -0
  49. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/langchain.py +0 -0
  50. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/llama_index.py +0 -0
  51. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/llama_index_graph.py +0 -0
  52. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/integrations/mem0.py +0 -0
  53. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/local/mcp_install.py +0 -0
  54. {lodedb-0.3.0 → lodedb-0.4.0}/src/lodedb/py.typed +0 -0
  55. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/Cargo.lock +0 -0
  56. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/Cargo.toml +0 -0
  57. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/Cargo.toml +0 -0
  58. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/README.md +0 -0
  59. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/build.rs +0 -0
  60. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/examples/dump_state.rs +0 -0
  61. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/examples/kernel_xtest.rs +0 -0
  62. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/codebook.rs +0 -0
  63. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/encode.rs +0 -0
  64. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/error.rs +0 -0
  65. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/id_map.rs +0 -0
  66. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/io.rs +0 -0
  67. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/lib.rs +0 -0
  68. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/pack.rs +0 -0
  69. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/rotation.rs +0 -0
  70. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/src/search.rs +0 -0
  71. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/codebook.rs +0 -0
  72. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/concurrent_search.rs +0 -0
  73. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/distortion.rs +0 -0
  74. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/encode.rs +0 -0
  75. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/encoded_rows.rs +0 -0
  76. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/filtering.rs +0 -0
  77. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/id_map.rs +0 -0
  78. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/input_validation.rs +0 -0
  79. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/io_versioning.rs +0 -0
  80. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/kernel_correctness.rs +0 -0
  81. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/lazy_init.rs +0 -0
  82. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/reconstruction.rs +0 -0
  83. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/rotation.rs +0 -0
  84. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/state_sequences.rs +0 -0
  85. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/swap_remove.rs +0 -0
  86. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec/tests/tqplus_calibration.rs +0 -0
  87. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/Cargo.toml +0 -0
  88. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/README.md +0 -0
  89. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/build.rs +0 -0
  90. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/__init__.py +0 -0
  91. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/_dedup.py +0 -0
  92. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/_persist.py +0 -0
  93. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/agno.py +0 -0
  94. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/haystack.py +0 -0
  95. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/langchain.py +0 -0
  96. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/python/turbovec/llama_index.py +0 -0
  97. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/src/lib.rs +0 -0
  98. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_agno.py +0 -0
  99. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_dedup.py +0 -0
  100. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_filtering.py +0 -0
  101. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_haystack.py +0 -0
  102. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_id_map.py +0 -0
  103. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_index.py +0 -0
  104. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_langchain.py +0 -0
  105. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_llama_index.py +0 -0
  106. {lodedb-0.3.0 → lodedb-0.4.0}/third_party/turbovec/turbovec-python/tests/test_security.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lodedb
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Programming Language :: Python :: 3 :: Only
@@ -8,21 +8,27 @@ Classifier: Topic :: Database :: Database Engines/Servers
8
8
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
9
9
  Requires-Dist: numpy>=2.0.0
10
10
  Requires-Dist: typer>=0.12.0
11
+ Requires-Dist: onnxruntime>=1.20.0
12
+ Requires-Dist: transformers>=4.40.0
11
13
  Requires-Dist: sentence-transformers>=3.0.0
12
14
  Requires-Dist: pyyaml>=6.0.0
13
15
  Requires-Dist: pytest>=8.3.0 ; extra == 'dev'
14
16
  Requires-Dist: ruff==0.15.17 ; extra == 'dev'
15
17
  Requires-Dist: cupy-cuda12x>=13.0.0 ; platform_system == 'Linux' and extra == 'gpu'
18
+ Requires-Dist: pillow>=10.0.0 ; extra == 'image'
16
19
  Requires-Dist: langchain-core>=0.3.0 ; extra == 'langchain'
17
20
  Requires-Dist: llama-index-core>=0.11.0 ; extra == 'llama-index'
18
21
  Requires-Dist: mcp>=1.0.0 ; extra == 'mcp'
19
22
  Requires-Dist: mem0ai>=2.0.0 ; extra == 'mem0'
23
+ Requires-Dist: optimum[exporters]>=1.23.0,<2.0.0 ; extra == 'onnx-export'
20
24
  Provides-Extra: dev
21
25
  Provides-Extra: gpu
26
+ Provides-Extra: image
22
27
  Provides-Extra: langchain
23
28
  Provides-Extra: llama-index
24
29
  Provides-Extra: mcp
25
30
  Provides-Extra: mem0
31
+ Provides-Extra: onnx-export
26
32
  License-File: LICENSE
27
33
  License-File: NOTICE
28
34
  Summary: Local-first, privacy-by-default embedded vector database. Your data never leaves your machine.
@@ -52,9 +58,20 @@ LodeDB instead of its default store. Over 17.5k documents, per framework default
52
58
  | vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
53
59
  |---|---|---|---|
54
60
  | On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
55
- | Single-query p50 (CPU) | **~410× faster** (0.8 vs 345 ms) | **~500× faster** (0.9 vs 427 ms) | **~25× faster** (0.9 vs 23 ms) |
56
- | Batched retrieval, 64 (GPU) | **~2,000×** (6,280 vs ~3 qps) | **~2,800×** (5,744 vs ~2 qps) | **~62×** (2,686 vs 43 qps) |
57
- | Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.9 vs 0.5 ms (both sub-ms) |
61
+ | Single-query p50 (CPU) | **~510× faster** (0.57 vs 294 ms) | **~610× faster** (0.57 vs 349 ms) | **~48× faster** (0.64 vs 31 ms) |
62
+ | Batched retrieval, 64 (GPU) | **~1,900×** (6,602 vs ~4 qps) | **~2,200×** (6,414 vs ~3 qps) | **~160×** (4,649 vs 29 qps) |
63
+ | Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.6 vs 0.5 ms (both sub-ms) |
64
+
65
+ LodeDB matches sqlite-vec/qdrant's per-add latency with a significantly more compact footprint:
66
+
67
+ | **embedded stores** | **durable add p50** | **single-query p50** | **batch-64/query** | **memory footprint** |
68
+ | --- | ---: | ---: | ---: | ---: |
69
+ | sqlite-vec | **0.4 ms** | 25.3 ms | 25.1 ms | 101 MB |
70
+ | qdrant | **0.5 ms** | 27.3 ms | 27.0 ms | 85 MB |
71
+ | **LodeDB** | **0.6 ms** | **0.49 ms** | **0.11 ms** | **29 MB** |
72
+ | pgvector | 2.3 ms | 30.4 ms | 30.6 ms | 50 MB |
73
+ | lancedb | 3.2 ms | 10.3 ms | 10.0 ms | 37 MB |
74
+ | chroma | 6.5 ms | 3.0 ms | 3.1 ms | 151 MB |
58
75
 
59
76
  [Full benchmark, all backends
60
77
  (FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
@@ -76,10 +93,16 @@ incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
76
93
  [How it works](#concurrency--durability).
77
94
  - **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
78
95
  (counts, bytes, latency), never raw payloads.
79
- - **Local embeddings**: `sentence-transformers` on CUDA, MPS, or CPU.
80
- - **Batteries included**: a `lodedb` CLI, a loopback dev server, an
81
- [MCP server](#use-as-an-mcp-server), and LangChain, LlamaIndex, and mem0 adapters
82
- (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`).
96
+ - **Local embeddings**: ONNX Runtime by default (lower per-query latency), with a PyTorch
97
+ `sentence-transformers` fallback; runs on CPU, CUDA, or MPS. Pick with `embedding_runtime=`.
98
+ - **Multimodal**: index images and text in one shared CLIP space (`model="clip"`) for
99
+ cross-modal search, or bring your own vectors from any model.
100
+ [How it works](#multimodal--bring-your-own-vectors).
101
+ - **Batteries included**: a `lodedb` CLI, a loopback/private-network dev server, an
102
+ [MCP server](#use-as-an-mcp-server), LangChain, LlamaIndex, and mem0 adapters
103
+ (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`), and a one-line
104
+ [PrivateGPT](https://github.com/zylon-ai/private-gpt) vector-store provider built on the
105
+ LlamaIndex adapter.
83
106
 
84
107
  > 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
85
108
  > available for commercial support, managed and at-scale serving, and on-prem / BYOC
@@ -97,7 +120,9 @@ the install with `lodedb doctor`. Optional extras:
97
120
 
98
121
  ```bash
99
122
  pip install "lodedb[gpu]" # GPU-resident scan (Linux/CUDA)
123
+ pip install "lodedb[image]" # image + text (CLIP) embedding (model="clip")
100
124
  pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
125
+ pip install "lodedb[onnx-export]" # export ONNX for a custom model (Optimum); presets need no export
101
126
  ```
102
127
 
103
128
  Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
@@ -155,7 +180,7 @@ Run with `uv run` (e.g. `uv run lodedb doctor`).
155
180
  ```python
156
181
  from lodedb import LodeDB
157
182
 
158
- db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality)
183
+ db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality) | "clip" (image+text)
159
184
 
160
185
  fox = db.add("the quick brown fox jumps", metadata={"topic": "animals"})
161
186
  db.add("a lazy dog sleeps all day", metadata={"topic": "animals"})
@@ -179,8 +204,8 @@ db.persist() # durable .tvim/.tvd/.jsd snapshot; replays on reopen
179
204
 
180
205
  Reopen with `LodeDB(path="./data")`; no migration step. Original text is kept in a
181
206
  `.tvtext` sidecar for `db.get`; pass `store_text=False` to keep none. Presets are `minilm`
182
- (384-dim) and `bge` (768-dim), with weights pulled from Hugging Face on first use. More in
183
- [`examples/`](examples/).
207
+ (384-dim), `bge` (768-dim), and `clip` (512-dim, image+text), with weights pulled from Hugging
208
+ Face on first use. More in [`examples/`](examples/).
184
209
 
185
210
  Need to read a store another process is writing to? Open it read-only. It takes no writer
186
211
  lock, so it never blocks on (or is blocked by) the writer:
@@ -259,6 +284,39 @@ reopened.search("E1234", k=5, mode="hybrid") # works after reopen, rebuilt from
259
284
  ```
260
285
  </details>
261
286
 
287
+ ## Multimodal & bring-your-own vectors
288
+
289
+ The storage and scan are modality-agnostic: TurboVec stores any normalized float32
290
+ vector, so an image, audio, or video embedding is indexed and scanned exactly like a
291
+ text one. There are two ways to use that.
292
+
293
+ Bring your own vectors. Open a vector-only index at your dimension and pass the
294
+ embeddings you already computed with any model (CLIP, SigLIP, ImageBind, an audio or
295
+ video encoder, a hosted API). No embedding model is bundled on this path:
296
+
297
+ ```python
298
+ db = LodeDB.open_vector_store("./media", vector_dim=512)
299
+ db.add_vectors(image_vector, id="img-001", metadata={"path": "photos/img-001.jpg"})
300
+ db.search_by_vector(query_vector, k=10)
301
+ ```
302
+
303
+ Or use the built-in `clip` preset for image and text in one shared space, so a text
304
+ query retrieves images and an image query retrieves images and text. It rides the base
305
+ sentence-transformers stack and adds only Pillow (`pip install 'lodedb[image]'`):
306
+
307
+ ```python
308
+ db = LodeDB("./gallery", model="clip") # downloads clip-ViT-B-32 on first use
309
+ db.add_image("photos/beach.jpg", metadata={"path": "photos/beach.jpg"})
310
+ db.search("a beach at sunset", k=5) # text -> image, cross-modal
311
+ db.search_by_image("photos/beach.jpg", k=5) # image -> image
312
+ ```
313
+
314
+ The raw image is never stored; keep it on disk and put its path in `metadata`. Keep one
315
+ embedding model per index (scores are only comparable within one space); the model
316
+ identity is pinned and re-enforced on reopen. To hold several encoders side by side, use
317
+ `LodeCollection` named spaces, and pass `embedder=` to drive an index with your own
318
+ model. See [`docs/multimodal.md`](docs/multimodal.md).
319
+
262
320
  ## GPU-resident index
263
321
 
264
322
  With the `[gpu]` extra on a CUDA host, LodeDB reconstructs the compact index into an fp16
@@ -342,7 +400,7 @@ and end-to-end single-query latency is 5.7 ms p50.
342
400
  lodedb doctor # capability report: embedding / GPU / TurboVec backend
343
401
  lodedb index ... # build / add to an on-disk index
344
402
  lodedb query ... # search
345
- lodedb serve # loopback dev server (127.0.0.1, no auth)
403
+ lodedb serve # dev server (127.0.0.1 by default; private LAN only, no auth)
346
404
  lodedb mcp # stdio MCP server for agent memory
347
405
  lodedb benchmark # local, metrics-only benchmark
348
406
  ```
@@ -453,7 +511,11 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star
453
511
  generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
454
512
  `commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
455
513
  publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
456
- out-of-process readers must see each write the instant it commits.
514
+ out-of-process readers must see each write the instant it commits. Note `<key>.wal` is
515
+ **payload-bearing before a checkpoint** (raw text under `store_text=True`, otherwise embedding
516
+ deltas plus, with `index_text=True`, lexical tokens), so treat it as sensitively as the data it
517
+ indexes; `persist()`/`close()` checkpoint and truncate it, and `generation` mode keeps no WAL.
518
+ See the [payload boundary](docs/architecture.md#persistence--payload-boundary) docs.
457
519
  - **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
458
520
 
459
521
  ## Limitations
@@ -14,9 +14,20 @@ LodeDB instead of its default store. Over 17.5k documents, per framework default
14
14
  | vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
15
15
  |---|---|---|---|
16
16
  | On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
17
- | Single-query p50 (CPU) | **~410× faster** (0.8 vs 345 ms) | **~500× faster** (0.9 vs 427 ms) | **~25× faster** (0.9 vs 23 ms) |
18
- | Batched retrieval, 64 (GPU) | **~2,000×** (6,280 vs ~3 qps) | **~2,800×** (5,744 vs ~2 qps) | **~62×** (2,686 vs 43 qps) |
19
- | Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.9 vs 0.5 ms (both sub-ms) |
17
+ | Single-query p50 (CPU) | **~510× faster** (0.57 vs 294 ms) | **~610× faster** (0.57 vs 349 ms) | **~48× faster** (0.64 vs 31 ms) |
18
+ | Batched retrieval, 64 (GPU) | **~1,900×** (6,602 vs ~4 qps) | **~2,200×** (6,414 vs ~3 qps) | **~160×** (4,649 vs 29 qps) |
19
+ | Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.6 vs 0.5 ms (both sub-ms) |
20
+
21
+ LodeDB matches sqlite-vec/qdrant's per-add latency with a significantly more compact footprint:
22
+
23
+ | **embedded stores** | **durable add p50** | **single-query p50** | **batch-64/query** | **memory footprint** |
24
+ | --- | ---: | ---: | ---: | ---: |
25
+ | sqlite-vec | **0.4 ms** | 25.3 ms | 25.1 ms | 101 MB |
26
+ | qdrant | **0.5 ms** | 27.3 ms | 27.0 ms | 85 MB |
27
+ | **LodeDB** | **0.6 ms** | **0.49 ms** | **0.11 ms** | **29 MB** |
28
+ | pgvector | 2.3 ms | 30.4 ms | 30.6 ms | 50 MB |
29
+ | lancedb | 3.2 ms | 10.3 ms | 10.0 ms | 37 MB |
30
+ | chroma | 6.5 ms | 3.0 ms | 3.1 ms | 151 MB |
20
31
 
21
32
  [Full benchmark, all backends
22
33
  (FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
@@ -38,10 +49,16 @@ incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
38
49
  [How it works](#concurrency--durability).
39
50
  - **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
40
51
  (counts, bytes, latency), never raw payloads.
41
- - **Local embeddings**: `sentence-transformers` on CUDA, MPS, or CPU.
42
- - **Batteries included**: a `lodedb` CLI, a loopback dev server, an
43
- [MCP server](#use-as-an-mcp-server), and LangChain, LlamaIndex, and mem0 adapters
44
- (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`).
52
+ - **Local embeddings**: ONNX Runtime by default (lower per-query latency), with a PyTorch
53
+ `sentence-transformers` fallback; runs on CPU, CUDA, or MPS. Pick with `embedding_runtime=`.
54
+ - **Multimodal**: index images and text in one shared CLIP space (`model="clip"`) for
55
+ cross-modal search, or bring your own vectors from any model.
56
+ [How it works](#multimodal--bring-your-own-vectors).
57
+ - **Batteries included**: a `lodedb` CLI, a loopback/private-network dev server, an
58
+ [MCP server](#use-as-an-mcp-server), LangChain, LlamaIndex, and mem0 adapters
59
+ (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`), and a one-line
60
+ [PrivateGPT](https://github.com/zylon-ai/private-gpt) vector-store provider built on the
61
+ LlamaIndex adapter.
45
62
 
46
63
  > 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
47
64
  > available for commercial support, managed and at-scale serving, and on-prem / BYOC
@@ -59,7 +76,9 @@ the install with `lodedb doctor`. Optional extras:
59
76
 
60
77
  ```bash
61
78
  pip install "lodedb[gpu]" # GPU-resident scan (Linux/CUDA)
79
+ pip install "lodedb[image]" # image + text (CLIP) embedding (model="clip")
62
80
  pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
81
+ pip install "lodedb[onnx-export]" # export ONNX for a custom model (Optimum); presets need no export
63
82
  ```
64
83
 
65
84
  Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
@@ -117,7 +136,7 @@ Run with `uv run` (e.g. `uv run lodedb doctor`).
117
136
  ```python
118
137
  from lodedb import LodeDB
119
138
 
120
- db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality)
139
+ db = LodeDB(path="./data", model="minilm") # "minilm" (fast) | "bge" (quality) | "clip" (image+text)
121
140
 
122
141
  fox = db.add("the quick brown fox jumps", metadata={"topic": "animals"})
123
142
  db.add("a lazy dog sleeps all day", metadata={"topic": "animals"})
@@ -141,8 +160,8 @@ db.persist() # durable .tvim/.tvd/.jsd snapshot; replays on reopen
141
160
 
142
161
  Reopen with `LodeDB(path="./data")`; no migration step. Original text is kept in a
143
162
  `.tvtext` sidecar for `db.get`; pass `store_text=False` to keep none. Presets are `minilm`
144
- (384-dim) and `bge` (768-dim), with weights pulled from Hugging Face on first use. More in
145
- [`examples/`](examples/).
163
+ (384-dim), `bge` (768-dim), and `clip` (512-dim, image+text), with weights pulled from Hugging
164
+ Face on first use. More in [`examples/`](examples/).
146
165
 
147
166
  Need to read a store another process is writing to? Open it read-only. It takes no writer
148
167
  lock, so it never blocks on (or is blocked by) the writer:
@@ -221,6 +240,39 @@ reopened.search("E1234", k=5, mode="hybrid") # works after reopen, rebuilt from
221
240
  ```
222
241
  </details>
223
242
 
243
+ ## Multimodal & bring-your-own vectors
244
+
245
+ The storage and scan are modality-agnostic: TurboVec stores any normalized float32
246
+ vector, so an image, audio, or video embedding is indexed and scanned exactly like a
247
+ text one. There are two ways to use that.
248
+
249
+ Bring your own vectors. Open a vector-only index at your dimension and pass the
250
+ embeddings you already computed with any model (CLIP, SigLIP, ImageBind, an audio or
251
+ video encoder, a hosted API). No embedding model is bundled on this path:
252
+
253
+ ```python
254
+ db = LodeDB.open_vector_store("./media", vector_dim=512)
255
+ db.add_vectors(image_vector, id="img-001", metadata={"path": "photos/img-001.jpg"})
256
+ db.search_by_vector(query_vector, k=10)
257
+ ```
258
+
259
+ Or use the built-in `clip` preset for image and text in one shared space, so a text
260
+ query retrieves images and an image query retrieves images and text. It rides the base
261
+ sentence-transformers stack and adds only Pillow (`pip install 'lodedb[image]'`):
262
+
263
+ ```python
264
+ db = LodeDB("./gallery", model="clip") # downloads clip-ViT-B-32 on first use
265
+ db.add_image("photos/beach.jpg", metadata={"path": "photos/beach.jpg"})
266
+ db.search("a beach at sunset", k=5) # text -> image, cross-modal
267
+ db.search_by_image("photos/beach.jpg", k=5) # image -> image
268
+ ```
269
+
270
+ The raw image is never stored; keep it on disk and put its path in `metadata`. Keep one
271
+ embedding model per index (scores are only comparable within one space); the model
272
+ identity is pinned and re-enforced on reopen. To hold several encoders side by side, use
273
+ `LodeCollection` named spaces, and pass `embedder=` to drive an index with your own
274
+ model. See [`docs/multimodal.md`](docs/multimodal.md).
275
+
224
276
  ## GPU-resident index
225
277
 
226
278
  With the `[gpu]` extra on a CUDA host, LodeDB reconstructs the compact index into an fp16
@@ -304,7 +356,7 @@ and end-to-end single-query latency is 5.7 ms p50.
304
356
  lodedb doctor # capability report: embedding / GPU / TurboVec backend
305
357
  lodedb index ... # build / add to an on-disk index
306
358
  lodedb query ... # search
307
- lodedb serve # loopback dev server (127.0.0.1, no auth)
359
+ lodedb serve # dev server (127.0.0.1 by default; private LAN only, no auth)
308
360
  lodedb mcp # stdio MCP server for agent memory
309
361
  lodedb benchmark # local, metrics-only benchmark
310
362
  ```
@@ -415,7 +467,11 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star
415
467
  generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
416
468
  `commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
417
469
  publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
418
- out-of-process readers must see each write the instant it commits.
470
+ out-of-process readers must see each write the instant it commits. Note `<key>.wal` is
471
+ **payload-bearing before a checkpoint** (raw text under `store_text=True`, otherwise embedding
472
+ deltas plus, with `index_text=True`, lexical tokens), so treat it as sensitively as the data it
473
+ indexes; `persist()`/`close()` checkpoint and truncate it, and `generation` mode keeps no WAL.
474
+ See the [payload boundary](docs/architecture.md#persistence--payload-boundary) docs.
419
475
  - **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
420
476
 
421
477
  ## Limitations
@@ -5,7 +5,7 @@ build-backend = "maturin"
5
5
  [project]
6
6
  name = "lodedb"
7
7
  # Static (maturin reads it here); keep in sync with src/lodedb/__init__.py __version__.
8
- version = "0.3.0"
8
+ version = "0.4.0"
9
9
  description = "Local-first, privacy-by-default embedded vector database. Your data never leaves your machine."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.11"
@@ -33,13 +33,19 @@ classifiers = [
33
33
  # Lean runtime set (see docs/architecture.md). The compact TurboVec CPU core +
34
34
  # .tvim/.tvd persistence is the bundled `lodedb._turbovec` extension (compiled
35
35
  # into the wheel, see [tool.maturin]), not a PyPI dependency. numpy = embedding/
36
- # scan I/O; typer = CLI; sentence-transformers = default embedding backend (pulls
37
- # torch/transformers); pyyaml = loads the bundled route registry. Heavy research
38
- # deps (faiss/modal/mteb/datasets/matplotlib/sklearn) are deliberately NOT here
39
- # and are guarded out by tests/test_import_boundary.py.
36
+ # scan I/O; typer = CLI; pyyaml = loads the bundled route registry.
37
+ # Embedding runtime: onnxruntime is the default (faster feature extraction; the
38
+ # preset models ship a prebuilt ONNX on the Hub, fetched on first use) with
39
+ # transformers for tokenization; sentence-transformers (which pulls torch) is the
40
+ # fallback when ONNX cannot be set up. Exporting ONNX for a model without a prebuilt
41
+ # graph is the opt-in [onnx-export] extra (Optimum), not part of the base install.
42
+ # Heavy research deps (faiss/modal/mteb/datasets/matplotlib/sklearn) are
43
+ # deliberately NOT here and are guarded out by tests/test_import_boundary.py.
40
44
  dependencies = [
41
45
  "numpy>=2.0.0",
42
46
  "typer>=0.12.0",
47
+ "onnxruntime>=1.20.0",
48
+ "transformers>=4.40.0",
43
49
  "sentence-transformers>=3.0.0",
44
50
  "pyyaml>=6.0.0",
45
51
  ]
@@ -48,6 +54,10 @@ dependencies = [
48
54
  # Opt-in GPU-resident exact scan on CUDA hosts (gpu_turbovec); absent CUDA, LodeDB uses
49
55
  # the CPU scan. cupy wheels are Linux/CUDA-only, so this resolves to nothing elsewhere.
50
56
  gpu = ["cupy-cuda12x>=13.0.0; platform_system == 'Linux'"]
57
+ # Export an ONNX graph for a model that does not ship a prebuilt one on the Hub
58
+ # (the built-in minilm/bge presets do, so this is only needed for custom models).
59
+ # Optimum runs the export in a subprocess, so it never imports into the lodedb process.
60
+ onnx-export = ["optimum[exporters]>=1.23.0,<2.0.0"]
51
61
  # Optional stdio MCP server so coding agents can use LodeDB as local memory.
52
62
  mcp = ["mcp>=1.0.0"]
53
63
  # Optional LangChain VectorStore adapter.
@@ -56,6 +66,11 @@ langchain = ["langchain-core>=0.3.0"]
56
66
  llama-index = ["llama-index-core>=0.11.0"]
57
67
  # Optional mem0 vector-store adapter.
58
68
  mem0 = ["mem0ai>=2.0.0"]
69
+ # Optional image + text (CLIP) embedding via the "clip" preset / add_image. The CLIP
70
+ # model runs on the existing sentence-transformers + torch stack (already in the base
71
+ # deps), so this adds only Pillow for decoding image files. Both are imported lazily,
72
+ # so a plain `import lodedb` pulls neither (guarded by tests/test_import_boundary.py).
73
+ image = ["pillow>=10.0.0"]
59
74
  # ruff is pinned (not floored) so `ruff format`/`ruff check` are reproducible across
60
75
  # contributors and CI — the formatter's output can change between ruff releases.
61
76
  dev = ["pytest>=8.3.0", "ruff==0.15.17"]
@@ -11,6 +11,8 @@ from __future__ import annotations
11
11
  from lodedb.local import (
12
12
  LOCAL_MODEL_PRESETS,
13
13
  ConcurrentWriterError,
14
+ ImageEmbeddingUnsupportedError,
15
+ LodeCollection,
14
16
  LodeDB,
15
17
  LodeSearchHit,
16
18
  ReadOnlyError,
@@ -19,11 +21,13 @@ from lodedb.local import (
19
21
  from lodedb.local.cli import app, main
20
22
 
21
23
  # Keep in sync with `version` in pyproject.toml (the release workflow asserts they match).
22
- __version__ = "0.3.0"
24
+ __version__ = "0.4.0"
23
25
 
24
26
  __all__ = [
25
27
  "LOCAL_MODEL_PRESETS",
26
28
  "ConcurrentWriterError",
29
+ "ImageEmbeddingUnsupportedError",
30
+ "LodeCollection",
27
31
  "LodeDB",
28
32
  "LodeSearchHit",
29
33
  "ReadOnlyError",