lodedb 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lodedb-0.2.0 → lodedb-0.3.0}/PKG-INFO +162 -17
- {lodedb-0.2.0 → lodedb-0.3.0}/README.md +157 -16
- {lodedb-0.2.0 → lodedb-0.3.0}/pyproject.toml +5 -1
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/__init__.py +1 -1
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/core.py +722 -64
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/index.py +36 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/runtime_policy.py +46 -0
- lodedb-0.3.0/src/lodedb/engine/wal_store.py +282 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/graph/knowledge_graph.py +60 -14
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/backends.py +16 -0
- lodedb-0.3.0/src/lodedb/local/cli.py +533 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/db.py +73 -8
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/doctor.py +48 -0
- lodedb-0.3.0/src/lodedb/local/integrations/__init__.py +17 -0
- lodedb-0.3.0/src/lodedb/local/integrations/llama_index.py +468 -0
- lodedb-0.3.0/src/lodedb/local/integrations/llama_index_graph.py +404 -0
- lodedb-0.3.0/src/lodedb/local/integrations/mem0.py +531 -0
- lodedb-0.3.0/src/lodedb/local/mcp_install.py +587 -0
- lodedb-0.3.0/src/lodedb/local/mcp_server.py +220 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/server.py +10 -2
- lodedb-0.2.0/src/lodedb/local/cli.py +0 -267
- lodedb-0.2.0/src/lodedb/local/integrations/__init__.py +0 -10
- lodedb-0.2.0/src/lodedb/local/mcp_server.py +0 -158
- {lodedb-0.2.0 → lodedb-0.3.0}/LICENSE +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/NOTICE +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/__main__.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/config.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/__init__.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/_atomic_io.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/_commit_manifest.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/_filelock.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/_filter_plan.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/_lexical.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/_predicate.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/document_text_store.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/embedding_backends.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/gpu_turbovec.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/lexical_index_store.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/mps_turbovec.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/route_profiles.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/route_registry.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/state_journal_store.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/turbovec_delta_store.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/turbovec_index.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/engine/turbovec_resident.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/graph/__init__.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/graph/_store.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/__init__.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/benchmark.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/integrations/langchain.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/local/presets.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/src/lodedb/py.typed +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/Cargo.lock +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/Cargo.toml +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/Cargo.toml +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/README.md +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/build.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/examples/dump_state.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/examples/kernel_xtest.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/codebook.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/encode.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/error.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/id_map.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/io.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/lib.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/pack.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/rotation.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/src/search.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/codebook.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/concurrent_search.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/distortion.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/encode.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/encoded_rows.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/filtering.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/id_map.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/input_validation.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/io_versioning.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/kernel_correctness.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/lazy_init.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/reconstruction.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/rotation.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/state_sequences.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/swap_remove.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec/tests/tqplus_calibration.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/Cargo.toml +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/README.md +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/build.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/__init__.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/_dedup.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/_persist.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/agno.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/haystack.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/langchain.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/python/turbovec/llama_index.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/src/lib.rs +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_agno.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_dedup.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_filtering.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_haystack.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_id_map.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_index.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_langchain.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_llama_index.py +0 -0
- {lodedb-0.2.0 → lodedb-0.3.0}/third_party/turbovec/turbovec-python/tests/test_security.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lodedb
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
@@ -14,11 +14,15 @@ Requires-Dist: pytest>=8.3.0 ; extra == 'dev'
|
|
|
14
14
|
Requires-Dist: ruff==0.15.17 ; extra == 'dev'
|
|
15
15
|
Requires-Dist: cupy-cuda12x>=13.0.0 ; platform_system == 'Linux' and extra == 'gpu'
|
|
16
16
|
Requires-Dist: langchain-core>=0.3.0 ; extra == 'langchain'
|
|
17
|
+
Requires-Dist: llama-index-core>=0.11.0 ; extra == 'llama-index'
|
|
17
18
|
Requires-Dist: mcp>=1.0.0 ; extra == 'mcp'
|
|
19
|
+
Requires-Dist: mem0ai>=2.0.0 ; extra == 'mem0'
|
|
18
20
|
Provides-Extra: dev
|
|
19
21
|
Provides-Extra: gpu
|
|
20
22
|
Provides-Extra: langchain
|
|
23
|
+
Provides-Extra: llama-index
|
|
21
24
|
Provides-Extra: mcp
|
|
25
|
+
Provides-Extra: mem0
|
|
22
26
|
License-File: LICENSE
|
|
23
27
|
License-File: NOTICE
|
|
24
28
|
Summary: Local-first, privacy-by-default embedded vector database. Your data never leaves your machine.
|
|
@@ -32,23 +36,34 @@ Project-URL: Homepage, https://github.com/Egoist-Machines/LodeDB
|
|
|
32
36
|
Project-URL: Issues, https://github.com/Egoist-Machines/LodeDB/issues
|
|
33
37
|
Project-URL: Repository, https://github.com/Egoist-Machines/LodeDB
|
|
34
38
|
|
|
35
|
-
|
|
39
|
+
<h1 align="center">LodeDB</h1>
|
|
40
|
+
<p align="center">🔥 the <b>fastest</b> and <i>most compact</i> embedded vector database in the world 🌍</p>
|
|
36
41
|
|
|
37
42
|
[](LICENSE)
|
|
38
43
|
[](pyproject.toml)
|
|
39
44
|
|
|
40
|
-
**A fast, exact embedded vector database for local RAG: in-process, on-disk, no server.**
|
|
41
|
-
|
|
42
45
|
*Built by [Egoist Machines, Inc.](https://egoistmachines.com) - efficient full-stack infrastructure
|
|
43
46
|
for reliable AI systems.*
|
|
44
47
|
|
|
45
|
-
|
|
46
|
-
|
|
48
|
+
LodeDB is great for local RAG; it's _extremely fast_, exact, in-process, and on-disk. We're the **best drop-in** durable memory backend for **LangChain, LlamaIndex, and mem0** across all metrics that matter: the most
|
|
49
|
+
compact on disk, GPU-accelerated, and sub-millisecond on durable writes. Point any of them at
|
|
50
|
+
LodeDB instead of its default store. Over 17.5k documents, per framework default:
|
|
51
|
+
|
|
52
|
+
| vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
|
|
53
|
+
|---|---|---|---|
|
|
54
|
+
| On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
|
|
55
|
+
| Single-query p50 (CPU) | **~410× faster** (0.8 vs 345 ms) | **~500× faster** (0.9 vs 427 ms) | **~25× faster** (0.9 vs 23 ms) |
|
|
56
|
+
| Batched retrieval, 64 (GPU) | **~2,000×** (6,280 vs ~3 qps) | **~2,800×** (5,744 vs ~2 qps) | **~62×** (2,686 vs 43 qps) |
|
|
57
|
+
| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.9 vs 0.5 ms (both sub-ms) |
|
|
58
|
+
|
|
59
|
+
[Full benchmark, all backends
|
|
60
|
+
(FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
|
|
61
|
+
|
|
62
|
+
Most embedded vector databases stop at the CPU. LodeDB runs the same on-disk index **on the
|
|
63
|
+
GPU** when you have one: batched search hits *24k queries/sec on an A10 and 50k qps on an L40S*,
|
|
47
64
|
2.8× to 4.8× the all-CPU ceiling, with recall unchanged. It also persists changed rows
|
|
48
65
|
incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
|
|
49
66
|
|
|
50
|
-
Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
|
|
51
|
-
|
|
52
67
|
- **GPU-resident batch search**: an fp16 copy of the index lives on the GPU, scored with a
|
|
53
68
|
tiled GEMM plus a streaming top-k (`[gpu]`, Linux/CUDA). [How it works](#gpu-resident-index).
|
|
54
69
|
- **O(changed) persistence**: commits only the rows that changed, 173× to 1,308× faster
|
|
@@ -62,8 +77,9 @@ Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
|
|
|
62
77
|
- **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
|
|
63
78
|
(counts, bytes, latency), never raw payloads.
|
|
64
79
|
- **Local embeddings**: `sentence-transformers` on CUDA, MPS, or CPU.
|
|
65
|
-
- **Batteries included**: a `lodedb` CLI, a loopback dev server, an
|
|
66
|
-
LangChain
|
|
80
|
+
- **Batteries included**: a `lodedb` CLI, a loopback dev server, an
|
|
81
|
+
[MCP server](#use-as-an-mcp-server), and LangChain, LlamaIndex, and mem0 adapters
|
|
82
|
+
(`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`).
|
|
67
83
|
|
|
68
84
|
> 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
|
|
69
85
|
> available for commercial support, managed and at-scale serving, and on-prem / BYOC
|
|
@@ -80,10 +96,43 @@ Python 3.11+, and bundle the TurboVec (Rust) core, so there's nothing to compile
|
|
|
80
96
|
the install with `lodedb doctor`. Optional extras:
|
|
81
97
|
|
|
82
98
|
```bash
|
|
83
|
-
pip install "lodedb[gpu]"
|
|
84
|
-
pip install "lodedb[mcp,langchain]"
|
|
99
|
+
pip install "lodedb[gpu]" # GPU-resident scan (Linux/CUDA)
|
|
100
|
+
pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
|
|
104
|
+
server in one step (details under [Use as an MCP server](#use-as-an-mcp-server)):
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
lodedb mcp install --client claude-code # or: claude-desktop | cursor | lm-studio | codex | all
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
<details>
|
|
111
|
+
<summary><b>Windows: NVIDIA GPU embeddings</b></summary>
|
|
112
|
+
|
|
113
|
+
On Windows, PyPI serves the CPU-only PyTorch build by default, so `pip install lodedb` (and
|
|
114
|
+
`uv`) leave embeddings on the CPU even on a CUDA machine, and no package metadata can override
|
|
115
|
+
which torch wheel pip resolves. `lodedb doctor` detects this and prints the fix; `lodedb doctor
|
|
116
|
+
--fix` reinstalls the CUDA build for you:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
lodedb doctor # flags a CPU-only PyTorch on Windows and prints the command
|
|
120
|
+
lodedb doctor --fix # reinstalls the CUDA build so embeddings use your NVIDIA GPU
|
|
85
121
|
```
|
|
86
122
|
|
|
123
|
+
Or reinstall manually, picking the index for your CUDA version (`cu121`, `cu124`, ...) from the
|
|
124
|
+
[PyTorch install guide](https://pytorch.org/get-started/locally/):
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
pip install torch --force-reinstall --no-deps --index-url https://download.pytorch.org/whl/cu121
|
|
128
|
+
uv pip install torch --reinstall --index-url https://download.pytorch.org/whl/cu121 # with uv
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
This is Windows-only: the default Linux PyPI wheel already bundles CUDA, and macOS uses CPU
|
|
132
|
+
or MPS.
|
|
133
|
+
|
|
134
|
+
</details>
|
|
135
|
+
|
|
87
136
|
<details>
|
|
88
137
|
<summary><b>Build from source</b> (contributors, or a platform without a wheel)</summary>
|
|
89
138
|
|
|
@@ -93,8 +142,8 @@ Linux). [uv](https://docs.astral.sh/uv/) builds and bundles the core for you:
|
|
|
93
142
|
```bash
|
|
94
143
|
git clone https://github.com/Egoist-Machines/LodeDB && cd LodeDB
|
|
95
144
|
uv sync # builds + bundles the TurboVec core via maturin
|
|
96
|
-
uv sync --extra mcp --extra langchain
|
|
97
|
-
uv sync --extra gpu
|
|
145
|
+
uv sync --extra mcp --extra langchain --extra llama-index --extra mem0 # + MCP server, adapters
|
|
146
|
+
uv sync --extra gpu # + GPU-resident scan (Linux/CUDA)
|
|
98
147
|
```
|
|
99
148
|
|
|
100
149
|
Run with `uv run` (e.g. `uv run lodedb doctor`).
|
|
@@ -158,7 +207,8 @@ db.search("E1234", k=5, mode="hybrid") # BM25 + RRF: surfaces it in the top-k
|
|
|
158
207
|
db.search("E1234", k=5, mode="lexical") # BM25 ranking alone, no vector scan
|
|
159
208
|
```
|
|
160
209
|
|
|
161
|
-
|
|
210
|
+
<details>
|
|
211
|
+
<summary><b>Prerequisites</b></summary>
|
|
162
212
|
|
|
163
213
|
`mode="hybrid"` and `mode="lexical"` build a BM25 index over your text, so they need a text
|
|
164
214
|
source enabled when you open the database. `mode="vector"` (the default) needs nothing.
|
|
@@ -172,8 +222,10 @@ source enabled when you open the database. `mode="vector"` (the default) needs n
|
|
|
172
222
|
Either source is enough and you can enable both. `store_text=True` is the default, so hybrid
|
|
173
223
|
works out of the box. With neither source enabled, a hybrid or lexical query raises a clear,
|
|
174
224
|
actionable error rather than silently degrading.
|
|
225
|
+
</details>
|
|
175
226
|
|
|
176
|
-
|
|
227
|
+
<details>
|
|
228
|
+
<summary><b>How it works</b></summary>
|
|
177
229
|
|
|
178
230
|
A `filter` constrains both rankers, so `mode="hybrid"` with a filter returns the true top-k of
|
|
179
231
|
the matching subset. The vector half of a hybrid query runs on the same scan as `mode="vector"`,
|
|
@@ -181,8 +233,10 @@ including the GPU-resident batch scan that serves `search_many`; only the BM25 r
|
|
|
181
233
|
fusion run on the CPU, and the vector kernel and on-disk format are untouched. The serving BM25
|
|
182
234
|
index lives in memory and is maintained incrementally: a small mutation folds just the changed
|
|
183
235
|
chunks into the existing index, so a single `add` never forces a full re-tokenization.
|
|
236
|
+
</details>
|
|
184
237
|
|
|
185
|
-
|
|
238
|
+
<details>
|
|
239
|
+
<summary><b>Durable lexical index (`index_text=True`)</b></summary>
|
|
186
240
|
|
|
187
241
|
By default the BM25 index is rebuilt from the retained raw text, so it needs `store_text=True`
|
|
188
242
|
and is re-tokenized on the first hybrid query after opening. Pass `index_text=True` to persist
|
|
@@ -203,6 +257,7 @@ db.close()
|
|
|
203
257
|
reopened = LodeDB(path="./data", index_text=True, store_text=False)
|
|
204
258
|
reopened.search("E1234", k=5, mode="hybrid") # works after reopen, rebuilt from persisted terms
|
|
205
259
|
```
|
|
260
|
+
</details>
|
|
206
261
|
|
|
207
262
|
## GPU-resident index
|
|
208
263
|
|
|
@@ -292,6 +347,85 @@ lodedb mcp # stdio MCP server for agent memory
|
|
|
292
347
|
lodedb benchmark # local, metrics-only benchmark
|
|
293
348
|
```
|
|
294
349
|
|
|
350
|
+
## Use as an MCP server
|
|
351
|
+
|
|
352
|
+
LodeDB ships a [Model Context Protocol](https://modelcontextprotocol.io) server, so an agent
|
|
353
|
+
can use a local on-disk database as long-term memory or a RAG store. It runs over stdio, adds
|
|
354
|
+
no storage logic of its own, and your data stays on the machine. Install the extra and point
|
|
355
|
+
your host at `lodedb mcp`:
|
|
356
|
+
|
|
357
|
+
```bash
|
|
358
|
+
pip install "lodedb[mcp]"
|
|
359
|
+
|
|
360
|
+
# for coding assistant:
|
|
361
|
+
lodedb mcp install --client claude-code # or: claude-desktop | cursor | lm-studio | codex | all
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
It exposes `lodedb_add`, `lodedb_search`, `lodedb_remove`, and `lodedb_stats`, plus
|
|
365
|
+
`lodedb_get` when text is available. `lodedb_search` returns each hit's stored text alongside
|
|
366
|
+
the score, id, and metadata, so a model can rank and answer in a single call rather than
|
|
367
|
+
chaining a follow-up lookup. It runs [hybrid search](#hybrid-search) (BM25 lexical + vector,
|
|
368
|
+
fused with RRF) by default when text is retained, so exact tokens like error codes and serials
|
|
369
|
+
surface next to semantic matches; with no text retained it falls back to a vector scan. Start
|
|
370
|
+
the server with `--exclude-text` to return metrics only (this also withdraws `lodedb_get`), or
|
|
371
|
+
`--no-store-text` to keep no text on disk at all. `lodedb_stats` is always metrics-only and raw
|
|
372
|
+
query text never leaves the process.
|
|
373
|
+
|
|
374
|
+
### One command install
|
|
375
|
+
|
|
376
|
+
`lodedb mcp install` writes the correct entry to a client's config for you, so you do not have to
|
|
377
|
+
find the file or hand-write the JSON/TOML:
|
|
378
|
+
|
|
379
|
+
```bash
|
|
380
|
+
lodedb mcp install --client claude-code # or: claude-desktop | cursor | lm-studio | codex | all
|
|
381
|
+
lodedb mcp install --client cursor --path ./data --model bge
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
It resolves the launch command for your environment, so `command` and `args` are correct even when
|
|
385
|
+
`lodedb` is not on `PATH` (it falls back to the `uv run --project ...` form, then an absolute path to
|
|
386
|
+
the entry point), and it resolves `--path` to an absolute path so the server opens the right
|
|
387
|
+
directory wherever the client starts it. The edit is idempotent (an existing `lodedb` entry is
|
|
388
|
+
updated, never duplicated) and never touches other servers in the file. It passes through the same
|
|
389
|
+
options as `lodedb mcp` (`--path`, `--model`, `--device`, `--exclude-text`, `--no-store-text`);
|
|
390
|
+
`--dry-run` prints the entry and target file without writing, and `lodedb mcp uninstall --client
|
|
391
|
+
<client>` removes it again. Override the config location with `--config <path>` (Claude Desktop and
|
|
392
|
+
LM Studio paths differ per OS), and use `--project <dir>` to write Cursor's project-level
|
|
393
|
+
`.cursor/mcp.json`. For Claude Code it runs `claude mcp add`; for the others it edits the config file
|
|
394
|
+
directly.
|
|
395
|
+
|
|
396
|
+
<details>
|
|
397
|
+
<summary><b>Register by hand</b> (Claude Code, Claude Desktop, Cursor, LM Studio, Codex)</summary>
|
|
398
|
+
|
|
399
|
+
The `lodedb` command must be on the host's `PATH`; if you installed into a virtual environment
|
|
400
|
+
(including a `uv` project) where it isn't, use the `uv run` form at the bottom.
|
|
401
|
+
|
|
402
|
+
**Claude Code, Claude Desktop, Cursor, LM Studio**: add the stdio entry to the host's MCP
|
|
403
|
+
config (`claude_desktop_config.json`, `.cursor/mcp.json`, or LM Studio's `mcp.json`), or run
|
|
404
|
+
`claude mcp add lodedb -- lodedb mcp --path ./data`:
|
|
405
|
+
|
|
406
|
+
```json
|
|
407
|
+
{ "mcpServers": { "lodedb": { "command": "lodedb", "args": ["mcp", "--path", "./data"] } } }
|
|
408
|
+
```
|
|
409
|
+
|
|
410
|
+
**Codex**: add to `~/.codex/config.toml`:
|
|
411
|
+
|
|
412
|
+
```toml
|
|
413
|
+
[mcp_servers.lodedb]
|
|
414
|
+
command = "lodedb"
|
|
415
|
+
args = ["mcp", "--path", "./data"]
|
|
416
|
+
```
|
|
417
|
+
|
|
418
|
+
**From a virtual environment (uv)**, when `lodedb` is not on `PATH`:
|
|
419
|
+
|
|
420
|
+
```json
|
|
421
|
+
{ "mcpServers": { "lodedb": { "command": "uv",
|
|
422
|
+
"args": ["run", "--project", "/path/to/LodeDB", "lodedb", "mcp", "--path", "/path/to/data"] } } }
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste starting point.
|
|
426
|
+
|
|
427
|
+
</details>
|
|
428
|
+
|
|
295
429
|
## Concurrency & durability
|
|
296
430
|
|
|
297
431
|
- **Single writer, many readers, per path.** One handle holds the path open for *writing* at
|
|
@@ -309,6 +443,17 @@ lodedb benchmark # local, metrics-only benchmark
|
|
|
309
443
|
- **Durability is `fast` by default.** Commits are *atomic* but not fsync'd. Pass
|
|
310
444
|
`durability="fsync"` (or `--durability fsync` / `LODEDB_DURABILITY=fsync`) to fsync each
|
|
311
445
|
file and its directory on commit for power-loss durability, at some commit-throughput cost.
|
|
446
|
+
- **WAL commit by default for low-latency durable writes.** Each `add`/`remove` appends one
|
|
447
|
+
framed record to a `<key>.wal` log and a full generation is checkpointed periodically, so a
|
|
448
|
+
durable single add costs roughly an order of magnitude less than publishing a whole generation
|
|
449
|
+
per write, into the sqlite-vec/qdrant range (see the comparison up top). The WAL is replayed
|
|
450
|
+
crash-atomically on reopen (a half-written trailing record is discarded), every writable open
|
|
451
|
+
folds it into a clean committed generation, and `close()`/`persist()` checkpoint it. WAL is
|
|
452
|
+
single-writer: a concurrent `open_readonly` reader still loads a consistent committed
|
|
453
|
+
generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
|
|
454
|
+
`commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
|
|
455
|
+
publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
|
|
456
|
+
out-of-process readers must see each write the instant it commits.
|
|
312
457
|
- **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
|
|
313
458
|
|
|
314
459
|
## Limitations
|
|
@@ -1,20 +1,31 @@
|
|
|
1
|
-
|
|
1
|
+
<h1 align="center">LodeDB</h1>
|
|
2
|
+
<p align="center">🔥 the <b>fastest</b> and <i>most compact</i> embedded vector database in the world 🌍</p>
|
|
2
3
|
|
|
3
4
|
[](LICENSE)
|
|
4
5
|
[](pyproject.toml)
|
|
5
6
|
|
|
6
|
-
**A fast, exact embedded vector database for local RAG: in-process, on-disk, no server.**
|
|
7
|
-
|
|
8
7
|
*Built by [Egoist Machines, Inc.](https://egoistmachines.com) - efficient full-stack infrastructure
|
|
9
8
|
for reliable AI systems.*
|
|
10
9
|
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
LodeDB is great for local RAG; it's _extremely fast_, exact, in-process, and on-disk. We're the **best drop-in** durable memory backend for **LangChain, LlamaIndex, and mem0** across all metrics that matter: the most
|
|
11
|
+
compact on disk, GPU-accelerated, and sub-millisecond on durable writes. Point any of them at
|
|
12
|
+
LodeDB instead of its default store. Over 17.5k documents, per framework default:
|
|
13
|
+
|
|
14
|
+
| vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
|
|
15
|
+
|---|---|---|---|
|
|
16
|
+
| On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
|
|
17
|
+
| Single-query p50 (CPU) | **~410× faster** (0.8 vs 345 ms) | **~500× faster** (0.9 vs 427 ms) | **~25× faster** (0.9 vs 23 ms) |
|
|
18
|
+
| Batched retrieval, 64 (GPU) | **~2,000×** (6,280 vs ~3 qps) | **~2,800×** (5,744 vs ~2 qps) | **~62×** (2,686 vs 43 qps) |
|
|
19
|
+
| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.9 vs 0.5 ms (both sub-ms) |
|
|
20
|
+
|
|
21
|
+
[Full benchmark, all backends
|
|
22
|
+
(FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
|
|
23
|
+
|
|
24
|
+
Most embedded vector databases stop at the CPU. LodeDB runs the same on-disk index **on the
|
|
25
|
+
GPU** when you have one: batched search hits *24k queries/sec on an A10 and 50k qps on an L40S*,
|
|
13
26
|
2.8× to 4.8× the all-CPU ceiling, with recall unchanged. It also persists changed rows
|
|
14
27
|
incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
|
|
15
28
|
|
|
16
|
-
Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
|
|
17
|
-
|
|
18
29
|
- **GPU-resident batch search**: an fp16 copy of the index lives on the GPU, scored with a
|
|
19
30
|
tiled GEMM plus a streaming top-k (`[gpu]`, Linux/CUDA). [How it works](#gpu-resident-index).
|
|
20
31
|
- **O(changed) persistence**: commits only the rows that changed, 173× to 1,308× faster
|
|
@@ -28,8 +39,9 @@ Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
|
|
|
28
39
|
- **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
|
|
29
40
|
(counts, bytes, latency), never raw payloads.
|
|
30
41
|
- **Local embeddings**: `sentence-transformers` on CUDA, MPS, or CPU.
|
|
31
|
-
- **Batteries included**: a `lodedb` CLI, a loopback dev server, an
|
|
32
|
-
LangChain
|
|
42
|
+
- **Batteries included**: a `lodedb` CLI, a loopback dev server, an
|
|
43
|
+
[MCP server](#use-as-an-mcp-server), and LangChain, LlamaIndex, and mem0 adapters
|
|
44
|
+
(`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`).
|
|
33
45
|
|
|
34
46
|
> 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
|
|
35
47
|
> available for commercial support, managed and at-scale serving, and on-prem / BYOC
|
|
@@ -46,10 +58,43 @@ Python 3.11+, and bundle the TurboVec (Rust) core, so there's nothing to compile
|
|
|
46
58
|
the install with `lodedb doctor`. Optional extras:
|
|
47
59
|
|
|
48
60
|
```bash
|
|
49
|
-
pip install "lodedb[gpu]"
|
|
50
|
-
pip install "lodedb[mcp,langchain]"
|
|
61
|
+
pip install "lodedb[gpu]" # GPU-resident scan (Linux/CUDA)
|
|
62
|
+
pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
|
|
66
|
+
server in one step (details under [Use as an MCP server](#use-as-an-mcp-server)):
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
lodedb mcp install --client claude-code # or: claude-desktop | cursor | lm-studio | codex | all
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
<details>
|
|
73
|
+
<summary><b>Windows: NVIDIA GPU embeddings</b></summary>
|
|
74
|
+
|
|
75
|
+
On Windows, PyPI serves the CPU-only PyTorch build by default, so `pip install lodedb` (and
|
|
76
|
+
`uv`) leave embeddings on the CPU even on a CUDA machine, and no package metadata can override
|
|
77
|
+
which torch wheel pip resolves. `lodedb doctor` detects this and prints the fix; `lodedb doctor
|
|
78
|
+
--fix` reinstalls the CUDA build for you:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
lodedb doctor # flags a CPU-only PyTorch on Windows and prints the command
|
|
82
|
+
lodedb doctor --fix # reinstalls the CUDA build so embeddings use your NVIDIA GPU
|
|
51
83
|
```
|
|
52
84
|
|
|
85
|
+
Or reinstall manually, picking the index for your CUDA version (`cu121`, `cu124`, ...) from the
|
|
86
|
+
[PyTorch install guide](https://pytorch.org/get-started/locally/):
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pip install torch --force-reinstall --no-deps --index-url https://download.pytorch.org/whl/cu121
|
|
90
|
+
uv pip install torch --reinstall --index-url https://download.pytorch.org/whl/cu121 # with uv
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
This is Windows-only: the default Linux PyPI wheel already bundles CUDA, and macOS uses CPU
|
|
94
|
+
or MPS.
|
|
95
|
+
|
|
96
|
+
</details>
|
|
97
|
+
|
|
53
98
|
<details>
|
|
54
99
|
<summary><b>Build from source</b> (contributors, or a platform without a wheel)</summary>
|
|
55
100
|
|
|
@@ -59,8 +104,8 @@ Linux). [uv](https://docs.astral.sh/uv/) builds and bundles the core for you:
|
|
|
59
104
|
```bash
|
|
60
105
|
git clone https://github.com/Egoist-Machines/LodeDB && cd LodeDB
|
|
61
106
|
uv sync # builds + bundles the TurboVec core via maturin
|
|
62
|
-
uv sync --extra mcp --extra langchain
|
|
63
|
-
uv sync --extra gpu
|
|
107
|
+
uv sync --extra mcp --extra langchain --extra llama-index --extra mem0 # + MCP server, adapters
|
|
108
|
+
uv sync --extra gpu # + GPU-resident scan (Linux/CUDA)
|
|
64
109
|
```
|
|
65
110
|
|
|
66
111
|
Run with `uv run` (e.g. `uv run lodedb doctor`).
|
|
@@ -124,7 +169,8 @@ db.search("E1234", k=5, mode="hybrid") # BM25 + RRF: surfaces it in the top-k
|
|
|
124
169
|
db.search("E1234", k=5, mode="lexical") # BM25 ranking alone, no vector scan
|
|
125
170
|
```
|
|
126
171
|
|
|
127
|
-
|
|
172
|
+
<details>
|
|
173
|
+
<summary><b>Prerequisites</b></summary>
|
|
128
174
|
|
|
129
175
|
`mode="hybrid"` and `mode="lexical"` build a BM25 index over your text, so they need a text
|
|
130
176
|
source enabled when you open the database. `mode="vector"` (the default) needs nothing.
|
|
@@ -138,8 +184,10 @@ source enabled when you open the database. `mode="vector"` (the default) needs n
|
|
|
138
184
|
Either source is enough and you can enable both. `store_text=True` is the default, so hybrid
|
|
139
185
|
works out of the box. With neither source enabled, a hybrid or lexical query raises a clear,
|
|
140
186
|
actionable error rather than silently degrading.
|
|
187
|
+
</details>
|
|
141
188
|
|
|
142
|
-
|
|
189
|
+
<details>
|
|
190
|
+
<summary><b>How it works</b></summary>
|
|
143
191
|
|
|
144
192
|
A `filter` constrains both rankers, so `mode="hybrid"` with a filter returns the true top-k of
|
|
145
193
|
the matching subset. The vector half of a hybrid query runs on the same scan as `mode="vector"`,
|
|
@@ -147,8 +195,10 @@ including the GPU-resident batch scan that serves `search_many`; only the BM25 r
|
|
|
147
195
|
fusion run on the CPU, and the vector kernel and on-disk format are untouched. The serving BM25
|
|
148
196
|
index lives in memory and is maintained incrementally: a small mutation folds just the changed
|
|
149
197
|
chunks into the existing index, so a single `add` never forces a full re-tokenization.
|
|
198
|
+
</details>
|
|
150
199
|
|
|
151
|
-
|
|
200
|
+
<details>
|
|
201
|
+
<summary><b>Durable lexical index (`index_text=True`)</b></summary>
|
|
152
202
|
|
|
153
203
|
By default the BM25 index is rebuilt from the retained raw text, so it needs `store_text=True`
|
|
154
204
|
and is re-tokenized on the first hybrid query after opening. Pass `index_text=True` to persist
|
|
@@ -169,6 +219,7 @@ db.close()
|
|
|
169
219
|
reopened = LodeDB(path="./data", index_text=True, store_text=False)
|
|
170
220
|
reopened.search("E1234", k=5, mode="hybrid") # works after reopen, rebuilt from persisted terms
|
|
171
221
|
```
|
|
222
|
+
</details>
|
|
172
223
|
|
|
173
224
|
## GPU-resident index
|
|
174
225
|
|
|
@@ -258,6 +309,85 @@ lodedb mcp # stdio MCP server for agent memory
|
|
|
258
309
|
lodedb benchmark # local, metrics-only benchmark
|
|
259
310
|
```
|
|
260
311
|
|
|
312
|
+
## Use as an MCP server
|
|
313
|
+
|
|
314
|
+
LodeDB ships a [Model Context Protocol](https://modelcontextprotocol.io) server, so an agent
|
|
315
|
+
can use a local on-disk database as long-term memory or a RAG store. It runs over stdio, adds
|
|
316
|
+
no storage logic of its own, and your data stays on the machine. Install the extra and point
|
|
317
|
+
your host at `lodedb mcp`:
|
|
318
|
+
|
|
319
|
+
```bash
|
|
320
|
+
pip install "lodedb[mcp]"
|
|
321
|
+
|
|
322
|
+
# for coding assistant:
|
|
323
|
+
lodedb mcp install --client claude-code # or: claude-desktop | cursor | lm-studio | codex | all
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
It exposes `lodedb_add`, `lodedb_search`, `lodedb_remove`, and `lodedb_stats`, plus
|
|
327
|
+
`lodedb_get` when text is available. `lodedb_search` returns each hit's stored text alongside
|
|
328
|
+
the score, id, and metadata, so a model can rank and answer in a single call rather than
|
|
329
|
+
chaining a follow-up lookup. It runs [hybrid search](#hybrid-search) (BM25 lexical + vector,
|
|
330
|
+
fused with RRF) by default when text is retained, so exact tokens like error codes and serials
|
|
331
|
+
surface next to semantic matches; with no text retained it falls back to a vector scan. Start
|
|
332
|
+
the server with `--exclude-text` to return metrics only (this also withdraws `lodedb_get`), or
|
|
333
|
+
`--no-store-text` to keep no text on disk at all. `lodedb_stats` is always metrics-only and raw
|
|
334
|
+
query text never leaves the process.
|
|
335
|
+
|
|
336
|
+
### One command install
|
|
337
|
+
|
|
338
|
+
`lodedb mcp install` writes the correct entry to a client's config for you, so you do not have to
|
|
339
|
+
find the file or hand-write the JSON/TOML:
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
lodedb mcp install --client claude-code # or: claude-desktop | cursor | lm-studio | codex | all
|
|
343
|
+
lodedb mcp install --client cursor --path ./data --model bge
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
It resolves the launch command for your environment, so `command` and `args` are correct even when
|
|
347
|
+
`lodedb` is not on `PATH` (it falls back to the `uv run --project ...` form, then an absolute path to
|
|
348
|
+
the entry point), and it resolves `--path` to an absolute path so the server opens the right
|
|
349
|
+
directory wherever the client starts it. The edit is idempotent (an existing `lodedb` entry is
|
|
350
|
+
updated, never duplicated) and never touches other servers in the file. It passes through the same
|
|
351
|
+
options as `lodedb mcp` (`--path`, `--model`, `--device`, `--exclude-text`, `--no-store-text`);
|
|
352
|
+
`--dry-run` prints the entry and target file without writing, and `lodedb mcp uninstall --client
|
|
353
|
+
<client>` removes it again. Override the config location with `--config <path>` (Claude Desktop and
|
|
354
|
+
LM Studio paths differ per OS), and use `--project <dir>` to write Cursor's project-level
|
|
355
|
+
`.cursor/mcp.json`. For Claude Code it runs `claude mcp add`; for the others it edits the config file
|
|
356
|
+
directly.
|
|
357
|
+
|
|
358
|
+
<details>
|
|
359
|
+
<summary><b>Register by hand</b> (Claude Code, Claude Desktop, Cursor, LM Studio, Codex)</summary>
|
|
360
|
+
|
|
361
|
+
The `lodedb` command must be on the host's `PATH`; if you installed into a virtual environment
|
|
362
|
+
(including a `uv` project) where it isn't, use the `uv run` form at the bottom.
|
|
363
|
+
|
|
364
|
+
**Claude Code, Claude Desktop, Cursor, LM Studio**: add the stdio entry to the host's MCP
|
|
365
|
+
config (`claude_desktop_config.json`, `.cursor/mcp.json`, or LM Studio's `mcp.json`), or run
|
|
366
|
+
`claude mcp add lodedb -- lodedb mcp --path ./data`:
|
|
367
|
+
|
|
368
|
+
```json
|
|
369
|
+
{ "mcpServers": { "lodedb": { "command": "lodedb", "args": ["mcp", "--path", "./data"] } } }
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
**Codex**: add to `~/.codex/config.toml`:
|
|
373
|
+
|
|
374
|
+
```toml
|
|
375
|
+
[mcp_servers.lodedb]
|
|
376
|
+
command = "lodedb"
|
|
377
|
+
args = ["mcp", "--path", "./data"]
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
**From a virtual environment (uv)**, when `lodedb` is not on `PATH`:
|
|
381
|
+
|
|
382
|
+
```json
|
|
383
|
+
{ "mcpServers": { "lodedb": { "command": "uv",
|
|
384
|
+
"args": ["run", "--project", "/path/to/LodeDB", "lodedb", "mcp", "--path", "/path/to/data"] } } }
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste starting point.
|
|
388
|
+
|
|
389
|
+
</details>
|
|
390
|
+
|
|
261
391
|
## Concurrency & durability
|
|
262
392
|
|
|
263
393
|
- **Single writer, many readers, per path.** One handle holds the path open for *writing* at
|
|
@@ -275,6 +405,17 @@ lodedb benchmark # local, metrics-only benchmark
|
|
|
275
405
|
- **Durability is `fast` by default.** Commits are *atomic* but not fsync'd. Pass
|
|
276
406
|
`durability="fsync"` (or `--durability fsync` / `LODEDB_DURABILITY=fsync`) to fsync each
|
|
277
407
|
file and its directory on commit for power-loss durability, at some commit-throughput cost.
|
|
408
|
+
- **WAL commit by default for low-latency durable writes.** Each `add`/`remove` appends one
|
|
409
|
+
framed record to a `<key>.wal` log and a full generation is checkpointed periodically, so a
|
|
410
|
+
durable single add costs roughly an order of magnitude less than publishing a whole generation
|
|
411
|
+
per write, into the sqlite-vec/qdrant range (see the comparison up top). The WAL is replayed
|
|
412
|
+
crash-atomically on reopen (a half-written trailing record is discarded), every writable open
|
|
413
|
+
folds it into a clean committed generation, and `close()`/`persist()` checkpoint it. WAL is
|
|
414
|
+
single-writer: a concurrent `open_readonly` reader still loads a consistent committed
|
|
415
|
+
generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
|
|
416
|
+
`commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
|
|
417
|
+
publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
|
|
418
|
+
out-of-process readers must see each write the instant it commits.
|
|
278
419
|
- **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
|
|
279
420
|
|
|
280
421
|
## Limitations
|
|
@@ -5,7 +5,7 @@ build-backend = "maturin"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "lodedb"
|
|
7
7
|
# Static (maturin reads it here); keep in sync with src/lodedb/__init__.py __version__.
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.3.0"
|
|
9
9
|
description = "Local-first, privacy-by-default embedded vector database. Your data never leaves your machine."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.11"
|
|
@@ -52,6 +52,10 @@ gpu = ["cupy-cuda12x>=13.0.0; platform_system == 'Linux'"]
|
|
|
52
52
|
mcp = ["mcp>=1.0.0"]
|
|
53
53
|
# Optional LangChain VectorStore adapter.
|
|
54
54
|
langchain = ["langchain-core>=0.3.0"]
|
|
55
|
+
# Optional LlamaIndex VectorStore adapter.
|
|
56
|
+
llama-index = ["llama-index-core>=0.11.0"]
|
|
57
|
+
# Optional mem0 vector-store adapter.
|
|
58
|
+
mem0 = ["mem0ai>=2.0.0"]
|
|
55
59
|
# ruff is pinned (not floored) so `ruff format`/`ruff check` are reproducible across
|
|
56
60
|
# contributors and CI — the formatter's output can change between ruff releases.
|
|
57
61
|
dev = ["pytest>=8.3.0", "ruff==0.15.17"]
|