lodedb 0.2.1__tar.gz → 0.4.0__tar.gz

{lodedb-0.2.1 → lodedb-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lodedb
-Version: 0.2.1
+Version: 0.4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -8,21 +8,27 @@ Classifier: Topic :: Database :: Database Engines/Servers
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Dist: numpy>=2.0.0
 Requires-Dist: typer>=0.12.0
+Requires-Dist: onnxruntime>=1.20.0
+Requires-Dist: transformers>=4.40.0
 Requires-Dist: sentence-transformers>=3.0.0
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: pytest>=8.3.0 ; extra == 'dev'
 Requires-Dist: ruff==0.15.17 ; extra == 'dev'
 Requires-Dist: cupy-cuda12x>=13.0.0 ; platform_system == 'Linux' and extra == 'gpu'
+Requires-Dist: pillow>=10.0.0 ; extra == 'image'
 Requires-Dist: langchain-core>=0.3.0 ; extra == 'langchain'
 Requires-Dist: llama-index-core>=0.11.0 ; extra == 'llama-index'
 Requires-Dist: mcp>=1.0.0 ; extra == 'mcp'
 Requires-Dist: mem0ai>=2.0.0 ; extra == 'mem0'
+Requires-Dist: optimum[exporters]>=1.23.0,<2.0.0 ; extra == 'onnx-export'
 Provides-Extra: dev
 Provides-Extra: gpu
+Provides-Extra: image
 Provides-Extra: langchain
 Provides-Extra: llama-index
 Provides-Extra: mcp
 Provides-Extra: mem0
+Provides-Extra: onnx-export
 License-File: LICENSE
 License-File: NOTICE
 Summary: Local-first, privacy-by-default embedded vector database. Your data never leaves your machine.
@@ -36,23 +42,45 @@ Project-URL: Homepage, https://github.com/Egoist-Machines/LodeDB
 Project-URL: Issues, https://github.com/Egoist-Machines/LodeDB/issues
 Project-URL: Repository, https://github.com/Egoist-Machines/LodeDB
 
-# LodeDB
+<h1 align="center">LodeDB</h1>
+<p align="center">🔥 the <b>fastest</b> and <i>most compact</i> embedded vector database in the world 🌍</p>
 
 [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](pyproject.toml)
 
-**A fast, exact embedded vector database for local RAG: in-process, on-disk, no server.**
-
 *Built by [Egoist Machines, Inc.](https://egoistmachines.com) - efficient full-stack infrastructure
 for reliable AI systems.*
 
-Most embedded vector databases stop at the CPU. LodeDB runs the same on-disk index on the
-GPU when you have one: batched search hits **24k queries/sec on an A10 and 50k qps on an L40S**,
+LodeDB is great for local RAG; it's _extremely fast_, exact, in-process, and on-disk. We're the **best drop-in** durable memory backend for **LangChain, LlamaIndex, and mem0** across all metrics that matter: the most
+compact on disk, GPU-accelerated, and sub-millisecond on durable writes. Point any of them at
+LodeDB instead of its default store. Over 17.5k documents, per framework default:
+
+| vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
+|---|---|---|---|
+| On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
+| Single-query p50 (CPU) | **~510× faster** (0.57 vs 294 ms) | **~610× faster** (0.57 vs 349 ms) | **~48× faster** (0.64 vs 31 ms) |
+| Batched retrieval, 64 (GPU) | **~1,900×** (6,602 vs ~4 qps) | **~2,200×** (6,414 vs ~3 qps) | **~160×** (4,649 vs 29 qps) |
+| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.6 vs 0.5 ms (both sub-ms) |
+
+LodeDB matches sqlite-vec/qdrant's per-add latency with a significantly more compact footprint:
+
+| **embedded stores** | **durable add p50** | **single-query p50** | **batch-64/query** | **memory footprint** |
+| --- | ---: | ---: | ---: | ---: |
+| sqlite-vec | **0.4 ms** | 25.3 ms | 25.1 ms | 101 MB |
+| qdrant | **0.5 ms** | 27.3 ms | 27.0 ms | 85 MB |
+| **LodeDB** | **0.6 ms** | **0.49 ms** | **0.11 ms** | **29 MB** |
+| pgvector | 2.3 ms | 30.4 ms | 30.6 ms | 50 MB |
+| lancedb | 3.2 ms | 10.3 ms | 10.0 ms | 37 MB |
+| chroma | 6.5 ms | 3.0 ms | 3.1 ms | 151 MB |
+
+[Full benchmark, all backends
+(FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
+
+Most embedded vector databases stop at the CPU. LodeDB runs the same on-disk index **on the
+GPU** when you have one: batched search hits *24k queries/sec on an A10 and 50k qps on an L40S*,
 2.8× to 4.8× the all-CPU ceiling, with recall unchanged. It also persists changed rows
 incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
 
-Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
-
 - **GPU-resident batch search**: an fp16 copy of the index lives on the GPU, scored with a
   tiled GEMM plus a streaming top-k (`[gpu]`, Linux/CUDA). [How it works](#gpu-resident-index).
 - **O(changed) persistence**: commits only the rows that changed, 173× to 1,308× faster
@@ -65,10 +93,16 @@ Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
   [How it works](#concurrency--durability).
 - **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
   (counts, bytes, latency), never raw payloads.
-- **Local embeddings**: `sentence-transformers` on CUDA, MPS, or CPU.
-- **Batteries included**: a `lodedb` CLI, a loopback dev server, an
-  [MCP server](#use-as-an-mcp-server), and LangChain, LlamaIndex, and mem0 adapters
-  (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`).
+- **Local embeddings**: ONNX Runtime by default (lower per-query latency), with a PyTorch
+  `sentence-transformers` fallback; runs on CPU, CUDA, or MPS. Pick with `embedding_runtime=`.
+- **Multimodal**: index images and text in one shared CLIP space (`model="clip"`) for
+  cross-modal search, or bring your own vectors from any model.
+  [How it works](#multimodal--bring-your-own-vectors).
+- **Batteries included**: a `lodedb` CLI, a loopback/private-network dev server, an
+  [MCP server](#use-as-an-mcp-server), LangChain, LlamaIndex, and mem0 adapters
+  (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`), and a one-line
+  [PrivateGPT](https://github.com/zylon-ai/private-gpt) vector-store provider built on the
+  LlamaIndex adapter.
 
 > 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
 > available for commercial support, managed and at-scale serving, and on-prem / BYOC
@@ -86,7 +120,9 @@ the install with `lodedb doctor`. Optional extras:
 
 ```bash
 pip install "lodedb[gpu]"                            # GPU-resident scan (Linux/CUDA)
+pip install "lodedb[image]"                          # image + text (CLIP) embedding (model="clip")
 pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
+pip install "lodedb[onnx-export]"                    # export ONNX for a custom model (Optimum); presets need no export
 ```
 
 Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
@@ -144,7 +180,7 @@ Run with `uv run` (e.g. `uv run lodedb doctor`).
 ```python
 from lodedb import LodeDB
 
-db = LodeDB(path="./data", model="minilm")   # "minilm" (fast) | "bge" (quality)
+db = LodeDB(path="./data", model="minilm")   # "minilm" (fast) | "bge" (quality) | "clip" (image+text)
 
 fox = db.add("the quick brown fox jumps", metadata={"topic": "animals"})
 db.add("a lazy dog sleeps all day", metadata={"topic": "animals"})
@@ -168,8 +204,8 @@ db.persist()    # durable .tvim/.tvd/.jsd snapshot; replays on reopen
 
 Reopen with `LodeDB(path="./data")`; no migration step. Original text is kept in a
 `.tvtext` sidecar for `db.get`; pass `store_text=False` to keep none. Presets are `minilm`
-(384-dim) and `bge` (768-dim), with weights pulled from Hugging Face on first use. More in
-[`examples/`](examples/).
+(384-dim), `bge` (768-dim), and `clip` (512-dim, image+text), with weights pulled from Hugging
+Face on first use. More in [`examples/`](examples/).
 
 Need to read a store another process is writing to? Open it read-only. It takes no writer
 lock, so it never blocks on (or is blocked by) the writer:
@@ -196,7 +232,8 @@ db.search("E1234", k=5, mode="hybrid")  # BM25 + RRF: surfaces it in the top-k
 db.search("E1234", k=5, mode="lexical") # BM25 ranking alone, no vector scan
 ```
 
-### Prerequisites
+<details>
+<summary><b>Prerequisites</b></summary>
 
 `mode="hybrid"` and `mode="lexical"` build a BM25 index over your text, so they need a text
 source enabled when you open the database. `mode="vector"` (the default) needs nothing.
@@ -210,8 +247,10 @@ source enabled when you open the database. `mode="vector"` (the default) needs n
 Either source is enough and you can enable both. `store_text=True` is the default, so hybrid
 works out of the box. With neither source enabled, a hybrid or lexical query raises a clear,
 actionable error rather than silently degrading.
+</details>
 
-### How it works
+<details>
+<summary><b>How it works</b></summary>
 
 A `filter` constrains both rankers, so `mode="hybrid"` with a filter returns the true top-k of
 the matching subset. The vector half of a hybrid query runs on the same scan as `mode="vector"`,
@@ -219,8 +258,10 @@ including the GPU-resident batch scan that serves `search_many`; only the BM25 r
 fusion run on the CPU, and the vector kernel and on-disk format are untouched. The serving BM25
 index lives in memory and is maintained incrementally: a small mutation folds just the changed
 chunks into the existing index, so a single `add` never forces a full re-tokenization.
+</details>
 
-### Durable lexical index (`index_text=True`)
+<details>
+<summary><b>Durable lexical index (`index_text=True`)</b></summary>
 
 By default the BM25 index is rebuilt from the retained raw text, so it needs `store_text=True`
 and is re-tokenized on the first hybrid query after opening. Pass `index_text=True` to persist
@@ -241,6 +282,40 @@ db.close()
 reopened = LodeDB(path="./data", index_text=True, store_text=False)
 reopened.search("E1234", k=5, mode="hybrid")  # works after reopen, rebuilt from persisted terms
 ```
+</details>
+
+## Multimodal & bring-your-own vectors
+
+The storage and scan are modality-agnostic: TurboVec stores any normalized float32
+vector, so an image, audio, or video embedding is indexed and scanned exactly like a
+text one. There are two ways to use that.
+
+Bring your own vectors. Open a vector-only index at your dimension and pass the
+embeddings you already computed with any model (CLIP, SigLIP, ImageBind, an audio or
+video encoder, a hosted API). No embedding model is bundled on this path:
+
+```python
+db = LodeDB.open_vector_store("./media", vector_dim=512)
+db.add_vectors(image_vector, id="img-001", metadata={"path": "photos/img-001.jpg"})
+db.search_by_vector(query_vector, k=10)
+```
+
+Or use the built-in `clip` preset for image and text in one shared space, so a text
+query retrieves images and an image query retrieves images and text. It rides the base
+sentence-transformers stack and adds only Pillow (`pip install 'lodedb[image]'`):
+
+```python
+db = LodeDB("./gallery", model="clip")            # downloads clip-ViT-B-32 on first use
+db.add_image("photos/beach.jpg", metadata={"path": "photos/beach.jpg"})
+db.search("a beach at sunset", k=5)               # text -> image, cross-modal
+db.search_by_image("photos/beach.jpg", k=5)       # image -> image
+```
+
+The raw image is never stored; keep it on disk and put its path in `metadata`. Keep one
+embedding model per index (scores are only comparable within one space); the model
+identity is pinned and re-enforced on reopen. To hold several encoders side by side, use
+`LodeCollection` named spaces, and pass `embedder=` to drive an index with your own
+model. See [`docs/multimodal.md`](docs/multimodal.md).
 
 ## GPU-resident index
 
@@ -325,7 +400,7 @@ and end-to-end single-query latency is 5.7 ms p50.
 lodedb doctor      # capability report: embedding / GPU / TurboVec backend
 lodedb index ...   # build / add to an on-disk index
 lodedb query ...   # search
-lodedb serve       # loopback dev server (127.0.0.1, no auth)
+lodedb serve       # dev server (127.0.0.1 by default; private LAN only, no auth)
 lodedb mcp         # stdio MCP server for agent memory
 lodedb benchmark   # local, metrics-only benchmark
 ```
@@ -339,6 +414,9 @@ your host at `lodedb mcp`:
 
 ```bash
 pip install "lodedb[mcp]"
+
+# for coding assistant:
+lodedb mcp install --client claude-code  # or: claude-desktop | cursor | lm-studio | codex | all
 ```
 
 It exposes `lodedb_add`, `lodedb_search`, `lodedb_remove`, and `lodedb_stats`, plus
@@ -351,7 +429,7 @@ the server with `--exclude-text` to return metrics only (this also withdraws `lo
 `--no-store-text` to keep no text on disk at all. `lodedb_stats` is always metrics-only and raw
 query text never leaves the process.
 
-### One command
+### One command install
 
 `lodedb mcp install` writes the correct entry to a client's config for you, so you do not have to
 find the file or hand-write the JSON/TOML:
@@ -423,6 +501,21 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star
 - **Durability is `fast` by default.** Commits are *atomic* but not fsync'd. Pass
   `durability="fsync"` (or `--durability fsync` / `LODEDB_DURABILITY=fsync`) to fsync each
   file and its directory on commit for power-loss durability, at some commit-throughput cost.
+- **WAL commit by default for low-latency durable writes.** Each `add`/`remove` appends one
+  framed record to a `<key>.wal` log and a full generation is checkpointed periodically, so a
+  durable single add costs roughly an order of magnitude less than publishing a whole generation
+  per write, into the sqlite-vec/qdrant range (see the comparison up top). The WAL is replayed
+  crash-atomically on reopen (a half-written trailing record is discarded), every writable open
+  folds it into a clean committed generation, and `close()`/`persist()` checkpoint it. WAL is
+  single-writer: a concurrent `open_readonly` reader still loads a consistent committed
+  generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
+  `commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
+  publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
+  out-of-process readers must see each write the instant it commits. Note `<key>.wal` is
+  **payload-bearing before a checkpoint** (raw text under `store_text=True`, otherwise embedding
+  deltas plus, with `index_text=True`, lexical tokens), so treat it as sensitively as the data it
+  indexes; `persist()`/`close()` checkpoint and truncate it, and `generation` mode keeps no WAL.
+  See the [payload boundary](docs/architecture.md#persistence--payload-boundary) docs.
 - **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
 
 ## Limitations

{lodedb-0.2.1 → lodedb-0.4.0}/README.md

@@ -1,20 +1,42 @@
-# LodeDB
+<h1 align="center">LodeDB</h1>
+<p align="center">🔥 the <b>fastest</b> and <i>most compact</i> embedded vector database in the world 🌍</p>
 
 [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
 [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](pyproject.toml)
 
-**A fast, exact embedded vector database for local RAG: in-process, on-disk, no server.**
-
 *Built by [Egoist Machines, Inc.](https://egoistmachines.com) - efficient full-stack infrastructure
 for reliable AI systems.*
 
-Most embedded vector databases stop at the CPU. LodeDB runs the same on-disk index on the
-GPU when you have one: batched search hits **24k queries/sec on an A10 and 50k qps on an L40S**,
+LodeDB is great for local RAG; it's _extremely fast_, exact, in-process, and on-disk. We're the **best drop-in** durable memory backend for **LangChain, LlamaIndex, and mem0** across all metrics that matter: the most
+compact on disk, GPU-accelerated, and sub-millisecond on durable writes. Point any of them at
+LodeDB instead of its default store. Over 17.5k documents, per framework default:
+
+| vs the framework's default store | LangChain `InMemoryVectorStore` | LlamaIndex `SimpleVectorStore` | mem0 Qdrant |
+|---|---|---|---|
+| On-disk footprint | **7.2× smaller** (28 vs 199 MB) | **5.3× smaller** (28 vs 145 MB) | **4.6× smaller** (15 vs 70 MB) |
+| Single-query p50 (CPU) | **~510× faster** (0.57 vs 294 ms) | **~610× faster** (0.57 vs 349 ms) | **~48× faster** (0.64 vs 31 ms) |
+| Batched retrieval, 64 (GPU) | **~1,900×** (6,602 vs ~4 qps) | **~2,200×** (6,414 vs ~3 qps) | **~160×** (4,649 vs 29 qps) |
+| Durable add of one memory | **~10,000× faster** (0.8 ms vs 8.6 s) | **~22,000× faster** (0.9 ms vs 19.3 s) | 0.6 vs 0.5 ms (both sub-ms) |
+
+LodeDB matches sqlite-vec/qdrant's per-add latency with a significantly more compact footprint:
+
+| **embedded stores** | **durable add p50** | **single-query p50** | **batch-64/query** | **memory footprint** |
+| --- | ---: | ---: | ---: | ---: |
+| sqlite-vec | **0.4 ms** | 25.3 ms | 25.1 ms | 101 MB |
+| qdrant | **0.5 ms** | 27.3 ms | 27.0 ms | 85 MB |
+| **LodeDB** | **0.6 ms** | **0.49 ms** | **0.11 ms** | **29 MB** |
+| pgvector | 2.3 ms | 30.4 ms | 30.6 ms | 50 MB |
+| lancedb | 3.2 ms | 10.3 ms | 10.0 ms | 37 MB |
+| chroma | 6.5 ms | 3.0 ms | 3.1 ms | 151 MB |
+
+[Full benchmark, all backends
+(FAISS, Chroma, Qdrant, LanceDB, sqlite-vec, pgvector), and method.](benchmarks/memory_integrations)
+
+Most embedded vector databases stop at the CPU. LodeDB runs the same on-disk index **on the
+GPU** when you have one: batched search hits *24k queries/sec on an A10 and 50k qps on an L40S*,
 2.8× to 4.8× the all-CPU ceiling, with recall unchanged. It also persists changed rows
 incrementally, so a commit stays **sub-millisecond even at 1M vectors**.
 
-Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
-
 - **GPU-resident batch search**: an fp16 copy of the index lives on the GPU, scored with a
   tiled GEMM plus a streaming top-k (`[gpu]`, Linux/CUDA). [How it works](#gpu-resident-index).
 - **O(changed) persistence**: commits only the rows that changed, 173× to 1,308× faster
@@ -27,10 +49,16 @@ Fast on a laptop. Faster on a GPU. Exact every time. Never phones home.
   [How it works](#concurrency--durability).
 - **Private by default**: text, ids, and vectors stay local; telemetry is metrics-only
   (counts, bytes, latency), never raw payloads.
-- **Local embeddings**: `sentence-transformers` on CUDA, MPS, or CPU.
-- **Batteries included**: a `lodedb` CLI, a loopback dev server, an
-  [MCP server](#use-as-an-mcp-server), and LangChain, LlamaIndex, and mem0 adapters
-  (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`).
+- **Local embeddings**: ONNX Runtime by default (lower per-query latency), with a PyTorch
+  `sentence-transformers` fallback; runs on CPU, CUDA, or MPS. Pick with `embedding_runtime=`.
+- **Multimodal**: index images and text in one shared CLIP space (`model="clip"`) for
+  cross-modal search, or bring your own vectors from any model.
+  [How it works](#multimodal--bring-your-own-vectors).
+- **Batteries included**: a `lodedb` CLI, a loopback/private-network dev server, an
+  [MCP server](#use-as-an-mcp-server), LangChain, LlamaIndex, and mem0 adapters
+  (`VectorStore`s, plus a LlamaIndex `PropertyGraphStore`), and a one-line
+  [PrivateGPT](https://github.com/zylon-ai/private-gpt) vector-store provider built on the
+  LlamaIndex adapter.
 
 > 🏢 **Enterprise** The LodeDB core is Apache-2.0 and free to use. Enterprise licensing is
 > available for commercial support, managed and at-scale serving, and on-prem / BYOC
@@ -48,7 +76,9 @@ the install with `lodedb doctor`. Optional extras:
 
 ```bash
 pip install "lodedb[gpu]"                            # GPU-resident scan (Linux/CUDA)
+pip install "lodedb[image]"                          # image + text (CLIP) embedding (model="clip")
 pip install "lodedb[mcp,langchain,llama-index,mem0]" # MCP server + LangChain/LlamaIndex/mem0 adapters
+pip install "lodedb[onnx-export]"                    # export ONNX for a custom model (Optimum); presets need no export
 ```
 
 Using LodeDB as memory for a coding assistant? After installing the `mcp` extra, register its
@@ -106,7 +136,7 @@ Run with `uv run` (e.g. `uv run lodedb doctor`).
 ```python
 from lodedb import LodeDB
 
-db = LodeDB(path="./data", model="minilm")   # "minilm" (fast) | "bge" (quality)
+db = LodeDB(path="./data", model="minilm")   # "minilm" (fast) | "bge" (quality) | "clip" (image+text)
 
 fox = db.add("the quick brown fox jumps", metadata={"topic": "animals"})
 db.add("a lazy dog sleeps all day", metadata={"topic": "animals"})
@@ -130,8 +160,8 @@ db.persist()    # durable .tvim/.tvd/.jsd snapshot; replays on reopen
 
 Reopen with `LodeDB(path="./data")`; no migration step. Original text is kept in a
 `.tvtext` sidecar for `db.get`; pass `store_text=False` to keep none. Presets are `minilm`
-(384-dim) and `bge` (768-dim), with weights pulled from Hugging Face on first use. More in
-[`examples/`](examples/).
+(384-dim), `bge` (768-dim), and `clip` (512-dim, image+text), with weights pulled from Hugging
+Face on first use. More in [`examples/`](examples/).
 
 Need to read a store another process is writing to? Open it read-only. It takes no writer
 lock, so it never blocks on (or is blocked by) the writer:
@@ -158,7 +188,8 @@ db.search("E1234", k=5, mode="hybrid")  # BM25 + RRF: surfaces it in the top-k
 db.search("E1234", k=5, mode="lexical") # BM25 ranking alone, no vector scan
 ```
 
-### Prerequisites
+<details>
+<summary><b>Prerequisites</b></summary>
 
 `mode="hybrid"` and `mode="lexical"` build a BM25 index over your text, so they need a text
 source enabled when you open the database. `mode="vector"` (the default) needs nothing.
@@ -172,8 +203,10 @@ source enabled when you open the database. `mode="vector"` (the default) needs n
 Either source is enough and you can enable both. `store_text=True` is the default, so hybrid
 works out of the box. With neither source enabled, a hybrid or lexical query raises a clear,
 actionable error rather than silently degrading.
+</details>
 
-### How it works
+<details>
+<summary><b>How it works</b></summary>
 
 A `filter` constrains both rankers, so `mode="hybrid"` with a filter returns the true top-k of
 the matching subset. The vector half of a hybrid query runs on the same scan as `mode="vector"`,
@@ -181,8 +214,10 @@ including the GPU-resident batch scan that serves `search_many`; only the BM25 r
 fusion run on the CPU, and the vector kernel and on-disk format are untouched. The serving BM25
 index lives in memory and is maintained incrementally: a small mutation folds just the changed
 chunks into the existing index, so a single `add` never forces a full re-tokenization.
+</details>
 
-### Durable lexical index (`index_text=True`)
+<details>
+<summary><b>Durable lexical index (`index_text=True`)</b></summary>
 
 By default the BM25 index is rebuilt from the retained raw text, so it needs `store_text=True`
 and is re-tokenized on the first hybrid query after opening. Pass `index_text=True` to persist
@@ -203,6 +238,40 @@ db.close()
 reopened = LodeDB(path="./data", index_text=True, store_text=False)
 reopened.search("E1234", k=5, mode="hybrid")  # works after reopen, rebuilt from persisted terms
 ```
+</details>
+
+## Multimodal & bring-your-own vectors
+
+The storage and scan are modality-agnostic: TurboVec stores any normalized float32
+vector, so an image, audio, or video embedding is indexed and scanned exactly like a
+text one. There are two ways to use that.
+
+Bring your own vectors. Open a vector-only index at your dimension and pass the
+embeddings you already computed with any model (CLIP, SigLIP, ImageBind, an audio or
+video encoder, a hosted API). No embedding model is bundled on this path:
+
+```python
+db = LodeDB.open_vector_store("./media", vector_dim=512)
+db.add_vectors(image_vector, id="img-001", metadata={"path": "photos/img-001.jpg"})
+db.search_by_vector(query_vector, k=10)
+```
+
+Or use the built-in `clip` preset for image and text in one shared space, so a text
+query retrieves images and an image query retrieves images and text. It rides the base
+sentence-transformers stack and adds only Pillow (`pip install 'lodedb[image]'`):
+
+```python
+db = LodeDB("./gallery", model="clip")            # downloads clip-ViT-B-32 on first use
+db.add_image("photos/beach.jpg", metadata={"path": "photos/beach.jpg"})
+db.search("a beach at sunset", k=5)               # text -> image, cross-modal
+db.search_by_image("photos/beach.jpg", k=5)       # image -> image
+```
+
+The raw image is never stored; keep it on disk and put its path in `metadata`. Keep one
+embedding model per index (scores are only comparable within one space); the model
+identity is pinned and re-enforced on reopen. To hold several encoders side by side, use
+`LodeCollection` named spaces, and pass `embedder=` to drive an index with your own
+model. See [`docs/multimodal.md`](docs/multimodal.md).
 
 ## GPU-resident index
 
@@ -287,7 +356,7 @@ and end-to-end single-query latency is 5.7 ms p50.
 lodedb doctor      # capability report: embedding / GPU / TurboVec backend
 lodedb index ...   # build / add to an on-disk index
 lodedb query ...   # search
-lodedb serve       # loopback dev server (127.0.0.1, no auth)
+lodedb serve       # dev server (127.0.0.1 by default; private LAN only, no auth)
 lodedb mcp         # stdio MCP server for agent memory
 lodedb benchmark   # local, metrics-only benchmark
 ```
@@ -301,6 +370,9 @@ your host at `lodedb mcp`:
 
 ```bash
 pip install "lodedb[mcp]"
+
+# for coding assistant:
+lodedb mcp install --client claude-code  # or: claude-desktop | cursor | lm-studio | codex | all
 ```
 
 It exposes `lodedb_add`, `lodedb_search`, `lodedb_remove`, and `lodedb_stats`, plus
@@ -313,7 +385,7 @@ the server with `--exclude-text` to return metrics only (this also withdraws `lo
 `--no-store-text` to keep no text on disk at all. `lodedb_stats` is always metrics-only and raw
 query text never leaves the process.
 
-### One command
+### One command install
 
 `lodedb mcp install` writes the correct entry to a client's config for you, so you do not have to
 find the file or hand-write the JSON/TOML:
@@ -385,6 +457,21 @@ See [`examples/mcp_config.json`](examples/mcp_config.json) for a copy-paste star
 - **Durability is `fast` by default.** Commits are *atomic* but not fsync'd. Pass
   `durability="fsync"` (or `--durability fsync` / `LODEDB_DURABILITY=fsync`) to fsync each
   file and its directory on commit for power-loss durability, at some commit-throughput cost.
+- **WAL commit by default for low-latency durable writes.** Each `add`/`remove` appends one
+  framed record to a `<key>.wal` log and a full generation is checkpointed periodically, so a
+  durable single add costs roughly an order of magnitude less than publishing a whole generation
+  per write, into the sqlite-vec/qdrant range (see the comparison up top). The WAL is replayed
+  crash-atomically on reopen (a half-written trailing record is discarded), every writable open
+  folds it into a clean committed generation, and `close()`/`persist()` checkpoint it. WAL is
+  single-writer: a concurrent `open_readonly` reader still loads a consistent committed
+  generation, but the last *checkpointed* one, not the writer's in-flight WAL. Pass
+  `commit_mode="generation"` (or `LODEDB_COMMIT_MODE=generation`) for the classic path that
+  publishes a crash-atomic, MVCC-readable generation on every write; pick it when many
+  out-of-process readers must see each write the instant it commits. Note `<key>.wal` is
+  **payload-bearing before a checkpoint** (raw text under `store_text=True`, otherwise embedding
+  deltas plus, with `index_text=True`, lexical tokens), so treat it as sensitively as the data it
+  indexes; `persist()`/`close()` checkpoint and truncate it, and `generation` mode keeps no WAL.
+  See the [payload boundary](docs/architecture.md#persistence--payload-boundary) docs.
 - **Local filesystems only.** The OS advisory lock is unreliable on NFS/SMB.
 
 ## Limitations

{lodedb-0.2.1 → lodedb-0.4.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "maturin"
 [project]
 name = "lodedb"
 # Static (maturin reads it here); keep in sync with src/lodedb/__init__.py __version__.
-version = "0.2.1"
+version = "0.4.0"
 description = "Local-first, privacy-by-default embedded vector database. Your data never leaves your machine."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -33,13 +33,19 @@ classifiers = [
 # Lean runtime set (see docs/architecture.md). The compact TurboVec CPU core +
 # .tvim/.tvd persistence is the bundled `lodedb._turbovec` extension (compiled
 # into the wheel, see [tool.maturin]), not a PyPI dependency. numpy = embedding/
-# scan I/O; typer = CLI; sentence-transformers = default embedding backend (pulls
-# torch/transformers); pyyaml = loads the bundled route registry. Heavy research
-# deps (faiss/modal/mteb/datasets/matplotlib/sklearn) are deliberately NOT here
-# and are guarded out by tests/test_import_boundary.py.
+# scan I/O; typer = CLI; pyyaml = loads the bundled route registry.
+# Embedding runtime: onnxruntime is the default (faster feature extraction; the
+# preset models ship a prebuilt ONNX on the Hub, fetched on first use) with
+# transformers for tokenization; sentence-transformers (which pulls torch) is the
+# fallback when ONNX cannot be set up. Exporting ONNX for a model without a prebuilt
+# graph is the opt-in [onnx-export] extra (Optimum), not part of the base install.
+# Heavy research deps (faiss/modal/mteb/datasets/matplotlib/sklearn) are
+# deliberately NOT here and are guarded out by tests/test_import_boundary.py.
 dependencies = [
   "numpy>=2.0.0",
   "typer>=0.12.0",
+  "onnxruntime>=1.20.0",
+  "transformers>=4.40.0",
   "sentence-transformers>=3.0.0",
   "pyyaml>=6.0.0",
 ]
@@ -48,6 +54,10 @@ dependencies = [
 # Opt-in GPU-resident exact scan on CUDA hosts (gpu_turbovec); absent CUDA, LodeDB uses
 # the CPU scan. cupy wheels are Linux/CUDA-only, so this resolves to nothing elsewhere.
 gpu = ["cupy-cuda12x>=13.0.0; platform_system == 'Linux'"]
+# Export an ONNX graph for a model that does not ship a prebuilt one on the Hub
+# (the built-in minilm/bge presets do, so this is only needed for custom models).
+# Optimum runs the export in a subprocess, so it never imports into the lodedb process.
+onnx-export = ["optimum[exporters]>=1.23.0,<2.0.0"]
 # Optional stdio MCP server so coding agents can use LodeDB as local memory.
 mcp = ["mcp>=1.0.0"]
 # Optional LangChain VectorStore adapter.
@@ -56,6 +66,11 @@ langchain = ["langchain-core>=0.3.0"]
 llama-index = ["llama-index-core>=0.11.0"]
 # Optional mem0 vector-store adapter.
 mem0 = ["mem0ai>=2.0.0"]
+# Optional image + text (CLIP) embedding via the "clip" preset / add_image. The CLIP
+# model runs on the existing sentence-transformers + torch stack (already in the base
+# deps), so this adds only Pillow for decoding image files. Both are imported lazily,
+# so a plain `import lodedb` pulls neither (guarded by tests/test_import_boundary.py).
+image = ["pillow>=10.0.0"]
 # ruff is pinned (not floored) so `ruff format`/`ruff check` are reproducible across
 # contributors and CI — the formatter's output can change between ruff releases.
 dev = ["pytest>=8.3.0", "ruff==0.15.17"]

{lodedb-0.2.1 → lodedb-0.4.0}/src/lodedb/__init__.py

@@ -11,6 +11,8 @@ from __future__ import annotations
 from lodedb.local import (
     LOCAL_MODEL_PRESETS,
     ConcurrentWriterError,
+    ImageEmbeddingUnsupportedError,
+    LodeCollection,
     LodeDB,
     LodeSearchHit,
     ReadOnlyError,
@@ -19,11 +21,13 @@ from lodedb.local import (
 from lodedb.local.cli import app, main
 
 # Keep in sync with `version` in pyproject.toml (the release workflow asserts they match).
-__version__ = "0.2.1"
+__version__ = "0.4.0"
 
 __all__ = [
     "LOCAL_MODEL_PRESETS",
     "ConcurrentWriterError",
+    "ImageEmbeddingUnsupportedError",
+    "LodeCollection",
     "LodeDB",
     "LodeSearchHit",
     "ReadOnlyError",