citadeldb-llamaindex 2.0.0__tar.gz

citadeldb_llamaindex-2.0.0/.gitignore

@@ -0,0 +1,18 @@
+/target
+**/*.rs.bk
+*.swp
+*.swo
+*~
+.DS_Store
+site/public/
+site/static/wasm/*.wasm
+site/static/wasm/*.js
+/notes/
+__pycache__/
+*.py[cod]
+# maturin build output; the .pyd is caught by the line above only by accident.
+*.pdb
+.pytest_cache/
+.mypy_cache/
+/dist/
+packaging/*/dist/

citadeldb_llamaindex-2.0.0/PKG-INFO

@@ -0,0 +1,106 @@
+Metadata-Version: 2.5
+Name: citadeldb-llamaindex
+Version: 2.0.0
+Summary: LlamaIndex vector storage backed by Citadel: encrypted at rest, with deletes that destroy the key
+Project-URL: Homepage, https://citadeldb.dev
+Project-URL: Repository, https://github.com/yp3y5akh0v/citadel
+Author: Yuriy Peysakhov
+License-Expression: Apache-2.0
+Keywords: encryption,llama-index,llamaindex,memory,rag,vector-store
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: citadeldb<3,>=2.0
+Requires-Dist: llama-index-core<0.15,>=0.12
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# citadeldb-llamaindex
+
+A [LlamaIndex](https://github.com/run-llama/llama_index) vector store backed by
+[Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and deletes
+that destroy the key, not just the row.
+
+```
+pip install citadeldb-llamaindex
+```
+
+```python
+from llama_index.core import Document, StorageContext, VectorStoreIndex
+from citadeldb_llamaindex import CitadelVectorStore
+
+store = CitadelVectorStore("corpus.cdl", key="your-passphrase", dim=1536)
+documents = [Document(text="the deployment failed because the disk was full")]
+
+index = VectorStoreIndex.from_documents(
+    documents,
+    storage_context=StorageContext.from_defaults(vector_store=store),
+)
+
+index.as_query_engine().query("why did the release break?")
+```
+
+`dim` must match your embedding model: 1536 for OpenAI `text-embedding-3-small`, 3072 for
+`text-embedding-3-large`, 1024 for `e5-large`.
+
+## Deletes destroy the key
+
+Every node is sealed under its own key. Deleting destroys that key and then removes the
+row, so any ciphertext surviving elsewhere stays unreadable.
+
+```python
+index.delete_ref_doc("doc-42")          # every node from that document
+store.forget_document("doc-42")         # the same, returning a count for the record
+store.clear()                           # the whole corpus
+```
+
+The node's key is gone, not just its entry in an index.
+
+## Filters
+
+Filtering matches LlamaIndex's own evaluator, and `similarity_top_k` is honoured: a filter
+matching only distant nodes still returns them, however many others outrank them. The same
+holds for `node_ids` and `doc_ids`, which name nodes exactly.
+
+String equality under a top-level `AND` is pushed into the scan so it narrows candidates
+before top-k. Everything else is evaluated afterwards, so the two agree: numbers are not
+pushed, because `EQ` here is Python's `==` (`1 == 1.0`) where the stored comparison is
+JSON-type exact.
+
+```python
+from llama_index.core.vector_stores.types import (
+    FilterOperator, MetadataFilter, MetadataFilters,
+)
+
+index.as_retriever(
+    filters=MetadataFilters(filters=[
+        MetadataFilter(key="year", value=2026, operator=FilterOperator.EQ),
+    ])
+).retrieve("...")
+```
+
+Filter semantics come from LlamaIndex's own evaluator, so every operator behaves exactly
+as it does with the reference store. Under `OR` or `NOT` nothing is pushed, because a
+pushed leaf would drop rows the filter keeps.
+
+## Notes
+
+LlamaIndex embeds before it calls a store, so a node arrives with its vector already
+computed and that vector is written straight onto the atom. Nothing here re-embeds, so
+writes and queries stay in one vector space.
+
+The node is stored whole, minus its text, which is kept once as the atom's searchable
+content and restored on read. Metadata, relationships and node type all round-trip.
+
+Citadel is embedded and one process owns the file. A path already open on this thread,
+under the same passphrase, is shared, so this can sit on the same database as another
+Citadel adapter; construct them on the same thread.
+
+## License
+
+Apache-2.0

citadeldb_llamaindex-2.0.0/README.md

@@ -0,0 +1,84 @@
+# citadeldb-llamaindex
+
+A [LlamaIndex](https://github.com/run-llama/llama_index) vector store backed by
+[Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and deletes
+that destroy the key, not just the row.
+
+```
+pip install citadeldb-llamaindex
+```
+
+```python
+from llama_index.core import Document, StorageContext, VectorStoreIndex
+from citadeldb_llamaindex import CitadelVectorStore
+
+store = CitadelVectorStore("corpus.cdl", key="your-passphrase", dim=1536)
+documents = [Document(text="the deployment failed because the disk was full")]
+
+index = VectorStoreIndex.from_documents(
+    documents,
+    storage_context=StorageContext.from_defaults(vector_store=store),
+)
+
+index.as_query_engine().query("why did the release break?")
+```
+
+`dim` must match your embedding model: 1536 for OpenAI `text-embedding-3-small`, 3072 for
+`text-embedding-3-large`, 1024 for `e5-large`.
+
+## Deletes destroy the key
+
+Every node is sealed under its own key. Deleting destroys that key and then removes the
+row, so any ciphertext surviving elsewhere stays unreadable.
+
+```python
+index.delete_ref_doc("doc-42")          # every node from that document
+store.forget_document("doc-42")         # the same, returning a count for the record
+store.clear()                           # the whole corpus
+```
+
+The node's key is gone, not just its entry in an index.
+
+## Filters
+
+Filtering matches LlamaIndex's own evaluator, and `similarity_top_k` is honoured: a filter
+matching only distant nodes still returns them, however many others outrank them. The same
+holds for `node_ids` and `doc_ids`, which name nodes exactly.
+
+String equality under a top-level `AND` is pushed into the scan so it narrows candidates
+before top-k. Everything else is evaluated afterwards, so the two agree: numbers are not
+pushed, because `EQ` here is Python's `==` (`1 == 1.0`) where the stored comparison is
+JSON-type exact.
+
+```python
+from llama_index.core.vector_stores.types import (
+    FilterOperator, MetadataFilter, MetadataFilters,
+)
+
+index.as_retriever(
+    filters=MetadataFilters(filters=[
+        MetadataFilter(key="year", value=2026, operator=FilterOperator.EQ),
+    ])
+).retrieve("...")
+```
+
+Filter semantics come from LlamaIndex's own evaluator, so every operator behaves exactly
+as it does with the reference store. Under `OR` or `NOT` nothing is pushed, because a
+pushed leaf would drop rows the filter keeps.
+
+## Notes
+
+LlamaIndex embeds before it calls a store, so a node arrives with its vector already
+computed and that vector is written straight onto the atom. Nothing here re-embeds, so
+writes and queries stay in one vector space.
+
+The node is stored whole, minus its text, which is kept once as the atom's searchable
+content and restored on read. Metadata, relationships and node type all round-trip.
+
+Citadel is embedded and one process owns the file. A path already open on this thread,
+under the same passphrase, is shared, so this can sit on the same database as another
+Citadel adapter; construct them on the same thread.
+
+## License
+
+Apache-2.0

citadeldb_llamaindex-2.0.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "citadeldb-llamaindex"
+dynamic = ["version"]
+description = "LlamaIndex vector storage backed by Citadel: encrypted at rest, with deletes that destroy the key"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{ name = "Yuriy Peysakhov" }]
+keywords = ["llamaindex", "llama-index", "rag", "vector-store", "memory", "encryption"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Topic :: Database",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = ["citadeldb>=2.0,<3", "llama-index-core>=0.12,<0.15"]
+
+[project.optional-dependencies]
+test = ["pytest>=8", "pytest-asyncio>=0.23"]
+
+[project.urls]
+Homepage = "https://citadeldb.dev"
+Repository = "https://github.com/yp3y5akh0v/citadel"
+
+# The version comes from the release tag, so there is nothing to bump.
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "../..", tag_regex = '^v(?P<version>\d+\.\d+\.\d+)$' }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/citadeldb_llamaindex"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

citadeldb_llamaindex-2.0.0/src/citadeldb_llamaindex/__init__.py

@@ -0,0 +1,13 @@
+"""LlamaIndex vector storage backed by Citadel, encrypted at rest."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .vector_store import CitadelVectorStore
+
+__all__ = ["CitadelVectorStore", "__version__"]
+
+
+try:
+    __version__ = version("citadeldb-llamaindex")
+except PackageNotFoundError:  # running from a source tree, never installed
+    __version__ = "0+unknown"

citadeldb_llamaindex-2.0.0/src/citadeldb_llamaindex/vector_store.py

@@ -0,0 +1,374 @@
+"""LlamaIndex vector store over an encrypted Citadel region."""
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import citadeldb
+from llama_index.core.schema import BaseNode, MetadataMode
+from llama_index.core.vector_stores.types import (
+    BasePydanticVectorStore,
+    FilterCondition,
+    FilterOperator,
+    MetadataFilters,
+    VectorStoreQuery,
+    VectorStoreQueryMode,
+    VectorStoreQueryResult,
+)
+from llama_index.core.vector_stores.utils import (
+    build_metadata_filter_fn,
+    metadata_dict_to_node,
+    node_to_metadata_dict,
+)
+from pydantic import PrivateAttr
+
+KIND = "node"
+DEFAULT_PATH = "llamaindex.cdl"
+DEFAULT_REGION = "nodes"
+# OpenAI text-embedding-ada-002 / 3-small, LlamaIndex's default width.
+DEFAULT_DIM = 1536
+# The region is read whole where containment cannot express the predicate.
+PAGE = 10_000
+
+# Recall fuses vector, keyword and recency, so hybrid needs no extra call.
+_SUPPORTED_MODES = (VectorStoreQueryMode.DEFAULT, VectorStoreQueryMode.HYBRID)
+
+
+# A Database is pinned to its opening thread, so workers take Memory, not self.
+
+
+def _pushdown(filters: MetadataFilters | None) -> dict[str, Any] | None:
+    """Equality leaves of a top-level AND; this narrows and never decides.
+
+    Only string leaves are pushed. Containment compares JSON types exactly where
+    EQ is Python `==`, which holds `1 == 1.0 == True`, so pushing a number or a
+    bool would drop rows `_keep` would have kept - deciding, not narrowing.
+    Equal strings are the one case where the two agree.
+    """
+    if filters is None or not filters.filters:
+        return None
+    if filters.condition != FilterCondition.AND:
+        return None
+    eq: dict[str, Any] = {}
+    for f in filters.filters:
+        if isinstance(f, MetadataFilters):
+            return None  # nested filters are refused downstream; do not guess here
+        if f.operator == FilterOperator.EQ and isinstance(f.value, str):
+            eq[f.key] = f.value
+    return {"meta": eq} if eq else None
+
+
+def _keep(hits: list[Any], filters: MetadataFilters | None) -> list[Any]:
+    """Hits whose metadata satisfies `filters`."""
+    if filters is None or not filters.filters:
+        return hits
+    by_id = {h.payload["nid"]: h.payload.get("meta", {}) for h in hits}
+    matches = build_metadata_filter_fn(lambda nid: by_id.get(nid, {}), filters)
+    return [h for h in hits if matches(h.payload["nid"])]
+
+
+def _ranked(
+    mem: Any,
+    region: str,
+    *,
+    want: int,
+    surviving: Any,
+    **recall: Any,
+) -> list[Any]:
+    """Ranked hits that pass `surviving`, widening until `want` of them or the
+    region runs out.
+
+    The engine answers `k` rows matching what was pushed to it; a predicate
+    containment cannot express is settled out here, and a fixed `k` would answer
+    short whenever the survivors rank below it. Fewer hits than asked for means
+    the region is exhausted, so the loop ends on the data rather than a constant.
+    Widening only appends: recall is ranked, so a larger `k` returns a superset
+    in the same order.
+    """
+    k = max(want, 32)
+    while True:
+        hits = mem.recall(region, k=k, **recall)
+        kept = surviving(hits)
+        if len(kept) >= want or len(hits) < k:
+            return kept[:want]
+        k *= 2
+
+
+def _node_of(hit: Any) -> BaseNode:
+    """Rebuild the node; the text is restored from the atom."""
+    return metadata_dict_to_node(hit.payload["meta"], text=hit.text)
+
+
+def _fetch(mem: Any, region: str, criterion: dict[str, Any] | None) -> list[Any]:
+    """Page to the end: one fetch is bounded, and a partial erase must not look whole."""
+    out: list[Any] = []
+    after = None
+    while True:
+        page = mem.fetch(region, KIND, payload_filter=criterion, limit=PAGE, after_id=after)
+        out.extend(page)
+        if len(page) < PAGE:
+            return out
+        after = page[-1].id
+
+
+def _add(mem: Any, region: str, dim: int, nodes: list[BaseNode]) -> list[str]:
+    atoms: dict[str, dict[str, Any]] = {}
+    ids: list[str] = []
+    for node in nodes:
+        if node.embedding is None:
+            raise ValueError(
+                f"node {node.node_id} has no embedding. LlamaIndex embeds before it "
+                f"calls a store, so index through VectorStoreIndex or set "
+                f"node.embedding yourself."
+            )
+        if len(node.embedding) != dim:
+            raise ValueError(
+                f"node {node.node_id} has a {len(node.embedding)}-dimension embedding "
+                f"but this region is {dim}. Build the store with "
+                f"dim={len(node.embedding)} to match your embedding model."
+            )
+        # An id repeated in one call keeps its last value, like the reference.
+        atoms[node.node_id] = {
+            "kind": KIND,
+            # What recall matches on, and the text the node is rebuilt with.
+            "text": node.get_content(metadata_mode=MetadataMode.NONE) or "",
+            # Storing the framework's vector keeps one vector space.
+            "embedding": list(node.embedding),
+            "payload": {
+                "nid": node.node_id,
+                "ref": node.ref_doc_id or "",
+                # remove_text: the text would otherwise be stored twice.
+                "meta": node_to_metadata_dict(node, remove_text=True),
+            },
+        }
+        ids.append(node.node_id)
+    if not atoms:
+        return ids
+    # Re-adding a stored id replaces it, as the reference store's dict does.
+    # Keyed on the node id and committed as one transaction, so two indexers
+    # adding the same node supersede rather than each leaving a row behind.
+    mem.remember_replacing_keyed_batch(
+        region, [(atom, nid) for nid, atom in atoms.items()]
+    )
+    # One id per input node, duplicates included.
+    return ids
+
+
+def _erase(mem: Any, region: str, hits: list[Any]) -> int:
+    if not hits:
+        return 0
+    return mem.forget(region, [h.id for h in hits]).erased_count
+
+
+def _delete_ref(mem: Any, region: str, ref_doc_id: str) -> int:
+    return _erase(mem, region, _fetch(mem, region, {"ref": ref_doc_id}))
+
+
+def _selected(
+    mem: Any,
+    region: str,
+    node_ids: list[str] | None,
+    filters: MetadataFilters | None,
+) -> list[Any]:
+    """Atoms named by `node_ids`, narrowed by `filters`. Both may be absent."""
+    if node_ids is not None:
+        wanted = set(node_ids)
+        # One indexed lookup per id beats scanning a region to find a handful.
+        hits = [h for nid in wanted for h in _fetch(mem, region, {"nid": nid})]
+    else:
+        hits = _fetch(mem, region, _pushdown(filters))
+    return _keep(hits, filters)
+
+
+def _get_nodes(
+    mem: Any, region: str, node_ids: list[str] | None, filters: MetadataFilters | None
+) -> list[BaseNode]:
+    return [_node_of(h) for h in _selected(mem, region, node_ids, filters)]
+
+
+def _delete_nodes(
+    mem: Any, region: str, node_ids: list[str] | None, filters: MetadataFilters | None
+) -> int:
+    return _erase(mem, region, _selected(mem, region, node_ids, filters))
+
+
+def _clear(mem: Any, region: str) -> int:
+    return _erase(mem, region, _fetch(mem, region, None))
+
+
+def _query(
+    mem: Any, region: str, query: VectorStoreQuery
+) -> VectorStoreQueryResult:
+    if query.mode not in _SUPPORTED_MODES:
+        raise NotImplementedError(
+            f"query mode {query.mode} is not supported; Citadel serves "
+            f"{' and '.join(m.value for m in _SUPPORTED_MODES)}"
+        )
+    if query.query_embedding is None:
+        raise ValueError(
+            "query_embedding is required: this store does not embed, so the query "
+            "vector has to come from the index's embed_model"
+        )
+    top_k = max(query.similarity_top_k, 0)
+    if top_k == 0:
+        return VectorStoreQueryResult(nodes=[], similarities=[], ids=[])
+
+    criterion = _pushdown(query.filters)
+    options = (
+        citadeldb.RecallOptions(payload_filter=criterion) if criterion else None
+    )
+
+    def surviving(hits: list[Any]) -> list[Any]:
+        hits = _keep(hits, query.filters)
+        if query.node_ids:
+            allowed = set(query.node_ids)
+            hits = [h for h in hits if h.payload["nid"] in allowed]
+        if query.doc_ids:
+            allowed = set(query.doc_ids)
+            hits = [h for h in hits if h.payload.get("ref") in allowed]
+        return hits
+
+    hits = _ranked(
+        mem,
+        region,
+        want=top_k,
+        surviving=surviving,
+        embedding=query.query_embedding,
+        kinds=[KIND],
+        options=options,
+    )
+    return VectorStoreQueryResult(
+        nodes=[_node_of(h) for h in hits],
+        # Distance can dip below zero at real widths; cosine cannot exceed 1.
+        similarities=[
+            min(1.0, 1.0 - h.distance) if h.distance is not None else h.score
+            for h in hits
+        ],
+        ids=[h.payload["nid"] for h in hits],
+    )
+
+
+class CitadelVectorStore(BasePydanticVectorStore):
+    """A LlamaIndex vector store backed by one encrypted Citadel region."""
+
+    stores_text: bool = True
+    is_embedding_query: bool = True
+
+    _db: Any = PrivateAttr(default=None)
+    _mem: Any = PrivateAttr(default=None)
+    _region: str = PrivateAttr(default=DEFAULT_REGION)
+    _dim: int = PrivateAttr(default=DEFAULT_DIM)
+
+    def __init__(
+        self,
+        path: str = DEFAULT_PATH,
+        key: str = "",
+        *,
+        region: str = DEFAULT_REGION,
+        dim: int = DEFAULT_DIM,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+        if not key:
+            raise ValueError("a passphrase is required: the corpus is the payload")
+        try:
+            self._db = citadeldb.connect(path, key=key, region_keys=True)
+        except citadeldb.OperationalError as e:
+            if "locked" not in str(e):
+                raise
+            raise RuntimeError(
+                f"{path} is open in another process. Citadel is embedded, so one "
+                f"process owns the file."
+            ) from e
+        self._mem = self._db.memory()
+        self._region = region
+        self._dim = dim
+        # Idempotent for a region of the same width, so a dim clash raises here.
+        self._mem.create_encrypted_region(region, citadeldb.MockEmbedder(dim=dim))
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "CitadelVectorStore"
+
+    @property
+    def client(self) -> Any:
+        """The Citadel memory engine backing this store."""
+        return self._mem
+
+    # ---- the abstract surface --------------------------------------------
+
+    def add(self, nodes: list[BaseNode], **kwargs: Any) -> list[str]:
+        return _add(self._mem, self._region, self._dim, list(nodes))
+
+    def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
+        _delete_ref(self._mem, self._region, ref_doc_id)
+
+    def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
+        return _query(self._mem, self._region, query)
+
+    # ---- the optional surface --------------------------------------------
+
+    def get_nodes(
+        self,
+        node_ids: list[str] | None = None,
+        filters: MetadataFilters | None = None,
+    ) -> list[BaseNode]:
+        return _get_nodes(self._mem, self._region, node_ids, filters)
+
+    def delete_nodes(
+        self,
+        node_ids: list[str] | None = None,
+        filters: MetadataFilters | None = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        _delete_nodes(self._mem, self._region, node_ids, filters)
+
+    def clear(self) -> None:
+        _clear(self._mem, self._region)
+
+    # ---- async -----------------------------------------------------------
+    # The bindings are sync, so a worker thread keeps the event loop free.
+
+    async def async_add(self, nodes: list[BaseNode], **kwargs: Any) -> list[str]:
+        return await asyncio.to_thread(
+            _add, self._mem, self._region, self._dim, list(nodes)
+        )
+
+    async def adelete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
+        await asyncio.to_thread(_delete_ref, self._mem, self._region, ref_doc_id)
+
+    async def aquery(
+        self, query: VectorStoreQuery, **kwargs: Any
+    ) -> VectorStoreQueryResult:
+        return await asyncio.to_thread(_query, self._mem, self._region, query)
+
+    async def aget_nodes(
+        self,
+        node_ids: list[str] | None = None,
+        filters: MetadataFilters | None = None,
+    ) -> list[BaseNode]:
+        return await asyncio.to_thread(
+            _get_nodes, self._mem, self._region, node_ids, filters
+        )
+
+    async def adelete_nodes(
+        self,
+        node_ids: list[str] | None = None,
+        filters: MetadataFilters | None = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        await asyncio.to_thread(
+            _delete_nodes, self._mem, self._region, node_ids, filters
+        )
+
+    async def aclear(self) -> None:
+        await asyncio.to_thread(_clear, self._mem, self._region)
+
+    # ---- beyond the protocol ---------------------------------------------
+
+    def forget_document(self, ref_doc_id: str) -> int:
+        """Destroy a source document's nodes, returning the number erased."""
+        return _delete_ref(self._mem, self._region, ref_doc_id)
+
+    def count(self) -> int:
+        return len(_fetch(self._mem, self._region, None))

citadeldb_llamaindex-2.0.0/tests/test_vector_store.py

@@ -0,0 +1,527 @@
+import pytest
+from llama_index.core.schema import NodeRelationship, RelatedNodeInfo, TextNode
+from llama_index.core.vector_stores.types import (
+    BasePydanticVectorStore,
+    FilterCondition,
+    FilterOperator,
+    MetadataFilter,
+    MetadataFilters,
+    VectorStoreQuery,
+    VectorStoreQueryMode,
+)
+
+from citadeldb_llamaindex import CitadelVectorStore
+from citadeldb_llamaindex.vector_store import DEFAULT_REGION, KIND
+
+DIM = 8
+
+
+def vec(axis: int) -> list[float]:
+    v = [0.0] * DIM
+    v[axis] = 1.0
+    return v
+
+
+def node(nid, text, axis, *, meta=None, ref=None):
+    n = TextNode(id_=nid, text=text, metadata=dict(meta or {}))
+    n.embedding = vec(axis)
+    if ref:
+        n.relationships[NodeRelationship.SOURCE] = RelatedNodeInfo(node_id=ref)
+    return n
+
+
+@pytest.fixture()
+def store(tmp_path):
+    # Each test gets its own file: a store is a corpus, not a scratch pad.
+    return CitadelVectorStore(str(tmp_path / "v.cdl"), key="test-passphrase", dim=DIM)
+
+
+def query(axis, k=10, **kw):
+    return VectorStoreQuery(query_embedding=vec(axis), similarity_top_k=k, **kw)
+
+
+# ---- conformance ---------------------------------------------------------
+
+
+def test_is_a_pydantic_vector_store(store):
+    assert isinstance(store, BasePydanticVectorStore)
+    assert store.stores_text is True
+    assert store.client is not None
+
+
+# ---- the embedding contract ----------------------------------------------
+
+
+def test_the_supplied_vector_is_the_one_stored(store):
+    """Identical text, different vectors: the only proof of no re-embed."""
+    store.add([node("a", "same text", 0), node("b", "same text", 7)])
+    res = store.query(query(7, k=1))
+    assert res.ids == ["b"]
+    assert res.similarities[0] == pytest.approx(1.0, abs=1e-3)
+
+
+def test_duplicate_content_keeps_its_own_vector(store):
+    """Vectors travel with their atom, so equal text cannot swap them."""
+    store.add([node("d1", "identical", 0), node("d2", "identical", 7)])
+    assert store.query(query(0, k=1)).ids == ["d1"]
+    assert store.query(query(7, k=1)).ids == ["d2"]
+
+
+def test_similarity_never_exceeds_one_at_a_realistic_width(tmp_path):
+    """Small test widths do not surface the sub-zero distance."""
+    import random
+
+    wide = 1536
+    s = CitadelVectorStore(str(tmp_path / "wide.cdl"), key="pw", dim=wide)
+    nodes = []
+    for seed in range(10):
+        r = random.Random(seed)
+        n = TextNode(id_=f"w{seed}", text="x")
+        n.embedding = [r.random() for _ in range(wide)]
+        nodes.append(n)
+    s.add(nodes)
+    for n in nodes:
+        res = s.query(VectorStoreQuery(query_embedding=n.embedding, similarity_top_k=1))
+        assert res.similarities[0] <= 1.0, (n.node_id, res.similarities[0])
+
+
+def test_a_node_without_an_embedding_names_the_fix(store):
+    bare = TextNode(id_="bare", text="no vector")
+    with pytest.raises(ValueError, match="no embedding"):
+        store.add([bare])
+
+
+def test_a_wrong_width_embedding_names_the_fix(store):
+    wide = TextNode(id_="wide", text="x")
+    wide.embedding = [0.1, 0.2, 0.3]
+    with pytest.raises(ValueError, match="dim=3"):
+        store.add([wide])
+
+
+# ---- round-trip ----------------------------------------------------------
+
+
+def test_text_and_metadata_round_trip(store):
+    store.add([node("m1", "the payload text", 0, meta={"page": 4, "tag": "intro"})])
+    got = store.query(query(0, k=1)).nodes[0]
+    assert got.get_content() == "the payload text"
+    assert got.metadata["page"] == 4
+    assert got.metadata["tag"] == "intro"
+    assert got.node_id == "m1"
+
+
+def test_relationships_round_trip(store):
+    """The node is stored whole, so its graph edges survive with it."""
+    store.add([node("r1", "child", 0, ref="parent-doc")])
+    got = store.query(query(0, k=1)).nodes[0]
+    assert got.ref_doc_id == "parent-doc"
+
+
+def test_text_is_not_stored_twice(store):
+    """The node JSON is stored with its text stripped."""
+    store.add([node("t1", "unique body text", 0)])
+    raw = store.client.fetch(DEFAULT_REGION, KIND, limit=5)[0]
+    assert "unique body text" not in raw.payload["meta"]["_node_content"]
+    assert raw.text == "unique body text"
+
+
+# ---- query ---------------------------------------------------------------
+
+
+def test_similarity_orders_results(store):
+    store.add([node("near", "a", 0), node("far", "b", 7)])
+    res = store.query(query(0, k=2))
+    assert res.ids[0] == "near"
+    assert res.similarities[0] > res.similarities[1]
+
+
+def test_top_k_is_respected(store):
+    store.add([node(f"n{i}", "t", i % DIM) for i in range(6)])
+    assert len(store.query(query(0, k=2)).ids) == 2
+
+
+def test_top_k_zero_is_empty(store):
+    store.add([node("z", "t", 0)])
+    res = store.query(query(0, k=0))
+    assert res.ids == [] and res.nodes == []
+
+
+def test_a_query_without_an_embedding_is_refused(store):
+    with pytest.raises(ValueError, match="query_embedding is required"):
+        store.query(VectorStoreQuery(similarity_top_k=1))
+
+
+def test_an_unsupported_mode_is_refused(store):
+    with pytest.raises(NotImplementedError, match="not supported"):
+        store.query(query(0, mode=VectorStoreQueryMode.SPARSE))
+
+
+def test_query_narrows_by_node_ids(store):
+    store.add([node("k1", "a", 0), node("k2", "b", 0)])
+    assert store.query(query(0, node_ids=["k2"])).ids == ["k2"]
+
+
+def test_query_narrows_by_doc_ids(store):
+    store.add([node("c1", "a", 0, ref="doc-a"), node("c2", "b", 0, ref="doc-b")])
+    assert store.query(query(0, doc_ids=["doc-b"])).ids == ["c2"]
+
+
+# ---- filters -------------------------------------------------------------
+
+
+def filters(*fs, condition=FilterCondition.AND):
+    return MetadataFilters(filters=list(fs), condition=condition)
+
+
+def seeded(store):
+    store.add([
+        node("x1", "one", 0, meta={"cat": "x", "n": 1}),
+        node("y2", "two", 1, meta={"cat": "y", "n": 2}),
+        node("z3", "three", 2, meta={"cat": "z", "n": 3}),
+    ])
+    return store
+
+
+def test_equality_filter(store):
+    seeded(store)
+    f = filters(MetadataFilter(key="cat", value="y", operator=FilterOperator.EQ))
+    assert store.query(query(0, filters=f)).ids == ["y2"]
+
+
+def test_comparison_filter(store):
+    seeded(store)
+    f = filters(MetadataFilter(key="n", value=2, operator=FilterOperator.GT))
+    assert store.query(query(0, filters=f)).ids == ["z3"]
+
+
+def test_in_filter(store):
+    seeded(store)
+    f = filters(MetadataFilter(key="cat", value=["x", "z"], operator=FilterOperator.IN))
+    assert sorted(store.query(query(0, filters=f)).ids) == ["x1", "z3"]
+
+
+def test_and_of_two_equalities(store):
+    seeded(store)
+    f = filters(
+        MetadataFilter(key="cat", value="x", operator=FilterOperator.EQ),
+        MetadataFilter(key="n", value=1, operator=FilterOperator.EQ),
+    )
+    assert store.query(query(0, filters=f)).ids == ["x1"]
+
+
+def test_or_does_not_push_equality_into_the_scan(store):
+    """Pushing a leaf of an OR would drop rows the filter keeps."""
+    seeded(store)
+    f = filters(
+        MetadataFilter(key="cat", value="x", operator=FilterOperator.EQ),
+        MetadataFilter(key="n", value=3, operator=FilterOperator.EQ),
+        condition=FilterCondition.OR,
+    )
+    assert sorted(store.query(query(0, filters=f)).ids) == ["x1", "z3"]
+
+
+def test_not_does_not_push_equality_into_the_scan(store):
+    seeded(store)
+    f = filters(
+        MetadataFilter(key="cat", value="x", operator=FilterOperator.EQ),
+        condition=FilterCondition.NOT,
+    )
+    assert sorted(store.query(query(0, filters=f)).ids) == ["y2", "z3"]
+
+
+def test_a_pushed_filter_agrees_with_an_unpushed_one(store):
+    """Pushdown must narrow, never decide: both paths answer the same."""
+    seeded(store)
+    pushed = filters(MetadataFilter(key="cat", value="z", operator=FilterOperator.EQ))
+    unpushed = filters(
+        MetadataFilter(key="cat", value="z", operator=FilterOperator.TEXT_MATCH)
+    )
+    assert store.query(query(0, filters=pushed)).ids == \
+        store.query(query(0, filters=unpushed)).ids
+
+
+def test_a_numeric_filter_is_not_decided_by_the_pushdown(store):
+    """Containment compares JSON types exactly where EQ is Python `==`, which
+    holds 1 == 1.0. Pushing a number would drop a row the filter keeps, on both
+    the query path and the erase path."""
+    store.add([node("n-int", "stored as an int", 0, meta={"year": 2024})])
+    same = filters(MetadataFilter(key="year", value=2024.0, operator=FilterOperator.EQ))
+
+    assert store.query(query(0, filters=same)).ids == ["n-int"]
+    assert [n.node_id for n in store.get_nodes(filters=same)] == ["n-int"]
+    # A silently partial erase is the worst direction for a store whose delete
+    # is irreversible key destruction.
+    store.delete_nodes(filters=same)
+    assert store.count() == 0
+
+
+def test_a_named_node_is_reachable_however_it_ranks(store):
+    """node_ids is an exact restriction, so a named node must come back even
+    when it ranks below a window of better-matching rows."""
+    store.add([node("wanted", "the one asked for", 1)])
+    store.add([node(f"chaff{i}", f"nearer {i}", 0) for i in range(400)])
+    got = store.query(query(0, k=1, node_ids=["wanted"]))
+    assert got.ids == ["wanted"]
+
+
+def test_an_unpushable_filter_survives_a_window_of_other_rows(store):
+    """TEXT_MATCH cannot be pushed, so it is settled after ranking; the only
+    matching row must still be found under a corpus that outranks it."""
+    store.add([node("gold", "the needle", 1, meta={"cat": "zebra"})])
+    store.add([node(f"c{i}", f"nearer {i}", 0, meta={"cat": "other"}) for i in range(400)])
+    unpushable = filters(
+        MetadataFilter(key="cat", value="zeb", operator=FilterOperator.TEXT_MATCH)
+    )
+    assert store.query(query(0, k=1, filters=unpushable)).ids == ["gold"]
+
+
+# ---- reads and deletes ---------------------------------------------------
+
+
+def test_get_nodes_by_id(store):
+    seeded(store)
+    got = store.get_nodes(node_ids=["y2"])
+    assert [n.node_id for n in got] == ["y2"]
+
+
+def test_get_nodes_by_filter(store):
+    seeded(store)
+    f = filters(MetadataFilter(key="n", value=1, operator=FilterOperator.EQ))
+    assert [n.node_id for n in store.get_nodes(filters=f)] == ["x1"]
+
+
+def test_delete_removes_a_whole_document(store):
+    store.add([
+        node("p1", "a", 0, ref="doc-1"),
+        node("p2", "b", 1, ref="doc-1"),
+        node("p3", "c", 2, ref="doc-2"),
+    ])
+    store.delete("doc-1")
+    assert sorted(store.query(query(0, k=9)).ids) == ["p3"]
+
+
+def test_delete_destroys_keys_rather_than_rows(store):
+    store.add([node("s1", "secret", 0, ref="doc-x")])
+    assert store.forget_document("doc-x") == 1
+    assert store.count() == 0
+
+
+def test_delete_nodes_by_id_and_filter(store):
+    seeded(store)
+    store.delete_nodes(node_ids=["x1"])
+    assert sorted(store.query(query(0, k=9)).ids) == ["y2", "z3"]
+    store.delete_nodes(
+        filters=filters(MetadataFilter(key="n", value=2, operator=FilterOperator.EQ))
+    )
+    assert store.query(query(0, k=9)).ids == ["z3"]
+
+
+def test_clear_empties_the_store(store):
+    seeded(store)
+    store.clear()
+    assert store.count() == 0
+    assert store.query(query(0, k=9)).ids == []
+
+
+def test_deleting_nothing_is_not_an_error(store):
+    store.delete("no-such-doc")
+    assert store.count() == 0
+
+
+def test_a_passphrase_is_required(tmp_path):
+    with pytest.raises(ValueError, match="passphrase"):
+        CitadelVectorStore(str(tmp_path / "nokey.cdl"), key="")
+
+
+# ---- async ---------------------------------------------------------------
+
+
+async def test_async_surface_round_trips(store):
+    await store.async_add([node("a1", "async body", 0)])
+    res = await store.aquery(query(0, k=1))
+    assert res.ids == ["a1"]
+    assert [n.node_id for n in await store.aget_nodes(node_ids=["a1"])] == ["a1"]
+    await store.adelete_nodes(node_ids=["a1"])
+    assert store.count() == 0
+    await store.aclear()
+
+
+async def test_the_event_loop_is_not_blocked(store):
+    """The base class would call the sync method straight through."""
+    import asyncio
+
+    ticks = 0
+
+    async def tick():
+        nonlocal ticks
+        while True:
+            ticks += 1
+            await asyncio.sleep(0)
+
+    ticker = asyncio.create_task(tick())
+    await asyncio.sleep(0)
+    await store.async_add([node(f"L{i}", "loop body", i % DIM) for i in range(40)])
+    await store.aquery(query(0, k=5))
+    ticker.cancel()
+    assert ticks > 1, "the loop made no progress during a store call"
+
+
+# ---- through LlamaIndex's own layer --------------------------------------
+
+
+# ---- edges ---------------------------------------------------------------
+
+
+def test_re_adding_the_same_node_id_replaces_it(store):
+    """Appending instead of replacing would double the corpus."""
+    store.add([node("same", "first version", 0)])
+    store.add([node("same", "second version", 0)])
+    assert store.count() == 1
+    assert store.query(query(0, k=5)).nodes[0].get_content() == "second version"
+
+
+def test_duplicate_ids_within_one_call_keep_the_last(store):
+    """The reference is dict-keyed by node id, so the last write wins."""
+    store.add([node("same", "first", 0), node("same", "second", 7)])
+    assert store.count() == 1
+    assert store.query(query(7, k=1)).nodes[0].get_content() == "second"
+
+
+def test_a_node_with_empty_text_is_storable(store):
+    """An image or placeholder node can carry no text at all."""
+    store.add([node("blank", "", 0)])
+    assert store.query(query(0, k=1)).ids == ["blank"]
+
+
+def test_adding_nothing_is_not_an_error(store):
+    assert store.add([]) == []
+    assert store.count() == 0
+
+
+def test_query_on_an_empty_store_is_empty(store):
+    res = store.query(query(0, k=5))
+    assert res.ids == [] and res.nodes == [] and res.similarities == []
+
+
+def test_empty_node_ids_selects_nothing_unlike_none(store):
+    """`[]` means none of them; `None` means no id restriction."""
+    store.add([node("a", "x", 0), node("b", "y", 1)])
+    assert store.get_nodes(node_ids=[]) == []
+    assert len(store.get_nodes()) == 2
+
+
+def test_delete_nodes_with_no_arguments_clears_everything(store):
+    """Matches SimpleVectorStore, whose filter is true for every node."""
+    store.add([node("a", "x", 0), node("b", "y", 1)])
+    store.delete_nodes()
+    assert store.count() == 0
+
+
+def test_a_negative_top_k_is_empty_not_a_tail_slice(store):
+    store.add([node("a", "x", 0)])
+    assert store.query(query(0, k=-1)).ids == []
+
+
+def test_nested_and_list_metadata_round_trip(store):
+    store.add([node("n", "t", 0, meta={"outer": {"inner": [1, 2]}})])
+    assert store.query(query(0, k=1)).nodes[0].metadata["outer"] == {"inner": [1, 2]}
+
+
+def test_contains_filter_on_a_list(store):
+    store.add([
+        node("t1", "a", 0, meta={"tags": ["red", "blue"]}),
+        node("t2", "b", 1, meta={"tags": ["green"]}),
+    ])
+    f = filters(MetadataFilter(key="tags", value="blue", operator=FilterOperator.CONTAINS))
+    assert store.query(query(0, filters=f)).ids == ["t1"]
+
+
+def test_a_filter_on_a_missing_key_excludes_rather_than_raises(store):
+    store.add([node("a", "x", 0, meta={"present": 1})])
+    f = filters(MetadataFilter(key="absent", value=1, operator=FilterOperator.EQ))
+    assert store.query(query(0, filters=f)).ids == []
+
+
+def test_unicode_round_trips(store):
+    store.add([node("u", "unicode - a and b", 0, meta={"k": "value"})])
+    got = store.query(query(0, k=1)).nodes[0]
+    assert got.get_content() == "unicode - a and b"
+
+
+def test_two_stores_share_one_database_file(tmp_path):
+    """The composition the shared-handle work exists to allow."""
+    path = str(tmp_path / "shared.cdl")
+    a = CitadelVectorStore(path, key="pw", region="corpus_a", dim=DIM)
+    b = CitadelVectorStore(path, key="pw", region="corpus_b", dim=DIM)
+    a.add([node("a1", "in a", 0)])
+    b.add([node("b1", "in b", 0)])
+    assert a.query(query(0, k=5)).ids == ["a1"]
+    assert b.query(query(0, k=5)).ids == ["b1"]
+
+
+def test_a_large_batch_keeps_every_node(store):
+    store.add([node(f"b{i}", f"body {i}", i % DIM) for i in range(300)])
+    assert store.count() == 300
+    assert len(store.query(query(0, k=50)).ids) == 50
+
+
+def test_it_survives_a_reopen(tmp_path):
+    """A region's embedder lives in memory, so reattach is the risk."""
+    import gc
+
+    p = str(tmp_path / "reopen.cdl")
+    first = CitadelVectorStore(p, key="pw", dim=DIM)
+    first.add([node("n1", "the disk was full", 0, meta={"page": 3})])
+    del first
+    gc.collect()
+
+    again = CitadelVectorStore(p, key="pw", dim=DIM)
+    assert again.count() == 1
+    got = again.get_nodes(node_ids=["n1"])[0]
+    assert got.get_content() == "the disk was full" and got.metadata["page"] == 3
+    assert again.query(query(0, k=1)).ids == ["n1"], "recall did not survive the reopen"
+
+
+def test_a_wrong_passphrase_cannot_reopen(tmp_path):
+    """Pins the encryption claim rather than inferring it from a reopen."""
+    import gc
+
+    import citadeldb
+
+    p = str(tmp_path / "enc.cdl")
+    first = CitadelVectorStore(p, key="right", dim=DIM)
+    first.add([node("n1", "the disk was full", 0)])
+    del first
+    gc.collect()
+
+    with pytest.raises(citadeldb.EncryptionError):
+        CitadelVectorStore(p, key="wrong", dim=DIM)
+
+
+def test_concurrent_writes_all_land(tmp_path):
+    """Indexing pipelines share one engine across threads."""
+    import concurrent.futures as cf
+
+    s = CitadelVectorStore(str(tmp_path / "conc.cdl"), key="pw", dim=DIM)
+    with cf.ThreadPoolExecutor(max_workers=4) as ex:
+        list(ex.map(lambda i: s.add([node(f"c{i}", f"body {i}", i % DIM)]), range(40)))
+    assert s.count() == 40
+
+
+def test_end_to_end_through_vector_store_index(tmp_path):
+    """The ABC is only half the integration; this drives LlamaIndex itself."""
+    from llama_index.core import StorageContext, VectorStoreIndex
+    from llama_index.core.embeddings import MockEmbedding
+    from llama_index.core.schema import Document
+
+    embed = MockEmbedding(embed_dim=DIM)
+    store = CitadelVectorStore(str(tmp_path / "e2e.cdl"), key="pw", dim=DIM)
+    index = VectorStoreIndex.from_documents(
+        [Document(text="the deploy failed because the disk was full"),
+         Document(text="lunch plans for friday")],
+        storage_context=StorageContext.from_defaults(vector_store=store),
+        embed_model=embed,
+    )
+    hits = index.as_retriever(similarity_top_k=2).retrieve("why did the release break?")
+    assert hits, "retrieval through VectorStoreIndex returned nothing"
+    assert store.count() == 2