citadeldb-haystack 2.0.0__tar.gz

citadeldb_haystack-2.0.0/.gitignore

@@ -0,0 +1,18 @@
+/target
+**/*.rs.bk
+*.swp
+*.swo
+*~
+.DS_Store
+site/public/
+site/static/wasm/*.wasm
+site/static/wasm/*.js
+/notes/
+__pycache__/
+*.py[cod]
+# maturin build output; the .pyd is caught by the line above only by accident.
+*.pdb
+.pytest_cache/
+.mypy_cache/
+/dist/
+packaging/*/dist/

citadeldb_haystack-2.0.0/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.5
+Name: citadeldb-haystack
+Version: 2.0.0
+Summary: Haystack document store backed by Citadel: encrypted at rest, with deletes that destroy the key
+Project-URL: Homepage, https://citadeldb.dev
+Project-URL: Repository, https://github.com/yp3y5akh0v/citadel
+Author: Yuriy Peysakhov
+License-Expression: Apache-2.0
+Keywords: document-store,encryption,haystack,rag,vector-store
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: citadeldb<3,>=2.0
+Requires-Dist: haystack-ai<4,>=2.9
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# citadeldb-haystack
+
+A [Haystack](https://github.com/deepset-ai/haystack) `DocumentStore` backed by
+[Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and deletes
+that destroy the key, not just the row.
+
+Passes deepset's own `DocumentStoreBaseTests` conformance suite.
+
+```
+pip install citadeldb-haystack
+```
+
+```python
+from haystack import Document
+from haystack.utils import Secret
+from citadeldb_haystack import CitadelDocumentStore
+
+store = CitadelDocumentStore("corpus.cdl", Secret.from_env_var("CITADEL_KEY"))
+# CITADEL_KEY must be set: an env-var secret is what lets a pipeline serialize.
+
+store.write_documents([Document(id="d1", content="...", meta={"chapter": "intro"})])
+store.filter_documents({"field": "meta.chapter", "operator": "==", "value": "intro"})
+```
+
+`dim` defaults to 768 and must match your embedding model.
+
+## The passphrase never lands in a pipeline file
+
+The passphrase is a Haystack `Secret`. Pipelines are serialized to disk, and a literal
+token refuses to serialize, so a passphrase cannot be written into a pipeline by accident:
+
+```python
+CitadelDocumentStore("literal.cdl", "literal-passphrase").to_dict()
+# ValueError: Cannot serialize token-based secret.
+
+CitadelDocumentStore("corpus.cdl", Secret.from_env_var("CITADEL_KEY")).to_dict()
+# {... "key": {"type": "env_var", "env_vars": ["CITADEL_KEY"], ...}}
+```
+
+Use `Secret.from_env_var` for any store that goes into a saved pipeline.
+
+## Deletes destroy the key
+
+Every document is sealed under its own key. Deleting destroys that key and then removes the
+row, so any ciphertext surviving elsewhere stays unreadable.
+
+```python
+store.delete_documents(["d1"])
+store.delete_all()             # returns the number erased
+```
+
+`DuplicatePolicy.NONE` falls back to `FAIL`, as `InMemoryDocumentStore` does, so an
+accidental re-write is reported rather than silently replacing a document whose key would
+then be destroyed.
+
+## Retrieval
+
+```python
+query_embedding = [0.0] * 768        # from your Haystack text embedder, `dim` wide
+
+store.embedding_retrieval(query_embedding, top_k=5,
+                          filters={"field": "meta.chapter", "operator": "==", "value": "intro"})
+```
+
+Filtering uses Haystack's own evaluator, so the whole filter language, date comparisons
+included, matches `InMemoryDocumentStore` operator for operator. `top_k` is `top_k`: a
+filter matching only distant documents still returns them, however many others outrank
+them.
+
+A top-level `AND` of string equality conditions is pushed into the scan, including nested
+paths like `meta.person.name`. Everything else is evaluated afterwards, so the two agree:
+nothing is pushed under `OR` or `NOT`, and numbers are not pushed either, because `==` here
+is Python's (`1 == 1.0`) where the stored comparison is JSON-type exact.
+
+## Notes
+
+Documents Haystack did not embed are stored, filterable and countable, but take no part in
+`embedding_retrieval`, as in `InMemoryDocumentStore`, which retrieves only documents that
+have embeddings.
+
+Citadel is embedded and one process owns the file. A path already open on this thread,
+under the same passphrase, is shared, so this can sit on the same database as another
+Citadel adapter; construct them on the same thread.
+
+## License
+
+Apache-2.0

citadeldb_haystack-2.0.0/README.md

@@ -0,0 +1,87 @@
+# citadeldb-haystack
+
+A [Haystack](https://github.com/deepset-ai/haystack) `DocumentStore` backed by
+[Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and deletes
+that destroy the key, not just the row.
+
+Passes deepset's own `DocumentStoreBaseTests` conformance suite.
+
+```
+pip install citadeldb-haystack
+```
+
+```python
+from haystack import Document
+from haystack.utils import Secret
+from citadeldb_haystack import CitadelDocumentStore
+
+store = CitadelDocumentStore("corpus.cdl", Secret.from_env_var("CITADEL_KEY"))
+# CITADEL_KEY must be set: an env-var secret is what lets a pipeline serialize.
+
+store.write_documents([Document(id="d1", content="...", meta={"chapter": "intro"})])
+store.filter_documents({"field": "meta.chapter", "operator": "==", "value": "intro"})
+```
+
+`dim` defaults to 768 and must match your embedding model.
+
+## The passphrase never lands in a pipeline file
+
+The passphrase is a Haystack `Secret`. Pipelines are serialized to disk, and a literal
+token refuses to serialize, so a passphrase cannot be written into a pipeline by accident:
+
+```python
+CitadelDocumentStore("literal.cdl", "literal-passphrase").to_dict()
+# ValueError: Cannot serialize token-based secret.
+
+CitadelDocumentStore("corpus.cdl", Secret.from_env_var("CITADEL_KEY")).to_dict()
+# {... "key": {"type": "env_var", "env_vars": ["CITADEL_KEY"], ...}}
+```
+
+Use `Secret.from_env_var` for any store that goes into a saved pipeline.
+
+## Deletes destroy the key
+
+Every document is sealed under its own key. Deleting destroys that key and then removes the
+row, so any ciphertext surviving elsewhere stays unreadable.
+
+```python
+store.delete_documents(["d1"])
+store.delete_all()             # returns the number erased
+```
+
+`DuplicatePolicy.NONE` falls back to `FAIL`, as `InMemoryDocumentStore` does, so an
+accidental re-write is reported rather than silently replacing a document whose key would
+then be destroyed.
+
+## Retrieval
+
+```python
+query_embedding = [0.0] * 768        # from your Haystack text embedder, `dim` wide
+
+store.embedding_retrieval(query_embedding, top_k=5,
+                          filters={"field": "meta.chapter", "operator": "==", "value": "intro"})
+```
+
+Filtering uses Haystack's own evaluator, so the whole filter language, date comparisons
+included, matches `InMemoryDocumentStore` operator for operator. `top_k` is `top_k`: a
+filter matching only distant documents still returns them, however many others outrank
+them.
+
+A top-level `AND` of string equality conditions is pushed into the scan, including nested
+paths like `meta.person.name`. Everything else is evaluated afterwards, so the two agree:
+nothing is pushed under `OR` or `NOT`, and numbers are not pushed either, because `==` here
+is Python's (`1 == 1.0`) where the stored comparison is JSON-type exact.
+
+## Notes
+
+Documents Haystack did not embed are stored, filterable and countable, but take no part in
+`embedding_retrieval`, as in `InMemoryDocumentStore`, which retrieves only documents that
+have embeddings.
+
+Citadel is embedded and one process owns the file. A path already open on this thread,
+under the same passphrase, is shared, so this can sit on the same database as another
+Citadel adapter; construct them on the same thread.
+
+## License
+
+Apache-2.0

citadeldb_haystack-2.0.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "citadeldb-haystack"
+dynamic = ["version"]
+description = "Haystack document store backed by Citadel: encrypted at rest, with deletes that destroy the key"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{ name = "Yuriy Peysakhov" }]
+keywords = ["haystack", "rag", "document-store", "vector-store", "encryption"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Topic :: Database",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+# The precomputed vector needs the `embedding` field added in citadeldb 2.0.
+dependencies = ["citadeldb>=2.0,<3", "haystack-ai>=2.9,<4"]
+
+[project.optional-dependencies]
+test = ["pytest>=8", "pytest-asyncio>=0.23"]
+
+[project.urls]
+Homepage = "https://citadeldb.dev"
+Repository = "https://github.com/yp3y5akh0v/citadel"
+
+# The version comes from the release tag, so there is nothing to bump.
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "../..", tag_regex = '^v(?P<version>\d+\.\d+\.\d+)$' }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/citadeldb_haystack"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

citadeldb_haystack-2.0.0/src/citadeldb_haystack/__init__.py

@@ -0,0 +1,14 @@
+"""Haystack document storage backed by Citadel, encrypted at rest."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .document_store import CitadelDocumentStore
+
+__all__ = ["CitadelDocumentStore", "__version__"]
+
+# Haystack wires stores by construction, so there is nothing to register.
+
+try:
+    __version__ = version("citadeldb-haystack")
+except PackageNotFoundError:  # running from a source tree, never installed
+    __version__ = "0+unknown"

citadeldb_haystack-2.0.0/src/citadeldb_haystack/document_store.py

@@ -0,0 +1,346 @@
+"""Haystack DocumentStore over an encrypted Citadel region."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from dataclasses import replace
+from typing import Any
+
+import citadeldb
+from haystack import default_from_dict, default_to_dict
+from haystack.dataclasses import Document
+from haystack.document_stores.errors import DuplicateDocumentError
+from haystack.document_stores.types import DuplicatePolicy
+from haystack.utils import Secret, deserialize_secrets_inplace
+from haystack.utils.filters import document_matches_filter
+
+KIND = "doc"
+DEFAULT_PATH = "haystack.cdl"
+DEFAULT_REGION = "documents"
+# The default width of Haystack's own embedders and of its conformance fixtures.
+DEFAULT_DIM = 768
+PAGE = 10_000
+# Haystack prefixes metadata fields; bare names address the document.
+_META_PREFIX = "meta."
+
+
+# A Database is pinned to its opening thread, so workers take Memory, not self.
+
+
+def _placeholder(text: str, dim: int) -> list[float]:
+    """A deterministic vector for a document Haystack did not embed."""
+    digest = hashlib.sha256(text.encode()).digest()
+    return [digest[i % len(digest)] / 255.0 for i in range(dim)]
+
+
+def _fetch(mem: Any, region: str, criterion: dict[str, Any] | None) -> list[Any]:
+    """Page to the end: one fetch is bounded, and a partial erase must not look whole."""
+    out: list[Any] = []
+    after = None
+    while True:
+        page = mem.fetch(region, KIND, payload_filter=criterion, limit=PAGE, after_id=after)
+        out.extend(page)
+        if len(page) < PAGE:
+            return out
+        after = page[-1].id
+
+
+def _erase(mem: Any, region: str, hits: list[Any]) -> int:
+    if not hits:
+        return 0
+    return mem.forget(region, [h.id for h in hits]).erased_count
+
+
+def _document(hit: Any) -> Document:
+    # Unflattened so the document round-trips whole.
+    return Document.from_dict(hit.payload["doc"])
+
+
+def _pushdown(filters: dict[str, Any] | None) -> dict[str, Any] | None:
+    """Equality leaves of a top-level AND; this narrows and never decides."""
+    if not filters:
+        return None
+    if filters.get("operator") == "AND":
+        conditions = filters.get("conditions", [])
+    elif "field" in filters:
+        conditions = [filters]  # a bare condition is an AND of one
+    else:
+        return None
+    leaves: list[tuple[tuple[str, ...], Any]] = []
+    for c in conditions:
+        field = c.get("field", "")
+        if (
+            c.get("operator") == "=="
+            and field.startswith(_META_PREFIX)
+            # Only strings: containment compares JSON types exactly where
+            # Haystack's `==` is Python's, which holds 1 == 1.0 == True, so
+            # pushing a number would decide rather than narrow. None means
+            # absent, which containment cannot express at all.
+            and isinstance(c.get("value"), str)
+        ):
+            path = tuple(p for p in field[len(_META_PREFIX):].split(".") if p)
+            if path:
+                leaves.append((path, c["value"]))
+    eq = _needle(leaves)
+    # The needle is shaped like the payload, not like the filter.
+    return {"doc": {"meta": eq}} if eq else None
+
+
+def _needle(leaves: list[tuple[tuple[str, ...], Any]]) -> dict[str, Any]:
+    """Nested-object needle for dotted metadata paths.
+
+    `meta.person.name` addresses a nested value, so it has to become
+    `{"person": {"name": v}}`; one flat key spelled with a dot can never exist in
+    the payload and would match nothing. Two leaves that collide, or one whose
+    path runs through another's, are dropped rather than merged - containment
+    holds one value per position, and guessing which wins would decide.
+    """
+    out: dict[str, Any] = {}
+    for path, value in leaves:
+        if any(
+            other is not path and (other[: len(path)] == path or path[: len(other)] == other)
+            for other, _ in leaves
+        ):
+            continue
+        node = out
+        for part in path[:-1]:
+            node = node.setdefault(part, {})
+        node[path[-1]] = value
+    return out
+
+
+def _filter(mem: Any, region: str, filters: dict[str, Any] | None) -> list[Document]:
+    docs = [_document(h) for h in _fetch(mem, region, _pushdown(filters))]
+    if not filters:
+        return docs
+    return [d for d in docs if document_matches_filter(filters, d)]
+
+
+def _atom(doc: Document, dim: int) -> dict[str, Any]:
+    if doc.embedding is not None and len(doc.embedding) != dim:
+        raise ValueError(
+            f"document {doc.id} has a {len(doc.embedding)}-dimension embedding but "
+            f"this region is {dim}. Build the store with dim={len(doc.embedding)} to "
+            f"match your embedding model."
+        )
+    return {
+        "kind": KIND,
+        "text": doc.content or "",
+        "embedding": (
+            list(doc.embedding)
+            if doc.embedding is not None
+            else _placeholder(doc.content or doc.id, dim)
+        ),
+        # Unflattened so the document round-trips whole. `emb` records whether
+        # the vector above is Haystack's or the placeholder, which is what keeps
+        # a hash of the text out of embedding_retrieval's answers.
+        "payload": {
+            "did": doc.id,
+            "emb": doc.embedding is not None,
+            "doc": doc.to_dict(flatten=False),
+        },
+    }
+
+
+def _write(
+    mem: Any,
+    region: str,
+    dim: int,
+    documents: list[Document],
+    policy: DuplicatePolicy,
+) -> int:
+    if policy == DuplicatePolicy.NONE:
+        policy = DuplicatePolicy.FAIL  # as InMemoryDocumentStore defaults
+
+    # OVERWRITE decides nothing from a prior read, so it does not pay for one:
+    # the keyed write below supersedes whatever the id already named.
+    seen: set[str] = set()
+    if policy != DuplicatePolicy.OVERWRITE:
+        # A live id set: a second copy in one batch collides like a stored one.
+        seen = {
+            h.payload["did"]
+            for doc in documents
+            for h in _fetch(mem, region, {"did": doc.id})
+        }
+    written = len(documents)
+    atoms: dict[str, dict[str, Any]] = {}
+    for doc in documents:
+        if policy != DuplicatePolicy.OVERWRITE and doc.id in seen:
+            if policy == DuplicatePolicy.FAIL:
+                raise DuplicateDocumentError(f"ID '{doc.id}' already exists.")
+            written -= 1  # SKIP: the copy already accepted stands
+            continue
+        # Last copy of an id in one batch wins, as the reference's dict does.
+        atoms[doc.id] = _atom(doc, dim)
+        seen.add(doc.id)
+    if atoms:
+        # Keyed on the document id and committed as one transaction: two writers
+        # of one id supersede rather than each adding a row, which a read out
+        # here cannot prevent. Every policy binds the key, so a document written
+        # under SKIP is still the one a later OVERWRITE replaces.
+        mem.remember_replacing_keyed_batch(
+            region, [(atom, did) for did, atom in atoms.items()]
+        )
+    return written
+
+
+def _delete(mem: Any, region: str, document_ids: list[str]) -> int:
+    # Contracted not to fail on an id that is not present.
+    return _erase(
+        mem,
+        region,
+        [h for did in dict.fromkeys(document_ids) for h in _fetch(mem, region, {"did": did})],
+    )
+
+
+def _count(mem: Any, region: str) -> int:
+    return len(_fetch(mem, region, None))
+
+
+class CitadelDocumentStore:
+    """A Haystack `DocumentStore` backed by one encrypted Citadel region."""
+
+    def __init__(
+        self,
+        path: str = DEFAULT_PATH,
+        key: Secret | str = Secret.from_env_var("CITADEL_KEY"),
+        *,
+        region: str = DEFAULT_REGION,
+        dim: int = DEFAULT_DIM,
+    ) -> None:
+        self._key = Secret.from_token(key) if isinstance(key, str) else key
+        passphrase = self._key.resolve_value()
+        if not passphrase:
+            raise ValueError("a passphrase is required: the corpus is the payload")
+        self._path = path
+        self._region = region
+        self._dim = dim
+        try:
+            self._db = citadeldb.connect(path, key=passphrase, region_keys=True)
+        except citadeldb.OperationalError as e:
+            if "locked" not in str(e):
+                raise
+            raise RuntimeError(
+                f"{path} is open in another process. Citadel is embedded, so one "
+                f"process owns the file."
+            ) from e
+        self._mem = self._db.memory()
+        # Idempotent for a region of the same width, so a dim clash raises here.
+        self._mem.create_encrypted_region(region, citadeldb.MockEmbedder(dim=dim))
+
+    # ---- serialization ----------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize for a pipeline file; a literal passphrase refuses."""
+        return default_to_dict(
+            self,
+            path=self._path,
+            key=self._key.to_dict(),
+            region=self._region,
+            dim=self._dim,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> CitadelDocumentStore:
+        deserialize_secrets_inplace(data["init_parameters"], keys=["key"])
+        return default_from_dict(cls, data)
+
+    # ---- the protocol -----------------------------------------------------
+
+    def count_documents(self) -> int:
+        return _count(self._mem, self._region)
+
+    def filter_documents(self, filters: dict[str, Any] | None = None) -> list[Document]:
+        return _filter(self._mem, self._region, filters)
+
+    def write_documents(
+        self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
+    ) -> int:
+        if not isinstance(documents, list) or any(
+            not isinstance(d, Document) for d in documents
+        ):
+            raise ValueError("Please provide a list of Documents.")
+        return _write(self._mem, self._region, self._dim, documents, policy)
+
+    def delete_documents(self, document_ids: list[str]) -> None:
+        _delete(self._mem, self._region, document_ids)
+
+    # ---- async ------------------------------------------------------------
+    # The bindings are sync, so a worker thread keeps the event loop free.
+
+    async def count_documents_async(self) -> int:
+        return await asyncio.to_thread(_count, self._mem, self._region)
+
+    async def filter_documents_async(
+        self, filters: dict[str, Any] | None = None
+    ) -> list[Document]:
+        return await asyncio.to_thread(_filter, self._mem, self._region, filters)
+
+    async def write_documents_async(
+        self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
+    ) -> int:
+        if not isinstance(documents, list) or any(
+            not isinstance(d, Document) for d in documents
+        ):
+            raise ValueError("Please provide a list of Documents.")
+        return await asyncio.to_thread(
+            _write, self._mem, self._region, self._dim, documents, policy
+        )
+
+    async def delete_documents_async(self, document_ids: list[str]) -> None:
+        await asyncio.to_thread(_delete, self._mem, self._region, document_ids)
+
+    # ---- beyond the protocol ----------------------------------------------
+
+    def embedding_retrieval(
+        self,
+        query_embedding: list[float],
+        top_k: int = 10,
+        filters: dict[str, Any] | None = None,
+    ) -> list[Document]:
+        """Documents ranked by recall, best first, with their score set.
+
+        Only documents Haystack embedded take part, as its own store does: an
+        unembedded one carries a hash of its text, and ranking that against a
+        real query vector produces a plausible score with no meaning behind it.
+        """
+        criterion = _pushdown(filters) or {}
+        criterion = {**criterion, "emb": True}
+        options = citadeldb.RecallOptions(payload_filter=criterion)
+
+        def surviving(hits: list[Any]) -> list[Document]:
+            out: list[Document] = []
+            for h in hits:
+                doc = _document(h)
+                if filters and not document_matches_filter(filters, doc):
+                    continue
+                # Mutating a shared Document would affect other pipeline steps.
+                out.append(
+                    replace(
+                        doc,
+                        score=min(1.0, 1.0 - h.distance)
+                        if h.distance is not None
+                        else h.score,
+                    )
+                )
+            return out
+
+        # A filter containment cannot express is settled above, so the window has
+        # to widen until top_k survive it rather than answer short.
+        k = max(top_k, 32)
+        while True:
+            hits = self._mem.recall(
+                self._region,
+                embedding=query_embedding,
+                k=k,
+                kinds=[KIND],
+                options=options,
+            )
+            out = surviving(hits)
+            if len(out) >= top_k or len(hits) < k:
+                return out[:top_k]
+            k *= 2
+
+    def delete_all(self) -> int:
+        """Destroy every document's key, returning the number erased."""
+        return _erase(self._mem, self._region, _fetch(self._mem, self._region, None))

citadeldb_haystack-2.0.0/tests/test_conformance.py

@@ -0,0 +1,23 @@
+"""deepset's DocumentStoreBaseTests run against CitadelDocumentStore."""
+import pytest
+from haystack.dataclasses import Document
+from haystack.document_stores.errors import DuplicateDocumentError
+from haystack.testing.document_store import DocumentStoreBaseTests
+
+from citadeldb_haystack import CitadelDocumentStore
+
+# The conformance fixtures embed at 768.
+DIM = 768
+
+
+class TestCitadelDocumentStore(DocumentStoreBaseTests):
+    @pytest.fixture
+    def document_store(self, tmp_path) -> CitadelDocumentStore:
+        return CitadelDocumentStore(str(tmp_path / "conformance.cdl"), "pw", dim=DIM)
+
+    def test_write_documents(self, document_store: CitadelDocumentStore):
+        """NONE falls back to FAIL, so an accidental re-write is reported."""
+        doc = Document(content="test doc")
+        assert document_store.write_documents([doc]) == 1
+        with pytest.raises(DuplicateDocumentError):
+            document_store.write_documents([doc])

citadeldb_haystack-2.0.0/tests/test_document_store.py

@@ -0,0 +1,528 @@
+import pytest
+from haystack import Pipeline
+from haystack.dataclasses import Document
+from haystack.document_stores.errors import DuplicateDocumentError
+from haystack.document_stores.types import DuplicatePolicy
+from haystack.utils import Secret
+
+from citadeldb_haystack import CitadelDocumentStore
+
+DIM = 8
+
+
+def vec(axis: int) -> list[float]:
+    v = [0.0] * DIM
+    v[axis] = 1.0
+    return v
+
+
+@pytest.fixture()
+def store(tmp_path):
+    return CitadelDocumentStore(str(tmp_path / "d.cdl"), "pw", dim=DIM)
+
+
+@pytest.fixture()
+def reference():
+    """The store whose behaviour the protocol is defined by."""
+    from haystack.document_stores.in_memory import InMemoryDocumentStore
+
+    return InMemoryDocumentStore()
+
+
+# ---- conformance ---------------------------------------------------------
+
+
+def test_satisfies_the_protocol(store):
+    """DocumentStore is not runtime_checkable, so conformance is by shape."""
+    for name in (
+        "to_dict", "from_dict", "count_documents", "filter_documents",
+        "write_documents", "delete_documents",
+    ):
+        assert callable(getattr(store, name)), name
+
+
+def test_an_equality_filter_returns_the_matching_documents(store):
+    """Exercises the pushed-down filter path, which conformance never hits."""
+    store.write_documents([
+        Document(id="a", content="1", meta={"cat": "x", "n": 1}),
+        Document(id="b", content="2", meta={"cat": "y", "n": 2}),
+    ])
+    got = store.filter_documents({"field": "meta.cat", "operator": "==", "value": "y"})
+    assert [d.id for d in got] == ["b"]
+
+
+def test_an_and_of_equalities_returns_the_matching_documents(store):
+    store.write_documents([
+        Document(id="a", content="1", meta={"cat": "x", "n": 1}),
+        Document(id="b", content="2", meta={"cat": "x", "n": 2}),
+    ])
+    got = store.filter_documents({
+        "operator": "AND",
+        "conditions": [
+            {"field": "meta.cat", "operator": "==", "value": "x"},
+            {"field": "meta.n", "operator": "==", "value": 2},
+        ],
+    })
+    assert [d.id for d in got] == ["b"]
+
+
+# ---- duplicate policy ----------------------------------------------------
+
+
+def test_skip_leaves_the_original_and_reports_fewer_written(store):
+    store.write_documents([Document(id="a", content="first")])
+    written = store.write_documents(
+        [Document(id="a", content="second"), Document(id="b", content="new")],
+        policy=DuplicatePolicy.SKIP,
+    )
+    assert written == 1
+    assert store.filter_documents({"field": "id", "operator": "==", "value": "a"})[
+        0
+    ].content == "first"
+
+
+def test_overwrite_replaces_and_counts_every_input(store):
+    store.write_documents([Document(id="a", content="first")])
+    assert (
+        store.write_documents(
+            [Document(id="a", content="second")], policy=DuplicatePolicy.OVERWRITE
+        )
+        == 1
+    )
+    assert store.count_documents() == 1
+
+
+def test_duplicate_ids_within_one_call_keep_the_last(store):
+    """The reference assigns storage[doc.id], so the last write wins."""
+    store.write_documents(
+        [Document(id="d", content="one"), Document(id="d", content="two")],
+        policy=DuplicatePolicy.OVERWRITE,
+    )
+    assert store.count_documents() == 1
+    assert store.filter_documents()[0].content == "two"
+
+
+def test_fail_raises_before_anything_is_written(store):
+    store.write_documents([Document(id="a", content="first")])
+    with pytest.raises(DuplicateDocumentError):
+        store.write_documents(
+            [Document(id="b", content="new"), Document(id="a", content="dup")]
+        )
+    assert store.count_documents() == 1, "a failed batch must not half-apply"
+
+
+def test_non_documents_are_refused(store):
+    with pytest.raises(ValueError, match="list of Documents"):
+        store.write_documents(["not a document"])
+
+
+# These compare against the reference directly, not against its docstring.
+
+
+def test_overwrite_with_duplicates_counts_every_input(store, reference):
+    """Two copies of one id is still two documents in input."""
+    docs = [Document(id="d", content="one"), Document(id="d", content="two")]
+    assert store.write_documents(docs, policy=DuplicatePolicy.OVERWRITE) == (
+        reference.write_documents(docs, policy=DuplicatePolicy.OVERWRITE)
+    )
+
+
+def test_skip_with_duplicates_counts_like_the_reference(store, reference):
+    docs = [Document(id="d", content="one"), Document(id="d", content="two")]
+    assert store.write_documents(docs, policy=DuplicatePolicy.SKIP) == (
+        reference.write_documents(docs, policy=DuplicatePolicy.SKIP)
+    )
+
+
+def test_fail_raises_on_a_duplicate_within_one_batch(store, reference):
+    """A second copy collides even though neither was stored before."""
+    docs = [Document(id="d", content="one"), Document(id="d", content="two")]
+    with pytest.raises(DuplicateDocumentError):
+        reference.write_documents(docs, policy=DuplicatePolicy.FAIL)
+    with pytest.raises(DuplicateDocumentError):
+        store.write_documents(docs, policy=DuplicatePolicy.FAIL)
+
+
+def test_skip_of_a_fresh_duplicate_stores_the_first(store):
+    """SKIP drops the later copy, so the earlier one is what survives."""
+    store.write_documents(
+        [Document(id="d", content="first"), Document(id="d", content="second")],
+        policy=DuplicatePolicy.SKIP,
+    )
+    assert store.count_documents() == 1
+    assert store.filter_documents()[0].content == "first"
+
+
+# ---- round-trip ----------------------------------------------------------
+
+
+def test_embeddings_round_trip(store):
+    """The conformance suite compares whole Documents, embeddings included."""
+    store.write_documents([Document(id="e", content="x", embedding=vec(3))])
+    assert store.filter_documents()[0].embedding == vec(3)
+
+
+def test_documents_without_embeddings_are_storable(store):
+    store.write_documents([Document(id="n", content="no vector")])
+    got = store.filter_documents()[0]
+    assert got.embedding is None and got.content == "no vector"
+
+
+def test_nested_metadata_round_trips(store):
+    store.write_documents([Document(id="m", content="x", meta={"o": {"i": [1, 2]}})])
+    assert store.filter_documents()[0].meta["o"] == {"i": [1, 2]}
+
+
+def test_a_wrong_width_embedding_names_the_fix(store):
+    with pytest.raises(ValueError, match="dim=3"):
+        store.write_documents([Document(id="w", content="x", embedding=[1.0, 2.0, 3.0])])
+
+
+# ---- deletes -------------------------------------------------------------
+
+
+def test_delete_destroys_keys(store):
+    store.write_documents([Document(id="s", content="secret")])
+    store.delete_documents(["s"])
+    assert store.count_documents() == 0
+
+
+def test_delete_tolerates_unknown_and_duplicate_ids(store):
+    store.write_documents([Document(id="a", content="x")])
+    store.delete_documents(["missing", "a", "a"])
+    assert store.count_documents() == 0
+
+
+def test_delete_all_empties_the_store(store):
+    store.write_documents([Document(content="a"), Document(content="b")])
+    assert store.delete_all() == 2
+    assert store.count_documents() == 0
+
+
+# ---- serialization -------------------------------------------------------
+
+
+def test_a_literal_passphrase_refuses_to_serialize(store):
+    """Haystack writes pipelines to disk; a passphrase must not travel."""
+    with pytest.raises(ValueError, match="Cannot serialize"):
+        store.to_dict()
+
+
+def test_an_env_var_passphrase_serializes_by_reference(tmp_path, monkeypatch):
+    monkeypatch.setenv("CITADEL_TEST_KEY", "pw")
+    s = CitadelDocumentStore(
+        str(tmp_path / "s.cdl"), Secret.from_env_var("CITADEL_TEST_KEY"), dim=DIM
+    )
+    data = s.to_dict()
+    assert data["init_parameters"]["key"]["env_vars"] == ["CITADEL_TEST_KEY"]
+    assert "pw" not in str(data), "the passphrase itself must never be serialized"
+
+
+def test_round_trip_through_from_dict(tmp_path, monkeypatch):
+    monkeypatch.setenv("CITADEL_TEST_KEY", "pw")
+    path = str(tmp_path / "r.cdl")
+    s = CitadelDocumentStore(path, Secret.from_env_var("CITADEL_TEST_KEY"), dim=DIM)
+    s.write_documents([Document(id="a", content="kept")])
+    revived = CitadelDocumentStore.from_dict(s.to_dict())
+    assert revived.count_documents() == 1
+
+
+def test_a_missing_passphrase_is_refused(tmp_path, monkeypatch):
+    monkeypatch.setenv("CITADEL_EMPTY_KEY", "")
+    with pytest.raises(ValueError, match="passphrase"):
+        CitadelDocumentStore(
+            str(tmp_path / "k.cdl"), Secret.from_env_var("CITADEL_EMPTY_KEY"), dim=DIM
+        )
+
+
+# ---- retrieval -----------------------------------------------------------
+
+
+def test_embedding_retrieval_ranks_and_scores(store):
+    store.write_documents([
+        Document(id="near", content="a", embedding=vec(0)),
+        Document(id="far", content="b", embedding=vec(7)),
+    ])
+    got = store.embedding_retrieval(vec(0), top_k=2)
+    assert [d.id for d in got] == ["near", "far"]
+    assert got[0].score is not None and 0.0 <= got[0].score <= 1.0
+
+
+def test_embedding_retrieval_respects_filters(store):
+    store.write_documents([
+        Document(id="x", content="a", meta={"cat": "x"}, embedding=vec(0)),
+        Document(id="y", content="b", meta={"cat": "y"}, embedding=vec(0)),
+    ])
+    got = store.embedding_retrieval(
+        vec(0), top_k=5, filters={"field": "meta.cat", "operator": "==", "value": "y"}
+    )
+    assert [d.id for d in got] == ["y"]
+
+
+def test_embedding_retrieval_respects_top_k(store):
+    store.write_documents(
+        [Document(id=f"d{i}", content="t", embedding=vec(i % DIM)) for i in range(6)]
+    )
+    assert len(store.embedding_retrieval(vec(0), top_k=2)) == 2
+
+
+# ---- edges ---------------------------------------------------------------
+
+
+def test_writing_nothing_is_not_an_error(store):
+    assert store.write_documents([]) == 0
+    assert store.count_documents() == 0
+
+
+def test_filtering_an_empty_store_is_empty(store):
+    assert store.filter_documents() == []
+    assert store.filter_documents({"field": "meta.x", "operator": "==", "value": 1}) == []
+
+
+def test_an_or_filter_is_not_narrowed_by_pushdown(store):
+    """Pushing a leaf of an OR would drop rows the filter keeps."""
+    store.write_documents([
+        Document(id="a", content="1", meta={"cat": "x"}),
+        Document(id="b", content="2", meta={"cat": "y"}),
+    ])
+    got = store.filter_documents({
+        "operator": "OR",
+        "conditions": [
+            {"field": "meta.cat", "operator": "==", "value": "x"},
+            {"field": "meta.cat", "operator": "==", "value": "y"},
+        ],
+    })
+    assert sorted(d.id for d in got) == ["a", "b"]
+
+
+def test_a_none_valued_equality_filter_is_not_pushed(store):
+    """None means absent, which containment cannot express."""
+    store.write_documents([
+        Document(id="has", content="1", meta={"n": 1}),
+        Document(id="hasnt", content="2", meta={}),
+    ])
+    got = store.filter_documents({"field": "meta.n", "operator": "==", "value": None})
+    assert [d.id for d in got] == ["hasnt"]
+
+
+def test_a_document_with_no_content_is_storable(store):
+    store.write_documents([Document(id="empty", content=None, meta={"k": 1})])
+    got = store.filter_documents()[0]
+    assert got.content is None and got.meta["k"] == 1
+
+
+def test_filtering_on_a_document_field_not_metadata(store):
+    store.write_documents([Document(id="wanted", content="x")])
+    got = store.filter_documents({"field": "id", "operator": "==", "value": "wanted"})
+    assert [d.id for d in got] == ["wanted"]
+
+
+def test_a_not_filter_is_not_narrowed_by_pushdown(store):
+    store.write_documents([
+        Document(id="a", content="1", meta={"cat": "x"}),
+        Document(id="b", content="2", meta={"cat": "y"}),
+    ])
+    got = store.filter_documents({
+        "operator": "NOT",
+        "conditions": [{"field": "meta.cat", "operator": "==", "value": "x"}],
+    })
+    assert [d.id for d in got] == ["b"]
+
+
+def test_retrieval_on_an_empty_store_is_empty(store):
+    assert store.embedding_retrieval(vec(0), top_k=5) == []
+
+
+def test_scores_do_not_leak_onto_stored_documents(store):
+    """Mutating a shared Document affects other pipeline steps."""
+    store.write_documents([Document(id="a", content="x", embedding=vec(0))])
+    store.embedding_retrieval(vec(0), top_k=1)
+    assert store.filter_documents()[0].score is None
+
+
+def test_it_survives_a_reopen(tmp_path):
+    """A region's embedder lives in memory, so a reopen must rebuild it."""
+    import gc
+
+    p = str(tmp_path / "reopen.cdl")
+    first = CitadelDocumentStore(p, "pw", dim=DIM)
+    first.write_documents([Document(id="d1", content="the disk was full",
+                                    meta={"chapter": "intro"}, embedding=vec(3))])
+    del first
+    gc.collect()
+
+    again = CitadelDocumentStore(p, "pw", dim=DIM)
+    assert again.count_documents() == 1
+    got = again.filter_documents(
+        {"field": "meta.chapter", "operator": "==", "value": "intro"}
+    )
+    assert [d.id for d in got] == ["d1"]
+    assert got[0].embedding == vec(3), "the embedding did not survive the reopen"
+    assert [d.id for d in again.embedding_retrieval(vec(3), top_k=1)] == ["d1"]
+
+
+def test_a_wrong_passphrase_cannot_reopen(tmp_path):
+    """Pins the encryption claim rather than inferring it from a reopen."""
+    import gc
+
+    import citadeldb
+
+    p = str(tmp_path / "enc.cdl")
+    first = CitadelDocumentStore(p, "right", dim=DIM)
+    first.write_documents([Document(id="d1", content="the disk was full",
+                                    embedding=vec(3))])
+    del first
+    gc.collect()
+
+    with pytest.raises(citadeldb.EncryptionError):
+        CitadelDocumentStore(p, "wrong", dim=DIM)
+
+
+def test_concurrent_writes_all_land(tmp_path):
+    """Indexing pipelines run in parallel; the engine is shared."""
+    import concurrent.futures as cf
+
+    s = CitadelDocumentStore(str(tmp_path / "conc.cdl"), "pw", dim=DIM)
+    with cf.ThreadPoolExecutor(max_workers=4) as ex:
+        list(ex.map(
+            lambda i: s.write_documents(
+                [Document(id=f"c{i}", content=f"body {i}", embedding=vec(i % DIM))],
+                policy=DuplicatePolicy.OVERWRITE,
+            ),
+            range(40),
+        ))
+    assert s.count_documents() == 40
+
+
+def test_a_nested_metadata_filter_finds_its_document(tmp_path):
+    """`meta.person.name` addresses a nested value. Pushing it as one flat key
+    spelled with a dot matches nothing, and the post-filter never sees the row."""
+    s = CitadelDocumentStore(str(tmp_path / "nested.cdl"), "pw", dim=DIM)
+    s.write_documents([
+        Document(id="n1", content="nested", meta={"person": {"name": "ada"}},
+                 embedding=vec(0)),
+        Document(id="n2", content="other", meta={"person": {"name": "bob"}},
+                 embedding=vec(1)),
+    ])
+    f = {"field": "meta.person.name", "operator": "==", "value": "ada"}
+    assert [d.id for d in s.filter_documents(filters=f)] == ["n1"]
+    assert [d.id for d in s.embedding_retrieval(vec(0), top_k=5, filters=f)] == ["n1"]
+
+
+def test_a_numeric_metadata_filter_is_not_decided_by_the_pushdown(tmp_path):
+    """Haystack's `==` is Python's, which holds 1 == 1.0; containment is type
+    strict, so pushing a number would drop a row the filter keeps."""
+    s = CitadelDocumentStore(str(tmp_path / "num.cdl"), "pw", dim=DIM)
+    s.write_documents([Document(id="i1", content="int", meta={"year": 2024},
+                                embedding=vec(0))])
+    f = {"field": "meta.year", "operator": "==", "value": 2024.0}
+    assert [d.id for d in s.filter_documents(filters=f)] == ["i1"]
+
+
+def test_unembedded_documents_stay_out_of_embedding_retrieval(tmp_path):
+    """Haystack's own store excludes them. A hash of the text scores plausibly
+    and would push the one real match out of top_k."""
+    s = CitadelDocumentStore(str(tmp_path / "unemb.cdl"), "pw", dim=DIM)
+    s.write_documents([Document(id=f"u{i}", content=f"unembedded {i}") for i in range(20)])
+    s.write_documents([Document(id="real", content="embedded", embedding=vec(0))])
+    assert s.count_documents() == 21
+    found = s.embedding_retrieval(vec(0), top_k=10)
+    assert [d.id for d in found] == ["real"]
+    # They are stored and readable, just not ranked against a query vector.
+    assert len(s.filter_documents()) == 21
+
+
+def test_an_unpushable_filter_survives_a_window_of_other_documents(tmp_path):
+    """`in` cannot be pushed, so it is settled after ranking; the only matching
+    document must still be found under a corpus that outranks it."""
+    s = CitadelDocumentStore(str(tmp_path / "window.cdl"), "pw", dim=DIM)
+    s.write_documents([Document(id="gold", content="needle", meta={"tag": "keep"},
+                                embedding=vec(1))])
+    s.write_documents([
+        Document(id=f"c{i}", content=f"nearer {i}", meta={"tag": "drop"},
+                 embedding=vec(0))
+        for i in range(400)
+    ])
+    f = {"field": "meta.tag", "operator": "in", "value": ["keep"]}
+    assert [d.id for d in s.embedding_retrieval(vec(0), top_k=1, filters=f)] == ["gold"]
+
+
+def test_concurrent_overwrites_of_one_id_leave_one_document(tmp_path):
+    """A document id is unique, so parallel pipelines overwriting the same one
+    must supersede rather than each add a row."""
+    import concurrent.futures as cf
+
+    s = CitadelDocumentStore(str(tmp_path / "oneid.cdl"), "pw", dim=DIM)
+    with cf.ThreadPoolExecutor(max_workers=8) as ex:
+        list(ex.map(
+            lambda i: s.write_documents(
+                [Document(id="same", content=f"v{i}", embedding=vec(i % DIM))],
+                policy=DuplicatePolicy.OVERWRITE,
+            ),
+            range(32),
+        ))
+    assert s.count_documents() == 1
+    # Retrieval must not fill top_k with copies of the one document either.
+    found = s.embedding_retrieval(query_embedding=vec(0), top_k=5)
+    assert [d.id for d in found] == ["same"]
+    s.delete_documents(["same"])
+    assert s.count_documents() == 0
+
+
+def test_two_stores_share_one_database_file(tmp_path):
+    path = str(tmp_path / "shared.cdl")
+    a = CitadelDocumentStore(path, "pw", region="a", dim=DIM)
+    b = CitadelDocumentStore(path, "pw", region="b", dim=DIM)
+    a.write_documents([Document(id="1", content="in a")])
+    b.write_documents([Document(id="2", content="in b")])
+    assert a.count_documents() == 1 and b.count_documents() == 1
+
+
+def test_a_large_batch_keeps_every_document(store):
+    store.write_documents([Document(id=f"b{i}", content=f"body {i}") for i in range(300)])
+    assert store.count_documents() == 300
+
+
+def test_it_serializes_inside_a_pipeline(tmp_path, monkeypatch):
+    """A store is only useful if a pipeline carrying it can be written out."""
+    monkeypatch.setenv("CITADEL_TEST_KEY", "pw")
+    s = CitadelDocumentStore(
+        str(tmp_path / "p.cdl"), Secret.from_env_var("CITADEL_TEST_KEY"), dim=DIM
+    )
+    from haystack.components.writers import DocumentWriter
+
+    pipe = Pipeline()
+    pipe.add_component("writer", DocumentWriter(document_store=s))
+    assert "CitadelDocumentStore" in str(pipe.to_dict())
+
+
+# ---- async ---------------------------------------------------------------
+
+
+async def test_async_surface_round_trips(store):
+    assert await store.write_documents_async([Document(id="a", content="x")]) == 1
+    assert await store.count_documents_async() == 1
+    assert len(await store.filter_documents_async()) == 1
+    await store.delete_documents_async(["a"])
+    assert await store.count_documents_async() == 0
+
+
+async def test_the_event_loop_is_not_blocked(store):
+    import asyncio
+
+    ticks = 0
+
+    async def tick():
+        nonlocal ticks
+        while True:
+            ticks += 1
+            await asyncio.sleep(0)
+
+    ticker = asyncio.create_task(tick())
+    await asyncio.sleep(0)
+    await store.write_documents_async(
+        [Document(id=f"L{i}", content=f"loop {i}") for i in range(40)]
+    )
+    await store.filter_documents_async()
+    ticker.cancel()
+    assert ticks > 1, "the loop made no progress during a store call"