citadeldb-haystack 2.1.0__tar.gz → 2.2.0__tar.gz

{citadeldb_haystack-2.1.0 → citadeldb_haystack-2.2.0}/.gitignore

@@ -5,6 +5,7 @@
 *~
 .DS_Store
 site/public/
+site/data/release.json
 site/static/wasm/*.wasm
 site/static/wasm/*.js
 /notes/

{citadeldb_haystack-2.1.0 → citadeldb_haystack-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: citadeldb-haystack
-Version: 2.1.0
+Version: 2.2.0
 Summary: Haystack document store backed by Citadel: encrypted at rest, with deletes that destroy the key
 Project-URL: Homepage, https://citadeldb.dev
 Project-URL: Repository, https://github.com/yp3y5akh0v/citadel
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
-Requires-Dist: citadeldb<3,>=2.0
+Requires-Dist: citadeldb<3,>=2.2
 Requires-Dist: haystack-ai<4,>=2.9
 Provides-Extra: test
 Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
@@ -26,19 +26,20 @@ A [Haystack](https://github.com/deepset-ai/haystack) `DocumentStore` backed by
 [Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and deletes
 that destroy the key, not just the row.
 
-Passes deepset's own `DocumentStoreBaseTests` conformance suite.
-
 ```
-pip install citadeldb-haystack
+pip install citadeldb-haystack sentence-transformers
 ```
 
+Requires `citadeldb>=2.2,<3` and `haystack-ai>=2.9,<4`. Set `CITADEL_KEY` before
+running the example. The embedding model downloads on first use and runs locally.
+
 ```python
 from haystack import Document
 from haystack.components.embedders import SentenceTransformersTextEmbedder
 from haystack.utils import Secret
 from citadeldb_haystack import CitadelDocumentStore
 
-embedder = SentenceTransformersTextEmbedder()
+embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-mpnet-base-v2")
 store = CitadelDocumentStore(
     "corpus.cdl",
     Secret.from_env_var("CITADEL_KEY"),
@@ -46,9 +47,10 @@ store = CitadelDocumentStore(
     dim=768,
     embedding_similarity_function="cosine",
 )
-# CITADEL_KEY must be set: an env-var secret is what lets a pipeline serialize.
-
-store.write_documents([Document(id="d1", content="...", meta={"chapter": "intro"})])
+store.write_documents([
+    Document(id="d1", content="The deployment failed because the disk was full.",
+             meta={"chapter": "intro"}),
+])
 store.filter_documents({"field": "meta.chapter", "operator": "==", "value": "intro"})
 ```
 
@@ -60,10 +62,10 @@ Embedders exposing `model_id`, `model`, or `model_name` record that identity aut
 in that order. For a custom component without any of those attributes, pass a stable
 `model_id=` explicitly; Citadel refuses to guess from the Python class name.
 
-## The passphrase never lands in a pipeline file
+## Pipeline serialization
 
-The passphrase is a Haystack `Secret`. Pipelines are serialized to disk, and a literal
-token refuses to serialize, so a passphrase cannot be written into a pipeline by accident:
+Use a Haystack environment-variable `Secret` for pipeline serialization. Literal
+passphrases cannot be serialized:
 
 ```python
 CitadelDocumentStore(
@@ -72,7 +74,8 @@ CitadelDocumentStore(
 # ValueError: Cannot serialize token-based secret.
 
 CitadelDocumentStore(
-    "corpus.cdl", Secret.from_env_var("CITADEL_KEY"), embedder=embedder, dim=768
+    "corpus.cdl", Secret.from_env_var("CITADEL_KEY"), embedder=embedder, dim=768,
+    embedding_similarity_function="cosine",
 ).to_dict()
 # {... "key": {"type": "env_var", "env_vars": ["CITADEL_KEY"], ...}}
 ```
@@ -81,22 +84,22 @@ Use `Secret.from_env_var` for any store that goes into a saved pipeline.
 
 ## Deletes destroy the key
 
-Every document is sealed under its own key. Deleting destroys that key and then removes the
-row, so any ciphertext surviving elsewhere stays unreadable.
+Every document is sealed under its own key. Deleting destroys that key and removes the
+row. Pre-erasure backups or snapshots containing keys, and exported plaintext, are outside
+that erasure.
 
 ```python
 store.delete_documents(["d1"])
 store.delete_all()  # returns the number erased
 ```
 
-`DuplicatePolicy.NONE` falls back to `FAIL`, as `InMemoryDocumentStore` does, so an
-accidental re-write is reported rather than silently replacing a document whose key would
-then be destroyed.
+`DuplicatePolicy.NONE` is treated as `FAIL`: duplicate ids are rejected.
 
 ## Retrieval
 
 ```python
-query_embedding = [0.0] * 768  # from your Haystack text embedder, `dim` wide
+embedder.warm_up()
+query_embedding = embedder.run(text="Why did the release break?")["embedding"]
 
 store.embedding_retrieval(
     query_embedding,
@@ -107,23 +110,18 @@ store.embedding_retrieval(
 )
 ```
 
-Filtering uses Haystack's own evaluator, so the whole filter language, date comparisons
-included, matches `InMemoryDocumentStore` operator for operator. `top_k` is `top_k`: a
-filter matching only distant documents still returns them, however many others outrank
-them.
-
-A top-level `AND` of string equality conditions is pushed into the scan, including nested
-paths like `meta.person.name`. Everything else is evaluated afterwards, so the two agree:
-nothing is pushed under `OR` or `NOT`, and numbers are not pushed either, because `==` here
-is Python's (`1 == 1.0`) where the stored comparison is JSON-type exact.
+Filters use Haystack's evaluator.
+Top-level `AND` string equalities, including nested paths such as `meta.person.name`,
+are passed to Citadel as payload filters. Other predicates filter ranked candidates; the
+search window expands until `top_k` matches survive or the region is exhausted.
 
 ## Notes
 
 The store requires a Haystack text embedder. Documents that arrive without a vector are
 embedded with it, while vectors already supplied by the pipeline are stored as-is. Pass the
 same model to the pipeline and store so both paths remain in one vector space. The store warms
-the embedder lazily before its first model call and serializes it, so pipeline round-trips retain
-the model instead of reopening with a hidden fallback.
+the embedder lazily before its first model call and includes its configuration in pipeline
+serialization.
 
 Citadel is embedded and one process owns the file. A path already open on this thread,
 under the same passphrase, is shared, so this can sit on the same database as another

{citadeldb_haystack-2.1.0 → citadeldb_haystack-2.2.0}/README.md

@@ -4,19 +4,20 @@ A [Haystack](https://github.com/deepset-ai/haystack) `DocumentStore` backed by
 [Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and deletes
 that destroy the key, not just the row.
 
-Passes deepset's own `DocumentStoreBaseTests` conformance suite.
-
 ```
-pip install citadeldb-haystack
+pip install citadeldb-haystack sentence-transformers
 ```
 
+Requires `citadeldb>=2.2,<3` and `haystack-ai>=2.9,<4`. Set `CITADEL_KEY` before
+running the example. The embedding model downloads on first use and runs locally.
+
 ```python
 from haystack import Document
 from haystack.components.embedders import SentenceTransformersTextEmbedder
 from haystack.utils import Secret
 from citadeldb_haystack import CitadelDocumentStore
 
-embedder = SentenceTransformersTextEmbedder()
+embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-mpnet-base-v2")
 store = CitadelDocumentStore(
     "corpus.cdl",
     Secret.from_env_var("CITADEL_KEY"),
@@ -24,9 +25,10 @@ store = CitadelDocumentStore(
     dim=768,
     embedding_similarity_function="cosine",
 )
-# CITADEL_KEY must be set: an env-var secret is what lets a pipeline serialize.
-
-store.write_documents([Document(id="d1", content="...", meta={"chapter": "intro"})])
+store.write_documents([
+    Document(id="d1", content="The deployment failed because the disk was full.",
+             meta={"chapter": "intro"}),
+])
 store.filter_documents({"field": "meta.chapter", "operator": "==", "value": "intro"})
 ```
 
@@ -38,10 +40,10 @@ Embedders exposing `model_id`, `model`, or `model_name` record that identity aut
 in that order. For a custom component without any of those attributes, pass a stable
 `model_id=` explicitly; Citadel refuses to guess from the Python class name.
 
-## The passphrase never lands in a pipeline file
+## Pipeline serialization
 
-The passphrase is a Haystack `Secret`. Pipelines are serialized to disk, and a literal
-token refuses to serialize, so a passphrase cannot be written into a pipeline by accident:
+Use a Haystack environment-variable `Secret` for pipeline serialization. Literal
+passphrases cannot be serialized:
 
 ```python
 CitadelDocumentStore(
@@ -50,7 +52,8 @@ CitadelDocumentStore(
 # ValueError: Cannot serialize token-based secret.
 
 CitadelDocumentStore(
-    "corpus.cdl", Secret.from_env_var("CITADEL_KEY"), embedder=embedder, dim=768
+    "corpus.cdl", Secret.from_env_var("CITADEL_KEY"), embedder=embedder, dim=768,
+    embedding_similarity_function="cosine",
 ).to_dict()
 # {... "key": {"type": "env_var", "env_vars": ["CITADEL_KEY"], ...}}
 ```
@@ -59,22 +62,22 @@ Use `Secret.from_env_var` for any store that goes into a saved pipeline.
 
 ## Deletes destroy the key
 
-Every document is sealed under its own key. Deleting destroys that key and then removes the
-row, so any ciphertext surviving elsewhere stays unreadable.
+Every document is sealed under its own key. Deleting destroys that key and removes the
+row. Pre-erasure backups or snapshots containing keys, and exported plaintext, are outside
+that erasure.
 
 ```python
 store.delete_documents(["d1"])
 store.delete_all()  # returns the number erased
 ```
 
-`DuplicatePolicy.NONE` falls back to `FAIL`, as `InMemoryDocumentStore` does, so an
-accidental re-write is reported rather than silently replacing a document whose key would
-then be destroyed.
+`DuplicatePolicy.NONE` is treated as `FAIL`: duplicate ids are rejected.
 
 ## Retrieval
 
 ```python
-query_embedding = [0.0] * 768  # from your Haystack text embedder, `dim` wide
+embedder.warm_up()
+query_embedding = embedder.run(text="Why did the release break?")["embedding"]
 
 store.embedding_retrieval(
     query_embedding,
@@ -85,23 +88,18 @@ store.embedding_retrieval(
 )
 ```
 
-Filtering uses Haystack's own evaluator, so the whole filter language, date comparisons
-included, matches `InMemoryDocumentStore` operator for operator. `top_k` is `top_k`: a
-filter matching only distant documents still returns them, however many others outrank
-them.
-
-A top-level `AND` of string equality conditions is pushed into the scan, including nested
-paths like `meta.person.name`. Everything else is evaluated afterwards, so the two agree:
-nothing is pushed under `OR` or `NOT`, and numbers are not pushed either, because `==` here
-is Python's (`1 == 1.0`) where the stored comparison is JSON-type exact.
+Filters use Haystack's evaluator.
+Top-level `AND` string equalities, including nested paths such as `meta.person.name`,
+are passed to Citadel as payload filters. Other predicates filter ranked candidates; the
+search window expands until `top_k` matches survive or the region is exhausted.
 
 ## Notes
 
 The store requires a Haystack text embedder. Documents that arrive without a vector are
 embedded with it, while vectors already supplied by the pipeline are stored as-is. Pass the
 same model to the pipeline and store so both paths remain in one vector space. The store warms
-the embedder lazily before its first model call and serializes it, so pipeline round-trips retain
-the model instead of reopening with a hidden fallback.
+the embedder lazily before its first model call and includes its configuration in pipeline
+serialization.
 
 Citadel is embedded and one process owns the file. A path already open on this thread,
 under the same passphrase, is shared, so this can sit on the same database as another

{citadeldb_haystack-2.1.0 → citadeldb_haystack-2.2.0}/pyproject.toml

@@ -18,8 +18,8 @@ classifiers = [
   "Topic :: Database",
   "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
-# The precomputed vector needs the `embedding` field added in citadeldb 2.0.
-dependencies = ["citadeldb>=2.0,<3", "haystack-ai>=2.9,<4"]
+# Reads the expanded AtomHit scoring fields added in citadeldb 2.2.
+dependencies = ["citadeldb>=2.2,<3", "haystack-ai>=2.9,<4"]
 
 [project.optional-dependencies]
 test = ["pytest>=8", "pytest-asyncio>=0.23"]

{citadeldb_haystack-2.1.0 → citadeldb_haystack-2.2.0}/src/citadeldb_haystack/document_store.py

@@ -119,10 +119,27 @@ class _HaystackEmbedder:
         return vector
 
     def embed(self, texts: list[str]) -> list[list[float]]:
-        return [self._one(text) for text in texts]
+        return self.embed_with_cancel(texts, None)
 
     def embed_queries(self, texts: list[str]) -> list[list[float]]:
-        return self.embed(texts)
+        return self.embed_queries_with_cancel(texts, None)
+
+    def embed_with_cancel(
+        self, texts: list[str], cancel_token: Any | None
+    ) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for text in texts:
+            if cancel_token is not None:
+                cancel_token.check()
+            vectors.append(self._one(text))
+            if cancel_token is not None:
+                cancel_token.check()
+        return vectors
+
+    def embed_queries_with_cancel(
+        self, texts: list[str], cancel_token: Any | None
+    ) -> list[list[float]]:
+        return self.embed_with_cancel(texts, cancel_token)
 
 
 def _fetch(mem: Any, region: str, criterion: dict[str, Any] | None) -> list[Any]:
@@ -449,7 +466,7 @@ class CitadelDocumentStore:
                 if filters and not document_matches_filter(filters, doc):
                     continue
                 if h.distance is None:
-                    score = h.score
+                    score = h.relevance if h.relevance is not None else 0.0
                 elif self.embedding_similarity_function == "dot_product":
                     score = -h.distance
                 else: