haystack-velesdb 7.0.0__tar.gz → 8.0.0__tar.gz

{haystack_velesdb-7.0.0/src/haystack_velesdb.egg-info → haystack_velesdb-8.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-velesdb
-Version: 7.0.0
+Version: 8.0.0
 Summary: Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database.
 Author-email: VelesDB Team <contact@wiscale.fr>
 License: MIT
@@ -20,8 +20,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: haystack-ai>=2.0.0
-Requires-Dist: velesdb>=3.8.0
-Requires-Dist: velesdb-common>=3.8.0
+Requires-Dist: velesdb>=5.1.0
+Requires-Dist: velesdb-common>=8.0.0
 Provides-Extra: dev
 Requires-Dist: pytest<9.0,>=7.0; extra == "dev"
 Dynamic: license-file
@@ -160,9 +160,14 @@ integers using the first 8 bytes of SHA-256 (~9.2 × 10¹⁸ slots).  For a
 is negligible for typical RAG workloads.  A `ValueError` is raised at write time
 if a collision is detected between a new document and an existing one.
 
-**Note on `scale_score`:** When `True` (default), cosine similarity scores
-are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
-in downstream re-ranking.
+**Note on `scale_score`:** When `True` (default), scores are higher-is-closer,
+as Haystack's `DocumentJoiner` and rankers expect: cosine similarities are
+normalised from `[-1, 1]` to `[0, 1]`, a `euclidean` or `hamming` distance `d`
+becomes `1 / (1 + d)`, and dot and Jaccard similarities are unchanged. A fused
+score (`fusion=...`) is never rescaled, but one that is still a distance
+(`average`, `maximum` or `weighted` over a distance metric) is mapped to
+`1 / (1 + d)`. The direction of each metric and strategy comes from
+velesdb-core, through `velesdb_common.scores`.
 
 **Note on streaming ingestion:** `stream_insert()` and `write_documents_streaming()`
 forward to the underlying `velesdb.Collection.stream_insert`, which requires

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0}/README.md

@@ -132,9 +132,14 @@ integers using the first 8 bytes of SHA-256 (~9.2 × 10¹⁸ slots).  For a
 is negligible for typical RAG workloads.  A `ValueError` is raised at write time
 if a collision is detected between a new document and an existing one.
 
-**Note on `scale_score`:** When `True` (default), cosine similarity scores
-are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
-in downstream re-ranking.
+**Note on `scale_score`:** When `True` (default), scores are higher-is-closer,
+as Haystack's `DocumentJoiner` and rankers expect: cosine similarities are
+normalised from `[-1, 1]` to `[0, 1]`, a `euclidean` or `hamming` distance `d`
+becomes `1 / (1 + d)`, and dot and Jaccard similarities are unchanged. A fused
+score (`fusion=...`) is never rescaled, but one that is still a distance
+(`average`, `maximum` or `weighted` over a distance metric) is mapped to
+`1 / (1 + d)`. The direction of each metric and strategy comes from
+velesdb-core, through `velesdb_common.scores`.
 
 **Note on streaming ingestion:** `stream_insert()` and `write_documents_streaming()`
 forward to the underlying `velesdb.Collection.stream_insert`, which requires

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "haystack-velesdb"
-version = "7.0.0"
+version = "8.0.0"
 description = "Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database."
 readme = "README.md"
 license = {text = "MIT"}
@@ -31,8 +31,8 @@ classifiers = [
 requires-python = ">=3.10"
 dependencies = [
     "haystack-ai>=2.0.0",
-    "velesdb>=3.8.0",
-    "velesdb-common>=3.8.0",
+    "velesdb>=5.1.0",
+    "velesdb-common>=8.0.0",
 ]
 
 [project.optional-dependencies]

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0}/src/haystack_velesdb/__init__.py

@@ -4,4 +4,4 @@ from haystack_velesdb.document_store import VelesDBDocumentStore
 from haystack_velesdb.retriever import VelesDBEmbeddingRetriever
 
 __all__ = ["VelesDBDocumentStore", "VelesDBEmbeddingRetriever"]
-__version__ = "7.0.0"
+__version__ = "8.0.0"

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0}/src/haystack_velesdb/document_store.py

@@ -17,6 +17,11 @@ import velesdb
 from velesdb_common.collection_admin import CollectionAdminMixin
 from velesdb_common.fusion import build_fusion_strategy
 from velesdb_common.ids import stable_hash_id
+from velesdb_common.scores import (
+    fused_higher_is_better,
+    score_to_similarity,
+    to_similarity,
+)
 from velesdb_common.security import (
     validate_collection_name,
     validate_metric,
@@ -225,8 +230,38 @@ def _doc_to_point(doc: Document, sparse_vector: Optional[dict] = None) -> dict:
     return point
 
 
+def _doc_score(
+    raw: Optional[float],
+    *,
+    scale_score: bool,
+    metric: str,
+    higher_is_better: Optional[bool],
+) -> Optional[float]:
+    """The ``Document.score`` of a VelesDB result score.
+
+    A fused score (``higher_is_better`` given, as core states it for the
+    strategy and metric) is always made higher-is-closer: Haystack ranks and
+    joins documents by descending score. A plain search score is scaled when
+    *scale_score* is set: cosine from ``[-1, 1]`` to ``[0, 1]``, a euclidean
+    or hamming distance ``d`` to ``1 / (1 + d)``, dot and Jaccard unchanged.
+    """
+    if raw is None:
+        return None
+    if higher_is_better is not None:
+        return to_similarity(raw, higher_is_better)
+    if not scale_score:
+        return raw
+    if metric == "cosine":
+        return (raw + 1.0) / 2.0
+    return score_to_similarity(metric, raw)
+
+
 def _result_to_doc(
-    result: dict, *, scale_score: bool = False, metric: str = "cosine"
+    result: dict,
+    *,
+    scale_score: bool = False,
+    metric: str = "cosine",
+    higher_is_better: Optional[bool] = None,
 ) -> Document:
     """Convert a VelesDB search or scroll result to a Haystack Document.
 
@@ -256,13 +291,12 @@ def _result_to_doc(
         )
     content = payload.get("content")
     meta = {k: v for k, v in payload.items() if k not in _RESERVED_PAYLOAD_KEYS}
-    raw_score: Optional[float] = result.get("score")
-    if scale_score and raw_score is not None and metric == "cosine":
-        # Normalise cosine similarity from [-1, 1] to [0, 1].
-        # Only meaningful for cosine; l2 and dot scores have different ranges.
-        score: Optional[float] = (raw_score + 1.0) / 2.0
-    else:
-        score = raw_score
+    score = _doc_score(
+        result.get("score"),
+        scale_score=scale_score,
+        metric=metric,
+        higher_is_better=higher_is_better,
+    )
     return Document(id=doc_id, content=content, meta=meta, score=score)
 
 
@@ -705,11 +739,13 @@ class VelesDBDocumentStore(CollectionAdminMixin):
             filters: Optional Haystack 2.x filter dict to restrict the search
                 space. Translated to VelesDB native shape before being
                 forwarded; ``meta.<key>`` is stripped to ``<key>``.
-            scale_score: When ``True`` and ``metric="cosine"``, scores are
-                normalised from ``[-1, 1]`` to ``[0, 1]``. Ignored for other
-                metrics, where raw scores are returned unchanged. Score
-                scaling does not apply to fused (``fusion``) results, whose
-                scores come from the fusion strategy rather than the metric.
+            scale_score: When ``True``, scores are higher-is-closer: cosine
+                normalised from ``[-1, 1]`` to ``[0, 1]``, a ``euclidean`` or
+                ``hamming`` distance ``d`` mapped to ``1 / (1 + d)``, dot and
+                Jaccard unchanged. When ``False``, raw scores are returned. A
+                fused (``fusion``) score is never rescaled, but a distance it
+                keeps (``average``, ``maximum`` or ``weighted`` over a distance
+                metric) is mapped to ``1 / (1 + d)`` either way.
             fusion: Optional fusion strategy name applied to the ranking —
                 one of ``"average"``, ``"maximum"``, ``"rrf"``,
                 ``"weighted"``, ``"relative_score"`` / ``"rsf"``. When set,
@@ -734,7 +770,7 @@ class VelesDBDocumentStore(CollectionAdminMixin):
         results: List[dict] = self._get_collection().search_request(
             velesdb.SearchOptions(
                 vector=query_embedding,
-                top_k=top_k,
+                k=top_k,
                 filter=veles_filter,
             )
         )
@@ -766,9 +802,14 @@ class VelesDBDocumentStore(CollectionAdminMixin):
             top_k=top_k,
             fusion=strategy,
         )
-        # Fused scores are strategy-derived, not metric similarities, so the
-        # cosine [-1, 1] -> [0, 1] rescaling is intentionally not applied.
-        return [_result_to_doc(r, metric=self._metric) for r in results]
+        # Fused scores are strategy-derived, not metric similarities: no
+        # cosine rescaling, but a score-space strategy over a distance metric
+        # still returns distances, which are made higher-is-closer.
+        higher_is_better = fused_higher_is_better(strategy, fusion, self._metric)
+        return [
+            _result_to_doc(r, metric=self._metric, higher_is_better=higher_is_better)
+            for r in results
+        ]
 
     # ------------------------------------------------------------------
     # Haystack pipeline serialisation

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0/src/haystack_velesdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haystack-velesdb
-Version: 7.0.0
+Version: 8.0.0
 Summary: Haystack 2.x DocumentStore for VelesDB: The Local AI Memory Database.
 Author-email: VelesDB Team <contact@wiscale.fr>
 License: MIT
@@ -20,8 +20,8 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: haystack-ai>=2.0.0
-Requires-Dist: velesdb>=3.8.0
-Requires-Dist: velesdb-common>=3.8.0
+Requires-Dist: velesdb>=5.1.0
+Requires-Dist: velesdb-common>=8.0.0
 Provides-Extra: dev
 Requires-Dist: pytest<9.0,>=7.0; extra == "dev"
 Dynamic: license-file
@@ -160,9 +160,14 @@ integers using the first 8 bytes of SHA-256 (~9.2 × 10¹⁸ slots).  For a
 is negligible for typical RAG workloads.  A `ValueError` is raised at write time
 if a collision is detected between a new document and an existing one.
 
-**Note on `scale_score`:** When `True` (default), cosine similarity scores
-are normalised from `[-1, 1]` to `[0, 1]` so they behave like probabilities
-in downstream re-ranking.
+**Note on `scale_score`:** When `True` (default), scores are higher-is-closer,
+as Haystack's `DocumentJoiner` and rankers expect: cosine similarities are
+normalised from `[-1, 1]` to `[0, 1]`, a `euclidean` or `hamming` distance `d`
+becomes `1 / (1 + d)`, and dot and Jaccard similarities are unchanged. A fused
+score (`fusion=...`) is never rescaled, but one that is still a distance
+(`average`, `maximum` or `weighted` over a distance metric) is mapped to
+`1 / (1 + d)`. The direction of each metric and strategy comes from
+velesdb-core, through `velesdb_common.scores`.
 
 **Note on streaming ingestion:** `stream_insert()` and `write_documents_streaming()`
 forward to the underlying `velesdb.Collection.stream_insert`, which requires

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0}/src/haystack_velesdb.egg-info/requires.txt

@@ -1,6 +1,6 @@
 haystack-ai>=2.0.0
-velesdb>=3.8.0
-velesdb-common>=3.8.0
+velesdb>=5.1.0
+velesdb-common>=8.0.0
 
 [dev]
 pytest<9.0,>=7.0

{haystack_velesdb-7.0.0 → haystack_velesdb-8.0.0}/tests/test_document_store.py

@@ -123,14 +123,14 @@ class _FakeSearchOptions:
         vector: Any = None,
         *,
         sparse_vector: Any = None,
-        top_k: int = 10,
+        k: int = 10,
         filter: Any = None,  # pylint: disable=redefined-builtin
         sparse_index_name: Any = None,
         include_vectors: bool = False,
     ) -> None:
         self.vector = vector
         self.sparse_vector = sparse_vector
-        self.top_k = top_k
+        self.k = k
         self.filter = filter
         self.sparse_index_name = sparse_index_name
         self.include_vectors = include_vectors
@@ -165,7 +165,7 @@ class _FakeCollection:
 
     def search_request(self, opts: Any) -> list:
         """Canonical search entry point — delegate to the legacy `search`."""
-        return self.search(opts.vector, top_k=opts.top_k, filter=opts.filter)
+        return self.search(opts.vector, top_k=opts.k, filter=opts.filter)
 
     def multi_query_search(
         self,
@@ -321,6 +321,18 @@ def _load_module() -> types.ModuleType:
     sys.modules["velesdb_common.collection_admin"] = vc_admin
     _admin_spec.loader.exec_module(vc_admin)
 
+    # The REAL velesdb_common.scores too: it imports velesdb lazily-guarded
+    # and, against this fake binding, uses its literal fallbacks, so the
+    # store's score direction is the canonical one.
+    _scores_path = (
+        Path(__file__).resolve().parents[2] / "common" / "src" / "velesdb_common" / "scores.py"
+    )
+    _scores_spec = importlib.util.spec_from_file_location("velesdb_common.scores", _scores_path)
+    assert _scores_spec and _scores_spec.loader
+    vc_scores = importlib.util.module_from_spec(_scores_spec)
+    sys.modules["velesdb_common.scores"] = vc_scores
+    _scores_spec.loader.exec_module(vc_scores)
+
     vc_sec = types.ModuleType("velesdb_common.security")
     vc_sec.validate_path = _passthrough  # type: ignore[attr-defined]
     vc_sec.validate_collection_name = _passthrough  # type: ignore[attr-defined]
@@ -397,12 +409,30 @@ def test_filter_documents_returns_all_when_none() -> None:
     assert len(store.filter_documents()) == 2
 
 
-def test_scale_score_not_applied_for_non_cosine_metric() -> None:
+def test_scale_score_maps_a_distance_to_a_higher_is_closer_score() -> None:
     store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_score_nc", metric="euclidean")
     store.write_documents([Document(id="z", content="raw", embedding=[1.0])])
     scaled = store.embedding_retrieval([1.0], scale_score=True)
-    # For euclidean metric scale_score should be a no-op — raw score returned.
-    assert scaled[0].score == 0.9
+    raw = store.embedding_retrieval([1.0], scale_score=False)
+    # A euclidean score is a distance: DocumentJoiner sorts descending, so the
+    # scaled score must grow as the distance shrinks.
+    assert scaled[0].score == 1.0 / (1.0 + 0.9)
+    assert raw[0].score == 0.9
+
+
+def test_scale_score_keeps_a_dot_similarity() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_score_dot", metric="dot")
+    store.write_documents([Document(id="d", content="dot", embedding=[1.0])])
+    assert store.embedding_retrieval([1.0], scale_score=True)[0].score == 0.9
+
+
+def test_score_space_fusion_over_a_distance_is_higher_is_closer() -> None:
+    store = _MOD.VelesDBDocumentStore(path="/tmp/hs", collection_name="t_fused_l2", metric="euclidean")
+    store.write_documents([Document(id="f", content="fused", embedding=[1.0])])
+    averaged = store.embedding_retrieval([1.0], fusion="average")
+    ranked = store.embedding_retrieval([1.0], fusion="rrf")
+    assert averaged[0].score == 1.0 / (1.0 + 0.9)  # a fused distance, mapped
+    assert ranked[0].score == 0.9  # RRF is already higher-is-better
 
 
 def test_scroll_limit_is_respected() -> None: