qdrant-haystack 4.1.1__tar.gz → 4.2.0__tar.gz

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [integrations/qdrant-v4.1.2] - 2024-07-15
+
+### 🐛 Bug Fixes
+
+- `qdrant` - Fallback to default filter policy when deserializing retrievers without the init parameter (#902)
+
+## [integrations/qdrant-v4.1.1] - 2024-07-10
+
+### 🚀 Features
+
+- Add filter_policy to qdrant integration (#819)
+
+### 🐛 Bug Fixes
+
+- Errors in convert_filters_to_qdrant (#870)
+
 ## [integrations/qdrant-v4.1.0] - 2024-07-03
 
 ### 🚀 Features

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qdrant-haystack
-Version: 4.1.1
+Version: 4.2.0
 Summary: An integration of Qdrant ANN vector database backend with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/examples/embedding_retrieval.py

@@ -12,6 +12,7 @@ from haystack.components.converters import MarkdownToDocument
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
 from haystack.components.preprocessors import DocumentSplitter
 from haystack.components.writers import DocumentWriter
+
 from haystack_integrations.components.retrievers.qdrant import QdrantEmbeddingRetriever
 from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/pyproject.toml

@@ -46,10 +46,12 @@ git_describe_command = 'git describe --tags --match="integrations/qdrant-v[0-9]*
 [tool.hatch.envs.default]
 dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "haystack-pydoc-tools"]
 [tool.hatch.envs.default.scripts]
-test = "pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
-test-cov = "coverage run -m pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+test-cov-retry = "test-cov --reruns 3 --reruns-delay 30 -x"
 cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
+cov-retry = ["test-cov-retry", "cov-report"]
 docs = ["pydoc-markdown pydoc/config.yml"]
 
 [[tool.hatch.envs.all.matrix]]
@@ -60,8 +62,8 @@ detached = true
 dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = ["ruff check {args:.}", "black --check --diff {args:.}"]
-fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
+style = ["ruff check {args:. --exclude tests/, examples/}", "black --check --diff {args:.}"]
+fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/, examples/}", "style"]
 all = ["style", "typing"]
 
 [tool.black]

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/src/haystack_integrations/components/retrievers/qdrant/retriever.py

@@ -4,9 +4,10 @@ from haystack import Document, component, default_from_dict, default_to_dict
 from haystack.dataclasses.sparse_embedding import SparseEmbedding
 from haystack.document_stores.types import FilterPolicy
 from haystack.document_stores.types.filter_policy import apply_filter_policy
-from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 from qdrant_client.http import models
 
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+
 
 @component
 class QdrantEmbeddingRetriever:
@@ -108,7 +109,10 @@ class QdrantEmbeddingRetriever:
         """
         document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
         data["init_parameters"]["document_store"] = document_store
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -249,7 +253,10 @@ class QdrantSparseEmbeddingRetriever:
         """
         document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
         data["init_parameters"]["document_store"] = document_store
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -394,7 +401,10 @@ class QdrantHybridRetriever:
         """
         document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
         data["init_parameters"]["document_store"] = document_store
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/src/haystack_integrations/document_stores/qdrant/document_store.py

@@ -15,7 +15,6 @@ from haystack.utils.filters import convert as convert_legacy_filters
 from qdrant_client import grpc
 from qdrant_client.http import models as rest
 from qdrant_client.http.exceptions import UnexpectedResponse
-from qdrant_client.hybrid.fusion import reciprocal_rank_fusion
 from tqdm import tqdm
 
 from .converters import (
@@ -537,20 +536,18 @@ class QdrantDocumentStore:
         qdrant_filters = convert_filters_to_qdrant(filters)
         query_indices = query_sparse_embedding.indices
         query_values = query_sparse_embedding.values
-        points = self.client.search(
+        points = self.client.query_points(
             collection_name=self.index,
-            query_vector=rest.NamedSparseVector(
-                name=SPARSE_VECTORS_NAME,
-                vector=rest.SparseVector(
-                    indices=query_indices,
-                    values=query_values,
-                ),
+            query=rest.SparseVector(
+                indices=query_indices,
+                values=query_values,
             ),
+            using=SPARSE_VECTORS_NAME,
             query_filter=qdrant_filters,
             limit=top_k,
             with_vectors=return_embedding,
             score_threshold=score_threshold,
-        )
+        ).points
         results = [
             convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings)
             for point in points
@@ -588,17 +585,15 @@ class QdrantDocumentStore:
         """
         qdrant_filters = convert_filters_to_qdrant(filters)
 
-        points = self.client.search(
+        points = self.client.query_points(
             collection_name=self.index,
-            query_vector=rest.NamedVector(
-                name=DENSE_VECTORS_NAME if self.use_sparse_embeddings else "",
-                vector=query_embedding,
-            ),
+            query=query_embedding,
+            using=DENSE_VECTORS_NAME if self.use_sparse_embeddings else None,
             query_filter=qdrant_filters,
             limit=top_k,
             with_vectors=return_embedding,
             score_threshold=score_threshold,
-        )
+        ).points
         results = [
             convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=self.use_sparse_embeddings)
             for point in points
@@ -655,46 +650,34 @@ class QdrantDocumentStore:
 
         qdrant_filters = convert_filters_to_qdrant(filters)
 
-        sparse_request = rest.SearchRequest(
-            vector=rest.NamedSparseVector(
-                name=SPARSE_VECTORS_NAME,
-                vector=rest.SparseVector(
-                    indices=query_sparse_embedding.indices,
-                    values=query_sparse_embedding.values,
-                ),
-            ),
-            filter=qdrant_filters,
-            limit=top_k,
-            with_payload=True,
-            with_vector=return_embedding,
-            score_threshold=score_threshold,
-        )
-
-        dense_request = rest.SearchRequest(
-            vector=rest.NamedVector(
-                name=DENSE_VECTORS_NAME,
-                vector=query_embedding,
-            ),
-            filter=qdrant_filters,
-            limit=top_k,
-            with_payload=True,
-            with_vector=return_embedding,
-        )
-
         try:
-            dense_request_response, sparse_request_response = self.client.search_batch(
-                collection_name=self.index, requests=[dense_request, sparse_request]
-            )
+            points = self.client.query_points(
+                collection_name=self.index,
+                prefetch=[
+                    rest.Prefetch(
+                        query=rest.SparseVector(
+                            indices=query_sparse_embedding.indices,
+                            values=query_sparse_embedding.values,
+                        ),
+                        using=SPARSE_VECTORS_NAME,
+                        filter=qdrant_filters,
+                    ),
+                    rest.Prefetch(
+                        query=query_embedding,
+                        using=DENSE_VECTORS_NAME,
+                        filter=qdrant_filters,
+                    ),
+                ],
+                query=rest.FusionQuery(fusion=rest.Fusion.RRF),
+                limit=top_k,
+                score_threshold=score_threshold,
+                with_payload=True,
+                with_vectors=return_embedding,
+            ).points
         except Exception as e:
             msg = "Error during hybrid search"
             raise QdrantStoreError(msg) from e
 
-        try:
-            points = reciprocal_rank_fusion(responses=[dense_request_response, sparse_request_response], limit=top_k)
-        except Exception as e:
-            msg = "Error while applying Reciprocal Rank Fusion"
-            raise QdrantStoreError(msg) from e
-
         results = [convert_qdrant_point_to_haystack_document(point, use_sparse_embeddings=True) for point in points]
 
         return results

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/src/haystack_integrations/document_stores/qdrant/migrate_to_sparse.py

@@ -1,9 +1,10 @@
 import logging
 import time
 
-from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 from qdrant_client.http import models
 
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
 logger.setLevel(logging.INFO)

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/tests/test_converters.py

@@ -1,9 +1,10 @@
 import numpy as np
+from qdrant_client.http import models as rest
+
 from haystack_integrations.document_stores.qdrant.converters import (
     convert_id,
     convert_qdrant_point_to_haystack_document,
 )
-from qdrant_client.http import models as rest
 
 
 def test_convert_id_is_deterministic():

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/tests/test_dict_converters.py

@@ -1,4 +1,5 @@
 from haystack.utils import Secret
+
 from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 
 

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/tests/test_document_store.py

@@ -12,12 +12,13 @@ from haystack.testing.document_store import (
     WriteDocumentsTest,
     _random_embeddings,
 )
+from qdrant_client.http import models as rest
+
 from haystack_integrations.document_stores.qdrant.document_store import (
     SPARSE_VECTORS_NAME,
     QdrantDocumentStore,
     QdrantStoreError,
 )
-from qdrant_client.http import models as rest
 
 
 class TestQdrantDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsTest):
@@ -113,19 +114,7 @@ class TestQdrantDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocu
         sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
         embedding = [0.1] * 768
 
-        with patch.object(document_store.client, "search_batch", side_effect=Exception("search_batch error")):
+        with patch.object(document_store.client, "query_points", side_effect=Exception("query_points")):
 
             with pytest.raises(QdrantStoreError):
                 document_store._query_hybrid(query_sparse_embedding=sparse_embedding, query_embedding=embedding)
-
-    @patch("haystack_integrations.document_stores.qdrant.document_store.reciprocal_rank_fusion")
-    def test_query_hybrid_reciprocal_rank_fusion_failure(self, mocked_fusion):
-        document_store = QdrantDocumentStore(location=":memory:", use_sparse_embeddings=True)
-
-        sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
-        embedding = [0.1] * 768
-
-        mocked_fusion.side_effect = Exception("reciprocal_rank_fusion error")
-
-        with pytest.raises(QdrantStoreError):
-            document_store._query_hybrid(query_sparse_embedding=sparse_embedding, query_embedding=embedding)

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/tests/test_filters.py

@@ -4,9 +4,10 @@ import pytest
 from haystack import Document
 from haystack.testing.document_store import FilterDocumentsTest
 from haystack.utils.filters import FilterError
-from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 from qdrant_client.http import models
 
+from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+
 
 class TestQdrantStoreBaseTests(FilterDocumentsTest):
     @pytest.fixture

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/tests/test_legacy_filters.py

@@ -5,6 +5,7 @@ from haystack import Document
 from haystack.document_stores.types import DocumentStore
 from haystack.testing.document_store import LegacyFilterDocumentsTest
 from haystack.utils.filters import FilterError
+
 from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 
 # The tests below are from haystack.testing.document_store.LegacyFilterDocumentsTest

{qdrant_haystack-4.1.1 → qdrant_haystack-4.2.0}/tests/test_retriever.py

@@ -8,6 +8,7 @@ from haystack.testing.document_store import (
     FilterableDocsFixtureMixin,
     _random_embeddings,
 )
+
 from haystack_integrations.components.retrievers.qdrant import (
     QdrantEmbeddingRetriever,
     QdrantHybridRetriever,
@@ -151,7 +152,10 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
             filters={"field": "meta.chapter", "operator": "==", "value": "abstract"},
             return_embedding=False,
         )["documents"]
-        assert len(results) == 3
+        # we need to combine init filter and run filter as the policy is MERGE
+        # when we combine these filters we use AND logical operator by default
+        # so the result should be 1 as we have only one document that matches both filters
+        assert len(results) == 1
 
         for document in results:
             assert document.embedding is None
@@ -296,6 +300,31 @@ class TestQdrantSparseEmbeddingRetriever(FilterableDocsFixtureMixin):
         assert retriever._return_embedding is True
         assert retriever._score_threshold is None
 
+    def test_from_dict_no_filter_policy(self):
+        data = {
+            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseEmbeddingRetriever",
+            "init_parameters": {
+                "document_store": {
+                    "init_parameters": {"location": ":memory:", "index": "test"},
+                    "type": "haystack_integrations.document_stores.qdrant.document_store.QdrantDocumentStore",
+                },
+                "filters": None,
+                "top_k": 5,
+                "scale_score": False,
+                "return_embedding": True,
+                "score_threshold": None,
+            },
+        }
+        retriever = QdrantSparseEmbeddingRetriever.from_dict(data)
+        assert isinstance(retriever._document_store, QdrantDocumentStore)
+        assert retriever._document_store.index == "test"
+        assert retriever._filters is None
+        assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
+        assert retriever._scale_score is False
+        assert retriever._return_embedding is True
+        assert retriever._score_threshold is None
+
     def test_run(self, filterable_docs: List[Document], generate_sparse_embedding):
         document_store = QdrantDocumentStore(location=":memory:", index="Boi", use_sparse_embeddings=True)
 
@@ -414,6 +443,29 @@ class TestQdrantHybridRetriever:
         assert retriever._return_embedding
         assert retriever._score_threshold is None
 
+    def test_from_dict_no_filter_policy(self):
+        data = {
+            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantHybridRetriever",
+            "init_parameters": {
+                "document_store": {
+                    "init_parameters": {"location": ":memory:", "index": "test"},
+                    "type": "haystack_integrations.document_stores.qdrant.document_store.QdrantDocumentStore",
+                },
+                "filters": None,
+                "top_k": 5,
+                "return_embedding": True,
+                "score_threshold": None,
+            },
+        }
+        retriever = QdrantHybridRetriever.from_dict(data)
+        assert isinstance(retriever._document_store, QdrantDocumentStore)
+        assert retriever._document_store.index == "test"
+        assert retriever._filters is None
+        assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
+        assert retriever._return_embedding
+        assert retriever._score_threshold is None
+
     def test_run(self):
         mock_store = Mock(spec=QdrantDocumentStore)
         sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])