qdrant-haystack 3.1.0__py3-none-any.whl → 3.2.1__py3-none-any.whl

haystack_integrations/document_stores/qdrant/converters.py

@@ -1,9 +1,12 @@
+import logging
 import uuid
 from typing import List, Union
 
 from haystack.dataclasses import Document
 from qdrant_client.http import models as rest
 
+logger = logging.getLogger(__name__)
+
 
 class HaystackToQdrant:
     """A converter from Haystack to Qdrant types."""
@@ -22,6 +25,17 @@ class HaystackToQdrant:
             vector = payload.pop(embedding_field) or {}
             _id = self.convert_id(payload.get("id"))
 
+            # TODO: remove as soon as we introduce the support for sparse embeddings in Qdrant
+            if "sparse_embedding" in payload:
+                sparse_embedding = payload.pop("sparse_embedding", None)
+                if sparse_embedding:
+                    logger.warning(
+                        "Document %s has the `sparse_embedding` field set,"
+                        "but storing sparse embeddings in Qdrant is not currently supported."
+                        "The `sparse_embedding` field will be ignored.",
+                        payload["id"],
+                    )
+
             point = rest.PointStruct(
                 payload=payload,
                 vector=vector,

haystack_integrations/document_stores/qdrant/document_store.py

@@ -63,6 +63,7 @@ class QdrantDocumentStore:
         path: Optional[str] = None,
         index: str = "Document",
         embedding_dim: int = 768,
+        on_disk: bool = False,  # noqa: FBT001, FBT002
         content_field: str = "content",
         name_field: str = "name",
         embedding_field: str = "embedding",
@@ -84,6 +85,7 @@ class QdrantDocumentStore:
         metadata: Optional[dict] = None,
         write_batch_size: int = 100,
         scroll_size: int = 10_000,
+        payload_fields_to_index: Optional[List[dict]] = None,
     ):
         super().__init__()
 
@@ -130,11 +132,13 @@ class QdrantDocumentStore:
         self.init_from = init_from
         self.wait_result_from_api = wait_result_from_api
         self.recreate_index = recreate_index
+        self.payload_fields_to_index = payload_fields_to_index
 
         # Make sure the collection is properly set up
-        self._set_up_collection(index, embedding_dim, recreate_index, similarity)
+        self._set_up_collection(index, embedding_dim, recreate_index, similarity, on_disk, payload_fields_to_index)
 
         self.embedding_dim = embedding_dim
+        self.on_disk = on_disk
         self.content_field = content_field
         self.name_field = name_field
         self.embedding_field = embedding_field
@@ -334,19 +338,36 @@ class QdrantDocumentStore:
             )
             raise QdrantStoreError(msg) from ke
 
+    def _create_payload_index(self, collection_name: str, payload_fields_to_index: Optional[List[dict]] = None):
+        """
+        Create payload index for the collection if payload_fields_to_index is provided
+        See: https://qdrant.tech/documentation/concepts/indexing/#payload-index
+        """
+        if payload_fields_to_index is not None:
+            for payload_index in payload_fields_to_index:
+                self.client.create_payload_index(
+                    collection_name=collection_name,
+                    field_name=payload_index["field_name"],
+                    field_schema=payload_index["field_schema"],
+                )
+
     def _set_up_collection(
         self,
         collection_name: str,
         embedding_dim: int,
         recreate_collection: bool,  # noqa: FBT001
         similarity: str,
+        on_disk: bool = False,  # noqa: FBT001, FBT002
+        payload_fields_to_index: Optional[List[dict]] = None,
     ):
         distance = self._get_distance(similarity)
 
         if recreate_collection:
             # There is no need to verify the current configuration of that
             # collection. It might be just recreated again.
-            self._recreate_collection(collection_name, distance, embedding_dim)
+            self._recreate_collection(collection_name, distance, embedding_dim, on_disk)
+            # Create Payload index if payload_fields_to_index is provided
+            self._create_payload_index(collection_name, payload_fields_to_index)
             return
 
         try:
@@ -360,7 +381,9 @@ class QdrantDocumentStore:
             # Qdrant local raises ValueError if the collection is not found, but
             # with the remote server UnexpectedResponse / RpcError is raised.
             # Until that's unified, we need to catch both.
-            self._recreate_collection(collection_name, distance, embedding_dim)
+            self._recreate_collection(collection_name, distance, embedding_dim, on_disk)
+            # Create Payload index if payload_fields_to_index is provided
+            self._create_payload_index(collection_name, payload_fields_to_index)
             return
 
         current_distance = collection_info.config.params.vectors.distance
@@ -384,11 +407,12 @@ class QdrantDocumentStore:
             )
             raise ValueError(msg)
 
-    def _recreate_collection(self, collection_name: str, distance, embedding_dim: int):
+    def _recreate_collection(self, collection_name: str, distance, embedding_dim: int, on_disk: bool):  # noqa: FBT001
         self.client.recreate_collection(
             collection_name=collection_name,
             vectors_config=rest.VectorParams(
                 size=embedding_dim,
+                on_disk=on_disk,
                 distance=distance,
             ),
             shard_number=self.shard_number,

haystack_integrations/document_stores/qdrant/filters.py

@@ -1,5 +1,5 @@
-from abc import ABC, abstractmethod
-from typing import Any, List, Optional, Union
+from datetime import datetime
+from typing import List, Optional, Union
 
 from haystack.utils.filters import COMPARISON_OPERATORS, LOGICAL_OPERATORS, FilterError
 from qdrant_client.http import models
@@ -10,15 +10,7 @@ COMPARISON_OPERATORS = COMPARISON_OPERATORS.keys()
 LOGICAL_OPERATORS = LOGICAL_OPERATORS.keys()
 
 
-class BaseFilterConverter(ABC):
-    """Converts Haystack filters to a format accepted by an external tool."""
-
-    @abstractmethod
-    def convert(self, filter_term: Optional[Union[List[dict], dict]]) -> Optional[Any]:
-        raise NotImplementedError
-
-
-class QdrantFilterConverter(BaseFilterConverter):
+class QdrantFilterConverter:
     """Converts Haystack filters to the format used by Qdrant."""
 
     def __init__(self):
@@ -141,36 +133,52 @@ class QdrantFilterConverter(BaseFilterConverter):
             must_not=[
                 (
                     models.FieldCondition(key=key, match=models.MatchText(text=item))
-                    if isinstance(item, str) and " " not in item
+                    if isinstance(item, str) and " " in item
                     else models.FieldCondition(key=key, match=models.MatchValue(value=item))
                 )
                 for item in value
             ]
         )
 
-    def _build_lt_condition(self, key: str, value: float) -> models.Condition:
-        if not isinstance(value, (int, float)):
-            msg = f"Value {value} is not an int or float"
-            raise FilterError(msg)
-        return models.FieldCondition(key=key, range=models.Range(lt=value))
+    def _build_lt_condition(self, key: str, value: Union[str, float, int]) -> models.Condition:
+        if isinstance(value, str) and is_datetime_string(value):
+            return models.FieldCondition(key=key, range=models.DatetimeRange(lt=value))
 
-    def _build_lte_condition(self, key: str, value: float) -> models.Condition:
-        if not isinstance(value, (int, float)):
-            msg = f"Value {value} is not an int or float"
-            raise FilterError(msg)
-        return models.FieldCondition(key=key, range=models.Range(lte=value))
+        if isinstance(value, (int, float)):
+            return models.FieldCondition(key=key, range=models.Range(lt=value))
 
-    def _build_gt_condition(self, key: str, value: float) -> models.Condition:
-        if not isinstance(value, (int, float)):
-            msg = f"Value {value} is not an int or float"
-            raise FilterError(msg)
-        return models.FieldCondition(key=key, range=models.Range(gt=value))
+        msg = f"Value {value} is not an int or float or datetime string"
+        raise FilterError(msg)
 
-    def _build_gte_condition(self, key: str, value: float) -> models.Condition:
-        if not isinstance(value, (int, float)):
-            msg = f"Value {value} is not an int or float"
-            raise FilterError(msg)
-        return models.FieldCondition(key=key, range=models.Range(gte=value))
+    def _build_lte_condition(self, key: str, value: Union[str, float, int]) -> models.Condition:
+        if isinstance(value, str) and is_datetime_string(value):
+            return models.FieldCondition(key=key, range=models.DatetimeRange(lte=value))
+
+        if isinstance(value, (int, float)):
+            return models.FieldCondition(key=key, range=models.Range(lte=value))
+
+        msg = f"Value {value} is not an int or float or datetime string"
+        raise FilterError(msg)
+
+    def _build_gt_condition(self, key: str, value: Union[str, float, int]) -> models.Condition:
+        if isinstance(value, str) and is_datetime_string(value):
+            return models.FieldCondition(key=key, range=models.DatetimeRange(gt=value))
+
+        if isinstance(value, (int, float)):
+            return models.FieldCondition(key=key, range=models.Range(gt=value))
+
+        msg = f"Value {value} is not an int or float or datetime string"
+        raise FilterError(msg)
+
+    def _build_gte_condition(self, key: str, value: Union[str, float, int]) -> models.Condition:
+        if isinstance(value, str) and is_datetime_string(value):
+            return models.FieldCondition(key=key, range=models.DatetimeRange(gte=value))
+
+        if isinstance(value, (int, float)):
+            return models.FieldCondition(key=key, range=models.Range(gte=value))
+
+        msg = f"Value {value} is not an int or float or datetime string"
+        raise FilterError(msg)
 
     def _build_has_id_condition(self, id_values: List[models.ExtendedPointId]) -> models.HasIdCondition:
         return models.HasIdCondition(
@@ -215,3 +223,11 @@ class QdrantFilterConverter(BaseFilterConverter):
                 return models.Filter(**{part_name: subfilter.must})
 
         return payload_filter
+
+
+def is_datetime_string(value: str) -> bool:
+    try:
+        datetime.fromisoformat(value)
+        return True
+    except ValueError:
+        return False

{qdrant_haystack-3.1.0.dist-info → qdrant_haystack-3.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: qdrant-haystack
-Version: 3.1.0
+Version: 3.2.1
 Summary: An integration of Qdrant ANN vector database backend with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md
@@ -10,15 +10,14 @@ License-Expression: Apache-2.0
 License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.7
-Requires-Dist: haystack-ai>=2.0.0b6
+Requires-Python: >=3.8
+Requires-Dist: haystack-ai
 Requires-Dist: qdrant-client
 Description-Content-Type: text/markdown
 

{qdrant_haystack-3.1.0.dist-info → qdrant_haystack-3.2.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 haystack_integrations/components/retrievers/qdrant/__init__.py,sha256=uX6yULYL7RExXbKb9wD7TTz7fKupJf97lUi1YAHgJcY,200
 haystack_integrations/components/retrievers/qdrant/retriever.py,sha256=Pu8qwKive3uvO3z4FnidgwXC0IUgxNRNuAeJrdE70rU,4423
 haystack_integrations/document_stores/qdrant/__init__.py,sha256=PuGxUj29V00f6UiCpTHRkzGufL8bJUML2iNwJnX2KwM,195
-haystack_integrations/document_stores/qdrant/converters.py,sha256=chTu0-nWLy6Hnzn76mXfEagOOWhXIUqdoWTBvcHK7PY,1797
-haystack_integrations/document_stores/qdrant/document_store.py,sha256=c4lobSgv0CRRPxZM-Q5AKwpiaSjf-D7oiqQBaQoL1sc,17529
-haystack_integrations/document_stores/qdrant/filters.py,sha256=zpmFQHhIP0IFClMADr84USA7p5Is237evutVQLFK3s4,8570
-qdrant_haystack-3.1.0.dist-info/METADATA,sha256=q8mahJGC6yCwguQJo9FzSeVYSVoK1LHn9wTgeKuhLEY,1851
-qdrant_haystack-3.1.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-qdrant_haystack-3.1.0.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
-qdrant_haystack-3.1.0.dist-info/RECORD,,
+haystack_integrations/document_stores/qdrant/converters.py,sha256=q_S3ATfX2KF4z9c2Z6t3mqz8GnULXAuKXIiGSSKGBJ0,2442
+haystack_integrations/document_stores/qdrant/document_store.py,sha256=-ahV-OuBjGn-299DbLDjq3a0CN_UdqK_-a7NIV2ngYw,18939
+haystack_integrations/document_stores/qdrant/filters.py,sha256=26sgZOdiXEJesk2NdB6NbQoAxEInpLNxO5pLkLnELKE,9170
+qdrant_haystack-3.2.1.dist-info/METADATA,sha256=f6i-52wn91Spzqrb80ZdXV4EhhFnOT16sUObZPLkNVE,1792
+qdrant_haystack-3.2.1.dist-info/WHEEL,sha256=as-1oFTWSeWBgyzh0O_qF439xqBe6AbBgt4MfYe5zwY,87
+qdrant_haystack-3.2.1.dist-info/licenses/LICENSE.txt,sha256=B05uMshqTA74s-0ltyHKI6yoPfJ3zYgQbvcXfDVGFf8,10280
+qdrant_haystack-3.2.1.dist-info/RECORD,,

{qdrant_haystack-3.1.0.dist-info → qdrant_haystack-3.2.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.21.1
+Generator: hatchling 1.22.5
 Root-Is-Purelib: true
 Tag: py3-none-any