qdrant-haystack 4.1.0__tar.gz → 4.1.2__tar.gz

{qdrant_haystack-4.1.0 → qdrant_haystack-4.1.2}/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## [integrations/qdrant-v4.1.1] - 2024-07-10
+
+### 🚀 Features
+
+- Add filter_policy to qdrant integration (#819)
+
+### 🐛 Bug Fixes
+
+- Errors in convert_filters_to_qdrant (#870)
+
+## [integrations/qdrant-v4.1.0] - 2024-07-03
+
+### 🚀 Features
+
+- Add `score_threshold` to Qdrant Retrievers (#860)
+- Qdrant - add support for BM42 (#864)
+
 ## [integrations/qdrant-v4.0.0] - 2024-07-02
 
 ### 🚜 Refactor

{qdrant_haystack-4.1.0 → qdrant_haystack-4.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qdrant-haystack
-Version: 4.1.0
+Version: 4.1.2
 Summary: An integration of Qdrant ANN vector database backend with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/qdrant/README.md
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.8
-Requires-Dist: haystack-ai>=2.0.1
+Requires-Dist: haystack-ai
 Requires-Dist: qdrant-client>=1.10.0
 Description-Content-Type: text/markdown
 

{qdrant_haystack-4.1.0 → qdrant_haystack-4.1.2}/pyproject.toml

@@ -25,7 +25,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai>=2.0.1", "qdrant-client>=1.10.0"]
+dependencies = ["haystack-ai", "qdrant-client>=1.10.0"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations"

{qdrant_haystack-4.1.0 → qdrant_haystack-4.1.2}/src/haystack_integrations/components/retrievers/qdrant/retriever.py

@@ -2,6 +2,8 @@ from typing import Any, Dict, List, Optional, Union
 
 from haystack import Document, component, default_from_dict, default_to_dict
 from haystack.dataclasses.sparse_embedding import SparseEmbedding
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 from qdrant_client.http import models
 
@@ -39,6 +41,7 @@ class QdrantEmbeddingRetriever:
         top_k: int = 10,
         scale_score: bool = False,
         return_embedding: bool = False,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
         score_threshold: Optional[float] = None,
     ):
         """
@@ -49,6 +52,7 @@ class QdrantEmbeddingRetriever:
         :param top_k: The maximum number of documents to retrieve.
         :param scale_score: Whether to scale the scores of the retrieved documents or not.
         :param return_embedding: Whether to return the embedding of the retrieved Documents.
+        :param filter_policy: Policy to determine how filters are applied.
         :param score_threshold: A minimal score threshold for the result.
             Score of the returned result might be higher or smaller than the threshold
              depending on the `similarity` function specified in the Document Store.
@@ -66,6 +70,9 @@ class QdrantEmbeddingRetriever:
         self._top_k = top_k
         self._scale_score = scale_score
         self._return_embedding = return_embedding
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
         self._score_threshold = score_threshold
 
     def to_dict(self) -> Dict[str, Any]:
@@ -80,6 +87,7 @@ class QdrantEmbeddingRetriever:
             document_store=self._document_store,
             filters=self._filters,
             top_k=self._top_k,
+            filter_policy=self._filter_policy.value,
             scale_score=self._scale_score,
             return_embedding=self._return_embedding,
             score_threshold=self._score_threshold,
@@ -100,6 +108,10 @@ class QdrantEmbeddingRetriever:
         """
         document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
         data["init_parameters"]["document_store"] = document_store
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -125,9 +137,11 @@ class QdrantEmbeddingRetriever:
             The retrieved documents.
 
         """
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
+
         docs = self._document_store._query_by_embedding(
             query_embedding=query_embedding,
-            filters=filters or self._filters,
+            filters=filters,
             top_k=top_k or self._top_k,
             scale_score=scale_score or self._scale_score,
             return_embedding=return_embedding or self._return_embedding,
@@ -171,6 +185,7 @@ class QdrantSparseEmbeddingRetriever:
         top_k: int = 10,
         scale_score: bool = False,
         return_embedding: bool = False,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
         score_threshold: Optional[float] = None,
     ):
         """
@@ -181,6 +196,7 @@ class QdrantSparseEmbeddingRetriever:
         :param top_k: The maximum number of documents to retrieve.
         :param scale_score: Whether to scale the scores of the retrieved documents or not.
         :param return_embedding: Whether to return the sparse embedding of the retrieved Documents.
+        :param filter_policy: Policy to determine how filters are applied. Defaults to "replace".
         :param score_threshold: A minimal score threshold for the result.
             Score of the returned result might be higher or smaller than the threshold
              depending on the Distance function used.
@@ -198,6 +214,9 @@ class QdrantSparseEmbeddingRetriever:
         self._top_k = top_k
         self._scale_score = scale_score
         self._return_embedding = return_embedding
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
         self._score_threshold = score_threshold
 
     def to_dict(self) -> Dict[str, Any]:
@@ -213,6 +232,7 @@ class QdrantSparseEmbeddingRetriever:
             filters=self._filters,
             top_k=self._top_k,
             scale_score=self._scale_score,
+            filter_policy=self._filter_policy.value,
             return_embedding=self._return_embedding,
             score_threshold=self._score_threshold,
         )
@@ -232,6 +252,10 @@ class QdrantSparseEmbeddingRetriever:
         """
         document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
         data["init_parameters"]["document_store"] = document_store
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -248,7 +272,9 @@ class QdrantSparseEmbeddingRetriever:
         Run the Sparse Embedding Retriever on the given input data.
 
         :param query_sparse_embedding: Sparse Embedding of the query.
-        :param filters: A dictionary with filters to narrow down the search space.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k: The maximum number of documents to return.
         :param scale_score: Whether to scale the scores of the retrieved documents or not.
         :param return_embedding: Whether to return the embedding of the retrieved Documents.
@@ -260,9 +286,11 @@ class QdrantSparseEmbeddingRetriever:
             The retrieved documents.
 
         """
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
+
         docs = self._document_store._query_by_sparse(
             query_sparse_embedding=query_sparse_embedding,
-            filters=filters or self._filters,
+            filters=filters,
             top_k=top_k or self._top_k,
             scale_score=scale_score or self._scale_score,
             return_embedding=return_embedding or self._return_embedding,
@@ -311,6 +339,7 @@ class QdrantHybridRetriever:
         filters: Optional[Union[Dict[str, Any], models.Filter]] = None,
         top_k: int = 10,
         return_embedding: bool = False,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
         score_threshold: Optional[float] = None,
     ):
         """
@@ -320,6 +349,7 @@ class QdrantHybridRetriever:
         :param filters: A dictionary with filters to narrow down the search space.
         :param top_k: The maximum number of documents to retrieve.
         :param return_embedding: Whether to return the embeddings of the retrieved Documents.
+        :param filter_policy: Policy to determine how filters are applied.
         :param score_threshold: A minimal score threshold for the result.
             Score of the returned result might be higher or smaller than the threshold
              depending on the Distance function used.
@@ -336,6 +366,9 @@ class QdrantHybridRetriever:
         self._filters = filters
         self._top_k = top_k
         self._return_embedding = return_embedding
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
         self._score_threshold = score_threshold
 
     def to_dict(self) -> Dict[str, Any]:
@@ -350,6 +383,7 @@ class QdrantHybridRetriever:
             document_store=self._document_store.to_dict(),
             filters=self._filters,
             top_k=self._top_k,
+            filter_policy=self._filter_policy.value,
             return_embedding=self._return_embedding,
             score_threshold=self._score_threshold,
         )
@@ -366,6 +400,10 @@ class QdrantHybridRetriever:
         """
         document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
         data["init_parameters"]["document_store"] = document_store
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -383,7 +421,9 @@ class QdrantHybridRetriever:
 
         :param query_embedding: Dense embedding of the query.
         :param query_sparse_embedding: Sparse embedding of the query.
-        :param filters: A dictionary with filters to narrow down the search space.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k: The maximum number of documents to return.
         :param return_embedding: Whether to return the embedding of the retrieved Documents.
         :param score_threshold: A minimal score threshold for the result.
@@ -394,10 +434,12 @@ class QdrantHybridRetriever:
             The retrieved documents.
 
         """
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
+
         docs = self._document_store._query_hybrid(
             query_embedding=query_embedding,
             query_sparse_embedding=query_sparse_embedding,
-            filters=filters or self._filters,
+            filters=filters,
             top_k=top_k or self._top_k,
             return_embedding=return_embedding or self._return_embedding,
             score_threshold=score_threshold or self._score_threshold,

qdrant_haystack-4.1.2/src/haystack_integrations/document_stores/qdrant/filters.py

@@ -0,0 +1,316 @@
+from datetime import datetime
+from typing import List, Optional, Union
+
+from haystack.utils.filters import COMPARISON_OPERATORS, LOGICAL_OPERATORS, FilterError
+from qdrant_client.http import models
+
+COMPARISON_OPERATORS = COMPARISON_OPERATORS.keys()
+LOGICAL_OPERATORS = LOGICAL_OPERATORS.keys()
+
+
+def convert_filters_to_qdrant(
+    filter_term: Optional[Union[List[dict], dict, models.Filter]] = None, is_parent_call: bool = True
+) -> Optional[Union[models.Filter, List[models.Filter], List[models.Condition]]]:
+    """Converts Haystack filters to the format used by Qdrant.
+
+    :param filter_term: the haystack filter to be converted to qdrant.
+    :param is_parent_call: indicates if this is the top-level call to the function. If True, the function returns
+      a single models.Filter object; if False, it may return a list of filters or conditions for further processing.
+
+    :returns: a single Qdrant Filter in the parent call or a list of such Filters in recursive calls.
+
+    :raises FilterError: If the invalid filter criteria is provided or if an unknown operator is encountered.
+
+    """
+
+    if isinstance(filter_term, models.Filter):
+        return filter_term
+    if not filter_term:
+        return None
+
+    must_clauses: List[models.Filter] = []
+    should_clauses: List[models.Filter] = []
+    must_not_clauses: List[models.Filter] = []
+    # Indicates if there are multiple same LOGICAL OPERATORS on each level
+    # and prevents them from being combined
+    same_operator_flag = False
+    conditions, qdrant_filter, current_level_operators = (
+        [],
+        [],
+        [],
+    )
+
+    if isinstance(filter_term, dict):
+        filter_term = [filter_term]
+
+    # ======== IDENTIFY FILTER ITEMS ON EACH LEVEL ========
+
+    for item in filter_term:
+        operator = item.get("operator")
+
+        # Check for repeated similar operators on each level
+        same_operator_flag = operator in current_level_operators and operator in LOGICAL_OPERATORS
+        if not same_operator_flag:
+            current_level_operators.append(operator)
+
+        if operator is None:
+            msg = "Operator not found in filters"
+            raise FilterError(msg)
+
+        if operator in LOGICAL_OPERATORS and "conditions" not in item:
+            msg = f"'conditions' not found for '{operator}'"
+            raise FilterError(msg)
+
+        if operator in LOGICAL_OPERATORS:
+            # Recursively process nested conditions
+            current_filter = convert_filters_to_qdrant(item.get("conditions", []), is_parent_call=False) or []
+
+            # When same_operator_flag is set to True,
+            # ensure each clause is appended as an independent list to avoid merging distinct clauses.
+            if operator == "AND":
+                must_clauses = [must_clauses, current_filter] if same_operator_flag else must_clauses + current_filter
+            elif operator == "OR":
+                should_clauses = (
+                    [should_clauses, current_filter] if same_operator_flag else should_clauses + current_filter
+                )
+            elif operator == "NOT":
+                must_not_clauses = (
+                    [must_not_clauses, current_filter] if same_operator_flag else must_not_clauses + current_filter
+                )
+
+        elif operator in COMPARISON_OPERATORS:
+            field = item.get("field")
+            value = item.get("value")
+            if field is None or value is None:
+                msg = f"'field' or 'value' not found for '{operator}'"
+                raise FilterError(msg)
+
+            parsed_conditions = _parse_comparison_operation(comparison_operation=operator, key=field, value=value)
+
+            # check if the parsed_conditions are models.Filter or models.Condition
+            for condition in parsed_conditions:
+                if isinstance(condition, models.Filter):
+                    qdrant_filter.append(condition)
+                else:
+                    conditions.append(condition)
+
+        else:
+            msg = f"Unknown operator {operator} used in filters"
+            raise FilterError(msg)
+
+    # ======== PROCESS FILTER ITEMS ON EACH LEVEL ========
+
+    # If same logical operators have separate clauses, create separate filters
+    if same_operator_flag:
+        qdrant_filter = build_filters_for_repeated_operators(
+            must_clauses, should_clauses, must_not_clauses, qdrant_filter
+        )
+
+    # else append a single Filter for existing clauses
+    elif must_clauses or should_clauses or must_not_clauses:
+        qdrant_filter.append(
+            models.Filter(
+                must=must_clauses or None,
+                should=should_clauses or None,
+                must_not=must_not_clauses or None,
+            )
+        )
+
+    # In case of parent call, a single Filter is returned
+    if is_parent_call:
+        # If qdrant_filter has just a single Filter in parent call,
+        # then it might be returned instead.
+        if len(qdrant_filter) == 1 and isinstance(qdrant_filter[0], models.Filter):
+            return qdrant_filter[0]
+        else:
+            must_clauses.extend(conditions)
+            return models.Filter(
+                must=must_clauses or None,
+                should=should_clauses or None,
+                must_not=must_not_clauses or None,
+            )
+
+    # Store conditions of each level in output of the loop
+    elif conditions:
+        qdrant_filter.extend(conditions)
+
+    return qdrant_filter
+
+
+def build_filters_for_repeated_operators(
+    must_clauses,
+    should_clauses,
+    must_not_clauses,
+    qdrant_filter,
+) -> List[models.Filter]:
+    """
+    Flattens the nested lists of clauses by creating separate Filters for each clause of a logical operator.
+
+    :param must_clauses: a nested list of must clauses or an empty list.
+    :param should_clauses: a nested list of should clauses or an empty list.
+    :param must_not_clauses: a nested list of must_not clauses or an empty list.
+    :param qdrant_filter: a list where the generated Filter objects will be appended.
+      This list will be modified in-place.
+
+
+    :returns: the modified `qdrant_filter` list with appended generated Filter objects.
+    """
+
+    if any(isinstance(i, list) for i in must_clauses):
+        for i in must_clauses:
+            qdrant_filter.append(
+                models.Filter(
+                    must=i or None,
+                    should=should_clauses or None,
+                    must_not=must_not_clauses or None,
+                )
+            )
+    if any(isinstance(i, list) for i in should_clauses):
+        for i in should_clauses:
+            qdrant_filter.append(
+                models.Filter(
+                    must=must_clauses or None,
+                    should=i or None,
+                    must_not=must_not_clauses or None,
+                )
+            )
+    if any(isinstance(i, list) for i in must_not_clauses):
+        for i in must_clauses:
+            qdrant_filter.append(
+                models.Filter(
+                    must=must_clauses or None,
+                    should=should_clauses or None,
+                    must_not=i or None,
+                )
+            )
+
+    return qdrant_filter
+
+
+def _parse_comparison_operation(
+    comparison_operation: str, key: str, value: Union[dict, List, str, float]
+) -> List[models.Condition]:
+    conditions: List[models.Condition] = []
+
+    condition_builder_mapping = {
+        "==": _build_eq_condition,
+        "in": _build_in_condition,
+        "!=": _build_ne_condition,
+        "not in": _build_nin_condition,
+        ">": _build_gt_condition,
+        ">=": _build_gte_condition,
+        "<": _build_lt_condition,
+        "<=": _build_lte_condition,
+    }
+
+    condition_builder = condition_builder_mapping.get(comparison_operation)
+
+    if condition_builder is None:
+        msg = f"Unknown operator {comparison_operation} used in filters"
+        raise ValueError(msg)
+
+    conditions.append(condition_builder(key, value))
+
+    return conditions
+
+
+def _build_eq_condition(key: str, value: models.ValueVariants) -> models.Condition:
+    if isinstance(value, str) and " " in value:
+        return models.FieldCondition(key=key, match=models.MatchText(text=value))
+    return models.FieldCondition(key=key, match=models.MatchValue(value=value))
+
+
+def _build_in_condition(key: str, value: List[models.ValueVariants]) -> models.Condition:
+    if not isinstance(value, list):
+        msg = f"Value {value} is not a list"
+        raise FilterError(msg)
+    return models.Filter(
+        should=[
+            (
+                models.FieldCondition(key=key, match=models.MatchText(text=item))
+                if isinstance(item, str) and " " not in item
+                else models.FieldCondition(key=key, match=models.MatchValue(value=item))
+            )
+            for item in value
+        ]
+    )
+
+
+def _build_ne_condition(key: str, value: models.ValueVariants) -> models.Condition:
+    return models.Filter(
+        must_not=[
+            (
+                models.FieldCondition(key=key, match=models.MatchText(text=value))
+                if isinstance(value, str) and " " not in value
+                else models.FieldCondition(key=key, match=models.MatchValue(value=value))
+            )
+        ]
+    )
+
+
+def _build_nin_condition(key: str, value: List[models.ValueVariants]) -> models.Condition:
+    if not isinstance(value, list):
+        msg = f"Value {value} is not a list"
+        raise FilterError(msg)
+    return models.Filter(
+        must_not=[
+            (
+                models.FieldCondition(key=key, match=models.MatchText(text=item))
+                if isinstance(item, str) and " " in item
+                else models.FieldCondition(key=key, match=models.MatchValue(value=item))
+            )
+            for item in value
+        ]
+    )
+
+
+def _build_lt_condition(key: str, value: Union[str, float, int]) -> models.Condition:
+    if isinstance(value, str) and is_datetime_string(value):
+        return models.FieldCondition(key=key, range=models.DatetimeRange(lt=value))
+
+    if isinstance(value, (int, float)):
+        return models.FieldCondition(key=key, range=models.Range(lt=value))
+
+    msg = f"Value {value} is not an int or float or datetime string"
+    raise FilterError(msg)
+
+
+def _build_lte_condition(key: str, value: Union[str, float, int]) -> models.Condition:
+    if isinstance(value, str) and is_datetime_string(value):
+        return models.FieldCondition(key=key, range=models.DatetimeRange(lte=value))
+
+    if isinstance(value, (int, float)):
+        return models.FieldCondition(key=key, range=models.Range(lte=value))
+
+    msg = f"Value {value} is not an int or float or datetime string"
+    raise FilterError(msg)
+
+
+def _build_gt_condition(key: str, value: Union[str, float, int]) -> models.Condition:
+    if isinstance(value, str) and is_datetime_string(value):
+        return models.FieldCondition(key=key, range=models.DatetimeRange(gt=value))
+
+    if isinstance(value, (int, float)):
+        return models.FieldCondition(key=key, range=models.Range(gt=value))
+
+    msg = f"Value {value} is not an int or float or datetime string"
+    raise FilterError(msg)
+
+
+def _build_gte_condition(key: str, value: Union[str, float, int]) -> models.Condition:
+    if isinstance(value, str) and is_datetime_string(value):
+        return models.FieldCondition(key=key, range=models.DatetimeRange(gte=value))
+
+    if isinstance(value, (int, float)):
+        return models.FieldCondition(key=key, range=models.Range(gte=value))
+
+    msg = f"Value {value} is not an int or float or datetime string"
+    raise FilterError(msg)
+
+
+def is_datetime_string(value: str) -> bool:
+    try:
+        datetime.fromisoformat(value)
+        return True
+    except ValueError:
+        return False

{qdrant_haystack-4.1.0 → qdrant_haystack-4.1.2}/tests/test_filters.py

@@ -61,6 +61,112 @@ class TestQdrantStoreBaseTests(FilterDocumentsTest):
             [d for d in filterable_docs if (d.meta.get("number") != 100 and d.meta.get("name") != "name_0")],
         )
 
+    def test_filter_criteria(self, document_store):
+        documents = [
+            Document(
+                content="This is test document 1.",
+                meta={"file_name": "file1", "classification": {"details": {"category1": 0.9, "category2": 0.3}}},
+            ),
+            Document(
+                content="This is test document 2.",
+                meta={"file_name": "file2", "classification": {"details": {"category1": 0.1, "category2": 0.7}}},
+            ),
+            Document(
+                content="This is test document 3.",
+                meta={"file_name": "file3", "classification": {"details": {"category1": 0.7, "category2": 0.9}}},
+            ),
+        ]
+
+        document_store.write_documents(documents)
+        filter_criteria = {
+            "operator": "AND",
+            "conditions": [
+                {"field": "meta.file_name", "operator": "in", "value": ["file1", "file2"]},
+                {
+                    "operator": "OR",
+                    "conditions": [
+                        {"field": "meta.classification.details.category1", "operator": ">=", "value": 0.85},
+                        {"field": "meta.classification.details.category2", "operator": ">=", "value": 0.85},
+                    ],
+                },
+            ],
+        }
+        result = document_store.filter_documents(filter_criteria)
+        self.assert_documents_are_equal(
+            result,
+            [
+                d
+                for d in documents
+                if (d.meta.get("file_name") in ["file1", "file2"])
+                and (
+                    (d.meta.get("classification").get("details").get("category1") >= 0.85)
+                    or (d.meta.get("classification").get("details").get("category2") >= 0.85)
+                )
+            ],
+        )
+
+    def test_complex_filter_criteria(self, document_store):
+        documents = [
+            Document(
+                content="This is test document 1.",
+                meta={
+                    "file_name": "file1",
+                    "classification": {"details": {"category1": 0.45, "category2": 0.5, "category3": 0.2}},
+                },
+            ),
+            Document(
+                content="This is test document 2.",
+                meta={
+                    "file_name": "file2",
+                    "classification": {"details": {"category1": 0.95, "category2": 0.85, "category3": 0.4}},
+                },
+            ),
+            Document(
+                content="This is test document 3.",
+                meta={
+                    "file_name": "file3",
+                    "classification": {"details": {"category1": 0.85, "category2": 0.7, "category3": 0.95}},
+                },
+            ),
+        ]
+
+        document_store.write_documents(documents)
+        filter_criteria = {
+            "operator": "AND",
+            "conditions": [
+                {"field": "meta.file_name", "operator": "in", "value": ["file1", "file2", "file3"]},
+                {
+                    "operator": "AND",
+                    "conditions": [
+                        {"field": "meta.classification.details.category1", "operator": ">=", "value": 0.85},
+                        {
+                            "operator": "OR",
+                            "conditions": [
+                                {"field": "meta.classification.details.category2", "operator": ">=", "value": 0.8},
+                                {"field": "meta.classification.details.category3", "operator": ">=", "value": 0.9},
+                            ],
+                        },
+                    ],
+                },
+            ],
+        }
+        result = document_store.filter_documents(filter_criteria)
+        self.assert_documents_are_equal(
+            result,
+            [
+                d
+                for d in documents
+                if (d.meta.get("file_name") in ["file1", "file2", "file3"])
+                and (
+                    (d.meta.get("classification").get("details").get("category1") >= 0.85)
+                    and (
+                        (d.meta.get("classification").get("details").get("category2") >= 0.8)
+                        or (d.meta.get("classification").get("details").get("category3") >= 0.9)
+                    )
+                )
+            ],
+        )
+
     # ======== OVERRIDES FOR NONE VALUED FILTERS ========
 
     def test_comparison_equal_with_none(self, document_store, filterable_docs):

{qdrant_haystack-4.1.0 → qdrant_haystack-4.1.2}/tests/test_retriever.py

@@ -1,7 +1,9 @@
 from typing import List
 from unittest.mock import Mock
 
+import pytest
 from haystack.dataclasses import Document, SparseEmbedding
+from haystack.document_stores.types import FilterPolicy
 from haystack.testing.document_store import (
     FilterableDocsFixtureMixin,
     _random_embeddings,
@@ -21,9 +23,16 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
         assert retriever._document_store == document_store
         assert retriever._filters is None
         assert retriever._top_k == 10
+        assert retriever._filter_policy == FilterPolicy.REPLACE
         assert retriever._return_embedding is False
         assert retriever._score_threshold is None
 
+        retriever = QdrantEmbeddingRetriever(document_store=document_store, filter_policy="replace")
+        assert retriever._filter_policy == FilterPolicy.REPLACE
+
+        with pytest.raises(ValueError):
+            QdrantEmbeddingRetriever(document_store=document_store, filter_policy="invalid")
+
     def test_to_dict(self):
         document_store = QdrantDocumentStore(location=":memory:", index="test", use_sparse_embeddings=False)
         retriever = QdrantEmbeddingRetriever(document_store=document_store)
@@ -73,6 +82,7 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
                 },
                 "filters": None,
                 "top_k": 10,
+                "filter_policy": "replace",
                 "scale_score": False,
                 "return_embedding": False,
                 "score_threshold": None,
@@ -89,6 +99,7 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
                 },
                 "filters": None,
                 "top_k": 5,
+                "filter_policy": "replace",
                 "scale_score": False,
                 "return_embedding": True,
                 "score_threshold": None,
@@ -99,6 +110,7 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
         assert retriever._document_store.index == "test"
         assert retriever._filters is None
         assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE
         assert retriever._scale_score is False
         assert retriever._return_embedding is True
         assert retriever._score_threshold is None
@@ -119,6 +131,31 @@ class TestQdrantRetriever(FilterableDocsFixtureMixin):
         for document in results:
             assert document.embedding is None
 
+    def test_run_filters(self, filterable_docs: List[Document]):
+        document_store = QdrantDocumentStore(location=":memory:", index="Boi", use_sparse_embeddings=False)
+
+        document_store.write_documents(filterable_docs)
+
+        retriever = QdrantEmbeddingRetriever(
+            document_store=document_store,
+            filters={"field": "meta.name", "operator": "==", "value": "name_0"},
+            filter_policy=FilterPolicy.MERGE,
+        )
+
+        results: List[Document] = retriever.run(query_embedding=_random_embeddings(768))["documents"]
+        assert len(results) == 3
+
+        results = retriever.run(
+            query_embedding=_random_embeddings(768),
+            top_k=5,
+            filters={"field": "meta.chapter", "operator": "==", "value": "abstract"},
+            return_embedding=False,
+        )["documents"]
+        assert len(results) == 3
+
+        for document in results:
+            assert document.embedding is None
+
     def test_run_with_score_threshold(self):
         document_store = QdrantDocumentStore(
             embedding_dim=4, location=":memory:", similarity="cosine", index="Boi", use_sparse_embeddings=False
@@ -167,9 +204,16 @@ class TestQdrantSparseEmbeddingRetriever(FilterableDocsFixtureMixin):
         assert retriever._document_store == document_store
         assert retriever._filters is None
         assert retriever._top_k == 10
+        assert retriever._filter_policy == FilterPolicy.REPLACE
         assert retriever._return_embedding is False
         assert retriever._score_threshold is None
 
+        retriever = QdrantSparseEmbeddingRetriever(document_store=document_store, filter_policy="replace")
+        assert retriever._filter_policy == FilterPolicy.REPLACE
+
+        with pytest.raises(ValueError):
+            QdrantSparseEmbeddingRetriever(document_store=document_store, filter_policy="invalid")
+
     def test_to_dict(self):
         document_store = QdrantDocumentStore(location=":memory:", index="test")
         retriever = QdrantSparseEmbeddingRetriever(document_store=document_store)
@@ -221,11 +265,38 @@ class TestQdrantSparseEmbeddingRetriever(FilterableDocsFixtureMixin):
                 "top_k": 10,
                 "scale_score": False,
                 "return_embedding": False,
+                "filter_policy": "replace",
                 "score_threshold": None,
             },
         }
 
     def test_from_dict(self):
+        data = {
+            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseEmbeddingRetriever",
+            "init_parameters": {
+                "document_store": {
+                    "init_parameters": {"location": ":memory:", "index": "test"},
+                    "type": "haystack_integrations.document_stores.qdrant.document_store.QdrantDocumentStore",
+                },
+                "filters": None,
+                "top_k": 5,
+                "scale_score": False,
+                "return_embedding": True,
+                "filter_policy": "replace",
+                "score_threshold": None,
+            },
+        }
+        retriever = QdrantSparseEmbeddingRetriever.from_dict(data)
+        assert isinstance(retriever._document_store, QdrantDocumentStore)
+        assert retriever._document_store.index == "test"
+        assert retriever._filters is None
+        assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE
+        assert retriever._scale_score is False
+        assert retriever._return_embedding is True
+        assert retriever._score_threshold is None
+
+    def test_from_dict_no_filter_policy(self):
         data = {
             "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantSparseEmbeddingRetriever",
             "init_parameters": {
@@ -245,6 +316,7 @@ class TestQdrantSparseEmbeddingRetriever(FilterableDocsFixtureMixin):
         assert retriever._document_store.index == "test"
         assert retriever._filters is None
         assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
         assert retriever._scale_score is False
         assert retriever._return_embedding is True
         assert retriever._score_threshold is None
@@ -278,9 +350,16 @@ class TestQdrantHybridRetriever:
         assert retriever._document_store == document_store
         assert retriever._filters is None
         assert retriever._top_k == 10
+        assert retriever._filter_policy == FilterPolicy.REPLACE
         assert retriever._return_embedding is False
         assert retriever._score_threshold is None
 
+        retriever = QdrantHybridRetriever(document_store=document_store, filter_policy="replace")
+        assert retriever._filter_policy == FilterPolicy.REPLACE
+
+        with pytest.raises(ValueError):
+            QdrantHybridRetriever(document_store=document_store, filter_policy="invalid")
+
     def test_to_dict(self):
         document_store = QdrantDocumentStore(location=":memory:", index="test")
         retriever = QdrantHybridRetriever(document_store=document_store, top_k=5, return_embedding=True)
@@ -330,12 +409,37 @@ class TestQdrantHybridRetriever:
                 },
                 "filters": None,
                 "top_k": 5,
+                "filter_policy": "replace",
                 "return_embedding": True,
                 "score_threshold": None,
             },
         }
 
     def test_from_dict(self):
+        data = {
+            "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantHybridRetriever",
+            "init_parameters": {
+                "document_store": {
+                    "init_parameters": {"location": ":memory:", "index": "test"},
+                    "type": "haystack_integrations.document_stores.qdrant.document_store.QdrantDocumentStore",
+                },
+                "filters": None,
+                "top_k": 5,
+                "filter_policy": "replace",
+                "return_embedding": True,
+                "score_threshold": None,
+            },
+        }
+        retriever = QdrantHybridRetriever.from_dict(data)
+        assert isinstance(retriever._document_store, QdrantDocumentStore)
+        assert retriever._document_store.index == "test"
+        assert retriever._filters is None
+        assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE
+        assert retriever._return_embedding
+        assert retriever._score_threshold is None
+
+    def test_from_dict_no_filter_policy(self):
         data = {
             "type": "haystack_integrations.components.retrievers.qdrant.retriever.QdrantHybridRetriever",
             "init_parameters": {
@@ -354,6 +458,7 @@ class TestQdrantHybridRetriever:
         assert retriever._document_store.index == "test"
         assert retriever._filters is None
         assert retriever._top_k == 5
+        assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
         assert retriever._return_embedding
         assert retriever._score_threshold is None
 

qdrant_haystack-4.1.0/src/haystack_integrations/document_stores/qdrant/filters.py

@@ -1,238 +0,0 @@
-from datetime import datetime
-from typing import List, Optional, Union
-
-from haystack.utils.filters import COMPARISON_OPERATORS, LOGICAL_OPERATORS, FilterError
-from qdrant_client.http import models
-
-from .converters import convert_id
-
-COMPARISON_OPERATORS = COMPARISON_OPERATORS.keys()
-LOGICAL_OPERATORS = LOGICAL_OPERATORS.keys()
-
-
-def convert_filters_to_qdrant(
-    filter_term: Optional[Union[List[dict], dict, models.Filter]] = None,
-) -> Optional[models.Filter]:
-    """Converts Haystack filters to the format used by Qdrant."""
-    if isinstance(filter_term, models.Filter):
-        return filter_term
-    if not filter_term:
-        return None
-
-    must_clauses, should_clauses, must_not_clauses = [], [], []
-
-    if isinstance(filter_term, dict):
-        filter_term = [filter_term]
-
-    for item in filter_term:
-        operator = item.get("operator")
-        if operator is None:
-            msg = "Operator not found in filters"
-            raise FilterError(msg)
-
-        if operator in LOGICAL_OPERATORS and "conditions" not in item:
-            msg = f"'conditions' not found for '{operator}'"
-            raise FilterError(msg)
-
-        if operator == "AND":
-            must_clauses.append(convert_filters_to_qdrant(item.get("conditions", [])))
-        elif operator == "OR":
-            should_clauses.append(convert_filters_to_qdrant(item.get("conditions", [])))
-        elif operator == "NOT":
-            must_not_clauses.append(convert_filters_to_qdrant(item.get("conditions", [])))
-        elif operator in COMPARISON_OPERATORS:
-            field = item.get("field")
-            value = item.get("value")
-            if field is None or value is None:
-                msg = f"'field' or 'value' not found for '{operator}'"
-                raise FilterError(msg)
-
-            must_clauses.extend(_parse_comparison_operation(comparison_operation=operator, key=field, value=value))
-        else:
-            msg = f"Unknown operator {operator} used in filters"
-            raise FilterError(msg)
-
-    payload_filter = models.Filter(
-        must=must_clauses or None,
-        should=should_clauses or None,
-        must_not=must_not_clauses or None,
-    )
-
-    filter_result = _squeeze_filter(payload_filter)
-
-    return filter_result
-
-
-def _parse_comparison_operation(
-    comparison_operation: str, key: str, value: Union[dict, List, str, float]
-) -> List[models.Condition]:
-    conditions: List[models.Condition] = []
-
-    condition_builder_mapping = {
-        "==": _build_eq_condition,
-        "in": _build_in_condition,
-        "!=": _build_ne_condition,
-        "not in": _build_nin_condition,
-        ">": _build_gt_condition,
-        ">=": _build_gte_condition,
-        "<": _build_lt_condition,
-        "<=": _build_lte_condition,
-    }
-
-    condition_builder = condition_builder_mapping.get(comparison_operation)
-
-    if condition_builder is None:
-        msg = f"Unknown operator {comparison_operation} used in filters"
-        raise ValueError(msg)
-
-    conditions.append(condition_builder(key, value))
-
-    return conditions
-
-
-def _build_eq_condition(key: str, value: models.ValueVariants) -> models.Condition:
-    if isinstance(value, str) and " " in value:
-        models.FieldCondition(key=key, match=models.MatchText(text=value))
-    return models.FieldCondition(key=key, match=models.MatchValue(value=value))
-
-
-def _build_in_condition(key: str, value: List[models.ValueVariants]) -> models.Condition:
-    if not isinstance(value, list):
-        msg = f"Value {value} is not a list"
-        raise FilterError(msg)
-    return models.Filter(
-        should=[
-            (
-                models.FieldCondition(key=key, match=models.MatchText(text=item))
-                if isinstance(item, str) and " " not in item
-                else models.FieldCondition(key=key, match=models.MatchValue(value=item))
-            )
-            for item in value
-        ]
-    )
-
-
-def _build_ne_condition(key: str, value: models.ValueVariants) -> models.Condition:
-    return models.Filter(
-        must_not=[
-            (
-                models.FieldCondition(key=key, match=models.MatchText(text=value))
-                if isinstance(value, str) and " " not in value
-                else models.FieldCondition(key=key, match=models.MatchValue(value=value))
-            )
-        ]
-    )
-
-
-def _build_nin_condition(key: str, value: List[models.ValueVariants]) -> models.Condition:
-    if not isinstance(value, list):
-        msg = f"Value {value} is not a list"
-        raise FilterError(msg)
-    return models.Filter(
-        must_not=[
-            (
-                models.FieldCondition(key=key, match=models.MatchText(text=item))
-                if isinstance(item, str) and " " in item
-                else models.FieldCondition(key=key, match=models.MatchValue(value=item))
-            )
-            for item in value
-        ]
-    )
-
-
-def _build_lt_condition(key: str, value: Union[str, float, int]) -> models.Condition:
-    if isinstance(value, str) and is_datetime_string(value):
-        return models.FieldCondition(key=key, range=models.DatetimeRange(lt=value))
-
-    if isinstance(value, (int, float)):
-        return models.FieldCondition(key=key, range=models.Range(lt=value))
-
-    msg = f"Value {value} is not an int or float or datetime string"
-    raise FilterError(msg)
-
-
-def _build_lte_condition(key: str, value: Union[str, float, int]) -> models.Condition:
-    if isinstance(value, str) and is_datetime_string(value):
-        return models.FieldCondition(key=key, range=models.DatetimeRange(lte=value))
-
-    if isinstance(value, (int, float)):
-        return models.FieldCondition(key=key, range=models.Range(lte=value))
-
-    msg = f"Value {value} is not an int or float or datetime string"
-    raise FilterError(msg)
-
-
-def _build_gt_condition(key: str, value: Union[str, float, int]) -> models.Condition:
-    if isinstance(value, str) and is_datetime_string(value):
-        return models.FieldCondition(key=key, range=models.DatetimeRange(gt=value))
-
-    if isinstance(value, (int, float)):
-        return models.FieldCondition(key=key, range=models.Range(gt=value))
-
-    msg = f"Value {value} is not an int or float or datetime string"
-    raise FilterError(msg)
-
-
-def _build_gte_condition(key: str, value: Union[str, float, int]) -> models.Condition:
-    if isinstance(value, str) and is_datetime_string(value):
-        return models.FieldCondition(key=key, range=models.DatetimeRange(gte=value))
-
-    if isinstance(value, (int, float)):
-        return models.FieldCondition(key=key, range=models.Range(gte=value))
-
-    msg = f"Value {value} is not an int or float or datetime string"
-    raise FilterError(msg)
-
-
-def _build_has_id_condition(id_values: List[models.ExtendedPointId]) -> models.HasIdCondition:
-    return models.HasIdCondition(
-        has_id=[
-            # Ids are converted into their internal representation
-            convert_id(item)
-            for item in id_values
-        ]
-    )
-
-
-def _squeeze_filter(payload_filter: models.Filter) -> models.Filter:
-    """
-    Simplify given payload filter, if the nested structure might be unnested.
-    That happens if there is a single clause in that filter.
-    :param payload_filter:
-    :returns:
-    """
-    filter_parts = {
-        "must": payload_filter.must,
-        "should": payload_filter.should,
-        "must_not": payload_filter.must_not,
-    }
-
-    total_clauses = sum(len(x) for x in filter_parts.values() if x is not None)
-    if total_clauses == 0 or total_clauses > 1:
-        return payload_filter
-
-    # Payload filter has just a single clause provided (either must, should
-    # or must_not). If that single clause is also of a models.Filter type,
-    # then it might be returned instead.
-    for part_name, filter_part in filter_parts.items():
-        if not filter_part:
-            continue
-
-        subfilter = filter_part[0]
-        if not isinstance(subfilter, models.Filter):
-            # The inner statement is a simple condition like models.FieldCondition
-            # so it cannot be simplified.
-            continue
-
-        if subfilter.must:
-            return models.Filter(**{part_name: subfilter.must})
-
-    return payload_filter
-
-
-def is_datetime_string(value: str) -> bool:
-    try:
-        datetime.fromisoformat(value)
-        return True
-    except ValueError:
-        return False