weaviate-haystack 2.1.1__tar.gz → 2.1.3__tar.gz

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/.gitignore

@@ -135,3 +135,12 @@ dmypy.json
 # Docs generation artifacts
 _readme_*.md
 .idea
+
+# macOS
+.DS_Store
+
+# http cache (requests-cache)
+**/http_cache.sqlite
+
+# ruff
+.ruff_cache

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [unreleased]
+
+### 🚀 Features
+
+- Add filter_policy to weaviate integration (#824)
+
+### 🐛 Bug Fixes
+
+- Weaviate filter error (#811)
+- Fix connection to Weaviate Cloud Service (#624)
+
+### ⚙️ Miscellaneous Tasks
+
+- Retry tests to reduce flakyness (#836)
+- Update ruff invocation to include check parameter (#853)
+
 ## [integrations/weaviate-v2.1.0] - 2024-06-10
 
 ### 🚀 Features

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: weaviate-haystack
-Version: 2.1.1
+Version: 2.1.3
 Summary: An integration of Weaviate vector database with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/pyproject.toml

@@ -63,7 +63,7 @@ detached = true
 dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = ["ruff {args:.}", "black --check --diff {args:.}"]
+style = ["ruff check {args:.}", "black --check --diff {args:.}"]
 fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
 all = ["style", "typing"]
 
@@ -75,7 +75,7 @@ skip-string-normalization = true
 [tool.ruff]
 target-version = "py38"
 line-length = 120
-select = [
+lint.select = [
   "A",
   "ARG",
   "B",
@@ -102,7 +102,7 @@ select = [
   "W",
   "YTT",
 ]
-ignore = [
+lint.ignore = [
   # Allow non-abstract empty methods in abstract base classes
   "B027",
   # Allow boolean positional values in function calls, like `dict.get(... True)`
@@ -118,18 +118,18 @@ ignore = [
   "PLR0913",
   "PLR0915",
 ]
-unfixable = [
+lint.unfixable = [
   # Don't touch unused imports
   "F401",
 ]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["src"]
 
-[tool.ruff.flake8-tidy-imports]
+[tool.ruff.lint.flake8-tidy-imports]
 ban-relative-imports = "parents"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/src/haystack_integrations/components/retrievers/weaviate/__init__.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from .bm25_retriever import WeaviateBM25Retriever
 from .embedding_retriever import WeaviateEmbeddingRetriever
 

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py

@@ -1,6 +1,12 @@
-from typing import Any, Dict, List, Optional
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Dict, List, Optional, Union
 
 from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
 
@@ -26,6 +32,7 @@ class WeaviateBM25Retriever:
         document_store: WeaviateDocumentStore,
         filters: Optional[Dict[str, Any]] = None,
         top_k: int = 10,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
     ):
         """
         Create a new instance of WeaviateBM25Retriever.
@@ -36,10 +43,14 @@ class WeaviateBM25Retriever:
             Custom filters applied when running the retriever
         :param top_k:
             Maximum number of documents to return
+        :param filter_policy: Policy to determine how filters are applied.
         """
         self._document_store = document_store
         self._filters = filters or {}
         self._top_k = top_k
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -52,6 +63,7 @@ class WeaviateBM25Retriever:
             self,
             filters=self._filters,
             top_k=self._top_k,
+            filter_policy=self._filter_policy.value,
             document_store=self._document_store.to_dict(),
         )
 
@@ -68,6 +80,11 @@ class WeaviateBM25Retriever:
         data["init_parameters"]["document_store"] = WeaviateDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
+
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -77,12 +94,14 @@ class WeaviateBM25Retriever:
 
         :param query:
             The query text.
-        :param filters:
-            Filters to use when running the retriever.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k:
             The maximum number of documents to return.
         """
-        filters = filters or self._filters
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
+
         top_k = top_k or self._top_k
         documents = self._document_store._bm25_retrieval(query=query, filters=filters, top_k=top_k)
         return {"documents": documents}

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py

@@ -1,6 +1,12 @@
-from typing import Any, Dict, List, Optional
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Dict, List, Optional, Union
 
 from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
 
@@ -18,6 +24,7 @@ class WeaviateEmbeddingRetriever:
         top_k: int = 10,
         distance: Optional[float] = None,
         certainty: Optional[float] = None,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
     ):
         """
         Creates a new instance of WeaviateEmbeddingRetriever.
@@ -32,6 +39,8 @@ class WeaviateEmbeddingRetriever:
             The maximum allowed distance between Documents' embeddings.
         :param certainty:
             Normalized distance between the result item and the search vector.
+        :param filter_policy:
+            Policy to determine how filters are applied.
         :raises ValueError:
             If both `distance` and `certainty` are provided.
             See https://weaviate.io/developers/weaviate/api/graphql/search-operators#variables to learn more about
@@ -46,6 +55,9 @@ class WeaviateEmbeddingRetriever:
         self._top_k = top_k
         self._distance = distance
         self._certainty = certainty
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -60,6 +72,7 @@ class WeaviateEmbeddingRetriever:
             top_k=self._top_k,
             distance=self._distance,
             certainty=self._certainty,
+            filter_policy=self._filter_policy.value,
             document_store=self._document_store.to_dict(),
         )
 
@@ -76,6 +89,12 @@ class WeaviateEmbeddingRetriever:
         data["init_parameters"]["document_store"] = WeaviateDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
+
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
+
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -92,8 +111,9 @@ class WeaviateEmbeddingRetriever:
 
         :param query_embedding:
             Embedding of the query.
-        :param filters:
-            Filters to use when running the retriever.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k:
             The maximum number of documents to return.
         :param distance:
@@ -105,7 +125,7 @@ class WeaviateEmbeddingRetriever:
             See https://weaviate.io/developers/weaviate/api/graphql/search-operators#variables to learn more about
             `distance` and `certainty` parameters.
         """
-        filters = filters or self._filters
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
         top_k = top_k or self._top_k
 
         distance = distance or self._distance

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/src/haystack_integrations/document_stores/weaviate/_filters.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Dict
 
 from dateutil import parser

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/src/haystack_integrations/document_stores/weaviate/auth.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field, fields
 from enum import Enum

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/tests/conftest.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from pathlib import Path
 
 import pytest

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/tests/test_auth.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from haystack_integrations.document_stores.weaviate.auth import (
     AuthApiKey,
     AuthBearerToken,

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/tests/test_bm25_retriever.py

@@ -1,5 +1,11 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from unittest.mock import Mock, patch
 
+import pytest
+from haystack.document_stores.types import FilterPolicy
 from haystack_integrations.components.retrievers.weaviate import WeaviateBM25Retriever
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
@@ -10,6 +16,13 @@ def test_init_default():
     assert retriever._document_store == mock_document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
+    assert retriever._filter_policy == FilterPolicy.REPLACE
+
+    retriever = WeaviateBM25Retriever(document_store=mock_document_store, filter_policy="replace")
+    assert retriever._filter_policy == FilterPolicy.REPLACE
+
+    with pytest.raises(ValueError):
+        WeaviateBM25Retriever(document_store=mock_document_store, filter_policy="keep_all")
 
 
 @patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
@@ -21,6 +34,7 @@ def test_to_dict(_mock_weaviate):
         "init_parameters": {
             "filters": {},
             "top_k": 10,
+            "filter_policy": "replace",
             "document_store": {
                 "type": "haystack_integrations.document_stores.weaviate.document_store.WeaviateDocumentStore",
                 "init_parameters": {
@@ -49,6 +63,45 @@ def test_to_dict(_mock_weaviate):
 
 @patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
 def test_from_dict(_mock_weaviate):
+    retriever = WeaviateBM25Retriever.from_dict(
+        {
+            "type": "haystack_integrations.components.retrievers.weaviate.bm25_retriever.WeaviateBM25Retriever",
+            "init_parameters": {
+                "filters": {},
+                "top_k": 10,
+                "filter_policy": "replace",
+                "document_store": {
+                    "type": "haystack_integrations.document_stores.weaviate.document_store.WeaviateDocumentStore",
+                    "init_parameters": {
+                        "url": None,
+                        "collection_settings": {
+                            "class": "Default",
+                            "invertedIndexConfig": {"indexNullState": True},
+                            "properties": [
+                                {"name": "_original_id", "dataType": ["text"]},
+                                {"name": "content", "dataType": ["text"]},
+                                {"name": "dataframe", "dataType": ["text"]},
+                                {"name": "blob_data", "dataType": ["blob"]},
+                                {"name": "blob_mime_type", "dataType": ["text"]},
+                                {"name": "score", "dataType": ["number"]},
+                            ],
+                        },
+                        "auth_client_secret": None,
+                        "additional_headers": None,
+                        "embedded_options": None,
+                        "additional_config": None,
+                    },
+                },
+            },
+        }
+    )
+    assert retriever._document_store
+    assert retriever._filters == {}
+    assert retriever._top_k == 10
+
+
+@patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
+def test_from_dict_no_filter_policy(_mock_weaviate):
     retriever = WeaviateBM25Retriever.from_dict(
         {
             "type": "haystack_integrations.components.retrievers.weaviate.bm25_retriever.WeaviateBM25Retriever",
@@ -83,6 +136,7 @@ def test_from_dict(_mock_weaviate):
     assert retriever._document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
+    assert retriever._filter_policy == FilterPolicy.REPLACE
 
 
 @patch("haystack_integrations.components.retrievers.weaviate.bm25_retriever.WeaviateDocumentStore")

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/tests/test_document_store.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import base64
 import os
 import random

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/tests/test_embedding_retriever.py

@@ -1,6 +1,11 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from unittest.mock import Mock, patch
 
 import pytest
+from haystack.document_stores.types import FilterPolicy
 from haystack_integrations.components.retrievers.weaviate import WeaviateEmbeddingRetriever
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
@@ -11,9 +16,16 @@ def test_init_default():
     assert retriever._document_store == mock_document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
+    assert retriever._filter_policy == FilterPolicy.REPLACE
     assert retriever._distance is None
     assert retriever._certainty is None
 
+    retriever = WeaviateEmbeddingRetriever(document_store=mock_document_store, filter_policy="replace")
+    assert retriever._filter_policy == FilterPolicy.REPLACE
+
+    with pytest.raises(ValueError):
+        WeaviateEmbeddingRetriever(document_store=mock_document_store, filter_policy="keep_all")
+
 
 def test_init_with_distance_and_certainty():
     mock_document_store = Mock(spec=WeaviateDocumentStore)
@@ -30,6 +42,7 @@ def test_to_dict(_mock_weaviate):
         "init_parameters": {
             "filters": {},
             "top_k": 10,
+            "filter_policy": "replace",
             "distance": None,
             "certainty": None,
             "document_store": {
@@ -60,6 +73,49 @@ def test_to_dict(_mock_weaviate):
 
 @patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
 def test_from_dict(_mock_weaviate):
+    retriever = WeaviateEmbeddingRetriever.from_dict(
+        {
+            "type": "haystack_integrations.components.retrievers.weaviate.embedding_retriever.WeaviateEmbeddingRetriever",  # noqa: E501
+            "init_parameters": {
+                "filters": {},
+                "top_k": 10,
+                "filter_policy": "replace",
+                "distance": None,
+                "certainty": None,
+                "document_store": {
+                    "type": "haystack_integrations.document_stores.weaviate.document_store.WeaviateDocumentStore",
+                    "init_parameters": {
+                        "url": None,
+                        "collection_settings": {
+                            "class": "Default",
+                            "invertedIndexConfig": {"indexNullState": True},
+                            "properties": [
+                                {"name": "_original_id", "dataType": ["text"]},
+                                {"name": "content", "dataType": ["text"]},
+                                {"name": "dataframe", "dataType": ["text"]},
+                                {"name": "blob_data", "dataType": ["blob"]},
+                                {"name": "blob_mime_type", "dataType": ["text"]},
+                                {"name": "score", "dataType": ["number"]},
+                            ],
+                        },
+                        "auth_client_secret": None,
+                        "additional_headers": None,
+                        "embedded_options": None,
+                        "additional_config": None,
+                    },
+                },
+            },
+        }
+    )
+    assert retriever._document_store
+    assert retriever._filters == {}
+    assert retriever._top_k == 10
+    assert retriever._distance is None
+    assert retriever._certainty is None
+
+
+@patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
+def test_from_dict_no_filter_policy(_mock_weaviate):
     retriever = WeaviateEmbeddingRetriever.from_dict(
         {
             "type": "haystack_integrations.components.retrievers.weaviate.embedding_retriever.WeaviateEmbeddingRetriever",  # noqa: E501
@@ -98,6 +154,7 @@ def test_from_dict(_mock_weaviate):
     assert retriever._top_k == 10
     assert retriever._distance is None
     assert retriever._certainty is None
+    assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
 
 
 @patch("haystack_integrations.components.retrievers.weaviate.bm25_retriever.WeaviateDocumentStore")

{weaviate_haystack-2.1.1 → weaviate_haystack-2.1.3}/tests/test_filters.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from haystack_integrations.document_stores.weaviate._filters import _invert_condition