weaviate-haystack 2.1.0__tar.gz → 2.1.2__tar.gz

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/.gitignore

@@ -135,3 +135,12 @@ dmypy.json
 # Docs generation artifacts
 _readme_*.md
 .idea
+
+# macOS
+.DS_Store
+
+# http cache (requests-cache)
+**/http_cache.sqlite
+
+# ruff
+.ruff_cache

weaviate_haystack-2.1.2/CHANGELOG.md

@@ -0,0 +1,62 @@
+# Changelog
+
+## [unreleased]
+
+### 🐛 Bug Fixes
+
+- Weaviate filter error (#811)
+- Fix connection to Weaviate Cloud Service (#624)
+
+### ⚙️ Miscellaneous Tasks
+
+- Retry tests to reduce flakyness (#836)
+
+## [integrations/weaviate-v2.1.0] - 2024-06-10
+
+### 🚀 Features
+
+- Defer the database connection to when it's needed (#802)
+
+### 🐛 Bug Fixes
+
+- Weaviate schema class name conversion which preserves PascalCase (#707)
+
+## [integrations/weaviate-v2.0.0] - 2024-03-25
+
+### 📚 Documentation
+
+- Disable-class-def (#556)
+- Fix docstrings (#586)
+
+### Weaviate
+
+- Migrate from weaviate python client v3  to v4 (#463)
+
+## [integrations/weaviate-v1.0.2] - 2024-02-27
+
+### 🐛 Bug Fixes
+
+- Fix order of API docs (#447)
+
+This PR will also push the docs to Readme
+- Fix weaviate auth tests (#488)
+
+
+
+### 📚 Documentation
+
+- Update category slug (#442)
+
+### Weaviate
+
+- Make retrievers return dicts (#491)
+
+## [integrations/weaviate-v1.0.0] - 2024-02-15
+
+### 🚀 Features
+
+- Generate weaviate API docs (#351)
+
+## [integrations/weaviate-v0.0.0] - 2024-01-10
+
+<!-- generated by git-cliff -->

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: weaviate-haystack
-Version: 2.1.0
+Version: 2.1.2
 Summary: An integration of Weaviate vector database with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/pyproject.toml

@@ -47,10 +47,10 @@ root = "../.."
 git_describe_command = 'git describe --tags --match="integrations/weaviate-v[0-9]*"'
 
 [tool.hatch.envs.default]
-dependencies = ["coverage[toml]>=6.5", "pytest", "ipython"]
+dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "ipython"]
 [tool.hatch.envs.default.scripts]
-test = "pytest {args:tests}"
-test-cov = "coverage run -m pytest {args:tests}"
+test = "pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+test-cov = "coverage run -m pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
 cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
 docs = ["pydoc-markdown pydoc/config.yml"]
@@ -63,7 +63,7 @@ detached = true
 dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = ["ruff {args:.}", "black --check --diff {args:.}"]
+style = ["ruff check {args:.}", "black --check --diff {args:.}"]
 fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
 all = ["style", "typing"]
 
@@ -75,7 +75,7 @@ skip-string-normalization = true
 [tool.ruff]
 target-version = "py38"
 line-length = 120
-select = [
+lint.select = [
   "A",
   "ARG",
   "B",
@@ -102,7 +102,7 @@ select = [
   "W",
   "YTT",
 ]
-ignore = [
+lint.ignore = [
   # Allow non-abstract empty methods in abstract base classes
   "B027",
   # Allow boolean positional values in function calls, like `dict.get(... True)`
@@ -118,18 +118,18 @@ ignore = [
   "PLR0913",
   "PLR0915",
 ]
-unfixable = [
+lint.unfixable = [
   # Don't touch unused imports
   "F401",
 ]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["src"]
 
-[tool.ruff.flake8-tidy-imports]
+[tool.ruff.lint.flake8-tidy-imports]
 ban-relative-imports = "parents"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 
@@ -141,12 +141,8 @@ parallel = false
 
 [tool.coverage.report]
 omit = ["*/tests/*", "*/__init__.py"]
-show_missing=true
-exclude_lines = [
-  "no cov",
-  "if __name__ == .__main__.:",
-  "if TYPE_CHECKING:",
-]
+show_missing = true
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
 
 
 [[tool.mypy.overrides]]

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/src/haystack_integrations/components/retrievers/weaviate/__init__.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from .bm25_retriever import WeaviateBM25Retriever
 from .embedding_retriever import WeaviateEmbeddingRetriever
 

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py

@@ -1,6 +1,12 @@
-from typing import Any, Dict, List, Optional
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Dict, List, Optional, Union
 
 from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
 
@@ -26,6 +32,7 @@ class WeaviateBM25Retriever:
         document_store: WeaviateDocumentStore,
         filters: Optional[Dict[str, Any]] = None,
         top_k: int = 10,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
     ):
         """
         Create a new instance of WeaviateBM25Retriever.
@@ -36,10 +43,14 @@ class WeaviateBM25Retriever:
             Custom filters applied when running the retriever
         :param top_k:
             Maximum number of documents to return
+        :param filter_policy: Policy to determine how filters are applied.
         """
         self._document_store = document_store
         self._filters = filters or {}
         self._top_k = top_k
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -52,6 +63,7 @@ class WeaviateBM25Retriever:
             self,
             filters=self._filters,
             top_k=self._top_k,
+            filter_policy=self._filter_policy.value,
             document_store=self._document_store.to_dict(),
         )
 
@@ -68,6 +80,7 @@ class WeaviateBM25Retriever:
         data["init_parameters"]["document_store"] = WeaviateDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
+        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -77,12 +90,14 @@ class WeaviateBM25Retriever:
 
         :param query:
             The query text.
-        :param filters:
-            Filters to use when running the retriever.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k:
             The maximum number of documents to return.
         """
-        filters = filters or self._filters
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
+
         top_k = top_k or self._top_k
         documents = self._document_store._bm25_retrieval(query=query, filters=filters, top_k=top_k)
         return {"documents": documents}

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/src/haystack_integrations/components/retrievers/weaviate/embedding_retriever.py

@@ -1,6 +1,12 @@
-from typing import Any, Dict, List, Optional
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Dict, List, Optional, Union
 
 from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.document_stores.types import FilterPolicy
+from haystack.document_stores.types.filter_policy import apply_filter_policy
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
 
@@ -18,6 +24,7 @@ class WeaviateEmbeddingRetriever:
         top_k: int = 10,
         distance: Optional[float] = None,
         certainty: Optional[float] = None,
+        filter_policy: Union[str, FilterPolicy] = FilterPolicy.REPLACE,
     ):
         """
         Creates a new instance of WeaviateEmbeddingRetriever.
@@ -32,6 +39,8 @@ class WeaviateEmbeddingRetriever:
             The maximum allowed distance between Documents' embeddings.
         :param certainty:
             Normalized distance between the result item and the search vector.
+        :param filter_policy:
+            Policy to determine how filters are applied.
         :raises ValueError:
             If both `distance` and `certainty` are provided.
             See https://weaviate.io/developers/weaviate/api/graphql/search-operators#variables to learn more about
@@ -46,6 +55,9 @@ class WeaviateEmbeddingRetriever:
         self._top_k = top_k
         self._distance = distance
         self._certainty = certainty
+        self._filter_policy = (
+            filter_policy if isinstance(filter_policy, FilterPolicy) else FilterPolicy.from_str(filter_policy)
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -60,6 +72,7 @@ class WeaviateEmbeddingRetriever:
             top_k=self._top_k,
             distance=self._distance,
             certainty=self._certainty,
+            filter_policy=self._filter_policy.value,
             document_store=self._document_store.to_dict(),
         )
 
@@ -76,6 +89,8 @@ class WeaviateEmbeddingRetriever:
         data["init_parameters"]["document_store"] = WeaviateDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
+
+        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
@@ -92,8 +107,9 @@ class WeaviateEmbeddingRetriever:
 
         :param query_embedding:
             Embedding of the query.
-        :param filters:
-            Filters to use when running the retriever.
+        :param filters: Filters applied to the retrieved Documents. The way runtime filters are applied depends on
+                        the `filter_policy` chosen at retriever initialization. See init method docstring for more
+                        details.
         :param top_k:
             The maximum number of documents to return.
         :param distance:
@@ -105,7 +121,7 @@ class WeaviateEmbeddingRetriever:
             See https://weaviate.io/developers/weaviate/api/graphql/search-operators#variables to learn more about
             `distance` and `certainty` parameters.
         """
-        filters = filters or self._filters
+        filters = apply_filter_policy(self._filter_policy, self._filters, filters)
         top_k = top_k or self._top_k
 
         distance = distance or self._distance

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/src/haystack_integrations/document_stores/weaviate/_filters.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Dict
 
 from dateutil import parser

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/src/haystack_integrations/document_stores/weaviate/auth.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field, fields
 from enum import Enum

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/src/haystack_integrations/document_stores/weaviate/document_store.py

@@ -12,6 +12,7 @@ from haystack.core.serialization import default_from_dict, default_to_dict
 from haystack.dataclasses.document import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types.policy import DuplicatePolicy
+from haystack.utils.filters import convert
 
 import weaviate
 from weaviate.collections.classes.data import DataObject
@@ -171,22 +172,33 @@ class WeaviateDocumentStore:
         if self._client:
             return self._client
 
-        # proxies, timeout_config, trust_env are part of additional_config now
-        # startup_period has been removed
-        self._client = weaviate.WeaviateClient(
-            connection_params=(
-                weaviate.connect.base.ConnectionParams.from_url(
-                    url=self._url, grpc_port=self._grpc_port, grpc_secure=self._grpc_secure
-                )
-                if self._url
-                else None
-            ),
-            auth_client_secret=self._auth_client_secret.resolve_value() if self._auth_client_secret else None,
-            additional_config=self._additional_config,
-            additional_headers=self._additional_headers,
-            embedded_options=self._embedded_options,
-            skip_init_checks=False,
-        )
+        if self._url and self._url.startswith("http") and self._url.endswith(".weaviate.network"):
+            # We use this utility function instead of using WeaviateClient directly like in other cases
+            # otherwise we'd have to parse the URL to get some information about the connection.
+            # This utility function does all that for us.
+            self._client = weaviate.connect_to_wcs(
+                self._url,
+                auth_credentials=self._auth_client_secret.resolve_value() if self._auth_client_secret else None,
+                headers=self._additional_headers,
+                additional_config=self._additional_config,
+            )
+        else:
+            # proxies, timeout_config, trust_env are part of additional_config now
+            # startup_period has been removed
+            self._client = weaviate.WeaviateClient(
+                connection_params=(
+                    weaviate.connect.base.ConnectionParams.from_url(
+                        url=self._url, grpc_port=self._grpc_port, grpc_secure=self._grpc_secure
+                    )
+                    if self._url
+                    else None
+                ),
+                auth_client_secret=self._auth_client_secret.resolve_value() if self._auth_client_secret else None,
+                additional_config=self._additional_config,
+                additional_headers=self._additional_headers,
+                embedded_options=self._embedded_options,
+                skip_init_checks=False,
+            )
 
         self._client.connect()
 
@@ -374,6 +386,9 @@ class WeaviateDocumentStore:
         :param filters: The filters to apply to the document list.
         :returns: A list of Documents that match the given filters.
         """
+        if filters and "operator" not in filters and "conditions" not in filters:
+            filters = convert(filters)
+
         result = []
         if filters:
             result = self._query_with_filters(filters)

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/tests/conftest.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from pathlib import Path
 
 import pytest

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/tests/test_auth.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from haystack_integrations.document_stores.weaviate.auth import (
     AuthApiKey,
     AuthBearerToken,

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/tests/test_bm25_retriever.py

@@ -1,5 +1,11 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from unittest.mock import Mock, patch
 
+import pytest
+from haystack.document_stores.types import FilterPolicy
 from haystack_integrations.components.retrievers.weaviate import WeaviateBM25Retriever
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
@@ -10,6 +16,13 @@ def test_init_default():
     assert retriever._document_store == mock_document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
+    assert retriever._filter_policy == FilterPolicy.REPLACE
+
+    retriever = WeaviateBM25Retriever(document_store=mock_document_store, filter_policy="replace")
+    assert retriever._filter_policy == FilterPolicy.REPLACE
+
+    with pytest.raises(ValueError):
+        WeaviateBM25Retriever(document_store=mock_document_store, filter_policy="keep_all")
 
 
 @patch("haystack_integrations.document_stores.weaviate.document_store.weaviate")
@@ -21,6 +34,7 @@ def test_to_dict(_mock_weaviate):
         "init_parameters": {
             "filters": {},
             "top_k": 10,
+            "filter_policy": "replace",
             "document_store": {
                 "type": "haystack_integrations.document_stores.weaviate.document_store.WeaviateDocumentStore",
                 "init_parameters": {
@@ -55,6 +69,7 @@ def test_from_dict(_mock_weaviate):
             "init_parameters": {
                 "filters": {},
                 "top_k": 10,
+                "filter_policy": "replace",
                 "document_store": {
                     "type": "haystack_integrations.document_stores.weaviate.document_store.WeaviateDocumentStore",
                     "init_parameters": {

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/tests/test_document_store.py

@@ -1,4 +1,9 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import base64
+import os
 import random
 from typing import List
 from unittest.mock import MagicMock, patch
@@ -16,6 +21,7 @@ from haystack.testing.document_store import (
     FilterDocumentsTest,
     WriteDocumentsTest,
 )
+from haystack.utils.auth import Secret
 from haystack_integrations.document_stores.weaviate.auth import AuthApiKey
 from haystack_integrations.document_stores.weaviate.document_store import (
     DOCUMENT_COLLECTION_PROPERTIES,
@@ -26,8 +32,6 @@ from numpy import array_equal as np_array_equal
 from numpy import float32 as np_float32
 from pandas import DataFrame
 from weaviate.collections.classes.data import DataObject
-
-# from weaviate.auth import AuthApiKey as WeaviateAuthApiKey
 from weaviate.config import AdditionalConfig, ConnectionConfig, Proxies, Timeout
 from weaviate.embedded import (
     DEFAULT_BINARY_PATH,
@@ -655,6 +659,15 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
         with pytest.raises(ValueError):
             document_store._embedding_retrieval(query_embedding=[], distance=0.1, certainty=0.1)
 
+    def test_filter_documents_with_legacy_filters(self, document_store):
+        docs = []
+        for index in range(10):
+            docs.append(Document(content="This is some content", meta={"index": index}))
+        document_store.write_documents(docs)
+        result = document_store.filter_documents({"content": {"$eq": "This is some content"}})
+
+        assert len(result) == 10
+
     def test_filter_documents_below_default_limit(self, document_store):
         docs = []
         for index in range(9998):
@@ -688,3 +701,24 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
             collection_settings=collection_settings,
         )
         assert doc_score._collection_settings["class"] == "Lower_case_name"
+
+    @pytest.mark.skipif(
+        not os.environ.get("WEAVIATE_API_KEY", None) and not os.environ.get("WEAVIATE_CLOUD_CLUSTER_URL", None),
+        reason="Both WEAVIATE_API_KEY and WEAVIATE_CLOUD_CLUSTER_URL are not set. Skipping test.",
+    )
+    def test_connect_to_weaviate_cloud(self):
+        document_store = WeaviateDocumentStore(
+            url=os.environ.get("WEAVIATE_CLOUD_CLUSTER_URL"),
+            auth_client_secret=AuthApiKey(api_key=Secret.from_env_var("WEAVIATE_API_KEY")),
+        )
+        assert document_store.client
+
+    def test_connect_to_local(self):
+        document_store = WeaviateDocumentStore(
+            url="http://localhost:8080",
+        )
+        assert document_store.client
+
+    def test_connect_to_embedded(self):
+        document_store = WeaviateDocumentStore(embedded_options=EmbeddedOptions())
+        assert document_store.client

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/tests/test_embedding_retriever.py

@@ -1,6 +1,11 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from unittest.mock import Mock, patch
 
 import pytest
+from haystack.document_stores.types import FilterPolicy
 from haystack_integrations.components.retrievers.weaviate import WeaviateEmbeddingRetriever
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 
@@ -11,9 +16,16 @@ def test_init_default():
     assert retriever._document_store == mock_document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
+    assert retriever._filter_policy == FilterPolicy.REPLACE
     assert retriever._distance is None
     assert retriever._certainty is None
 
+    retriever = WeaviateEmbeddingRetriever(document_store=mock_document_store, filter_policy="replace")
+    assert retriever._filter_policy == FilterPolicy.REPLACE
+
+    with pytest.raises(ValueError):
+        WeaviateEmbeddingRetriever(document_store=mock_document_store, filter_policy="keep_all")
+
 
 def test_init_with_distance_and_certainty():
     mock_document_store = Mock(spec=WeaviateDocumentStore)
@@ -30,6 +42,7 @@ def test_to_dict(_mock_weaviate):
         "init_parameters": {
             "filters": {},
             "top_k": 10,
+            "filter_policy": "replace",
             "distance": None,
             "certainty": None,
             "document_store": {
@@ -66,6 +79,7 @@ def test_from_dict(_mock_weaviate):
             "init_parameters": {
                 "filters": {},
                 "top_k": 10,
+                "filter_policy": "replace",
                 "distance": None,
                 "certainty": None,
                 "document_store": {

{weaviate_haystack-2.1.0 → weaviate_haystack-2.1.2}/tests/test_filters.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from haystack_integrations.document_stores.weaviate._filters import _invert_condition