weaviate-haystack 2.0.0__tar.gz → 2.1.1__tar.gz

weaviate_haystack-2.1.1/CHANGELOG.md

@@ -0,0 +1,51 @@
+# Changelog
+
+## [integrations/weaviate-v2.1.0] - 2024-06-10
+
+### 🚀 Features
+
+- Defer the database connection to when it's needed (#802)
+
+### 🐛 Bug Fixes
+
+- Weaviate schema class name conversion which preserves PascalCase (#707)
+
+## [integrations/weaviate-v2.0.0] - 2024-03-25
+
+### 📚 Documentation
+
+- Disable-class-def (#556)
+- Fix docstrings (#586)
+
+### Weaviate
+
+- Migrate from weaviate python client v3  to v4 (#463)
+
+## [integrations/weaviate-v1.0.2] - 2024-02-27
+
+### 🐛 Bug Fixes
+
+- Fix order of API docs (#447)
+
+This PR will also push the docs to Readme
+- Fix weaviate auth tests (#488)
+
+
+
+### 📚 Documentation
+
+- Update category slug (#442)
+
+### Weaviate
+
+- Make retrievers return dicts (#491)
+
+## [integrations/weaviate-v1.0.0] - 2024-02-15
+
+### 🚀 Features
+
+- Generate weaviate API docs (#351)
+
+## [integrations/weaviate-v0.0.0] - 2024-01-10
+
+<!-- generated by git-cliff -->

{weaviate_haystack-2.0.0 → weaviate_haystack-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: weaviate-haystack
-Version: 2.0.0
+Version: 2.1.1
 Summary: An integration of Weaviate vector database with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md
@@ -9,6 +9,7 @@ Author-email: deepset GmbH <info@deepset.ai>
 License-Expression: Apache-2.0
 License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9

{weaviate_haystack-2.0.0 → weaviate_haystack-2.1.1}/pydoc/config.yml

@@ -18,7 +18,7 @@ processors:
   - type: smart
   - type: crossref
 renderer:
-  type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer
+  type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
   excerpt: Weaviate integration for Haystack
   category_slug: integrations-api
   title: Weaviate

{weaviate_haystack-2.0.0 → weaviate_haystack-2.1.1}/pyproject.toml

@@ -12,6 +12,7 @@ license = "Apache-2.0"
 keywords = []
 authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
 classifiers = [
+  "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3.8",
@@ -46,10 +47,10 @@ root = "../.."
 git_describe_command = 'git describe --tags --match="integrations/weaviate-v[0-9]*"'
 
 [tool.hatch.envs.default]
-dependencies = ["coverage[toml]>=6.5", "pytest", "ipython"]
+dependencies = ["coverage[toml]>=6.5", "pytest", "pytest-rerunfailures", "ipython"]
 [tool.hatch.envs.default.scripts]
-test = "pytest {args:tests}"
-test-cov = "coverage run -m pytest {args:tests}"
+test = "pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+test-cov = "coverage run -m pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
 cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
 docs = ["pydoc-markdown pydoc/config.yml"]
@@ -140,12 +141,8 @@ parallel = false
 
 [tool.coverage.report]
 omit = ["*/tests/*", "*/__init__.py"]
-show_missing=true
-exclude_lines = [
-  "no cov",
-  "if __name__ == .__main__.:",
-  "if TYPE_CHECKING:",
-]
+show_missing = true
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
 
 
 [[tool.mypy.overrides]]

{weaviate_haystack-2.0.0 → weaviate_haystack-2.1.1}/src/haystack_integrations/document_stores/weaviate/document_store.py

@@ -12,6 +12,7 @@ from haystack.core.serialization import default_from_dict, default_to_dict
 from haystack.dataclasses.document import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types.policy import DuplicatePolicy
+from haystack.utils.filters import convert
 
 import weaviate
 from weaviate.collections.classes.data import DataObject
@@ -139,47 +140,83 @@ class WeaviateDocumentStore:
         :param grpc_secure:
             Whether to use a secure channel for the underlying gRPC API.
         """
-        # proxies, timeout_config, trust_env are part of additional_config now
-        # startup_period has been removed
-        self._client = weaviate.WeaviateClient(
-            connection_params=(
-                weaviate.connect.base.ConnectionParams.from_url(url=url, grpc_port=grpc_port, grpc_secure=grpc_secure)
-                if url
-                else None
-            ),
-            auth_client_secret=auth_client_secret.resolve_value() if auth_client_secret else None,
-            additional_config=additional_config,
-            additional_headers=additional_headers,
-            embedded_options=embedded_options,
-            skip_init_checks=False,
+        self._url = url
+        self._auth_client_secret = auth_client_secret
+        self._additional_headers = additional_headers
+        self._embedded_options = embedded_options
+        self._additional_config = additional_config
+        self._grpc_port = grpc_port
+        self._grpc_secure = grpc_secure
+        self._client = None
+        self._collection = None
+        # Store the connection settings dictionary
+        self._collection_settings = collection_settings or {
+            "class": "Default",
+            "invertedIndexConfig": {"indexNullState": True},
+            "properties": DOCUMENT_COLLECTION_PROPERTIES,
+        }
+        self._clean_connection_settings()
+
+    def _clean_connection_settings(self):
+        # Set the class if not set
+        _class_name = self._collection_settings.get("class", "Default")
+        _class_name = _class_name[0].upper() + _class_name[1:]
+        self._collection_settings["class"] = _class_name
+        # Set the properties if they're not set
+        self._collection_settings["properties"] = self._collection_settings.get(
+            "properties", DOCUMENT_COLLECTION_PROPERTIES
         )
+
+    @property
+    def client(self):
+        if self._client:
+            return self._client
+
+        if self._url and self._url.startswith("http") and self._url.endswith(".weaviate.network"):
+            # We use this utility function instead of using WeaviateClient directly like in other cases
+            # otherwise we'd have to parse the URL to get some information about the connection.
+            # This utility function does all that for us.
+            self._client = weaviate.connect_to_wcs(
+                self._url,
+                auth_credentials=self._auth_client_secret.resolve_value() if self._auth_client_secret else None,
+                headers=self._additional_headers,
+                additional_config=self._additional_config,
+            )
+        else:
+            # proxies, timeout_config, trust_env are part of additional_config now
+            # startup_period has been removed
+            self._client = weaviate.WeaviateClient(
+                connection_params=(
+                    weaviate.connect.base.ConnectionParams.from_url(
+                        url=self._url, grpc_port=self._grpc_port, grpc_secure=self._grpc_secure
+                    )
+                    if self._url
+                    else None
+                ),
+                auth_client_secret=self._auth_client_secret.resolve_value() if self._auth_client_secret else None,
+                additional_config=self._additional_config,
+                additional_headers=self._additional_headers,
+                embedded_options=self._embedded_options,
+                skip_init_checks=False,
+            )
+
         self._client.connect()
 
         # Test connection, it will raise an exception if it fails.
         self._client.collections._get_all(simple=True)
+        if not self._client.collections.exists(self._collection_settings["class"]):
+            self._client.collections.create_from_dict(self._collection_settings)
 
-        if collection_settings is None:
-            collection_settings = {
-                "class": "Default",
-                "invertedIndexConfig": {"indexNullState": True},
-                "properties": DOCUMENT_COLLECTION_PROPERTIES,
-            }
-        else:
-            # Set the class if not set
-            collection_settings["class"] = collection_settings.get("class", "default").capitalize()
-            # Set the properties if they're not set
-            collection_settings["properties"] = collection_settings.get("properties", DOCUMENT_COLLECTION_PROPERTIES)
+        return self._client
 
-        if not self._client.collections.exists(collection_settings["class"]):
-            self._client.collections.create_from_dict(collection_settings)
+    @property
+    def collection(self):
+        if self._collection:
+            return self._collection
 
-        self._url = url
-        self._collection_settings = collection_settings
-        self._auth_client_secret = auth_client_secret
-        self._additional_headers = additional_headers
-        self._embedded_options = embedded_options
-        self._additional_config = additional_config
-        self._collection = self._client.collections.get(collection_settings["class"])
+        client = self.client
+        self._collection = client.collections.get(self._collection_settings["class"])
+        return self._collection
 
     def to_dict(self) -> Dict[str, Any]:
         """
@@ -228,7 +265,7 @@ class WeaviateDocumentStore:
         """
         Returns the number of documents present in the DocumentStore.
         """
-        total = self._collection.aggregate.over_all(total_count=True).total_count
+        total = self.collection.aggregate.over_all(total_count=True).total_count
         return total if total else 0
 
     def _to_data_object(self, document: Document) -> Dict[str, Any]:
@@ -300,16 +337,16 @@ class WeaviateDocumentStore:
         return Document.from_dict(document_data)
 
     def _query(self) -> List[Dict[str, Any]]:
-        properties = [p.name for p in self._collection.config.get().properties]
+        properties = [p.name for p in self.collection.config.get().properties]
         try:
-            result = self._collection.iterator(include_vector=True, return_properties=properties)
+            result = self.collection.iterator(include_vector=True, return_properties=properties)
         except weaviate.exceptions.WeaviateQueryError as e:
             msg = f"Failed to query documents in Weaviate. Error: {e.message}"
             raise DocumentStoreError(msg) from e
         return result
 
     def _query_with_filters(self, filters: Dict[str, Any]) -> List[Dict[str, Any]]:
-        properties = [p.name for p in self._collection.config.get().properties]
+        properties = [p.name for p in self.collection.config.get().properties]
         # When querying with filters we need to paginate using limit and offset as using
         # a cursor with after is not possible. See the official docs:
         # https://weaviate.io/developers/weaviate/api/graphql/additional-operators#cursor-with-after
@@ -325,7 +362,7 @@ class WeaviateDocumentStore:
         # Keep querying until we get all documents matching the filters
         while partial_result is None or len(partial_result.objects) == DEFAULT_QUERY_LIMIT:
             try:
-                partial_result = self._collection.query.fetch_objects(
+                partial_result = self.collection.query.fetch_objects(
                     filters=convert_filters(filters),
                     include_vector=True,
                     limit=DEFAULT_QUERY_LIMIT,
@@ -349,6 +386,9 @@ class WeaviateDocumentStore:
         :param filters: The filters to apply to the document list.
         :returns: A list of Documents that match the given filters.
         """
+        if filters and "operator" not in filters and "conditions" not in filters:
+            filters = convert(filters)
+
         result = []
         if filters:
             result = self._query_with_filters(filters)
@@ -363,7 +403,7 @@ class WeaviateDocumentStore:
         Raises in case of errors.
         """
 
-        with self._client.batch.dynamic() as batch:
+        with self.client.batch.dynamic() as batch:
             for doc in documents:
                 if not isinstance(doc, Document):
                     msg = f"Expected a Document, got '{type(doc)}' instead."
@@ -371,11 +411,11 @@ class WeaviateDocumentStore:
 
                 batch.add_object(
                     properties=self._to_data_object(doc),
-                    collection=self._collection.name,
+                    collection=self.collection.name,
                     uuid=generate_uuid5(doc.id),
                     vector=doc.embedding,
                 )
-        if failed_objects := self._client.batch.failed_objects:
+        if failed_objects := self.client.batch.failed_objects:
             # We fallback to use the UUID if the _original_id is not present, this is just to be
             mapped_objects = {}
             for obj in failed_objects:
@@ -411,12 +451,12 @@ class WeaviateDocumentStore:
                 msg = f"Expected a Document, got '{type(doc)}' instead."
                 raise ValueError(msg)
 
-            if policy == DuplicatePolicy.SKIP and self._collection.data.exists(uuid=generate_uuid5(doc.id)):
+            if policy == DuplicatePolicy.SKIP and self.collection.data.exists(uuid=generate_uuid5(doc.id)):
                 # This Document already exists, we skip it
                 continue
 
             try:
-                self._collection.data.insert(
+                self.collection.data.insert(
                     uuid=generate_uuid5(doc.id),
                     properties=self._to_data_object(doc),
                     vector=doc.embedding,
@@ -452,13 +492,13 @@ class WeaviateDocumentStore:
         :param document_ids: The object_ids to delete.
         """
         weaviate_ids = [generate_uuid5(doc_id) for doc_id in document_ids]
-        self._collection.data.delete_many(where=weaviate.classes.query.Filter.by_id().contains_any(weaviate_ids))
+        self.collection.data.delete_many(where=weaviate.classes.query.Filter.by_id().contains_any(weaviate_ids))
 
     def _bm25_retrieval(
         self, query: str, filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None
     ) -> List[Document]:
-        properties = [p.name for p in self._collection.config.get().properties]
-        result = self._collection.query.bm25(
+        properties = [p.name for p in self.collection.config.get().properties]
+        result = self.collection.query.bm25(
             query=query,
             filters=convert_filters(filters) if filters else None,
             limit=top_k,
@@ -482,8 +522,8 @@ class WeaviateDocumentStore:
             msg = "Can't use 'distance' and 'certainty' parameters together"
             raise ValueError(msg)
 
-        properties = [p.name for p in self._collection.config.get().properties]
-        result = self._collection.query.near_vector(
+        properties = [p.name for p in self.collection.config.get().properties]
+        result = self.collection.query.near_vector(
             near_vector=query_embedding,
             distance=distance,
             certainty=certainty,

{weaviate_haystack-2.0.0 → weaviate_haystack-2.1.1}/tests/test_document_store.py

@@ -1,4 +1,5 @@
 import base64
+import os
 import random
 from typing import List
 from unittest.mock import MagicMock, patch
@@ -16,6 +17,7 @@ from haystack.testing.document_store import (
     FilterDocumentsTest,
     WriteDocumentsTest,
 )
+from haystack.utils.auth import Secret
 from haystack_integrations.document_stores.weaviate.auth import AuthApiKey
 from haystack_integrations.document_stores.weaviate.document_store import (
     DOCUMENT_COLLECTION_PROPERTIES,
@@ -26,8 +28,6 @@ from numpy import array_equal as np_array_equal
 from numpy import float32 as np_float32
 from pandas import DataFrame
 from weaviate.collections.classes.data import DataObject
-
-# from weaviate.auth import AuthApiKey as WeaviateAuthApiKey
 from weaviate.config import AdditionalConfig, ConnectionConfig, Proxies, Timeout
 from weaviate.embedded import (
     DEFAULT_BINARY_PATH,
@@ -38,6 +38,12 @@ from weaviate.embedded import (
 )
 
 
+@patch("haystack_integrations.document_stores.weaviate.document_store.weaviate.WeaviateClient")
+def test_init_is_lazy(_mock_client):
+    _ = WeaviateDocumentStore()
+    _mock_client.assert_not_called()
+
+
 @pytest.mark.integration
 class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDocumentsTest, FilterDocumentsTest):
     @pytest.fixture
@@ -57,7 +63,7 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
             collection_settings=collection_settings,
         )
         yield store
-        store._client.collections.delete(collection_settings["class"])
+        store.client.collections.delete(collection_settings["class"])
 
     @pytest.fixture
     def filterable_docs(self) -> List[Document]:
@@ -150,12 +156,12 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
                 assert received_meta.get(key) == expected_meta.get(key)
 
     @patch("haystack_integrations.document_stores.weaviate.document_store.weaviate.WeaviateClient")
-    def test_init(self, mock_weaviate_client_class, monkeypatch):
+    def test_connection(self, mock_weaviate_client_class, monkeypatch):
         mock_client = MagicMock()
         mock_client.collections.exists.return_value = False
         mock_weaviate_client_class.return_value = mock_client
         monkeypatch.setenv("WEAVIATE_API_KEY", "my_api_key")
-        WeaviateDocumentStore(
+        ds = WeaviateDocumentStore(
             collection_settings={"class": "My_collection"},
             auth_client_secret=AuthApiKey(),
             additional_headers={"X-HuggingFace-Api-Key": "MY_HUGGINGFACE_KEY"},
@@ -170,8 +176,11 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
             ),
         )
 
-        # Verify client is created with correct parameters
+        # Trigger the actual database connection by accessing the `client` property so we
+        # can assert the setup was good
+        _ = ds.client
 
+        # Verify client is created with correct parameters
         mock_weaviate_client_class.assert_called_once_with(
             auth_client_secret=AuthApiKey().resolve_value(),
             connection_params=None,
@@ -646,6 +655,15 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
         with pytest.raises(ValueError):
             document_store._embedding_retrieval(query_embedding=[], distance=0.1, certainty=0.1)
 
+    def test_filter_documents_with_legacy_filters(self, document_store):
+        docs = []
+        for index in range(10):
+            docs.append(Document(content="This is some content", meta={"index": index}))
+        document_store.write_documents(docs)
+        result = document_store.filter_documents({"content": {"$eq": "This is some content"}})
+
+        assert len(result) == 10
+
     def test_filter_documents_below_default_limit(self, document_store):
         docs = []
         for index in range(9998):
@@ -664,3 +682,39 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
         document_store.write_documents(docs)
         with pytest.raises(DocumentStoreError):
             document_store.filter_documents({"field": "content", "operator": "==", "value": "This is some content"})
+
+    def test_schema_class_name_conversion_preserves_pascal_case(self):
+        collection_settings = {"class": "CaseDocument"}
+        doc_score = WeaviateDocumentStore(
+            url="http://localhost:8080",
+            collection_settings=collection_settings,
+        )
+        assert doc_score._collection_settings["class"] == "CaseDocument"
+
+        collection_settings = {"class": "lower_case_name"}
+        doc_score = WeaviateDocumentStore(
+            url="http://localhost:8080",
+            collection_settings=collection_settings,
+        )
+        assert doc_score._collection_settings["class"] == "Lower_case_name"
+
+    @pytest.mark.skipif(
+        not os.environ.get("WEAVIATE_API_KEY", None) and not os.environ.get("WEAVIATE_CLOUD_CLUSTER_URL", None),
+        reason="Both WEAVIATE_API_KEY and WEAVIATE_CLOUD_CLUSTER_URL are not set. Skipping test.",
+    )
+    def test_connect_to_weaviate_cloud(self):
+        document_store = WeaviateDocumentStore(
+            url=os.environ.get("WEAVIATE_CLOUD_CLUSTER_URL"),
+            auth_client_secret=AuthApiKey(api_key=Secret.from_env_var("WEAVIATE_API_KEY")),
+        )
+        assert document_store.client
+
+    def test_connect_to_local(self):
+        document_store = WeaviateDocumentStore(
+            url="http://localhost:8080",
+        )
+        assert document_store.client
+
+    def test_connect_to_embedded(self):
+        document_store = WeaviateDocumentStore(embedded_options=EmbeddedOptions())
+        assert document_store.client