weaviate-haystack 7.2.0__tar.gz → 7.4.0__tar.gz

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## [integrations/weaviate-v7.3.0] - 2026-03-11
+
+### 🚀 Features
+
+- Add missing async methods for `WeaviateDocumentStore` (#2929)
+
+
+## [integrations/weaviate-v7.2.0] - 2026-03-06
+
+### 🐛 Bug Fixes
+
+- Remove unnecessary connection test and add `close`/`close_async` methods to `WeaviateDocumentStore` (#2891)
+
+### 📚 Documentation
+
+- Simplify pydoc configs (#2855)
+
+### 🧪 Testing
+
+- Replacing each `DocumentStore` specific tests and used the generalised ones from `haystack.testing.document_store` (#2812)
+- Fix Weaviate tests to include grpc_config (#2871)
+
+
 ## [integrations/weaviate-v7.1.0] - 2026-01-29
 
 ### 🚀 Features

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: weaviate-haystack
-Version: 7.2.0
+Version: 7.4.0
 Summary: An integration of Weaviate vector database with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.10
 Requires-Dist: haystack-ai>=2.24.0
 Requires-Dist: python-dateutil
-Requires-Dist: weaviate-client>=4.9
+Requires-Dist: weaviate-client>=4.20
 Description-Content-Type: text/markdown
 
 # weaviate-haystack

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/docker-compose.yml

@@ -7,7 +7,7 @@ services:
     - '8080'
     - --scheme
     - http
-    image: semitechnologies/weaviate:1.30.17
+    image: semitechnologies/weaviate:1.36.2
     ports:
     - 8080:8080
     - 50051:50051

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 ]
 dependencies = [
   "haystack-ai>=2.24.0",
-  "weaviate-client>=4.9",
+  "weaviate-client>=4.20",
   "python-dateutil",
 ]
 

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/src/haystack_integrations/components/retrievers/weaviate/hybrid_retriever.py

@@ -23,7 +23,7 @@ class WeaviateHybridRetriever:
         document_store: WeaviateDocumentStore,
         filters: dict[str, Any] | None = None,
         top_k: int = 10,
-        alpha: float | None = None,
+        alpha: float = 0.7,
         max_vector_distance: float | None = None,
         filter_policy: str | FilterPolicy = FilterPolicy.REPLACE,
     ):
@@ -46,7 +46,7 @@ class WeaviateHybridRetriever:
             - `alpha = 1.0`: only vector similarity scoring is used.
             - Values in between blend the two; higher values favor the vector score, lower values favor BM25.
 
-            If `None`, the Weaviate server default is used.
+            By default, 0.7 is used which is the Weaviate server default.
 
             See the official Weaviate docs on Hybrid Search parameters for more details:
             - [Hybrid search parameters](https://weaviate.io/developers/weaviate/search/hybrid#parameters)
@@ -66,7 +66,7 @@ class WeaviateHybridRetriever:
             Policy to determine how filters are applied.
         """
 
-        if alpha is not None and not 0.0 <= alpha <= 1.0:
+        if not 0.0 <= alpha <= 1.0:
             msg = f"alpha ({alpha}) must be in the range [0.0, 1.0]"
             raise ValueError(msg)
 

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/src/haystack_integrations/document_stores/weaviate/document_store.py

@@ -6,7 +6,7 @@ import base64
 import datetime
 import json
 from dataclasses import asdict
-from typing import Any
+from typing import Any, NoReturn
 
 from haystack import logging
 from haystack.core.serialization import default_from_dict, default_to_dict
@@ -22,9 +22,13 @@ from weaviate.collections.classes.aggregate import (
     _MetricsInteger,
     _MetricsNumber,
 )
-from weaviate.collections.classes.data import DataObject
+from weaviate.collections.classes.batch import ErrorObject
+from weaviate.collections.classes.internal import Object
+from weaviate.collections.collection import Collection, CollectionAsync
+from weaviate.collections.iterator import _ObjectAIterator, _ObjectIterator
 from weaviate.config import AdditionalConfig
 from weaviate.embedded import EmbeddedOptions
+from weaviate.types import VECTORS
 from weaviate.util import generate_uuid5
 
 from ._filters import convert_filters, validate_filters
@@ -105,7 +109,7 @@ class WeaviateDocumentStore:
         additional_config: AdditionalConfig | None = None,
         grpc_port: int = 50051,
         grpc_secure: bool = False,
-    ):
+    ) -> None:
         """
         Create a new instance of WeaviateDocumentStore and connects to the Weaviate instance.
 
@@ -267,7 +271,7 @@ class WeaviateDocumentStore:
         return self._async_client
 
     @property
-    def collection(self):
+    def collection(self) -> Collection[dict[str, Any], None]:
         if self._collection:
             return self._collection
 
@@ -276,7 +280,7 @@ class WeaviateDocumentStore:
         return self._collection
 
     @property
-    async def async_collection(self):
+    async def async_collection(self) -> CollectionAsync[dict[str, Any], None]:
         if self._async_collection:
             return self._async_collection
 
@@ -352,6 +356,14 @@ class WeaviateDocumentStore:
         total = self.collection.aggregate.over_all(total_count=True).total_count
         return total if total else 0
 
+    async def count_documents_async(self) -> int:
+        """
+        Asynchronously returns the number of documents present in the DocumentStore.
+        """
+        collection = await self.async_collection
+        total = (await collection.aggregate.over_all(total_count=True)).total_count
+        return total if total else 0
+
     def count_documents_by_filter(self, filters: dict[str, Any]) -> int:
         """
         Returns the number of documents that match the provided filters.
@@ -733,15 +745,6 @@ class WeaviateDocumentStore:
         # The embedding vector is stored separately from the rest of the data
         del data["embedding"]
 
-        # _split_overlap meta field is unsupported because of a bug
-        # https://github.com/deepset-ai/haystack-core-integrations/issues/1172
-        if "_split_overlap" in data:
-            data.pop("_split_overlap")
-            logger.warning(
-                "Document {id} has the unsupported `_split_overlap` meta field. It will be ignored.",
-                id=data["_original_id"],
-            )
-
         if "sparse_embedding" in data:
             sparse_embedding = data.pop("sparse_embedding", None)
             if sparse_embedding:
@@ -755,7 +758,7 @@ class WeaviateDocumentStore:
         return data
 
     @staticmethod
-    def _to_document(data: DataObject[dict[str, Any], None]) -> Document:
+    def _to_document(data: Object[dict[str, Any], None]) -> Document:
         """
         Converts a data object read from Weaviate into a Document.
         """
@@ -793,7 +796,7 @@ class WeaviateDocumentStore:
 
         return Document.from_dict(document_data)
 
-    def _query(self) -> list[DataObject[dict[str, Any], None]]:
+    def _query(self) -> _ObjectIterator[dict[str, Any], None]:
         properties = [p.name for p in self.collection.config.get().properties]
         try:
             result = self.collection.iterator(include_vector=True, return_properties=properties)
@@ -802,7 +805,17 @@ class WeaviateDocumentStore:
             raise DocumentStoreError(msg) from e
         return result
 
-    def _query_with_filters(self, filters: dict[str, Any]) -> list[DataObject[dict[str, Any], None]]:
+    async def _query_async(self) -> _ObjectAIterator[dict[str, Any], None]:
+        collection = await self.async_collection
+        properties = [p.name for p in (await collection.config.get()).properties]
+        try:
+            result = collection.iterator(include_vector=True, return_properties=properties)
+        except weaviate.exceptions.WeaviateQueryError as e:
+            msg = f"Failed to query documents in Weaviate. Error: {e.message}"
+            raise DocumentStoreError(msg) from e
+        return result
+
+    def _query_with_filters(self, filters: dict[str, Any]) -> list[Object[dict[str, Any], None]]:
         properties = [p.name for p in self.collection.config.get().properties]
         # When querying with filters we need to paginate using limit and offset as using
         # a cursor with after is not possible. See the official docs:
@@ -833,6 +846,38 @@ class WeaviateDocumentStore:
             offset += DEFAULT_QUERY_LIMIT
         return result
 
+    async def _query_with_filters_async(self, filters: dict[str, Any]) -> list[Object[dict[str, Any], None]]:
+        collection = await self.async_collection
+        properties = [p.name for p in (await collection.config.get()).properties]
+        # When querying with filters we need to paginate using limit and offset as using
+        # a cursor with after is not possible. See the official docs:
+        # https://weaviate.io/developers/weaviate/api/graphql/additional-operators#cursor-with-after
+        #
+        # Nonetheless there's also another issue, paginating with limit and offset is not efficient
+        # and it's still restricted by the QUERY_MAXIMUM_RESULTS environment variable.
+        # If the sum of limit and offset is greater than QUERY_MAXIMUM_RESULTS an error is raised.
+        # See the official docs for more:
+        # https://weaviate.io/developers/weaviate/api/graphql/additional-operators#performance-considerations
+        offset = 0
+        partial_result = None
+        result = []
+        # Keep querying until we get all documents matching the filters
+        while partial_result is None or len(partial_result.objects) == DEFAULT_QUERY_LIMIT:
+            try:
+                partial_result = await collection.query.fetch_objects(
+                    filters=convert_filters(filters),
+                    include_vector=True,
+                    limit=DEFAULT_QUERY_LIMIT,
+                    offset=offset,
+                    return_properties=properties,
+                )
+            except weaviate.exceptions.WeaviateQueryError as e:
+                msg = f"Failed to query documents in Weaviate. Error: {e.message}"
+                raise DocumentStoreError(msg) from e
+            result.extend(partial_result.objects)
+            offset += DEFAULT_QUERY_LIMIT
+        return result
+
     def filter_documents(self, filters: dict[str, Any] | None = None) -> list[Document]:
         """
         Returns the documents that match the filters provided.
@@ -852,9 +897,48 @@ class WeaviateDocumentStore:
         result = []
         if filters:
             result = self._query_with_filters(filters)
-        else:
-            result = self._query()
-        return [WeaviateDocumentStore._to_document(doc) for doc in result]
+            return [WeaviateDocumentStore._to_document(doc) for doc in result]
+        result_iter = self._query()
+        return [WeaviateDocumentStore._to_document(doc) for doc in result_iter]
+
+    async def filter_documents_async(self, filters: dict[str, Any] | None = None) -> list[Document]:
+        """
+        Asynchronously returns the documents that match the filters provided.
+
+        For a detailed specification of the filters, refer to the
+        DocumentStore.filter_documents() protocol documentation.
+
+        Note: The ``contains`` filter operator is case-sensitive (substring
+        matching). For case-insensitive matching, normalize the value before
+        building the filter.
+
+        :param filters: The filters to apply to the document list.
+        :returns: A list of Documents that match the given filters.
+        """
+        validate_filters(filters)
+
+        result = []
+        if filters:
+            result = await self._query_with_filters_async(filters)
+            return [WeaviateDocumentStore._to_document(doc) for doc in result]
+        result_iter = await self._query_async()
+        return [WeaviateDocumentStore._to_document(doc) async for doc in result_iter]
+
+    @staticmethod
+    def _handle_failed_objects(failed_objects: list[ErrorObject]) -> NoReturn:
+        # We fall back to use the UUID if the _original_id is not present, this is just to be
+        mapped_objects = {}
+        for obj in failed_objects:
+            properties = obj.object_.properties or {}
+            # We get the object uuid just in case the _original_id is not present.
+            # That's extremely unlikely to happen but let's stay on the safe side.
+            id_ = properties.get("_original_id", obj.object_.uuid)
+            mapped_objects[id_] = obj.message
+
+        msg = "\n".join(
+            [f"Failed to write object with id '{id_}'. Error: '{message}'" for id_, message in mapped_objects.items()]
+        )
+        raise DocumentStoreError(msg)
 
     def _batch_write(self, documents: list[Document]) -> int:
         """
@@ -876,22 +960,35 @@ class WeaviateDocumentStore:
                     vector=doc.embedding,
                 )
         if failed_objects := self.client.batch.failed_objects:
-            # We fall back to use the UUID if the _original_id is not present, this is just to be
-            mapped_objects = {}
-            for obj in failed_objects:
-                properties = obj.object_.properties or {}
-                # We get the object uuid just in case the _original_id is not present.
-                # That's extremely unlikely to happen but let's stay on the safe side.
-                id_ = properties.get("_original_id", obj.object_.uuid)
-                mapped_objects[id_] = obj.message
-
-            msg = "\n".join(
-                [
-                    f"Failed to write object with id '{id_}'. Error: '{message}'"
-                    for id_, message in mapped_objects.items()
-                ]
-            )
-            raise DocumentStoreError(msg)
+            self._handle_failed_objects(failed_objects)
+
+        # If the document already exists we get no status message back from Weaviate.
+        # So we assume that all Documents were written.
+        return len(documents)
+
+    async def _batch_write_async(self, documents: list[Document]) -> int:
+        """
+        Asynchronously writes document to Weaviate in batches.
+        Documents with the same id will be overwritten.
+        Raises in case of errors.
+        """
+        client = await self.async_client
+
+        async with client.batch.stream() as batch:
+            for doc in documents:
+                if not isinstance(doc, Document):
+                    msg = f"Expected a Document, got '{type(doc)}' instead."
+                    raise ValueError(msg)
+
+                await batch.add_object(
+                    properties=WeaviateDocumentStore._to_data_object(doc),
+                    collection=(await self.async_collection).name,
+                    uuid=generate_uuid5(doc.id),
+                    vector=doc.embedding,
+                )
+
+        if failed_objects := client.batch.failed_objects:
+            self._handle_failed_objects(failed_objects)
 
         # If the document already exists we get no status message back from Weaviate.
         # So we assume that all Documents were written.
@@ -931,6 +1028,43 @@ class WeaviateDocumentStore:
             raise DuplicateDocumentError(msg)
         return written
 
+    async def _write_async(self, documents: list[Document], policy: DuplicatePolicy) -> int:
+        """
+        Asynchronously writes documents to Weaviate using the specified policy.
+        This doesn't use the batch API, so it's slower than _batch_write.
+        If policy is set to SKIP it will skip any document that already exists.
+        If policy is set to FAIL it will raise an exception if any of the documents already exists.
+        """
+        collection = await self.async_collection
+
+        duplicate_errors_ids = []
+        for doc in documents:
+            if not isinstance(doc, Document):
+                msg = f"Expected a Document, got '{type(doc)}' instead."
+                raise ValueError(msg)
+
+            if policy == DuplicatePolicy.SKIP and await (await self.async_collection).data.exists(
+                uuid=generate_uuid5(doc.id)
+            ):
+                # This Document already exists, continue
+                continue
+
+            try:
+                await collection.data.insert(
+                    uuid=generate_uuid5(doc.id),
+                    properties=WeaviateDocumentStore._to_data_object(doc),
+                    vector=doc.embedding,
+                )
+
+            except weaviate.exceptions.UnexpectedStatusCodeError:
+                if policy == DuplicatePolicy.FAIL:
+                    duplicate_errors_ids.append(doc.id)
+
+        if duplicate_errors_ids:
+            msg = f"IDs '{', '.join(duplicate_errors_ids)}' already exist in the document store."
+            raise DuplicateDocumentError(msg)
+        return len(documents)
+
     def write_documents(self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
         """
         Writes documents to Weaviate using the specified policy.
@@ -939,12 +1073,59 @@ class WeaviateDocumentStore:
         We can't use the batch API for other policies as it doesn't return any information whether the document
         already exists or not. That prevents us from returning errors when using the FAIL policy or skipping a
         Document when using the SKIP policy.
+
+        :param documents:
+            A list of documents to write into the document store.
+        :param policy:
+            DuplicatePolicy to apply when a document with the same ID already exists in the document store.
+
+        :raises ValueError:
+            When input is not valid.
+        :raises DuplicateDocumentError:
+            When duplicate documents are found and using a FAIL policy.
+        :raises DocumentStoreError:
+            When documents have failed to be batch written.
+
+        :returns:
+            The number of documents written.
         """
         if policy in [DuplicatePolicy.NONE, DuplicatePolicy.OVERWRITE]:
             return self._batch_write(documents)
 
         return self._write(documents, policy)
 
+    async def write_documents_async(
+        self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
+    ) -> int:
+        """
+        Asynchronously writes documents to Weaviate using the specified policy.
+        We recommend using a OVERWRITE policy as it's faster than other policies for Weaviate since it uses
+        the batch API.
+        We can't use the batch API for other policies as it doesn't return any information whether the document
+        already exists or not. That prevents us from returning errors when using the FAIL policy or skipping a
+        Document when using the SKIP policy.
+
+
+        :param documents:
+            A list of documents to write into the document store.
+        :param policy:
+            DuplicatePolicy to apply when a document with the same ID already exists in the document store.
+
+        :raises ValueError:
+            When input is not valid.
+        :raises DuplicateDocumentError:
+            When duplicate documents are found and using a FAIL policy.
+        :raises DocumentStoreError:
+            When documents have failed to be batch written.
+
+        :returns:
+            The number of documents written.
+        """
+        if policy in [DuplicatePolicy.NONE, DuplicatePolicy.OVERWRITE]:
+            return await self._batch_write_async(documents)
+
+        return await self._write_async(documents, policy)
+
     def delete_documents(self, document_ids: list[str]) -> None:
         """
         Deletes all documents with matching document_ids from the DocumentStore.
@@ -954,6 +1135,16 @@ class WeaviateDocumentStore:
         weaviate_ids = [generate_uuid5(doc_id) for doc_id in document_ids]
         self.collection.data.delete_many(where=weaviate.classes.query.Filter.by_id().contains_any(weaviate_ids))
 
+    async def delete_documents_async(self, document_ids: list[str]) -> None:
+        """
+        Asynchronously deletes all documents with matching document_ids from the DocumentStore.
+
+        :param document_ids: The object_ids to delete.
+        """
+        weaviate_ids = [generate_uuid5(doc_id) for doc_id in document_ids]
+        collection = await self.async_collection
+        await collection.data.delete_many(where=weaviate.classes.query.Filter.by_id().contains_any(weaviate_ids))
+
     def delete_all_documents(self, *, recreate_index: bool = False, batch_size: int = 1000) -> None:
         """
         Deletes all documents in a collection.
@@ -1003,6 +1194,57 @@ class WeaviateDocumentStore:
                     "Make sure to specify a deletion `batch_size` which is less than `QUERY_MAXIMUM_RESULTS`.",
                 )
 
+    async def delete_all_documents_async(self, *, recreate_index: bool = False, batch_size: int = 1000) -> None:
+        """
+        Asynchronously deletes all documents in a collection.
+
+        If recreate_index is False, it keeps the collection but deletes documents iteratively.
+        If recreate_index is True, the collection is dropped and faithfully recreated.
+        This is recommended for performance reasons.
+
+        :param recreate_index: Use drop and recreate strategy. (recommended for performance)
+        :param batch_size: Only relevant if recreate_index is false. Defines the deletion batch size.
+            Note that this parameter needs to be less or equal to the set `QUERY_MAXIMUM_RESULTS` variable
+            set for the weaviate deployment (default is 10000).
+            Reference: https://docs.weaviate.io/weaviate/manage-objects/delete#delete-all-objects
+        """
+        client = await self.async_client
+
+        if recreate_index:
+            # get current up-to-date config from server, so we can recreate the collection faithfully
+            cfg = (await client.collections.get(self._collection_settings["class"]).config.get()).to_dict()
+            class_name = cfg.get("class", self._collection_settings["class"])
+
+            await client.collections.delete(class_name)
+            await client.collections.create_from_dict(cfg)
+
+            self._collection_settings = cfg
+            self._async_collection = client.collections.get(class_name)
+            return
+
+        uuids = []
+        batch_size = max(1, int(batch_size))
+
+        collection = await self.async_collection
+        async for obj in collection.iterator(return_properties=[], include_vector=False):
+            uuids.append(obj.uuid)
+            if len(uuids) >= batch_size:
+                res = await collection.data.delete_many(where=weaviate.classes.query.Filter.by_id().contains_any(uuids))
+                if res.successful < len(uuids):
+                    logger.warning(
+                        "Not all documents in the batch have been deleted. "
+                        "Make sure to specify a deletion `batch_size` which is less than `QUERY_MAXIMUM_RESULTS`.",
+                    )
+                uuids.clear()
+
+        if uuids:
+            res = await collection.data.delete_many(where=weaviate.classes.query.Filter.by_id().contains_any(uuids))
+            if res.successful < len(uuids):
+                logger.warning(
+                    "Not all documents have been deleted. "
+                    "Make sure to specify a deletion `batch_size` which is less than `QUERY_MAXIMUM_RESULTS`.",
+                )
+
     def delete_by_filter(self, filters: dict[str, Any]) -> int:
         """
         Deletes all documents that match the provided filters.
@@ -1095,11 +1337,9 @@ class WeaviateDocumentStore:
 
                     # Update the object, preserving the vector
                     # Get the vector from the object to preserve it during replace
-                    vector = None
-                    if isinstance(obj.vector, list):
+                    vector: VECTORS | None = None
+                    if isinstance(obj.vector, (list, dict)):
                         vector = obj.vector
-                    elif isinstance(obj.vector, dict):
-                        vector = obj.vector.get("default")
 
                     self.collection.data.replace(
                         uuid=obj.uuid,
@@ -1191,11 +1431,9 @@ class WeaviateDocumentStore:
 
                     # Update the object, preserving the vector
                     # Get the vector from the object to preserve it during replace
-                    vector = None
-                    if isinstance(obj.vector, list):
+                    vector: VECTORS | None = None
+                    if isinstance(obj.vector, (list, dict)):
                         vector = obj.vector
-                    elif isinstance(obj.vector, dict):
-                        vector = obj.vector.get("default")
 
                     await collection.data.replace(
                         uuid=obj.uuid,
@@ -1322,7 +1560,7 @@ class WeaviateDocumentStore:
         query_embedding: list[float],
         filters: dict[str, Any] | None = None,
         top_k: int | None = None,
-        alpha: float | None = None,
+        alpha: float = 0.7,
         max_vector_distance: float | None = None,
     ) -> list[Document]:
         properties = [p.name for p in self.collection.config.get().properties]
@@ -1347,7 +1585,7 @@ class WeaviateDocumentStore:
         query_embedding: list[float],
         filters: dict[str, Any] | None = None,
         top_k: int | None = None,
-        alpha: float | None = None,
+        alpha: float = 0.7,
         max_vector_distance: float | None = None,
     ) -> list[Document]:
         collection = await self.async_collection

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/tests/test_document_store.py

@@ -464,29 +464,32 @@ class TestWeaviateDocumentStore(DocumentStoreBaseExtendedTests):
             ],
         )
 
-    def test_meta_split_overlap_is_skipped(self, document_store):
+    def test_split_overlap_preserved(self, document_store):
+        """Split overlap meta is written and read back correctly."""
+        overlap = [
+            {"range": [3.0, 13.0], "doc_id": "34326b7e6be489cb4c031152fc378cb50479ca5fcc3861e7e61dfb2e4e4e968b"},
+            {"range": [0.0, 13.0], "doc_id": "780f791c09d499c0bf01f87bce047b45c44224d36c79f0c9d8c1405a3197fc1a"},
+        ]
         doc = Document(
-            content="The moonlight shimmered ",
+            id="6edd24e8b01f3cd6e4b71fef7d57b52f17664e14db5ab01b8ef429f97add3620",
+            content="an eighth test. ",
             meta={
-                "source_id": "62049ba1d1e1d5ebb1f6230b0b00c5356b8706c56e0b9c36b1dfc86084cd75f0",
-                "page_number": 1,
-                "split_id": 0,
-                "split_idx_start": 0,
-                "_split_overlap": [
-                    {"doc_id": "68ed48ba830048c5d7815874ed2de794722e6d10866b6c55349a914fd9a0df65", "range": (0, 20)}
-                ],
+                "_split_overlap": overlap,
+                "page_number": 1.0,
+                "split_id": 33.0,
+                "split_idx_start": 159.0,
+                "source_id": "fdbde6d217f04d3dd60c01f36541794f3153a61f13b4ca669655f4c5610c1664",
             },
         )
         document_store.write_documents([doc])
-
         written_doc = document_store.filter_documents()[0]
-
-        assert written_doc.content == "The moonlight shimmered "
-        assert written_doc.meta["source_id"] == "62049ba1d1e1d5ebb1f6230b0b00c5356b8706c56e0b9c36b1dfc86084cd75f0"
-        assert written_doc.meta["page_number"] == 1.0
-        assert written_doc.meta["split_id"] == 0.0
-        assert written_doc.meta["split_idx_start"] == 0.0
-        assert "_split_overlap" not in written_doc.meta
+        assert "_split_overlap" in written_doc.meta
+        written_overlap = written_doc.meta["_split_overlap"]
+        assert len(written_overlap) == 2
+        assert written_overlap[0]["doc_id"] == overlap[0]["doc_id"]
+        assert list(written_overlap[0]["range"]) == [3, 13]
+        assert written_overlap[1]["doc_id"] == overlap[1]["doc_id"]
+        assert list(written_overlap[1]["range"]) == [0, 13]
 
     def test_bm25_retrieval(self, document_store):
         document_store.write_documents(

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/tests/test_document_store_async.py

@@ -2,11 +2,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import logging
 from collections.abc import AsyncGenerator
+from pathlib import Path
 
 import pytest
 import pytest_asyncio
+from haystack.dataclasses.byte_stream import ByteStream
 from haystack.dataclasses.document import Document
+from haystack.document_stores.errors import DocumentStoreError
 
 from haystack_integrations.document_stores.weaviate import WeaviateDocumentStore
 from haystack_integrations.document_stores.weaviate.document_store import DOCUMENT_COLLECTION_PROPERTIES
@@ -31,8 +35,7 @@ class TestWeaviateDocumentStoreAsync:
             collection_settings=collection_settings,
         )
         yield store
-        store.client.collections.delete(collection_settings["class"])
-        store.close()
+        await (await store.async_client).collections.delete(collection_settings["class"])
         await store.close_async()
 
     @pytest.mark.asyncio
@@ -50,20 +53,68 @@ class TestWeaviateDocumentStoreAsync:
         assert await document_store.async_client is not None
         assert await document_store.async_collection is not None
 
-        document_store.write_documents(
-            [
-                Document(content="Haskell is a functional programming language"),
-                Document(content="Lisp is a functional programming language"),
-                Document(content="Python is an object oriented programming language"),
-            ]
+        assert await document_store.count_documents_async() == 0
+
+    @pytest.mark.asyncio
+    async def test_filter_documents_with_blob_data_async(
+        self, document_store: WeaviateDocumentStore, test_files_path: Path
+    ) -> None:
+        image = ByteStream.from_file_path(test_files_path / "robot1.jpg", mime_type="image/jpeg")
+        doc = Document(content="test doc", blob=image)
+        assert await document_store.write_documents_async([doc]) == 1
+
+        docs = await document_store.filter_documents_async()
+
+        assert len(docs) == 1
+        assert docs[0].blob == image
+
+    @pytest.mark.asyncio
+    async def test_filter_documents_below_default_limit_async(self, document_store: WeaviateDocumentStore) -> None:
+        docs = []
+        for index in range(9998):
+            docs.append(Document(content="This is some content", meta={"index": index}))
+        await document_store.write_documents_async(docs)
+        result = await document_store.filter_documents_async(
+            {"field": "content", "operator": "==", "value": "This is some content"}
         )
-        filters = {"field": "content", "operator": "==", "value": "Haskell"}
 
-        assert await document_store.count_documents_by_filter_async(filters) == 1
+        assert len(result) == 9998
+
+    @pytest.mark.asyncio
+    async def test_filter_documents_over_default_limit(self, document_store: WeaviateDocumentStore) -> None:
+        docs = []
+        for index in range(10000):
+            docs.append(Document(content="This is some content", meta={"index": index}))
+        await document_store.write_documents_async(docs)
+        with pytest.raises(DocumentStoreError):
+            await document_store.filter_documents_async(
+                {"field": "content", "operator": "==", "value": "This is some content"}
+            )
+
+    @pytest.mark.asyncio
+    async def test_write_documents_async(self, document_store: WeaviateDocumentStore) -> None:
+        """
+        Test write_documents() with default policy overwrites existing documents.
+        """
+        doc = Document(content="test doc")
+        assert await document_store.write_documents_async([doc]) == 1
+        assert await document_store.count_documents_async() == 1
+
+        doc.content = "test doc 2"
+        assert await document_store.write_documents_async([doc]) == 1
+        assert await document_store.count_documents_async() == 1
+
+    @pytest.mark.asyncio
+    async def test_write_documents_with_blob_data_async(
+        self, document_store: WeaviateDocumentStore, test_files_path: Path
+    ) -> None:
+        image = ByteStream.from_file_path(test_files_path / "robot1.jpg", mime_type="image/jpeg")
+        doc = Document(content="test doc", blob=image)
+        assert await document_store.write_documents_async([doc]) == 1
 
     @pytest.mark.asyncio
     async def test_bm25_retrieval_async(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="Haskell is a functional programming language"),
                 Document(content="Lisp is a functional programming language"),
@@ -80,7 +131,7 @@ class TestWeaviateDocumentStoreAsync:
 
     @pytest.mark.asyncio
     async def test_bm25_retrieval_async_with_filters(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="Haskell is a functional programming language"),
                 Document(content="Lisp is a functional programming language"),
@@ -96,7 +147,7 @@ class TestWeaviateDocumentStoreAsync:
 
     @pytest.mark.asyncio
     async def test_embedding_retrieval_async(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="The document", embedding=[1.0, 1.0, 1.0, 1.0]),
                 Document(content="Another document", embedding=[0.8, 0.8, 0.8, 1.0]),
@@ -112,7 +163,7 @@ class TestWeaviateDocumentStoreAsync:
 
     @pytest.mark.asyncio
     async def test_embedding_retrieval_async_with_filters(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="The document I want", embedding=[1.0, 1.0, 1.0, 1.0]),
                 Document(content="Another document", embedding=[0.8, 0.8, 0.8, 1.0]),
@@ -134,7 +185,7 @@ class TestWeaviateDocumentStoreAsync:
 
     @pytest.mark.asyncio
     async def test_hybrid_retrieval_async(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="Haskell is a functional programming language", embedding=[1.0, 0.8, 0.2, 0.1]),
                 Document(content="Lisp is a functional programming language", embedding=[0.9, 0.7, 0.3, 0.2]),
@@ -154,7 +205,7 @@ class TestWeaviateDocumentStoreAsync:
 
     @pytest.mark.asyncio
     async def test_hybrid_retrieval_async_with_filters(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="Haskell is a functional programming language", embedding=[1.0, 0.8, 0.2, 0.1]),
                 Document(content="Lisp is a functional programming language", embedding=[0.9, 0.7, 0.3, 0.2]),
@@ -173,7 +224,7 @@ class TestWeaviateDocumentStoreAsync:
 
     @pytest.mark.asyncio
     async def test_hybrid_retrieval_async_with_alpha(self, document_store):
-        document_store.write_documents(
+        await document_store.write_documents_async(
             [
                 Document(content="Haskell is a functional programming language", embedding=[1.0, 0.8, 0.2, 0.1]),
                 Document(content="Python is an object oriented language", embedding=[0.1, 0.2, 0.8, 0.9]),
@@ -205,7 +256,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 2", meta={"category": "TypeB"}),
             Document(content="Doc 3", meta={"category": "TypeA"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         # delete documents with category="TypeA"
         deleted_count = await document_store.delete_by_filter_async(
@@ -215,7 +266,7 @@ class TestWeaviateDocumentStoreAsync:
         assert document_store.count_documents() == 1
 
         # verify only category TypeB remains
-        remaining_docs = document_store.filter_documents()
+        remaining_docs = await document_store.filter_documents_async()
         assert len(remaining_docs) == 1
         assert remaining_docs[0].meta["category"] == "TypeB"
 
@@ -226,7 +277,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 2", meta={"category": "TypeB", "status": "draft"}),
             Document(content="Doc 3", meta={"category": "TypeA", "status": "draft"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
         assert document_store.count_documents() == 3
 
         # update status for category="TypeA" documents
@@ -236,7 +287,7 @@ class TestWeaviateDocumentStoreAsync:
         assert updated_count == 2
 
         # Verify the updates
-        published_docs = document_store.filter_documents(
+        published_docs = await document_store.filter_documents_async(
             filters={"field": "meta.status", "operator": "==", "value": "published"}
         )
         assert len(published_docs) == 2
@@ -257,7 +308,7 @@ class TestWeaviateDocumentStoreAsync:
                     meta={"index": index, "status": "draft", "category": "test"},
                 )
             )
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         # update all documents should trigger pagination (3 pages)
         updated_count = await document_store.update_by_filter_async(
@@ -266,7 +317,7 @@ class TestWeaviateDocumentStoreAsync:
         )
         assert updated_count == 250
 
-        published_docs = document_store.filter_documents(
+        published_docs = await document_store.filter_documents_async(
             filters={"field": "status", "operator": "==", "value": "published"}
         )
         assert len(published_docs) == 250
@@ -276,6 +327,17 @@ class TestWeaviateDocumentStoreAsync:
             assert "index" in doc.meta
             assert 0 <= doc.meta["index"] < 250
 
+    @pytest.mark.asyncio
+    async def test_count_documents_async(self, document_store: WeaviateDocumentStore) -> None:
+        docs = [
+            Document(content="Doc 1", meta={"category": "TypeA"}),
+            Document(content="Doc 2", meta={"category": "TypeB"}),
+            Document(content="Doc 3", meta={"category": "TypeA"}),
+            Document(content="Doc 4", meta={"category": "TypeA"}),
+        ]
+        await document_store.write_documents_async(docs)
+        assert await document_store.count_documents_async() == 4
+
     @pytest.mark.asyncio
     async def test_count_documents_by_filter_async(self, document_store):
         docs = [
@@ -284,8 +346,8 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 3", meta={"category": "TypeA"}),
             Document(content="Doc 4", meta={"category": "TypeA"}),
         ]
-        document_store.write_documents(docs)
-        assert document_store.count_documents() == 4
+        await document_store.write_documents_async(docs)
+        assert await document_store.count_documents_async() == 4
 
         count = await document_store.count_documents_by_filter_async(
             filters={"field": "meta.category", "operator": "==", "value": "TypeA"}
@@ -325,7 +387,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 3", meta={"number": 20}),
             Document(content="Doc 4", meta={"number": 15}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         result = await document_store.get_metadata_field_min_max_async("number")
         assert result["min"] == 5
@@ -337,7 +399,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 1", meta={"number": 100}),
             Document(content="Doc 2", meta={"number": 200}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         result = await document_store.get_metadata_field_min_max_async("meta.number")
         assert result["min"] == 100
@@ -362,7 +424,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 4", meta={"category": "TypeC", "status": "published"}),
             Document(content="Doc 5", meta={"category": "TypeA", "status": "archived"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         result = await document_store.count_unique_metadata_by_filter_async(
             filters={"field": "meta.category", "operator": "==", "value": "TypeA"}, metadata_fields=["status"]
@@ -388,7 +450,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 1", meta={"category": "TypeA"}),
             Document(content="Doc 2", meta={"category": "TypeB"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         result = await document_store.count_unique_metadata_by_filter_async(
             filters={"field": "meta.category", "operator": "in", "value": ["TypeA", "TypeB"]},
@@ -401,7 +463,7 @@ class TestWeaviateDocumentStoreAsync:
         docs = [
             Document(content="Doc 1", meta={"category": "TypeA"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         result = await document_store.count_unique_metadata_by_filter_async(
             filters={"field": "meta.category", "operator": "==", "value": "NonExistent"},
@@ -426,7 +488,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 4", meta={"category": "TypeC"}),
             Document(content="Doc 5", meta={"category": "TypeB"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         values, total_count = await document_store.get_metadata_field_unique_values_async("category")
         assert total_count == 3
@@ -438,7 +500,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 1", meta={"category": "TypeA"}),
             Document(content="Doc 2", meta={"category": "TypeB"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         values, total_count = await document_store.get_metadata_field_unique_values_async("meta.category")
         assert total_count == 2
@@ -452,7 +514,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Python is great", meta={"category": "TypeC"}),
             Document(content="JavaScript tutorial", meta={"category": "TypeD"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         values, total_count = await document_store.get_metadata_field_unique_values_async(
             "category", search_term="Python"
@@ -469,7 +531,7 @@ class TestWeaviateDocumentStoreAsync:
             Document(content="Doc 4", meta={"category": "TypeD"}),
             Document(content="Doc 5", meta={"category": "TypeE"}),
         ]
-        document_store.write_documents(docs)
+        await document_store.write_documents_async(docs)
 
         values, total_count = await document_store.get_metadata_field_unique_values_async("category", from_=0, size=2)
         assert total_count == 5
@@ -491,3 +553,39 @@ class TestWeaviateDocumentStoreAsync:
         values, total_count = await document_store.get_metadata_field_unique_values_async("category")
         assert total_count == 0
         assert values == []
+
+    @pytest.mark.asyncio
+    async def test_delete_all_documents_recreate_async(self, document_store: WeaviateDocumentStore) -> None:
+        docs = [Document(content="test doc 1"), Document(content="test doc 2")]
+        assert await document_store.write_documents_async(docs) == 2
+        assert await document_store.count_documents_async() == 2
+
+        cls = document_store._collection_settings["class"]
+        collection = (await document_store.async_client).collections.get(cls)
+        previous_config = (await collection.config.get()).to_dict()
+
+        await document_store.delete_all_documents_async(recreate_index=True)
+        assert await document_store.count_documents_async() == 0
+
+        new_config = (await (await document_store.async_client).collections.get(cls).config.get()).to_dict()
+        assert previous_config == new_config
+
+    @pytest.mark.asyncio
+    async def test_delete_all_documents_batch_size_async(self, document_store: WeaviateDocumentStore) -> None:
+        docs = [Document(content=str(i)) for i in range(0, 5)]
+        assert await document_store.write_documents_async(docs) == 5
+        await document_store.delete_all_documents_async(batch_size=2)
+        assert await document_store.count_documents_async() == 0
+
+    @pytest.mark.asyncio
+    async def test_delete_all_documents_excessive_batch_size_async(
+        self, document_store: WeaviateDocumentStore, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Test that the deletion is not complete if the batch size exceeds the QUERY_MAXIMUM_RESULTS."""
+        # assume QUERY_MAXIMUM_RESULTS == 10000 with standard deployment
+        docs = [Document(content=str(i)) for i in range(0, 10005)]
+        assert await document_store.write_documents_async(docs) == 10005
+        with caplog.at_level(logging.WARNING):
+            await document_store.delete_all_documents_async(batch_size=20000)
+        assert await document_store.count_documents_async() == 5
+        assert "Not all documents have been deleted." in caplog.text

{weaviate_haystack-7.2.0 → weaviate_haystack-7.4.0}/tests/test_hybrid_retriever.py

@@ -17,7 +17,7 @@ def test_init_default():
     assert retriever._document_store == mock_document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
-    assert retriever._alpha is None
+    assert retriever._alpha == 0.7
     assert retriever._max_vector_distance is None
     assert retriever._filter_policy == FilterPolicy.REPLACE
 
@@ -56,7 +56,7 @@ def test_to_dict(_mock_weaviate):
         "init_parameters": {
             "filters": {},
             "top_k": 10,
-            "alpha": None,
+            "alpha": 0.7,
             "max_vector_distance": None,
             "filter_policy": "replace",
             "document_store": {
@@ -112,7 +112,7 @@ def test_from_dict(_mock_weaviate):
             "init_parameters": {
                 "filters": {},
                 "top_k": 10,
-                "alpha": None,
+                "alpha": 0.7,
                 "max_vector_distance": None,
                 "filter_policy": "replace",
                 "document_store": {
@@ -142,7 +142,7 @@ def test_from_dict(_mock_weaviate):
     assert retriever._document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
-    assert retriever._alpha is None
+    assert retriever._alpha == 0.7
     assert retriever._max_vector_distance is None
 
 
@@ -200,7 +200,12 @@ def test_run_basic():
     assert "documents" in result
     assert len(result["documents"]) == 1
     mock_document_store._hybrid_retrieval.assert_called_once_with(
-        query="test query", query_embedding=[0.1, 0.2, 0.3], filters={}, top_k=10, alpha=None, max_vector_distance=None
+        query="test query",
+        query_embedding=[0.1, 0.2, 0.3],
+        filters={},
+        top_k=10,
+        alpha=0.7,
+        max_vector_distance=None,
     )
 
 
@@ -217,7 +222,7 @@ def test_run_with_runtime_filters():
         query_embedding=[0.1, 0.2, 0.3],
         filters={"runtime": "filter"},
         top_k=10,
-        alpha=None,
+        alpha=0.7,
         max_vector_distance=None,
     )
 
@@ -259,7 +264,7 @@ def test_run_empty_query():
     assert "documents" in result
     assert len(result["documents"]) == 0
     mock_document_store._hybrid_retrieval.assert_called_once_with(
-        query="", query_embedding=[0.1, 0.2, 0.3], filters={}, top_k=10, alpha=None, max_vector_distance=None
+        query="", query_embedding=[0.1, 0.2, 0.3], filters={}, top_k=10, alpha=0.7, max_vector_distance=None
     )
 
 
@@ -288,7 +293,7 @@ def test_from_dict_no_filter_policy(_mock_weaviate):
             "init_parameters": {
                 "filters": {},
                 "top_k": 10,
-                "alpha": None,
+                "alpha": 0.7,
                 "max_vector_distance": None,
                 # filter_policy intentionally omitted
                 "document_store": {
@@ -318,7 +323,7 @@ def test_from_dict_no_filter_policy(_mock_weaviate):
     assert retriever._document_store
     assert retriever._filters == {}
     assert retriever._top_k == 10
-    assert retriever._alpha is None
+    assert retriever._alpha == 0.7
     assert retriever._max_vector_distance is None
     assert retriever._filter_policy == FilterPolicy.REPLACE
 
@@ -381,7 +386,7 @@ def test_run_with_max_vector_distance_zero_runtime():
         query_embedding=[0.1, 0.2],
         filters={},
         top_k=10,
-        alpha=None,
+        alpha=0.7,
         max_vector_distance=0.0,
     )
 
@@ -402,7 +407,7 @@ def test_run_with_max_vector_distance_zero_init_and_none_runtime():
         query_embedding=[0.1, 0.2],
         filters={},
         top_k=10,
-        alpha=None,
+        alpha=0.7,
         max_vector_distance=0.0,
     )