weaviate-haystack 4.0.2__tar.gz → 5.0.0__tar.gz

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [integrations/weaviate-v4.0.2] - 2024-11-13
+
+### 🐛 Bug Fixes
+
+- Dependency for weaviate document store (#1186)
+
 ## [integrations/weaviate-v4.0.1] - 2024-11-11
 
 ## [integrations/weaviate-v4.0.0] - 2024-10-18

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/PKG-INFO

@@ -1,16 +1,16 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: weaviate-haystack
-Version: 4.0.2
+Version: 5.0.0
 Summary: An integration of Weaviate vector database with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
 Author-email: deepset GmbH <info@deepset.ai>
-License: Apache-2.0
+License-Expression: Apache-2.0
+License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/pyproject.toml

@@ -15,7 +15,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
@@ -160,3 +159,6 @@ module = [
   "grpc",
 ]
 ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+markers = ["integration: integration tests"]

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/src/haystack_integrations/document_stores/weaviate/_filters.py

@@ -6,7 +6,6 @@ from typing import Any, Dict
 
 from dateutil import parser
 from haystack.errors import FilterError
-from pandas import DataFrame
 
 import weaviate
 from weaviate.collections.classes.filters import Filter, FilterReturn
@@ -129,7 +128,7 @@ def _greater_than(field: str, value: Any) -> FilterReturn:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return weaviate.classes.query.Filter.by_property(field).greater_than(_handle_date(value))
@@ -151,7 +150,7 @@ def _greater_than_equal(field: str, value: Any) -> FilterReturn:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return weaviate.classes.query.Filter.by_property(field).greater_or_equal(_handle_date(value))
@@ -173,7 +172,7 @@ def _less_than(field: str, value: Any) -> FilterReturn:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return weaviate.classes.query.Filter.by_property(field).less_than(_handle_date(value))
@@ -195,7 +194,7 @@ def _less_than_equal(field: str, value: Any) -> FilterReturn:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return weaviate.classes.query.Filter.by_property(field).less_or_equal(_handle_date(value))
@@ -248,8 +247,6 @@ def _parse_comparison_condition(condition: Dict[str, Any]) -> FilterReturn:
         raise FilterError(msg)
     operator: str = condition["operator"]
     value: Any = condition["value"]
-    if isinstance(value, DataFrame):
-        value = value.to_json()
 
     return COMPARISON_OPERATORS[operator](field, value)
 

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/src/haystack_integrations/document_stores/weaviate/document_store.py

@@ -4,10 +4,10 @@
 import base64
 import datetime
 import json
-import logging
 from dataclasses import asdict
 from typing import Any, Dict, List, Optional
 
+from haystack import logging
 from haystack.core.serialization import default_from_dict, default_to_dict
 from haystack.dataclasses.document import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
@@ -39,7 +39,6 @@ logger = logging.getLogger(__name__)
 DOCUMENT_COLLECTION_PROPERTIES = [
     {"name": "_original_id", "dataType": ["text"]},
     {"name": "content", "dataType": ["text"]},
-    {"name": "dataframe", "dataType": ["text"]},
     {"name": "blob_data", "dataType": ["blob"]},
     {"name": "blob_mime_type", "dataType": ["text"]},
     {"name": "score", "dataType": ["number"]},
@@ -105,7 +104,6 @@ class WeaviateDocumentStore:
             properties:
             - _original_id: text
             - content: text
-            - dataframe: text
             - blob_data: blob
             - blob_mime_type: text
             - score: number
@@ -291,17 +289,28 @@ class WeaviateDocumentStore:
         if "_split_overlap" in data:
             data.pop("_split_overlap")
             logger.warning(
-                "Document %s has the unsupported `_split_overlap` meta field. It will be ignored.", data["_original_id"]
+                "Document {id} has the unsupported `_split_overlap` meta field. It will be ignored.",
+                id=data["_original_id"],
             )
 
+        if "dataframe" in data:
+            dataframe = data.pop("dataframe")
+            if dataframe:
+                logger.warning(
+                    "Document {id} has the `dataframe` field set. "
+                    "WeaviateDocumentStore no longer supports dataframes and this field will be ignored. "
+                    "The `dataframe` field will soon be removed from Haystack Document.",
+                    id=data["_original_id"],
+                )
+
         if "sparse_embedding" in data:
             sparse_embedding = data.pop("sparse_embedding", None)
             if sparse_embedding:
                 logger.warning(
-                    "Document %s has the `sparse_embedding` field set,"
+                    "Document {id} has the `sparse_embedding` field set,"
                     "but storing sparse embeddings in Weaviate is not currently supported."
                     "The `sparse_embedding` field will be ignored.",
-                    data["_original_id"],
+                    id=data["_original_id"],
                 )
 
         return data
@@ -329,6 +338,16 @@ class WeaviateDocumentStore:
         document_data.pop("blob_data", None)
         document_data.pop("blob_mime_type", None)
 
+        if "dataframe" in document_data:
+            dataframe = document_data.pop("dataframe")
+            if dataframe:
+                logger.warning(
+                    "Document {id} has the `dataframe` field set. "
+                    "WeaviateDocumentStore no longer supports dataframes and this field will be ignored. "
+                    "The `dataframe` field will soon be removed from Haystack Document.",
+                    id=document_data["id"],
+                )
+
         for key, value in document_data.items():
             if isinstance(value, datetime.datetime):
                 document_data[key] = value.strftime("%Y-%m-%dT%H:%M:%SZ")

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/tests/test_bm25_retriever.py

@@ -46,7 +46,6 @@ def test_to_dict(_mock_weaviate):
                         "properties": [
                             {"name": "_original_id", "dataType": ["text"]},
                             {"name": "content", "dataType": ["text"]},
-                            {"name": "dataframe", "dataType": ["text"]},
                             {"name": "blob_data", "dataType": ["blob"]},
                             {"name": "blob_mime_type", "dataType": ["text"]},
                             {"name": "score", "dataType": ["number"]},
@@ -81,7 +80,6 @@ def test_from_dict(_mock_weaviate):
                             "properties": [
                                 {"name": "_original_id", "dataType": ["text"]},
                                 {"name": "content", "dataType": ["text"]},
-                                {"name": "dataframe", "dataType": ["text"]},
                                 {"name": "blob_data", "dataType": ["blob"]},
                                 {"name": "blob_mime_type", "dataType": ["text"]},
                                 {"name": "score", "dataType": ["number"]},
@@ -119,7 +117,6 @@ def test_from_dict_no_filter_policy(_mock_weaviate):
                             "properties": [
                                 {"name": "_original_id", "dataType": ["text"]},
                                 {"name": "content", "dataType": ["text"]},
-                                {"name": "dataframe", "dataType": ["text"]},
                                 {"name": "blob_data", "dataType": ["blob"]},
                                 {"name": "blob_mime_type", "dataType": ["text"]},
                                 {"name": "score", "dataType": ["number"]},

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/tests/test_document_store.py

@@ -4,7 +4,6 @@
 
 import base64
 import os
-import random
 from typing import List
 from unittest.mock import MagicMock, patch
 
@@ -14,12 +13,11 @@ from haystack.dataclasses.byte_stream import ByteStream
 from haystack.dataclasses.document import Document
 from haystack.document_stores.errors import DocumentStoreError
 from haystack.testing.document_store import (
-    TEST_EMBEDDING_1,
-    TEST_EMBEDDING_2,
     CountDocumentsTest,
     DeleteDocumentsTest,
     FilterDocumentsTest,
     WriteDocumentsTest,
+    create_filterable_docs,
 )
 from haystack.utils.auth import Secret
 from numpy import array as np_array
@@ -78,60 +76,10 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
         Weaviate forces RFC 3339 date strings.
         The original fixture uses ISO 8601 date strings.
         """
-        documents = []
-        for i in range(3):
-            documents.append(
-                Document(
-                    content=f"A Foo Document {i}",
-                    meta={
-                        "name": f"name_{i}",
-                        "page": "100",
-                        "chapter": "intro",
-                        "number": 2,
-                        "date": "1969-07-21T20:17:40Z",
-                    },
-                    embedding=[random.random() for _ in range(768)],  # noqa: S311
-                )
-            )
-            documents.append(
-                Document(
-                    content=f"A Bar Document {i}",
-                    meta={
-                        "name": f"name_{i}",
-                        "page": "123",
-                        "chapter": "abstract",
-                        "number": -2,
-                        "date": "1972-12-11T19:54:58Z",
-                    },
-                    embedding=[random.random() for _ in range(768)],  # noqa: S311
-                )
-            )
-            documents.append(
-                Document(
-                    content=f"A Foobar Document {i}",
-                    meta={
-                        "name": f"name_{i}",
-                        "page": "90",
-                        "chapter": "conclusion",
-                        "number": -10,
-                        "date": "1989-11-09T17:53:00Z",
-                    },
-                    embedding=[random.random() for _ in range(768)],  # noqa: S311
-                )
-            )
-            documents.append(
-                Document(
-                    content=f"Document {i} without embedding",
-                    meta={"name": f"name_{i}", "no_embedding": True, "chapter": "conclusion"},
-                )
-            )
-            documents.append(Document(dataframe=DataFrame([i]), meta={"name": f"table_doc_{i}"}))
-            documents.append(
-                Document(content=f"Doc {i} with zeros emb", meta={"name": "zeros_doc"}, embedding=TEST_EMBEDDING_1)
-            )
-            documents.append(
-                Document(content=f"Doc {i} with ones emb", meta={"name": "ones_doc"}, embedding=TEST_EMBEDDING_2)
-            )
+        documents = create_filterable_docs()
+        for i in range(len(documents)):
+            if date := documents[i].meta.get("date"):
+                documents[i].meta["date"] = f"{date}Z"
         return documents
 
     def assert_documents_are_equal(self, received: List[Document], expected: List[Document]):
@@ -238,7 +186,6 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
                     "properties": [
                         {"name": "_original_id", "dataType": ["text"]},
                         {"name": "content", "dataType": ["text"]},
-                        {"name": "dataframe", "dataType": ["text"]},
                         {"name": "blob_data", "dataType": ["blob"]},
                         {"name": "blob_mime_type", "dataType": ["text"]},
                         {"name": "score", "dataType": ["number"]},
@@ -320,7 +267,6 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
             "properties": [
                 {"name": "_original_id", "dataType": ["text"]},
                 {"name": "content", "dataType": ["text"]},
-                {"name": "dataframe", "dataType": ["text"]},
                 {"name": "blob_data", "dataType": ["blob"]},
                 {"name": "blob_mime_type", "dataType": ["text"]},
                 {"name": "score", "dataType": ["number"]},
@@ -348,7 +294,6 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
         assert data == {
             "_original_id": doc.id,
             "content": doc.content,
-            "dataframe": None,
             "score": None,
         }
 
@@ -365,11 +310,20 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
             "content": doc.content,
             "blob_data": base64.b64encode(image.data).decode(),
             "blob_mime_type": "image/jpeg",
-            "dataframe": None,
             "score": None,
             "key": "value",
         }
 
+    def test_to_data_object_skips_dataframe(self, document_store):
+        doc = Document(id="test_id", content="test content")
+        doc.dataframe = DataFrame([{"a": 1, "b": 2}, {"a": 3, "b": 4}])
+
+        data = document_store._to_data_object(doc)
+
+        assert data["_original_id"] == "test_id"
+        assert data["content"] == "test content"
+        assert "dataframe" not in data
+
     def test_to_document(self, document_store, test_files_path):
         image = ByteStream.from_file_path(test_files_path / "robot1.jpg", mime_type="image/jpeg")
         data = DataObject(
@@ -378,7 +332,6 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
                 "content": "some content",
                 "blob_data": base64.b64encode(image.data).decode(),
                 "blob_mime_type": "image/jpeg",
-                "dataframe": None,
                 "score": None,
                 "key": "value",
             },
@@ -393,6 +346,20 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
         assert doc.score is None
         assert doc.meta == {"key": "value"}
 
+    def test_to_document_skips_dataframe(self, document_store):
+        data = DataObject(
+            properties={
+                "_original_id": "test_id",
+                "content": "test content",
+                "dataframe": {"a": [1, 2, 3]},
+            }
+        )
+
+        doc = document_store._to_document(data)
+        assert doc.id == "test_id"
+        assert doc.content == "test content"
+        assert not hasattr(doc, "dataframe") or doc.dataframe is None
+
     def test_write_documents(self, document_store):
         """
         Test write_documents() with default policy overwrites existing documents.
@@ -504,10 +471,6 @@ class TestWeaviateDocumentStore(CountDocumentsTest, WriteDocumentsTest, DeleteDo
             ],
         )
 
-    @pytest.mark.skip(reason="Weaviate for some reason is not returning what we expect")
-    def test_comparison_not_equal_with_dataframe(self, document_store, filterable_docs):
-        return super().test_comparison_not_equal_with_dataframe(document_store, filterable_docs)
-
     def test_meta_split_overlap_is_skipped(self, document_store):
         doc = Document(
             content="The moonlight shimmered ",

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/tests/test_embedding_retriever.py

@@ -56,7 +56,6 @@ def test_to_dict(_mock_weaviate):
                         "properties": [
                             {"name": "_original_id", "dataType": ["text"]},
                             {"name": "content", "dataType": ["text"]},
-                            {"name": "dataframe", "dataType": ["text"]},
                             {"name": "blob_data", "dataType": ["blob"]},
                             {"name": "blob_mime_type", "dataType": ["text"]},
                             {"name": "score", "dataType": ["number"]},
@@ -93,7 +92,6 @@ def test_from_dict(_mock_weaviate):
                             "properties": [
                                 {"name": "_original_id", "dataType": ["text"]},
                                 {"name": "content", "dataType": ["text"]},
-                                {"name": "dataframe", "dataType": ["text"]},
                                 {"name": "blob_data", "dataType": ["blob"]},
                                 {"name": "blob_mime_type", "dataType": ["text"]},
                                 {"name": "score", "dataType": ["number"]},
@@ -135,7 +133,6 @@ def test_from_dict_no_filter_policy(_mock_weaviate):
                             "properties": [
                                 {"name": "_original_id", "dataType": ["text"]},
                                 {"name": "content", "dataType": ["text"]},
-                                {"name": "dataframe", "dataType": ["text"]},
                                 {"name": "blob_data", "dataType": ["blob"]},
                                 {"name": "blob_mime_type", "dataType": ["text"]},
                                 {"name": "score", "dataType": ["number"]},

{weaviate_haystack-4.0.2 → weaviate_haystack-5.0.0}/src/haystack_integrations/document_stores/weaviate/__init__.py

@@ -5,10 +5,10 @@ from .auth import AuthApiKey, AuthBearerToken, AuthClientCredentials, AuthClient
 from .document_store import WeaviateDocumentStore
 
 __all__ = [
-    "WeaviateDocumentStore",
     "AuthApiKey",
     "AuthBearerToken",
     "AuthClientCredentials",
     "AuthClientPassword",
     "AuthCredentials",
+    "WeaviateDocumentStore",
 ]