elasticsearch-haystack 1.0.1__tar.gz → 2.0.0__tar.gz

{elasticsearch_haystack-1.0.1 → elasticsearch_haystack-2.0.0}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [integrations/elasticsearch-v1.0.1] - 2024-10-28
+
+### ⚙️ Miscellaneous Tasks
+
+- Update changelog after removing legacy filters (#1083)
+- Update ruff linting scripts and settings (#1105)
+- Adopt uv as installer (#1142)
+
 ## [integrations/elasticsearch-v1.0.0] - 2024-09-12
 
 ### 🚀 Features
@@ -69,8 +77,6 @@ This PR will also push the docs to Readme
 
 - Fix project urls (#96)
 
-
-
 ### 🚜 Refactor
 
 - Use `hatch_vcs` to manage integrations versioning (#103)
@@ -81,15 +87,12 @@ This PR will also push the docs to Readme
 
 - Fix import and increase version (#77)
 
-
-
 ## [integrations/elasticsearch-v0.1.0] - 2023-12-04
 
 ### 🐛 Bug Fixes
 
 - Fix license headers
 
-
 ## [integrations/elasticsearch-v0.0.2] - 2023-11-29
 
 <!-- generated by git-cliff -->

{elasticsearch_haystack-1.0.1 → elasticsearch_haystack-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: elasticsearch-haystack
-Version: 1.0.1
+Version: 2.0.0
 Summary: Haystack 2.x Document Store for ElasticSearch
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues

{elasticsearch_haystack-1.0.1 → elasticsearch_haystack-2.0.0}/pyproject.toml

@@ -159,5 +159,5 @@ minversion = "6.0"
 markers = ["unit: unit tests", "integration: integration tests"]
 
 [[tool.mypy.overrides]]
-module = ["haystack.*", "haystack_integrations.*", "pytest.*"]
+module = ["haystack.*", "haystack_integrations.*", "numpy.*", "pytest.*"]
 ignore_missing_imports = true

{elasticsearch_haystack-1.0.1 → elasticsearch_haystack-2.0.0}/src/haystack_integrations/document_stores/elasticsearch/document_store.py

@@ -258,6 +258,15 @@ class ElasticsearchDocumentStore:
         elasticsearch_actions = []
         for doc in documents:
             doc_dict = doc.to_dict()
+            if "dataframe" in doc_dict:
+                dataframe = doc_dict.pop("dataframe")
+                if dataframe:
+                    logger.warning(
+                        "Document %s has the `dataframe` field set,"
+                        "ElasticsearchDocumentStore no longer supports dataframes and this field will be ignored. "
+                        "The `dataframe` field will soon be removed from Haystack Document.",
+                        doc.id,
+                    )
             if "sparse_embedding" in doc_dict:
                 sparse_embedding = doc_dict.pop("sparse_embedding", None)
                 if sparse_embedding:
@@ -322,6 +331,15 @@ class ElasticsearchDocumentStore:
             data["metadata"]["highlighted"] = hit["highlight"]
         data["score"] = hit["_score"]
 
+        if "dataframe" in data:
+            dataframe = data.pop("dataframe")
+            if dataframe:
+                logger.warning(
+                    "Document %s has the `dataframe` field set,"
+                    "ElasticsearchDocumentStore no longer supports dataframes and this field will be ignored. "
+                    "The `dataframe` field will soon be removed from Haystack Document.",
+                    data["id"],
+                )
         return Document.from_dict(data)
 
     def delete_documents(self, document_ids: List[str]) -> None:

{elasticsearch_haystack-1.0.1 → elasticsearch_haystack-2.0.0}/src/haystack_integrations/document_stores/elasticsearch/filters.py

@@ -5,7 +5,6 @@ from datetime import datetime
 from typing import Any, Dict, List
 
 from haystack.errors import FilterError
-from pandas import DataFrame
 
 
 def _normalize_filters(filters: Dict[str, Any]) -> Dict[str, Any]:
@@ -57,7 +56,7 @@ def _equal(field: str, value: Any) -> Dict[str, Any]:
                 }
             }
         }
-    if field in ["text", "dataframe"]:
+    if field == "text":
         # We want to fully match the text field.
         return {"match": {field: {"query": value, "minimum_should_match": "100%"}}}
     return {"term": {field: value}}
@@ -69,7 +68,7 @@ def _not_equal(field: str, value: Any) -> Dict[str, Any]:
 
     if isinstance(value, list):
         return {"bool": {"must_not": {"terms": {field: value}}}}
-    if field in ["text", "dataframe"]:
+    if field == "text":
         # We want to fully match the text field.
         return {"bool": {"must_not": {"match": {field: {"query": value, "minimum_should_match": "100%"}}}}}
 
@@ -92,7 +91,7 @@ def _greater_than(field: str, value: Any) -> Dict[str, Any]:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return {"range": {field: {"gt": value}}}
@@ -114,7 +113,7 @@ def _greater_than_equal(field: str, value: Any) -> Dict[str, Any]:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return {"range": {field: {"gte": value}}}
@@ -136,7 +135,7 @@ def _less_than(field: str, value: Any) -> Dict[str, Any]:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return {"range": {field: {"lt": value}}}
@@ -158,7 +157,7 @@ def _less_than_equal(field: str, value: Any) -> Dict[str, Any]:
                 "Strings are only comparable if they are ISO formatted dates."
             )
             raise FilterError(msg) from exc
-    if type(value) in [list, DataFrame]:
+    if isinstance(value, list):
         msg = f"Filter value can't be of type {type(value)} using operators '>', '>=', '<', '<='"
         raise FilterError(msg)
     return {"range": {field: {"lte": value}}}
@@ -212,8 +211,6 @@ def _parse_comparison_condition(condition: Dict[str, Any]) -> Dict[str, Any]:
         raise FilterError(msg)
     operator: str = condition["operator"]
     value: Any = condition["value"]
-    if isinstance(value, DataFrame):
-        value = value.to_json()
 
     return COMPARISON_OPERATORS[operator](field, value)
 

{elasticsearch_haystack-1.0.1 → elasticsearch_haystack-2.0.0}/tests/test_document_store.py

@@ -12,6 +12,7 @@ from haystack.dataclasses.document import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import DocumentStoreBaseTests
+from pandas import DataFrame
 
 from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
@@ -130,6 +131,31 @@ class TestDocumentStore(DocumentStoreBaseTests):
         with pytest.raises(DuplicateDocumentError):
             document_store.write_documents(docs, DuplicatePolicy.FAIL)
 
+    def test_write_documents_dataframe_ignored(self, document_store: ElasticsearchDocumentStore):
+        doc = Document(id="1", content="test")
+        doc.dataframe = DataFrame({"a": [1, 2, 3]})
+
+        document_store.write_documents([doc])
+
+        res = document_store.filter_documents()
+        assert len(res) == 1
+
+        assert res[0].id == "1"
+        assert res[0].content == "test"
+
+        assert not hasattr(res[0], "dataframe") or res[0].dataframe is None
+
+    def test_deserialize_document_dataframe_ignored(self, document_store: ElasticsearchDocumentStore):
+        hit = {
+            "_source": {"id": "1", "content": "test", "dataframe": {"a": [1, 2, 3]}},
+            "_score": 1.0,
+        }
+        doc = document_store._deserialize_document(hit)
+        assert doc.id == "1"
+        assert doc.content == "test"
+        assert doc.score == 1.0
+        assert not hasattr(doc, "dataframe") or doc.dataframe is None
+
     def test_bm25_retrieval(self, document_store: ElasticsearchDocumentStore):
         document_store.write_documents(
             [