elasticsearch-haystack 0.7.0__tar.gz → 1.0.0__tar.gz

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/CHANGELOG.md

@@ -5,6 +5,16 @@
 ### 🚀 Features
 
 - Defer the database connection to when it's needed (#766)
+- Add filter_policy to elasticsearch integration (#825)
+
+### 🐛 Bug Fixes
+
+- `ElasticSearch` - Fallback to default filter policy when deserializing retrievers without the init parameter (#898)
+
+### ⚙️ Miscellaneous Tasks
+
+- Retry tests to reduce flakyness (#836)
+- Update ruff invocation to include check parameter (#853)
 
 ## [integrations/elasticsearch-v0.5.0] - 2024-05-24
 

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: elasticsearch-haystack
-Version: 0.7.0
+Version: 1.0.0
 Summary: Haystack 2.x Document Store for ElasticSearch
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/pyproject.toml

@@ -49,10 +49,12 @@ dependencies = [
   "haystack-pydoc-tools",
 ]
 [tool.hatch.envs.default.scripts]
-test = "pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
-test-cov = "coverage run -m pytest --reruns 3 --reruns-delay 30 -x {args:tests}"
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+test-cov-retry = "test-cov --reruns 3 --reruns-delay 30 -x"
 cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
+cov-retry = ["test-cov-retry", "cov-report"]
 docs = ["pydoc-markdown pydoc/config.yml"]
 
 [[tool.hatch.envs.all.matrix]]
@@ -63,8 +65,8 @@ detached = true
 dependencies = ["black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243"]
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
-style = ["ruff check {args:.}", "black --check --diff {args:.}"]
-fmt = ["black {args:.}", "ruff --fix {args:.}", "style"]
+style = ["ruff check {args:. --exclude tests/}", "black --check --diff {args:.}"]
+fmt = ["black {args:.}", "ruff --fix {args:. --exclude tests/}", "style"]
 all = ["style", "typing"]
 
 [tool.hatch.metadata]

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/src/haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py

@@ -7,6 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
 from haystack.document_stores.types.filter_policy import apply_filter_policy
+
 from haystack_integrations.document_stores.elasticsearch.document_store import ElasticsearchDocumentStore
 
 
@@ -108,7 +109,10 @@ class ElasticsearchBM25Retriever:
         data["init_parameters"]["document_store"] = ElasticsearchDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/src/haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py

@@ -7,6 +7,7 @@ from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
 from haystack.document_stores.types.filter_policy import apply_filter_policy
+
 from haystack_integrations.document_stores.elasticsearch.document_store import ElasticsearchDocumentStore
 
 
@@ -106,7 +107,10 @@ class ElasticsearchEmbeddingRetriever:
         data["init_parameters"]["document_store"] = ElasticsearchDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
-        data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(data["init_parameters"]["filter_policy"])
+        # Pipelines serialized with old versions of the component might not
+        # have the filter_policy field.
+        if filter_policy := data["init_parameters"].get("filter_policy"):
+            data["init_parameters"]["filter_policy"] = FilterPolicy.from_str(filter_policy)
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/src/haystack_integrations/document_stores/elasticsearch/document_store.py

@@ -12,7 +12,6 @@ from haystack import default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
-from haystack.utils.filters import convert
 from haystack.version import __version__ as haystack_version
 
 from elasticsearch import Elasticsearch, helpers  # type: ignore[import-not-found]
@@ -224,7 +223,8 @@ class ElasticsearchDocumentStore:
         :returns: List of `Document`s that match the filters.
         """
         if filters and "operator" not in filters and "conditions" not in filters:
-            filters = convert(filters)
+            msg = "Invalid filter syntax. See https://docs.haystack.deepset.ai/docs/metadata-filtering for details."
+            raise ValueError(msg)
 
         query = {"bool": {"filter": _normalize_filters(filters)}} if filters else None
         documents = self._search_documents(query=query)

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/tests/test_bm25_retriever.py

@@ -6,6 +6,7 @@ from unittest.mock import Mock, patch
 import pytest
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
+
 from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchBM25Retriever
 from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
@@ -77,6 +78,30 @@ def test_from_dict(_mock_elasticsearch_client):
     assert retriever._filter_policy == FilterPolicy.REPLACE
 
 
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_from_dict_no_filter_policy(_mock_elasticsearch_client):
+    data = {
+        "type": "haystack_integrations.components.retrievers.elasticsearch.bm25_retriever.ElasticsearchBM25Retriever",
+        "init_parameters": {
+            "document_store": {
+                "init_parameters": {"hosts": "some fake host", "index": "default"},
+                "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
+            },
+            "filters": {},
+            "fuzziness": "AUTO",
+            "top_k": 10,
+            "scale_score": True,
+        },
+    }
+    retriever = ElasticsearchBM25Retriever.from_dict(data)
+    assert retriever._document_store
+    assert retriever._filters == {}
+    assert retriever._fuzziness == "AUTO"
+    assert retriever._top_k == 10
+    assert retriever._scale_score
+    assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
+
+
 def test_run():
     mock_store = Mock(spec=ElasticsearchDocumentStore)
     mock_store._bm25_retrieval.return_value = [Document(content="Test doc")]

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/tests/test_document_store.py

@@ -12,6 +12,7 @@ from haystack.dataclasses.document import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import DocumentStoreBaseTests
+
 from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
 

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/tests/test_embedding_retriever.py

@@ -6,6 +6,7 @@ from unittest.mock import Mock, patch
 import pytest
 from haystack.dataclasses import Document
 from haystack.document_stores.types import FilterPolicy
+
 from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchEmbeddingRetriever
 from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
@@ -74,6 +75,29 @@ def test_from_dict(_mock_elasticsearch_client):
     assert retriever._num_candidates is None
 
 
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_from_dict_no_filter_policy(_mock_elasticsearch_client):
+    t = "haystack_integrations.components.retrievers.elasticsearch.embedding_retriever.ElasticsearchEmbeddingRetriever"
+    data = {
+        "type": t,
+        "init_parameters": {
+            "document_store": {
+                "init_parameters": {"hosts": "some fake host", "index": "default"},
+                "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
+            },
+            "filters": {},
+            "top_k": 10,
+            "num_candidates": None,
+        },
+    }
+    retriever = ElasticsearchEmbeddingRetriever.from_dict(data)
+    assert retriever._document_store
+    assert retriever._filters == {}
+    assert retriever._top_k == 10
+    assert retriever._num_candidates is None
+    assert retriever._filter_policy == FilterPolicy.REPLACE  # defaults to REPLACE
+
+
 def test_run():
     mock_store = Mock(spec=ElasticsearchDocumentStore)
     mock_store._embedding_retrieval.return_value = [Document(content="Test doc", embedding=[0.1, 0.2])]

{elasticsearch_haystack-0.7.0 → elasticsearch_haystack-1.0.0}/tests/test_filters.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import pytest
 from haystack.errors import FilterError
+
 from haystack_integrations.document_stores.elasticsearch.filters import _normalize_filters, _normalize_ranges
 
 filters_data = [