elasticsearch-haystack 3.1.0__tar.gz → 4.1.0__tar.gz

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [integrations/elasticsearch-v4.0.0] - 2025-09-24
+
+### 🚀 Features
+
+- [**breaking**] Adding `api_token` and `apit_token_id` authentication support to `ElasticSearchDocumentStore` (#2292)
+
+### 🧹 Chores
+
+- Remove black (#1985)
+- Standardize readmes - part 1 (#2202)
+- Standardize readmes - part 2 (#2205)
+
+
+## [integrations/elasticsearch-v3.1.0] - 2025-06-12
+
+### 🐛 Bug Fixes
+
+- Fix Elasticsearch types + add py.typed (#1923)
+
+### 🧹 Chores
+
+- Align core-integrations Hatch scripts (#1898)
+- Update md files for new hatch scripts (#1911)
+
+
 ## [integrations/elasticsearch-v3.0.1] - 2025-05-27
 
 

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elasticsearch-haystack
-Version: 3.1.0
+Version: 4.1.0
 Summary: Haystack 2.x Document Store for ElasticSearch
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -24,35 +24,19 @@ Requires-Dist: elasticsearch<9,>=8
 Requires-Dist: haystack-ai>=2.4.0
 Description-Content-Type: text/markdown
 
-[![test](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml)
+# elasticsearch-haystack
 
 [![PyPI - Version](https://img.shields.io/pypi/v/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
 
-# Elasticsearch Document Store
+- [Integration page](https://haystack.deepset.ai/integrations/elasticsearch-document-store)
+- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/elasticsearch/CHANGELOG.md)
 
-Document Store for Haystack 2.x, supports ElasticSearch 8.
+---
 
-## Installation
+## Contributing
 
-```console
-pip install elasticsearch-haystack
-```
+Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
-## Testing
-
-To run tests first start a Docker container running ElasticSearch. We provide a utility `docker-compose.yml` for that:
-
-```console
-docker-compose up
-```
-
-Then run tests:
-
-```console
-hatch run test:all
-```
-
-## License
-
-`elasticsearch-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
+To run integration tests locally, you need a Docker container running ElasticSearch.
+Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.

elasticsearch_haystack-4.1.0/README.md

@@ -0,0 +1,16 @@
+# elasticsearch-haystack
+
+[![PyPI - Version](https://img.shields.io/pypi/v/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
+
+- [Integration page](https://haystack.deepset.ai/integrations/elasticsearch-document-store)
+- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/elasticsearch/CHANGELOG.md)
+
+---
+
+## Contributing
+
+Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
+
+To run integration tests locally, you need a Docker container running ElasticSearch.
+Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/pyproject.toml

@@ -82,11 +82,6 @@ disallow_incomplete_defs = true
 [tool.hatch.metadata]
 allow-direct-references = true
 
-[tool.black]
-target-version = ["py38"]
-line-length = 120
-skip-string-normalization = true
-
 [tool.ruff]
 target-version = "py38"
 line-length = 120

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/src/haystack_integrations/document_stores/elasticsearch/document_store.py

@@ -1,8 +1,14 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
+
+# ruff: noqa: FBT002, FBT001    boolean-type-hint-positional-argument and boolean-default-value-positional-argument
+# ruff: noqa: B008              function-call-in-default-argument
+# ruff: noqa: S101              disable checks for uses of the assert keyword
+
+
 from collections.abc import Mapping
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 import numpy as np
 from elastic_transport import NodeConfig
@@ -10,6 +16,7 @@ from haystack import default_from_dict, default_to_dict, logging
 from haystack.dataclasses import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
+from haystack.utils import Secret, deserialize_secrets_inplace
 from haystack.version import __version__ as haystack_version
 
 from elasticsearch import AsyncElasticsearch, Elasticsearch, helpers
@@ -38,13 +45,16 @@ class ElasticsearchDocumentStore:
 
     Usage example (Elastic Cloud):
     ```python
-    from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
-    document_store = ElasticsearchDocumentStore(cloud_id="YOUR_CLOUD_ID", api_key="YOUR_API_KEY")
+    from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
+    document_store = ElasticsearchDocumentStore(
+        api_key_id=Secret.from_env_var("ELASTIC_API_KEY_ID", strict=False),
+        api_key=Secret.from_env_var("ELASTIC_API_KEY", strict=False),
+    )
     ```
 
     Usage example (self-hosted Elasticsearch instance):
     ```python
-    from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
+    from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
     document_store = ElasticsearchDocumentStore(hosts="http://localhost:9200")
     ```
     In the above example we connect with security disabled just to show the basic usage.
@@ -63,6 +73,8 @@ class ElasticsearchDocumentStore:
         hosts: Optional[Hosts] = None,
         custom_mapping: Optional[Dict[str, Any]] = None,
         index: str = "default",
+        api_key: Secret = Secret.from_env_var("ELASTIC_API_KEY", strict=False),
+        api_key_id: Secret = Secret.from_env_var("ELASTIC_API_KEY_ID", strict=False),
         embedding_similarity_function: Literal["cosine", "dot_product", "l2_norm", "max_inner_product"] = "cosine",
         **kwargs: Any,
     ):
@@ -80,9 +92,16 @@ class ElasticsearchDocumentStore:
         For the full list of supported kwargs, see the official Elasticsearch
         [reference](https://elasticsearch-py.readthedocs.io/en/stable/api.html#module-elasticsearch)
 
+        Authentication is provided via Secret objects, which by default are loaded from environment variables.
+        You can either provide both `api_key_id` and `api_key`, or just `api_key` containing a base64-encoded string
+        of `id:secret`. Secret instances can also be loaded from a token using the `Secret.from_token()` method.
+
         :param hosts: List of hosts running the Elasticsearch client.
         :param custom_mapping: Custom mapping for the index. If not provided, a default mapping will be used.
         :param index: Name of index in Elasticsearch.
+        :param api_key: A Secret object containing the API key for authenticating or base64-encoded with the
+                        concatenated secret and id for authenticating with Elasticsearch (separated by “:”).
+        :param api_key_id: A Secret object containing the API key ID for authenticating with Elasticsearch.
         :param embedding_similarity_function: The similarity function used to compare Documents embeddings.
             This parameter only takes effect if the index does not yet exist and is created.
             To choose the most appropriate function, look for information about your embedding model.
@@ -94,6 +113,8 @@ class ElasticsearchDocumentStore:
         self._client: Optional[Elasticsearch] = None
         self._async_client: Optional[AsyncElasticsearch] = None
         self._index = index
+        self._api_key = api_key
+        self._api_key_id = api_key_id
         self._embedding_similarity_function = embedding_similarity_function
         self._custom_mapping = custom_mapping
         self._kwargs = kwargs
@@ -103,6 +124,29 @@ class ElasticsearchDocumentStore:
             msg = "custom_mapping must be a dictionary"
             raise ValueError(msg)
 
+        if not self._custom_mapping:
+            self._default_mappings = {
+                "properties": {
+                    "embedding": {
+                        "type": "dense_vector",
+                        "index": True,
+                        "similarity": self._embedding_similarity_function,
+                    },
+                    "content": {"type": "text"},
+                },
+                "dynamic_templates": [
+                    {
+                        "strings": {
+                            "path_match": "*",
+                            "match_mapping_type": "string",
+                            "mapping": {
+                                "type": "keyword",
+                            },
+                        }
+                    }
+                ],
+            }
+
     def _ensure_initialized(self):
         """
         Ensures both sync and async clients are initialized and the index exists.
@@ -111,14 +155,18 @@ class ElasticsearchDocumentStore:
             headers = self._kwargs.pop("headers", {})
             headers["user-agent"] = f"haystack-py-ds/{haystack_version}"
 
+            api_key = self._handle_auth()
+
             # Initialize both sync and async clients
             self._client = Elasticsearch(
                 self._hosts,
+                api_key=api_key,
                 headers=headers,
                 **self._kwargs,
             )
             self._async_client = AsyncElasticsearch(
                 self._hosts,
+                api_key=api_key,
                 headers=headers,
                 **self._kwargs,
             )
@@ -130,27 +178,7 @@ class ElasticsearchDocumentStore:
                 mappings = self._custom_mapping
             else:
                 # Configure mapping for the embedding field if none is provided
-                mappings = {
-                    "properties": {
-                        "embedding": {
-                            "type": "dense_vector",
-                            "index": True,
-                            "similarity": self._embedding_similarity_function,
-                        },
-                        "content": {"type": "text"},
-                    },
-                    "dynamic_templates": [
-                        {
-                            "strings": {
-                                "path_match": "*",
-                                "match_mapping_type": "string",
-                                "mapping": {
-                                    "type": "keyword",
-                                },
-                            }
-                        }
-                    ],
-                }
+                mappings = self._default_mappings
 
             # Create the index if it doesn't exist
             if not self._client.indices.exists(index=self._index):
@@ -158,13 +186,56 @@ class ElasticsearchDocumentStore:
 
             self._initialized = True
 
+    def _handle_auth(self) -> Optional[Union[str, Tuple[str, str]]]:
+        """
+        Handles authentication for the Elasticsearch client.
+
+        There are three possible scenarios.
+
+        1) Authentication with both api_key and api_key_id, either as Secrets or as environment variables. In this case,
+           use both for authentication.
+
+        2) Authentication with only api_key, either as a Secret or as an environment variable. In this case, the api_key
+           must be a base64-encoded string that encodes both id and secret <id:secret>.
+
+        3) There's no authentication, neither api_key nor api_key_id are provided as a Secret nor defined as
+           environment variables. In this case, the client will connect without authentication.
+
+        :returns:
+            api_key: Optional[Union[str, Tuple[str, str]]]
+
+        """
+
+        api_key: Optional[Union[str, Tuple[str, str]]]  # make the type checker happy
+
+        api_key_resolved = self._api_key.resolve_value()
+        api_key_id_resolved = self._api_key_id.resolve_value()
+
+        # Scenario 1: both are found, use them
+        if api_key_id_resolved and api_key_resolved:
+            api_key = (api_key_id_resolved, api_key_resolved)
+            return api_key
+
+        # Scenario 2: only api_key is set, must be a base64-encoded string that encodes id and secret (separated by “:”)
+        elif api_key_resolved and not api_key_id_resolved:
+            return api_key_resolved
+
+        # Error: only api_key_id is found, raise an error
+        elif api_key_id_resolved and not api_key_resolved:
+            msg = "api_key_id is provided but api_key is missing."
+            raise ValueError(msg)
+
+        else:
+            # Scenario 3: neither found, no authentication
+            return None
+
     @property
     def client(self) -> Elasticsearch:
         """
         Returns the synchronous Elasticsearch client, initializing it if necessary.
         """
         self._ensure_initialized()
-        assert self._client is not None  # noqa: S101
+        assert self._client is not None
         return self._client
 
     @property
@@ -173,7 +244,7 @@ class ElasticsearchDocumentStore:
         Returns the asynchronous Elasticsearch client, initializing it if necessary.
         """
         self._ensure_initialized()
-        assert self._async_client is not None  # noqa: S101
+        assert self._async_client is not None
         return self._async_client
 
     def to_dict(self) -> Dict[str, Any]:
@@ -191,6 +262,8 @@ class ElasticsearchDocumentStore:
             hosts=self._hosts,
             custom_mapping=self._custom_mapping,
             index=self._index,
+            api_key=self._api_key.to_dict(),
+            api_key_id=self._api_key_id.to_dict(),
             embedding_similarity_function=self._embedding_similarity_function,
             **self._kwargs,
         )
@@ -205,6 +278,7 @@ class ElasticsearchDocumentStore:
         :returns:
             Deserialized component.
         """
+        deserialize_secrets_inplace(data, keys=["api_key", "api_key_id"])
         return default_from_dict(cls, data)
 
     def count_documents(self) -> int:
@@ -384,7 +458,7 @@ class ElasticsearchDocumentStore:
 
         if errors:
             # with stats_only=False, errors is guaranteed to be a list of dicts
-            assert isinstance(errors, list)  # noqa: S101
+            assert isinstance(errors, list)
             duplicate_errors_ids = []
             other_errors = []
             for e in errors:
@@ -463,7 +537,7 @@ class ElasticsearchDocumentStore:
             )
             if failed:
                 # with stats_only=False, failed is guaranteed to be a list of dicts
-                assert isinstance(failed, list)  # noqa: S101
+                assert isinstance(failed, list)
                 if policy == DuplicatePolicy.FAIL:
                     for error in failed:
                         if "create" in error and error["create"]["status"] == DOC_ALREADY_EXISTS:
@@ -490,6 +564,14 @@ class ElasticsearchDocumentStore:
             raise_on_error=False,
         )
 
+    def _prepare_delete_all_request(self, *, is_async: bool) -> Dict[str, Any]:
+        return {
+            "index": self._index,
+            "body": {"query": {"match_all": {}}},  # Delete all documents
+            "wait_for_completion": False if is_async else True,  # block until done (set False for async)
+            "refresh": True,  # Ensure changes are visible immediately
+        }
+
     async def delete_documents_async(self, document_ids: List[str]) -> None:
         """
         Asynchronously deletes all documents with a matching document_ids from the document store.
@@ -509,6 +591,92 @@ class ElasticsearchDocumentStore:
             msg = f"Failed to delete documents from Elasticsearch: {e!s}"
             raise DocumentStoreError(msg) from e
 
+    def delete_all_documents(self, recreate_index: bool = False) -> None:
+        """
+        Deletes all documents in the document store.
+
+        A fast way to clear all documents from the document store while preserving any index settings and mappings.
+
+        :param recreate_index: If True, the index will be deleted and recreated with the original mappings and
+            settings. If False, all documents will be deleted using the `delete_by_query` API.
+        """
+        self._ensure_initialized()  # _ensure_initialized ensures _client is not None and an index exists
+
+        if recreate_index:
+            # get the current index mappings and settings
+            index_name = self._index
+            mappings = self._client.indices.get(index=self._index)[index_name]["mappings"]  # type: ignore
+            settings = self._client.indices.get(index=self._index)[index_name]["settings"]  # type: ignore
+
+            # remove settings that cannot be set during index creation
+            settings["index"].pop("uuid", None)
+            settings["index"].pop("creation_date", None)
+            settings["index"].pop("provided_name", None)
+            settings["index"].pop("version", None)
+
+            self._client.indices.delete(index=self._index)  # type: ignore
+            self._client.indices.create(index=self._index, settings=settings, mappings=mappings)  # type: ignore
+
+            # delete index
+            self._client.indices.delete(index=self._index)  # type: ignore
+
+            # recreate with mappings
+            self._client.indices.create(index=self._index, mappings=mappings)  # type: ignore
+
+        else:
+            result = self._client.delete_by_query(**self._prepare_delete_all_request(is_async=False))  # type: ignore
+            logger.info(
+                "Deleted all the {n_docs} documents from the index '{index}'.",
+                index=self._index,
+                n_docs=result["deleted"],
+            )
+
+    async def delete_all_documents_async(self, recreate_index: bool = False) -> None:
+        """
+        Asynchronously deletes all documents in the document store.
+
+        A fast way to clear all documents from the document store while preserving any index settings and mappings.
+        :param recreate_index: If True, the index will be deleted and recreated with the original mappings and
+            settings. If False, all documents will be deleted using the `delete_by_query` API.
+        """
+        self._ensure_initialized()  # ensures _async_client is not None
+
+        try:
+            if recreate_index:
+                # get the current index mappings and settings
+                index_name = self._index
+                index_info = await self._async_client.indices.get(index=self._index)  # type: ignore
+                mappings = index_info[index_name]["mappings"]
+                settings = index_info[index_name]["settings"]
+
+                # remove settings that cannot be set during index creation
+                settings["index"].pop("uuid", None)
+                settings["index"].pop("creation_date", None)
+                settings["index"].pop("provided_name", None)
+                settings["index"].pop("version", None)
+
+                # delete index
+                await self._async_client.indices.delete(index=self._index)  # type: ignore
+
+                # recreate with settings and mappings
+                await self._async_client.indices.create(index=self._index, settings=settings, mappings=mappings)  # type: ignore
+
+            else:
+                # use delete_by_query for more efficient deletion without index recreation
+                # For async, we need to wait for completion to get the deleted count
+                delete_request = self._prepare_delete_all_request(is_async=True)
+                delete_request["wait_for_completion"] = True  # Override to wait for completion in async
+                result = await self._async_client.delete_by_query(**delete_request)  # type: ignore
+                logger.info(
+                    "Deleted all the {n_docs} documents from the index '{index}'.",
+                    index=self._index,
+                    n_docs=result["deleted"],
+                )
+
+        except Exception as e:
+            msg = f"Failed to delete all documents from Elasticsearch: {e!s}"
+            raise DocumentStoreError(msg) from e
+
     def _bm25_retrieval(
         self,
         query: str,

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/tests/test_bm25_retriever.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 from unittest.mock import Mock, patch
 
 import pytest
@@ -38,6 +39,20 @@ def test_to_dict(_mock_elasticsearch_client):
             "document_store": {
                 "init_parameters": {
                     "hosts": "some fake host",
+                    "api_key": {
+                        "env_vars": [
+                            "ELASTIC_API_KEY",
+                        ],
+                        "strict": False,
+                        "type": "env_var",
+                    },
+                    "api_key_id": {
+                        "env_vars": [
+                            "ELASTIC_API_KEY_ID",
+                        ],
+                        "strict": False,
+                        "type": "env_var",
+                    },
                     "custom_mapping": None,
                     "index": "default",
                     "embedding_similarity_function": "cosine",

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/tests/test_document_store.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import random
+import time
 from typing import List
 from unittest.mock import Mock, patch
 
@@ -13,6 +14,8 @@ from haystack.dataclasses.sparse_embedding import SparseEmbedding
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import DocumentStoreBaseTests
+from haystack.utils import Secret
+from haystack.utils.auth import TokenSecret
 
 from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
@@ -46,6 +49,20 @@ def test_to_dict(_mock_elasticsearch_client):
     assert res == {
         "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
         "init_parameters": {
+            "api_key": {
+                "env_vars": [
+                    "ELASTIC_API_KEY",
+                ],
+                "strict": False,
+                "type": "env_var",
+            },
+            "api_key_id": {
+                "env_vars": [
+                    "ELASTIC_API_KEY_ID",
+                ],
+                "strict": False,
+                "type": "env_var",
+            },
             "hosts": "some hosts",
             "custom_mapping": None,
             "index": "default",
@@ -62,6 +79,8 @@ def test_from_dict(_mock_elasticsearch_client):
             "hosts": "some hosts",
             "custom_mapping": None,
             "index": "default",
+            "api_key": None,
+            "api_key_id": None,
             "embedding_similarity_function": "cosine",
         },
     }
@@ -69,9 +88,135 @@ def test_from_dict(_mock_elasticsearch_client):
     assert document_store._hosts == "some hosts"
     assert document_store._index == "default"
     assert document_store._custom_mapping is None
+    assert document_store._api_key is None
+    assert document_store._api_key_id is None
     assert document_store._embedding_similarity_function == "cosine"
 
 
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_to_dict_with_api_keys_env_vars(_mock_elasticsearch_client, monkeypatch):
+    monkeypatch.setenv("ELASTIC_API_KEY", "test-api-key")
+    monkeypatch.setenv("ELASTIC_API_KEY_ID", "test-api-key-id")
+    document_store = ElasticsearchDocumentStore(hosts="https://localhost:9200")
+    document_store.client()
+    res = document_store.to_dict()
+    assert res["init_parameters"]["api_key"] == {"type": "env_var", "env_vars": ["ELASTIC_API_KEY"], "strict": False}
+    assert res["init_parameters"]["api_key_id"] == {
+        "type": "env_var",
+        "env_vars": ["ELASTIC_API_KEY_ID"],
+        "strict": False,
+    }
+
+
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_to_dict_with_api_keys_as_secret(_mock_elasticsearch_client, monkeypatch):
+    monkeypatch.setenv("ELASTIC_API_KEY", "test-api-key")
+    monkeypatch.setenv("ELASTIC_API_KEY_ID", "test-api-key-id")
+    with pytest.raises(ValueError):
+        document_store = ElasticsearchDocumentStore(
+            hosts="https://localhost:9200",
+            api_key=TokenSecret(_token="test-api-key"),
+            api_key_id=TokenSecret(_token="test-api-key-id"),
+        )
+        document_store.client()
+        _ = document_store.to_dict()
+
+
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_from_dict_with_api_keys_env_vars(_mock_elasticsearch_client):
+    data = {
+        "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
+        "init_parameters": {
+            "hosts": "some hosts",
+            "custom_mapping": None,
+            "index": "default",
+            "api_key": {"type": "env_var", "env_vars": ["ELASTIC_API_KEY"], "strict": False},
+            "api_key_id": {"type": "env_var", "env_vars": ["ELASTIC_API_KEY_ID"], "strict": False},
+            "embedding_similarity_function": "cosine",
+        },
+    }
+
+    document_store = ElasticsearchDocumentStore.from_dict(data)
+    assert document_store._api_key == {"type": "env_var", "env_vars": ["ELASTIC_API_KEY"], "strict": False}
+    assert document_store._api_key_id == {"type": "env_var", "env_vars": ["ELASTIC_API_KEY_ID"], "strict": False}
+
+
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_api_key_validation_only_api_key(_mock_elasticsearch_client):
+    api_key = Secret.from_token("test_api_key")
+
+    document_store = ElasticsearchDocumentStore(hosts="https://localhost:9200", api_key=api_key)
+    document_store.client()
+    assert document_store._api_key == api_key
+    # not passing the api_key_id makes it default to reading from env var
+    assert document_store._api_key_id == Secret.from_env_var("ELASTIC_API_KEY_ID", strict=False)
+
+
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+def test_api_key_validation_only_api_key_id_raises_error(_mock_elasticsearch_client):
+    api_key_id = Secret.from_token("test_api_key_id")
+    with pytest.raises(ValueError, match="api_key_id is provided but api_key is missing"):
+        es = ElasticsearchDocumentStore(hosts="https://localhost:9200", api_key_id=api_key_id)
+        es.client()
+
+
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.AsyncElasticsearch")
+def test_client_initialization_with_api_key_tuple(_mock_async_es, _mock_es):
+    api_key = Secret.from_token("test_api_key")
+    api_key_id = Secret.from_token("test_api_key_id")
+
+    # Mock the client.info() call to avoid actual connection
+    mock_client = Mock()
+    mock_client.info.return_value = {"version": {"number": "8.0.0"}}
+    _mock_es.return_value = mock_client
+
+    document_store = ElasticsearchDocumentStore(hosts="https://localhost:9200", api_key=api_key, api_key_id=api_key_id)
+
+    # Access client to trigger initialization
+    _ = document_store.client
+
+    # Check that Elasticsearch was called with the correct api_key tuple
+    _mock_es.assert_called_once()
+    call_args = _mock_es.call_args
+    assert call_args[0][0] == "https://localhost:9200"  # hosts
+    assert call_args[1]["api_key"] == ("test_api_key_id", "test_api_key")
+
+    # Check that AsyncElasticsearch was called with the same api_key tuple
+    _mock_async_es.assert_called_once()
+    async_call_args = _mock_async_es.call_args
+    assert async_call_args[0][0] == "https://localhost:9200"  # hosts
+    assert async_call_args[1]["api_key"] == ("test_api_key_id", "test_api_key")
+
+
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.AsyncElasticsearch")
+def test_client_initialization_with_api_key_string(_mock_async_es, _mock_es):
+    api_key = Secret.from_token("test_api_key")
+
+    # Mock the client.info() call to avoid actual connection
+    mock_client = Mock()
+    mock_client.info.return_value = {"version": {"number": "8.0.0"}}
+    _mock_es.return_value = mock_client
+
+    document_store = ElasticsearchDocumentStore(hosts="testhost", api_key=api_key)
+
+    # Access client to trigger initialization
+    _ = document_store.client
+
+    # Check that Elasticsearch was called with the correct api_key string
+    _mock_es.assert_called_once()
+    call_args = _mock_es.call_args
+    assert call_args[0][0] == "testhost"  # hosts
+    assert call_args[1]["api_key"] == "test_api_key"
+
+    # Check that AsyncElasticsearch was called with the same api_key string
+    _mock_async_es.assert_called_once()
+    async_call_args = _mock_async_es.call_args
+    assert async_call_args[0][0] == "testhost"  # hosts
+    assert async_call_args[1]["api_key"] == "test_api_key"
+
+
 @pytest.mark.integration
 class TestDocumentStore(DocumentStoreBaseTests):
     """
@@ -339,6 +484,59 @@ class TestDocumentStore(DocumentStoreBaseTests):
             mappings=custom_mapping,
         )
 
+    def test_delete_all_documents_index_recreation(self, document_store: ElasticsearchDocumentStore):
+        # populate the index with some documents
+        docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
+        document_store.write_documents(docs)
+
+        # capture index structure before deletion
+        assert document_store._client is not None
+        index_info_before = document_store._client.indices.get(index=document_store._index)
+        mappings_before = index_info_before[document_store._index]["mappings"]
+        settings_before = index_info_before[document_store._index]["settings"]
+
+        # delete all documents
+        document_store.delete_all_documents(recreate_index=True)
+        assert document_store.count_documents() == 0
+
+        # verify index structure is preserved
+        index_info_after = document_store._client.indices.get(index=document_store._index)
+        mappings_after = index_info_after[document_store._index]["mappings"]
+        assert mappings_after == mappings_before, "delete_all_documents should preserve index mappings"
+
+        settings_after = index_info_after[document_store._index]["settings"]
+        settings_after["index"].pop("uuid", None)
+        settings_after["index"].pop("creation_date", None)
+        settings_before["index"].pop("uuid", None)
+        settings_before["index"].pop("creation_date", None)
+        assert settings_after == settings_before, "delete_all_documents should preserve index settings"
+
+        # verify index can accept new documents and retrieve
+        new_doc = Document(id="4", content="New document after delete all")
+        document_store.write_documents([new_doc])
+        assert document_store.count_documents() == 1
+
+        results = document_store.filter_documents()
+        assert len(results) == 1
+        assert results[0].content == "New document after delete all"
+
+    def test_delete_all_documents_no_index_recreation(self, document_store: ElasticsearchDocumentStore):
+        docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
+        document_store.write_documents(docs)
+        assert document_store.count_documents() == 2
+
+        document_store.delete_all_documents(recreate_index=False)
+        time.sleep(2)  # need to wait for the deletion to be reflected in count_documents
+        assert document_store.count_documents() == 0
+
+        new_doc = Document(id="3", content="New document after delete all")
+        document_store.write_documents([new_doc])
+        assert document_store.count_documents() == 1
+
+        results = document_store.filter_documents()
+        assert len(results) == 1
+        assert results[0].content == "New document after delete all"
+
 
 @pytest.mark.integration
 class TestElasticsearchDocumentStoreAsync:
@@ -487,3 +685,79 @@ class TestElasticsearchDocumentStoreAsync:
         assert len(results) == 1
         assert results[0].id == "1"
         assert not hasattr(results[0], "sparse_embedding") or results[0].sparse_embedding is None
+
+    @pytest.mark.asyncio
+    async def test_delete_all_documents_async(self, document_store):
+        docs = [
+            Document(id="1", content="First document", meta={"category": "test"}),
+            Document(id="2", content="Second document", meta={"category": "test"}),
+            Document(id="3", content="Third document", meta={"category": "other"}),
+        ]
+        await document_store.write_documents_async(docs)
+        assert await document_store.count_documents_async() == 3
+
+        # delete all documents
+        await document_store.delete_all_documents_async(recreate_index=False)
+        assert await document_store.count_documents_async() == 0
+
+        # verify index still exists and can accept new documents and retrieve
+        new_doc = Document(id="4", content="New document after delete all")
+        await document_store.write_documents_async([new_doc])
+        assert await document_store.count_documents_async() == 1
+
+        results = await document_store.filter_documents_async()
+        assert len(results) == 1
+        assert results[0].id == "4"
+        assert results[0].content == "New document after delete all"
+
+    @pytest.mark.asyncio
+    async def test_delete_all_documents_async_index_recreation(self, document_store):
+        # populate the index with some documents
+        docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
+        await document_store.write_documents_async(docs)
+
+        # capture index structure before deletion
+        assert document_store._async_client is not None
+        index_info_before = await document_store._async_client.indices.get(index=document_store._index)
+        mappings_before = index_info_before[document_store._index]["mappings"]
+        settings_before = index_info_before[document_store._index]["settings"]
+
+        # delete all documents with index recreation
+        await document_store.delete_all_documents_async(recreate_index=True)
+        assert await document_store.count_documents_async() == 0
+
+        # verify index structure is preserved
+        index_info_after = await document_store._async_client.indices.get(index=document_store._index)
+        mappings_after = index_info_after[document_store._index]["mappings"]
+        assert mappings_after == mappings_before, "delete_all_documents_async should preserve index mappings"
+
+        settings_after = index_info_after[document_store._index]["settings"]
+        settings_after["index"].pop("uuid", None)
+        settings_after["index"].pop("creation_date", None)
+        settings_before["index"].pop("uuid", None)
+        settings_before["index"].pop("creation_date", None)
+        assert settings_after == settings_before, "delete_all_documents_async should preserve index settings"
+
+        # verify index can accept new documents and retrieve
+        new_doc = Document(id="4", content="New document after delete all")
+        await document_store.write_documents_async([new_doc])
+        assert await document_store.count_documents_async() == 1
+
+        results = await document_store.filter_documents_async()
+        assert len(results) == 1
+        assert results[0].content == "New document after delete all"
+
+    @pytest.mark.asyncio
+    async def test_delete_all_documents_async_no_index_recreation(self, document_store):
+        docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
+        await document_store.write_documents_async(docs)
+        assert await document_store.count_documents_async() == 2
+
+        await document_store.delete_all_documents_async(recreate_index=False)
+        # Need to wait for the deletion to be reflected in count_documents
+        time.sleep(2)
+        assert await document_store.count_documents_async() == 0
+
+        new_doc = Document(id="3", content="New document after delete all")
+        await document_store.write_documents_async([new_doc])
+        assert await document_store.count_documents_async() == 1

{elasticsearch_haystack-3.1.0 → elasticsearch_haystack-4.1.0}/tests/test_embedding_retriever.py

@@ -37,6 +37,20 @@ def test_to_dict(_mock_elasticsearch_client):
         "init_parameters": {
             "document_store": {
                 "init_parameters": {
+                    "api_key": {
+                        "env_vars": [
+                            "ELASTIC_API_KEY",
+                        ],
+                        "strict": False,
+                        "type": "env_var",
+                    },
+                    "api_key_id": {
+                        "env_vars": [
+                            "ELASTIC_API_KEY_ID",
+                        ],
+                        "strict": False,
+                        "type": "env_var",
+                    },
                     "hosts": "some fake host",
                     "custom_mapping": None,
                     "index": "default",

elasticsearch_haystack-3.1.0/README.md

@@ -1,32 +0,0 @@
-[![test](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml)
-
-[![PyPI - Version](https://img.shields.io/pypi/v/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
-[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
-
-# Elasticsearch Document Store
-
-Document Store for Haystack 2.x, supports ElasticSearch 8.
-
-## Installation
-
-```console
-pip install elasticsearch-haystack
-```
-
-## Testing
-
-To run tests first start a Docker container running ElasticSearch. We provide a utility `docker-compose.yml` for that:
-
-```console
-docker-compose up
-```
-
-Then run tests:
-
-```console
-hatch run test:all
-```
-
-## License
-
-`elasticsearch-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.