elasticsearch-haystack 3.0.1__py3-none-any.whl → 4.0.0__py3-none-any.whl

{elasticsearch_haystack-3.0.1.dist-info → elasticsearch_haystack-4.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elasticsearch-haystack
-Version: 3.0.1
+Version: 4.0.0
 Summary: Haystack 2.x Document Store for ElasticSearch
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -24,35 +24,19 @@ Requires-Dist: elasticsearch<9,>=8
 Requires-Dist: haystack-ai>=2.4.0
 Description-Content-Type: text/markdown
 
-[![test](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/elasticsearch.yml)
+# elasticsearch-haystack
 
 [![PyPI - Version](https://img.shields.io/pypi/v/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/elasticsearch-haystack.svg)](https://pypi.org/project/elasticsearch-haystack)
 
-# Elasticsearch Document Store
+- [Integration page](https://haystack.deepset.ai/integrations/elasticsearch-document-store)
+- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/elasticsearch/CHANGELOG.md)
 
-Document Store for Haystack 2.x, supports ElasticSearch 8.
+---
 
-## Installation
+## Contributing
 
-```console
-pip install elasticsearch-haystack
-```
+Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
 
-## Testing
-
-To run tests first start a Docker container running ElasticSearch. We provide a utility `docker-compose.yml` for that:
-
-```console
-docker-compose up
-```
-
-Then run tests:
-
-```console
-hatch run test
-```
-
-## License
-
-`elasticsearch-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
+To run integration tests locally, you need a Docker container running ElasticSearch.
+Use the provided `docker-compose.yml` file to start the container: `docker compose up -d`.

elasticsearch_haystack-4.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+haystack_integrations/components/retrievers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haystack_integrations/components/retrievers/elasticsearch/__init__.py,sha256=cSJBsYjz_T4kK-M-auAHVUnYIcgUqqwwQe_hsF0_IG4,307
+haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py,sha256=HsR42EolOBEIuLNwQ_8FeSmrHMJ6WscYulcoXSHoaYQ,7098
+haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py,sha256=-6eIHW5cU4k8-jAsUsCb15hJRalpkUhzy_dNxr5HUZo,7404
+haystack_integrations/document_stores/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haystack_integrations/document_stores/elasticsearch/__init__.py,sha256=YTfu94dtVUBogbJFr1aJrKuaI6-Bw9VuHfPoyU7M8os,207
+haystack_integrations/document_stores/elasticsearch/document_store.py,sha256=1KnQdBsD-QdvncxFZ1oaNTA9-vRJ7xXCDaKr5JcTVnE,31062
+haystack_integrations/document_stores/elasticsearch/filters.py,sha256=Umip-PP4uFjuWeB1JWkKhaKClQ0VpiykoDlDu99wIV0,9759
+elasticsearch_haystack-4.0.0.dist-info/METADATA,sha256=Ukr9NBPT0Vwi8WhAA5agHeOQfB3_0_6RcGQDx_yJB2w,2105
+elasticsearch_haystack-4.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+elasticsearch_haystack-4.0.0.dist-info/licenses/LICENSE,sha256=_M2kulivnaiTHiW-5CRlZrPmH47tt04pBgAgeDvfYi4,11342
+elasticsearch_haystack-4.0.0.dist-info/RECORD,,

haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py

@@ -116,7 +116,9 @@ class ElasticsearchBM25Retriever:
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
-    def run(self, query: str, filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None):
+    def run(
+        self, query: str, filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None
+    ) -> Dict[str, List[Document]]:
         """
         Retrieve documents using the BM25 keyword-based algorithm.
 
@@ -139,7 +141,9 @@ class ElasticsearchBM25Retriever:
         return {"documents": docs}
 
     @component.output_types(documents=List[Document])
-    async def run_async(self, query: str, filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None):
+    async def run_async(
+        self, query: str, filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None
+    ) -> Dict[str, List[Document]]:
         """
         Asynchronously retrieve documents using the BM25 keyword-based algorithm.
 

haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py

@@ -114,7 +114,9 @@ class ElasticsearchEmbeddingRetriever:
         return default_from_dict(cls, data)
 
     @component.output_types(documents=List[Document])
-    def run(self, query_embedding: List[float], filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None):
+    def run(
+        self, query_embedding: List[float], filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None
+    ) -> Dict[str, List[Document]]:
         """
         Retrieve documents using a vector similarity metric.
 
@@ -139,7 +141,7 @@ class ElasticsearchEmbeddingRetriever:
     @component.output_types(documents=List[Document])
     async def run_async(
         self, query_embedding: List[float], filters: Optional[Dict[str, Any]] = None, top_k: Optional[int] = None
-    ):
+    ) -> Dict[str, List[Document]]:
         """
         Asynchronously retrieve documents using a vector similarity metric.
 

haystack_integrations/document_stores/elasticsearch/document_store.py

@@ -1,20 +1,20 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 from collections.abc import Mapping
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 import numpy as np
-
-# There are no import stubs for elastic_transport and elasticsearch so mypy fails
-from elastic_transport import NodeConfig  # type: ignore[import-not-found]
+from elastic_transport import NodeConfig
 from haystack import default_from_dict, default_to_dict, logging
 from haystack.dataclasses import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
+from haystack.utils import Secret, deserialize_secrets_inplace
 from haystack.version import __version__ as haystack_version
 
-from elasticsearch import AsyncElasticsearch, Elasticsearch, helpers  # type: ignore[import-not-found]
+from elasticsearch import AsyncElasticsearch, Elasticsearch, helpers
 
 from .filters import _normalize_filters
 
@@ -40,13 +40,16 @@ class ElasticsearchDocumentStore:
 
     Usage example (Elastic Cloud):
     ```python
-    from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
-    document_store = ElasticsearchDocumentStore(cloud_id="YOUR_CLOUD_ID", api_key="YOUR_API_KEY")
+    from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
+    document_store = ElasticsearchDocumentStore(
+        api_key_id=Secret.from_env_var("ELASTIC_API_KEY_ID", strict=False),
+        api_key=Secret.from_env_var("ELASTIC_API_KEY", strict=False),
+    )
     ```
 
     Usage example (self-hosted Elasticsearch instance):
     ```python
-    from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
+    from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
     document_store = ElasticsearchDocumentStore(hosts="http://localhost:9200")
     ```
     In the above example we connect with security disabled just to show the basic usage.
@@ -65,8 +68,10 @@ class ElasticsearchDocumentStore:
         hosts: Optional[Hosts] = None,
         custom_mapping: Optional[Dict[str, Any]] = None,
         index: str = "default",
+        api_key: Secret = Secret.from_env_var("ELASTIC_API_KEY", strict=False),  # noqa: B008
+        api_key_id: Secret = Secret.from_env_var("ELASTIC_API_KEY_ID", strict=False),  # noqa: B008
         embedding_similarity_function: Literal["cosine", "dot_product", "l2_norm", "max_inner_product"] = "cosine",
-        **kwargs,
+        **kwargs: Any,
     ):
         """
         Creates a new ElasticsearchDocumentStore instance.
@@ -82,9 +87,16 @@ class ElasticsearchDocumentStore:
         For the full list of supported kwargs, see the official Elasticsearch
         [reference](https://elasticsearch-py.readthedocs.io/en/stable/api.html#module-elasticsearch)
 
+        Authentication is provided via Secret objects, which by default are loaded from environment variables.
+        You can either provide both `api_key_id` and `api_key`, or just `api_key` containing a base64-encoded string
+        of `id:secret`. Secret instances can also be loaded from a token using the `Secret.from_token()` method.
+
         :param hosts: List of hosts running the Elasticsearch client.
         :param custom_mapping: Custom mapping for the index. If not provided, a default mapping will be used.
         :param index: Name of index in Elasticsearch.
+        :param api_key: A Secret object containing the API key for authenticating or base64-encoded with the
+                        concatenated secret and id for authenticating with Elasticsearch (separated by “:”).
+        :param api_key_id: A Secret object containing the API key ID for authenticating with Elasticsearch.
         :param embedding_similarity_function: The similarity function used to compare Documents embeddings.
             This parameter only takes effect if the index does not yet exist and is created.
             To choose the most appropriate function, look for information about your embedding model.
@@ -93,9 +105,11 @@ class ElasticsearchDocumentStore:
         :param **kwargs: Optional arguments that `Elasticsearch` takes.
         """
         self._hosts = hosts
-        self._client = None
-        self._async_client = None
+        self._client: Optional[Elasticsearch] = None
+        self._async_client: Optional[AsyncElasticsearch] = None
         self._index = index
+        self._api_key = api_key
+        self._api_key_id = api_key_id
         self._embedding_similarity_function = embedding_similarity_function
         self._custom_mapping = custom_mapping
         self._kwargs = kwargs
@@ -113,14 +127,18 @@ class ElasticsearchDocumentStore:
             headers = self._kwargs.pop("headers", {})
             headers["user-agent"] = f"haystack-py-ds/{haystack_version}"
 
+            api_key = self._handle_auth()
+
             # Initialize both sync and async clients
             self._client = Elasticsearch(
                 self._hosts,
+                api_key=api_key,
                 headers=headers,
                 **self._kwargs,
             )
             self._async_client = AsyncElasticsearch(
                 self._hosts,
+                api_key=api_key,
                 headers=headers,
                 **self._kwargs,
             )
@@ -160,12 +178,56 @@ class ElasticsearchDocumentStore:
 
             self._initialized = True
 
+    def _handle_auth(self) -> Optional[Union[str, Tuple[str, str]]]:
+        """
+        Handles authentication for the Elasticsearch client.
+
+        There are three possible scenarios.
+
+        1) Authentication with both api_key and api_key_id, either as Secrets or as environment variables. In this case,
+           use both for authentication.
+
+        2) Authentication with only api_key, either as a Secret or as an environment variable. In this case, the api_key
+           must be a base64-encoded string that encodes both id and secret <id:secret>.
+
+        3) There's no authentication, neither api_key nor api_key_id are provided as a Secret nor defined as
+           environment variables. In this case, the client will connect without authentication.
+
+        :returns:
+            api_key: Optional[Union[str, Tuple[str, str]]]
+
+        """
+
+        api_key: Optional[Union[str, Tuple[str, str]]]  # make the type checker happy
+
+        api_key_resolved = self._api_key.resolve_value()
+        api_key_id_resolved = self._api_key_id.resolve_value()
+
+        # Scenario 1: both are found, use them
+        if api_key_id_resolved and api_key_resolved:
+            api_key = (api_key_id_resolved, api_key_resolved)
+            return api_key
+
+        # Scenario 2: only api_key is set, must be a base64-encoded string that encodes id and secret (separated by “:”)
+        elif api_key_resolved and not api_key_id_resolved:
+            return api_key_resolved
+
+        # Error: only api_key_id is found, raise an error
+        elif api_key_id_resolved and not api_key_resolved:
+            msg = "api_key_id is provided but api_key is missing."
+            raise ValueError(msg)
+
+        else:
+            # Scenario 3: neither found, no authentication
+            return None
+
     @property
     def client(self) -> Elasticsearch:
         """
         Returns the synchronous Elasticsearch client, initializing it if necessary.
         """
         self._ensure_initialized()
+        assert self._client is not None  # noqa: S101
         return self._client
 
     @property
@@ -174,6 +236,7 @@ class ElasticsearchDocumentStore:
         Returns the asynchronous Elasticsearch client, initializing it if necessary.
         """
         self._ensure_initialized()
+        assert self._async_client is not None  # noqa: S101
         return self._async_client
 
     def to_dict(self) -> Dict[str, Any]:
@@ -191,6 +254,8 @@ class ElasticsearchDocumentStore:
             hosts=self._hosts,
             custom_mapping=self._custom_mapping,
             index=self._index,
+            api_key=self._api_key.to_dict(),
+            api_key_id=self._api_key_id.to_dict(),
             embedding_similarity_function=self._embedding_similarity_function,
             **self._kwargs,
         )
@@ -205,6 +270,7 @@ class ElasticsearchDocumentStore:
         :returns:
             Deserialized component.
         """
+        deserialize_secrets_inplace(data, keys=["api_key", "api_key_id"])
         return default_from_dict(cls, data)
 
     def count_documents(self) -> int:
@@ -226,7 +292,7 @@ class ElasticsearchDocumentStore:
         result = await self._async_client.count(index=self._index)  # type: ignore
         return result["count"]
 
-    def _search_documents(self, **kwargs) -> List[Document]:
+    def _search_documents(self, **kwargs: Any) -> List[Document]:
         """
         Calls the Elasticsearch client's search method and handles pagination.
         """
@@ -253,7 +319,7 @@ class ElasticsearchDocumentStore:
                 break
         return documents
 
-    async def _search_documents_async(self, **kwargs) -> List[Document]:
+    async def _search_documents_async(self, **kwargs: Any) -> List[Document]:
         """
         Asynchronously calls the Elasticsearch client's search method and handles pagination.
         """
@@ -379,9 +445,12 @@ class ElasticsearchDocumentStore:
             refresh="wait_for",
             index=self._index,
             raise_on_error=False,
+            stats_only=False,
         )
 
         if errors:
+            # with stats_only=False, errors is guaranteed to be a list of dicts
+            assert isinstance(errors, list)  # noqa: S101
             duplicate_errors_ids = []
             other_errors = []
             for e in errors:
@@ -451,13 +520,16 @@ class ElasticsearchDocumentStore:
 
         try:
             success, failed = await helpers.async_bulk(
-                client=self._async_client,
+                client=self.async_client,
                 actions=actions,
                 index=self._index,
                 refresh=True,
                 raise_on_error=False,
+                stats_only=False,
             )
             if failed:
+                # with stats_only=False, failed is guaranteed to be a list of dicts
+                assert isinstance(failed, list)  # noqa: S101
                 if policy == DuplicatePolicy.FAIL:
                     for error in failed:
                         if "create" in error and error["create"]["status"] == DOC_ALREADY_EXISTS:
@@ -494,7 +566,7 @@ class ElasticsearchDocumentStore:
 
         try:
             await helpers.async_bulk(
-                client=self._async_client,
+                client=self.async_client,
                 actions=({"_op_type": "delete", "_id": id_} for id_ in document_ids),
                 index=self._index,
                 refresh=True,
@@ -551,6 +623,8 @@ class ElasticsearchDocumentStore:
 
         if scale_score:
             for doc in documents:
+                if doc.score is None:
+                    continue
                 doc.score = float(1 / (1 + np.exp(-np.asarray(doc.score / BM25_SCALING_FACTOR))))
 
         return documents

elasticsearch_haystack-3.0.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-haystack_integrations/components/retrievers/elasticsearch/__init__.py,sha256=cSJBsYjz_T4kK-M-auAHVUnYIcgUqqwwQe_hsF0_IG4,307
-haystack_integrations/components/retrievers/elasticsearch/bm25_retriever.py,sha256=ISHc6elYXoDXDvC62_3bMMCk_Dv67jvZIgQBCZ1ZHdw,7012
-haystack_integrations/components/retrievers/elasticsearch/embedding_retriever.py,sha256=jHDLMeecpf-DhvbRM1AAq2kIJn7xMNTR9vkm-FhHH7k,7332
-haystack_integrations/document_stores/elasticsearch/__init__.py,sha256=YTfu94dtVUBogbJFr1aJrKuaI6-Bw9VuHfPoyU7M8os,207
-haystack_integrations/document_stores/elasticsearch/document_store.py,sha256=pZ0pPyOCPTCKNYD4q5YbLrslSGTIbVPj60U18-BImX8,27406
-haystack_integrations/document_stores/elasticsearch/filters.py,sha256=Umip-PP4uFjuWeB1JWkKhaKClQ0VpiykoDlDu99wIV0,9759
-elasticsearch_haystack-3.0.1.dist-info/METADATA,sha256=defq5KselqQwwMLUa3jfrmXjjVxH15cLSn7iKlDKgJk,2257
-elasticsearch_haystack-3.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-elasticsearch_haystack-3.0.1.dist-info/licenses/LICENSE,sha256=_M2kulivnaiTHiW-5CRlZrPmH47tt04pBgAgeDvfYi4,11342
-elasticsearch_haystack-3.0.1.dist-info/RECORD,,