apache-airflow-providers-elasticsearch 5.0.1rc1__py3-none-any.whl → 5.0.2rc1__py3-none-any.whl

airflow/providers/elasticsearch/__init__.py

@@ -28,7 +28,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "5.0.1"
+__version__ = "5.0.2"
 
 try:
     from airflow import __version__ as airflow_version

airflow/providers/elasticsearch/get_provider_info.py

@@ -29,6 +29,7 @@ def get_provider_info():
         "description": "`Elasticsearch <https://www.elastic.co/elasticsearch>`__\n",
         "suspended": False,
         "versions": [
+            "5.0.2",
             "5.0.1",
             "5.0.0",
             "4.5.1",

airflow/providers/elasticsearch/hooks/elasticsearch.py

@@ -19,16 +19,18 @@ from __future__ import annotations
 
 import warnings
 from functools import cached_property
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib import parse
 
 from elasticsearch import Elasticsearch
 
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.hooks.base import BaseHook
-from airflow.models.connection import Connection as AirflowConnection
 from airflow.providers.common.sql.hooks.sql import DbApiHook
 
+if TYPE_CHECKING:
+    from airflow.models.connection import Connection as AirflowConnection
+
 
 def connect(
     host: str = "localhost",
@@ -92,13 +94,13 @@ class ElasticsearchSQLHook(DbApiHook):
         conn_id = getattr(self, self.conn_name_attr)
         conn = self.connection or self.get_connection(conn_id)
 
-        conn_args = dict(
-            host=conn.host,
-            port=conn.port,
-            user=conn.login or None,
-            password=conn.password or None,
-            scheme=conn.schema or "http",
-        )
+        conn_args = {
+            "host": conn.host,
+            "port": conn.port,
+            "user": conn.login or None,
+            "password": conn.password or None,
+            "scheme": conn.schema or "http",
+        }
 
         if conn.extra_dejson.get("http_compress", False):
             conn_args["http_compress"] = bool(["http_compress"])
@@ -116,11 +118,11 @@ class ElasticsearchSQLHook(DbApiHook):
 
         login = ""
         if conn.login:
-            login = "{conn.login}:{conn.password}@".format(conn=conn)
+            login = f"{conn.login}:{conn.password}@"
         host = conn.host
         if conn.port is not None:
             host += f":{conn.port}"
-        uri = "{conn.conn_type}+{conn.schema}://{login}{host}/".format(conn=conn, login=login, host=host)
+        uri = f"{conn.conn_type}+{conn.schema}://{login}{host}/"
 
         extras_length = len(conn.extra_dejson)
         if not extras_length:

airflow/providers/elasticsearch/log/es_response.py

@@ -38,7 +38,7 @@ class AttributeList:
         return _wrap(val)
 
     def __iter__(self):
-        return map(lambda i: _wrap(i), self._l_)
+        return (_wrap(i) for i in self._l_)
 
     def __bool__(self):
         return bool(self._l_)

airflow/providers/elasticsearch/log/es_task_handler.py

@@ -17,11 +17,11 @@
 # under the License.
 from __future__ import annotations
 
+import inspect
 import logging
 import sys
 import warnings
 from collections import defaultdict
-from datetime import datetime
 from operator import attrgetter
 from time import time
 from typing import TYPE_CHECKING, Any, Callable, List, Tuple
@@ -31,11 +31,11 @@ from urllib.parse import quote, urlparse
 import elasticsearch
 import pendulum
 from elasticsearch.exceptions import NotFoundError
+from typing_extensions import Literal
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.models.dagrun import DagRun
-from airflow.models.taskinstance import TaskInstance
 from airflow.providers.elasticsearch.log.es_json_formatter import ElasticsearchJSONFormatter
 from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse, Hit
 from airflow.utils import timezone
@@ -43,6 +43,11 @@ from airflow.utils.log.file_task_handler import FileTaskHandler
 from airflow.utils.log.logging_mixin import ExternalLoggingMixin, LoggingMixin
 from airflow.utils.session import create_session
 
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from airflow.models.taskinstance import TaskInstance
+
 LOG_LINE_DEFAULTS = {"exc_text": "", "stack_info": ""}
 # Elasticsearch hosted log type
 EsLogMsgType = List[Tuple[str, str]]
@@ -53,6 +58,32 @@ EsLogMsgType = List[Tuple[str, str]]
 USE_PER_RUN_LOG_ID = hasattr(DagRun, "get_log_template")
 
 
+VALID_ES_CONFIG_KEYS = set(inspect.signature(elasticsearch.Elasticsearch.__init__).parameters.keys())
+# Remove `self` from the valid set of kwargs
+VALID_ES_CONFIG_KEYS.remove("self")
+
+
+def get_es_kwargs_from_config() -> dict[str, Any]:
+    elastic_search_config = conf.getsection("elasticsearch_configs")
+    kwargs_dict = (
+        {key: value for key, value in elastic_search_config.items() if key in VALID_ES_CONFIG_KEYS}
+        if elastic_search_config
+        else {}
+    )
+    # For elasticsearch>8 retry_timeout have changed for elasticsearch to retry_on_timeout
+    # in Elasticsearch() compared to previous versions.
+    # Read more at: https://elasticsearch-py.readthedocs.io/en/v8.8.2/api.html#module-elasticsearch
+    if (
+        elastic_search_config
+        and "retry_timeout" in elastic_search_config
+        and not kwargs_dict.get("retry_on_timeout")
+    ):
+        retry_timeout = elastic_search_config.get("retry_timeout")
+        if retry_timeout is not None:
+            kwargs_dict["retry_on_timeout"] = retry_timeout
+    return kwargs_dict
+
+
 class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMixin):
     """
     ElasticsearchTaskHandler is a python log handler that reads logs from Elasticsearch.
@@ -92,17 +123,14 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         host: str = "http://localhost:9200",
         frontend: str = "localhost:5601",
         index_patterns: str | None = conf.get("elasticsearch", "index_patterns", fallback="_all"),
-        es_kwargs: dict | None = conf.getsection("elasticsearch_configs"),
+        es_kwargs: dict | None | Literal["default_es_kwargs"] = "default_es_kwargs",
         *,
         filename_template: str | None = None,
         log_id_template: str | None = None,
     ):
         es_kwargs = es_kwargs or {}
-        # For elasticsearch>8,arguments like retry_timeout have changed for elasticsearch to retry_on_timeout
-        # in Elasticsearch() compared to previous versions.
-        # Read more at: https://elasticsearch-py.readthedocs.io/en/v8.8.2/api.html#module-elasticsearch
-        if es_kwargs.get("retry_timeout"):
-            es_kwargs["retry_on_timeout"] = es_kwargs.pop("retry_timeout")
+        if es_kwargs == "default_es_kwargs":
+            es_kwargs = get_es_kwargs_from_config()
         host = self.format_url(host)
         super().__init__(base_log_folder, filename_template)
         self.closed = False
@@ -361,7 +389,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         if self.json_format:
             self.formatter = ElasticsearchJSONFormatter(
                 fmt=self.formatter._fmt,
-                json_fields=self.json_fields + [self.offset_field],
+                json_fields=[*self.json_fields, self.offset_field],
                 extras={
                     "dag_id": str(ti.dag_id),
                     "task_id": str(ti.task_id),

{apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info → apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-elasticsearch
-Version: 5.0.1rc1
+Version: 5.0.2rc1
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-elasticsearch package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
 Author: Apache Software Foundation
 Author-email: dev@airflow.apache.org
 License: Apache License 2.0
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.1/
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.2/
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.2/changelog.html
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
@@ -75,7 +75,7 @@ Requires-Dist: apache-airflow-providers-common-sql ; extra == 'common.sql'
 
 Package ``apache-airflow-providers-elasticsearch``
 
-Release: ``5.0.1rc1``
+Release: ``5.0.2rc1``
 
 
 `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -88,7 +88,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
 are in ``airflow.providers.elasticsearch`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.2/>`_.
 
 
 Installation
@@ -131,4 +131,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.0.2/changelog.html>`_.

apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/RECORD

@@ -0,0 +1,15 @@
+airflow/providers/elasticsearch/__init__.py,sha256=pfeNE3S4fPn5x9VoC5yExvm6mm1gaQxnXW2PmkOD1qM,1582
+airflow/providers/elasticsearch/get_provider_info.py,sha256=jWyHVfIRIhk9gKlhk92mUPKebrKEwaDF1aMNsQocaow,7680
+airflow/providers/elasticsearch/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=qgpyV4pjPj8b2EwjWHm8eGcewubQBTHY32K_xjBDeP8,6472
+airflow/providers/elasticsearch/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/elasticsearch/log/es_json_formatter.py,sha256=IBCC6U8XUIfZ1vZKeDAQWCwNCXhP5yG2oOjSJesuGV0,1640
+airflow/providers/elasticsearch/log/es_response.py,sha256=txtpMOadBbuiV91nCw16hgTurTOdawv0DrjDVsedVF4,5369
+airflow/providers/elasticsearch/log/es_task_handler.py,sha256=9XcgfW5N4sg-_mNCcO3Ty-ymea_hK6FZKMYzZypKI3Q,23850
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/METADATA,sha256=lZAnKAhKyo0jywfOJTDoVYFJocIgfNtbRqCLKZJyFoQ,5951
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/entry_points.txt,sha256=dAJ1Ib0gGGXb1p2Gd9pMCFzyIXRCqHax_eno9jpWDWk,110
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
+apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info/RECORD,,

{apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info → apache_airflow_providers_elasticsearch-5.0.2rc1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.1)
+Generator: bdist_wheel (0.41.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/RECORD

@@ -1,15 +0,0 @@
-airflow/providers/elasticsearch/__init__.py,sha256=txdMTGaxqJAdwftO-0rvZHI1JIyvGxNZpXOHY-AX3aI,1582
-airflow/providers/elasticsearch/get_provider_info.py,sha256=HkjZldBjbIx-zS1HcUNRrBRDtv1uxZqxbzq4h8pm6wg,7659
-airflow/providers/elasticsearch/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=r5JKHB20SBQS6bGy0o1t4_G2DIxe2HANU0YTLWP41JM,6481
-airflow/providers/elasticsearch/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/elasticsearch/log/es_json_formatter.py,sha256=IBCC6U8XUIfZ1vZKeDAQWCwNCXhP5yG2oOjSJesuGV0,1640
-airflow/providers/elasticsearch/log/es_response.py,sha256=3ZSIdVcI--7_Ouw32HicIijS0D15NoRxlNaN-FvnBC8,5374
-airflow/providers/elasticsearch/log/es_task_handler.py,sha256=rD659Y86XC5iEGG488opPLJpmLcyUPcau3uMJTUsUCE,22986
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/METADATA,sha256=pdBp6tF2B1V471wkSseDpGVLKLlRA4LWqH3JZI7d2QY,5951
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/entry_points.txt,sha256=dAJ1Ib0gGGXb1p2Gd9pMCFzyIXRCqHax_eno9jpWDWk,110
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
-apache_airflow_providers_elasticsearch-5.0.1rc1.dist-info/RECORD,,