apache-airflow-providers-elasticsearch 6.3.1__py3-none-any.whl → 6.3.2__py3-none-any.whl

airflow/providers/elasticsearch/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "6.3.1"
+__version__ = "6.3.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

airflow/providers/elasticsearch/hooks/elasticsearch.py

@@ -26,11 +26,7 @@ from urllib import parse
 from elasticsearch import Elasticsearch
 
 from airflow.providers.common.sql.hooks.sql import DbApiHook
-
-try:
-    from airflow.sdk import BaseHook
-except ImportError:
-    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
+from airflow.providers.elasticsearch.version_compat import BaseHook
 
 if TYPE_CHECKING:
     from elastic_transport import ObjectApiResponse

airflow/providers/elasticsearch/log/es_task_handler.py

@@ -29,7 +29,7 @@ import time
 from collections import defaultdict
 from collections.abc import Callable
 from operator import attrgetter
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any, Literal, cast
 from urllib.parse import quote, urlparse
 
 # Using `from elasticsearch import *` would break elasticsearch mocking used in unit test.
@@ -56,6 +56,7 @@ if TYPE_CHECKING:
     from datetime import datetime
 
     from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
+    from airflow.utils.log.file_task_handler import LogMetadata
 
 
 LOG_LINE_DEFAULTS = {"exc_text": "", "stack_info": ""}
@@ -66,6 +67,7 @@ LOG_LINE_DEFAULTS = {"exc_text": "", "stack_info": ""}
 # not exist, the task handler should use the log_id_template attribute instead.
 USE_PER_RUN_LOG_ID = hasattr(DagRun, "get_log_template")
 
+TASK_LOG_FIELDS = ["timestamp", "event", "level", "chan", "logger"]
 
 VALID_ES_CONFIG_KEYS = set(inspect.signature(elasticsearch.Elasticsearch.__init__).parameters.keys())
 # Remove `self` from the valid set of kwargs
@@ -159,11 +161,11 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         es_kwargs = es_kwargs or {}
         if es_kwargs == "default_es_kwargs":
             es_kwargs = get_es_kwargs_from_config()
-        host = self.format_url(host)
+        self.host = self.format_url(host)
         super().__init__(base_log_folder)
         self.closed = False
 
-        self.client = elasticsearch.Elasticsearch(host, **es_kwargs)
+        self.client = elasticsearch.Elasticsearch(self.host, **es_kwargs)
         # in airflow.cfg, host of elasticsearch has to be http://dockerhostXxxx:9200
 
         self.frontend = frontend
@@ -184,7 +186,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         )
 
         self.formatter: logging.Formatter
-        self.handler: logging.FileHandler | logging.StreamHandler  # type: ignore[assignment]
+        self.handler: logging.FileHandler | logging.StreamHandler | None = None
         self._doc_type_map: dict[Any, Any] = {}
         self._doc_type: list[Any] = []
 
@@ -286,7 +288,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
     def _group_logs_by_host(self, response: ElasticSearchResponse) -> dict[str, list[Hit]]:
         grouped_logs = defaultdict(list)
         for hit in response:
-            key = getattr_nested(hit, self.host_field, None) or "default_host"
+            key = getattr_nested(hit, self.host_field, None) or self.host
             grouped_logs[key].append(hit)
         return grouped_logs
 
@@ -294,8 +296,8 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         return True
 
     def _read(
-        self, ti: TaskInstance, try_number: int, metadata: dict | None = None
-    ) -> tuple[EsLogMsgType, dict]:
+        self, ti: TaskInstance, try_number: int, metadata: LogMetadata | None = None
+    ) -> tuple[EsLogMsgType, LogMetadata]:
         """
         Endpoint for streaming log.
 
@@ -306,7 +308,9 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         :return: a list of tuple with host and log documents, metadata.
         """
         if not metadata:
-            metadata = {"offset": 0}
+            # LogMetadata(TypedDict) is used as type annotation for log_reader; added ignore to suppress mypy error
+            metadata = {"offset": 0}  # type: ignore[assignment]
+        metadata = cast("LogMetadata", metadata)
         if "offset" not in metadata:
             metadata["offset"] = 0
 
@@ -346,7 +350,9 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
                     "Otherwise, the logs for this task instance may have been removed."
                 )
                 if AIRFLOW_V_3_0_PLUS:
-                    return missing_log_message, metadata
+                    from airflow.utils.log.file_task_handler import StructuredLogMessage
+
+                    return [StructuredLogMessage(event=missing_log_message)], metadata
                 return [("", missing_log_message)], metadata  # type: ignore[list-item]
             if (
                 # Assume end of log after not receiving new log for N min,
@@ -375,11 +381,16 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
                         sources=[host for host in logs_by_host.keys()],
                     ),  # type: ignore[call-arg]
                     StructuredLogMessage(event="::endgroup::"),
-                ]  # type: ignore[misc]
+                ]
 
+                # Flatten all hits, filter to only desired fields, and construct StructuredLogMessage objects
                 message = header + [
-                    StructuredLogMessage(event=concat_logs(hits)) for hits in logs_by_host.values()
-                ]  # type: ignore[misc]
+                    StructuredLogMessage(
+                        **{k: v for k, v in hit.to_dict().items() if k.lower() in TASK_LOG_FIELDS}
+                    )
+                    for hits in logs_by_host.values()
+                    for hit in hits
+                ]
             else:
                 message = [
                     (host, concat_logs(hits))  # type: ignore[misc]
@@ -421,7 +432,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         index_patterns = self._get_index_patterns(ti)
         try:
-            max_log_line = self.client.count(index=index_patterns, query=query)["count"]  # type: ignore
+            max_log_line = self.client.count(index=index_patterns, query=query)["count"]
         except NotFoundError as e:
             self.log.exception("The target index pattern %s does not exist", index_patterns)
             raise e
@@ -508,7 +519,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         # Reopen the file stream, because FileHandler.close() would be called
         # first in logging.shutdown() and the stream in it would be set to None.
-        if self.handler.stream is None or self.handler.stream.closed:  # type: ignore[attr-defined]
+        if self.handler.stream is None or self.handler.stream.closed:
             self.handler.stream = self.handler._open()  # type: ignore[union-attr]
 
         # Mark the end of file using end of log mark,

airflow/providers/elasticsearch/version_compat.py

@@ -33,10 +33,18 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
 
 
 AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
+AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0)
+
+if AIRFLOW_V_3_1_PLUS:
+    from airflow.sdk import BaseHook
+else:
+    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
 
 if AIRFLOW_V_3_0_PLUS:
     from airflow.utils.log.file_task_handler import StructuredLogMessage
 
     EsLogMsgType = list[StructuredLogMessage] | str
 else:
-    EsLogMsgType = list[tuple[str, str]]  # type: ignore[misc]
+    EsLogMsgType = list[tuple[str, str]]  # type: ignore[assignment,misc]
+
+__all__ = ["AIRFLOW_V_3_0_PLUS", "AIRFLOW_V_3_1_PLUS", "BaseHook", "EsLogMsgType"]

{apache_airflow_providers_elasticsearch-6.3.1.dist-info → apache_airflow_providers_elasticsearch-6.3.2.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-elasticsearch
-Version: 6.3.1
+Version: 6.3.2
 Summary: Provider package apache-airflow-providers-elasticsearch for Apache Airflow
 Keywords: airflow-provider,elasticsearch,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.10
+Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -18,13 +18,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow>=2.10.0
 Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
 Requires-Dist: elasticsearch>=8.10,<9
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.2/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.2
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -55,8 +56,9 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-elasticsearch``
 
-Release: ``6.3.1``
+Release: ``6.3.2``
 
+Release Date: ``|PypiReleaseDate|``
 
 `Elasticsearch <https://www.elastic.co/elasticsearch>`__
 
@@ -68,7 +70,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
 are in ``airflow.providers.elasticsearch`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.2/>`_.
 
 Installation
 ------------
@@ -77,7 +79,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-elasticsearch``
 
-The package supports the following python versions: 3.10,3.11,3.12
+The package supports the following python versions: 3.10,3.11,3.12,3.13
 
 Requirements
 ------------
@@ -110,5 +112,5 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.2/changelog.html>`_.
 

{apache_airflow_providers_elasticsearch-6.3.1.dist-info → apache_airflow_providers_elasticsearch-6.3.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
 airflow/providers/elasticsearch/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/elasticsearch/__init__.py,sha256=tQJpEmqy_cj4UtUzW3ZEOYEQzXdrmsVE9FiwNNX7wwk,1502
+airflow/providers/elasticsearch/__init__.py,sha256=QyVuquS4SFu8rW3mF_Z6J8B-_xBf749nRxohrJ4Fp7Y,1502
 airflow/providers/elasticsearch/get_provider_info.py,sha256=473iBNJ4Kiy-DvbOa6CqDOS3pPEs6x7WBwkhR1QTuiw,8106
-airflow/providers/elasticsearch/version_compat.py,sha256=ZwhtGeuktt0--08loaNiOkmkdSO14adQdRFsw64qQdo,1755
+airflow/providers/elasticsearch/version_compat.py,sha256=VoYY99deuWiC8ya1zauXtFNSb6MBplEtBvc6zAgb6oM,2073
 airflow/providers/elasticsearch/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=oDYFA-V-VEa-DWWBpuYEUOrYyccbd7Dxftpe6xpHLtM,8661
+airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=hHbNKnsy8bNHjV-ufZYBXSOw2dj5JmicOub7VTohy7A,8583
 airflow/providers/elasticsearch/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/elasticsearch/log/es_json_formatter.py,sha256=DwWPDJtZLr_6Mdae1-XVEgmE1XErFIanSzxWovs50ig,1796
 airflow/providers/elasticsearch/log/es_response.py,sha256=LdMBuVBTydaC42HooYSttAjTK-CpPA4r_KHl38msMnk,6046
-airflow/providers/elasticsearch/log/es_task_handler.py,sha256=VdiYogPgpQBnzREFdK95oqYx-MLLOtg0Wr0XD3OI5lg,28299
-apache_airflow_providers_elasticsearch-6.3.1.dist-info/entry_points.txt,sha256=jpgAUVmTsdtWQ4nru2FJQKP9JBN4OPHK-ybfYc3_BOs,109
-apache_airflow_providers_elasticsearch-6.3.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_elasticsearch-6.3.1.dist-info/METADATA,sha256=nvwzlDqXPAFuWsXm6WyUzaRBuQnJ810BVtoq512vrdk,5163
-apache_airflow_providers_elasticsearch-6.3.1.dist-info/RECORD,,
+airflow/providers/elasticsearch/log/es_task_handler.py,sha256=nqjmWbHkXNfroDeQn611jMOODpLKNuCA-SUkJ1QwLM8,28928
+apache_airflow_providers_elasticsearch-6.3.2.dist-info/entry_points.txt,sha256=jpgAUVmTsdtWQ4nru2FJQKP9JBN4OPHK-ybfYc3_BOs,109
+apache_airflow_providers_elasticsearch-6.3.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_elasticsearch-6.3.2.dist-info/METADATA,sha256=kx20yrITa4jQD7i3y0EDS6_zSsDDNyR5DhYrmGp3ESU,5255
+apache_airflow_providers_elasticsearch-6.3.2.dist-info/RECORD,,