apache-airflow-providers-elasticsearch 5.4.0rc1__py3-none-any.whl → 5.4.1__py3-none-any.whl

airflow/providers/elasticsearch/__init__.py

@@ -25,18 +25,15 @@ from __future__ import annotations
 
 import packaging.version
 
-__all__ = ["__version__"]
+from airflow import __version__ as airflow_version
 
-__version__ = "5.4.0"
+__all__ = ["__version__"]
 
-try:
-    from airflow import __version__ as airflow_version
-except ImportError:
-    from airflow.version import version as airflow_version
+__version__ = "5.4.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.6.0"
+    "2.7.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-elasticsearch:{__version__}` needs Apache Airflow 2.6.0+"
+        f"The package `apache-airflow-providers-elasticsearch:{__version__}` needs Apache Airflow 2.7.0+"
     )

airflow/providers/elasticsearch/get_provider_info.py

@@ -28,9 +28,13 @@ def get_provider_info():
         "name": "Elasticsearch",
         "description": "`Elasticsearch <https://www.elastic.co/elasticsearch>`__\n",
         "state": "ready",
-        "source-date-epoch": 1705912070,
+        "source-date-epoch": 1716287658,
         "versions": [
+            "5.4.1",
             "5.4.0",
+            "5.3.4",
+            "5.3.3",
+            "5.3.2",
             "5.3.1",
             "5.3.0",
             "5.2.0",
@@ -66,7 +70,7 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.6.0",
+            "apache-airflow>=2.7.0",
             "apache-airflow-providers-common-sql>=1.3.1",
             "elasticsearch>=8.10,<9",
         ],
@@ -160,12 +164,19 @@ def get_provider_info():
                         "default": "offset",
                     },
                     "index_patterns": {
-                        "description": "Comma separated list of index patterns to use when searching for logs (default: `_all`).\n",
+                        "description": "Comma separated list of index patterns to use when searching for logs (default: `_all`).\nThe index_patterns_callable takes precedence over this.\n",
                         "version_added": "2.6.0",
                         "type": "string",
                         "example": "something-*",
                         "default": "_all",
                     },
+                    "index_patterns_callable": {
+                        "description": "A string representing the full path to the Python callable path which accept TI object and\nreturn comma separated list of index patterns. This will takes precedence over index_patterns.\n",
+                        "version_added": "5.5.0",
+                        "type": "string",
+                        "example": "module.callable",
+                        "default": "",
+                    },
                 },
             },
             "elasticsearch_configs": {

airflow/providers/elasticsearch/hooks/elasticsearch.py

@@ -17,11 +17,11 @@
 # under the License.
 from __future__ import annotations
 
-import warnings
 from functools import cached_property
 from typing import TYPE_CHECKING, Any
 from urllib import parse
 
+from deprecated import deprecated
 from elasticsearch import Elasticsearch
 
 from airflow.exceptions import AirflowProviderDeprecationWarning
@@ -138,6 +138,10 @@ class ElasticsearchSQLHook(DbApiHook):
         return uri
 
 
+@deprecated(
+    reason="Please use `airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook`.",
+    category=AirflowProviderDeprecationWarning,
+)
 class ElasticsearchHook(ElasticsearchSQLHook):
     """
     This class is deprecated and was renamed to ElasticsearchSQLHook.
@@ -146,12 +150,6 @@ class ElasticsearchHook(ElasticsearchSQLHook):
     """
 
     def __init__(self, *args, **kwargs):
-        warnings.warn(
-            """This class is deprecated.
-            Please use `airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchSQLHook`.""",
-            AirflowProviderDeprecationWarning,
-            stacklevel=3,
-        )
         super().__init__(*args, **kwargs)
 
 
@@ -164,8 +162,8 @@ class ElasticsearchPythonHook(BaseHook):
                                 Example: {"ca_cert":"/path/to/cert", "basic_auth": "(user, pass)"}
     """
 
-    def __init__(self, hosts: list[Any], es_conn_args: dict | None = None, **kwargs):
-        super().__init__(**kwargs)
+    def __init__(self, hosts: list[Any], es_conn_args: dict | None = None):
+        super().__init__()
         self.hosts = hosts
         self.es_conn_args = es_conn_args or {}
 

airflow/providers/elasticsearch/log/es_json_formatter.py

@@ -16,6 +16,8 @@
 # under the License.
 from __future__ import annotations
 
+from datetime import datetime
+
 import pendulum
 
 from airflow.utils.log.json_formatter import JSONFormatter
@@ -30,7 +32,9 @@ class ElasticsearchJSONFormatter(JSONFormatter):
 
     def formatTime(self, record, datefmt=None):
         """Return the creation time of the LogRecord in ISO 8601 date/time format in the local time zone."""
-        dt = pendulum.from_timestamp(record.created, tz=pendulum.local_timezone())
+        # TODO: Use airflow.utils.timezone.from_timestamp(record.created, tz="local")
+        #  as soon as min Airflow 2.9.0
+        dt = datetime.fromtimestamp(record.created, tz=pendulum.local_timezone())
         s = dt.strftime(datefmt or self.default_time_format)
         if self.default_msec_format:
             s = self.default_msec_format % (s, record.msecs)

airflow/providers/elasticsearch/log/es_response.py

@@ -34,15 +34,18 @@ class AttributeList:
         self._l_ = _list
 
     def __getitem__(self, k):
+        """Retrieve an item or a slice from the list. If the item is a dictionary, it is wrapped in an AttributeDict."""
         val = self._l_[k]
         if isinstance(val, slice):
             return AttributeList(val)
         return _wrap(val)
 
     def __iter__(self):
+        """Provide an iterator for the list or the dictionary."""
         return (_wrap(i) for i in self._l_)
 
     def __bool__(self):
+        """Check if the list is non-empty."""
         return bool(self._l_)
 
 
@@ -53,12 +56,14 @@ class AttributeDict:
         super().__setattr__("_d_", d)
 
     def __getattr__(self, attr_name):
+        """Retrieve an item as an attribute from the dictionary."""
         try:
             return self.__getitem__(attr_name)
         except KeyError:
             raise AttributeError(f"{self.__class__.__name__!r} object has no attribute {attr_name!r}")
 
     def __getitem__(self, key):
+        """Retrieve an item using a key from the dictionary."""
         return _wrap(self._d_[key])
 
     def to_dict(self):
@@ -120,14 +125,17 @@ class ElasticSearchResponse(AttributeDict):
         super().__init__(response)
 
     def __iter__(self) -> Iterator[Hit]:
+        """Provide an iterator over the hits in the Elasticsearch response."""
         return iter(self.hits)
 
     def __getitem__(self, key):
+        """Retrieve a specific hit or a slice of hits from the Elasticsearch response."""
         if isinstance(key, (slice, int)):
             return self.hits[key]
         return super().__getitem__(key)
 
     def __bool__(self):
+        """Evaluate the presence of hits in the Elasticsearch response."""
         return bool(self.hits)
 
     @property

airflow/providers/elasticsearch/log/es_task_handler.py

@@ -41,6 +41,7 @@ from airflow.providers.elasticsearch.log.es_response import ElasticSearchRespons
 from airflow.utils import timezone
 from airflow.utils.log.file_task_handler import FileTaskHandler
 from airflow.utils.log.logging_mixin import ExternalLoggingMixin, LoggingMixin
+from airflow.utils.module_loading import import_string
 from airflow.utils.session import create_session
 
 if TYPE_CHECKING:
@@ -108,7 +109,7 @@ def _ensure_ti(ti: TaskInstanceKey | TaskInstance, session) -> TaskInstance:
         .one_or_none()
     )
     if isinstance(val, TaskInstance):
-        val._try_number = ti.try_number
+        val.try_number = ti.try_number
         return val
     else:
         raise AirflowException(f"Could not find TaskInstance for {ti}")
@@ -152,7 +153,8 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         offset_field: str = "offset",
         host: str = "http://localhost:9200",
         frontend: str = "localhost:5601",
-        index_patterns: str | None = conf.get("elasticsearch", "index_patterns", fallback="_all"),
+        index_patterns: str = conf.get("elasticsearch", "index_patterns"),
+        index_patterns_callable: str = conf.get("elasticsearch", "index_patterns_callable", fallback=""),
         es_kwargs: dict | None | Literal["default_es_kwargs"] = "default_es_kwargs",
         *,
         filename_template: str | None = None,
@@ -184,6 +186,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         self.host_field = host_field
         self.offset_field = offset_field
         self.index_patterns = index_patterns
+        self.index_patterns_callable = index_patterns_callable
         self.context_set = False
 
         self.formatter: logging.Formatter
@@ -213,6 +216,19 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         return host
 
+    def _get_index_patterns(self, ti: TaskInstance | None) -> str:
+        """
+        Get index patterns by calling index_patterns_callable, if provided, or the configured index_patterns.
+
+        :param ti: A TaskInstance object or None.
+        """
+        if self.index_patterns_callable:
+            self.log.debug("Using index_patterns_callable: %s", self.index_patterns_callable)
+            index_pattern_callable_obj = import_string(self.index_patterns_callable)
+            return index_pattern_callable_obj(ti)
+        self.log.debug("Using index_patterns: %s", self.index_patterns)
+        return self.index_patterns
+
     def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> str:
         from airflow.models.taskinstance import TaskInstanceKey
 
@@ -225,6 +241,8 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
             else:
                 log_id_template = self.log_id_template
 
+        if TYPE_CHECKING:
+            assert ti.task
         try:
             dag = ti.task.dag
         except AttributeError:  # ti.task is not always set.
@@ -300,7 +318,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         offset = metadata["offset"]
         log_id = self._render_log_id(ti, try_number)
-        response = self._es_read(log_id, offset)
+        response = self._es_read(log_id, offset, ti)
         if response is not None and response.hits:
             logs_by_host = self._group_logs_by_host(response)
             next_offset = attrgetter(self.offset_field)(response[-1])
@@ -370,12 +388,13 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         # Just a safe-guard to preserve backwards-compatibility
         return hit.message
 
-    def _es_read(self, log_id: str, offset: int | str) -> ElasticSearchResponse | None:
+    def _es_read(self, log_id: str, offset: int | str, ti: TaskInstance) -> ElasticSearchResponse | None:
         """
         Return the logs matching log_id in Elasticsearch and next offset or ''.
 
         :param log_id: the log_id of the log to read.
         :param offset: the offset start to read log from.
+        :param ti: the task instance object
 
         :meta private:
         """
@@ -386,16 +405,17 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
             }
         }
 
+        index_patterns = self._get_index_patterns(ti)
         try:
-            max_log_line = self.client.count(index=self.index_patterns, query=query)["count"]  # type: ignore
+            max_log_line = self.client.count(index=index_patterns, query=query)["count"]  # type: ignore
         except NotFoundError as e:
-            self.log.exception("The target index pattern %s does not exist", self.index_patterns)
+            self.log.exception("The target index pattern %s does not exist", index_patterns)
             raise e
 
         if max_log_line != 0:
             try:
                 res = self.client.search(
-                    index=self.index_patterns,
+                    index=index_patterns,
                     query=query,
                     sort=[self.offset_field],
                     size=self.MAX_LINE_PER_PAGE,

{apache_airflow_providers_elasticsearch-5.4.0rc1.dist-info → apache_airflow_providers_elasticsearch-5.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-elasticsearch
-Version: 5.4.0rc1
+Version: 5.4.1
 Summary: Provider package apache-airflow-providers-elasticsearch for Apache Airflow
 Keywords: airflow-provider,elasticsearch,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -19,14 +19,15 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1.dev0
-Requires-Dist: apache-airflow>=2.6.0.dev0
+Requires-Dist: apache-airflow-providers-common-sql>=1.3.1
+Requires-Dist: apache-airflow>=2.7.0
 Requires-Dist: elasticsearch>=8.10,<9
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -77,7 +78,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-elasticsearch``
 
-Release: ``5.4.0.rc1``
+Release: ``5.4.1``
 
 
 `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -90,7 +91,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
 are in ``airflow.providers.elasticsearch`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/>`_.
 
 Installation
 ------------
@@ -99,7 +100,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-elasticsearch``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11
+The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -107,7 +108,7 @@ Requirements
 =======================================  ==================
 PIP package                              Version required
 =======================================  ==================
-``apache-airflow``                       ``>=2.6.0``
+``apache-airflow``                       ``>=2.7.0``
 ``apache-airflow-providers-common-sql``  ``>=1.3.1``
 ``elasticsearch``                        ``>=8.10,<9``
 =======================================  ==================
@@ -132,4 +133,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html>`_.

apache_airflow_providers_elasticsearch-5.4.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+airflow/providers/elasticsearch/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
+airflow/providers/elasticsearch/__init__.py,sha256=leQgUnvVqq9KHi7Uffo5O3HuM6OldRNLpqcwJxlBFjE,1500
+airflow/providers/elasticsearch/get_provider_info.py,sha256=b7kZ8E03LE7dotRl0fc2ZbPpfpf8EkJ7unuUmpxNwSQ,8507
+airflow/providers/elasticsearch/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=nraT_vR1vm0VZ13G4lIJ9YEI0igsm-A66BjFlf-yNmE,6362
+airflow/providers/elasticsearch/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/elasticsearch/log/es_json_formatter.py,sha256=DwWPDJtZLr_6Mdae1-XVEgmE1XErFIanSzxWovs50ig,1796
+airflow/providers/elasticsearch/log/es_response.py,sha256=tEFz1pAUzIg6Zw43ZjiHFTOB72kNP-czbEwMbymUZ88,6037
+airflow/providers/elasticsearch/log/es_task_handler.py,sha256=vgEQWlZwCp_K0jTv7KKwv6jPgXLXRCeSQ5833Wto7ts,26244
+apache_airflow_providers_elasticsearch-5.4.1.dist-info/entry_points.txt,sha256=jpgAUVmTsdtWQ4nru2FJQKP9JBN4OPHK-ybfYc3_BOs,109
+apache_airflow_providers_elasticsearch-5.4.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_elasticsearch-5.4.1.dist-info/METADATA,sha256=73QEnaM9VMtHOLQ1edfLeNQ0h5hxu2HNPxi_i9LJjiY,6217
+apache_airflow_providers_elasticsearch-5.4.1.dist-info/RECORD,,

apache_airflow_providers_elasticsearch-5.4.0rc1.dist-info/RECORD

@@ -1,13 +0,0 @@
-airflow/providers/elasticsearch/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
-airflow/providers/elasticsearch/__init__.py,sha256=OyxIHdA-PcxE5vOaA2ZbScRgTrXifx4xfo9nQmWyBEo,1588
-airflow/providers/elasticsearch/get_provider_info.py,sha256=HCp7W6EQYrczJLrDOsl5Bg22st23Vw7Q20daeKyxc6o,7878
-airflow/providers/elasticsearch/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=V2tMQKKGZgWwTnrc7n_-IW6XrFIY6oAygqgcSpXTUDk,6448
-airflow/providers/elasticsearch/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/elasticsearch/log/es_json_formatter.py,sha256=IBCC6U8XUIfZ1vZKeDAQWCwNCXhP5yG2oOjSJesuGV0,1640
-airflow/providers/elasticsearch/log/es_response.py,sha256=TSZSkf63q-uM6dju4FJXx-GVcmsbxSdx9tMdcBTXmaA,5428
-airflow/providers/elasticsearch/log/es_task_handler.py,sha256=zXMZM13Yvc80pZNmftxL6YXcuIrCs-2wQ9-KWCUYut0,25257
-apache_airflow_providers_elasticsearch-5.4.0rc1.dist-info/entry_points.txt,sha256=jpgAUVmTsdtWQ4nru2FJQKP9JBN4OPHK-ybfYc3_BOs,109
-apache_airflow_providers_elasticsearch-5.4.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_elasticsearch-5.4.0rc1.dist-info/METADATA,sha256=aeEiQsqw8chv_CwivuLryg0Q00CxOQOsjPKKWnexnsE,6178
-apache_airflow_providers_elasticsearch-5.4.0rc1.dist-info/RECORD,,