apache-airflow-providers-elasticsearch 5.4.0rc2__py3-none-any.whl → 5.4.1rc1__py3-none-any.whl

airflow/providers/elasticsearch/__init__.py

@@ -25,14 +25,11 @@ from __future__ import annotations
 
 import packaging.version
 
-__all__ = ["__version__"]
+from airflow import __version__ as airflow_version
 
-__version__ = "5.4.0"
+__all__ = ["__version__"]
 
-try:
-    from airflow import __version__ as airflow_version
-except ImportError:
-    from airflow.version import version as airflow_version
+__version__ = "5.4.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

airflow/providers/elasticsearch/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Elasticsearch",
         "description": "`Elasticsearch <https://www.elastic.co/elasticsearch>`__\n",
         "state": "ready",
-        "source-date-epoch": 1714476311,
+        "source-date-epoch": 1716287658,
         "versions": [
+            "5.4.1",
             "5.4.0",
             "5.3.4",
             "5.3.3",
@@ -163,12 +164,19 @@ def get_provider_info():
                         "default": "offset",
                     },
                     "index_patterns": {
-                        "description": "Comma separated list of index patterns to use when searching for logs (default: `_all`).\n",
+                        "description": "Comma separated list of index patterns to use when searching for logs (default: `_all`).\nThe index_patterns_callable takes precedence over this.\n",
                         "version_added": "2.6.0",
                         "type": "string",
                         "example": "something-*",
                         "default": "_all",
                     },
+                    "index_patterns_callable": {
+                        "description": "A string representing the full path to the Python callable path which accept TI object and\nreturn comma separated list of index patterns. This will takes precedence over index_patterns.\n",
+                        "version_added": "5.5.0",
+                        "type": "string",
+                        "example": "module.callable",
+                        "default": "",
+                    },
                 },
             },
             "elasticsearch_configs": {

airflow/providers/elasticsearch/log/es_task_handler.py

@@ -41,6 +41,7 @@ from airflow.providers.elasticsearch.log.es_response import ElasticSearchRespons
 from airflow.utils import timezone
 from airflow.utils.log.file_task_handler import FileTaskHandler
 from airflow.utils.log.logging_mixin import ExternalLoggingMixin, LoggingMixin
+from airflow.utils.module_loading import import_string
 from airflow.utils.session import create_session
 
 if TYPE_CHECKING:
@@ -108,7 +109,7 @@ def _ensure_ti(ti: TaskInstanceKey | TaskInstance, session) -> TaskInstance:
         .one_or_none()
     )
     if isinstance(val, TaskInstance):
-        val._try_number = ti.try_number
+        val.try_number = ti.try_number
         return val
     else:
         raise AirflowException(f"Could not find TaskInstance for {ti}")
@@ -152,7 +153,8 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         offset_field: str = "offset",
         host: str = "http://localhost:9200",
         frontend: str = "localhost:5601",
-        index_patterns: str | None = conf.get("elasticsearch", "index_patterns", fallback="_all"),
+        index_patterns: str = conf.get("elasticsearch", "index_patterns"),
+        index_patterns_callable: str = conf.get("elasticsearch", "index_patterns_callable", fallback=""),
         es_kwargs: dict | None | Literal["default_es_kwargs"] = "default_es_kwargs",
         *,
         filename_template: str | None = None,
@@ -184,6 +186,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         self.host_field = host_field
         self.offset_field = offset_field
         self.index_patterns = index_patterns
+        self.index_patterns_callable = index_patterns_callable
         self.context_set = False
 
         self.formatter: logging.Formatter
@@ -213,6 +216,19 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         return host
 
+    def _get_index_patterns(self, ti: TaskInstance | None) -> str:
+        """
+        Get index patterns by calling index_patterns_callable, if provided, or the configured index_patterns.
+
+        :param ti: A TaskInstance object or None.
+        """
+        if self.index_patterns_callable:
+            self.log.debug("Using index_patterns_callable: %s", self.index_patterns_callable)
+            index_pattern_callable_obj = import_string(self.index_patterns_callable)
+            return index_pattern_callable_obj(ti)
+        self.log.debug("Using index_patterns: %s", self.index_patterns)
+        return self.index_patterns
+
     def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> str:
         from airflow.models.taskinstance import TaskInstanceKey
 
@@ -302,7 +318,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         offset = metadata["offset"]
         log_id = self._render_log_id(ti, try_number)
-        response = self._es_read(log_id, offset)
+        response = self._es_read(log_id, offset, ti)
         if response is not None and response.hits:
             logs_by_host = self._group_logs_by_host(response)
             next_offset = attrgetter(self.offset_field)(response[-1])
@@ -372,12 +388,13 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         # Just a safe-guard to preserve backwards-compatibility
         return hit.message
 
-    def _es_read(self, log_id: str, offset: int | str) -> ElasticSearchResponse | None:
+    def _es_read(self, log_id: str, offset: int | str, ti: TaskInstance) -> ElasticSearchResponse | None:
         """
         Return the logs matching log_id in Elasticsearch and next offset or ''.
 
         :param log_id: the log_id of the log to read.
         :param offset: the offset start to read log from.
+        :param ti: the task instance object
 
         :meta private:
         """
@@ -388,16 +405,17 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
             }
         }
 
+        index_patterns = self._get_index_patterns(ti)
         try:
-            max_log_line = self.client.count(index=self.index_patterns, query=query)["count"]  # type: ignore
+            max_log_line = self.client.count(index=index_patterns, query=query)["count"]  # type: ignore
         except NotFoundError as e:
-            self.log.exception("The target index pattern %s does not exist", self.index_patterns)
+            self.log.exception("The target index pattern %s does not exist", index_patterns)
             raise e
 
         if max_log_line != 0:
             try:
                 res = self.client.search(
-                    index=self.index_patterns,
+                    index=index_patterns,
                     query=query,
                     sort=[self.offset_field],
                     size=self.MAX_LINE_PER_PAGE,

{apache_airflow_providers_elasticsearch-5.4.0rc2.dist-info → apache_airflow_providers_elasticsearch-5.4.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-elasticsearch
-Version: 5.4.0rc2
+Version: 5.4.1rc1
 Summary: Provider package apache-airflow-providers-elasticsearch for Apache Airflow
 Keywords: airflow-provider,elasticsearch,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -26,8 +26,8 @@ Requires-Dist: apache-airflow>=2.7.0rc0
 Requires-Dist: elasticsearch>=8.10,<9
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -78,7 +78,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-elasticsearch``
 
-Release: ``5.4.0.rc2``
+Release: ``5.4.1.rc1``
 
 
 `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -91,7 +91,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
 are in ``airflow.providers.elasticsearch`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/>`_.
 
 Installation
 ------------
@@ -133,4 +133,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html>`_.

{apache_airflow_providers_elasticsearch-5.4.0rc2.dist-info → apache_airflow_providers_elasticsearch-5.4.1rc1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 airflow/providers/elasticsearch/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
-airflow/providers/elasticsearch/__init__.py,sha256=Shw3-aCrJQlbq0E5bUNubbMMcqFxQpVjhIjKUtNr0mo,1588
-airflow/providers/elasticsearch/get_provider_info.py,sha256=u5KPbym8U7gGdhXy3oBQHXTa35PDIDNw_-Ns05iuJx0,7941
+airflow/providers/elasticsearch/__init__.py,sha256=leQgUnvVqq9KHi7Uffo5O3HuM6OldRNLpqcwJxlBFjE,1500
+airflow/providers/elasticsearch/get_provider_info.py,sha256=b7kZ8E03LE7dotRl0fc2ZbPpfpf8EkJ7unuUmpxNwSQ,8507
 airflow/providers/elasticsearch/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/elasticsearch/hooks/elasticsearch.py,sha256=nraT_vR1vm0VZ13G4lIJ9YEI0igsm-A66BjFlf-yNmE,6362
 airflow/providers/elasticsearch/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/elasticsearch/log/es_json_formatter.py,sha256=DwWPDJtZLr_6Mdae1-XVEgmE1XErFIanSzxWovs50ig,1796
 airflow/providers/elasticsearch/log/es_response.py,sha256=tEFz1pAUzIg6Zw43ZjiHFTOB72kNP-czbEwMbymUZ88,6037
-airflow/providers/elasticsearch/log/es_task_handler.py,sha256=HgPdS8KI7ntuZTkkM_Fe6dDtgo-eUaGhM8eBfJ46YR8,25310
-apache_airflow_providers_elasticsearch-5.4.0rc2.dist-info/entry_points.txt,sha256=jpgAUVmTsdtWQ4nru2FJQKP9JBN4OPHK-ybfYc3_BOs,109
-apache_airflow_providers_elasticsearch-5.4.0rc2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_elasticsearch-5.4.0rc2.dist-info/METADATA,sha256=oKJKAQsjfXXOIOKHj7nfBk7KfBkESSjDxOFA6rBB8kM,6230
-apache_airflow_providers_elasticsearch-5.4.0rc2.dist-info/RECORD,,
+airflow/providers/elasticsearch/log/es_task_handler.py,sha256=vgEQWlZwCp_K0jTv7KKwv6jPgXLXRCeSQ5833Wto7ts,26244
+apache_airflow_providers_elasticsearch-5.4.1rc1.dist-info/entry_points.txt,sha256=jpgAUVmTsdtWQ4nru2FJQKP9JBN4OPHK-ybfYc3_BOs,109
+apache_airflow_providers_elasticsearch-5.4.1rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_elasticsearch-5.4.1rc1.dist-info/METADATA,sha256=Ls3_xbeJ1BIoytibTNsvykeIdAkDa2oiz6oxjjWgtKY,6230
+apache_airflow_providers_elasticsearch-5.4.1rc1.dist-info/RECORD,,