apache-airflow-providers-elasticsearch 5.4.0rc2__tar.gz → 5.4.1__tar.gz

{apache_airflow_providers_elasticsearch-5.4.0rc2 → apache_airflow_providers_elasticsearch-5.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-elasticsearch
-Version: 5.4.0rc2
+Version: 5.4.1
 Summary: Provider package apache-airflow-providers-elasticsearch for Apache Airflow
 Keywords: airflow-provider,elasticsearch,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,13 +21,13 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1rc0
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.3.1
+Requires-Dist: apache-airflow>=2.7.0
 Requires-Dist: elasticsearch>=8.10,<9
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -78,7 +78,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-elasticsearch``
 
-Release: ``5.4.0.rc2``
+Release: ``5.4.1``
 
 
 `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -91,7 +91,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
 are in ``airflow.providers.elasticsearch`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/>`_.
 
 Installation
 ------------
@@ -133,4 +133,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html>`_.

{apache_airflow_providers_elasticsearch-5.4.0rc2 → apache_airflow_providers_elasticsearch-5.4.1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-elasticsearch``
 
-Release: ``5.4.0.rc2``
+Release: ``5.4.1``
 
 
 `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -55,7 +55,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
 are in ``airflow.providers.elasticsearch`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/>`_.
 
 Installation
 ------------
@@ -97,4 +97,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html>`_.

{apache_airflow_providers_elasticsearch-5.4.0rc2 → apache_airflow_providers_elasticsearch-5.4.1}/airflow/providers/elasticsearch/__init__.py

@@ -25,14 +25,11 @@ from __future__ import annotations
 
 import packaging.version
 
-__all__ = ["__version__"]
+from airflow import __version__ as airflow_version
 
-__version__ = "5.4.0"
+__all__ = ["__version__"]
 
-try:
-    from airflow import __version__ as airflow_version
-except ImportError:
-    from airflow.version import version as airflow_version
+__version__ = "5.4.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

{apache_airflow_providers_elasticsearch-5.4.0rc2 → apache_airflow_providers_elasticsearch-5.4.1}/airflow/providers/elasticsearch/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Elasticsearch",
         "description": "`Elasticsearch <https://www.elastic.co/elasticsearch>`__\n",
         "state": "ready",
-        "source-date-epoch": 1714476311,
+        "source-date-epoch": 1716287658,
         "versions": [
+            "5.4.1",
             "5.4.0",
             "5.3.4",
             "5.3.3",
@@ -163,12 +164,19 @@ def get_provider_info():
                         "default": "offset",
                     },
                     "index_patterns": {
-                        "description": "Comma separated list of index patterns to use when searching for logs (default: `_all`).\n",
+                        "description": "Comma separated list of index patterns to use when searching for logs (default: `_all`).\nThe index_patterns_callable takes precedence over this.\n",
                         "version_added": "2.6.0",
                         "type": "string",
                         "example": "something-*",
                         "default": "_all",
                     },
+                    "index_patterns_callable": {
+                        "description": "A string representing the full path to the Python callable path which accept TI object and\nreturn comma separated list of index patterns. This will takes precedence over index_patterns.\n",
+                        "version_added": "5.5.0",
+                        "type": "string",
+                        "example": "module.callable",
+                        "default": "",
+                    },
                 },
             },
             "elasticsearch_configs": {

{apache_airflow_providers_elasticsearch-5.4.0rc2 → apache_airflow_providers_elasticsearch-5.4.1}/airflow/providers/elasticsearch/log/es_task_handler.py

@@ -41,6 +41,7 @@ from airflow.providers.elasticsearch.log.es_response import ElasticSearchRespons
 from airflow.utils import timezone
 from airflow.utils.log.file_task_handler import FileTaskHandler
 from airflow.utils.log.logging_mixin import ExternalLoggingMixin, LoggingMixin
+from airflow.utils.module_loading import import_string
 from airflow.utils.session import create_session
 
 if TYPE_CHECKING:
@@ -108,7 +109,7 @@ def _ensure_ti(ti: TaskInstanceKey | TaskInstance, session) -> TaskInstance:
         .one_or_none()
     )
     if isinstance(val, TaskInstance):
-        val._try_number = ti.try_number
+        val.try_number = ti.try_number
         return val
     else:
         raise AirflowException(f"Could not find TaskInstance for {ti}")
@@ -152,7 +153,8 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         offset_field: str = "offset",
         host: str = "http://localhost:9200",
         frontend: str = "localhost:5601",
-        index_patterns: str | None = conf.get("elasticsearch", "index_patterns", fallback="_all"),
+        index_patterns: str = conf.get("elasticsearch", "index_patterns"),
+        index_patterns_callable: str = conf.get("elasticsearch", "index_patterns_callable", fallback=""),
         es_kwargs: dict | None | Literal["default_es_kwargs"] = "default_es_kwargs",
         *,
         filename_template: str | None = None,
@@ -184,6 +186,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         self.host_field = host_field
         self.offset_field = offset_field
         self.index_patterns = index_patterns
+        self.index_patterns_callable = index_patterns_callable
         self.context_set = False
 
         self.formatter: logging.Formatter
@@ -213,6 +216,19 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         return host
 
+    def _get_index_patterns(self, ti: TaskInstance | None) -> str:
+        """
+        Get index patterns by calling index_patterns_callable, if provided, or the configured index_patterns.
+
+        :param ti: A TaskInstance object or None.
+        """
+        if self.index_patterns_callable:
+            self.log.debug("Using index_patterns_callable: %s", self.index_patterns_callable)
+            index_pattern_callable_obj = import_string(self.index_patterns_callable)
+            return index_pattern_callable_obj(ti)
+        self.log.debug("Using index_patterns: %s", self.index_patterns)
+        return self.index_patterns
+
     def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> str:
         from airflow.models.taskinstance import TaskInstanceKey
 
@@ -302,7 +318,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
 
         offset = metadata["offset"]
         log_id = self._render_log_id(ti, try_number)
-        response = self._es_read(log_id, offset)
+        response = self._es_read(log_id, offset, ti)
         if response is not None and response.hits:
             logs_by_host = self._group_logs_by_host(response)
             next_offset = attrgetter(self.offset_field)(response[-1])
@@ -372,12 +388,13 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
         # Just a safe-guard to preserve backwards-compatibility
         return hit.message
 
-    def _es_read(self, log_id: str, offset: int | str) -> ElasticSearchResponse | None:
+    def _es_read(self, log_id: str, offset: int | str, ti: TaskInstance) -> ElasticSearchResponse | None:
         """
         Return the logs matching log_id in Elasticsearch and next offset or ''.
 
         :param log_id: the log_id of the log to read.
         :param offset: the offset start to read log from.
+        :param ti: the task instance object
 
         :meta private:
         """
@@ -388,16 +405,17 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix
             }
         }
 
+        index_patterns = self._get_index_patterns(ti)
         try:
-            max_log_line = self.client.count(index=self.index_patterns, query=query)["count"]  # type: ignore
+            max_log_line = self.client.count(index=index_patterns, query=query)["count"]  # type: ignore
         except NotFoundError as e:
-            self.log.exception("The target index pattern %s does not exist", self.index_patterns)
+            self.log.exception("The target index pattern %s does not exist", index_patterns)
             raise e
 
         if max_log_line != 0:
             try:
                 res = self.client.search(
-                    index=self.index_patterns,
+                    index=index_patterns,
                     query=query,
                     sort=[self.offset_field],
                     size=self.MAX_LINE_PER_PAGE,

{apache_airflow_providers_elasticsearch-5.4.0rc2 → apache_airflow_providers_elasticsearch-5.4.1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-elasticsearch"
-version = "5.4.0.rc2"
+version = "5.4.1"
 description = "Provider package apache-airflow-providers-elasticsearch for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -56,14 +56,14 @@ classifiers = [
 ]
 requires-python = "~=3.8"
 dependencies = [
-    "apache-airflow-providers-common-sql>=1.3.1rc0",
-    "apache-airflow>=2.7.0rc0",
+    "apache-airflow-providers-common-sql>=1.3.1",
+    "apache-airflow>=2.7.0",
     "elasticsearch>=8.10,<9",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/5.4.1/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"