apache-airflow-providers-elasticsearch 6.3.0__tar.gz → 6.3.1rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of apache-airflow-providers-elasticsearch might be problematic. Click here for more details.

Files changed (46) hide show
  1. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/PKG-INFO +10 -11
  2. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/README.rst +4 -4
  3. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/changelog.rst +15 -0
  4. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/index.rst +3 -3
  5. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/provider.yaml +2 -1
  6. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/pyproject.toml +6 -7
  7. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/__init__.py +1 -1
  8. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/hooks/elasticsearch.py +10 -6
  9. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/log/es_task_handler.py +3 -11
  10. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/version_compat.py +7 -0
  11. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/test_es_task_handler.py +26 -2
  12. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/.latest-doc-only-change.txt +0 -0
  13. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/commits.rst +0 -0
  14. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/conf.py +0 -0
  15. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/configurations-ref.rst +0 -0
  16. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/connections/elasticsearch.rst +0 -0
  17. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/hooks/elasticsearch_python_hook.rst +0 -0
  18. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/hooks/elasticsearch_sql_hook.rst +0 -0
  19. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/hooks/index.rst +0 -0
  20. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/installing-providers-from-sources.rst +0 -0
  21. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/integration-logos/Elasticsearch.png +0 -0
  22. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/logging/index.rst +0 -0
  23. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/redirects.txt +0 -0
  24. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/docs/security.rst +0 -0
  25. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/__init__.py +0 -0
  26. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/__init__.py +0 -0
  27. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/LICENSE +0 -0
  28. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/get_provider_info.py +0 -0
  29. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/hooks/__init__.py +0 -0
  30. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/log/__init__.py +0 -0
  31. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/log/es_json_formatter.py +0 -0
  32. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/src/airflow/providers/elasticsearch/log/es_response.py +0 -0
  33. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/conftest.py +0 -0
  34. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/system/__init__.py +0 -0
  35. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/system/elasticsearch/__init__.py +0 -0
  36. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/system/elasticsearch/example_elasticsearch_query.py +0 -0
  37. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/__init__.py +0 -0
  38. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/__init__.py +0 -0
  39. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/hooks/__init__.py +0 -0
  40. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/hooks/test_elasticsearch.py +0 -0
  41. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/__init__.py +0 -0
  42. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/elasticmock/__init__.py +0 -0
  43. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/elasticmock/fake_elasticsearch.py +0 -0
  44. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/elasticmock/utilities/__init__.py +0 -0
  45. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/test_es_json_formatter.py +0 -0
  46. {apache_airflow_providers_elasticsearch-6.3.0 → apache_airflow_providers_elasticsearch-6.3.1rc1}/tests/unit/elasticsearch/log/test_es_response.py +0 -0
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: apache-airflow-providers-elasticsearch
3
- Version: 6.3.0
3
+ Version: 6.3.1rc1
4
4
  Summary: Provider package apache-airflow-providers-elasticsearch for Apache Airflow
5
5
  Keywords: airflow-provider,elasticsearch,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
7
7
  Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
8
- Requires-Python: ~=3.9
8
+ Requires-Python: ~=3.10
9
9
  Description-Content-Type: text/x-rst
10
10
  Classifier: Development Status :: 5 - Production/Stable
11
11
  Classifier: Environment :: Console
@@ -15,17 +15,16 @@ Classifier: Intended Audience :: System Administrators
15
15
  Classifier: Framework :: Apache Airflow
16
16
  Classifier: Framework :: Apache Airflow :: Provider
17
17
  Classifier: License :: OSI Approved :: Apache Software License
18
- Classifier: Programming Language :: Python :: 3.9
19
18
  Classifier: Programming Language :: Python :: 3.10
20
19
  Classifier: Programming Language :: Python :: 3.11
21
20
  Classifier: Programming Language :: Python :: 3.12
22
21
  Classifier: Topic :: System :: Monitoring
23
- Requires-Dist: apache-airflow>=2.10.0
24
- Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
22
+ Requires-Dist: apache-airflow>=2.10.0rc1
23
+ Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
25
24
  Requires-Dist: elasticsearch>=8.10,<9
26
25
  Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
27
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0/changelog.html
28
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0
26
+ Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html
27
+ Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1
29
28
  Project-URL: Mastodon, https://fosstodon.org/@airflow
30
29
  Project-URL: Slack Chat, https://s.apache.org/airflow-slack
31
30
  Project-URL: Source Code, https://github.com/apache/airflow
@@ -56,7 +55,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
56
55
 
57
56
  Package ``apache-airflow-providers-elasticsearch``
58
57
 
59
- Release: ``6.3.0``
58
+ Release: ``6.3.1``
60
59
 
61
60
 
62
61
  `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -69,7 +68,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
69
68
  are in ``airflow.providers.elasticsearch`` python package.
70
69
 
71
70
  You can find package information and changelog for the provider
72
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0/>`_.
71
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/>`_.
73
72
 
74
73
  Installation
75
74
  ------------
@@ -78,7 +77,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
78
77
  for the minimum Airflow version supported) via
79
78
  ``pip install apache-airflow-providers-elasticsearch``
80
79
 
81
- The package supports the following python versions: 3.9,3.10,3.11,3.12
80
+ The package supports the following python versions: 3.10,3.11,3.12
82
81
 
83
82
  Requirements
84
83
  ------------
@@ -111,5 +110,5 @@ Dependent package
111
110
  ============================================================================================================ ==============
112
111
 
113
112
  The changelog for the provider package can be found in the
114
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0/changelog.html>`_.
113
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html>`_.
115
114
 
@@ -23,7 +23,7 @@
23
23
 
24
24
  Package ``apache-airflow-providers-elasticsearch``
25
25
 
26
- Release: ``6.3.0``
26
+ Release: ``6.3.1``
27
27
 
28
28
 
29
29
  `Elasticsearch <https://www.elastic.co/elasticsearch>`__
@@ -36,7 +36,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
36
36
  are in ``airflow.providers.elasticsearch`` python package.
37
37
 
38
38
  You can find package information and changelog for the provider
39
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0/>`_.
39
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/>`_.
40
40
 
41
41
  Installation
42
42
  ------------
@@ -45,7 +45,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
45
45
  for the minimum Airflow version supported) via
46
46
  ``pip install apache-airflow-providers-elasticsearch``
47
47
 
48
- The package supports the following python versions: 3.9,3.10,3.11,3.12
48
+ The package supports the following python versions: 3.10,3.11,3.12
49
49
 
50
50
  Requirements
51
51
  ------------
@@ -78,4 +78,4 @@ Dependent package
78
78
  ============================================================================================================ ==============
79
79
 
80
80
  The changelog for the provider package can be found in the
81
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0/changelog.html>`_.
81
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html>`_.
@@ -27,6 +27,21 @@
27
27
  Changelog
28
28
  ---------
29
29
 
30
+ 6.3.1
31
+ .....
32
+
33
+ Misc
34
+ ~~~~
35
+
36
+ * ``Move 'BaseHook' implementation to task SDK (#51873)``
37
+ * ``Provider Migration: Update elasticsearch for Airflow 3.0 compatibility (#52628)``
38
+ * ``Disable UP038 ruff rule and revert mandatory 'X | Y' in insintance checks (#52644)``
39
+ * ``Drop support for Python 3.9 (#52072)``
40
+
41
+ .. Below changes are excluded from the changelog. Move them to
42
+ appropriate section above if needed. Do not delete the lines(!):
43
+ * ``Removed pytestmark db_test from the elasticsearch providers tests (#52139)``
44
+
30
45
  6.3.0
31
46
  .....
32
47
 
@@ -79,7 +79,7 @@ apache-airflow-providers-elasticsearch package
79
79
  `Elasticsearch <https://www.elastic.co/elasticsearch>`__
80
80
 
81
81
 
82
- Release: 6.3.0
82
+ Release: 6.3.1
83
83
 
84
84
  Provider package
85
85
  ----------------
@@ -132,5 +132,5 @@ Downloading official packages
132
132
  You can download officially released packages and verify their checksums and signatures from the
133
133
  `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
134
134
 
135
- * `The apache-airflow-providers-elasticsearch 6.3.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.0.tar.gz.sha512>`__)
136
- * `The apache-airflow-providers-elasticsearch 6.3.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.0-py3-none-any.whl.sha512>`__)
135
+ * `The apache-airflow-providers-elasticsearch 6.3.1 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1.tar.gz.sha512>`__)
136
+ * `The apache-airflow-providers-elasticsearch 6.3.1 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1-py3-none-any.whl.sha512>`__)
@@ -22,12 +22,13 @@ description: |
22
22
  `Elasticsearch <https://www.elastic.co/elasticsearch>`__
23
23
 
24
24
  state: ready
25
- source-date-epoch: 1747132714
25
+ source-date-epoch: 1751473399
26
26
  # Note that those versions are maintained by release manager - do not update them manually
27
27
  # with the exception of case where other provider in sources has >= new provider version.
28
28
  # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
29
29
  # to be done in the same PR
30
30
  versions:
31
+ - 6.3.1
31
32
  - 6.3.0
32
33
  - 6.2.2
33
34
  - 6.2.1
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
25
25
 
26
26
  [project]
27
27
  name = "apache-airflow-providers-elasticsearch"
28
- version = "6.3.0"
28
+ version = "6.3.1rc1"
29
29
  description = "Provider package apache-airflow-providers-elasticsearch for Apache Airflow"
30
30
  readme = "README.rst"
31
31
  authors = [
@@ -44,21 +44,20 @@ classifiers = [
44
44
  "Framework :: Apache Airflow",
45
45
  "Framework :: Apache Airflow :: Provider",
46
46
  "License :: OSI Approved :: Apache Software License",
47
- "Programming Language :: Python :: 3.9",
48
47
  "Programming Language :: Python :: 3.10",
49
48
  "Programming Language :: Python :: 3.11",
50
49
  "Programming Language :: Python :: 3.12",
51
50
  "Topic :: System :: Monitoring",
52
51
  ]
53
- requires-python = "~=3.9"
52
+ requires-python = "~=3.10"
54
53
 
55
54
  # The dependencies should be modified in place in the generated file.
56
55
  # Any change in the dependencies is preserved when the file is regenerated
57
56
  # Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
58
57
  # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
59
58
  dependencies = [
60
- "apache-airflow>=2.10.0",
61
- "apache-airflow-providers-common-sql>=1.27.0",
59
+ "apache-airflow>=2.10.0rc1",
60
+ "apache-airflow-providers-common-sql>=1.27.0rc1",
62
61
  "elasticsearch>=8.10,<9",
63
62
  ]
64
63
 
@@ -98,8 +97,8 @@ apache-airflow-providers-common-sql = {workspace = true}
98
97
  apache-airflow-providers-standard = {workspace = true}
99
98
 
100
99
  [project.urls]
101
- "Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0"
102
- "Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.0/changelog.html"
100
+ "Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1"
101
+ "Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html"
103
102
  "Bug Tracker" = "https://github.com/apache/airflow/issues"
104
103
  "Source Code" = "https://github.com/apache/airflow"
105
104
  "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "6.3.0"
32
+ __version__ = "6.3.1"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.10.0"
@@ -20,14 +20,18 @@ from __future__ import annotations
20
20
  from collections.abc import Iterable, Mapping
21
21
  from copy import deepcopy
22
22
  from functools import cached_property
23
- from typing import TYPE_CHECKING, Any
23
+ from typing import TYPE_CHECKING, Any, cast
24
24
  from urllib import parse
25
25
 
26
26
  from elasticsearch import Elasticsearch
27
27
 
28
- from airflow.hooks.base import BaseHook
29
28
  from airflow.providers.common.sql.hooks.sql import DbApiHook
30
29
 
30
+ try:
31
+ from airflow.sdk import BaseHook
32
+ except ImportError:
33
+ from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
34
+
31
35
  if TYPE_CHECKING:
32
36
  from elastic_transport import ObjectApiResponse
33
37
 
@@ -179,8 +183,8 @@ class ElasticsearchSQLHook(DbApiHook):
179
183
  conn = self.connection
180
184
 
181
185
  conn_args = {
182
- "host": conn.host,
183
- "port": conn.port,
186
+ "host": cast("str", conn.host),
187
+ "port": cast("int", conn.port),
184
188
  "user": conn.login or None,
185
189
  "password": conn.password or None,
186
190
  "scheme": conn.schema or "http",
@@ -191,7 +195,7 @@ class ElasticsearchSQLHook(DbApiHook):
191
195
  if conn_args.get("http_compress", False):
192
196
  conn_args["http_compress"] = bool(conn_args["http_compress"])
193
197
 
194
- return connect(**conn_args)
198
+ return connect(**conn_args) # type: ignore[arg-type]
195
199
 
196
200
  def get_uri(self) -> str:
197
201
  conn = self.connection
@@ -199,7 +203,7 @@ class ElasticsearchSQLHook(DbApiHook):
199
203
  login = ""
200
204
  if conn.login:
201
205
  login = f"{conn.login}:{conn.password}@"
202
- host = conn.host
206
+ host = conn.host or ""
203
207
  if conn.port is not None:
204
208
  host += f":{conn.port}"
205
209
  uri = f"{conn.conn_type}+{conn.schema}://{login}{host}/"
@@ -27,8 +27,9 @@ import shutil
27
27
  import sys
28
28
  import time
29
29
  from collections import defaultdict
30
+ from collections.abc import Callable
30
31
  from operator import attrgetter
31
- from typing import TYPE_CHECKING, Any, Callable, Literal
32
+ from typing import TYPE_CHECKING, Any, Literal
32
33
  from urllib.parse import quote, urlparse
33
34
 
34
35
  # Using `from elasticsearch import *` would break elasticsearch mocking used in unit test.
@@ -44,7 +45,7 @@ from airflow.providers.elasticsearch.log.es_json_formatter import (
44
45
  ElasticsearchJSONFormatter,
45
46
  )
46
47
  from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse, Hit
47
- from airflow.providers.elasticsearch.version_compat import AIRFLOW_V_3_0_PLUS
48
+ from airflow.providers.elasticsearch.version_compat import AIRFLOW_V_3_0_PLUS, EsLogMsgType
48
49
  from airflow.utils import timezone
49
50
  from airflow.utils.log.file_task_handler import FileTaskHandler
50
51
  from airflow.utils.log.logging_mixin import ExternalLoggingMixin, LoggingMixin
@@ -56,15 +57,6 @@ if TYPE_CHECKING:
56
57
 
57
58
  from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
58
59
 
59
- if AIRFLOW_V_3_0_PLUS:
60
- from typing import Union
61
-
62
- from airflow.utils.log.file_task_handler import StructuredLogMessage
63
-
64
- EsLogMsgType = Union[list[StructuredLogMessage], str]
65
- else:
66
- EsLogMsgType = list[tuple[str, str]] # type: ignore[misc]
67
-
68
60
 
69
61
  LOG_LINE_DEFAULTS = {"exc_text": "", "stack_info": ""}
70
62
  # Elasticsearch hosted log type
@@ -33,3 +33,10 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
33
33
 
34
34
 
35
35
  AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
36
+
37
+ if AIRFLOW_V_3_0_PLUS:
38
+ from airflow.utils.log.file_task_handler import StructuredLogMessage
39
+
40
+ EsLogMsgType = list[StructuredLogMessage] | str
41
+ else:
42
+ EsLogMsgType = list[tuple[str, str]] # type: ignore[misc]
@@ -51,8 +51,6 @@ from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
51
51
  from unit.elasticsearch.log.elasticmock import elasticmock
52
52
  from unit.elasticsearch.log.elasticmock.utilities import SearchFailedException
53
53
 
54
- pytestmark = pytest.mark.db_test
55
-
56
54
  ES_PROVIDER_YAML_FILE = AIRFLOW_PROVIDERS_ROOT_PATH / "elasticsearch" / "provider.yaml"
57
55
 
58
56
 
@@ -202,6 +200,7 @@ class TestElasticsearchTaskHandler:
202
200
  )
203
201
  assert handler.index_patterns == patterns
204
202
 
203
+ @pytest.mark.db_test
205
204
  def test_read(self, ti):
206
205
  ts = pendulum.now()
207
206
  logs, metadatas = self.es_task_handler.read(
@@ -227,6 +226,7 @@ class TestElasticsearchTaskHandler:
227
226
  assert metadata["offset"] == "1"
228
227
  assert timezone.parse(metadata["last_log_timestamp"]) > ts
229
228
 
229
+ @pytest.mark.db_test
230
230
  def test_read_with_patterns(self, ti):
231
231
  ts = pendulum.now()
232
232
  with mock.patch.object(self.es_task_handler, "index_patterns", new="test_*,other_*"):
@@ -253,6 +253,7 @@ class TestElasticsearchTaskHandler:
253
253
  assert metadata["offset"] == "1"
254
254
  assert timezone.parse(metadata["last_log_timestamp"]) > ts
255
255
 
256
+ @pytest.mark.db_test
256
257
  def test_read_with_patterns_no_match(self, ti):
257
258
  ts = pendulum.now()
258
259
  with mock.patch.object(self.es_task_handler, "index_patterns", new="test_other_*,test_another_*"):
@@ -276,6 +277,7 @@ class TestElasticsearchTaskHandler:
276
277
  # last_log_timestamp won't change if no log lines read.
277
278
  assert timezone.parse(metadata["last_log_timestamp"]) == ts
278
279
 
280
+ @pytest.mark.db_test
279
281
  def test_read_with_missing_index(self, ti):
280
282
  ts = pendulum.now()
281
283
  with mock.patch.object(self.es_task_handler, "index_patterns", new="nonexistent,test_*"):
@@ -286,6 +288,7 @@ class TestElasticsearchTaskHandler:
286
288
  {"offset": 0, "last_log_timestamp": str(ts), "end_of_log": False},
287
289
  )
288
290
 
291
+ @pytest.mark.db_test
289
292
  @pytest.mark.parametrize("seconds", [3, 6])
290
293
  def test_read_missing_logs(self, seconds, create_task_instance):
291
294
  """
@@ -330,6 +333,7 @@ class TestElasticsearchTaskHandler:
330
333
  assert metadatas[0]["offset"] == "0"
331
334
  assert timezone.parse(metadatas[0]["last_log_timestamp"]) == ts
332
335
 
336
+ @pytest.mark.db_test
333
337
  def test_read_with_match_phrase_query(self, ti):
334
338
  similar_log_id = (
335
339
  f"{TestElasticsearchTaskHandler.TASK_ID}-"
@@ -374,6 +378,7 @@ class TestElasticsearchTaskHandler:
374
378
  assert metadata["offset"] == "1"
375
379
  assert timezone.parse(metadata["last_log_timestamp"]) > ts
376
380
 
381
+ @pytest.mark.db_test
377
382
  def test_read_with_none_metadata(self, ti):
378
383
  logs, metadatas = self.es_task_handler.read(ti, 1)
379
384
  if AIRFLOW_V_3_0_PLUS:
@@ -395,6 +400,7 @@ class TestElasticsearchTaskHandler:
395
400
  assert metadata["offset"] == "1"
396
401
  assert timezone.parse(metadata["last_log_timestamp"]) < pendulum.now()
397
402
 
403
+ @pytest.mark.db_test
398
404
  def test_read_nonexistent_log(self, ti):
399
405
  ts = pendulum.now()
400
406
  # In ElasticMock, search is going to return all documents with matching index
@@ -420,6 +426,7 @@ class TestElasticsearchTaskHandler:
420
426
  # last_log_timestamp won't change if no log lines read.
421
427
  assert timezone.parse(metadata["last_log_timestamp"]) == ts
422
428
 
429
+ @pytest.mark.db_test
423
430
  def test_read_with_empty_metadata(self, ti):
424
431
  ts = pendulum.now()
425
432
  logs, metadatas = self.es_task_handler.read(ti, 1, {})
@@ -466,6 +473,7 @@ class TestElasticsearchTaskHandler:
466
473
  # if not last_log_timestamp is provided.
467
474
  assert timezone.parse(metadata["last_log_timestamp"]) > ts
468
475
 
476
+ @pytest.mark.db_test
469
477
  def test_read_timeout(self, ti):
470
478
  ts = pendulum.now().subtract(minutes=5)
471
479
 
@@ -498,6 +506,7 @@ class TestElasticsearchTaskHandler:
498
506
  assert str(offset) == metadata["offset"]
499
507
  assert timezone.parse(metadata["last_log_timestamp"]) == ts
500
508
 
509
+ @pytest.mark.db_test
501
510
  def test_read_as_download_logs(self, ti):
502
511
  ts = pendulum.now()
503
512
  logs, metadatas = self.es_task_handler.read(
@@ -529,6 +538,7 @@ class TestElasticsearchTaskHandler:
529
538
  assert metadata["offset"] == "1"
530
539
  assert timezone.parse(metadata["last_log_timestamp"]) > ts
531
540
 
541
+ @pytest.mark.db_test
532
542
  def test_read_raises(self, ti):
533
543
  with mock.patch.object(self.es_task_handler.log, "exception") as mock_exception:
534
544
  with mock.patch.object(self.es_task_handler.client, "search") as mock_execute:
@@ -552,10 +562,12 @@ class TestElasticsearchTaskHandler:
552
562
  assert metadata["offset"] == "0"
553
563
  assert not metadata["end_of_log"]
554
564
 
565
+ @pytest.mark.db_test
555
566
  def test_set_context(self, ti):
556
567
  self.es_task_handler.set_context(ti)
557
568
  assert self.es_task_handler.mark_end_on_close
558
569
 
570
+ @pytest.mark.db_test
559
571
  def test_set_context_w_json_format_and_write_stdout(self, ti):
560
572
  formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
561
573
  self.es_task_handler.formatter = formatter
@@ -563,6 +575,7 @@ class TestElasticsearchTaskHandler:
563
575
  self.es_task_handler.json_format = True
564
576
  self.es_task_handler.set_context(ti)
565
577
 
578
+ @pytest.mark.db_test
566
579
  def test_read_with_json_format(self, ti):
567
580
  ts = pendulum.now()
568
581
  formatter = logging.Formatter(
@@ -592,6 +605,7 @@ class TestElasticsearchTaskHandler:
592
605
  else:
593
606
  assert logs[0][0][1] == expected_message
594
607
 
608
+ @pytest.mark.db_test
595
609
  def test_read_with_json_format_with_custom_offset_and_host_fields(self, ti):
596
610
  ts = pendulum.now()
597
611
  formatter = logging.Formatter(
@@ -624,6 +638,7 @@ class TestElasticsearchTaskHandler:
624
638
  else:
625
639
  assert logs[0][0][1] == expected_message
626
640
 
641
+ @pytest.mark.db_test
627
642
  def test_read_with_custom_offset_and_host_fields(self, ti):
628
643
  ts = pendulum.now()
629
644
  # Delete the existing log entry as it doesn't have the new offset and host fields
@@ -648,6 +663,7 @@ class TestElasticsearchTaskHandler:
648
663
  else:
649
664
  assert self.test_message == logs[0][0][1]
650
665
 
666
+ @pytest.mark.db_test
651
667
  def test_close(self, ti):
652
668
  formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
653
669
  self.es_task_handler.formatter = formatter
@@ -664,6 +680,7 @@ class TestElasticsearchTaskHandler:
664
680
  assert log_line.endswith(self.end_of_log_mark.strip())
665
681
  assert self.es_task_handler.closed
666
682
 
683
+ @pytest.mark.db_test
667
684
  def test_close_no_mark_end(self, ti):
668
685
  ti.raw = True
669
686
  self.es_task_handler.set_context(ti)
@@ -674,6 +691,7 @@ class TestElasticsearchTaskHandler:
674
691
  assert self.end_of_log_mark not in log_file.read()
675
692
  assert self.es_task_handler.closed
676
693
 
694
+ @pytest.mark.db_test
677
695
  def test_close_closed(self, ti):
678
696
  self.es_task_handler.closed = True
679
697
  self.es_task_handler.set_context(ti)
@@ -683,6 +701,7 @@ class TestElasticsearchTaskHandler:
683
701
  ) as log_file:
684
702
  assert len(log_file.read()) == 0
685
703
 
704
+ @pytest.mark.db_test
686
705
  def test_close_with_no_handler(self, ti):
687
706
  self.es_task_handler.set_context(ti)
688
707
  self.es_task_handler.handler = None
@@ -693,6 +712,7 @@ class TestElasticsearchTaskHandler:
693
712
  assert len(log_file.read()) == 0
694
713
  assert self.es_task_handler.closed
695
714
 
715
+ @pytest.mark.db_test
696
716
  def test_close_with_no_stream(self, ti):
697
717
  self.es_task_handler.set_context(ti)
698
718
  self.es_task_handler.handler.stream = None
@@ -712,6 +732,7 @@ class TestElasticsearchTaskHandler:
712
732
  assert self.end_of_log_mark in log_file.read()
713
733
  assert self.es_task_handler.closed
714
734
 
735
+ @pytest.mark.db_test
715
736
  def test_render_log_id(self, ti):
716
737
  assert self.es_task_handler._render_log_id(ti, 1) == self.LOG_ID
717
738
 
@@ -722,6 +743,7 @@ class TestElasticsearchTaskHandler:
722
743
  clean_logical_date = self.es_task_handler._clean_date(datetime(2016, 7, 8, 9, 10, 11, 12))
723
744
  assert clean_logical_date == "2016_07_08T09_10_11_000012"
724
745
 
746
+ @pytest.mark.db_test
725
747
  @pytest.mark.parametrize(
726
748
  "json_format, es_frontend, expected_url",
727
749
  [
@@ -781,6 +803,7 @@ class TestElasticsearchTaskHandler:
781
803
  self.es_task_handler.frontend = frontend
782
804
  assert self.es_task_handler.supports_external_link == expected
783
805
 
806
+ @pytest.mark.db_test
784
807
  @mock.patch("sys.__stdout__", new_callable=StringIO)
785
808
  def test_dynamic_offset(self, stdout_mock, ti, time_machine):
786
809
  # arrange
@@ -843,6 +866,7 @@ class TestElasticsearchTaskHandler:
843
866
  filename_template=None,
844
867
  )
845
868
 
869
+ @pytest.mark.db_test
846
870
  def test_write_to_es(self, ti):
847
871
  self.es_task_handler.write_to_es = True
848
872
  self.es_task_handler.json_format = True