apache-airflow-providers-elasticsearch 6.3.0rc1__tar.gz → 6.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-elasticsearch might be problematic. Click here for more details.
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/PKG-INFO +10 -11
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/README.rst +4 -4
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/changelog.rst +15 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/index.rst +3 -3
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/provider.yaml +2 -1
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/pyproject.toml +6 -7
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/__init__.py +1 -1
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/hooks/elasticsearch.py +10 -6
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/log/es_task_handler.py +3 -11
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/version_compat.py +7 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/test_es_task_handler.py +26 -2
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/commits.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/conf.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/configurations-ref.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/connections/elasticsearch.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/hooks/elasticsearch_python_hook.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/hooks/elasticsearch_sql_hook.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/hooks/index.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/integration-logos/Elasticsearch.png +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/logging/index.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/redirects.txt +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/docs/security.rst +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/LICENSE +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/get_provider_info.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/hooks/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/log/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/log/es_json_formatter.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/src/airflow/providers/elasticsearch/log/es_response.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/conftest.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/system/elasticsearch/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/system/elasticsearch/example_elasticsearch_query.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/hooks/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/hooks/test_elasticsearch.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/elasticmock/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/elasticmock/fake_elasticsearch.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/elasticmock/utilities/__init__.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/test_es_json_formatter.py +0 -0
- {apache_airflow_providers_elasticsearch-6.3.0rc1 → apache_airflow_providers_elasticsearch-6.3.1}/tests/unit/elasticsearch/log/test_es_response.py +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-elasticsearch
|
|
3
|
-
Version: 6.3.
|
|
3
|
+
Version: 6.3.1
|
|
4
4
|
Summary: Provider package apache-airflow-providers-elasticsearch for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,elasticsearch,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
7
7
|
Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
8
|
-
Requires-Python: ~=3.
|
|
8
|
+
Requires-Python: ~=3.10
|
|
9
9
|
Description-Content-Type: text/x-rst
|
|
10
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
11
11
|
Classifier: Environment :: Console
|
|
@@ -15,17 +15,16 @@ Classifier: Intended Audience :: System Administrators
|
|
|
15
15
|
Classifier: Framework :: Apache Airflow
|
|
16
16
|
Classifier: Framework :: Apache Airflow :: Provider
|
|
17
17
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
21
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.27.
|
|
22
|
+
Requires-Dist: apache-airflow>=2.10.0
|
|
23
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
|
|
25
24
|
Requires-Dist: elasticsearch>=8.10,<9
|
|
26
25
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
27
|
-
Project-URL: Changelog, https://airflow.
|
|
28
|
-
Project-URL: Documentation, https://airflow.
|
|
26
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html
|
|
27
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1
|
|
29
28
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
30
29
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
31
30
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -56,7 +55,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
|
56
55
|
|
|
57
56
|
Package ``apache-airflow-providers-elasticsearch``
|
|
58
57
|
|
|
59
|
-
Release: ``6.3.
|
|
58
|
+
Release: ``6.3.1``
|
|
60
59
|
|
|
61
60
|
|
|
62
61
|
`Elasticsearch <https://www.elastic.co/elasticsearch>`__
|
|
@@ -69,7 +68,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
|
|
|
69
68
|
are in ``airflow.providers.elasticsearch`` python package.
|
|
70
69
|
|
|
71
70
|
You can find package information and changelog for the provider
|
|
72
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.
|
|
71
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/>`_.
|
|
73
72
|
|
|
74
73
|
Installation
|
|
75
74
|
------------
|
|
@@ -78,7 +77,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
|
78
77
|
for the minimum Airflow version supported) via
|
|
79
78
|
``pip install apache-airflow-providers-elasticsearch``
|
|
80
79
|
|
|
81
|
-
The package supports the following python versions: 3.
|
|
80
|
+
The package supports the following python versions: 3.10,3.11,3.12
|
|
82
81
|
|
|
83
82
|
Requirements
|
|
84
83
|
------------
|
|
@@ -111,5 +110,5 @@ Dependent package
|
|
|
111
110
|
============================================================================================================ ==============
|
|
112
111
|
|
|
113
112
|
The changelog for the provider package can be found in the
|
|
114
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.
|
|
113
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html>`_.
|
|
115
114
|
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-elasticsearch``
|
|
25
25
|
|
|
26
|
-
Release: ``6.3.
|
|
26
|
+
Release: ``6.3.1``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`Elasticsearch <https://www.elastic.co/elasticsearch>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``elasticsearch`` provider. All classes for this
|
|
|
36
36
|
are in ``airflow.providers.elasticsearch`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -45,7 +45,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
|
45
45
|
for the minimum Airflow version supported) via
|
|
46
46
|
``pip install apache-airflow-providers-elasticsearch``
|
|
47
47
|
|
|
48
|
-
The package supports the following python versions: 3.
|
|
48
|
+
The package supports the following python versions: 3.10,3.11,3.12
|
|
49
49
|
|
|
50
50
|
Requirements
|
|
51
51
|
------------
|
|
@@ -78,4 +78,4 @@ Dependent package
|
|
|
78
78
|
============================================================================================================ ==============
|
|
79
79
|
|
|
80
80
|
The changelog for the provider package can be found in the
|
|
81
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.
|
|
81
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html>`_.
|
|
@@ -27,6 +27,21 @@
|
|
|
27
27
|
Changelog
|
|
28
28
|
---------
|
|
29
29
|
|
|
30
|
+
6.3.1
|
|
31
|
+
.....
|
|
32
|
+
|
|
33
|
+
Misc
|
|
34
|
+
~~~~
|
|
35
|
+
|
|
36
|
+
* ``Move 'BaseHook' implementation to task SDK (#51873)``
|
|
37
|
+
* ``Provider Migration: Update elasticsearch for Airflow 3.0 compatibility (#52628)``
|
|
38
|
+
* ``Disable UP038 ruff rule and revert mandatory 'X | Y' in insintance checks (#52644)``
|
|
39
|
+
* ``Drop support for Python 3.9 (#52072)``
|
|
40
|
+
|
|
41
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
42
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
43
|
+
* ``Removed pytestmark db_test from the elasticsearch providers tests (#52139)``
|
|
44
|
+
|
|
30
45
|
6.3.0
|
|
31
46
|
.....
|
|
32
47
|
|
|
@@ -79,7 +79,7 @@ apache-airflow-providers-elasticsearch package
|
|
|
79
79
|
`Elasticsearch <https://www.elastic.co/elasticsearch>`__
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
Release: 6.3.
|
|
82
|
+
Release: 6.3.1
|
|
83
83
|
|
|
84
84
|
Provider package
|
|
85
85
|
----------------
|
|
@@ -132,5 +132,5 @@ Downloading official packages
|
|
|
132
132
|
You can download officially released packages and verify their checksums and signatures from the
|
|
133
133
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
134
134
|
|
|
135
|
-
* `The apache-airflow-providers-elasticsearch 6.3.
|
|
136
|
-
* `The apache-airflow-providers-elasticsearch 6.3.
|
|
135
|
+
* `The apache-airflow-providers-elasticsearch 6.3.1 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1.tar.gz.sha512>`__)
|
|
136
|
+
* `The apache-airflow-providers-elasticsearch 6.3.1 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_elasticsearch-6.3.1-py3-none-any.whl.sha512>`__)
|
|
@@ -22,12 +22,13 @@ description: |
|
|
|
22
22
|
`Elasticsearch <https://www.elastic.co/elasticsearch>`__
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
|
-
source-date-epoch:
|
|
25
|
+
source-date-epoch: 1751473399
|
|
26
26
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
27
27
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
28
28
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
29
29
|
# to be done in the same PR
|
|
30
30
|
versions:
|
|
31
|
+
- 6.3.1
|
|
31
32
|
- 6.3.0
|
|
32
33
|
- 6.2.2
|
|
33
34
|
- 6.2.1
|
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-elasticsearch"
|
|
28
|
-
version = "6.3.
|
|
28
|
+
version = "6.3.1"
|
|
29
29
|
description = "Provider package apache-airflow-providers-elasticsearch for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
authors = [
|
|
@@ -44,21 +44,20 @@ classifiers = [
|
|
|
44
44
|
"Framework :: Apache Airflow",
|
|
45
45
|
"Framework :: Apache Airflow :: Provider",
|
|
46
46
|
"License :: OSI Approved :: Apache Software License",
|
|
47
|
-
"Programming Language :: Python :: 3.9",
|
|
48
47
|
"Programming Language :: Python :: 3.10",
|
|
49
48
|
"Programming Language :: Python :: 3.11",
|
|
50
49
|
"Programming Language :: Python :: 3.12",
|
|
51
50
|
"Topic :: System :: Monitoring",
|
|
52
51
|
]
|
|
53
|
-
requires-python = "~=3.
|
|
52
|
+
requires-python = "~=3.10"
|
|
54
53
|
|
|
55
54
|
# The dependencies should be modified in place in the generated file.
|
|
56
55
|
# Any change in the dependencies is preserved when the file is regenerated
|
|
57
56
|
# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
|
|
58
57
|
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
|
59
58
|
dependencies = [
|
|
60
|
-
"apache-airflow>=2.10.
|
|
61
|
-
"apache-airflow-providers-common-sql>=1.27.
|
|
59
|
+
"apache-airflow>=2.10.0",
|
|
60
|
+
"apache-airflow-providers-common-sql>=1.27.0",
|
|
62
61
|
"elasticsearch>=8.10,<9",
|
|
63
62
|
]
|
|
64
63
|
|
|
@@ -98,8 +97,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
98
97
|
apache-airflow-providers-standard = {workspace = true}
|
|
99
98
|
|
|
100
99
|
[project.urls]
|
|
101
|
-
"Documentation" = "https://airflow.
|
|
102
|
-
"Changelog" = "https://airflow.
|
|
100
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1"
|
|
101
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-elasticsearch/6.3.1/changelog.html"
|
|
103
102
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
104
103
|
"Source Code" = "https://github.com/apache/airflow"
|
|
105
104
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "6.3.
|
|
32
|
+
__version__ = "6.3.1"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -20,14 +20,18 @@ from __future__ import annotations
|
|
|
20
20
|
from collections.abc import Iterable, Mapping
|
|
21
21
|
from copy import deepcopy
|
|
22
22
|
from functools import cached_property
|
|
23
|
-
from typing import TYPE_CHECKING, Any
|
|
23
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
24
24
|
from urllib import parse
|
|
25
25
|
|
|
26
26
|
from elasticsearch import Elasticsearch
|
|
27
27
|
|
|
28
|
-
from airflow.hooks.base import BaseHook
|
|
29
28
|
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
30
29
|
|
|
30
|
+
try:
|
|
31
|
+
from airflow.sdk import BaseHook
|
|
32
|
+
except ImportError:
|
|
33
|
+
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
|
34
|
+
|
|
31
35
|
if TYPE_CHECKING:
|
|
32
36
|
from elastic_transport import ObjectApiResponse
|
|
33
37
|
|
|
@@ -179,8 +183,8 @@ class ElasticsearchSQLHook(DbApiHook):
|
|
|
179
183
|
conn = self.connection
|
|
180
184
|
|
|
181
185
|
conn_args = {
|
|
182
|
-
"host": conn.host,
|
|
183
|
-
"port": conn.port,
|
|
186
|
+
"host": cast("str", conn.host),
|
|
187
|
+
"port": cast("int", conn.port),
|
|
184
188
|
"user": conn.login or None,
|
|
185
189
|
"password": conn.password or None,
|
|
186
190
|
"scheme": conn.schema or "http",
|
|
@@ -191,7 +195,7 @@ class ElasticsearchSQLHook(DbApiHook):
|
|
|
191
195
|
if conn_args.get("http_compress", False):
|
|
192
196
|
conn_args["http_compress"] = bool(conn_args["http_compress"])
|
|
193
197
|
|
|
194
|
-
return connect(**conn_args)
|
|
198
|
+
return connect(**conn_args) # type: ignore[arg-type]
|
|
195
199
|
|
|
196
200
|
def get_uri(self) -> str:
|
|
197
201
|
conn = self.connection
|
|
@@ -199,7 +203,7 @@ class ElasticsearchSQLHook(DbApiHook):
|
|
|
199
203
|
login = ""
|
|
200
204
|
if conn.login:
|
|
201
205
|
login = f"{conn.login}:{conn.password}@"
|
|
202
|
-
host = conn.host
|
|
206
|
+
host = conn.host or ""
|
|
203
207
|
if conn.port is not None:
|
|
204
208
|
host += f":{conn.port}"
|
|
205
209
|
uri = f"{conn.conn_type}+{conn.schema}://{login}{host}/"
|
|
@@ -27,8 +27,9 @@ import shutil
|
|
|
27
27
|
import sys
|
|
28
28
|
import time
|
|
29
29
|
from collections import defaultdict
|
|
30
|
+
from collections.abc import Callable
|
|
30
31
|
from operator import attrgetter
|
|
31
|
-
from typing import TYPE_CHECKING, Any,
|
|
32
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
32
33
|
from urllib.parse import quote, urlparse
|
|
33
34
|
|
|
34
35
|
# Using `from elasticsearch import *` would break elasticsearch mocking used in unit test.
|
|
@@ -44,7 +45,7 @@ from airflow.providers.elasticsearch.log.es_json_formatter import (
|
|
|
44
45
|
ElasticsearchJSONFormatter,
|
|
45
46
|
)
|
|
46
47
|
from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse, Hit
|
|
47
|
-
from airflow.providers.elasticsearch.version_compat import AIRFLOW_V_3_0_PLUS
|
|
48
|
+
from airflow.providers.elasticsearch.version_compat import AIRFLOW_V_3_0_PLUS, EsLogMsgType
|
|
48
49
|
from airflow.utils import timezone
|
|
49
50
|
from airflow.utils.log.file_task_handler import FileTaskHandler
|
|
50
51
|
from airflow.utils.log.logging_mixin import ExternalLoggingMixin, LoggingMixin
|
|
@@ -56,15 +57,6 @@ if TYPE_CHECKING:
|
|
|
56
57
|
|
|
57
58
|
from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
|
|
58
59
|
|
|
59
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
60
|
-
from typing import Union
|
|
61
|
-
|
|
62
|
-
from airflow.utils.log.file_task_handler import StructuredLogMessage
|
|
63
|
-
|
|
64
|
-
EsLogMsgType = Union[list[StructuredLogMessage], str]
|
|
65
|
-
else:
|
|
66
|
-
EsLogMsgType = list[tuple[str, str]] # type: ignore[misc]
|
|
67
|
-
|
|
68
60
|
|
|
69
61
|
LOG_LINE_DEFAULTS = {"exc_text": "", "stack_info": ""}
|
|
70
62
|
# Elasticsearch hosted log type
|
|
@@ -33,3 +33,10 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
|
36
|
+
|
|
37
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
38
|
+
from airflow.utils.log.file_task_handler import StructuredLogMessage
|
|
39
|
+
|
|
40
|
+
EsLogMsgType = list[StructuredLogMessage] | str
|
|
41
|
+
else:
|
|
42
|
+
EsLogMsgType = list[tuple[str, str]] # type: ignore[misc]
|
|
@@ -51,8 +51,6 @@ from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
|
|
|
51
51
|
from unit.elasticsearch.log.elasticmock import elasticmock
|
|
52
52
|
from unit.elasticsearch.log.elasticmock.utilities import SearchFailedException
|
|
53
53
|
|
|
54
|
-
pytestmark = pytest.mark.db_test
|
|
55
|
-
|
|
56
54
|
ES_PROVIDER_YAML_FILE = AIRFLOW_PROVIDERS_ROOT_PATH / "elasticsearch" / "provider.yaml"
|
|
57
55
|
|
|
58
56
|
|
|
@@ -202,6 +200,7 @@ class TestElasticsearchTaskHandler:
|
|
|
202
200
|
)
|
|
203
201
|
assert handler.index_patterns == patterns
|
|
204
202
|
|
|
203
|
+
@pytest.mark.db_test
|
|
205
204
|
def test_read(self, ti):
|
|
206
205
|
ts = pendulum.now()
|
|
207
206
|
logs, metadatas = self.es_task_handler.read(
|
|
@@ -227,6 +226,7 @@ class TestElasticsearchTaskHandler:
|
|
|
227
226
|
assert metadata["offset"] == "1"
|
|
228
227
|
assert timezone.parse(metadata["last_log_timestamp"]) > ts
|
|
229
228
|
|
|
229
|
+
@pytest.mark.db_test
|
|
230
230
|
def test_read_with_patterns(self, ti):
|
|
231
231
|
ts = pendulum.now()
|
|
232
232
|
with mock.patch.object(self.es_task_handler, "index_patterns", new="test_*,other_*"):
|
|
@@ -253,6 +253,7 @@ class TestElasticsearchTaskHandler:
|
|
|
253
253
|
assert metadata["offset"] == "1"
|
|
254
254
|
assert timezone.parse(metadata["last_log_timestamp"]) > ts
|
|
255
255
|
|
|
256
|
+
@pytest.mark.db_test
|
|
256
257
|
def test_read_with_patterns_no_match(self, ti):
|
|
257
258
|
ts = pendulum.now()
|
|
258
259
|
with mock.patch.object(self.es_task_handler, "index_patterns", new="test_other_*,test_another_*"):
|
|
@@ -276,6 +277,7 @@ class TestElasticsearchTaskHandler:
|
|
|
276
277
|
# last_log_timestamp won't change if no log lines read.
|
|
277
278
|
assert timezone.parse(metadata["last_log_timestamp"]) == ts
|
|
278
279
|
|
|
280
|
+
@pytest.mark.db_test
|
|
279
281
|
def test_read_with_missing_index(self, ti):
|
|
280
282
|
ts = pendulum.now()
|
|
281
283
|
with mock.patch.object(self.es_task_handler, "index_patterns", new="nonexistent,test_*"):
|
|
@@ -286,6 +288,7 @@ class TestElasticsearchTaskHandler:
|
|
|
286
288
|
{"offset": 0, "last_log_timestamp": str(ts), "end_of_log": False},
|
|
287
289
|
)
|
|
288
290
|
|
|
291
|
+
@pytest.mark.db_test
|
|
289
292
|
@pytest.mark.parametrize("seconds", [3, 6])
|
|
290
293
|
def test_read_missing_logs(self, seconds, create_task_instance):
|
|
291
294
|
"""
|
|
@@ -330,6 +333,7 @@ class TestElasticsearchTaskHandler:
|
|
|
330
333
|
assert metadatas[0]["offset"] == "0"
|
|
331
334
|
assert timezone.parse(metadatas[0]["last_log_timestamp"]) == ts
|
|
332
335
|
|
|
336
|
+
@pytest.mark.db_test
|
|
333
337
|
def test_read_with_match_phrase_query(self, ti):
|
|
334
338
|
similar_log_id = (
|
|
335
339
|
f"{TestElasticsearchTaskHandler.TASK_ID}-"
|
|
@@ -374,6 +378,7 @@ class TestElasticsearchTaskHandler:
|
|
|
374
378
|
assert metadata["offset"] == "1"
|
|
375
379
|
assert timezone.parse(metadata["last_log_timestamp"]) > ts
|
|
376
380
|
|
|
381
|
+
@pytest.mark.db_test
|
|
377
382
|
def test_read_with_none_metadata(self, ti):
|
|
378
383
|
logs, metadatas = self.es_task_handler.read(ti, 1)
|
|
379
384
|
if AIRFLOW_V_3_0_PLUS:
|
|
@@ -395,6 +400,7 @@ class TestElasticsearchTaskHandler:
|
|
|
395
400
|
assert metadata["offset"] == "1"
|
|
396
401
|
assert timezone.parse(metadata["last_log_timestamp"]) < pendulum.now()
|
|
397
402
|
|
|
403
|
+
@pytest.mark.db_test
|
|
398
404
|
def test_read_nonexistent_log(self, ti):
|
|
399
405
|
ts = pendulum.now()
|
|
400
406
|
# In ElasticMock, search is going to return all documents with matching index
|
|
@@ -420,6 +426,7 @@ class TestElasticsearchTaskHandler:
|
|
|
420
426
|
# last_log_timestamp won't change if no log lines read.
|
|
421
427
|
assert timezone.parse(metadata["last_log_timestamp"]) == ts
|
|
422
428
|
|
|
429
|
+
@pytest.mark.db_test
|
|
423
430
|
def test_read_with_empty_metadata(self, ti):
|
|
424
431
|
ts = pendulum.now()
|
|
425
432
|
logs, metadatas = self.es_task_handler.read(ti, 1, {})
|
|
@@ -466,6 +473,7 @@ class TestElasticsearchTaskHandler:
|
|
|
466
473
|
# if not last_log_timestamp is provided.
|
|
467
474
|
assert timezone.parse(metadata["last_log_timestamp"]) > ts
|
|
468
475
|
|
|
476
|
+
@pytest.mark.db_test
|
|
469
477
|
def test_read_timeout(self, ti):
|
|
470
478
|
ts = pendulum.now().subtract(minutes=5)
|
|
471
479
|
|
|
@@ -498,6 +506,7 @@ class TestElasticsearchTaskHandler:
|
|
|
498
506
|
assert str(offset) == metadata["offset"]
|
|
499
507
|
assert timezone.parse(metadata["last_log_timestamp"]) == ts
|
|
500
508
|
|
|
509
|
+
@pytest.mark.db_test
|
|
501
510
|
def test_read_as_download_logs(self, ti):
|
|
502
511
|
ts = pendulum.now()
|
|
503
512
|
logs, metadatas = self.es_task_handler.read(
|
|
@@ -529,6 +538,7 @@ class TestElasticsearchTaskHandler:
|
|
|
529
538
|
assert metadata["offset"] == "1"
|
|
530
539
|
assert timezone.parse(metadata["last_log_timestamp"]) > ts
|
|
531
540
|
|
|
541
|
+
@pytest.mark.db_test
|
|
532
542
|
def test_read_raises(self, ti):
|
|
533
543
|
with mock.patch.object(self.es_task_handler.log, "exception") as mock_exception:
|
|
534
544
|
with mock.patch.object(self.es_task_handler.client, "search") as mock_execute:
|
|
@@ -552,10 +562,12 @@ class TestElasticsearchTaskHandler:
|
|
|
552
562
|
assert metadata["offset"] == "0"
|
|
553
563
|
assert not metadata["end_of_log"]
|
|
554
564
|
|
|
565
|
+
@pytest.mark.db_test
|
|
555
566
|
def test_set_context(self, ti):
|
|
556
567
|
self.es_task_handler.set_context(ti)
|
|
557
568
|
assert self.es_task_handler.mark_end_on_close
|
|
558
569
|
|
|
570
|
+
@pytest.mark.db_test
|
|
559
571
|
def test_set_context_w_json_format_and_write_stdout(self, ti):
|
|
560
572
|
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
|
561
573
|
self.es_task_handler.formatter = formatter
|
|
@@ -563,6 +575,7 @@ class TestElasticsearchTaskHandler:
|
|
|
563
575
|
self.es_task_handler.json_format = True
|
|
564
576
|
self.es_task_handler.set_context(ti)
|
|
565
577
|
|
|
578
|
+
@pytest.mark.db_test
|
|
566
579
|
def test_read_with_json_format(self, ti):
|
|
567
580
|
ts = pendulum.now()
|
|
568
581
|
formatter = logging.Formatter(
|
|
@@ -592,6 +605,7 @@ class TestElasticsearchTaskHandler:
|
|
|
592
605
|
else:
|
|
593
606
|
assert logs[0][0][1] == expected_message
|
|
594
607
|
|
|
608
|
+
@pytest.mark.db_test
|
|
595
609
|
def test_read_with_json_format_with_custom_offset_and_host_fields(self, ti):
|
|
596
610
|
ts = pendulum.now()
|
|
597
611
|
formatter = logging.Formatter(
|
|
@@ -624,6 +638,7 @@ class TestElasticsearchTaskHandler:
|
|
|
624
638
|
else:
|
|
625
639
|
assert logs[0][0][1] == expected_message
|
|
626
640
|
|
|
641
|
+
@pytest.mark.db_test
|
|
627
642
|
def test_read_with_custom_offset_and_host_fields(self, ti):
|
|
628
643
|
ts = pendulum.now()
|
|
629
644
|
# Delete the existing log entry as it doesn't have the new offset and host fields
|
|
@@ -648,6 +663,7 @@ class TestElasticsearchTaskHandler:
|
|
|
648
663
|
else:
|
|
649
664
|
assert self.test_message == logs[0][0][1]
|
|
650
665
|
|
|
666
|
+
@pytest.mark.db_test
|
|
651
667
|
def test_close(self, ti):
|
|
652
668
|
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
|
653
669
|
self.es_task_handler.formatter = formatter
|
|
@@ -664,6 +680,7 @@ class TestElasticsearchTaskHandler:
|
|
|
664
680
|
assert log_line.endswith(self.end_of_log_mark.strip())
|
|
665
681
|
assert self.es_task_handler.closed
|
|
666
682
|
|
|
683
|
+
@pytest.mark.db_test
|
|
667
684
|
def test_close_no_mark_end(self, ti):
|
|
668
685
|
ti.raw = True
|
|
669
686
|
self.es_task_handler.set_context(ti)
|
|
@@ -674,6 +691,7 @@ class TestElasticsearchTaskHandler:
|
|
|
674
691
|
assert self.end_of_log_mark not in log_file.read()
|
|
675
692
|
assert self.es_task_handler.closed
|
|
676
693
|
|
|
694
|
+
@pytest.mark.db_test
|
|
677
695
|
def test_close_closed(self, ti):
|
|
678
696
|
self.es_task_handler.closed = True
|
|
679
697
|
self.es_task_handler.set_context(ti)
|
|
@@ -683,6 +701,7 @@ class TestElasticsearchTaskHandler:
|
|
|
683
701
|
) as log_file:
|
|
684
702
|
assert len(log_file.read()) == 0
|
|
685
703
|
|
|
704
|
+
@pytest.mark.db_test
|
|
686
705
|
def test_close_with_no_handler(self, ti):
|
|
687
706
|
self.es_task_handler.set_context(ti)
|
|
688
707
|
self.es_task_handler.handler = None
|
|
@@ -693,6 +712,7 @@ class TestElasticsearchTaskHandler:
|
|
|
693
712
|
assert len(log_file.read()) == 0
|
|
694
713
|
assert self.es_task_handler.closed
|
|
695
714
|
|
|
715
|
+
@pytest.mark.db_test
|
|
696
716
|
def test_close_with_no_stream(self, ti):
|
|
697
717
|
self.es_task_handler.set_context(ti)
|
|
698
718
|
self.es_task_handler.handler.stream = None
|
|
@@ -712,6 +732,7 @@ class TestElasticsearchTaskHandler:
|
|
|
712
732
|
assert self.end_of_log_mark in log_file.read()
|
|
713
733
|
assert self.es_task_handler.closed
|
|
714
734
|
|
|
735
|
+
@pytest.mark.db_test
|
|
715
736
|
def test_render_log_id(self, ti):
|
|
716
737
|
assert self.es_task_handler._render_log_id(ti, 1) == self.LOG_ID
|
|
717
738
|
|
|
@@ -722,6 +743,7 @@ class TestElasticsearchTaskHandler:
|
|
|
722
743
|
clean_logical_date = self.es_task_handler._clean_date(datetime(2016, 7, 8, 9, 10, 11, 12))
|
|
723
744
|
assert clean_logical_date == "2016_07_08T09_10_11_000012"
|
|
724
745
|
|
|
746
|
+
@pytest.mark.db_test
|
|
725
747
|
@pytest.mark.parametrize(
|
|
726
748
|
"json_format, es_frontend, expected_url",
|
|
727
749
|
[
|
|
@@ -781,6 +803,7 @@ class TestElasticsearchTaskHandler:
|
|
|
781
803
|
self.es_task_handler.frontend = frontend
|
|
782
804
|
assert self.es_task_handler.supports_external_link == expected
|
|
783
805
|
|
|
806
|
+
@pytest.mark.db_test
|
|
784
807
|
@mock.patch("sys.__stdout__", new_callable=StringIO)
|
|
785
808
|
def test_dynamic_offset(self, stdout_mock, ti, time_machine):
|
|
786
809
|
# arrange
|
|
@@ -843,6 +866,7 @@ class TestElasticsearchTaskHandler:
|
|
|
843
866
|
filename_template=None,
|
|
844
867
|
)
|
|
845
868
|
|
|
869
|
+
@pytest.mark.db_test
|
|
846
870
|
def test_write_to_es(self, ti):
|
|
847
871
|
self.es_task_handler.write_to_es = True
|
|
848
872
|
self.es_task_handler.json_format = True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|