apache-airflow-providers-openlineage 1.2.1__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-openlineage might be problematic. Click here for more details.
- {apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info → apache_airflow_providers_openlineage-1.3.0}/PKG-INFO +31 -22
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/README.rst +11 -4
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0/airflow/providers/openlineage}/LICENSE +52 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/__init__.py +4 -5
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/get_provider_info.py +4 -3
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/adapter.py +53 -13
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/facets.py +2 -2
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/listener.py +11 -2
- apache_airflow_providers_openlineage-1.3.0/pyproject.toml +84 -0
- apache-airflow-providers-openlineage-1.2.1/MANIFEST.in +0 -31
- apache-airflow-providers-openlineage-1.2.1/NOTICE +0 -6
- apache-airflow-providers-openlineage-1.2.1/PKG-INFO +0 -130
- apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/SOURCES.txt +0 -31
- apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/dependency_links.txt +0 -1
- apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/entry_points.txt +0 -5
- apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/not-zip-safe +0 -1
- apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/requires.txt +0 -8
- apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/top_level.txt +0 -1
- apache-airflow-providers-openlineage-1.2.1/pyproject.toml +0 -185
- apache-airflow-providers-openlineage-1.2.1/setup.cfg +0 -68
- apache-airflow-providers-openlineage-1.2.1/setup.py +0 -49
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/__init__.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/base.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/bash.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/manager.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/python.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/__init__.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/macros.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/openlineage.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/sqlparser.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/utils/__init__.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/utils/sql.py +0 -0
- {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/utils/utils.py +0 -0
|
@@ -1,19 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-openlineage
|
|
3
|
-
Version: 1.
|
|
4
|
-
Summary: Provider
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/
|
|
11
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html
|
|
12
|
-
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
13
|
-
Project-URL: Source Code, https://github.com/apache/airflow
|
|
14
|
-
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
15
|
-
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
16
|
-
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
3
|
+
Version: 1.3.0
|
|
4
|
+
Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
|
|
5
|
+
Keywords: airflow-provider,openlineage,airflow,integration
|
|
6
|
+
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
7
|
+
Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
8
|
+
Requires-Python: ~=3.8
|
|
9
|
+
Description-Content-Type: text/x-rst
|
|
17
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
18
11
|
Classifier: Environment :: Console
|
|
19
12
|
Classifier: Environment :: Web Environment
|
|
@@ -27,11 +20,20 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
27
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
28
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
29
22
|
Classifier: Topic :: System :: Monitoring
|
|
30
|
-
Requires-
|
|
31
|
-
|
|
23
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.6.0
|
|
24
|
+
Requires-Dist: apache-airflow>=2.7.0
|
|
25
|
+
Requires-Dist: attrs>=22.2
|
|
26
|
+
Requires-Dist: openlineage-integration-common>=0.28.0
|
|
27
|
+
Requires-Dist: openlineage-python>=0.28.0
|
|
28
|
+
Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
|
|
29
|
+
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
30
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html
|
|
31
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0
|
|
32
|
+
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
33
|
+
Project-URL: Source Code, https://github.com/apache/airflow
|
|
34
|
+
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
35
|
+
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
32
36
|
Provides-Extra: common.sql
|
|
33
|
-
License-File: LICENSE
|
|
34
|
-
License-File: NOTICE
|
|
35
37
|
|
|
36
38
|
|
|
37
39
|
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
@@ -50,7 +52,8 @@ License-File: NOTICE
|
|
|
50
52
|
KIND, either express or implied. See the License for the
|
|
51
53
|
specific language governing permissions and limitations
|
|
52
54
|
under the License.
|
|
53
|
-
|
|
55
|
+
|
|
56
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
54
57
|
or more contributor license agreements. See the NOTICE file
|
|
55
58
|
distributed with this work for additional information
|
|
56
59
|
regarding copyright ownership. The ASF licenses this file
|
|
@@ -67,10 +70,16 @@ License-File: NOTICE
|
|
|
67
70
|
specific language governing permissions and limitations
|
|
68
71
|
under the License.
|
|
69
72
|
|
|
73
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
74
|
+
OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
75
|
+
|
|
76
|
+
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
77
|
+
`PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
78
|
+
|
|
70
79
|
|
|
71
80
|
Package ``apache-airflow-providers-openlineage``
|
|
72
81
|
|
|
73
|
-
Release: ``1.
|
|
82
|
+
Release: ``1.3.0``
|
|
74
83
|
|
|
75
84
|
|
|
76
85
|
`OpenLineage <https://openlineage.io/>`__
|
|
@@ -83,7 +92,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
|
|
|
83
92
|
are in ``airflow.providers.openlineage`` python package.
|
|
84
93
|
|
|
85
94
|
You can find package information and changelog for the provider
|
|
86
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
95
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/>`_.
|
|
87
96
|
|
|
88
97
|
Installation
|
|
89
98
|
------------
|
|
@@ -127,4 +136,4 @@ Dependent package
|
|
|
127
136
|
============================================================================================================ ==============
|
|
128
137
|
|
|
129
138
|
The changelog for the provider package can be found in the
|
|
130
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
139
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html>`_.
|
{apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/README.rst
RENAMED
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
KIND, either express or implied. See the License for the
|
|
16
16
|
specific language governing permissions and limitations
|
|
17
17
|
under the License.
|
|
18
|
-
|
|
18
|
+
|
|
19
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
19
20
|
or more contributor license agreements. See the NOTICE file
|
|
20
21
|
distributed with this work for additional information
|
|
21
22
|
regarding copyright ownership. The ASF licenses this file
|
|
@@ -32,10 +33,16 @@
|
|
|
32
33
|
specific language governing permissions and limitations
|
|
33
34
|
under the License.
|
|
34
35
|
|
|
36
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
37
|
+
OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
38
|
+
|
|
39
|
+
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
40
|
+
`PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
41
|
+
|
|
35
42
|
|
|
36
43
|
Package ``apache-airflow-providers-openlineage``
|
|
37
44
|
|
|
38
|
-
Release: ``1.
|
|
45
|
+
Release: ``1.3.0``
|
|
39
46
|
|
|
40
47
|
|
|
41
48
|
`OpenLineage <https://openlineage.io/>`__
|
|
@@ -48,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
|
|
|
48
55
|
are in ``airflow.providers.openlineage`` python package.
|
|
49
56
|
|
|
50
57
|
You can find package information and changelog for the provider
|
|
51
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
58
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/>`_.
|
|
52
59
|
|
|
53
60
|
Installation
|
|
54
61
|
------------
|
|
@@ -92,4 +99,4 @@ Dependent package
|
|
|
92
99
|
============================================================================================================ ==============
|
|
93
100
|
|
|
94
101
|
The changelog for the provider package can be found in the
|
|
95
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.
|
|
102
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html>`_.
|
|
@@ -199,3 +199,55 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
============================================================================
|
|
204
|
+
APACHE AIRFLOW SUBCOMPONENTS:
|
|
205
|
+
|
|
206
|
+
The Apache Airflow project contains subcomponents with separate copyright
|
|
207
|
+
notices and license terms. Your use of the source code for the these
|
|
208
|
+
subcomponents is subject to the terms and conditions of the following
|
|
209
|
+
licenses.
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
========================================================================
|
|
213
|
+
Third party Apache 2.0 licenses
|
|
214
|
+
========================================================================
|
|
215
|
+
|
|
216
|
+
The following components are provided under the Apache 2.0 License.
|
|
217
|
+
See project link for details. The text of each license is also included
|
|
218
|
+
at licenses/LICENSE-[project].txt.
|
|
219
|
+
|
|
220
|
+
(ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
|
|
221
|
+
(ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
|
|
222
|
+
(ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
|
|
223
|
+
(ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
|
|
224
|
+
|
|
225
|
+
========================================================================
|
|
226
|
+
MIT licenses
|
|
227
|
+
========================================================================
|
|
228
|
+
|
|
229
|
+
The following components are provided under the MIT License. See project link for details.
|
|
230
|
+
The text of each license is also included at licenses/LICENSE-[project].txt.
|
|
231
|
+
|
|
232
|
+
(MIT License) jquery v3.5.1 (https://jquery.org/license/)
|
|
233
|
+
(MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
|
|
234
|
+
(MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
|
|
235
|
+
(MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
|
|
236
|
+
(MIT License) dataTables v1.10.25 (https://datatables.net)
|
|
237
|
+
(MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
|
|
238
|
+
(MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
|
|
239
|
+
(MIT License) MomentJS v2.24.0 (http://momentjs.com/)
|
|
240
|
+
(MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
|
|
241
|
+
|
|
242
|
+
========================================================================
|
|
243
|
+
BSD 3-Clause licenses
|
|
244
|
+
========================================================================
|
|
245
|
+
The following components are provided under the BSD 3-Clause license. See project links for details.
|
|
246
|
+
The text of each license is also included at licenses/LICENSE-[project].txt.
|
|
247
|
+
|
|
248
|
+
(BSD 3 License) d3 v5.16.0 (https://d3js.org)
|
|
249
|
+
(BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
|
|
250
|
+
(BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
|
|
251
|
+
|
|
252
|
+
========================================================================
|
|
253
|
+
See licenses/LICENSES-ui.txt for packages used in `/airflow/www`
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
#
|
|
2
1
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
2
|
# or more contributor license agreements. See the NOTICE file
|
|
4
3
|
# distributed with this work for additional information
|
|
@@ -19,8 +18,8 @@
|
|
|
19
18
|
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
20
19
|
# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
|
|
21
20
|
#
|
|
22
|
-
# IF YOU WANT TO MODIFY
|
|
23
|
-
# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/
|
|
21
|
+
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
22
|
+
# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
24
23
|
#
|
|
25
24
|
from __future__ import annotations
|
|
26
25
|
|
|
@@ -28,7 +27,7 @@ import packaging.version
|
|
|
28
27
|
|
|
29
28
|
__all__ = ["__version__"]
|
|
30
29
|
|
|
31
|
-
__version__ = "1.
|
|
30
|
+
__version__ = "1.3.0"
|
|
32
31
|
|
|
33
32
|
try:
|
|
34
33
|
from airflow import __version__ as airflow_version
|
|
@@ -39,5 +38,5 @@ if packaging.version.parse(packaging.version.parse(airflow_version).base_version
|
|
|
39
38
|
"2.7.0"
|
|
40
39
|
):
|
|
41
40
|
raise RuntimeError(
|
|
42
|
-
f"The package `apache-airflow-providers-openlineage:{__version__}`
|
|
41
|
+
f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.7.0+"
|
|
43
42
|
)
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
19
19
|
# OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
20
20
|
#
|
|
21
|
-
# IF YOU WANT TO MODIFY
|
|
22
|
-
# `get_provider_info_TEMPLATE.py.jinja2` IN the `
|
|
21
|
+
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
22
|
+
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
def get_provider_info():
|
|
@@ -28,7 +28,8 @@ def get_provider_info():
|
|
|
28
28
|
"name": "OpenLineage Airflow",
|
|
29
29
|
"description": "`OpenLineage <https://openlineage.io/>`__\n",
|
|
30
30
|
"suspended": False,
|
|
31
|
-
"
|
|
31
|
+
"source-date-epoch": 1701983402,
|
|
32
|
+
"versions": ["1.3.0", "1.2.1", "1.2.0", "1.1.1", "1.1.0", "1.0.2", "1.0.1", "1.0.0"],
|
|
32
33
|
"dependencies": [
|
|
33
34
|
"apache-airflow>=2.7.0",
|
|
34
35
|
"apache-airflow-providers-common-sql>=1.6.0",
|
|
@@ -38,6 +38,7 @@ from openlineage.client.run import Job, Run, RunEvent, RunState
|
|
|
38
38
|
from airflow.configuration import conf
|
|
39
39
|
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
|
|
40
40
|
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
|
|
41
|
+
from airflow.stats import Stats
|
|
41
42
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
42
43
|
|
|
43
44
|
if TYPE_CHECKING:
|
|
@@ -96,7 +97,8 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
96
97
|
with open(path) as config_file:
|
|
97
98
|
return yaml.safe_load(config_file)
|
|
98
99
|
|
|
99
|
-
|
|
100
|
+
@staticmethod
|
|
101
|
+
def build_dag_run_id(dag_id, dag_run_id):
|
|
100
102
|
return str(uuid.uuid3(uuid.NAMESPACE_URL, f"{_DAG_NAMESPACE}.{dag_id}.{dag_run_id}"))
|
|
101
103
|
|
|
102
104
|
@staticmethod
|
|
@@ -113,8 +115,10 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
113
115
|
self._client = self.get_or_create_openlineage_client()
|
|
114
116
|
redacted_event: RunEvent = self._redacter.redact(event, max_depth=20) # type: ignore[assignment]
|
|
115
117
|
try:
|
|
116
|
-
|
|
118
|
+
with Stats.timer("ol.emit.attempts"):
|
|
119
|
+
return self._client.emit(redacted_event)
|
|
117
120
|
except Exception as e:
|
|
121
|
+
Stats.incr("ol.emit.failed")
|
|
118
122
|
self.log.warning("Failed to emit OpenLineage event of id %s", event.run.runId)
|
|
119
123
|
self.log.debug("OpenLineage emission failure: %s", e)
|
|
120
124
|
|
|
@@ -160,17 +164,19 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
160
164
|
|
|
161
165
|
if not run_facets:
|
|
162
166
|
run_facets = {}
|
|
167
|
+
if task:
|
|
168
|
+
run_facets = {**task.run_facets, **run_facets}
|
|
163
169
|
run_facets["processing_engine"] = processing_engine_version_facet # type: ignore
|
|
164
170
|
event = RunEvent(
|
|
165
171
|
eventType=RunState.START,
|
|
166
172
|
eventTime=event_time,
|
|
167
173
|
run=self._build_run(
|
|
168
|
-
run_id,
|
|
169
|
-
job_name,
|
|
170
|
-
parent_job_name,
|
|
171
|
-
parent_run_id,
|
|
172
|
-
nominal_start_time,
|
|
173
|
-
nominal_end_time,
|
|
174
|
+
run_id=run_id,
|
|
175
|
+
job_name=job_name,
|
|
176
|
+
parent_job_name=parent_job_name,
|
|
177
|
+
parent_run_id=parent_run_id,
|
|
178
|
+
nominal_start_time=nominal_start_time,
|
|
179
|
+
nominal_end_time=nominal_end_time,
|
|
174
180
|
run_facets=run_facets,
|
|
175
181
|
),
|
|
176
182
|
job=self._build_job(
|
|
@@ -186,19 +192,36 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
186
192
|
)
|
|
187
193
|
self.emit(event)
|
|
188
194
|
|
|
189
|
-
def complete_task(
|
|
195
|
+
def complete_task(
|
|
196
|
+
self,
|
|
197
|
+
run_id: str,
|
|
198
|
+
job_name: str,
|
|
199
|
+
parent_job_name: str | None,
|
|
200
|
+
parent_run_id: str | None,
|
|
201
|
+
end_time: str,
|
|
202
|
+
task: OperatorLineage,
|
|
203
|
+
):
|
|
190
204
|
"""
|
|
191
205
|
Emits openlineage event of type COMPLETE.
|
|
192
206
|
|
|
193
207
|
:param run_id: globally unique identifier of task in dag run
|
|
194
208
|
:param job_name: globally unique identifier of task between dags
|
|
209
|
+
:param parent_job_name: the name of the parent job (typically the DAG,
|
|
210
|
+
but possibly a task group)
|
|
211
|
+
:param parent_run_id: identifier of job spawning this task
|
|
195
212
|
:param end_time: time of task completion
|
|
196
213
|
:param task: metadata container with information extracted from operator
|
|
197
214
|
"""
|
|
198
215
|
event = RunEvent(
|
|
199
216
|
eventType=RunState.COMPLETE,
|
|
200
217
|
eventTime=end_time,
|
|
201
|
-
run=self._build_run(
|
|
218
|
+
run=self._build_run(
|
|
219
|
+
run_id=run_id,
|
|
220
|
+
job_name=job_name,
|
|
221
|
+
parent_job_name=parent_job_name,
|
|
222
|
+
parent_run_id=parent_run_id,
|
|
223
|
+
run_facets=task.run_facets,
|
|
224
|
+
),
|
|
202
225
|
job=self._build_job(job_name, job_facets=task.job_facets),
|
|
203
226
|
inputs=task.inputs,
|
|
204
227
|
outputs=task.outputs,
|
|
@@ -206,20 +229,37 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
206
229
|
)
|
|
207
230
|
self.emit(event)
|
|
208
231
|
|
|
209
|
-
def fail_task(
|
|
232
|
+
def fail_task(
|
|
233
|
+
self,
|
|
234
|
+
run_id: str,
|
|
235
|
+
job_name: str,
|
|
236
|
+
parent_job_name: str | None,
|
|
237
|
+
parent_run_id: str | None,
|
|
238
|
+
end_time: str,
|
|
239
|
+
task: OperatorLineage,
|
|
240
|
+
):
|
|
210
241
|
"""
|
|
211
242
|
Emits openlineage event of type FAIL.
|
|
212
243
|
|
|
213
244
|
:param run_id: globally unique identifier of task in dag run
|
|
214
245
|
:param job_name: globally unique identifier of task between dags
|
|
246
|
+
:param parent_job_name: the name of the parent job (typically the DAG,
|
|
247
|
+
but possibly a task group)
|
|
248
|
+
:param parent_run_id: identifier of job spawning this task
|
|
215
249
|
:param end_time: time of task completion
|
|
216
250
|
:param task: metadata container with information extracted from operator
|
|
217
251
|
"""
|
|
218
252
|
event = RunEvent(
|
|
219
253
|
eventType=RunState.FAIL,
|
|
220
254
|
eventTime=end_time,
|
|
221
|
-
run=self._build_run(
|
|
222
|
-
|
|
255
|
+
run=self._build_run(
|
|
256
|
+
run_id=run_id,
|
|
257
|
+
job_name=job_name,
|
|
258
|
+
parent_job_name=parent_job_name,
|
|
259
|
+
parent_run_id=parent_run_id,
|
|
260
|
+
run_facets=task.run_facets,
|
|
261
|
+
),
|
|
262
|
+
job=self._build_job(job_name, job_facets=task.job_facets),
|
|
223
263
|
inputs=task.inputs,
|
|
224
264
|
outputs=task.outputs,
|
|
225
265
|
producer=_PRODUCER,
|
|
@@ -28,7 +28,7 @@ class AirflowMappedTaskRunFacet(BaseFacet):
|
|
|
28
28
|
mapIndex: int
|
|
29
29
|
operatorClass: str
|
|
30
30
|
|
|
31
|
-
_additional_skip_redact
|
|
31
|
+
_additional_skip_redact = ["operatorClass"]
|
|
32
32
|
|
|
33
33
|
@classmethod
|
|
34
34
|
def from_task_instance(cls, task_instance):
|
|
@@ -63,7 +63,7 @@ class UnknownOperatorInstance(RedactMixin):
|
|
|
63
63
|
properties: dict[str, object]
|
|
64
64
|
type: str = "operator"
|
|
65
65
|
|
|
66
|
-
_skip_redact
|
|
66
|
+
_skip_redact = ["name", "type"]
|
|
67
67
|
|
|
68
68
|
|
|
69
69
|
@define(slots=False)
|
|
@@ -101,7 +101,6 @@ class OpenLineageListener:
|
|
|
101
101
|
owners=dag.owner.split(", "),
|
|
102
102
|
task=task_metadata,
|
|
103
103
|
run_facets={
|
|
104
|
-
**task_metadata.run_facets,
|
|
105
104
|
**get_custom_facets(task_instance),
|
|
106
105
|
**get_airflow_run_facet(dagrun, dag, task_instance, task, task_uuid),
|
|
107
106
|
},
|
|
@@ -115,6 +114,7 @@ class OpenLineageListener:
|
|
|
115
114
|
|
|
116
115
|
dagrun = task_instance.dag_run
|
|
117
116
|
task = task_instance.task
|
|
117
|
+
dag = task.dag
|
|
118
118
|
|
|
119
119
|
task_uuid = OpenLineageAdapter.build_task_instance_run_id(
|
|
120
120
|
task.task_id, task_instance.execution_date, task_instance.try_number - 1
|
|
@@ -122,6 +122,8 @@ class OpenLineageListener:
|
|
|
122
122
|
|
|
123
123
|
@print_warning(self.log)
|
|
124
124
|
def on_success():
|
|
125
|
+
parent_run_id = OpenLineageAdapter.build_dag_run_id(dag.dag_id, dagrun.run_id)
|
|
126
|
+
|
|
125
127
|
task_metadata = self.extractor_manager.extract_metadata(
|
|
126
128
|
dagrun, task, complete=True, task_instance=task_instance
|
|
127
129
|
)
|
|
@@ -131,6 +133,8 @@ class OpenLineageListener:
|
|
|
131
133
|
self.adapter.complete_task(
|
|
132
134
|
run_id=task_uuid,
|
|
133
135
|
job_name=get_job_name(task),
|
|
136
|
+
parent_job_name=dag.dag_id,
|
|
137
|
+
parent_run_id=parent_run_id,
|
|
134
138
|
end_time=end_date.isoformat(),
|
|
135
139
|
task=task_metadata,
|
|
136
140
|
)
|
|
@@ -143,13 +147,16 @@ class OpenLineageListener:
|
|
|
143
147
|
|
|
144
148
|
dagrun = task_instance.dag_run
|
|
145
149
|
task = task_instance.task
|
|
150
|
+
dag = task.dag
|
|
146
151
|
|
|
147
152
|
task_uuid = OpenLineageAdapter.build_task_instance_run_id(
|
|
148
|
-
task.task_id, task_instance.execution_date, task_instance.try_number
|
|
153
|
+
task.task_id, task_instance.execution_date, task_instance.try_number
|
|
149
154
|
)
|
|
150
155
|
|
|
151
156
|
@print_warning(self.log)
|
|
152
157
|
def on_failure():
|
|
158
|
+
parent_run_id = OpenLineageAdapter.build_dag_run_id(dag.dag_id, dagrun.run_id)
|
|
159
|
+
|
|
153
160
|
task_metadata = self.extractor_manager.extract_metadata(
|
|
154
161
|
dagrun, task, complete=True, task_instance=task_instance
|
|
155
162
|
)
|
|
@@ -159,6 +166,8 @@ class OpenLineageListener:
|
|
|
159
166
|
self.adapter.fail_task(
|
|
160
167
|
run_id=task_uuid,
|
|
161
168
|
job_name=get_job_name(task),
|
|
169
|
+
parent_job_name=dag.dag_id,
|
|
170
|
+
parent_run_id=parent_run_id,
|
|
162
171
|
end_time=end_date.isoformat(),
|
|
163
172
|
task=task_metadata,
|
|
164
173
|
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
|
|
2
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
# or more contributor license agreements. See the NOTICE file
|
|
4
|
+
# distributed with this work for additional information
|
|
5
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
# to you under the Apache License, Version 2.0 (the
|
|
7
|
+
# "License"); you may not use this file except in compliance
|
|
8
|
+
# with the License. You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing,
|
|
13
|
+
# software distributed under the License is distributed on an
|
|
14
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
+
# KIND, either express or implied. See the License for the
|
|
16
|
+
# specific language governing permissions and limitations
|
|
17
|
+
# under the License.
|
|
18
|
+
|
|
19
|
+
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
20
|
+
# OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
21
|
+
|
|
22
|
+
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
23
|
+
# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
24
|
+
#
|
|
25
|
+
[build-system]
|
|
26
|
+
requires = ["flit_core >=3.2,<4"]
|
|
27
|
+
build-backend = "flit_core.buildapi"
|
|
28
|
+
|
|
29
|
+
[project]
|
|
30
|
+
name = "apache-airflow-providers-openlineage"
|
|
31
|
+
version = "1.3.0"
|
|
32
|
+
description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
|
|
33
|
+
readme = "README.rst"
|
|
34
|
+
authors = [
|
|
35
|
+
{name="Apache Software Foundation", email="dev@airflow.apache.org"},
|
|
36
|
+
]
|
|
37
|
+
maintainers = [
|
|
38
|
+
{name="Apache Software Foundation", email="dev@airflow.apache.org"},
|
|
39
|
+
]
|
|
40
|
+
keywords = [ "airflow-provider", "openlineage", "airflow", "integration" ]
|
|
41
|
+
classifiers = [
|
|
42
|
+
"Development Status :: 5 - Production/Stable",
|
|
43
|
+
"Environment :: Console",
|
|
44
|
+
"Environment :: Web Environment",
|
|
45
|
+
"Intended Audience :: Developers",
|
|
46
|
+
"Intended Audience :: System Administrators",
|
|
47
|
+
"Framework :: Apache Airflow",
|
|
48
|
+
"Framework :: Apache Airflow :: Provider",
|
|
49
|
+
"License :: OSI Approved :: Apache Software License",
|
|
50
|
+
"Programming Language :: Python :: 3.8",
|
|
51
|
+
"Programming Language :: Python :: 3.9",
|
|
52
|
+
"Programming Language :: Python :: 3.10",
|
|
53
|
+
"Programming Language :: Python :: 3.11",
|
|
54
|
+
"Topic :: System :: Monitoring",
|
|
55
|
+
]
|
|
56
|
+
requires-python = "~=3.8"
|
|
57
|
+
dependencies = [
|
|
58
|
+
"apache-airflow-providers-common-sql>=1.6.0",
|
|
59
|
+
"apache-airflow>=2.7.0",
|
|
60
|
+
"attrs>=22.2",
|
|
61
|
+
"openlineage-integration-common>=0.28.0",
|
|
62
|
+
"openlineage-python>=0.28.0",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
[project.urls]
|
|
66
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0"
|
|
67
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html"
|
|
68
|
+
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
69
|
+
"Source Code" = "https://github.com/apache/airflow"
|
|
70
|
+
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
71
|
+
"Twitter" = "https://twitter.com/ApacheAirflow"
|
|
72
|
+
"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
|
|
73
|
+
|
|
74
|
+
[project.entry-points."apache_airflow_provider"]
|
|
75
|
+
provider_info = "airflow.providers.openlineage.get_provider_info:get_provider_info"
|
|
76
|
+
[project.entry-points."airflow.plugins"]
|
|
77
|
+
openlineage = "airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin"
|
|
78
|
+
[project.optional-dependencies]
|
|
79
|
+
"common.sql" = [
|
|
80
|
+
"apache-airflow-providers-common-sql",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[tool.flit.module]
|
|
84
|
+
name = "airflow.providers.openlineage"
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
|
|
19
|
-
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
20
|
-
# OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
21
|
-
|
|
22
|
-
# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
|
|
23
|
-
# `MANIFEST_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
include NOTICE
|
|
29
|
-
include LICENSE
|
|
30
|
-
include CHANGELOG.rst
|
|
31
|
-
global-exclude __pycache__ *.pyc
|
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: apache-airflow-providers-openlineage
|
|
3
|
-
Version: 1.2.1
|
|
4
|
-
Summary: Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
|
|
5
|
-
Home-page: https://airflow.apache.org/
|
|
6
|
-
Download-URL: https://archive.apache.org/dist/airflow/providers
|
|
7
|
-
Author: Apache Software Foundation
|
|
8
|
-
Author-email: dev@airflow.apache.org
|
|
9
|
-
License: Apache License 2.0
|
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/
|
|
11
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html
|
|
12
|
-
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
13
|
-
Project-URL: Source Code, https://github.com/apache/airflow
|
|
14
|
-
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
15
|
-
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
16
|
-
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
17
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
18
|
-
Classifier: Environment :: Console
|
|
19
|
-
Classifier: Environment :: Web Environment
|
|
20
|
-
Classifier: Intended Audience :: Developers
|
|
21
|
-
Classifier: Intended Audience :: System Administrators
|
|
22
|
-
Classifier: Framework :: Apache Airflow
|
|
23
|
-
Classifier: Framework :: Apache Airflow :: Provider
|
|
24
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
25
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
26
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
27
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
28
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
29
|
-
Classifier: Topic :: System :: Monitoring
|
|
30
|
-
Requires-Python: ~=3.8
|
|
31
|
-
Description-Content-Type: text/x-rst
|
|
32
|
-
Provides-Extra: common.sql
|
|
33
|
-
License-File: LICENSE
|
|
34
|
-
License-File: NOTICE
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
38
|
-
or more contributor license agreements. See the NOTICE file
|
|
39
|
-
distributed with this work for additional information
|
|
40
|
-
regarding copyright ownership. The ASF licenses this file
|
|
41
|
-
to you under the Apache License, Version 2.0 (the
|
|
42
|
-
"License"); you may not use this file except in compliance
|
|
43
|
-
with the License. You may obtain a copy of the License at
|
|
44
|
-
|
|
45
|
-
.. http://www.apache.org/licenses/LICENSE-2.0
|
|
46
|
-
|
|
47
|
-
.. Unless required by applicable law or agreed to in writing,
|
|
48
|
-
software distributed under the License is distributed on an
|
|
49
|
-
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
50
|
-
KIND, either express or implied. See the License for the
|
|
51
|
-
specific language governing permissions and limitations
|
|
52
|
-
under the License.
|
|
53
|
-
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
54
|
-
or more contributor license agreements. See the NOTICE file
|
|
55
|
-
distributed with this work for additional information
|
|
56
|
-
regarding copyright ownership. The ASF licenses this file
|
|
57
|
-
to you under the Apache License, Version 2.0 (the
|
|
58
|
-
"License"); you may not use this file except in compliance
|
|
59
|
-
with the License. You may obtain a copy of the License at
|
|
60
|
-
|
|
61
|
-
.. http://www.apache.org/licenses/LICENSE-2.0
|
|
62
|
-
|
|
63
|
-
.. Unless required by applicable law or agreed to in writing,
|
|
64
|
-
software distributed under the License is distributed on an
|
|
65
|
-
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
66
|
-
KIND, either express or implied. See the License for the
|
|
67
|
-
specific language governing permissions and limitations
|
|
68
|
-
under the License.
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
Package ``apache-airflow-providers-openlineage``
|
|
72
|
-
|
|
73
|
-
Release: ``1.2.1``
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
`OpenLineage <https://openlineage.io/>`__
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
Provider package
|
|
80
|
-
----------------
|
|
81
|
-
|
|
82
|
-
This is a provider package for ``openlineage`` provider. All classes for this provider package
|
|
83
|
-
are in ``airflow.providers.openlineage`` python package.
|
|
84
|
-
|
|
85
|
-
You can find package information and changelog for the provider
|
|
86
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/>`_.
|
|
87
|
-
|
|
88
|
-
Installation
|
|
89
|
-
------------
|
|
90
|
-
|
|
91
|
-
You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
|
|
92
|
-
for the minimum Airflow version supported) via
|
|
93
|
-
``pip install apache-airflow-providers-openlineage``
|
|
94
|
-
|
|
95
|
-
The package supports the following python versions: 3.8,3.9,3.10,3.11
|
|
96
|
-
|
|
97
|
-
Requirements
|
|
98
|
-
------------
|
|
99
|
-
|
|
100
|
-
======================================= ==================
|
|
101
|
-
PIP package Version required
|
|
102
|
-
======================================= ==================
|
|
103
|
-
``apache-airflow`` ``>=2.7.0``
|
|
104
|
-
``apache-airflow-providers-common-sql`` ``>=1.6.0``
|
|
105
|
-
``attrs`` ``>=22.2``
|
|
106
|
-
``openlineage-integration-common`` ``>=0.28.0``
|
|
107
|
-
``openlineage-python`` ``>=0.28.0``
|
|
108
|
-
======================================= ==================
|
|
109
|
-
|
|
110
|
-
Cross provider package dependencies
|
|
111
|
-
-----------------------------------
|
|
112
|
-
|
|
113
|
-
Those are dependencies that might be needed in order to use all the features of the package.
|
|
114
|
-
You need to install the specified provider packages in order to use them.
|
|
115
|
-
|
|
116
|
-
You can install such cross-provider dependencies when installing from PyPI. For example:
|
|
117
|
-
|
|
118
|
-
.. code-block:: bash
|
|
119
|
-
|
|
120
|
-
pip install apache-airflow-providers-openlineage[common.sql]
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
============================================================================================================ ==============
|
|
124
|
-
Dependent package Extra
|
|
125
|
-
============================================================================================================ ==============
|
|
126
|
-
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
127
|
-
============================================================================================================ ==============
|
|
128
|
-
|
|
129
|
-
The changelog for the provider package can be found in the
|
|
130
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html>`_.
|
apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/SOURCES.txt
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
MANIFEST.in
|
|
3
|
-
NOTICE
|
|
4
|
-
README.rst
|
|
5
|
-
pyproject.toml
|
|
6
|
-
setup.cfg
|
|
7
|
-
setup.py
|
|
8
|
-
airflow/providers/openlineage/__init__.py
|
|
9
|
-
airflow/providers/openlineage/get_provider_info.py
|
|
10
|
-
airflow/providers/openlineage/sqlparser.py
|
|
11
|
-
airflow/providers/openlineage/extractors/__init__.py
|
|
12
|
-
airflow/providers/openlineage/extractors/base.py
|
|
13
|
-
airflow/providers/openlineage/extractors/bash.py
|
|
14
|
-
airflow/providers/openlineage/extractors/manager.py
|
|
15
|
-
airflow/providers/openlineage/extractors/python.py
|
|
16
|
-
airflow/providers/openlineage/plugins/__init__.py
|
|
17
|
-
airflow/providers/openlineage/plugins/adapter.py
|
|
18
|
-
airflow/providers/openlineage/plugins/facets.py
|
|
19
|
-
airflow/providers/openlineage/plugins/listener.py
|
|
20
|
-
airflow/providers/openlineage/plugins/macros.py
|
|
21
|
-
airflow/providers/openlineage/plugins/openlineage.py
|
|
22
|
-
airflow/providers/openlineage/utils/__init__.py
|
|
23
|
-
airflow/providers/openlineage/utils/sql.py
|
|
24
|
-
airflow/providers/openlineage/utils/utils.py
|
|
25
|
-
apache_airflow_providers_openlineage.egg-info/PKG-INFO
|
|
26
|
-
apache_airflow_providers_openlineage.egg-info/SOURCES.txt
|
|
27
|
-
apache_airflow_providers_openlineage.egg-info/dependency_links.txt
|
|
28
|
-
apache_airflow_providers_openlineage.egg-info/entry_points.txt
|
|
29
|
-
apache_airflow_providers_openlineage.egg-info/not-zip-safe
|
|
30
|
-
apache_airflow_providers_openlineage.egg-info/requires.txt
|
|
31
|
-
apache_airflow_providers_openlineage.egg-info/top_level.txt
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
airflow
|
|
@@ -1,185 +0,0 @@
|
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
|
3
|
-
# distributed with this work for additional information
|
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
|
6
|
-
# "License"); you may not use this file except in compliance
|
|
7
|
-
# with the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
|
12
|
-
# software distributed under the License is distributed on an
|
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
-
# KIND, either express or implied. See the License for the
|
|
15
|
-
# specific language governing permissions and limitations
|
|
16
|
-
# under the License.
|
|
17
|
-
[tool.black]
|
|
18
|
-
line-length = 110
|
|
19
|
-
target-version = ['py38', 'py39', 'py310', 'py311']
|
|
20
|
-
|
|
21
|
-
# Editable installs are currently broken using setuptools 64.0.0 and above. The problem is tracked in
|
|
22
|
-
# https://github.com/pypa/setuptools/issues/3548. We're also discussing how we could potentially fix
|
|
23
|
-
# this problem on our end in issue https://github.com/apache/airflow/issues/30764. Until then we need
|
|
24
|
-
# to use one of the following workarounds locally for editable installs:
|
|
25
|
-
# 1) Pin setuptools <= 63.4.3 below in the [build-system] section.
|
|
26
|
-
# 2) Include your airflow source code directory in PYTHONPATH.
|
|
27
|
-
[build-system]
|
|
28
|
-
requires = ['setuptools==67.2.0']
|
|
29
|
-
build-backend = "setuptools.build_meta"
|
|
30
|
-
|
|
31
|
-
[tool.ruff]
|
|
32
|
-
target-version = "py38"
|
|
33
|
-
typing-modules = ["airflow.typing_compat"]
|
|
34
|
-
line-length = 110
|
|
35
|
-
extend-exclude = [
|
|
36
|
-
".eggs",
|
|
37
|
-
"airflow/_vendor/*",
|
|
38
|
-
"airflow/providers/google/ads/_vendor/*",
|
|
39
|
-
# The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
|
|
40
|
-
# ignore them in ruff
|
|
41
|
-
"airflow/providers/common/sql/*/*.pyi",
|
|
42
|
-
"airflow/migrations/versions/*.py",
|
|
43
|
-
"tests/dags/test_imports.py",
|
|
44
|
-
]
|
|
45
|
-
|
|
46
|
-
extend-select = [
|
|
47
|
-
"I", # Missing required import (auto-fixable)
|
|
48
|
-
"UP", # Pyupgrade
|
|
49
|
-
"RUF100", # Unused noqa (auto-fixable)
|
|
50
|
-
|
|
51
|
-
# We ignore more pydocstyle than we enable, so be more selective at what we enable
|
|
52
|
-
"D101",
|
|
53
|
-
"D106",
|
|
54
|
-
"D2",
|
|
55
|
-
"D3",
|
|
56
|
-
"D400",
|
|
57
|
-
# "D401", # Not enabled by ruff, but we don't want it
|
|
58
|
-
"D402",
|
|
59
|
-
"D403",
|
|
60
|
-
"D412",
|
|
61
|
-
"D419",
|
|
62
|
-
"TCH", # Rules around TYPE_CHECKING blocks
|
|
63
|
-
"TID251", # Specific modules or module members that may not be imported or accessed
|
|
64
|
-
"TID253", # Ban certain modules from being imported at module level
|
|
65
|
-
]
|
|
66
|
-
extend-ignore = [
|
|
67
|
-
"D203",
|
|
68
|
-
"D212",
|
|
69
|
-
"D213",
|
|
70
|
-
"D214",
|
|
71
|
-
"D215",
|
|
72
|
-
"E731",
|
|
73
|
-
"TCH003", # Do not move imports from stdlib to TYPE_CHECKING block
|
|
74
|
-
]
|
|
75
|
-
|
|
76
|
-
namespace-packages = ["airflow/providers"]
|
|
77
|
-
|
|
78
|
-
[tool.pytest.ini_options]
|
|
79
|
-
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
|
|
80
|
-
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
|
|
81
|
-
# * And we focus on use native pytest capabilities rather than adopt another frameworks.
|
|
82
|
-
addopts = "-rasl --verbosity=2 -p no:flaky -p no:nose --asyncio-mode=strict"
|
|
83
|
-
norecursedirs = [
|
|
84
|
-
".eggs",
|
|
85
|
-
"airflow",
|
|
86
|
-
"tests/dags_with_system_exit",
|
|
87
|
-
"tests/test_utils",
|
|
88
|
-
"tests/dags_corrupted",
|
|
89
|
-
"tests/dags",
|
|
90
|
-
"tests/system/providers/google/cloud/dataproc/resources",
|
|
91
|
-
"tests/system/providers/google/cloud/gcs/resources",
|
|
92
|
-
]
|
|
93
|
-
log_level = "INFO"
|
|
94
|
-
filterwarnings = [
|
|
95
|
-
"error::pytest.PytestCollectionWarning",
|
|
96
|
-
"ignore::DeprecationWarning:flask_appbuilder.filemanager",
|
|
97
|
-
"ignore::DeprecationWarning:flask_appbuilder.widgets",
|
|
98
|
-
# https://github.com/dpgaspar/Flask-AppBuilder/pull/1940
|
|
99
|
-
"ignore::DeprecationWarning:flask_sqlalchemy",
|
|
100
|
-
# https://github.com/dpgaspar/Flask-AppBuilder/pull/1903
|
|
101
|
-
"ignore::DeprecationWarning:apispec.utils",
|
|
102
|
-
]
|
|
103
|
-
python_files = [
|
|
104
|
-
"test_*.py",
|
|
105
|
-
"example_*.py",
|
|
106
|
-
]
|
|
107
|
-
testpaths = [
|
|
108
|
-
"tests",
|
|
109
|
-
]
|
|
110
|
-
|
|
111
|
-
[tool.ruff.isort]
|
|
112
|
-
required-imports = ["from __future__ import annotations"]
|
|
113
|
-
combine-as-imports = true
|
|
114
|
-
|
|
115
|
-
[tool.ruff.per-file-ignores]
|
|
116
|
-
"airflow/models/__init__.py" = ["F401", "TCH004"]
|
|
117
|
-
"airflow/models/sqla_models.py" = ["F401"]
|
|
118
|
-
|
|
119
|
-
# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
|
|
120
|
-
# needed for the test to work
|
|
121
|
-
"tests/decorators/test_python.py" = ["I002"]
|
|
122
|
-
|
|
123
|
-
# The Pydantic representations of SqlAlchemy Models are not parsed well with Pydantic
|
|
124
|
-
# when __future__.annotations is used so we need to skip them from upgrading
|
|
125
|
-
# Pydantic also require models to be imported during execution
|
|
126
|
-
"airflow/serialization/pydantic/*.py" = ["I002", "UP007", "TCH001"]
|
|
127
|
-
|
|
128
|
-
# Ignore pydoc style from these
|
|
129
|
-
"*.pyi" = ["D"]
|
|
130
|
-
"scripts/*" = ["D"]
|
|
131
|
-
"docs/*" = ["D"]
|
|
132
|
-
"provider_packages/*" = ["D"]
|
|
133
|
-
"*/example_dags/*" = ["D"]
|
|
134
|
-
"chart/*" = ["D"]
|
|
135
|
-
# In addition ignore top level imports, e.g. pandas, numpy in tests and dev
|
|
136
|
-
"dev/*" = ["D", "TID253"]
|
|
137
|
-
"tests/*" = ["D", "TID253"]
|
|
138
|
-
"docker_tests/*" = ["D", "TID253"]
|
|
139
|
-
"kubernetes_tests/*" = ["D", "TID253"]
|
|
140
|
-
|
|
141
|
-
# All of the modules which have an extra license header (i.e. that we copy from another project) need to
|
|
142
|
-
# ignore E402 -- module level import not at top level
|
|
143
|
-
"airflow/api/auth/backend/kerberos_auth.py" = ["E402"]
|
|
144
|
-
"airflow/security/kerberos.py" = ["E402"]
|
|
145
|
-
"airflow/security/utils.py" = ["E402"]
|
|
146
|
-
"tests/providers/elasticsearch/log/elasticmock/__init__.py" = ["E402"]
|
|
147
|
-
"tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py" = ["E402"]
|
|
148
|
-
|
|
149
|
-
[tool.ruff.flake8-tidy-imports.banned-api]
|
|
150
|
-
"airflow.AirflowException".msg = "Use airflow.exceptions.AirflowException instead."
|
|
151
|
-
"airflow.Dataset".msg = "Use airflow.datasets.Dataset instead."
|
|
152
|
-
|
|
153
|
-
[tool.ruff.flake8-tidy-imports]
|
|
154
|
-
# Ban certain modules from being imported at module level, instead requiring
|
|
155
|
-
# that they're imported lazily (e.g., within a function definition).
|
|
156
|
-
banned-module-level-imports = ["numpy", "pandas"]
|
|
157
|
-
|
|
158
|
-
[tool.ruff.flake8-type-checking]
|
|
159
|
-
exempt-modules = ["typing", "typing_extensions"]
|
|
160
|
-
|
|
161
|
-
[tool.coverage.run]
|
|
162
|
-
branch = true
|
|
163
|
-
relative_files = true
|
|
164
|
-
source = ["airflow"]
|
|
165
|
-
omit = [
|
|
166
|
-
"airflow/_vendor/**",
|
|
167
|
-
"airflow/contrib/**",
|
|
168
|
-
"airflow/example_dags/**",
|
|
169
|
-
"airflow/migrations/**",
|
|
170
|
-
"airflow/providers/**/example_dags/**",
|
|
171
|
-
"airflow/www/node_modules/**",
|
|
172
|
-
"airflow/providers/google/ads/_vendor/**",
|
|
173
|
-
]
|
|
174
|
-
|
|
175
|
-
[tool.coverage.report]
|
|
176
|
-
skip_empty = true
|
|
177
|
-
exclude_also = [
|
|
178
|
-
"def __repr__",
|
|
179
|
-
"raise AssertionError",
|
|
180
|
-
"raise NotImplementedError",
|
|
181
|
-
"if __name__ == .__main__.:",
|
|
182
|
-
"@(abc\\.)?abstractmethod",
|
|
183
|
-
"@(typing(_extensions)?\\.)?overload",
|
|
184
|
-
"if (typing(_extensions)?\\.)?TYPE_CHECKING:"
|
|
185
|
-
]
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
[metadata]
|
|
2
|
-
name = apache-airflow-providers-openlineage
|
|
3
|
-
summary = Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
|
|
4
|
-
description = Provider package apache-airflow-providers-openlineage for Apache Airflow
|
|
5
|
-
long_description = file: README.rst
|
|
6
|
-
long_description_content_type = text/x-rst
|
|
7
|
-
author = Apache Software Foundation
|
|
8
|
-
author_email = dev@airflow.apache.org
|
|
9
|
-
url = https://airflow.apache.org/
|
|
10
|
-
download_url = https://archive.apache.org/dist/airflow/providers
|
|
11
|
-
license = Apache License 2.0
|
|
12
|
-
license_files =
|
|
13
|
-
LICENSE
|
|
14
|
-
NOTICE
|
|
15
|
-
classifiers =
|
|
16
|
-
Development Status :: 5 - Production/Stable
|
|
17
|
-
Environment :: Console
|
|
18
|
-
Environment :: Web Environment
|
|
19
|
-
Intended Audience :: Developers
|
|
20
|
-
Intended Audience :: System Administrators
|
|
21
|
-
Framework :: Apache Airflow
|
|
22
|
-
Framework :: Apache Airflow :: Provider
|
|
23
|
-
License :: OSI Approved :: Apache Software License
|
|
24
|
-
Programming Language :: Python :: 3.8
|
|
25
|
-
Programming Language :: Python :: 3.9
|
|
26
|
-
Programming Language :: Python :: 3.10
|
|
27
|
-
Programming Language :: Python :: 3.11
|
|
28
|
-
Topic :: System :: Monitoring
|
|
29
|
-
project_urls =
|
|
30
|
-
Documentation=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/
|
|
31
|
-
Changelog=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html
|
|
32
|
-
Bug Tracker=https://github.com/apache/airflow/issues
|
|
33
|
-
Source Code=https://github.com/apache/airflow
|
|
34
|
-
Slack Chat=https://s.apache.org/airflow-slack
|
|
35
|
-
Twitter=https://twitter.com/ApacheAirflow
|
|
36
|
-
YouTube=https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
37
|
-
|
|
38
|
-
[bdist_wheel]
|
|
39
|
-
python_tag = py3
|
|
40
|
-
|
|
41
|
-
[options]
|
|
42
|
-
zip_safe = False
|
|
43
|
-
include_package_data = True
|
|
44
|
-
python_requires = ~=3.8
|
|
45
|
-
packages = find:
|
|
46
|
-
setup_requires =
|
|
47
|
-
setuptools
|
|
48
|
-
wheel
|
|
49
|
-
install_requires =
|
|
50
|
-
apache-airflow-providers-common-sql>=1.6.0
|
|
51
|
-
apache-airflow>=2.7.0
|
|
52
|
-
attrs>=22.2
|
|
53
|
-
openlineage-integration-common>=0.28.0
|
|
54
|
-
openlineage-python>=0.28.0
|
|
55
|
-
|
|
56
|
-
[options.entry_points]
|
|
57
|
-
apache_airflow_provider =
|
|
58
|
-
provider_info=airflow.providers.openlineage.get_provider_info:get_provider_info
|
|
59
|
-
airflow.plugins =
|
|
60
|
-
openlineage=airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin
|
|
61
|
-
|
|
62
|
-
[files]
|
|
63
|
-
packages = airflow.providers.openlineage
|
|
64
|
-
|
|
65
|
-
[egg_info]
|
|
66
|
-
tag_build =
|
|
67
|
-
tag_date = 0
|
|
68
|
-
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
|
|
19
|
-
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
20
|
-
# OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
21
|
-
#
|
|
22
|
-
# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
|
|
23
|
-
# `SETUP_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
|
|
24
|
-
|
|
25
|
-
"""Setup.py for the apache-airflow-providers-openlineage package."""
|
|
26
|
-
|
|
27
|
-
from setuptools import find_namespace_packages, setup
|
|
28
|
-
|
|
29
|
-
version = "1.2.1"
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def do_setup():
|
|
33
|
-
"""Perform the package apache-airflow-providers-openlineage setup."""
|
|
34
|
-
setup(
|
|
35
|
-
version=version,
|
|
36
|
-
extras_require={"common.sql": ["apache-airflow-providers-common-sql"]},
|
|
37
|
-
packages=find_namespace_packages(
|
|
38
|
-
include=[
|
|
39
|
-
"airflow.providers.openlineage",
|
|
40
|
-
"airflow.providers.openlineage.*",
|
|
41
|
-
"airflow.providers.openlineage_vendor",
|
|
42
|
-
"airflow.providers.openlineage_vendor.*",
|
|
43
|
-
],
|
|
44
|
-
),
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if __name__ == "__main__":
|
|
49
|
-
do_setup()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|