apache-airflow-providers-openlineage 1.2.1__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of apache-airflow-providers-openlineage might be problematic. Click here for more details.

Files changed (33) hide show
  1. {apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info → apache_airflow_providers_openlineage-1.3.0}/PKG-INFO +31 -22
  2. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/README.rst +11 -4
  3. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0/airflow/providers/openlineage}/LICENSE +52 -0
  4. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/__init__.py +4 -5
  5. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/get_provider_info.py +4 -3
  6. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/adapter.py +53 -13
  7. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/facets.py +2 -2
  8. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/listener.py +11 -2
  9. apache_airflow_providers_openlineage-1.3.0/pyproject.toml +84 -0
  10. apache-airflow-providers-openlineage-1.2.1/MANIFEST.in +0 -31
  11. apache-airflow-providers-openlineage-1.2.1/NOTICE +0 -6
  12. apache-airflow-providers-openlineage-1.2.1/PKG-INFO +0 -130
  13. apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/SOURCES.txt +0 -31
  14. apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/dependency_links.txt +0 -1
  15. apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/entry_points.txt +0 -5
  16. apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/not-zip-safe +0 -1
  17. apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/requires.txt +0 -8
  18. apache-airflow-providers-openlineage-1.2.1/apache_airflow_providers_openlineage.egg-info/top_level.txt +0 -1
  19. apache-airflow-providers-openlineage-1.2.1/pyproject.toml +0 -185
  20. apache-airflow-providers-openlineage-1.2.1/setup.cfg +0 -68
  21. apache-airflow-providers-openlineage-1.2.1/setup.py +0 -49
  22. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/__init__.py +0 -0
  23. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/base.py +0 -0
  24. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/bash.py +0 -0
  25. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/manager.py +0 -0
  26. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/extractors/python.py +0 -0
  27. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/__init__.py +0 -0
  28. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/macros.py +0 -0
  29. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/plugins/openlineage.py +0 -0
  30. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/sqlparser.py +0 -0
  31. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/utils/__init__.py +0 -0
  32. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/utils/sql.py +0 -0
  33. {apache-airflow-providers-openlineage-1.2.1 → apache_airflow_providers_openlineage-1.3.0}/airflow/providers/openlineage/utils/utils.py +0 -0
@@ -1,19 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: apache-airflow-providers-openlineage
3
- Version: 1.2.1
4
- Summary: Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
5
- Home-page: https://airflow.apache.org/
6
- Download-URL: https://archive.apache.org/dist/airflow/providers
7
- Author: Apache Software Foundation
8
- Author-email: dev@airflow.apache.org
9
- License: Apache License 2.0
10
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/
11
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html
12
- Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
13
- Project-URL: Source Code, https://github.com/apache/airflow
14
- Project-URL: Slack Chat, https://s.apache.org/airflow-slack
15
- Project-URL: Twitter, https://twitter.com/ApacheAirflow
16
- Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
3
+ Version: 1.3.0
4
+ Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
5
+ Keywords: airflow-provider,openlineage,airflow,integration
6
+ Author-email: Apache Software Foundation <dev@airflow.apache.org>
7
+ Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
8
+ Requires-Python: ~=3.8
9
+ Description-Content-Type: text/x-rst
17
10
  Classifier: Development Status :: 5 - Production/Stable
18
11
  Classifier: Environment :: Console
19
12
  Classifier: Environment :: Web Environment
@@ -27,11 +20,20 @@ Classifier: Programming Language :: Python :: 3.9
27
20
  Classifier: Programming Language :: Python :: 3.10
28
21
  Classifier: Programming Language :: Python :: 3.11
29
22
  Classifier: Topic :: System :: Monitoring
30
- Requires-Python: ~=3.8
31
- Description-Content-Type: text/x-rst
23
+ Requires-Dist: apache-airflow-providers-common-sql>=1.6.0
24
+ Requires-Dist: apache-airflow>=2.7.0
25
+ Requires-Dist: attrs>=22.2
26
+ Requires-Dist: openlineage-integration-common>=0.28.0
27
+ Requires-Dist: openlineage-python>=0.28.0
28
+ Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
29
+ Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
30
+ Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html
31
+ Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0
32
+ Project-URL: Slack Chat, https://s.apache.org/airflow-slack
33
+ Project-URL: Source Code, https://github.com/apache/airflow
34
+ Project-URL: Twitter, https://twitter.com/ApacheAirflow
35
+ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
32
36
  Provides-Extra: common.sql
33
- License-File: LICENSE
34
- License-File: NOTICE
35
37
 
36
38
 
37
39
  .. Licensed to the Apache Software Foundation (ASF) under one
@@ -50,7 +52,8 @@ License-File: NOTICE
50
52
  KIND, either express or implied. See the License for the
51
53
  specific language governing permissions and limitations
52
54
  under the License.
53
- .. Licensed to the Apache Software Foundation (ASF) under one
55
+
56
+ .. Licensed to the Apache Software Foundation (ASF) under one
54
57
  or more contributor license agreements. See the NOTICE file
55
58
  distributed with this work for additional information
56
59
  regarding copyright ownership. The ASF licenses this file
@@ -67,10 +70,16 @@ License-File: NOTICE
67
70
  specific language governing permissions and limitations
68
71
  under the License.
69
72
 
73
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
74
+ OVERWRITTEN WHEN PREPARING PACKAGES.
75
+
76
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
77
+ `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
78
+
70
79
 
71
80
  Package ``apache-airflow-providers-openlineage``
72
81
 
73
- Release: ``1.2.1``
82
+ Release: ``1.3.0``
74
83
 
75
84
 
76
85
  `OpenLineage <https://openlineage.io/>`__
@@ -83,7 +92,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
83
92
  are in ``airflow.providers.openlineage`` python package.
84
93
 
85
94
  You can find package information and changelog for the provider
86
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/>`_.
95
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/>`_.
87
96
 
88
97
  Installation
89
98
  ------------
@@ -127,4 +136,4 @@ Dependent package
127
136
  ============================================================================================================ ==============
128
137
 
129
138
  The changelog for the provider package can be found in the
130
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html>`_.
139
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html>`_.
@@ -15,7 +15,8 @@
15
15
  KIND, either express or implied. See the License for the
16
16
  specific language governing permissions and limitations
17
17
  under the License.
18
- .. Licensed to the Apache Software Foundation (ASF) under one
18
+
19
+ .. Licensed to the Apache Software Foundation (ASF) under one
19
20
  or more contributor license agreements. See the NOTICE file
20
21
  distributed with this work for additional information
21
22
  regarding copyright ownership. The ASF licenses this file
@@ -32,10 +33,16 @@
32
33
  specific language governing permissions and limitations
33
34
  under the License.
34
35
 
36
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
37
+ OVERWRITTEN WHEN PREPARING PACKAGES.
38
+
39
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
40
+ `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
41
+
35
42
 
36
43
  Package ``apache-airflow-providers-openlineage``
37
44
 
38
- Release: ``1.2.1``
45
+ Release: ``1.3.0``
39
46
 
40
47
 
41
48
  `OpenLineage <https://openlineage.io/>`__
@@ -48,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
48
55
  are in ``airflow.providers.openlineage`` python package.
49
56
 
50
57
  You can find package information and changelog for the provider
51
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/>`_.
58
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/>`_.
52
59
 
53
60
  Installation
54
61
  ------------
@@ -92,4 +99,4 @@ Dependent package
92
99
  ============================================================================================================ ==============
93
100
 
94
101
  The changelog for the provider package can be found in the
95
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html>`_.
102
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html>`_.
@@ -199,3 +199,55 @@ distributed under the License is distributed on an "AS IS" BASIS,
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+ ============================================================================
204
+ APACHE AIRFLOW SUBCOMPONENTS:
205
+
206
+ The Apache Airflow project contains subcomponents with separate copyright
207
+ notices and license terms. Your use of the source code for the these
208
+ subcomponents is subject to the terms and conditions of the following
209
+ licenses.
210
+
211
+
212
+ ========================================================================
213
+ Third party Apache 2.0 licenses
214
+ ========================================================================
215
+
216
+ The following components are provided under the Apache 2.0 License.
217
+ See project link for details. The text of each license is also included
218
+ at licenses/LICENSE-[project].txt.
219
+
220
+ (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
221
+ (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
222
+ (ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
223
+ (ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
224
+
225
+ ========================================================================
226
+ MIT licenses
227
+ ========================================================================
228
+
229
+ The following components are provided under the MIT License. See project link for details.
230
+ The text of each license is also included at licenses/LICENSE-[project].txt.
231
+
232
+ (MIT License) jquery v3.5.1 (https://jquery.org/license/)
233
+ (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
234
+ (MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
235
+ (MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
236
+ (MIT License) dataTables v1.10.25 (https://datatables.net)
237
+ (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
238
+ (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
239
+ (MIT License) MomentJS v2.24.0 (http://momentjs.com/)
240
+ (MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
241
+
242
+ ========================================================================
243
+ BSD 3-Clause licenses
244
+ ========================================================================
245
+ The following components are provided under the BSD 3-Clause license. See project links for details.
246
+ The text of each license is also included at licenses/LICENSE-[project].txt.
247
+
248
+ (BSD 3 License) d3 v5.16.0 (https://d3js.org)
249
+ (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
250
+ (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
251
+
252
+ ========================================================================
253
+ See licenses/LICENSES-ui.txt for packages used in `/airflow/www`
@@ -1,4 +1,3 @@
1
- #
2
1
  # Licensed to the Apache Software Foundation (ASF) under one
3
2
  # or more contributor license agreements. See the NOTICE file
4
3
  # distributed with this work for additional information
@@ -19,8 +18,8 @@
19
18
  # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
20
19
  # OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
21
20
  #
22
- # IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
23
- # `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
21
+ # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
22
+ # `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
24
23
  #
25
24
  from __future__ import annotations
26
25
 
@@ -28,7 +27,7 @@ import packaging.version
28
27
 
29
28
  __all__ = ["__version__"]
30
29
 
31
- __version__ = "1.2.1"
30
+ __version__ = "1.3.0"
32
31
 
33
32
  try:
34
33
  from airflow import __version__ as airflow_version
@@ -39,5 +38,5 @@ if packaging.version.parse(packaging.version.parse(airflow_version).base_version
39
38
  "2.7.0"
40
39
  ):
41
40
  raise RuntimeError(
42
- f"The package `apache-airflow-providers-openlineage:{__version__}` requires Apache Airflow 2.7.0+"
41
+ f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.7.0+"
43
42
  )
@@ -18,8 +18,8 @@
18
18
  # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
19
19
  # OVERWRITTEN WHEN PREPARING PACKAGES.
20
20
  #
21
- # IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
22
- # `get_provider_info_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
21
+ # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
22
+ # `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
23
23
 
24
24
 
25
25
  def get_provider_info():
@@ -28,7 +28,8 @@ def get_provider_info():
28
28
  "name": "OpenLineage Airflow",
29
29
  "description": "`OpenLineage <https://openlineage.io/>`__\n",
30
30
  "suspended": False,
31
- "versions": ["1.2.1", "1.2.0", "1.1.1", "1.1.0", "1.0.2", "1.0.1", "1.0.0"],
31
+ "source-date-epoch": 1701983402,
32
+ "versions": ["1.3.0", "1.2.1", "1.2.0", "1.1.1", "1.1.0", "1.0.2", "1.0.1", "1.0.0"],
32
33
  "dependencies": [
33
34
  "apache-airflow>=2.7.0",
34
35
  "apache-airflow-providers-common-sql>=1.6.0",
@@ -38,6 +38,7 @@ from openlineage.client.run import Job, Run, RunEvent, RunState
38
38
  from airflow.configuration import conf
39
39
  from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
40
40
  from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
41
+ from airflow.stats import Stats
41
42
  from airflow.utils.log.logging_mixin import LoggingMixin
42
43
 
43
44
  if TYPE_CHECKING:
@@ -96,7 +97,8 @@ class OpenLineageAdapter(LoggingMixin):
96
97
  with open(path) as config_file:
97
98
  return yaml.safe_load(config_file)
98
99
 
99
- def build_dag_run_id(self, dag_id, dag_run_id):
100
+ @staticmethod
101
+ def build_dag_run_id(dag_id, dag_run_id):
100
102
  return str(uuid.uuid3(uuid.NAMESPACE_URL, f"{_DAG_NAMESPACE}.{dag_id}.{dag_run_id}"))
101
103
 
102
104
  @staticmethod
@@ -113,8 +115,10 @@ class OpenLineageAdapter(LoggingMixin):
113
115
  self._client = self.get_or_create_openlineage_client()
114
116
  redacted_event: RunEvent = self._redacter.redact(event, max_depth=20) # type: ignore[assignment]
115
117
  try:
116
- return self._client.emit(redacted_event)
118
+ with Stats.timer("ol.emit.attempts"):
119
+ return self._client.emit(redacted_event)
117
120
  except Exception as e:
121
+ Stats.incr("ol.emit.failed")
118
122
  self.log.warning("Failed to emit OpenLineage event of id %s", event.run.runId)
119
123
  self.log.debug("OpenLineage emission failure: %s", e)
120
124
 
@@ -160,17 +164,19 @@ class OpenLineageAdapter(LoggingMixin):
160
164
 
161
165
  if not run_facets:
162
166
  run_facets = {}
167
+ if task:
168
+ run_facets = {**task.run_facets, **run_facets}
163
169
  run_facets["processing_engine"] = processing_engine_version_facet # type: ignore
164
170
  event = RunEvent(
165
171
  eventType=RunState.START,
166
172
  eventTime=event_time,
167
173
  run=self._build_run(
168
- run_id,
169
- job_name,
170
- parent_job_name,
171
- parent_run_id,
172
- nominal_start_time,
173
- nominal_end_time,
174
+ run_id=run_id,
175
+ job_name=job_name,
176
+ parent_job_name=parent_job_name,
177
+ parent_run_id=parent_run_id,
178
+ nominal_start_time=nominal_start_time,
179
+ nominal_end_time=nominal_end_time,
174
180
  run_facets=run_facets,
175
181
  ),
176
182
  job=self._build_job(
@@ -186,19 +192,36 @@ class OpenLineageAdapter(LoggingMixin):
186
192
  )
187
193
  self.emit(event)
188
194
 
189
- def complete_task(self, run_id: str, job_name: str, end_time: str, task: OperatorLineage):
195
+ def complete_task(
196
+ self,
197
+ run_id: str,
198
+ job_name: str,
199
+ parent_job_name: str | None,
200
+ parent_run_id: str | None,
201
+ end_time: str,
202
+ task: OperatorLineage,
203
+ ):
190
204
  """
191
205
  Emits openlineage event of type COMPLETE.
192
206
 
193
207
  :param run_id: globally unique identifier of task in dag run
194
208
  :param job_name: globally unique identifier of task between dags
209
+ :param parent_job_name: the name of the parent job (typically the DAG,
210
+ but possibly a task group)
211
+ :param parent_run_id: identifier of job spawning this task
195
212
  :param end_time: time of task completion
196
213
  :param task: metadata container with information extracted from operator
197
214
  """
198
215
  event = RunEvent(
199
216
  eventType=RunState.COMPLETE,
200
217
  eventTime=end_time,
201
- run=self._build_run(run_id, job_name=job_name, run_facets=task.run_facets),
218
+ run=self._build_run(
219
+ run_id=run_id,
220
+ job_name=job_name,
221
+ parent_job_name=parent_job_name,
222
+ parent_run_id=parent_run_id,
223
+ run_facets=task.run_facets,
224
+ ),
202
225
  job=self._build_job(job_name, job_facets=task.job_facets),
203
226
  inputs=task.inputs,
204
227
  outputs=task.outputs,
@@ -206,20 +229,37 @@ class OpenLineageAdapter(LoggingMixin):
206
229
  )
207
230
  self.emit(event)
208
231
 
209
- def fail_task(self, run_id: str, job_name: str, end_time: str, task: OperatorLineage):
232
+ def fail_task(
233
+ self,
234
+ run_id: str,
235
+ job_name: str,
236
+ parent_job_name: str | None,
237
+ parent_run_id: str | None,
238
+ end_time: str,
239
+ task: OperatorLineage,
240
+ ):
210
241
  """
211
242
  Emits openlineage event of type FAIL.
212
243
 
213
244
  :param run_id: globally unique identifier of task in dag run
214
245
  :param job_name: globally unique identifier of task between dags
246
+ :param parent_job_name: the name of the parent job (typically the DAG,
247
+ but possibly a task group)
248
+ :param parent_run_id: identifier of job spawning this task
215
249
  :param end_time: time of task completion
216
250
  :param task: metadata container with information extracted from operator
217
251
  """
218
252
  event = RunEvent(
219
253
  eventType=RunState.FAIL,
220
254
  eventTime=end_time,
221
- run=self._build_run(run_id, job_name=job_name, run_facets=task.run_facets),
222
- job=self._build_job(job_name),
255
+ run=self._build_run(
256
+ run_id=run_id,
257
+ job_name=job_name,
258
+ parent_job_name=parent_job_name,
259
+ parent_run_id=parent_run_id,
260
+ run_facets=task.run_facets,
261
+ ),
262
+ job=self._build_job(job_name, job_facets=task.job_facets),
223
263
  inputs=task.inputs,
224
264
  outputs=task.outputs,
225
265
  producer=_PRODUCER,
@@ -28,7 +28,7 @@ class AirflowMappedTaskRunFacet(BaseFacet):
28
28
  mapIndex: int
29
29
  operatorClass: str
30
30
 
31
- _additional_skip_redact: list[str] = ["operatorClass"]
31
+ _additional_skip_redact = ["operatorClass"]
32
32
 
33
33
  @classmethod
34
34
  def from_task_instance(cls, task_instance):
@@ -63,7 +63,7 @@ class UnknownOperatorInstance(RedactMixin):
63
63
  properties: dict[str, object]
64
64
  type: str = "operator"
65
65
 
66
- _skip_redact: list[str] = ["name", "type"]
66
+ _skip_redact = ["name", "type"]
67
67
 
68
68
 
69
69
  @define(slots=False)
@@ -101,7 +101,6 @@ class OpenLineageListener:
101
101
  owners=dag.owner.split(", "),
102
102
  task=task_metadata,
103
103
  run_facets={
104
- **task_metadata.run_facets,
105
104
  **get_custom_facets(task_instance),
106
105
  **get_airflow_run_facet(dagrun, dag, task_instance, task, task_uuid),
107
106
  },
@@ -115,6 +114,7 @@ class OpenLineageListener:
115
114
 
116
115
  dagrun = task_instance.dag_run
117
116
  task = task_instance.task
117
+ dag = task.dag
118
118
 
119
119
  task_uuid = OpenLineageAdapter.build_task_instance_run_id(
120
120
  task.task_id, task_instance.execution_date, task_instance.try_number - 1
@@ -122,6 +122,8 @@ class OpenLineageListener:
122
122
 
123
123
  @print_warning(self.log)
124
124
  def on_success():
125
+ parent_run_id = OpenLineageAdapter.build_dag_run_id(dag.dag_id, dagrun.run_id)
126
+
125
127
  task_metadata = self.extractor_manager.extract_metadata(
126
128
  dagrun, task, complete=True, task_instance=task_instance
127
129
  )
@@ -131,6 +133,8 @@ class OpenLineageListener:
131
133
  self.adapter.complete_task(
132
134
  run_id=task_uuid,
133
135
  job_name=get_job_name(task),
136
+ parent_job_name=dag.dag_id,
137
+ parent_run_id=parent_run_id,
134
138
  end_time=end_date.isoformat(),
135
139
  task=task_metadata,
136
140
  )
@@ -143,13 +147,16 @@ class OpenLineageListener:
143
147
 
144
148
  dagrun = task_instance.dag_run
145
149
  task = task_instance.task
150
+ dag = task.dag
146
151
 
147
152
  task_uuid = OpenLineageAdapter.build_task_instance_run_id(
148
- task.task_id, task_instance.execution_date, task_instance.try_number - 1
153
+ task.task_id, task_instance.execution_date, task_instance.try_number
149
154
  )
150
155
 
151
156
  @print_warning(self.log)
152
157
  def on_failure():
158
+ parent_run_id = OpenLineageAdapter.build_dag_run_id(dag.dag_id, dagrun.run_id)
159
+
153
160
  task_metadata = self.extractor_manager.extract_metadata(
154
161
  dagrun, task, complete=True, task_instance=task_instance
155
162
  )
@@ -159,6 +166,8 @@ class OpenLineageListener:
159
166
  self.adapter.fail_task(
160
167
  run_id=task_uuid,
161
168
  job_name=get_job_name(task),
169
+ parent_job_name=dag.dag_id,
170
+ parent_run_id=parent_run_id,
162
171
  end_time=end_date.isoformat(),
163
172
  task=task_metadata,
164
173
  )
@@ -0,0 +1,84 @@
1
+
2
+ # Licensed to the Apache Software Foundation (ASF) under one
3
+ # or more contributor license agreements. See the NOTICE file
4
+ # distributed with this work for additional information
5
+ # regarding copyright ownership. The ASF licenses this file
6
+ # to you under the Apache License, Version 2.0 (the
7
+ # "License"); you may not use this file except in compliance
8
+ # with the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing,
13
+ # software distributed under the License is distributed on an
14
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ # KIND, either express or implied. See the License for the
16
+ # specific language governing permissions and limitations
17
+ # under the License.
18
+
19
+ # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
20
+ # OVERWRITTEN WHEN PREPARING PACKAGES.
21
+
22
+ # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
23
+ # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
24
+ #
25
+ [build-system]
26
+ requires = ["flit_core >=3.2,<4"]
27
+ build-backend = "flit_core.buildapi"
28
+
29
+ [project]
30
+ name = "apache-airflow-providers-openlineage"
31
+ version = "1.3.0"
32
+ description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
33
+ readme = "README.rst"
34
+ authors = [
35
+ {name="Apache Software Foundation", email="dev@airflow.apache.org"},
36
+ ]
37
+ maintainers = [
38
+ {name="Apache Software Foundation", email="dev@airflow.apache.org"},
39
+ ]
40
+ keywords = [ "airflow-provider", "openlineage", "airflow", "integration" ]
41
+ classifiers = [
42
+ "Development Status :: 5 - Production/Stable",
43
+ "Environment :: Console",
44
+ "Environment :: Web Environment",
45
+ "Intended Audience :: Developers",
46
+ "Intended Audience :: System Administrators",
47
+ "Framework :: Apache Airflow",
48
+ "Framework :: Apache Airflow :: Provider",
49
+ "License :: OSI Approved :: Apache Software License",
50
+ "Programming Language :: Python :: 3.8",
51
+ "Programming Language :: Python :: 3.9",
52
+ "Programming Language :: Python :: 3.10",
53
+ "Programming Language :: Python :: 3.11",
54
+ "Topic :: System :: Monitoring",
55
+ ]
56
+ requires-python = "~=3.8"
57
+ dependencies = [
58
+ "apache-airflow-providers-common-sql>=1.6.0",
59
+ "apache-airflow>=2.7.0",
60
+ "attrs>=22.2",
61
+ "openlineage-integration-common>=0.28.0",
62
+ "openlineage-python>=0.28.0",
63
+ ]
64
+
65
+ [project.urls]
66
+ "Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0"
67
+ "Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.3.0/changelog.html"
68
+ "Bug Tracker" = "https://github.com/apache/airflow/issues"
69
+ "Source Code" = "https://github.com/apache/airflow"
70
+ "Slack Chat" = "https://s.apache.org/airflow-slack"
71
+ "Twitter" = "https://twitter.com/ApacheAirflow"
72
+ "YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
73
+
74
+ [project.entry-points."apache_airflow_provider"]
75
+ provider_info = "airflow.providers.openlineage.get_provider_info:get_provider_info"
76
+ [project.entry-points."airflow.plugins"]
77
+ openlineage = "airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin"
78
+ [project.optional-dependencies]
79
+ "common.sql" = [
80
+ "apache-airflow-providers-common-sql",
81
+ ]
82
+
83
+ [tool.flit.module]
84
+ name = "airflow.providers.openlineage"
@@ -1,31 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one
3
- # or more contributor license agreements. See the NOTICE file
4
- # distributed with this work for additional information
5
- # regarding copyright ownership. The ASF licenses this file
6
- # to you under the Apache License, Version 2.0 (the
7
- # "License"); you may not use this file except in compliance
8
- # with the License. You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing,
13
- # software distributed under the License is distributed on an
14
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- # KIND, either express or implied. See the License for the
16
- # specific language governing permissions and limitations
17
- # under the License.
18
-
19
- # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
20
- # OVERWRITTEN WHEN PREPARING PACKAGES.
21
-
22
- # IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
23
- # `MANIFEST_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
24
-
25
-
26
-
27
-
28
- include NOTICE
29
- include LICENSE
30
- include CHANGELOG.rst
31
- global-exclude __pycache__ *.pyc
@@ -1,6 +0,0 @@
1
- Apache Airflow
2
- Copyright 2016-2021 The Apache Software Foundation
3
-
4
- This product includes software developed at The Apache Software
5
- Foundation (http://www.apache.org/).
6
- =======================================================================
@@ -1,130 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: apache-airflow-providers-openlineage
3
- Version: 1.2.1
4
- Summary: Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
5
- Home-page: https://airflow.apache.org/
6
- Download-URL: https://archive.apache.org/dist/airflow/providers
7
- Author: Apache Software Foundation
8
- Author-email: dev@airflow.apache.org
9
- License: Apache License 2.0
10
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/
11
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html
12
- Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
13
- Project-URL: Source Code, https://github.com/apache/airflow
14
- Project-URL: Slack Chat, https://s.apache.org/airflow-slack
15
- Project-URL: Twitter, https://twitter.com/ApacheAirflow
16
- Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
17
- Classifier: Development Status :: 5 - Production/Stable
18
- Classifier: Environment :: Console
19
- Classifier: Environment :: Web Environment
20
- Classifier: Intended Audience :: Developers
21
- Classifier: Intended Audience :: System Administrators
22
- Classifier: Framework :: Apache Airflow
23
- Classifier: Framework :: Apache Airflow :: Provider
24
- Classifier: License :: OSI Approved :: Apache Software License
25
- Classifier: Programming Language :: Python :: 3.8
26
- Classifier: Programming Language :: Python :: 3.9
27
- Classifier: Programming Language :: Python :: 3.10
28
- Classifier: Programming Language :: Python :: 3.11
29
- Classifier: Topic :: System :: Monitoring
30
- Requires-Python: ~=3.8
31
- Description-Content-Type: text/x-rst
32
- Provides-Extra: common.sql
33
- License-File: LICENSE
34
- License-File: NOTICE
35
-
36
-
37
- .. Licensed to the Apache Software Foundation (ASF) under one
38
- or more contributor license agreements. See the NOTICE file
39
- distributed with this work for additional information
40
- regarding copyright ownership. The ASF licenses this file
41
- to you under the Apache License, Version 2.0 (the
42
- "License"); you may not use this file except in compliance
43
- with the License. You may obtain a copy of the License at
44
-
45
- .. http://www.apache.org/licenses/LICENSE-2.0
46
-
47
- .. Unless required by applicable law or agreed to in writing,
48
- software distributed under the License is distributed on an
49
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
50
- KIND, either express or implied. See the License for the
51
- specific language governing permissions and limitations
52
- under the License.
53
- .. Licensed to the Apache Software Foundation (ASF) under one
54
- or more contributor license agreements. See the NOTICE file
55
- distributed with this work for additional information
56
- regarding copyright ownership. The ASF licenses this file
57
- to you under the Apache License, Version 2.0 (the
58
- "License"); you may not use this file except in compliance
59
- with the License. You may obtain a copy of the License at
60
-
61
- .. http://www.apache.org/licenses/LICENSE-2.0
62
-
63
- .. Unless required by applicable law or agreed to in writing,
64
- software distributed under the License is distributed on an
65
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
66
- KIND, either express or implied. See the License for the
67
- specific language governing permissions and limitations
68
- under the License.
69
-
70
-
71
- Package ``apache-airflow-providers-openlineage``
72
-
73
- Release: ``1.2.1``
74
-
75
-
76
- `OpenLineage <https://openlineage.io/>`__
77
-
78
-
79
- Provider package
80
- ----------------
81
-
82
- This is a provider package for ``openlineage`` provider. All classes for this provider package
83
- are in ``airflow.providers.openlineage`` python package.
84
-
85
- You can find package information and changelog for the provider
86
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/>`_.
87
-
88
- Installation
89
- ------------
90
-
91
- You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
92
- for the minimum Airflow version supported) via
93
- ``pip install apache-airflow-providers-openlineage``
94
-
95
- The package supports the following python versions: 3.8,3.9,3.10,3.11
96
-
97
- Requirements
98
- ------------
99
-
100
- ======================================= ==================
101
- PIP package Version required
102
- ======================================= ==================
103
- ``apache-airflow`` ``>=2.7.0``
104
- ``apache-airflow-providers-common-sql`` ``>=1.6.0``
105
- ``attrs`` ``>=22.2``
106
- ``openlineage-integration-common`` ``>=0.28.0``
107
- ``openlineage-python`` ``>=0.28.0``
108
- ======================================= ==================
109
-
110
- Cross provider package dependencies
111
- -----------------------------------
112
-
113
- Those are dependencies that might be needed in order to use all the features of the package.
114
- You need to install the specified provider packages in order to use them.
115
-
116
- You can install such cross-provider dependencies when installing from PyPI. For example:
117
-
118
- .. code-block:: bash
119
-
120
- pip install apache-airflow-providers-openlineage[common.sql]
121
-
122
-
123
- ============================================================================================================ ==============
124
- Dependent package Extra
125
- ============================================================================================================ ==============
126
- `apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
127
- ============================================================================================================ ==============
128
-
129
- The changelog for the provider package can be found in the
130
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html>`_.
@@ -1,31 +0,0 @@
1
- LICENSE
2
- MANIFEST.in
3
- NOTICE
4
- README.rst
5
- pyproject.toml
6
- setup.cfg
7
- setup.py
8
- airflow/providers/openlineage/__init__.py
9
- airflow/providers/openlineage/get_provider_info.py
10
- airflow/providers/openlineage/sqlparser.py
11
- airflow/providers/openlineage/extractors/__init__.py
12
- airflow/providers/openlineage/extractors/base.py
13
- airflow/providers/openlineage/extractors/bash.py
14
- airflow/providers/openlineage/extractors/manager.py
15
- airflow/providers/openlineage/extractors/python.py
16
- airflow/providers/openlineage/plugins/__init__.py
17
- airflow/providers/openlineage/plugins/adapter.py
18
- airflow/providers/openlineage/plugins/facets.py
19
- airflow/providers/openlineage/plugins/listener.py
20
- airflow/providers/openlineage/plugins/macros.py
21
- airflow/providers/openlineage/plugins/openlineage.py
22
- airflow/providers/openlineage/utils/__init__.py
23
- airflow/providers/openlineage/utils/sql.py
24
- airflow/providers/openlineage/utils/utils.py
25
- apache_airflow_providers_openlineage.egg-info/PKG-INFO
26
- apache_airflow_providers_openlineage.egg-info/SOURCES.txt
27
- apache_airflow_providers_openlineage.egg-info/dependency_links.txt
28
- apache_airflow_providers_openlineage.egg-info/entry_points.txt
29
- apache_airflow_providers_openlineage.egg-info/not-zip-safe
30
- apache_airflow_providers_openlineage.egg-info/requires.txt
31
- apache_airflow_providers_openlineage.egg-info/top_level.txt
@@ -1,5 +0,0 @@
1
- [airflow.plugins]
2
- openlineage = airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin
3
-
4
- [apache_airflow_provider]
5
- provider_info = airflow.providers.openlineage.get_provider_info:get_provider_info
@@ -1,8 +0,0 @@
1
- apache-airflow-providers-common-sql>=1.6.0
2
- apache-airflow>=2.7.0
3
- attrs>=22.2
4
- openlineage-integration-common>=0.28.0
5
- openlineage-python>=0.28.0
6
-
7
- [common.sql]
8
- apache-airflow-providers-common-sql
@@ -1,185 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
- [tool.black]
18
- line-length = 110
19
- target-version = ['py38', 'py39', 'py310', 'py311']
20
-
21
- # Editable installs are currently broken using setuptools 64.0.0 and above. The problem is tracked in
22
- # https://github.com/pypa/setuptools/issues/3548. We're also discussing how we could potentially fix
23
- # this problem on our end in issue https://github.com/apache/airflow/issues/30764. Until then we need
24
- # to use one of the following workarounds locally for editable installs:
25
- # 1) Pin setuptools <= 63.4.3 below in the [build-system] section.
26
- # 2) Include your airflow source code directory in PYTHONPATH.
27
- [build-system]
28
- requires = ['setuptools==67.2.0']
29
- build-backend = "setuptools.build_meta"
30
-
31
- [tool.ruff]
32
- target-version = "py38"
33
- typing-modules = ["airflow.typing_compat"]
34
- line-length = 110
35
- extend-exclude = [
36
- ".eggs",
37
- "airflow/_vendor/*",
38
- "airflow/providers/google/ads/_vendor/*",
39
- # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
40
- # ignore them in ruff
41
- "airflow/providers/common/sql/*/*.pyi",
42
- "airflow/migrations/versions/*.py",
43
- "tests/dags/test_imports.py",
44
- ]
45
-
46
- extend-select = [
47
- "I", # Missing required import (auto-fixable)
48
- "UP", # Pyupgrade
49
- "RUF100", # Unused noqa (auto-fixable)
50
-
51
- # We ignore more pydocstyle than we enable, so be more selective at what we enable
52
- "D101",
53
- "D106",
54
- "D2",
55
- "D3",
56
- "D400",
57
- # "D401", # Not enabled by ruff, but we don't want it
58
- "D402",
59
- "D403",
60
- "D412",
61
- "D419",
62
- "TCH", # Rules around TYPE_CHECKING blocks
63
- "TID251", # Specific modules or module members that may not be imported or accessed
64
- "TID253", # Ban certain modules from being imported at module level
65
- ]
66
- extend-ignore = [
67
- "D203",
68
- "D212",
69
- "D213",
70
- "D214",
71
- "D215",
72
- "E731",
73
- "TCH003", # Do not move imports from stdlib to TYPE_CHECKING block
74
- ]
75
-
76
- namespace-packages = ["airflow/providers"]
77
-
78
- [tool.pytest.ini_options]
79
- # * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
80
- # * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
81
- # * And we focus on use native pytest capabilities rather than adopt another frameworks.
82
- addopts = "-rasl --verbosity=2 -p no:flaky -p no:nose --asyncio-mode=strict"
83
- norecursedirs = [
84
- ".eggs",
85
- "airflow",
86
- "tests/dags_with_system_exit",
87
- "tests/test_utils",
88
- "tests/dags_corrupted",
89
- "tests/dags",
90
- "tests/system/providers/google/cloud/dataproc/resources",
91
- "tests/system/providers/google/cloud/gcs/resources",
92
- ]
93
- log_level = "INFO"
94
- filterwarnings = [
95
- "error::pytest.PytestCollectionWarning",
96
- "ignore::DeprecationWarning:flask_appbuilder.filemanager",
97
- "ignore::DeprecationWarning:flask_appbuilder.widgets",
98
- # https://github.com/dpgaspar/Flask-AppBuilder/pull/1940
99
- "ignore::DeprecationWarning:flask_sqlalchemy",
100
- # https://github.com/dpgaspar/Flask-AppBuilder/pull/1903
101
- "ignore::DeprecationWarning:apispec.utils",
102
- ]
103
- python_files = [
104
- "test_*.py",
105
- "example_*.py",
106
- ]
107
- testpaths = [
108
- "tests",
109
- ]
110
-
111
- [tool.ruff.isort]
112
- required-imports = ["from __future__ import annotations"]
113
- combine-as-imports = true
114
-
115
- [tool.ruff.per-file-ignores]
116
- "airflow/models/__init__.py" = ["F401", "TCH004"]
117
- "airflow/models/sqla_models.py" = ["F401"]
118
-
119
- # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
120
- # needed for the test to work
121
- "tests/decorators/test_python.py" = ["I002"]
122
-
123
- # The Pydantic representations of SqlAlchemy Models are not parsed well with Pydantic
124
- # when __future__.annotations is used so we need to skip them from upgrading
125
- # Pydantic also require models to be imported during execution
126
- "airflow/serialization/pydantic/*.py" = ["I002", "UP007", "TCH001"]
127
-
128
- # Ignore pydoc style from these
129
- "*.pyi" = ["D"]
130
- "scripts/*" = ["D"]
131
- "docs/*" = ["D"]
132
- "provider_packages/*" = ["D"]
133
- "*/example_dags/*" = ["D"]
134
- "chart/*" = ["D"]
135
- # In addition ignore top level imports, e.g. pandas, numpy in tests and dev
136
- "dev/*" = ["D", "TID253"]
137
- "tests/*" = ["D", "TID253"]
138
- "docker_tests/*" = ["D", "TID253"]
139
- "kubernetes_tests/*" = ["D", "TID253"]
140
-
141
- # All of the modules which have an extra license header (i.e. that we copy from another project) need to
142
- # ignore E402 -- module level import not at top level
143
- "airflow/api/auth/backend/kerberos_auth.py" = ["E402"]
144
- "airflow/security/kerberos.py" = ["E402"]
145
- "airflow/security/utils.py" = ["E402"]
146
- "tests/providers/elasticsearch/log/elasticmock/__init__.py" = ["E402"]
147
- "tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py" = ["E402"]
148
-
149
- [tool.ruff.flake8-tidy-imports.banned-api]
150
- "airflow.AirflowException".msg = "Use airflow.exceptions.AirflowException instead."
151
- "airflow.Dataset".msg = "Use airflow.datasets.Dataset instead."
152
-
153
- [tool.ruff.flake8-tidy-imports]
154
- # Ban certain modules from being imported at module level, instead requiring
155
- # that they're imported lazily (e.g., within a function definition).
156
- banned-module-level-imports = ["numpy", "pandas"]
157
-
158
- [tool.ruff.flake8-type-checking]
159
- exempt-modules = ["typing", "typing_extensions"]
160
-
161
- [tool.coverage.run]
162
- branch = true
163
- relative_files = true
164
- source = ["airflow"]
165
- omit = [
166
- "airflow/_vendor/**",
167
- "airflow/contrib/**",
168
- "airflow/example_dags/**",
169
- "airflow/migrations/**",
170
- "airflow/providers/**/example_dags/**",
171
- "airflow/www/node_modules/**",
172
- "airflow/providers/google/ads/_vendor/**",
173
- ]
174
-
175
- [tool.coverage.report]
176
- skip_empty = true
177
- exclude_also = [
178
- "def __repr__",
179
- "raise AssertionError",
180
- "raise NotImplementedError",
181
- "if __name__ == .__main__.:",
182
- "@(abc\\.)?abstractmethod",
183
- "@(typing(_extensions)?\\.)?overload",
184
- "if (typing(_extensions)?\\.)?TYPE_CHECKING:"
185
- ]
@@ -1,68 +0,0 @@
1
- [metadata]
2
- name = apache-airflow-providers-openlineage
3
- summary = Provider for Apache Airflow. Implements apache-airflow-providers-openlineage package
4
- description = Provider package apache-airflow-providers-openlineage for Apache Airflow
5
- long_description = file: README.rst
6
- long_description_content_type = text/x-rst
7
- author = Apache Software Foundation
8
- author_email = dev@airflow.apache.org
9
- url = https://airflow.apache.org/
10
- download_url = https://archive.apache.org/dist/airflow/providers
11
- license = Apache License 2.0
12
- license_files =
13
- LICENSE
14
- NOTICE
15
- classifiers =
16
- Development Status :: 5 - Production/Stable
17
- Environment :: Console
18
- Environment :: Web Environment
19
- Intended Audience :: Developers
20
- Intended Audience :: System Administrators
21
- Framework :: Apache Airflow
22
- Framework :: Apache Airflow :: Provider
23
- License :: OSI Approved :: Apache Software License
24
- Programming Language :: Python :: 3.8
25
- Programming Language :: Python :: 3.9
26
- Programming Language :: Python :: 3.10
27
- Programming Language :: Python :: 3.11
28
- Topic :: System :: Monitoring
29
- project_urls =
30
- Documentation=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/
31
- Changelog=https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.2.1/changelog.html
32
- Bug Tracker=https://github.com/apache/airflow/issues
33
- Source Code=https://github.com/apache/airflow
34
- Slack Chat=https://s.apache.org/airflow-slack
35
- Twitter=https://twitter.com/ApacheAirflow
36
- YouTube=https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
37
-
38
- [bdist_wheel]
39
- python_tag = py3
40
-
41
- [options]
42
- zip_safe = False
43
- include_package_data = True
44
- python_requires = ~=3.8
45
- packages = find:
46
- setup_requires =
47
- setuptools
48
- wheel
49
- install_requires =
50
- apache-airflow-providers-common-sql>=1.6.0
51
- apache-airflow>=2.7.0
52
- attrs>=22.2
53
- openlineage-integration-common>=0.28.0
54
- openlineage-python>=0.28.0
55
-
56
- [options.entry_points]
57
- apache_airflow_provider =
58
- provider_info=airflow.providers.openlineage.get_provider_info:get_provider_info
59
- airflow.plugins =
60
- openlineage=airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin
61
-
62
- [files]
63
- packages = airflow.providers.openlineage
64
-
65
- [egg_info]
66
- tag_build =
67
- tag_date = 0
68
-
@@ -1,49 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one
3
- # or more contributor license agreements. See the NOTICE file
4
- # distributed with this work for additional information
5
- # regarding copyright ownership. The ASF licenses this file
6
- # to you under the Apache License, Version 2.0 (the
7
- # "License"); you may not use this file except in compliance
8
- # with the License. You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing,
13
- # software distributed under the License is distributed on an
14
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- # KIND, either express or implied. See the License for the
16
- # specific language governing permissions and limitations
17
- # under the License.
18
-
19
- # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
20
- # OVERWRITTEN WHEN PREPARING PACKAGES.
21
- #
22
- # IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
23
- # `SETUP_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
24
-
25
- """Setup.py for the apache-airflow-providers-openlineage package."""
26
-
27
- from setuptools import find_namespace_packages, setup
28
-
29
- version = "1.2.1"
30
-
31
-
32
- def do_setup():
33
- """Perform the package apache-airflow-providers-openlineage setup."""
34
- setup(
35
- version=version,
36
- extras_require={"common.sql": ["apache-airflow-providers-common-sql"]},
37
- packages=find_namespace_packages(
38
- include=[
39
- "airflow.providers.openlineage",
40
- "airflow.providers.openlineage.*",
41
- "airflow.providers.openlineage_vendor",
42
- "airflow.providers.openlineage_vendor.*",
43
- ],
44
- ),
45
- )
46
-
47
-
48
- if __name__ == "__main__":
49
- do_setup()