apache-airflow-providers-openlineage 2.3.0rc1__tar.gz → 2.4.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-openlineage might be problematic. Click here for more details.
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/PKG-INFO +6 -6
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/README.rst +3 -3
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/changelog.rst +33 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/index.rst +3 -3
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/provider.yaml +3 -2
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/pyproject.toml +3 -3
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/__init__.py +1 -1
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/get_provider_info.py +1 -1
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/plugins/adapter.py +125 -55
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/plugins/listener.py +57 -13
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/plugins/macros.py +11 -1
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/plugins/openlineage.py +12 -1
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/utils/utils.py +5 -2
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_base_complex_dag.py +132 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_base_simple_dag.py +62 -0
- apache_airflow_providers_openlineage-2.3.0rc1/tests/system/openlineage/example_openlineage_defer.py → apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_defer_simple_dag.py +25 -16
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_mapped_simple_dag.py +85 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_asset_or_time_dag.py +78 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_cron_dag.py +55 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_list_complex_assets_dag.py +67 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_list_multiple_assets_dag.py +56 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_list_single_asset_dag.py +56 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_multiple_assets_dag.py +59 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_single_asset_dag.py +56 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_schedule_timetable_dag.py +65 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_task_groups_dag.py +62 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_trigger_dag.py +75 -0
- apache_airflow_providers_openlineage-2.3.0rc1/tests/system/openlineage/example_openlineage.py → apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/example_openlineage_versioned_dag.py +17 -7
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/__init__.py +75 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_base_complex_dag__af2.json +1884 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_base_complex_dag__af3.json +1840 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_base_simple_dag__af2.json +353 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_base_simple_dag__af3.json +345 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_defer_simple_dag.json +32 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_mapped_simple_dag.json +38 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_asset_or_time_dag__af2.json +188 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_asset_or_time_dag__af3.json +215 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_cron_dag__af2.json +59 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_cron_dag__af3.json +59 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_list_complex_assets_dag__af2.json +1 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_list_complex_assets_dag__af3.json +206 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_list_multiple_assets_dag__af2.json +101 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_list_multiple_assets_dag__af3.json +116 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_list_single_asset_dag__af2.json +89 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_list_single_asset_dag__af3.json +95 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_multiple_assets_dag__af2.json +164 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_multiple_assets_dag__af3.json +188 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_single_asset_dag__af2.json +74 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_single_asset_dag__af3.json +80 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_timetable_dag__af2.json +71 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_schedule_timetable_dag__af3.json +71 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_task_groups_dag.json +94 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_trigger_dag.json +255 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_versioned_dag__af2.json +1 -0
- apache_airflow_providers_openlineage-2.4.0rc1/tests/system/openlineage/expected_events/openlineage_versioned_dag__af3.json +53 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/system/openlineage/operator.py +4 -1
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_adapter.py +184 -9
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_listener.py +281 -27
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/utils/test_utils.py +4 -0
- apache_airflow_providers_openlineage-2.3.0rc1/tests/system/openlineage/example_openlineage.json +0 -169
- apache_airflow_providers_openlineage-2.3.0rc1/tests/system/openlineage/example_openlineage_defer.json +0 -38
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/commits.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/conf.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/configurations-ref.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/guides/developer.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/guides/structure.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/guides/user.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/integration-logos/openlineage.svg +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/macros.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/security.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/docs/supported_classes.rst +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/LICENSE +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/conf.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/extractors/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/extractors/base.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/extractors/bash.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/extractors/manager.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/extractors/python.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/facets/AirflowDagRunFacet.json +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/facets/AirflowDebugRunFacet.json +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/facets/AirflowJobFacet.json +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/facets/AirflowRunFacet.json +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/facets/AirflowStateRunFacet.json +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/facets/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/plugins/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/plugins/facets.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/sqlparser.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/utils/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/utils/selective_enable.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/utils/spark.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/utils/sql.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/src/airflow/providers/openlineage/version_compat.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/conftest.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/integration/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/integration/openlineage/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/integration/openlineage/operators/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/system/openlineage/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/system/openlineage/conftest.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/system/openlineage/transport/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/system/openlineage/transport/variable.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/dags/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/dags/test_openlineage_execution.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/extractors/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/extractors/test_base.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/extractors/test_bash.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/extractors/test_manager.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/extractors/test_python.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/log_config.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/openlineage_configs/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/openlineage_configs/http.yaml +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_execution.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_facets.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_macros.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_openlineage.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/plugins/test_utils.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/test_conf.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/test_sqlparser.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/utils/__init__.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/utils/custom_facet_fixture.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/utils/test_selective_enable.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/utils/test_spark.py +0 -0
- {apache_airflow_providers_openlineage-2.3.0rc1 → apache_airflow_providers_openlineage-2.4.0rc1}/tests/unit/openlineage/utils/test_sql.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-openlineage
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0rc1
|
|
4
4
|
Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,openlineage,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -27,8 +27,8 @@ Requires-Dist: attrs>=22.2
|
|
|
27
27
|
Requires-Dist: openlineage-integration-common>=1.31.0
|
|
28
28
|
Requires-Dist: openlineage-python>=1.31.0
|
|
29
29
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
30
|
-
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
31
|
-
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
30
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.4.0/changelog.html
|
|
31
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.4.0
|
|
32
32
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
33
33
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
34
34
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -59,7 +59,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
|
59
59
|
|
|
60
60
|
Package ``apache-airflow-providers-openlineage``
|
|
61
61
|
|
|
62
|
-
Release: ``2.
|
|
62
|
+
Release: ``2.4.0``
|
|
63
63
|
|
|
64
64
|
|
|
65
65
|
`OpenLineage <https://openlineage.io/>`__
|
|
@@ -72,7 +72,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
|
|
|
72
72
|
are in ``airflow.providers.openlineage`` python package.
|
|
73
73
|
|
|
74
74
|
You can find package information and changelog for the provider
|
|
75
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
75
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.4.0/>`_.
|
|
76
76
|
|
|
77
77
|
Installation
|
|
78
78
|
------------
|
|
@@ -118,5 +118,5 @@ Dependent package
|
|
|
118
118
|
================================================================================================================== =================
|
|
119
119
|
|
|
120
120
|
The changelog for the provider package can be found in the
|
|
121
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
121
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.4.0/changelog.html>`_.
|
|
122
122
|
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-openlineage``
|
|
25
25
|
|
|
26
|
-
Release: ``2.
|
|
26
|
+
Release: ``2.4.0``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`OpenLineage <https://openlineage.io/>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
|
|
|
36
36
|
are in ``airflow.providers.openlineage`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.4.0/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -82,4 +82,4 @@ Dependent package
|
|
|
82
82
|
================================================================================================================== =================
|
|
83
83
|
|
|
84
84
|
The changelog for the provider package can be found in the
|
|
85
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
85
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.4.0/changelog.html>`_.
|
|
@@ -26,6 +26,39 @@
|
|
|
26
26
|
Changelog
|
|
27
27
|
---------
|
|
28
28
|
|
|
29
|
+
2.4.0
|
|
30
|
+
.....
|
|
31
|
+
|
|
32
|
+
Features
|
|
33
|
+
~~~~~~~~
|
|
34
|
+
|
|
35
|
+
* ``feat: Add NominalTimeRunFacet to all OL events (#51404)``
|
|
36
|
+
* ``feat: Add TagsJobFacet to DAGRun OpenLineage events (#51303)``
|
|
37
|
+
* ``feat: Add Airflow-specific OL system tests validation and more tests (#51084)``
|
|
38
|
+
* ``feat: merge TimeDeltaSensorAsync to TimeDeltaSensor (#51133)``
|
|
39
|
+
* ``expose OpenLineage's lineage_root_* macros in plugin (#50532)``
|
|
40
|
+
|
|
41
|
+
Bug Fixes
|
|
42
|
+
~~~~~~~~~
|
|
43
|
+
|
|
44
|
+
* ``fix: Use task owner for TASK level Ownership facet (#51305)``
|
|
45
|
+
* ``Fix openlineage doc error (#51356)``
|
|
46
|
+
* ``Fix OpenLineage macro _get_logical_date (#51210)``
|
|
47
|
+
* ``Fix failing static checks (#51197)``
|
|
48
|
+
* ``Fix simple grammar mistakes in doc (#51138)``
|
|
49
|
+
* ``Fixes issue RuntimeTaskInstance does not contain log_url | added during taskrunner startup (#50376)``
|
|
50
|
+
|
|
51
|
+
Misc
|
|
52
|
+
~~~~
|
|
53
|
+
|
|
54
|
+
* ``nit: task-level facets should not overwrite integration-level facets (#51690)``
|
|
55
|
+
* ``Make duration in 'List Dag Run' page sortable (#51495)``
|
|
56
|
+
* ``import MappedOperator from airflow.sdk.definitions.mappedoperator (#51492)``
|
|
57
|
+
|
|
58
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
59
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
60
|
+
* ``tests: Adjust OL system test after ownership facet changes (#51394)``
|
|
61
|
+
|
|
29
62
|
2.3.0
|
|
30
63
|
.....
|
|
31
64
|
|
|
@@ -81,7 +81,7 @@ apache-airflow-providers-openlineage package
|
|
|
81
81
|
`OpenLineage <https://openlineage.io/>`__
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
Release: 2.
|
|
84
|
+
Release: 2.4.0
|
|
85
85
|
|
|
86
86
|
Provider package
|
|
87
87
|
----------------
|
|
@@ -138,5 +138,5 @@ Downloading official packages
|
|
|
138
138
|
You can download officially released packages and verify their checksums and signatures from the
|
|
139
139
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
140
140
|
|
|
141
|
-
* `The apache-airflow-providers-openlineage 2.
|
|
142
|
-
* `The apache-airflow-providers-openlineage 2.
|
|
141
|
+
* `The apache-airflow-providers-openlineage 2.4.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_openlineage-2.4.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_openlineage-2.4.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_openlineage-2.4.0.tar.gz.sha512>`__)
|
|
142
|
+
* `The apache-airflow-providers-openlineage 2.4.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_openlineage-2.4.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_openlineage-2.4.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_openlineage-2.4.0-py3-none-any.whl.sha512>`__)
|
|
@@ -22,12 +22,13 @@ description: |
|
|
|
22
22
|
`OpenLineage <https://openlineage.io/>`__
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
|
-
source-date-epoch:
|
|
25
|
+
source-date-epoch: 1749896859
|
|
26
26
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
27
27
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
28
28
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
29
29
|
# to be done in the same PR
|
|
30
30
|
versions:
|
|
31
|
+
- 2.4.0
|
|
31
32
|
- 2.3.0
|
|
32
33
|
- 2.2.0
|
|
33
34
|
- 2.1.3
|
|
@@ -74,7 +75,7 @@ config:
|
|
|
74
75
|
openlineage:
|
|
75
76
|
description: |
|
|
76
77
|
This section applies settings for OpenLineage integration.
|
|
77
|
-
More about configuration and
|
|
78
|
+
More about configuration and its precedence can be found in the `user's guide
|
|
78
79
|
<https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html#transport-setup>`_.
|
|
79
80
|
|
|
80
81
|
options:
|
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-openlineage"
|
|
28
|
-
version = "2.
|
|
28
|
+
version = "2.4.0rc1"
|
|
29
29
|
description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
authors = [
|
|
@@ -105,8 +105,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
105
105
|
apache-airflow-providers-standard = {workspace = true}
|
|
106
106
|
|
|
107
107
|
[project.urls]
|
|
108
|
-
"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
109
|
-
"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.
|
|
108
|
+
"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.4.0"
|
|
109
|
+
"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.4.0/changelog.html"
|
|
110
110
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
111
111
|
"Source Code" = "https://github.com/apache/airflow"
|
|
112
112
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "2.
|
|
32
|
+
__version__ = "2.4.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -42,7 +42,7 @@ def get_provider_info():
|
|
|
42
42
|
],
|
|
43
43
|
"config": {
|
|
44
44
|
"openlineage": {
|
|
45
|
-
"description": "This section applies settings for OpenLineage integration.\nMore about configuration and
|
|
45
|
+
"description": "This section applies settings for OpenLineage integration.\nMore about configuration and its precedence can be found in the `user's guide\n<https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html#transport-setup>`_.\n",
|
|
46
46
|
"options": {
|
|
47
47
|
"disabled": {
|
|
48
48
|
"description": "Disable sending events without uninstalling the OpenLineage Provider by setting this to true.\n",
|
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
import os
|
|
20
20
|
import traceback
|
|
21
21
|
from contextlib import ExitStack
|
|
22
|
-
from typing import TYPE_CHECKING
|
|
22
|
+
from typing import TYPE_CHECKING, Literal
|
|
23
23
|
|
|
24
24
|
import yaml
|
|
25
25
|
from openlineage.client import OpenLineageClient, set_producer
|
|
@@ -32,7 +32,7 @@ from openlineage.client.facet_v2 import (
|
|
|
32
32
|
job_type_job,
|
|
33
33
|
nominal_time_run,
|
|
34
34
|
ownership_job,
|
|
35
|
-
|
|
35
|
+
tags_job,
|
|
36
36
|
)
|
|
37
37
|
from openlineage.client.uuid import generate_static_uuid
|
|
38
38
|
|
|
@@ -63,11 +63,8 @@ _PRODUCER = f"https://github.com/apache/airflow/tree/providers-openlineage/{OPEN
|
|
|
63
63
|
|
|
64
64
|
set_producer(_PRODUCER)
|
|
65
65
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
# otherwise the `JobTypeJobFacet._producer` will be set with the default value
|
|
69
|
-
_JOB_TYPE_DAG = job_type_job.JobTypeJobFacet(jobType="DAG", integration="AIRFLOW", processingType="BATCH")
|
|
70
|
-
_JOB_TYPE_TASK = job_type_job.JobTypeJobFacet(jobType="TASK", integration="AIRFLOW", processingType="BATCH")
|
|
66
|
+
_JOB_TYPE_DAG: Literal["DAG"] = "DAG"
|
|
67
|
+
_JOB_TYPE_TASK: Literal["TASK"] = "TASK"
|
|
71
68
|
|
|
72
69
|
|
|
73
70
|
class OpenLineageAdapter(LoggingMixin):
|
|
@@ -187,10 +184,10 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
187
184
|
job_name: str,
|
|
188
185
|
job_description: str,
|
|
189
186
|
event_time: str,
|
|
190
|
-
code_location: str | None,
|
|
191
187
|
nominal_start_time: str | None,
|
|
192
188
|
nominal_end_time: str | None,
|
|
193
|
-
owners: list[str],
|
|
189
|
+
owners: list[str] | None,
|
|
190
|
+
tags: list[str] | None,
|
|
194
191
|
task: OperatorLineage | None,
|
|
195
192
|
run_facets: dict[str, RunFacet] | None = None,
|
|
196
193
|
) -> RunEvent:
|
|
@@ -201,17 +198,16 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
201
198
|
:param job_name: globally unique identifier of task in dag
|
|
202
199
|
:param job_description: user provided description of job
|
|
203
200
|
:param event_time:
|
|
204
|
-
:param code_location: file path or URL of DAG file
|
|
205
201
|
:param nominal_start_time: scheduled time of dag run
|
|
206
202
|
:param nominal_end_time: following schedule of dag run
|
|
207
|
-
:param owners: list of owners
|
|
203
|
+
:param owners: list of owners
|
|
204
|
+
:param tags: list of tags
|
|
208
205
|
:param task: metadata container with information extracted from operator
|
|
209
206
|
:param run_facets: custom run facets
|
|
210
207
|
"""
|
|
211
208
|
run_facets = run_facets or {}
|
|
212
209
|
if task:
|
|
213
210
|
run_facets = {**task.run_facets, **run_facets}
|
|
214
|
-
run_facets = {**run_facets, **get_processing_engine_facet()} # type: ignore
|
|
215
211
|
event = RunEvent(
|
|
216
212
|
eventType=RunState.START,
|
|
217
213
|
eventTime=event_time,
|
|
@@ -225,8 +221,8 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
225
221
|
job_name=job_name,
|
|
226
222
|
job_type=_JOB_TYPE_TASK,
|
|
227
223
|
job_description=job_description,
|
|
228
|
-
|
|
229
|
-
|
|
224
|
+
job_owners=owners,
|
|
225
|
+
job_tags=tags,
|
|
230
226
|
job_facets=task.job_facets if task else None,
|
|
231
227
|
),
|
|
232
228
|
inputs=task.inputs if task else [],
|
|
@@ -241,6 +237,10 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
241
237
|
job_name: str,
|
|
242
238
|
end_time: str,
|
|
243
239
|
task: OperatorLineage,
|
|
240
|
+
nominal_start_time: str | None,
|
|
241
|
+
nominal_end_time: str | None,
|
|
242
|
+
owners: list[str] | None,
|
|
243
|
+
tags: list[str] | None,
|
|
244
244
|
run_facets: dict[str, RunFacet] | None = None,
|
|
245
245
|
) -> RunEvent:
|
|
246
246
|
"""
|
|
@@ -249,21 +249,32 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
249
249
|
:param run_id: globally unique identifier of task in dag run
|
|
250
250
|
:param job_name: globally unique identifier of task between dags
|
|
251
251
|
:param end_time: time of task completion
|
|
252
|
+
:param tags: list of tags
|
|
253
|
+
:param nominal_start_time: scheduled time of dag run
|
|
254
|
+
:param nominal_end_time: following schedule of dag run
|
|
252
255
|
:param task: metadata container with information extracted from operator
|
|
256
|
+
:param owners: list of owners
|
|
253
257
|
:param run_facets: additional run facets
|
|
254
258
|
"""
|
|
255
259
|
run_facets = run_facets or {}
|
|
256
260
|
if task:
|
|
257
261
|
run_facets = {**task.run_facets, **run_facets}
|
|
258
|
-
run_facets = {**run_facets, **get_processing_engine_facet()} # type: ignore
|
|
259
262
|
event = RunEvent(
|
|
260
263
|
eventType=RunState.COMPLETE,
|
|
261
264
|
eventTime=end_time,
|
|
262
265
|
run=self._build_run(
|
|
263
266
|
run_id=run_id,
|
|
267
|
+
nominal_start_time=nominal_start_time,
|
|
268
|
+
nominal_end_time=nominal_end_time,
|
|
264
269
|
run_facets=run_facets,
|
|
265
270
|
),
|
|
266
|
-
job=self._build_job(
|
|
271
|
+
job=self._build_job(
|
|
272
|
+
job_name,
|
|
273
|
+
job_type=_JOB_TYPE_TASK,
|
|
274
|
+
job_facets=task.job_facets,
|
|
275
|
+
job_owners=owners,
|
|
276
|
+
job_tags=tags,
|
|
277
|
+
),
|
|
267
278
|
inputs=task.inputs,
|
|
268
279
|
outputs=task.outputs,
|
|
269
280
|
producer=_PRODUCER,
|
|
@@ -276,6 +287,10 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
276
287
|
job_name: str,
|
|
277
288
|
end_time: str,
|
|
278
289
|
task: OperatorLineage,
|
|
290
|
+
nominal_start_time: str | None,
|
|
291
|
+
nominal_end_time: str | None,
|
|
292
|
+
owners: list[str] | None,
|
|
293
|
+
tags: list[str] | None,
|
|
279
294
|
error: str | BaseException | None = None,
|
|
280
295
|
run_facets: dict[str, RunFacet] | None = None,
|
|
281
296
|
) -> RunEvent:
|
|
@@ -287,13 +302,16 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
287
302
|
:param end_time: time of task completion
|
|
288
303
|
:param task: metadata container with information extracted from operator
|
|
289
304
|
:param run_facets: custom run facets
|
|
305
|
+
:param tags: list of tags
|
|
306
|
+
:param nominal_start_time: scheduled time of dag run
|
|
307
|
+
:param nominal_end_time: following schedule of dag run
|
|
308
|
+
:param owners: list of owners
|
|
290
309
|
:param error: error
|
|
291
310
|
:param run_facets: additional run facets
|
|
292
311
|
"""
|
|
293
312
|
run_facets = run_facets or {}
|
|
294
313
|
if task:
|
|
295
314
|
run_facets = {**task.run_facets, **run_facets}
|
|
296
|
-
run_facets = {**run_facets, **get_processing_engine_facet()} # type: ignore
|
|
297
315
|
|
|
298
316
|
if error:
|
|
299
317
|
stack_trace = None
|
|
@@ -310,9 +328,17 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
310
328
|
eventTime=end_time,
|
|
311
329
|
run=self._build_run(
|
|
312
330
|
run_id=run_id,
|
|
331
|
+
nominal_start_time=nominal_start_time,
|
|
332
|
+
nominal_end_time=nominal_end_time,
|
|
313
333
|
run_facets=run_facets,
|
|
314
334
|
),
|
|
315
|
-
job=self._build_job(
|
|
335
|
+
job=self._build_job(
|
|
336
|
+
job_name,
|
|
337
|
+
job_type=_JOB_TYPE_TASK,
|
|
338
|
+
job_facets=task.job_facets,
|
|
339
|
+
job_owners=owners,
|
|
340
|
+
job_tags=tags,
|
|
341
|
+
),
|
|
316
342
|
inputs=task.inputs,
|
|
317
343
|
outputs=task.outputs,
|
|
318
344
|
producer=_PRODUCER,
|
|
@@ -324,9 +350,10 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
324
350
|
dag_id: str,
|
|
325
351
|
logical_date: datetime,
|
|
326
352
|
start_date: datetime,
|
|
327
|
-
nominal_start_time: str,
|
|
328
|
-
nominal_end_time: str,
|
|
329
|
-
owners: list[str],
|
|
353
|
+
nominal_start_time: str | None,
|
|
354
|
+
nominal_end_time: str | None,
|
|
355
|
+
owners: list[str] | None,
|
|
356
|
+
tags: list[str],
|
|
330
357
|
run_facets: dict[str, RunFacet],
|
|
331
358
|
clear_number: int,
|
|
332
359
|
description: str | None = None,
|
|
@@ -340,8 +367,9 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
340
367
|
job_name=dag_id,
|
|
341
368
|
job_type=_JOB_TYPE_DAG,
|
|
342
369
|
job_description=description,
|
|
343
|
-
|
|
370
|
+
job_owners=owners,
|
|
344
371
|
job_facets=job_facets,
|
|
372
|
+
job_tags=tags,
|
|
345
373
|
),
|
|
346
374
|
run=self._build_run(
|
|
347
375
|
run_id=self.build_dag_run_id(
|
|
@@ -349,7 +377,7 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
349
377
|
),
|
|
350
378
|
nominal_start_time=nominal_start_time,
|
|
351
379
|
nominal_end_time=nominal_end_time,
|
|
352
|
-
run_facets={**run_facets, **get_airflow_debug_facet()
|
|
380
|
+
run_facets={**run_facets, **get_airflow_debug_facet()},
|
|
353
381
|
),
|
|
354
382
|
inputs=[],
|
|
355
383
|
outputs=[],
|
|
@@ -368,24 +396,34 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
368
396
|
run_id: str,
|
|
369
397
|
end_date: datetime,
|
|
370
398
|
logical_date: datetime,
|
|
399
|
+
nominal_start_time: str | None,
|
|
400
|
+
nominal_end_time: str | None,
|
|
401
|
+
tags: list[str] | None,
|
|
371
402
|
clear_number: int,
|
|
372
403
|
dag_run_state: DagRunState,
|
|
373
404
|
task_ids: list[str],
|
|
405
|
+
owners: list[str] | None,
|
|
374
406
|
run_facets: dict[str, RunFacet],
|
|
375
407
|
):
|
|
376
408
|
try:
|
|
377
409
|
event = RunEvent(
|
|
378
410
|
eventType=RunState.COMPLETE,
|
|
379
411
|
eventTime=end_date.isoformat(),
|
|
380
|
-
job=self._build_job(
|
|
381
|
-
|
|
382
|
-
|
|
412
|
+
job=self._build_job(
|
|
413
|
+
job_name=dag_id,
|
|
414
|
+
job_type=_JOB_TYPE_DAG,
|
|
415
|
+
job_owners=owners,
|
|
416
|
+
job_tags=tags,
|
|
417
|
+
),
|
|
418
|
+
run=self._build_run(
|
|
419
|
+
run_id=self.build_dag_run_id(
|
|
383
420
|
dag_id=dag_id, logical_date=logical_date, clear_number=clear_number
|
|
384
421
|
),
|
|
385
|
-
|
|
422
|
+
nominal_start_time=nominal_start_time,
|
|
423
|
+
nominal_end_time=nominal_end_time,
|
|
424
|
+
run_facets={
|
|
386
425
|
**get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
|
|
387
426
|
**get_airflow_debug_facet(),
|
|
388
|
-
**get_processing_engine_facet(),
|
|
389
427
|
**run_facets,
|
|
390
428
|
},
|
|
391
429
|
),
|
|
@@ -406,9 +444,13 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
406
444
|
run_id: str,
|
|
407
445
|
end_date: datetime,
|
|
408
446
|
logical_date: datetime,
|
|
447
|
+
nominal_start_time: str | None,
|
|
448
|
+
nominal_end_time: str | None,
|
|
449
|
+
tags: list[str] | None,
|
|
409
450
|
clear_number: int,
|
|
410
451
|
dag_run_state: DagRunState,
|
|
411
452
|
task_ids: list[str],
|
|
453
|
+
owners: list[str] | None,
|
|
412
454
|
msg: str,
|
|
413
455
|
run_facets: dict[str, RunFacet],
|
|
414
456
|
):
|
|
@@ -416,20 +458,24 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
416
458
|
event = RunEvent(
|
|
417
459
|
eventType=RunState.FAIL,
|
|
418
460
|
eventTime=end_date.isoformat(),
|
|
419
|
-
job=self._build_job(
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
461
|
+
job=self._build_job(
|
|
462
|
+
job_name=dag_id,
|
|
463
|
+
job_type=_JOB_TYPE_DAG,
|
|
464
|
+
job_owners=owners,
|
|
465
|
+
job_tags=tags,
|
|
466
|
+
),
|
|
467
|
+
run=self._build_run(
|
|
468
|
+
run_id=self.build_dag_run_id(
|
|
469
|
+
dag_id=dag_id, logical_date=logical_date, clear_number=clear_number
|
|
425
470
|
),
|
|
426
|
-
|
|
471
|
+
nominal_start_time=nominal_start_time,
|
|
472
|
+
nominal_end_time=nominal_end_time,
|
|
473
|
+
run_facets={
|
|
427
474
|
"errorMessage": error_message_run.ErrorMessageRunFacet(
|
|
428
475
|
message=msg, programmingLanguage="python"
|
|
429
476
|
),
|
|
430
477
|
**get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
|
|
431
478
|
**get_airflow_debug_facet(),
|
|
432
|
-
**get_processing_engine_facet(),
|
|
433
479
|
**run_facets,
|
|
434
480
|
},
|
|
435
481
|
),
|
|
@@ -452,49 +498,73 @@ class OpenLineageAdapter(LoggingMixin):
|
|
|
452
498
|
run_facets: dict[str, RunFacet] | None = None,
|
|
453
499
|
) -> Run:
|
|
454
500
|
facets: dict[str, RunFacet] = {}
|
|
501
|
+
if run_facets:
|
|
502
|
+
facets.update(run_facets)
|
|
455
503
|
if nominal_start_time:
|
|
456
504
|
facets.update(
|
|
457
|
-
{
|
|
505
|
+
{
|
|
506
|
+
"nominalTime": nominal_time_run.NominalTimeRunFacet(
|
|
507
|
+
nominalStartTime=nominal_start_time,
|
|
508
|
+
nominalEndTime=nominal_end_time,
|
|
509
|
+
producer=_PRODUCER,
|
|
510
|
+
)
|
|
511
|
+
}
|
|
458
512
|
)
|
|
459
|
-
|
|
460
|
-
facets.update(run_facets)
|
|
513
|
+
facets.update(get_processing_engine_facet())
|
|
461
514
|
|
|
462
515
|
return Run(run_id, facets)
|
|
463
516
|
|
|
464
517
|
@staticmethod
|
|
465
518
|
def _build_job(
|
|
466
519
|
job_name: str,
|
|
467
|
-
job_type:
|
|
520
|
+
job_type: Literal["DAG", "TASK"],
|
|
468
521
|
job_description: str | None = None,
|
|
469
|
-
|
|
470
|
-
|
|
522
|
+
job_owners: list[str] | None = None,
|
|
523
|
+
job_tags: list[str] | None = None,
|
|
471
524
|
job_facets: dict[str, JobFacet] | None = None,
|
|
472
525
|
):
|
|
473
526
|
facets: dict[str, JobFacet] = {}
|
|
474
|
-
|
|
527
|
+
if job_facets:
|
|
528
|
+
facets.update(job_facets)
|
|
475
529
|
if job_description:
|
|
476
530
|
facets.update(
|
|
477
|
-
{
|
|
531
|
+
{
|
|
532
|
+
"documentation": documentation_job.DocumentationJobFacet(
|
|
533
|
+
description=job_description, producer=_PRODUCER
|
|
534
|
+
)
|
|
535
|
+
}
|
|
478
536
|
)
|
|
479
|
-
if
|
|
537
|
+
if job_owners:
|
|
480
538
|
facets.update(
|
|
481
539
|
{
|
|
482
|
-
"
|
|
483
|
-
|
|
540
|
+
"ownership": ownership_job.OwnershipJobFacet(
|
|
541
|
+
owners=[ownership_job.Owner(name=owner) for owner in sorted(job_owners)],
|
|
542
|
+
producer=_PRODUCER,
|
|
484
543
|
)
|
|
485
544
|
}
|
|
486
545
|
)
|
|
487
|
-
if
|
|
546
|
+
if job_tags:
|
|
488
547
|
facets.update(
|
|
489
548
|
{
|
|
490
|
-
"
|
|
491
|
-
|
|
549
|
+
"tags": tags_job.TagsJobFacet(
|
|
550
|
+
tags=[
|
|
551
|
+
tags_job.TagsJobFacetFields(
|
|
552
|
+
key=tag,
|
|
553
|
+
value=tag,
|
|
554
|
+
source="AIRFLOW",
|
|
555
|
+
)
|
|
556
|
+
for tag in sorted(job_tags)
|
|
557
|
+
],
|
|
558
|
+
producer=_PRODUCER,
|
|
492
559
|
)
|
|
493
560
|
}
|
|
494
561
|
)
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
562
|
+
facets.update(
|
|
563
|
+
{
|
|
564
|
+
"jobType": job_type_job.JobTypeJobFacet(
|
|
565
|
+
jobType=job_type, integration="AIRFLOW", processingType="BATCH", producer=_PRODUCER
|
|
566
|
+
)
|
|
567
|
+
}
|
|
568
|
+
)
|
|
499
569
|
|
|
500
|
-
return Job(conf.namespace(), job_name, facets)
|
|
570
|
+
return Job(namespace=conf.namespace(), name=job_name, facets=facets)
|