apache-airflow-providers-databricks 7.7.3rc1__tar.gz → 7.7.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-databricks might be problematic. Click here for more details.
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/PKG-INFO +12 -13
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/README.rst +4 -5
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/changelog.rst +23 -4
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/index.rst +4 -5
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/provider.yaml +2 -1
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/pyproject.toml +9 -9
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/__init__.py +1 -1
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/operators/databricks.py +2 -8
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/operators/databricks_repos.py +2 -6
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/operators/databricks_sql.py +2 -2
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/operators/databricks_workflow.py +3 -7
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/plugins/databricks_workflow.py +94 -103
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/sensors/databricks.py +2 -7
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/sensors/databricks_partition.py +2 -11
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/sensors/databricks_sql.py +2 -11
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/version_compat.py +0 -12
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/hooks/test_databricks.py +1 -1
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/hooks/test_databricks_sql.py +1 -2
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/operators/test_databricks.py +9 -12
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/sensors/test_databricks.py +0 -2
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/commits.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/conf.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/connections/databricks.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/img/databricks_workflow_task_group_airflow_graph_view.png +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/img/workflow_plugin_launch_task.png +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/img/workflow_plugin_single_task.png +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/img/workflow_run_databricks_graph_view.png +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/integration-logos/Databricks.png +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/copy_into.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/index.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/jobs_create.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/notebook.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/repos_create.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/repos_delete.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/repos_update.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/run_now.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/sql.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/sql_statements.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/submit_run.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/task.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/operators/workflow.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/plugins/index.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/plugins/workflow.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/docs/security.rst +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/LICENSE +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/exceptions.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/get_provider_info.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/hooks/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/hooks/databricks.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/hooks/databricks_base.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/hooks/databricks_sql.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/operators/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/plugins/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/sensors/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/triggers/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/triggers/databricks.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/utils/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/utils/databricks.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/utils/mixins.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/src/airflow/providers/databricks/utils/openlineage.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/conftest.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/databricks/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/databricks/example_databricks.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/databricks/example_databricks_repos.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/databricks/example_databricks_sensors.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/databricks/example_databricks_sql.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/system/databricks/example_databricks_workflow.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/hooks/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/hooks/test_databricks_azure_workload_identity.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/hooks/test_databricks_azure_workload_identity_async.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/hooks/test_databricks_base.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/operators/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/operators/test_databricks_copy.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/operators/test_databricks_repos.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/operators/test_databricks_sql.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/operators/test_databricks_workflow.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/plugins/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/plugins/test_databricks_workflow.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/sensors/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/sensors/test_databricks_partition.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/sensors/test_databricks_sql.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/test_exceptions.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/triggers/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/triggers/test_databricks.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/utils/__init__.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/utils/test_databricks.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/utils/test_mixins.py +0 -0
- {apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/tests/unit/databricks/utils/test_openlineage.py +0 -0
{apache_airflow_providers_databricks-7.7.3rc1 → apache_airflow_providers_databricks-7.7.4}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-databricks
|
|
3
|
-
Version: 7.7.
|
|
3
|
+
Version: 7.7.4
|
|
4
4
|
Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,databricks,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,9 +20,9 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.
|
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.27.
|
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.8.0
|
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
|
|
26
26
|
Requires-Dist: requests>=2.32.0,<3
|
|
27
27
|
Requires-Dist: databricks-sql-connector>=4.0.0
|
|
28
28
|
Requires-Dist: databricks-sqlalchemy>=1.0.2
|
|
@@ -33,13 +33,13 @@ Requires-Dist: pandas>=2.2.3; python_version >="3.13"
|
|
|
33
33
|
Requires-Dist: pyarrow>=16.1.0; python_version < '3.13'
|
|
34
34
|
Requires-Dist: pyarrow>=18.0.0; python_version >= '3.13'
|
|
35
35
|
Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
|
|
36
|
-
Requires-Dist: apache-airflow-providers-fab>=2.2.
|
|
37
|
-
Requires-Dist: apache-airflow-providers-openlineage>=2.3.
|
|
36
|
+
Requires-Dist: apache-airflow-providers-fab>=2.2.0 ; extra == "fab" and ( python_version < '3.13')
|
|
37
|
+
Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
|
|
38
38
|
Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
|
|
39
39
|
Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
|
|
40
40
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
41
|
-
Project-URL: Changelog, https://airflow.
|
|
42
|
-
Project-URL: Documentation, https://airflow.
|
|
41
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html
|
|
42
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4
|
|
43
43
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
44
44
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
45
45
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -75,7 +75,7 @@ Provides-Extra: standard
|
|
|
75
75
|
|
|
76
76
|
Package ``apache-airflow-providers-databricks``
|
|
77
77
|
|
|
78
|
-
Release: ``7.7.
|
|
78
|
+
Release: ``7.7.4``
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
`Databricks <https://databricks.com/>`__
|
|
@@ -88,7 +88,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
|
|
|
88
88
|
are in ``airflow.providers.databricks`` python package.
|
|
89
89
|
|
|
90
90
|
You can find package information and changelog for the provider
|
|
91
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.
|
|
91
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/>`_.
|
|
92
92
|
|
|
93
93
|
Installation
|
|
94
94
|
------------
|
|
@@ -106,7 +106,7 @@ Requirements
|
|
|
106
106
|
PIP package Version required
|
|
107
107
|
========================================== ======================================
|
|
108
108
|
``apache-airflow`` ``>=2.10.0``
|
|
109
|
-
``apache-airflow-providers-common-compat`` ``>=1.
|
|
109
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
110
110
|
``apache-airflow-providers-common-sql`` ``>=1.27.0``
|
|
111
111
|
``requests`` ``>=2.32.0,<3``
|
|
112
112
|
``databricks-sql-connector`` ``>=4.0.0``
|
|
@@ -137,7 +137,6 @@ Dependent package
|
|
|
137
137
|
================================================================================================================== =================
|
|
138
138
|
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
139
139
|
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
140
|
-
`apache-airflow-providers-fab <https://airflow.apache.org/docs/apache-airflow-providers-fab>`_ ``fab``
|
|
141
140
|
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
142
141
|
================================================================================================================== =================
|
|
143
142
|
|
|
@@ -155,5 +154,5 @@ Extra Dependencies
|
|
|
155
154
|
================== ================================================================
|
|
156
155
|
|
|
157
156
|
The changelog for the provider package can be found in the
|
|
158
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.
|
|
157
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html>`_.
|
|
159
158
|
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-databricks``
|
|
25
25
|
|
|
26
|
-
Release: ``7.7.
|
|
26
|
+
Release: ``7.7.4``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`Databricks <https://databricks.com/>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
|
|
|
36
36
|
are in ``airflow.providers.databricks`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -54,7 +54,7 @@ Requirements
|
|
|
54
54
|
PIP package Version required
|
|
55
55
|
========================================== ======================================
|
|
56
56
|
``apache-airflow`` ``>=2.10.0``
|
|
57
|
-
``apache-airflow-providers-common-compat`` ``>=1.
|
|
57
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
58
58
|
``apache-airflow-providers-common-sql`` ``>=1.27.0``
|
|
59
59
|
``requests`` ``>=2.32.0,<3``
|
|
60
60
|
``databricks-sql-connector`` ``>=4.0.0``
|
|
@@ -85,7 +85,6 @@ Dependent package
|
|
|
85
85
|
================================================================================================================== =================
|
|
86
86
|
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
87
87
|
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
88
|
-
`apache-airflow-providers-fab <https://airflow.apache.org/docs/apache-airflow-providers-fab>`_ ``fab``
|
|
89
88
|
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
90
89
|
================================================================================================================== =================
|
|
91
90
|
|
|
@@ -103,4 +102,4 @@ Extra Dependencies
|
|
|
103
102
|
================== ================================================================
|
|
104
103
|
|
|
105
104
|
The changelog for the provider package can be found in the
|
|
106
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.
|
|
105
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html>`_.
|
|
@@ -26,11 +26,32 @@
|
|
|
26
26
|
Changelog
|
|
27
27
|
---------
|
|
28
28
|
|
|
29
|
-
7.7.
|
|
29
|
+
7.7.4
|
|
30
30
|
.....
|
|
31
31
|
|
|
32
|
+
Bug Fixes
|
|
33
|
+
~~~~~~~~~
|
|
34
|
+
|
|
35
|
+
* ``Fix Databricks provider import error without fab provider (#56702)``
|
|
36
|
+
|
|
37
|
+
Misc
|
|
38
|
+
~~~~
|
|
39
|
+
|
|
40
|
+
* ``Migrate databricks provider to ''common.compat'' (#56993)``
|
|
41
|
+
|
|
42
|
+
Doc-only
|
|
43
|
+
~~~~~~~~
|
|
44
|
+
|
|
45
|
+
* ``Remove placeholder Release Date in changelog and index files (#56056)``
|
|
46
|
+
|
|
47
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
48
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
49
|
+
* ``Remove 'pytest.importorskip("flask_appbuilder")' from tests (#56679)``
|
|
50
|
+
* ``Enable PT011 rule to prvoider tests (#56320)``
|
|
51
|
+
|
|
52
|
+
7.7.3
|
|
53
|
+
.....
|
|
32
54
|
|
|
33
|
-
Release Date: ``|PypiReleaseDate|``
|
|
34
55
|
|
|
35
56
|
Bug Fixes
|
|
36
57
|
~~~~~~~~~
|
|
@@ -49,8 +70,6 @@ Misc
|
|
|
49
70
|
.....
|
|
50
71
|
|
|
51
72
|
|
|
52
|
-
Release Date: ``|PypiReleaseDate|``
|
|
53
|
-
|
|
54
73
|
Bug Fixes
|
|
55
74
|
~~~~~~~~~
|
|
56
75
|
|
|
@@ -78,7 +78,7 @@ apache-airflow-providers-databricks package
|
|
|
78
78
|
`Databricks <https://databricks.com/>`__
|
|
79
79
|
|
|
80
80
|
|
|
81
|
-
Release: 7.7.
|
|
81
|
+
Release: 7.7.4
|
|
82
82
|
|
|
83
83
|
Provider package
|
|
84
84
|
----------------
|
|
@@ -102,7 +102,7 @@ The minimum Apache Airflow version supported by this provider distribution is ``
|
|
|
102
102
|
PIP package Version required
|
|
103
103
|
========================================== ======================================
|
|
104
104
|
``apache-airflow`` ``>=2.10.0``
|
|
105
|
-
``apache-airflow-providers-common-compat`` ``>=1.
|
|
105
|
+
``apache-airflow-providers-common-compat`` ``>=1.8.0``
|
|
106
106
|
``apache-airflow-providers-common-sql`` ``>=1.27.0``
|
|
107
107
|
``requests`` ``>=2.32.0,<3``
|
|
108
108
|
``databricks-sql-connector`` ``>=4.0.0``
|
|
@@ -133,7 +133,6 @@ Dependent package
|
|
|
133
133
|
================================================================================================================== =================
|
|
134
134
|
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
|
135
135
|
`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_ ``common.sql``
|
|
136
|
-
`apache-airflow-providers-fab <https://airflow.apache.org/docs/apache-airflow-providers-fab>`_ ``fab``
|
|
137
136
|
`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_ ``openlineage``
|
|
138
137
|
================================================================================================================== =================
|
|
139
138
|
|
|
@@ -143,5 +142,5 @@ Downloading official packages
|
|
|
143
142
|
You can download officially released packages and verify their checksums and signatures from the
|
|
144
143
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
145
144
|
|
|
146
|
-
* `The apache-airflow-providers-databricks 7.7.
|
|
147
|
-
* `The apache-airflow-providers-databricks 7.7.
|
|
145
|
+
* `The apache-airflow-providers-databricks 7.7.4 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.4.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.4.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.4.tar.gz.sha512>`__)
|
|
146
|
+
* `The apache-airflow-providers-databricks 7.7.4 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.4-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.4-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.4-py3-none-any.whl.sha512>`__)
|
|
@@ -22,12 +22,13 @@ description: |
|
|
|
22
22
|
`Databricks <https://databricks.com/>`__
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
|
-
source-date-epoch:
|
|
25
|
+
source-date-epoch: 1761116588
|
|
26
26
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
27
27
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
28
28
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
29
29
|
# to be done in the same PR
|
|
30
30
|
versions:
|
|
31
|
+
- 7.7.4
|
|
31
32
|
- 7.7.3
|
|
32
33
|
- 7.7.2
|
|
33
34
|
- 7.7.1
|
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-databricks"
|
|
28
|
-
version = "7.7.
|
|
28
|
+
version = "7.7.4"
|
|
29
29
|
description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
authors = [
|
|
@@ -57,9 +57,9 @@ requires-python = ">=3.10"
|
|
|
57
57
|
# Make sure to run ``prek update-providers-dependencies --all-files``
|
|
58
58
|
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
|
59
59
|
dependencies = [
|
|
60
|
-
"apache-airflow>=2.10.
|
|
61
|
-
"apache-airflow-providers-common-compat>=1.
|
|
62
|
-
"apache-airflow-providers-common-sql>=1.27.
|
|
60
|
+
"apache-airflow>=2.10.0",
|
|
61
|
+
"apache-airflow-providers-common-compat>=1.8.0",
|
|
62
|
+
"apache-airflow-providers-common-sql>=1.27.0",
|
|
63
63
|
"requests>=2.32.0,<3",
|
|
64
64
|
"databricks-sql-connector>=4.0.0",
|
|
65
65
|
"databricks-sqlalchemy>=1.0.2",
|
|
@@ -82,13 +82,13 @@ dependencies = [
|
|
|
82
82
|
"azure-identity>=1.3.1",
|
|
83
83
|
]
|
|
84
84
|
"fab" = [
|
|
85
|
-
"apache-airflow-providers-fab>=2.2.
|
|
85
|
+
"apache-airflow-providers-fab>=2.2.0; python_version < '3.13'"
|
|
86
86
|
]
|
|
87
87
|
"standard" = [
|
|
88
88
|
"apache-airflow-providers-standard"
|
|
89
89
|
]
|
|
90
90
|
"openlineage" = [
|
|
91
|
-
"apache-airflow-providers-openlineage>=2.3.
|
|
91
|
+
"apache-airflow-providers-openlineage>=2.3.0"
|
|
92
92
|
]
|
|
93
93
|
|
|
94
94
|
[dependency-groups]
|
|
@@ -98,13 +98,13 @@ dev = [
|
|
|
98
98
|
"apache-airflow-devel-common",
|
|
99
99
|
"apache-airflow-providers-common-compat",
|
|
100
100
|
"apache-airflow-providers-common-sql",
|
|
101
|
-
"apache-airflow-providers-fab",
|
|
102
101
|
"apache-airflow-providers-openlineage",
|
|
103
102
|
# Additional devel dependencies (do not remove this line and add extra development dependencies)
|
|
104
103
|
"deltalake>=1.1.3",
|
|
105
104
|
"apache-airflow-providers-fab>=2.2.0; python_version < '3.13'",
|
|
106
105
|
"apache-airflow-providers-microsoft-azure",
|
|
107
106
|
"apache-airflow-providers-common-sql[pandas,polars]",
|
|
107
|
+
"apache-airflow-providers-fab",
|
|
108
108
|
]
|
|
109
109
|
|
|
110
110
|
# To build docs:
|
|
@@ -133,8 +133,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
133
133
|
apache-airflow-providers-standard = {workspace = true}
|
|
134
134
|
|
|
135
135
|
[project.urls]
|
|
136
|
-
"Documentation" = "https://airflow.
|
|
137
|
-
"Changelog" = "https://airflow.
|
|
136
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4"
|
|
137
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html"
|
|
138
138
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
139
139
|
"Source Code" = "https://github.com/apache/airflow"
|
|
140
140
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "7.7.
|
|
32
|
+
__version__ = "7.7.4"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.10.0"
|
|
@@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Any
|
|
|
28
28
|
|
|
29
29
|
from airflow.configuration import conf
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseOperator, BaseOperatorLink, XCom
|
|
31
32
|
from airflow.providers.databricks.hooks.databricks import (
|
|
32
33
|
DatabricksHook,
|
|
33
34
|
RunLifeCycleState,
|
|
@@ -51,7 +52,7 @@ from airflow.providers.databricks.utils.databricks import (
|
|
|
51
52
|
validate_trigger_event,
|
|
52
53
|
)
|
|
53
54
|
from airflow.providers.databricks.utils.mixins import DatabricksSQLStatementsMixin
|
|
54
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
55
|
+
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
55
56
|
|
|
56
57
|
if TYPE_CHECKING:
|
|
57
58
|
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
@@ -62,13 +63,6 @@ if TYPE_CHECKING:
|
|
|
62
63
|
from airflow.sdk import TaskGroup
|
|
63
64
|
from airflow.sdk.types import Context, Logger
|
|
64
65
|
|
|
65
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
66
|
-
from airflow.sdk import BaseOperatorLink
|
|
67
|
-
from airflow.sdk.execution_time.xcom import XCom
|
|
68
|
-
else:
|
|
69
|
-
from airflow.models import XCom
|
|
70
|
-
from airflow.models.baseoperatorlink import BaseOperatorLink # type: ignore[no-redef]
|
|
71
|
-
|
|
72
66
|
DEFER_METHOD_NAME = "execute_complete"
|
|
73
67
|
XCOM_RUN_ID_KEY = "run_id"
|
|
74
68
|
XCOM_JOB_ID_KEY = "job_id"
|
|
@@ -26,15 +26,11 @@ from typing import TYPE_CHECKING
|
|
|
26
26
|
from urllib.parse import urlsplit
|
|
27
27
|
|
|
28
28
|
from airflow.exceptions import AirflowException
|
|
29
|
+
from airflow.providers.common.compat.sdk import BaseOperator
|
|
29
30
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook
|
|
30
|
-
from airflow.providers.databricks.version_compat import BaseOperator
|
|
31
31
|
|
|
32
32
|
if TYPE_CHECKING:
|
|
33
|
-
|
|
34
|
-
from airflow.sdk.definitions.context import Context
|
|
35
|
-
except ImportError:
|
|
36
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
37
|
-
from airflow.utils.context import Context
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
38
34
|
|
|
39
35
|
|
|
40
36
|
class DatabricksReposCreateOperator(BaseOperator):
|
|
@@ -28,12 +28,12 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
28
28
|
from databricks.sql.utils import ParamEscaper
|
|
29
29
|
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseOperator
|
|
31
32
|
from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
|
|
32
33
|
from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
|
|
33
|
-
from airflow.providers.databricks.version_compat import BaseOperator
|
|
34
34
|
|
|
35
35
|
if TYPE_CHECKING:
|
|
36
|
-
from airflow.
|
|
36
|
+
from airflow.providers.common.compat.sdk import Context
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class DatabricksSqlOperator(SQLExecuteQueryOperator):
|
|
@@ -26,24 +26,20 @@ from typing import TYPE_CHECKING, Any
|
|
|
26
26
|
from mergedeep import merge
|
|
27
27
|
|
|
28
28
|
from airflow.exceptions import AirflowException
|
|
29
|
+
from airflow.providers.common.compat.sdk import BaseOperator, TaskGroup
|
|
29
30
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook, RunLifeCycleState
|
|
30
31
|
from airflow.providers.databricks.plugins.databricks_workflow import (
|
|
31
32
|
WorkflowJobRepairAllFailedLink,
|
|
32
33
|
WorkflowJobRunLink,
|
|
33
34
|
store_databricks_job_run_link,
|
|
34
35
|
)
|
|
35
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
36
|
-
|
|
37
|
-
try:
|
|
38
|
-
from airflow.sdk import TaskGroup
|
|
39
|
-
except ImportError:
|
|
40
|
-
from airflow.utils.task_group import TaskGroup # type: ignore[no-redef]
|
|
36
|
+
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
41
37
|
|
|
42
38
|
if TYPE_CHECKING:
|
|
43
39
|
from types import TracebackType
|
|
44
40
|
|
|
45
41
|
from airflow.models.taskmixin import DAGNode
|
|
46
|
-
from airflow.
|
|
42
|
+
from airflow.providers.common.compat.sdk import Context
|
|
47
43
|
|
|
48
44
|
|
|
49
45
|
@dataclass
|
|
@@ -21,21 +21,13 @@ import os
|
|
|
21
21
|
from typing import TYPE_CHECKING, Any
|
|
22
22
|
from urllib.parse import unquote
|
|
23
23
|
|
|
24
|
-
from flask import flash, redirect, request, url_for
|
|
25
|
-
from flask_appbuilder import BaseView
|
|
26
|
-
from flask_appbuilder.api import expose
|
|
27
|
-
|
|
28
24
|
from airflow.exceptions import AirflowException, TaskInstanceNotFound
|
|
29
25
|
from airflow.models.dagrun import DagRun
|
|
30
26
|
from airflow.models.taskinstance import TaskInstance, TaskInstanceKey, clear_task_instances
|
|
31
27
|
from airflow.plugins_manager import AirflowPlugin
|
|
28
|
+
from airflow.providers.common.compat.sdk import BaseOperatorLink, TaskGroup, XCom
|
|
32
29
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook
|
|
33
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
34
|
-
|
|
35
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
36
|
-
from airflow.providers.fab.www import auth
|
|
37
|
-
else:
|
|
38
|
-
from airflow.www import auth # type: ignore
|
|
30
|
+
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
39
31
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
40
32
|
from airflow.utils.state import TaskInstanceState
|
|
41
33
|
|
|
@@ -43,24 +35,15 @@ if TYPE_CHECKING:
|
|
|
43
35
|
from sqlalchemy.orm.session import Session
|
|
44
36
|
|
|
45
37
|
from airflow.models import BaseOperator
|
|
38
|
+
from airflow.providers.common.compat.sdk import Context
|
|
46
39
|
from airflow.providers.databricks.operators.databricks import DatabricksTaskBaseOperator
|
|
47
40
|
from airflow.sdk.types import Logger
|
|
48
|
-
from airflow.utils.context import Context
|
|
49
41
|
|
|
50
42
|
|
|
51
43
|
REPAIR_WAIT_ATTEMPTS = os.getenv("DATABRICKS_REPAIR_WAIT_ATTEMPTS", 20)
|
|
52
44
|
REPAIR_WAIT_DELAY = os.getenv("DATABRICKS_REPAIR_WAIT_DELAY", 0.5)
|
|
53
45
|
|
|
54
46
|
|
|
55
|
-
def get_auth_decorator():
|
|
56
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
57
|
-
from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
|
|
58
|
-
else:
|
|
59
|
-
from airflow.auth.managers.models.resource_details import DagAccessEntity
|
|
60
|
-
|
|
61
|
-
return auth.has_access_dag("POST", DagAccessEntity.RUN)
|
|
62
|
-
|
|
63
|
-
|
|
64
47
|
def get_databricks_task_ids(
|
|
65
48
|
group_id: str, task_map: dict[str, DatabricksTaskBaseOperator], log: Logger
|
|
66
49
|
) -> list[str]:
|
|
@@ -86,7 +69,64 @@ def get_databricks_task_ids(
|
|
|
86
69
|
# TODO: Need to re-think on how to support the currently unavailable repair functionality in Airflow 3. Probably a
|
|
87
70
|
# good time to re-evaluate this would be once the plugin functionality is expanded in Airflow 3.1.
|
|
88
71
|
if not AIRFLOW_V_3_0_PLUS:
|
|
72
|
+
from flask import flash, redirect, request, url_for
|
|
73
|
+
from flask_appbuilder import BaseView
|
|
74
|
+
from flask_appbuilder.api import expose
|
|
75
|
+
|
|
89
76
|
from airflow.utils.session import NEW_SESSION, provide_session
|
|
77
|
+
from airflow.www import auth
|
|
78
|
+
|
|
79
|
+
def get_auth_decorator():
|
|
80
|
+
from airflow.auth.managers.models.resource_details import DagAccessEntity
|
|
81
|
+
|
|
82
|
+
return auth.has_access_dag("POST", DagAccessEntity.RUN)
|
|
83
|
+
|
|
84
|
+
class RepairDatabricksTasks(BaseView, LoggingMixin):
|
|
85
|
+
"""Repair databricks tasks from Airflow."""
|
|
86
|
+
|
|
87
|
+
default_view = "repair"
|
|
88
|
+
|
|
89
|
+
@expose("/repair_databricks_job/<string:dag_id>/<string:run_id>", methods=("GET",))
|
|
90
|
+
@get_auth_decorator()
|
|
91
|
+
def repair(self, dag_id: str, run_id: str):
|
|
92
|
+
return_url = self._get_return_url(dag_id, run_id)
|
|
93
|
+
|
|
94
|
+
tasks_to_repair = request.values.get("tasks_to_repair")
|
|
95
|
+
self.log.info("Tasks to repair: %s", tasks_to_repair)
|
|
96
|
+
if not tasks_to_repair:
|
|
97
|
+
flash("No tasks to repair. Not sending repair request.")
|
|
98
|
+
return redirect(return_url)
|
|
99
|
+
|
|
100
|
+
databricks_conn_id = request.values.get("databricks_conn_id")
|
|
101
|
+
databricks_run_id = request.values.get("databricks_run_id")
|
|
102
|
+
|
|
103
|
+
if not databricks_conn_id:
|
|
104
|
+
flash("No Databricks connection ID provided. Cannot repair tasks.")
|
|
105
|
+
return redirect(return_url)
|
|
106
|
+
|
|
107
|
+
if not databricks_run_id:
|
|
108
|
+
flash("No Databricks run ID provided. Cannot repair tasks.")
|
|
109
|
+
return redirect(return_url)
|
|
110
|
+
|
|
111
|
+
self.log.info("Repairing databricks job %s", databricks_run_id)
|
|
112
|
+
res = _repair_task(
|
|
113
|
+
databricks_conn_id=databricks_conn_id,
|
|
114
|
+
databricks_run_id=int(databricks_run_id),
|
|
115
|
+
tasks_to_repair=tasks_to_repair.split(","),
|
|
116
|
+
logger=self.log,
|
|
117
|
+
)
|
|
118
|
+
self.log.info("Repairing databricks job query for run %s sent", databricks_run_id)
|
|
119
|
+
|
|
120
|
+
self.log.info("Clearing tasks to rerun in airflow")
|
|
121
|
+
|
|
122
|
+
run_id = unquote(run_id)
|
|
123
|
+
_clear_task_instances(dag_id, run_id, tasks_to_repair.split(","), self.log)
|
|
124
|
+
flash(f"Databricks repair job is starting!: {res}")
|
|
125
|
+
return redirect(return_url)
|
|
126
|
+
|
|
127
|
+
@staticmethod
|
|
128
|
+
def _get_return_url(dag_id: str, run_id: str) -> str:
|
|
129
|
+
return url_for("Airflow.grid", dag_id=dag_id, dag_run_id=run_id)
|
|
90
130
|
|
|
91
131
|
def _get_dag(dag_id: str, session: Session):
|
|
92
132
|
from airflow.models.serialized_dag import SerializedDagModel
|
|
@@ -140,46 +180,45 @@ if not AIRFLOW_V_3_0_PLUS:
|
|
|
140
180
|
raise TaskInstanceNotFound("Task instance not found")
|
|
141
181
|
return ti
|
|
142
182
|
|
|
183
|
+
def _repair_task(
|
|
184
|
+
databricks_conn_id: str,
|
|
185
|
+
databricks_run_id: int,
|
|
186
|
+
tasks_to_repair: list[str],
|
|
187
|
+
logger: Logger,
|
|
188
|
+
) -> int:
|
|
189
|
+
"""
|
|
190
|
+
Repair a Databricks task using the Databricks API.
|
|
143
191
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
databricks_run_id: int,
|
|
147
|
-
tasks_to_repair: list[str],
|
|
148
|
-
logger: Logger,
|
|
149
|
-
) -> int:
|
|
150
|
-
"""
|
|
151
|
-
Repair a Databricks task using the Databricks API.
|
|
152
|
-
|
|
153
|
-
This function allows the Airflow retry function to create a repair job for Databricks.
|
|
154
|
-
It uses the Databricks API to get the latest repair ID before sending the repair query.
|
|
192
|
+
This function allows the Airflow retry function to create a repair job for Databricks.
|
|
193
|
+
It uses the Databricks API to get the latest repair ID before sending the repair query.
|
|
155
194
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
195
|
+
:param databricks_conn_id: The Databricks connection ID.
|
|
196
|
+
:param databricks_run_id: The Databricks run ID.
|
|
197
|
+
:param tasks_to_repair: A list of Databricks task IDs to repair.
|
|
198
|
+
:param logger: The logger to use for logging.
|
|
199
|
+
:return: None
|
|
200
|
+
"""
|
|
201
|
+
hook = DatabricksHook(databricks_conn_id=databricks_conn_id)
|
|
202
|
+
|
|
203
|
+
repair_history_id = hook.get_latest_repair_id(databricks_run_id)
|
|
204
|
+
logger.debug("Latest repair ID is %s", repair_history_id)
|
|
205
|
+
logger.debug(
|
|
206
|
+
"Sending repair query for tasks %s on run %s",
|
|
207
|
+
tasks_to_repair,
|
|
208
|
+
databricks_run_id,
|
|
209
|
+
)
|
|
171
210
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
211
|
+
run_data = hook.get_run(databricks_run_id)
|
|
212
|
+
repair_json = {
|
|
213
|
+
"run_id": databricks_run_id,
|
|
214
|
+
"latest_repair_id": repair_history_id,
|
|
215
|
+
"rerun_tasks": tasks_to_repair,
|
|
216
|
+
}
|
|
178
217
|
|
|
179
|
-
|
|
180
|
-
|
|
218
|
+
if "overriding_parameters" in run_data:
|
|
219
|
+
repair_json["overriding_parameters"] = run_data["overriding_parameters"]
|
|
181
220
|
|
|
182
|
-
|
|
221
|
+
return hook.repair_run(repair_json)
|
|
183
222
|
|
|
184
223
|
|
|
185
224
|
def get_launch_task_id(task_group: TaskGroup) -> str:
|
|
@@ -461,54 +500,6 @@ class WorkflowJobRepairSingleTaskLink(BaseOperatorLink, LoggingMixin):
|
|
|
461
500
|
return url_for("RepairDatabricksTasks.repair", **query_params)
|
|
462
501
|
|
|
463
502
|
|
|
464
|
-
class RepairDatabricksTasks(BaseView, LoggingMixin):
|
|
465
|
-
"""Repair databricks tasks from Airflow."""
|
|
466
|
-
|
|
467
|
-
default_view = "repair"
|
|
468
|
-
|
|
469
|
-
@expose("/repair_databricks_job/<string:dag_id>/<string:run_id>", methods=("GET",))
|
|
470
|
-
@get_auth_decorator()
|
|
471
|
-
def repair(self, dag_id: str, run_id: str):
|
|
472
|
-
return_url = self._get_return_url(dag_id, run_id)
|
|
473
|
-
|
|
474
|
-
tasks_to_repair = request.values.get("tasks_to_repair")
|
|
475
|
-
self.log.info("Tasks to repair: %s", tasks_to_repair)
|
|
476
|
-
if not tasks_to_repair:
|
|
477
|
-
flash("No tasks to repair. Not sending repair request.")
|
|
478
|
-
return redirect(return_url)
|
|
479
|
-
|
|
480
|
-
databricks_conn_id = request.values.get("databricks_conn_id")
|
|
481
|
-
databricks_run_id = request.values.get("databricks_run_id")
|
|
482
|
-
|
|
483
|
-
if not databricks_conn_id:
|
|
484
|
-
flash("No Databricks connection ID provided. Cannot repair tasks.")
|
|
485
|
-
return redirect(return_url)
|
|
486
|
-
|
|
487
|
-
if not databricks_run_id:
|
|
488
|
-
flash("No Databricks run ID provided. Cannot repair tasks.")
|
|
489
|
-
return redirect(return_url)
|
|
490
|
-
|
|
491
|
-
self.log.info("Repairing databricks job %s", databricks_run_id)
|
|
492
|
-
res = _repair_task(
|
|
493
|
-
databricks_conn_id=databricks_conn_id,
|
|
494
|
-
databricks_run_id=int(databricks_run_id),
|
|
495
|
-
tasks_to_repair=tasks_to_repair.split(","),
|
|
496
|
-
logger=self.log,
|
|
497
|
-
)
|
|
498
|
-
self.log.info("Repairing databricks job query for run %s sent", databricks_run_id)
|
|
499
|
-
|
|
500
|
-
self.log.info("Clearing tasks to rerun in airflow")
|
|
501
|
-
|
|
502
|
-
run_id = unquote(run_id)
|
|
503
|
-
_clear_task_instances(dag_id, run_id, tasks_to_repair.split(","), self.log)
|
|
504
|
-
flash(f"Databricks repair job is starting!: {res}")
|
|
505
|
-
return redirect(return_url)
|
|
506
|
-
|
|
507
|
-
@staticmethod
|
|
508
|
-
def _get_return_url(dag_id: str, run_id: str) -> str:
|
|
509
|
-
return url_for("Airflow.grid", dag_id=dag_id, dag_run_id=run_id)
|
|
510
|
-
|
|
511
|
-
|
|
512
503
|
class DatabricksWorkflowPlugin(AirflowPlugin):
|
|
513
504
|
"""
|
|
514
505
|
Databricks Workflows plugin for Airflow.
|
|
@@ -24,18 +24,13 @@ from typing import TYPE_CHECKING, Any
|
|
|
24
24
|
|
|
25
25
|
from airflow.configuration import conf
|
|
26
26
|
from airflow.exceptions import AirflowException
|
|
27
|
+
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
27
28
|
from airflow.providers.databricks.hooks.databricks import DatabricksHook, SQLStatementState
|
|
28
29
|
from airflow.providers.databricks.operators.databricks import DEFER_METHOD_NAME
|
|
29
30
|
from airflow.providers.databricks.utils.mixins import DatabricksSQLStatementsMixin
|
|
30
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
31
|
-
|
|
32
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
33
|
-
from airflow.sdk.bases.sensor import BaseSensorOperator
|
|
34
|
-
else:
|
|
35
|
-
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
|
36
31
|
|
|
37
32
|
if TYPE_CHECKING:
|
|
38
|
-
from airflow.
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
39
34
|
|
|
40
35
|
XCOM_STATEMENT_ID_KEY = "statement_id"
|
|
41
36
|
|
|
@@ -28,21 +28,12 @@ from typing import TYPE_CHECKING, Any
|
|
|
28
28
|
from databricks.sql.utils import ParamEscaper
|
|
29
29
|
|
|
30
30
|
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
31
32
|
from airflow.providers.common.sql.hooks.handlers import fetch_all_handler
|
|
32
33
|
from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
|
|
33
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
34
|
-
|
|
35
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
36
|
-
from airflow.sdk import BaseSensorOperator
|
|
37
|
-
else:
|
|
38
|
-
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
|
39
34
|
|
|
40
35
|
if TYPE_CHECKING:
|
|
41
|
-
|
|
42
|
-
from airflow.sdk.definitions.context import Context
|
|
43
|
-
except ImportError:
|
|
44
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
45
|
-
from airflow.utils.context import Context
|
|
36
|
+
from airflow.providers.common.compat.sdk import Context
|
|
46
37
|
|
|
47
38
|
|
|
48
39
|
class DatabricksPartitionSensor(BaseSensorOperator):
|
|
@@ -25,21 +25,12 @@ from functools import cached_property
|
|
|
25
25
|
from typing import TYPE_CHECKING, Any
|
|
26
26
|
|
|
27
27
|
from airflow.exceptions import AirflowException
|
|
28
|
+
from airflow.providers.common.compat.sdk import BaseSensorOperator
|
|
28
29
|
from airflow.providers.common.sql.hooks.handlers import fetch_all_handler
|
|
29
30
|
from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
|
|
30
|
-
from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
|
|
31
|
-
|
|
32
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
33
|
-
from airflow.sdk import BaseSensorOperator
|
|
34
|
-
else:
|
|
35
|
-
from airflow.sensors.base import BaseSensorOperator # type: ignore[no-redef]
|
|
36
31
|
|
|
37
32
|
if TYPE_CHECKING:
|
|
38
|
-
|
|
39
|
-
from airflow.sdk.definitions.context import Context
|
|
40
|
-
except ImportError:
|
|
41
|
-
# TODO: Remove once provider drops support for Airflow 2
|
|
42
|
-
from airflow.utils.context import Context
|
|
33
|
+
from airflow.providers.common.compat.sdk import Context
|
|
43
34
|
|
|
44
35
|
|
|
45
36
|
class DatabricksSqlSensor(BaseSensorOperator):
|
|
@@ -34,18 +34,6 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
|
34
34
|
|
|
35
35
|
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
|
36
36
|
|
|
37
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
38
|
-
from airflow.sdk import BaseOperator, BaseOperatorLink, TaskGroup
|
|
39
|
-
from airflow.sdk.execution_time.xcom import XCom
|
|
40
|
-
else:
|
|
41
|
-
from airflow.models import BaseOperator, XCom
|
|
42
|
-
from airflow.models.baseoperatorlink import BaseOperatorLink # type: ignore[no-redef]
|
|
43
|
-
from airflow.utils.task_group import TaskGroup # type: ignore[no-redef]
|
|
44
|
-
|
|
45
37
|
__all__ = [
|
|
46
38
|
"AIRFLOW_V_3_0_PLUS",
|
|
47
|
-
"BaseOperator",
|
|
48
|
-
"BaseOperatorLink",
|
|
49
|
-
"TaskGroup",
|
|
50
|
-
"XCom",
|
|
51
39
|
]
|
|
@@ -358,7 +358,7 @@ class TestDatabricksHook:
|
|
|
358
358
|
assert host == HOST
|
|
359
359
|
|
|
360
360
|
def test_init_bad_retry_limit(self):
|
|
361
|
-
with pytest.raises(ValueError):
|
|
361
|
+
with pytest.raises(ValueError, match="Retry limit must be greater than or equal to 1"):
|
|
362
362
|
DatabricksHook(retry_limit=0)
|
|
363
363
|
|
|
364
364
|
def test_do_api_call_retries_with_retryable_error(self):
|
|
@@ -373,9 +373,8 @@ def test_query(
|
|
|
373
373
|
],
|
|
374
374
|
)
|
|
375
375
|
def test_no_query(databricks_hook, empty_statement):
|
|
376
|
-
with pytest.raises(ValueError
|
|
376
|
+
with pytest.raises(ValueError, match="List of SQL statements is empty"):
|
|
377
377
|
databricks_hook.run(sql=empty_statement)
|
|
378
|
-
assert err.value.args[0] == "List of SQL statements is empty"
|
|
379
378
|
|
|
380
379
|
|
|
381
380
|
@pytest.mark.parametrize(
|
|
@@ -2404,10 +2404,10 @@ class TestDatabricksNotebookOperator:
|
|
|
2404
2404
|
existing_cluster_id="existing_cluster_id",
|
|
2405
2405
|
databricks_conn_id="test_conn_id",
|
|
2406
2406
|
)
|
|
2407
|
-
with pytest.raises(
|
|
2407
|
+
with pytest.raises(
|
|
2408
|
+
ValueError, match="Both new_cluster and existing_cluster_id are set. Only one should be set."
|
|
2409
|
+
):
|
|
2408
2410
|
operator._get_run_json()
|
|
2409
|
-
exception_message = "Both new_cluster and existing_cluster_id are set. Only one should be set."
|
|
2410
|
-
assert str(exc_info.value) == exception_message
|
|
2411
2411
|
|
|
2412
2412
|
def test_both_new_and_existing_cluster_unset(self):
|
|
2413
2413
|
operator = DatabricksNotebookOperator(
|
|
@@ -2416,10 +2416,8 @@ class TestDatabricksNotebookOperator:
|
|
|
2416
2416
|
source="test_source",
|
|
2417
2417
|
databricks_conn_id="test_conn_id",
|
|
2418
2418
|
)
|
|
2419
|
-
with pytest.raises(ValueError
|
|
2419
|
+
with pytest.raises(ValueError, match="Must specify either existing_cluster_id or new_cluster."):
|
|
2420
2420
|
operator._get_run_json()
|
|
2421
|
-
exception_message = "Must specify either existing_cluster_id or new_cluster."
|
|
2422
|
-
assert str(exc_info.value) == exception_message
|
|
2423
2421
|
|
|
2424
2422
|
def test_job_runs_forever_by_default(self):
|
|
2425
2423
|
operator = DatabricksNotebookOperator(
|
|
@@ -2442,13 +2440,12 @@ class TestDatabricksNotebookOperator:
|
|
|
2442
2440
|
existing_cluster_id="existing_cluster_id",
|
|
2443
2441
|
execution_timeout=timedelta(seconds=0),
|
|
2444
2442
|
)
|
|
2445
|
-
with pytest.raises(
|
|
2443
|
+
with pytest.raises(
|
|
2444
|
+
ValueError,
|
|
2445
|
+
match="If you've set an `execution_timeout` for the task, ensure it's not `0`. "
|
|
2446
|
+
"Set it instead to `None` if you desire the task to run indefinitely.",
|
|
2447
|
+
):
|
|
2446
2448
|
operator._get_run_json()
|
|
2447
|
-
exception_message = (
|
|
2448
|
-
"If you've set an `execution_timeout` for the task, ensure it's not `0`. "
|
|
2449
|
-
"Set it instead to `None` if you desire the task to run indefinitely."
|
|
2450
|
-
)
|
|
2451
|
-
assert str(exc_info.value) == exception_message
|
|
2452
2449
|
|
|
2453
2450
|
def test_extend_workflow_notebook_packages(self):
|
|
2454
2451
|
"""Test that the operator can extend the notebook packages of a Databricks workflow task group."""
|
|
@@ -21,8 +21,6 @@ from unittest import mock
|
|
|
21
21
|
|
|
22
22
|
import pytest
|
|
23
23
|
|
|
24
|
-
pytest.importorskip("flask_appbuilder") # Remove after upgrading to FAB5
|
|
25
|
-
|
|
26
24
|
from airflow.exceptions import AirflowException, TaskDeferred
|
|
27
25
|
from airflow.providers.databricks.hooks.databricks import SQLStatementState
|
|
28
26
|
from airflow.providers.databricks.sensors.databricks import DatabricksSQLStatementsSensor
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|