apache-airflow-providers-databricks 7.0.0rc2__tar.gz → 7.2.0__tar.gz

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apache-airflow-providers-databricks
-Version: 7.0.0rc2
+Version: 7.2.0
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,20 +20,19 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: aiohttp>=3.9.2, <4
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
-Requires-Dist: apache-airflow>=2.9.0rc0
+Requires-Dist: apache-airflow>=2.9.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
+Requires-Dist: requests>=2.27.0,<3
 Requires-Dist: databricks-sql-connector>=3.0.0
+Requires-Dist: aiohttp>=3.9.2, <4
 Requires-Dist: mergedeep>=1.3.4
-Requires-Dist: pandas>=1.5.3,<2.2;python_version<"3.9"
-Requires-Dist: pandas>=2.1.2,<2.2;python_version>="3.9"
+Requires-Dist: pandas>=2.1.2,<2.2
 Requires-Dist: pyarrow>=14.0.1
-Requires-Dist: requests>=2.27.0,<3
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://x.com/ApacheAirflow
@@ -42,23 +41,6 @@ Provides-Extra: azure-identity
 Provides-Extra: sdk
 
 
-.. Licensed to the Apache Software Foundation (ASF) under one
-   or more contributor license agreements.  See the NOTICE file
-   distributed with this work for additional information
-   regarding copyright ownership.  The ASF licenses this file
-   to you under the Apache License, Version 2.0 (the
-   "License"); you may not use this file except in compliance
-   with the License.  You may obtain a copy of the License at
-
-..   http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
-   software distributed under the License is distributed on an
-   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-   KIND, either express or implied.  See the License for the
-   specific language governing permissions and limitations
-   under the License.
-
  .. Licensed to the Apache Software Foundation (ASF) under one
     or more contributor license agreements.  See the NOTICE file
     distributed with this work for additional information
@@ -76,8 +58,7 @@ Provides-Extra: sdk
     specific language governing permissions and limitations
     under the License.
 
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-    OVERWRITTEN WHEN PREPARING PACKAGES.
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 
  .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
     `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -85,7 +66,7 @@ Provides-Extra: sdk
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.0.0.rc2``
+Release: ``7.2.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -98,7 +79,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0/>`_.
 
 Installation
 ------------
@@ -112,19 +93,18 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  =========================================
+=======================================  ==================
 PIP package                              Version required
-=======================================  =========================================
+=======================================  ==================
 ``apache-airflow``                       ``>=2.9.0``
 ``apache-airflow-providers-common-sql``  ``>=1.20.0``
 ``requests``                             ``>=2.27.0,<3``
 ``databricks-sql-connector``             ``>=3.0.0``
 ``aiohttp``                              ``>=3.9.2,<4``
 ``mergedeep``                            ``>=1.3.4``
-``pandas``                               ``>=2.1.2,<2.2; python_version >= "3.9"``
-``pandas``                               ``>=1.5.3,<2.2; python_version < "3.9"``
+``pandas``                               ``>=2.1.2,<2.2``
 ``pyarrow``                              ``>=14.0.1``
-=======================================  =========================================
+=======================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -146,4 +126,5 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0/changelog.html>`_.
+

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0}/README.rst

@@ -1,21 +1,4 @@
 
-.. Licensed to the Apache Software Foundation (ASF) under one
-   or more contributor license agreements.  See the NOTICE file
-   distributed with this work for additional information
-   regarding copyright ownership.  The ASF licenses this file
-   to you under the Apache License, Version 2.0 (the
-   "License"); you may not use this file except in compliance
-   with the License.  You may obtain a copy of the License at
-
-..   http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
-   software distributed under the License is distributed on an
-   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-   KIND, either express or implied.  See the License for the
-   specific language governing permissions and limitations
-   under the License.
-
  .. Licensed to the Apache Software Foundation (ASF) under one
     or more contributor license agreements.  See the NOTICE file
     distributed with this work for additional information
@@ -33,8 +16,7 @@
     specific language governing permissions and limitations
     under the License.
 
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-    OVERWRITTEN WHEN PREPARING PACKAGES.
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 
  .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
     `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -42,7 +24,7 @@
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.0.0.rc2``
+Release: ``7.2.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -55,7 +37,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0/>`_.
 
 Installation
 ------------
@@ -69,19 +51,18 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  =========================================
+=======================================  ==================
 PIP package                              Version required
-=======================================  =========================================
+=======================================  ==================
 ``apache-airflow``                       ``>=2.9.0``
 ``apache-airflow-providers-common-sql``  ``>=1.20.0``
 ``requests``                             ``>=2.27.0,<3``
 ``databricks-sql-connector``             ``>=3.0.0``
 ``aiohttp``                              ``>=3.9.2,<4``
 ``mergedeep``                            ``>=1.3.4``
-``pandas``                               ``>=2.1.2,<2.2; python_version >= "3.9"``
-``pandas``                               ``>=1.5.3,<2.2; python_version < "3.9"``
+``pandas``                               ``>=2.1.2,<2.2``
 ``pyarrow``                              ``>=14.0.1``
-=======================================  =========================================
+=======================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -103,4 +84,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0/changelog.html>`_.

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0}/pyproject.toml

@@ -1,4 +1,3 @@
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,10 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 
-# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
 # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
 [build-system]
 requires = ["flit_core==3.10.1"]
@@ -27,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-databricks"
-version = "7.0.0.rc2"
+version = "7.2.0"
 description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -53,21 +51,41 @@ classifiers = [
     "Topic :: System :: Monitoring",
 ]
 requires-python = "~=3.9"
+
+# The dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
 dependencies = [
-    "aiohttp>=3.9.2, <4",
-    "apache-airflow-providers-common-sql>=1.20.0rc0",
-    "apache-airflow>=2.9.0rc0",
+    "apache-airflow>=2.9.0",
+    "apache-airflow-providers-common-sql>=1.20.0",
+    "requests>=2.27.0,<3",
     "databricks-sql-connector>=3.0.0",
+    "aiohttp>=3.9.2, <4",
     "mergedeep>=1.3.4",
-    "pandas>=1.5.3,<2.2;python_version<\"3.9\"",
-    "pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
+    "pandas>=2.1.2,<2.2",
     "pyarrow>=14.0.1",
-    "requests>=2.27.0,<3",
+]
+
+# The optional dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+[project.optional-dependencies]
+# pip install apache-airflow-providers-databricks[sdk]
+"sdk" = [
+    "databricks-sdk==0.10.0",
+]
+"azure-identity" = [
+    "azure-identity>=1.3.1",
+]
+
+# The dependency groups should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+[dependency-groups]
+dev = [
+    "deltalake>=0.12.0",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.0.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.2.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -76,15 +94,12 @@ dependencies = [
 
 [project.entry-points."apache_airflow_provider"]
 provider_info = "airflow.providers.databricks.get_provider_info:get_provider_info"
+
 [project.entry-points."airflow.plugins"]
 databricks_workflow = "airflow.providers.databricks.plugins.databricks_workflow:DatabricksWorkflowPlugin"
-[project.optional-dependencies]
-"sdk" = [
-    "databricks-sdk==0.10.0",
-]
-"azure-identity" = [
-    "azure-identity>=1.3.1",
-]
 
 [tool.flit.module]
 name = "airflow.providers.databricks"
+
+[tool.pytest.ini_options]
+ignore = "tests/system/"

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/LICENSE

@@ -199,55 +199,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-
-============================================================================
-   APACHE AIRFLOW SUBCOMPONENTS:
-
-   The Apache Airflow project contains subcomponents with separate copyright
-   notices and license terms. Your use of the source code for the these
-   subcomponents is subject to the terms and conditions of the following
-   licenses.
-
-
-========================================================================
-Third party Apache 2.0 licenses
-========================================================================
-
-The following components are provided under the Apache 2.0 License.
-See project link for details. The text of each license is also included
-at 3rd-party-licenses/LICENSE-[project].txt.
-
-    (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
-    (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
-    (ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
-    (ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
-
-========================================================================
-MIT licenses
-========================================================================
-
-The following components are provided under the MIT License. See project link for details.
-The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
-
-    (MIT License) jquery v3.5.1 (https://jquery.org/license/)
-    (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
-    (MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
-    (MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
-    (MIT License) dataTables v1.10.25 (https://datatables.net)
-    (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
-    (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
-    (MIT License) MomentJS v2.24.0 (http://momentjs.com/)
-    (MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
-
-========================================================================
-BSD 3-Clause licenses
-========================================================================
-The following components are provided under the BSD 3-Clause license. See project links for details.
-The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
-
-    (BSD 3 License) d3 v5.16.0 (https://d3js.org)
-    (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
-    (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
-
-========================================================================
-See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.0.0"
+__version__ = "7.2.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/get_provider_info.py

@@ -15,8 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 #
 # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
 # `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -28,8 +27,9 @@ def get_provider_info():
         "name": "Databricks",
         "description": "`Databricks <https://databricks.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1734533222,
+        "source-date-epoch": 1739959376,
         "versions": [
+            "7.2.0",
             "7.0.0",
             "6.13.0",
             "6.12.0",
@@ -77,30 +77,6 @@ def get_provider_info():
             "1.0.1",
             "1.0.0",
         ],
-        "dependencies": [
-            "apache-airflow>=2.9.0",
-            "apache-airflow-providers-common-sql>=1.20.0",
-            "requests>=2.27.0,<3",
-            "databricks-sql-connector>=3.0.0",
-            "aiohttp>=3.9.2, <4",
-            "mergedeep>=1.3.4",
-            'pandas>=2.1.2,<2.2;python_version>="3.9"',
-            'pandas>=1.5.3,<2.2;python_version<"3.9"',
-            "pyarrow>=14.0.1",
-        ],
-        "additional-extras": [
-            {
-                "name": "sdk",
-                "description": "Install Databricks SDK",
-                "dependencies": ["databricks-sdk==0.10.0"],
-            },
-            {
-                "name": "azure-identity",
-                "description": "Install Azure Identity client library",
-                "dependencies": ["azure-identity>=1.3.1"],
-            },
-        ],
-        "devel-dependencies": ["deltalake>=0.12.0"],
         "integrations": [
             {
                 "integration-name": "Databricks",
@@ -112,7 +88,7 @@ def get_provider_info():
                     "/docs/apache-airflow-providers-databricks/operators/run_now.rst",
                     "/docs/apache-airflow-providers-databricks/operators/task.rst",
                 ],
-                "logo": "/integration-logos/databricks/Databricks.png",
+                "logo": "/docs/integration-logos/Databricks.png",
                 "tags": ["service"],
             },
             {
@@ -122,7 +98,6 @@ def get_provider_info():
                     "/docs/apache-airflow-providers-databricks/operators/sql.rst",
                     "/docs/apache-airflow-providers-databricks/operators/copy_into.rst",
                 ],
-                "logo": "/integration-logos/databricks/Databricks.png",
                 "tags": ["service"],
             },
             {
@@ -133,14 +108,12 @@ def get_provider_info():
                     "/docs/apache-airflow-providers-databricks/operators/repos_update.rst",
                     "/docs/apache-airflow-providers-databricks/operators/repos_delete.rst",
                 ],
-                "logo": "/integration-logos/databricks/Databricks.png",
                 "tags": ["service"],
             },
             {
                 "integration-name": "Databricks Workflow",
                 "external-doc-url": "https://docs.databricks.com/en/workflows/index.html",
                 "how-to-guide": ["/docs/apache-airflow-providers-databricks/operators/workflow.rst"],
-                "logo": "/integration-logos/databricks/Databricks.png",
                 "tags": ["service"],
             },
         ],
@@ -203,4 +176,19 @@ def get_provider_info():
             }
         ],
         "extra-links": ["airflow.providers.databricks.operators.databricks.DatabricksJobRunLink"],
+        "dependencies": [
+            "apache-airflow>=2.9.0",
+            "apache-airflow-providers-common-sql>=1.20.0",
+            "requests>=2.27.0,<3",
+            "databricks-sql-connector>=3.0.0",
+            "aiohttp>=3.9.2, <4",
+            "mergedeep>=1.3.4",
+            "pandas>=2.1.2,<2.2",
+            "pyarrow>=14.0.1",
+        ],
+        "optional-dependencies": {
+            "sdk": ["databricks-sdk==0.10.0"],
+            "azure-identity": ["azure-identity>=1.3.1"],
+        },
+        "devel-dependencies": ["deltalake>=0.12.0"],
     }

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/hooks/databricks_sql.py

@@ -31,14 +31,13 @@ from typing import (
     overload,
 )
 
-from databricks import sql  # type: ignore[attr-defined]
-from databricks.sql.types import Row
-
 from airflow.exceptions import AirflowException
 from airflow.models.connection import Connection as AirflowConnection
 from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results
 from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout
 from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook
+from databricks import sql  # type: ignore[attr-defined]
+from databricks.sql.types import Row
 
 if TYPE_CHECKING:
     from databricks.sql.client import Connection

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/operators/databricks.py

@@ -19,6 +19,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import time
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
@@ -966,6 +967,8 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
 
     :param caller: The name of the caller operator to be used in the logs.
     :param databricks_conn_id: The name of the Airflow connection to use.
+    :param databricks_task_key: An optional task_key used to refer to the task by Databricks API. By
+        default this will be set to the hash of ``dag_id + task_id``.
     :param databricks_retry_args: An optional dictionary with arguments passed to ``tenacity.Retrying`` class.
     :param databricks_retry_delay: Number of seconds to wait between retries.
     :param databricks_retry_limit: Amount of times to retry if the Databricks backend is unreachable.
@@ -986,6 +989,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
         self,
         caller: str = "DatabricksTaskBaseOperator",
         databricks_conn_id: str = "databricks_default",
+        databricks_task_key: str = "",
         databricks_retry_args: dict[Any, Any] | None = None,
         databricks_retry_delay: int = 1,
         databricks_retry_limit: int = 3,
@@ -1000,6 +1004,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
     ):
         self.caller = caller
         self.databricks_conn_id = databricks_conn_id
+        self._databricks_task_key = databricks_task_key
         self.databricks_retry_args = databricks_retry_args
         self.databricks_retry_delay = databricks_retry_delay
         self.databricks_retry_limit = databricks_retry_limit
@@ -1037,17 +1042,21 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
             caller=caller,
         )
 
-    def _get_databricks_task_id(self, task_id: str) -> str:
-        """Get the databricks task ID using dag_id and task_id. Removes illegal characters."""
-        task_id = f"{self.dag_id}__{task_id.replace('.', '__')}"
-        if len(task_id) > 100:
-            self.log.warning(
-                "The generated task_key '%s' exceeds 100 characters and will be truncated by the Databricks API. "
-                "This will cause failure when trying to monitor the task. task_key is generated by ",
-                "concatenating dag_id and task_id.",
-                task_id,
+    @cached_property
+    def databricks_task_key(self) -> str:
+        return self._generate_databricks_task_key()
+
+    def _generate_databricks_task_key(self, task_id: str | None = None) -> str:
+        """Create a databricks task key using the hash of dag_id and task_id."""
+        if not self._databricks_task_key or len(self._databricks_task_key) > 100:
+            self.log.info(
+                "databricks_task_key has not be provided or the provided one exceeds 100 characters and will be truncated by the Databricks API. This will cause failure when trying to monitor the task. A task_key will be generated using the hash value of dag_id+task_id"
             )
-        return task_id
+            task_id = task_id or self.task_id
+            task_key = f"{self.dag_id}__{task_id}".encode()
+            self._databricks_task_key = hashlib.md5(task_key).hexdigest()
+            self.log.info("Generated databricks task_key: %s", self._databricks_task_key)
+        return self._databricks_task_key
 
     @property
     def _databricks_workflow_task_group(self) -> DatabricksWorkflowTaskGroup | None:
@@ -1077,7 +1086,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
     def _get_run_json(self) -> dict[str, Any]:
         """Get run json to be used for task submissions."""
         run_json = {
-            "run_name": self._get_databricks_task_id(self.task_id),
+            "run_name": self.databricks_task_key,
             **self._get_task_base_json(),
         }
         if self.new_cluster and self.existing_cluster_id:
@@ -1090,7 +1099,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
             raise ValueError("Must specify either existing_cluster_id or new_cluster.")
         return run_json
 
-    def _launch_job(self, context: Context | None = None) -> int:
+    def _launch_job(self, context: Context | None = None) -> int | None:
         """Launch the job on Databricks."""
         run_json = self._get_run_json()
         self.databricks_run_id = self._hook.submit_run(run_json)
@@ -1127,9 +1136,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
         # building the {task_key: task} map below.
         sorted_task_runs = sorted(tasks, key=lambda x: x["start_time"])
 
-        return {task["task_key"]: task for task in sorted_task_runs}[
-            self._get_databricks_task_id(self.task_id)
-        ]
+        return {task["task_key"]: task for task in sorted_task_runs}[self.databricks_task_key]
 
     def _convert_to_databricks_workflow_task(
         self, relevant_upstreams: list[BaseOperator], context: Context | None = None
@@ -1137,9 +1144,9 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
         """Convert the operator to a Databricks workflow task that can be a task in a workflow."""
         base_task_json = self._get_task_base_json()
         result = {
-            "task_key": self._get_databricks_task_id(self.task_id),
+            "task_key": self.databricks_task_key,
             "depends_on": [
-                {"task_key": self._get_databricks_task_id(task_id)}
+                {"task_key": self._generate_databricks_task_key(task_id)}
                 for task_id in self.upstream_task_ids
                 if task_id in relevant_upstreams
             ],
@@ -1172,7 +1179,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
         run_state = RunState(**run["state"])
         self.log.info(
             "Current state of the the databricks task %s is %s",
-            self._get_databricks_task_id(self.task_id),
+            self.databricks_task_key,
             run_state.life_cycle_state,
         )
         if self.deferrable and not run_state.is_terminal:
@@ -1194,7 +1201,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
             run_state = RunState(**run["state"])
             self.log.info(
                 "Current state of the databricks task %s is %s",
-                self._get_databricks_task_id(self.task_id),
+                self.databricks_task_key,
                 run_state.life_cycle_state,
             )
         self._handle_terminal_run_state(run_state)

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/operators/databricks_repos.py

@@ -30,7 +30,11 @@ from airflow.models import BaseOperator
 from airflow.providers.databricks.hooks.databricks import DatabricksHook
 
 if TYPE_CHECKING:
-    from airflow.utils.context import Context
+    try:
+        from airflow.sdk.definitions.context import Context
+    except ImportError:
+        # TODO: Remove once provider drops support for Airflow 2
+        from airflow.utils.context import Context
 
 
 class DatabricksReposCreateOperator(BaseOperator):

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/operators/databricks_sql.py

@@ -24,12 +24,11 @@ import json
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, ClassVar
 
-from databricks.sql.utils import ParamEscaper
-
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
 from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
 from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
+from databricks.sql.utils import ParamEscaper
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/operators/databricks_workflow.py

@@ -93,7 +93,7 @@ class _CreateDatabricksWorkflowOperator(BaseOperator):
     """
 
     operator_extra_links = (WorkflowJobRunLink(), WorkflowJobRepairAllFailedLink())
-    template_fields = ("notebook_params",)
+    template_fields = ("notebook_params", "job_clusters")
     caller = "_CreateDatabricksWorkflowOperator"
 
     def __init__(

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/plugins/databricks_workflow.py

@@ -44,6 +44,8 @@ from airflow.www.views import AirflowBaseView
 if TYPE_CHECKING:
     from sqlalchemy.orm.session import Session
 
+    from airflow.providers.databricks.operators.databricks import DatabricksTaskBaseOperator
+
 
 REPAIR_WAIT_ATTEMPTS = os.getenv("DATABRICKS_REPAIR_WAIT_ATTEMPTS", 20)
 REPAIR_WAIT_DELAY = os.getenv("DATABRICKS_REPAIR_WAIT_DELAY", 0.5)
@@ -57,18 +59,8 @@ def get_auth_decorator():
     return auth.has_access_dag("POST", DagAccessEntity.RUN)
 
 
-def _get_databricks_task_id(task: BaseOperator) -> str:
-    """
-    Get the databricks task ID using dag_id and task_id. removes illegal characters.
-
-    :param task: The task to get the databricks task ID for.
-    :return: The databricks task ID.
-    """
-    return f"{task.dag_id}__{task.task_id.replace('.', '__')}"
-
-
 def get_databricks_task_ids(
-    group_id: str, task_map: dict[str, BaseOperator], log: logging.Logger
+    group_id: str, task_map: dict[str, DatabricksTaskBaseOperator], log: logging.Logger
 ) -> list[str]:
     """
     Return a list of all Databricks task IDs for a dictionary of Airflow tasks.
@@ -83,7 +75,7 @@ def get_databricks_task_ids(
     for task_id, task in task_map.items():
         if task_id == f"{group_id}.launch":
             continue
-        databricks_task_id = _get_databricks_task_id(task)
+        databricks_task_id = task.databricks_task_key
         log.debug("databricks task id for task %s is %s", task_id, databricks_task_id)
         task_ids.append(databricks_task_id)
     return task_ids
@@ -112,7 +104,7 @@ def _clear_task_instances(
     dag = airflow_app.dag_bag.get_dag(dag_id)
     log.debug("task_ids %s to clear", str(task_ids))
     dr: DagRun = _get_dagrun(dag, run_id, session=session)
-    tis_to_clear = [ti for ti in dr.get_task_instances() if _get_databricks_task_id(ti) in task_ids]
+    tis_to_clear = [ti for ti in dr.get_task_instances() if ti.databricks_task_key in task_ids]
     clear_task_instances(tis_to_clear, session)
 
 
@@ -327,7 +319,7 @@ class WorkflowJobRepairAllFailedLink(BaseOperatorLink, LoggingMixin):
 
         tasks_to_run = {ti: t for ti, t in task_group_sub_tasks if ti in failed_and_skipped_tasks}
 
-        return ",".join(get_databricks_task_ids(task_group.group_id, tasks_to_run, log))
+        return ",".join(get_databricks_task_ids(task_group.group_id, tasks_to_run, log))  # type: ignore[arg-type]
 
     @staticmethod
     def _get_failed_and_skipped_tasks(dr: DagRun) -> list[str]:
@@ -390,7 +382,7 @@ class WorkflowJobRepairSingleTaskLink(BaseOperatorLink, LoggingMixin):
             "databricks_conn_id": metadata.conn_id,
             "databricks_run_id": metadata.run_id,
             "run_id": ti_key.run_id,
-            "tasks_to_repair": _get_databricks_task_id(task),
+            "tasks_to_repair": task.databricks_task_key,
         }
         return url_for("RepairDatabricksTasks.repair", **query_params)
 

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/sensors/databricks_partition.py

@@ -25,15 +25,18 @@ from datetime import datetime
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Callable
 
-from databricks.sql.utils import ParamEscaper
-
 from airflow.exceptions import AirflowException
 from airflow.providers.common.sql.hooks.sql import fetch_all_handler
 from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
 from airflow.sensors.base import BaseSensorOperator
+from databricks.sql.utils import ParamEscaper
 
 if TYPE_CHECKING:
-    from airflow.utils.context import Context
+    try:
+        from airflow.sdk.definitions.context import Context
+    except ImportError:
+        # TODO: Remove once provider drops support for Airflow 2
+        from airflow.utils.context import Context
 
 
 class DatabricksPartitionSensor(BaseSensorOperator):

{apache_airflow_providers_databricks-7.0.0rc2 → apache_airflow_providers_databricks-7.2.0/src}/airflow/providers/databricks/sensors/databricks_sql.py

@@ -30,7 +30,11 @@ from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
 from airflow.sensors.base import BaseSensorOperator
 
 if TYPE_CHECKING:
-    from airflow.utils.context import Context
+    try:
+        from airflow.sdk.definitions.context import Context
+    except ImportError:
+        # TODO: Remove once provider drops support for Airflow 2
+        from airflow.utils.context import Context
 
 
 class DatabricksSqlSensor(BaseSensorOperator):