apache-airflow-providers-databricks 7.5.0rc1__tar.gz → 7.6.0rc1__tar.gz

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.5.0rc1
+Version: 7.6.0rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.9
+Requires-Python: ~=3.10
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,7 +15,6 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -28,16 +27,17 @@ Requires-Dist: databricks-sql-connector>=3.0.0
 Requires-Dist: databricks-sqlalchemy>=1.0.2
 Requires-Dist: aiohttp>=3.9.2, <4
 Requires-Dist: mergedeep>=1.3.4
-Requires-Dist: pandas>=2.1.2,<2.2
-Requires-Dist: pyarrow>=14.0.1
+Requires-Dist: pandas>=2.1.2; python_version <"3.13"
+Requires-Dist: pandas>=2.2.3; python_version >="3.13"
+Requires-Dist: pyarrow>=16.1.0
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: apache-airflow-providers-fab ; extra == "fab"
 Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.5.0/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.5.0
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.6.0/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.6.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -73,7 +73,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.5.0``
+Release: ``7.6.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -86,7 +86,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.5.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.6.0/>`_.
 
 Installation
 ------------
@@ -95,14 +95,14 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
-The package supports the following python versions: 3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.10,3.11,3.12
 
 Requirements
 ------------
 
-==========================================  ==================
+==========================================  =====================================
 PIP package                                 Version required
-==========================================  ==================
+==========================================  =====================================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
@@ -111,9 +111,10 @@ PIP package                                 Version required
 ``databricks-sqlalchemy``                   ``>=1.0.2``
 ``aiohttp``                                 ``>=3.9.2,<4``
 ``mergedeep``                               ``>=1.3.4``
-``pandas``                                  ``>=2.1.2,<2.2``
-``pyarrow``                                 ``>=14.0.1``
-==========================================  ==================
+``pandas``                                  ``>=2.1.2; python_version < "3.13"``
+``pandas``                                  ``>=2.2.3; python_version >= "3.13"``
+``pyarrow``                                 ``>=16.1.0``
+==========================================  =====================================
 
 Cross provider package dependencies
 -----------------------------------
@@ -138,5 +139,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.5.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.6.0/changelog.html>`_.
 

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.5.0``
+Release: ``7.6.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -36,7 +36,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.5.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.6.0/>`_.
 
 Installation
 ------------
@@ -45,14 +45,14 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
-The package supports the following python versions: 3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.10,3.11,3.12
 
 Requirements
 ------------
 
-==========================================  ==================
+==========================================  =====================================
 PIP package                                 Version required
-==========================================  ==================
+==========================================  =====================================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
@@ -61,9 +61,10 @@ PIP package                                 Version required
 ``databricks-sqlalchemy``                   ``>=1.0.2``
 ``aiohttp``                                 ``>=3.9.2,<4``
 ``mergedeep``                               ``>=1.3.4``
-``pandas``                                  ``>=2.1.2,<2.2``
-``pyarrow``                                 ``>=14.0.1``
-==========================================  ==================
+``pandas``                                  ``>=2.1.2; python_version < "3.13"``
+``pandas``                                  ``>=2.2.3; python_version >= "3.13"``
+``pyarrow``                                 ``>=16.1.0``
+==========================================  =====================================
 
 Cross provider package dependencies
 -----------------------------------
@@ -88,4 +89,4 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.5.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.6.0/changelog.html>`_.

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/docs/changelog.rst

@@ -26,6 +26,41 @@
 Changelog
 ---------
 
+7.6.0
+.....
+
+Features
+~~~~~~~~
+
+* ``Refactor Databricks hook to use HTTP method constants and auto-prepend api/ to endpoint paths (#52385)``
+
+Bug Fixes
+~~~~~~~~~
+
+* ``Fix: Unclosed aiohttp ClientSession and TCPConnector in DatabricksRunNowOperator (deferrable=True) (#52119)``
+
+Misc
+~~~~
+
+* ``Move 'BaseHook' implementation to task SDK (#51873)``
+* ``Disable UP038 ruff rule and revert mandatory 'X | Y' in insintance checks (#52644)``
+* ``Bump pyarrow to 16.1.0 minimum version for several providers (#52635)``
+* ``Upgrade ruff to latest version (0.12.1) (#52562)``
+* ``feat: Add explicit support for DatabricksHook to Openlineage helper (#52253)``
+* ``Replace 'models.BaseOperator' to Task SDK one for DBT & Databricks (#52377)``
+* ``Drop support for Python 3.9 (#52072)``
+* ``Use BaseSensorOperator from task sdk in providers (#52296)``
+* ``Enable DatabricksJobRunLink for Databricks plugin, skip provide_session usage in Airflow3 (#52228)``
+* ``Add deprecation to 'airflow/sensors/base.py' (#52249)``
+* ``Bump upper binding on pandas in all providers (#52060)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``Make sure all test version imports come from test_common (#52425)``
+   * ``Clean up messy default connection overrides in provider tests (#52137)``
+   * ``Remove pytest.mark.db_test where possible from databricks provider. (#52033)``
+   * ``Relax databricks test a bit to support compatibility with older version of it (#51787)``
+
 7.5.0
 .....
 

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/docs/index.rst

@@ -78,7 +78,7 @@ apache-airflow-providers-databricks package
 `Databricks <https://databricks.com/>`__
 
 
-Release: 7.5.0
+Release: 7.6.0
 
 Provider package
 ----------------
@@ -98,9 +98,9 @@ Requirements
 
 The minimum Apache Airflow version supported by this provider distribution is ``2.10.0``.
 
-==========================================  ==================
+==========================================  =====================================
 PIP package                                 Version required
-==========================================  ==================
+==========================================  =====================================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
@@ -109,9 +109,10 @@ PIP package                                 Version required
 ``databricks-sqlalchemy``                   ``>=1.0.2``
 ``aiohttp``                                 ``>=3.9.2,<4``
 ``mergedeep``                               ``>=1.3.4``
-``pandas``                                  ``>=2.1.2,<2.2``
-``pyarrow``                                 ``>=14.0.1``
-==========================================  ==================
+``pandas``                                  ``>=2.1.2; python_version < "3.13"``
+``pandas``                                  ``>=2.2.3; python_version >= "3.13"``
+``pyarrow``                                 ``>=16.1.0``
+==========================================  =====================================
 
 Cross provider package dependencies
 -----------------------------------
@@ -141,5 +142,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-databricks 7.5.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.5.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.5.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.5.0.tar.gz.sha512>`__)
-* `The apache-airflow-providers-databricks 7.5.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.5.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.5.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.5.0-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-databricks 7.6.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.6.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.6.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.6.0.tar.gz.sha512>`__)
+* `The apache-airflow-providers-databricks 7.6.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.6.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.6.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.6.0-py3-none-any.whl.sha512>`__)

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/provider.yaml

@@ -22,12 +22,13 @@ description: |
     `Databricks <https://databricks.com/>`__
 
 state: ready
-source-date-epoch: 1749896357
+source-date-epoch: 1751473030
 # Note that those versions are maintained by release manager - do not update them manually
 # with the exception of case where other provider in sources has >= new provider version.
 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
 # to be done in the same PR
 versions:
+  - 7.6.0
   - 7.5.0
   - 7.4.0
   - 7.3.2

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-databricks"
-version = "7.5.0rc1"
+version = "7.6.0rc1"
 description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -44,13 +44,12 @@ classifiers = [
     "Framework :: Apache Airflow",
     "Framework :: Apache Airflow :: Provider",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
 ]
-requires-python = "~=3.9"
+requires-python = "~=3.10"
 
 # The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
@@ -65,8 +64,9 @@ dependencies = [
     "databricks-sqlalchemy>=1.0.2",
     "aiohttp>=3.9.2, <4",
     "mergedeep>=1.3.4",
-    "pandas>=2.1.2,<2.2",
-    "pyarrow>=14.0.1",
+    'pandas>=2.1.2; python_version <"3.13"',
+    'pandas>=2.2.3; python_version >="3.13"',
+    "pyarrow>=16.1.0",
 ]
 
 # The optional dependencies should be modified in place in the generated file
@@ -130,8 +130,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.5.0"
-"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.5.0/changelog.html"
+"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.6.0"
+"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.6.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.5.0"
+__version__ = "7.6.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/hooks/databricks.py

@@ -37,33 +37,36 @@ from requests import exceptions as requests_exceptions
 from airflow.exceptions import AirflowException
 from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook
 
-GET_CLUSTER_ENDPOINT = ("GET", "api/2.0/clusters/get")
-RESTART_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/restart")
-START_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/start")
-TERMINATE_CLUSTER_ENDPOINT = ("POST", "api/2.0/clusters/delete")
-
-CREATE_ENDPOINT = ("POST", "api/2.1/jobs/create")
-RESET_ENDPOINT = ("POST", "api/2.1/jobs/reset")
-UPDATE_ENDPOINT = ("POST", "api/2.1/jobs/update")
-RUN_NOW_ENDPOINT = ("POST", "api/2.1/jobs/run-now")
-SUBMIT_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/submit")
-GET_RUN_ENDPOINT = ("GET", "api/2.1/jobs/runs/get")
-CANCEL_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/cancel")
-DELETE_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/delete")
-REPAIR_RUN_ENDPOINT = ("POST", "api/2.1/jobs/runs/repair")
-OUTPUT_RUNS_JOB_ENDPOINT = ("GET", "api/2.1/jobs/runs/get-output")
-CANCEL_ALL_RUNS_ENDPOINT = ("POST", "api/2.1/jobs/runs/cancel-all")
-
-INSTALL_LIBS_ENDPOINT = ("POST", "api/2.0/libraries/install")
-UNINSTALL_LIBS_ENDPOINT = ("POST", "api/2.0/libraries/uninstall")
-
-LIST_JOBS_ENDPOINT = ("GET", "api/2.1/jobs/list")
-LIST_PIPELINES_ENDPOINT = ("GET", "api/2.0/pipelines")
-
-WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "api/2.0/workspace/get-status")
-
-SPARK_VERSIONS_ENDPOINT = ("GET", "api/2.0/clusters/spark-versions")
-SQL_STATEMENTS_ENDPOINT = "api/2.0/sql/statements"
+GET_CLUSTER_ENDPOINT = ("GET", "2.0/clusters/get")
+RESTART_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/restart")
+START_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/start")
+TERMINATE_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/delete")
+
+CREATE_ENDPOINT = ("POST", "2.1/jobs/create")
+RESET_ENDPOINT = ("POST", "2.1/jobs/reset")
+UPDATE_ENDPOINT = ("POST", "2.1/jobs/update")
+RUN_NOW_ENDPOINT = ("POST", "2.1/jobs/run-now")
+SUBMIT_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/submit")
+GET_RUN_ENDPOINT = ("GET", "2.1/jobs/runs/get")
+CANCEL_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/cancel")
+DELETE_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/delete")
+REPAIR_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/repair")
+OUTPUT_RUNS_JOB_ENDPOINT = ("GET", "2.1/jobs/runs/get-output")
+CANCEL_ALL_RUNS_ENDPOINT = ("POST", "2.1/jobs/runs/cancel-all")
+
+INSTALL_LIBS_ENDPOINT = ("POST", "2.0/libraries/install")
+UNINSTALL_LIBS_ENDPOINT = ("POST", "2.0/libraries/uninstall")
+UPDATE_REPO_ENDPOINT = ("PATCH", "2.0/repos/")
+DELETE_REPO_ENDPOINT = ("DELETE", "2.0/repos/")
+CREATE_REPO_ENDPOINT = ("POST", "2.0/repos")
+
+LIST_JOBS_ENDPOINT = ("GET", "2.1/jobs/list")
+LIST_PIPELINES_ENDPOINT = ("GET", "2.0/pipelines")
+
+WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "2.0/workspace/get-status")
+
+SPARK_VERSIONS_ENDPOINT = ("GET", "2.0/clusters/spark-versions")
+SQL_STATEMENTS_ENDPOINT = "2.0/sql/statements"
 
 
 class RunLifeCycleState(Enum):
@@ -718,7 +721,8 @@ class DatabricksHook(BaseDatabricksHook):
         :param json: payload
         :return: metadata from update
         """
-        repos_endpoint = ("PATCH", f"api/2.0/repos/{repo_id}")
+        method, base_path = UPDATE_REPO_ENDPOINT
+        repos_endpoint = (method, f"{base_path}/{repo_id}")
         return self._do_api_call(repos_endpoint, json)
 
     def delete_repo(self, repo_id: str):
@@ -728,7 +732,8 @@ class DatabricksHook(BaseDatabricksHook):
         :param repo_id: ID of Databricks Repos
         :return:
         """
-        repos_endpoint = ("DELETE", f"api/2.0/repos/{repo_id}")
+        method, base_path = DELETE_REPO_ENDPOINT
+        repos_endpoint = (method, f"{base_path}/{repo_id}")
         self._do_api_call(repos_endpoint)
 
     def create_repo(self, json: dict[str, Any]) -> dict:
@@ -738,8 +743,7 @@ class DatabricksHook(BaseDatabricksHook):
         :param json: payload
         :return:
         """
-        repos_endpoint = ("POST", "api/2.0/repos")
-        return self._do_api_call(repos_endpoint, json)
+        return self._do_api_call(CREATE_REPO_ENDPOINT, json)
 
     def get_repo_by_path(self, path: str) -> str | None:
         """

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/hooks/databricks_base.py

@@ -50,9 +50,13 @@ from tenacity import (
 
 from airflow import __version__
 from airflow.exceptions import AirflowException, AirflowOptionalProviderFeatureException
-from airflow.hooks.base import BaseHook
 from airflow.providers_manager import ProvidersManager
 
+try:
+    from airflow.sdk import BaseHook
+except ImportError:
+    from airflow.hooks.base import BaseHook as BaseHook  # type: ignore
+
 if TYPE_CHECKING:
     from airflow.models import Connection
 
@@ -135,7 +139,7 @@ class BaseDatabricksHook(BaseHook):
 
     @cached_property
     def databricks_conn(self) -> Connection:
-        return self.get_connection(self.databricks_conn_id)
+        return self.get_connection(self.databricks_conn_id)  # type: ignore[return-value]
 
     def get_conn(self) -> Connection:
         return self.databricks_conn
@@ -353,14 +357,15 @@ class BaseDatabricksHook(BaseHook):
             async for attempt in self._a_get_retry_object():
                 with attempt:
                     if self.databricks_conn.extra_dejson.get("use_azure_managed_identity", False):
-                        token = await AsyncManagedIdentityCredential().get_token(f"{resource}/.default")
+                        async with AsyncManagedIdentityCredential() as credential:
+                            token = await credential.get_token(f"{resource}/.default")
                     else:
-                        credential = AsyncClientSecretCredential(
+                        async with AsyncClientSecretCredential(
                             client_id=self.databricks_conn.login,
                             client_secret=self.databricks_conn.password,
                             tenant_id=self.databricks_conn.extra_dejson["azure_tenant_id"],
-                        )
-                        token = await credential.get_token(f"{resource}/.default")
+                        ) as credential:
+                            token = await credential.get_token(f"{resource}/.default")
                     jsn = {
                         "access_token": token.token,
                         "token_type": "Bearer",
@@ -636,8 +641,9 @@ class BaseDatabricksHook(BaseHook):
         """
         method, endpoint = endpoint_info
 
-        # TODO: get rid of explicit 'api/' in the endpoint specification
-        url = self._endpoint_url(endpoint)
+        # Automatically prepend 'api/' prefix to all endpoint paths
+        full_endpoint = f"api/{endpoint}"
+        url = self._endpoint_url(full_endpoint)
 
         aad_headers = self._get_aad_headers()
         headers = {**self.user_agent_header, **aad_headers}
@@ -703,7 +709,8 @@ class BaseDatabricksHook(BaseHook):
         """
         method, endpoint = endpoint_info
 
-        url = self._endpoint_url(endpoint)
+        full_endpoint = f"api/{endpoint}"
+        url = self._endpoint_url(full_endpoint)
 
         aad_headers = await self._a_get_aad_headers()
         headers = {**self.user_agent_header, **aad_headers}

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/hooks/databricks_sql.py

@@ -18,14 +18,13 @@ from __future__ import annotations
 
 import threading
 from collections import namedtuple
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
 from contextlib import closing
 from copy import copy
 from datetime import timedelta
 from typing import (
     TYPE_CHECKING,
     Any,
-    Callable,
     TypeVar,
     cast,
     overload,
@@ -345,10 +344,9 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
 
     def get_openlineage_database_specific_lineage(self, task_instance) -> OperatorLineage | None:
         """
-        Generate OpenLineage metadata for a Databricks task instance based on executed query IDs.
+        Emit separate OpenLineage events for each Databricks query, based on executed query IDs.
 
-        If a single query ID is present, attach an `ExternalQueryRunFacet` to the lineage metadata.
-        If multiple query IDs are present, emits separate OpenLineage events for each query instead.
+        If a single query ID is present, also add an `ExternalQueryRunFacet` to the returned lineage metadata.
 
         Note that `get_openlineage_database_specific_lineage` is usually called after task's execution,
         so if multiple query IDs are present, both START and COMPLETE event for each query will be emitted
@@ -369,13 +367,22 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         from airflow.providers.openlineage.sqlparser import SQLParser
 
         if not self.query_ids:
-            self.log.debug("openlineage: no databricks query ids found.")
+            self.log.info("OpenLineage could not find databricks query ids.")
             return None
 
         self.log.debug("openlineage: getting connection to get database info")
         connection = self.get_connection(self.get_conn_id())
         namespace = SQLParser.create_namespace(self.get_openlineage_database_info(connection))
 
+        self.log.info("Separate OpenLineage events will be emitted for each Databricks query_id.")
+        emit_openlineage_events_for_databricks_queries(
+            task_instance=task_instance,
+            hook=self,
+            query_ids=self.query_ids,
+            query_for_extra_metadata=True,
+            query_source_namespace=namespace,
+        )
+
         if len(self.query_ids) == 1:
             self.log.debug("Attaching ExternalQueryRunFacet with single query_id to OpenLineage event.")
             return OperatorLineage(
@@ -386,12 +393,4 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
                 }
             )
 
-        self.log.info("Multiple query_ids found. Separate OpenLineage event will be emitted for each query.")
-        emit_openlineage_events_for_databricks_queries(
-            query_ids=self.query_ids,
-            query_source_namespace=namespace,
-            task_instance=task_instance,
-            hook=self,
-        )
-
         return None

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/operators/databricks.py

@@ -29,7 +29,6 @@ from typing import TYPE_CHECKING, Any
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
-from airflow.models import BaseOperator
 from airflow.providers.databricks.hooks.databricks import (
     DatabricksHook,
     RunLifeCycleState,
@@ -42,13 +41,14 @@ from airflow.providers.databricks.operators.databricks_workflow import (
 from airflow.providers.databricks.plugins.databricks_workflow import (
     WorkflowJobRepairSingleTaskLink,
     WorkflowJobRunLink,
+    store_databricks_job_run_link,
 )
 from airflow.providers.databricks.triggers.databricks import (
     DatabricksExecutionTrigger,
 )
 from airflow.providers.databricks.utils.databricks import normalise_json_content, validate_trigger_event
 from airflow.providers.databricks.utils.mixins import DatabricksSQLStatementsMixin
-from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
+from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS, BaseOperator
 
 if TYPE_CHECKING:
     from airflow.models.taskinstancekey import TaskInstanceKey
@@ -1214,10 +1214,16 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
         super().__init__(**kwargs)
 
         if self._databricks_workflow_task_group is not None:
-            self.operator_extra_links = (
-                WorkflowJobRunLink(),
-                WorkflowJobRepairSingleTaskLink(),
-            )
+            # Conditionally set operator_extra_links based on Airflow version. In Airflow 3, only show the job run link.
+            # In Airflow 2, show the job run link and the repair link.
+            # TODO: Once we expand the plugin functionality in Airflow 3.1, this can be re-evaluated on how to handle the repair link.
+            if AIRFLOW_V_3_0_PLUS:
+                self.operator_extra_links = (WorkflowJobRunLink(),)
+            else:
+                self.operator_extra_links = (
+                    WorkflowJobRunLink(),
+                    WorkflowJobRepairSingleTaskLink(),
+                )
         else:
             # Databricks does not support repair for non-workflow tasks, hence do not show the repair link.
             self.operator_extra_links = (DatabricksJobRunLink(),)
@@ -1427,6 +1433,15 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
             )
             self.databricks_run_id = workflow_run_metadata.run_id
             self.databricks_conn_id = workflow_run_metadata.conn_id
+
+            # Store operator links in XCom for Airflow 3 compatibility
+            if AIRFLOW_V_3_0_PLUS:
+                # Store the job run link
+                store_databricks_job_run_link(
+                    context=context,
+                    metadata=workflow_run_metadata,
+                    logger=self.log,
+                )
         else:
             self._launch_job(context=context)
         if self.wait_for_termination:

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/operators/databricks_repos.py

@@ -26,8 +26,8 @@ from typing import TYPE_CHECKING
 from urllib.parse import urlsplit
 
 from airflow.exceptions import AirflowException
-from airflow.models import BaseOperator
 from airflow.providers.databricks.hooks.databricks import DatabricksHook
+from airflow.providers.databricks.version_compat import BaseOperator
 
 if TYPE_CHECKING:
     try:

{apache_airflow_providers_databricks-7.5.0rc1 → apache_airflow_providers_databricks-7.6.0rc1}/src/airflow/providers/databricks/operators/databricks_sql.py

@@ -28,9 +28,9 @@ from typing import TYPE_CHECKING, Any, ClassVar
 from databricks.sql.utils import ParamEscaper
 
 from airflow.exceptions import AirflowException
-from airflow.models import BaseOperator
 from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
 from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
+from airflow.providers.databricks.version_compat import BaseOperator
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context