apache-airflow-providers-databricks 7.7.4rc1__py3-none-any.whl → 7.8.0__py3-none-any.whl

airflow/providers/databricks/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.7.4"
+__version__ = "7.8.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.10.0"
+    "2.11.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.10.0+"
+        f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.11.0+"
     )

airflow/providers/databricks/get_provider_info.py

@@ -45,6 +45,7 @@ def get_provider_info():
                 "external-doc-url": "https://databricks.com/product/databricks-sql",
                 "how-to-guide": [
                     "/docs/apache-airflow-providers-databricks/operators/sql.rst",
+                    "/docs/apache-airflow-providers-databricks/operators/sql_statements.rst",
                     "/docs/apache-airflow-providers-databricks/operators/copy_into.rst",
                 ],
                 "tags": ["service"],

airflow/providers/databricks/hooks/databricks.py

@@ -134,6 +134,9 @@ class RunState:
             and self.state_message == other.state_message
         )
 
+    def __hash__(self):
+        return hash((self.life_cycle_state, self.result_state, self.state_message))
+
     def __repr__(self) -> str:
         return str(self.__dict__)
 
@@ -183,6 +186,9 @@ class ClusterState:
     def __eq__(self, other) -> bool:
         return self.state == other.state and self.state_message == other.state_message
 
+    def __hash__(self):
+        return hash((self.state, self.state_message))
+
     def __repr__(self) -> str:
         return str(self.__dict__)
 
@@ -244,6 +250,9 @@ class SQLStatementState:
             and self.error_message == other.error_message
         )
 
+    def __hash__(self):
+        return hash((self.state, self.error_code, self.error_message))
+
     def __repr__(self) -> str:
         return str(self.__dict__)
 

airflow/providers/databricks/hooks/databricks_base.py

@@ -166,12 +166,12 @@ class BaseDatabricksHook(BaseHook):
         return ua_string
 
     @cached_property
-    def host(self) -> str:
+    def host(self) -> str | None:
+        host = None
         if "host" in self.databricks_conn.extra_dejson:
             host = self._parse_host(self.databricks_conn.extra_dejson["host"])
-        else:
+        elif self.databricks_conn.host:
             host = self._parse_host(self.databricks_conn.host)
-
         return host
 
     async def __aenter__(self):
@@ -207,6 +207,11 @@ class BaseDatabricksHook(BaseHook):
         # In this case, host = xx.cloud.databricks.com
         return host
 
+    def _get_connection_attr(self, attr_name: str) -> str:
+        if not (attr := getattr(self.databricks_conn, attr_name)):
+            raise ValueError(f"`{attr_name}` must be present in Connection")
+        return attr
+
     def _get_retry_object(self) -> Retrying:
         """
         Instantiate a retry object.
@@ -235,7 +240,7 @@ class BaseDatabricksHook(BaseHook):
                 with attempt:
                     resp = requests.post(
                         resource,
-                        auth=HTTPBasicAuth(self.databricks_conn.login, self.databricks_conn.password),
+                        auth=HTTPBasicAuth(self._get_connection_attr("login"), self.databricks_conn.password),
                         data="grant_type=client_credentials&scope=all-apis",
                         headers={
                             **self.user_agent_header,
@@ -271,7 +276,9 @@ class BaseDatabricksHook(BaseHook):
                 with attempt:
                     async with self._session.post(
                         resource,
-                        auth=aiohttp.BasicAuth(self.databricks_conn.login, self.databricks_conn.password),
+                        auth=aiohttp.BasicAuth(
+                            self._get_connection_attr("login"), self.databricks_conn.password
+                        ),
                         data="grant_type=client_credentials&scope=all-apis",
                         headers={
                             **self.user_agent_header,
@@ -316,7 +323,7 @@ class BaseDatabricksHook(BaseHook):
                         token = ManagedIdentityCredential().get_token(f"{resource}/.default")
                     else:
                         credential = ClientSecretCredential(
-                            client_id=self.databricks_conn.login,
+                            client_id=self._get_connection_attr("login"),
                             client_secret=self.databricks_conn.password,
                             tenant_id=self.databricks_conn.extra_dejson["azure_tenant_id"],
                         )
@@ -364,7 +371,7 @@ class BaseDatabricksHook(BaseHook):
                             token = await credential.get_token(f"{resource}/.default")
                     else:
                         async with AsyncClientSecretCredential(
-                            client_id=self.databricks_conn.login,
+                            client_id=self._get_connection_attr("login"),
                             client_secret=self.databricks_conn.password,
                             tenant_id=self.databricks_conn.extra_dejson["azure_tenant_id"],
                         ) as credential:
@@ -678,7 +685,7 @@ class BaseDatabricksHook(BaseHook):
             auth = _TokenAuth(token)
         else:
             self.log.info("Using basic auth.")
-            auth = HTTPBasicAuth(self.databricks_conn.login, self.databricks_conn.password)
+            auth = HTTPBasicAuth(self._get_connection_attr("login"), self.databricks_conn.password)
 
         request_func: Any
         if method == "GET":
@@ -745,7 +752,7 @@ class BaseDatabricksHook(BaseHook):
             auth = BearerAuth(token)
         else:
             self.log.info("Using basic auth.")
-            auth = aiohttp.BasicAuth(self.databricks_conn.login, self.databricks_conn.password)
+            auth = aiohttp.BasicAuth(self._get_connection_attr("login"), self.databricks_conn.password)
 
         request_func: Any
         if method == "GET":

airflow/providers/databricks/hooks/databricks_sql.py

@@ -184,13 +184,13 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
             "catalog": self.catalog,
             "schema": self.schema,
         }
-        url_query = {k: v for k, v in url_query.items() if v is not None}
+        url_query_formatted: dict[str, str] = {k: v for k, v in url_query.items() if v is not None}
         return URL.create(
             drivername="databricks",
             username="token",
             password=self._get_token(raise_error=True),
             host=self.host,
-            query=url_query,
+            query=url_query_formatted,
         )
 
     def get_uri(self) -> str:
@@ -278,7 +278,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
             self.log.info("Running statement: %s, parameters: %s", sql_statement, parameters)
             # when using AAD tokens, it could expire if previous query run longer than token lifetime
             conn = self.get_conn()
-            with closing(conn.cursor()) as cur:
+            with closing(conn.cursor()):
                 self.set_autocommit(conn, autocommit)
 
                 with closing(conn.cursor()) as cur:

airflow/providers/databricks/plugins/databricks_workflow.py

@@ -17,7 +17,6 @@
 
 from __future__ import annotations
 
-import os
 from typing import TYPE_CHECKING, Any
 from urllib.parse import unquote
 
@@ -40,10 +39,6 @@ if TYPE_CHECKING:
     from airflow.sdk.types import Logger
 
 
-REPAIR_WAIT_ATTEMPTS = os.getenv("DATABRICKS_REPAIR_WAIT_ATTEMPTS", 20)
-REPAIR_WAIT_DELAY = os.getenv("DATABRICKS_REPAIR_WAIT_DELAY", 0.5)
-
-
 def get_databricks_task_ids(
     group_id: str, task_map: dict[str, DatabricksTaskBaseOperator], log: Logger
 ) -> list[str]:
@@ -148,7 +143,7 @@ if not AIRFLOW_V_3_0_PLUS:
         if not session:
             raise AirflowException("Session not provided.")
 
-        return session.query(DagRun).filter(DagRun.dag_id == dag.dag_id, DagRun.run_id == run_id).first()
+        return session.query(DagRun).filter(DagRun.dag_id == dag.dag_id, DagRun.run_id == run_id).one()
 
     @provide_session
     def _clear_task_instances(

airflow/providers/databricks/sensors/databricks_partition.py

@@ -189,9 +189,10 @@ class DatabricksPartitionSensor(BaseSensorOperator):
         formatted_opts = ""
         if opts:
             output_list = []
-            for partition_col, partition_value in opts.items():
-                if escape_key:
-                    partition_col = self.escaper.escape_item(partition_col)
+            for partition_col_raw, partition_value in opts.items():
+                partition_col = (
+                    self.escaper.escape_item(partition_col_raw) if escape_key else partition_col_raw
+                )
                 if partition_col in partition_columns:
                     if isinstance(partition_value, list):
                         output_list.append(f"""{partition_col} in {tuple(partition_value)}""")

{apache_airflow_providers_databricks-7.7.4rc1.dist-info → apache_airflow_providers_databricks-7.8.0.dist-info}/METADATA

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.7.4rc1
+Version: 7.8.0
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
 Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
+License-Expression: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Environment :: Web Environment
@@ -14,15 +15,16 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.10.0rc1
-Requires-Dist: apache-airflow-providers-common-compat>=1.8.0rc1
-Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: apache-airflow>=2.11.0
+Requires-Dist: apache-airflow-providers-common-compat>=1.8.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
 Requires-Dist: requests>=2.32.0,<3
 Requires-Dist: databricks-sql-connector>=4.0.0
 Requires-Dist: databricks-sqlalchemy>=1.0.2
@@ -33,13 +35,13 @@ Requires-Dist: pandas>=2.2.3; python_version >="3.13"
 Requires-Dist: pyarrow>=16.1.0; python_version < '3.13'
 Requires-Dist: pyarrow>=18.0.0; python_version >= '3.13'
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
-Requires-Dist: apache-airflow-providers-fab>=2.2.0rc1 ; extra == "fab" and ( python_version < '3.13')
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
+Requires-Dist: apache-airflow-providers-fab>=2.2.0 ; extra == "fab" and ( python_version < '3.13')
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.4
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -75,7 +77,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.7.4``
+Release: ``7.8.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -88,7 +90,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0/>`_.
 
 Installation
 ------------
@@ -105,7 +107,7 @@ Requirements
 ==========================================  ======================================
 PIP package                                 Version required
 ==========================================  ======================================
-``apache-airflow``                          ``>=2.10.0``
+``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-common-compat``  ``>=1.8.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
 ``requests``                                ``>=2.32.0,<3``
@@ -154,5 +156,5 @@ Extra               Dependencies
 ==================  ================================================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.4/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0/changelog.html>`_.
 

{apache_airflow_providers_databricks-7.7.4rc1.dist-info → apache_airflow_providers_databricks-7.8.0.dist-info}/RECORD

@@ -1,22 +1,21 @@
-airflow/providers/databricks/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/databricks/__init__.py,sha256=47IUUh1RLJbiG9at92L1--O2ozh0fap2Swxx9hN7ySs,1499
+airflow/providers/databricks/__init__.py,sha256=jK9hWZ8jptf_y_7T6PywlGiLdB4zq1EsGECyxMGFi8A,1499
 airflow/providers/databricks/exceptions.py,sha256=85RklmLOI_PnTzfXNIUd5fAu2aMMUhelwumQAX0wANE,1261
-airflow/providers/databricks/get_provider_info.py,sha256=NZ-rY6k6ctDZN7rDngN7mAzq7RMhLag5NwfnuBNcKuw,5644
+airflow/providers/databricks/get_provider_info.py,sha256=LfK0AwIARVh4tX5146-J2VRZwfe6GP3xjLyltA7X7iU,5738
 airflow/providers/databricks/version_compat.py,sha256=RQbdCueLOaFZWekpQmF0BoAoJInW8EoyvJ3Ah-HbrPo,1577
 airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/hooks/databricks.py,sha256=rkd1J73Zc8IRnMBapxRHWUvyNAEVyV5j14iBDtRdzFo,29470
-airflow/providers/databricks/hooks/databricks_base.py,sha256=PpXH3LmHpGiWKaxBuInBCV2q7aKwLWs4xyKSx4E-sJ4,36486
-airflow/providers/databricks/hooks/databricks_sql.py,sha256=-oGJxteTW1L7L0MLpiNeucWs3q_k2n0Ax2rLSNGc0F8,17726
+airflow/providers/databricks/hooks/databricks.py,sha256=uOBPUUAEc9eHBdvMgNyJzWivIFCt_GQgR4UlRrRxqgM,29754
+airflow/providers/databricks/hooks/databricks_base.py,sha256=m-m2AKqD3-6mEfvuwgo7Era47zGzsjKbpLTRQNjiUS4,36864
+airflow/providers/databricks/hooks/databricks_sql.py,sha256=xougOWuFgQzhBzFcuYkbX-lo0FpKCQztXoBETJEzesg,17755
 airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/databricks/operators/databricks.py,sha256=bVhFNTeGboHkmaJkYDYEyF0V1QUOB_RnsvwaCuEtIew,79316
 airflow/providers/databricks/operators/databricks_repos.py,sha256=VRZye45ZMlDxti6ZJjuouox5umiMoeQ-BKugPpE7jnM,13155
 airflow/providers/databricks/operators/databricks_sql.py,sha256=gwpkr660qpk4dUve98RB-hniaMzuXL6znQZZGilJxi0,21842
 airflow/providers/databricks/operators/databricks_workflow.py,sha256=QLsR0pGLWvvQbutsjj4RWwBE-z6tkWiYLHj6waMv8ZE,15019
 airflow/providers/databricks/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/databricks/plugins/databricks_workflow.py,sha256=f0p-PefGpkrbJcmcG2_dGjqGwIhILG5sXBAxclWOJM8,20173
+airflow/providers/databricks/plugins/databricks_workflow.py,sha256=5vyG2WNM25ptSv5IwAndUTqKAOmTneOWy_pAtqBKcgc,20020
 airflow/providers/databricks/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/databricks/sensors/databricks.py,sha256=RrjSzncvppdp5U8RYHd975MCIQIb_s1VQoxm9Aqbvac,6262
-airflow/providers/databricks/sensors/databricks_partition.py,sha256=FfhSFgxjJapiOhg50UpjczAxQGEv64gD_q0IlrjeUtY,9890
+airflow/providers/databricks/sensors/databricks_partition.py,sha256=qPDy8oxg-Lo-jnHy1EbxmA5GIjC6t0XnFJ1E3aAmUgg,9940
 airflow/providers/databricks/sensors/databricks_sql.py,sha256=shq7ng4LCiaD4Q7lorm4g1A7aijmq3nVUnCFlYtoI7c,5633
 airflow/providers/databricks/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/databricks/triggers/databricks.py,sha256=DQbXLw1W_e3Iw-hsDph7vPuHc2caj623V7WmA2_PftM,8672
@@ -24,7 +23,9 @@ airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2
 airflow/providers/databricks/utils/databricks.py,sha256=ecvzZbC4KdXds47VeSayot9EO-RQnTRJTEwKITH7waQ,5117
 airflow/providers/databricks/utils/mixins.py,sha256=WUmkt3AmXalmV6zOUIJZWbTldxYunAZOstddDhKCC94,7407
 airflow/providers/databricks/utils/openlineage.py,sha256=1jT5Woh9YifawdP-VFWsabfF-ecuCjPlzD5P_W4DAhI,15078
-apache_airflow_providers_databricks-7.7.4rc1.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
-apache_airflow_providers_databricks-7.7.4rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_databricks-7.7.4rc1.dist-info/METADATA,sha256=lbuHUvAn5Skwkypk_EDFdgEVPxroNlt4SkT_RTYUnfs,7771
-apache_airflow_providers_databricks-7.7.4rc1.dist-info/RECORD,,
+apache_airflow_providers_databricks-7.8.0.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
+apache_airflow_providers_databricks-7.8.0.dist-info/licenses/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+apache_airflow_providers_databricks-7.8.0.dist-info/licenses/NOTICE,sha256=E3-_E02gwwSEFzeeWPKmnIjOoos3hW28CLISV6sYrbQ,168
+apache_airflow_providers_databricks-7.8.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_databricks-7.8.0.dist-info/METADATA,sha256=ag4y8wdimVeG_xpU6SGjicRyOsUe66vk2zyUTQ8h9c0,7750
+apache_airflow_providers_databricks-7.8.0.dist-info/RECORD,,

apache_airflow_providers_databricks-7.8.0.dist-info/licenses/NOTICE

@@ -0,0 +1,5 @@
+Apache Airflow
+Copyright 2016-2025 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).