apache-airflow-providers-databricks 5.1.0rc2__py3-none-any.whl → 6.0.0rc1__py3-none-any.whl

airflow/providers/databricks/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "5.1.0"
+__version__ = "6.0.0"
 
 try:
     from airflow import __version__ as airflow_version

airflow/providers/databricks/get_provider_info.py

@@ -28,9 +28,9 @@ def get_provider_info():
         "name": "Databricks",
         "description": "`Databricks <https://databricks.com/>`__\n",
         "suspended": False,
-        "source-date-epoch": 1701983371,
+        "source-date-epoch": 1703288125,
         "versions": [
-            "5.1.0",
+            "6.0.0",
             "5.0.1",
             "5.0.0",
             "4.7.0",
@@ -65,7 +65,7 @@ def get_provider_info():
         ],
         "dependencies": [
             "apache-airflow>=2.6.0",
-            "apache-airflow-providers-common-sql>=1.8.1",
+            "apache-airflow-providers-common-sql>=1.10.0",
             "requests>=2.27,<3",
             "databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0",
             "aiohttp>=3.6.3, <4",

airflow/providers/databricks/hooks/databricks.py

@@ -55,7 +55,7 @@ INSTALL_LIBS_ENDPOINT = ("POST", "api/2.0/libraries/install")
 UNINSTALL_LIBS_ENDPOINT = ("POST", "api/2.0/libraries/uninstall")
 
 LIST_JOBS_ENDPOINT = ("GET", "api/2.1/jobs/list")
-LIST_PIPELINES_ENDPOINT = ("GET", "/api/2.0/pipelines")
+LIST_PIPELINES_ENDPOINT = ("GET", "api/2.0/pipelines")
 
 WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "api/2.0/workspace/get-status")
 
@@ -322,8 +322,8 @@ class DatabricksHook(BaseDatabricksHook):
             payload["filter"] = filter
 
         while has_more:
-            if next_token:
-                payload["page_token"] = next_token
+            if next_token is not None:
+                payload = {**payload, "page_token": next_token}
             response = self._do_api_call(LIST_PIPELINES_ENDPOINT, payload)
             pipelines = response.get("statuses", [])
             all_pipelines += pipelines
@@ -345,11 +345,11 @@ class DatabricksHook(BaseDatabricksHook):
 
         if len(matching_pipelines) > 1:
             raise AirflowException(
-                f"There are more than one job with name {pipeline_name}. "
+                f"There are more than one pipelines with name {pipeline_name}. "
                 "Please delete duplicated pipelines first"
             )
 
-        if not pipeline_name:
+        if not pipeline_name or len(matching_pipelines) == 0:
             return None
         else:
             return matching_pipelines[0]["pipeline_id"]

airflow/providers/databricks/hooks/databricks_sql.py

@@ -16,19 +16,32 @@
 # under the License.
 from __future__ import annotations
 
+import warnings
+from collections import namedtuple
 from contextlib import closing
 from copy import copy
-from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, TypeVar, overload
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Sequence,
+    TypeVar,
+    cast,
+    overload,
+)
 
 from databricks import sql  # type: ignore[attr-defined]
-from databricks.sql.types import Row
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results
 from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook
 
 if TYPE_CHECKING:
     from databricks.sql.client import Connection
+    from databricks.sql.types import Row
 
 LIST_SQL_ENDPOINTS_ENDPOINT = ("GET", "api/2.0/sql/endpoints")
 
@@ -52,6 +65,10 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         on every request
     :param catalog: An optional initial catalog to use. Requires DBR version 9.0+
     :param schema: An optional initial schema to use. Requires DBR version 9.0+
+    :param return_tuple: Return a ``namedtuple`` object instead of a ``databricks.sql.Row`` object. Default
+        to False. In a future release of the provider, this will become True by default. This parameter
+        ensures backward-compatibility during the transition phase to common tuple objects for all hooks based
+        on DbApiHook. This flag will also be removed in a future release.
     :param kwargs: Additional parameters internal to Databricks SQL Connector parameters
     """
 
@@ -68,6 +85,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         catalog: str | None = None,
         schema: str | None = None,
         caller: str = "DatabricksSqlHook",
+        return_tuple: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(databricks_conn_id, caller=caller)
@@ -80,8 +98,18 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         self.http_headers = http_headers
         self.catalog = catalog
         self.schema = schema
+        self.return_tuple = return_tuple
         self.additional_params = kwargs
 
+        if not self.return_tuple:
+            warnings.warn(
+                """Returning a raw `databricks.sql.Row` object is deprecated. A namedtuple will be
+                returned instead in a future release of the databricks provider. Set `return_tuple=True` to
+                enable this behavior.""",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
+
     def _get_extra_config(self) -> dict[str, Any | None]:
         extra_params = copy(self.databricks_conn.extra_dejson)
         for arg in ["http_path", "session_configuration", *self.extra_parameters]:
@@ -167,7 +195,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         handler: Callable[[Any], T] = ...,
         split_statements: bool = ...,
         return_last: bool = ...,
-    ) -> T | list[T]:
+    ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None:
         ...
 
     def run(
@@ -178,7 +206,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         handler: Callable[[Any], T] | None = None,
         split_statements: bool = True,
         return_last: bool = True,
-    ) -> T | list[T] | None:
+    ) -> tuple | list[tuple] | list[list[tuple] | tuple] | None:
         """
         Run a command or a list of commands.
 
@@ -223,7 +251,12 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
                 with closing(conn.cursor()) as cur:
                     self._run_command(cur, sql_statement, parameters)
                     if handler is not None:
-                        result = self._make_serializable(handler(cur))
+                        raw_result = handler(cur)
+                        if self.return_tuple:
+                            result = self._make_common_data_structure(raw_result)
+                        else:
+                            # Returning raw result is deprecated, and do not comply with current common.sql interface
+                            result = raw_result  # type: ignore[assignment]
                         if return_single_query_results(sql, return_last, split_statements):
                             results = [result]
                             self.descriptions = [cur.description]
@@ -241,14 +274,20 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         else:
             return results
 
-    @staticmethod
-    def _make_serializable(result):
-        """Transform the databricks Row objects into JSON-serializable lists."""
+    def _make_common_data_structure(self, result: Sequence[Row] | Row) -> list[tuple] | tuple:
+        """Transform the databricks Row objects into namedtuple."""
+        # Below ignored lines respect namedtuple docstring, but mypy do not support dynamically
+        # instantiated namedtuple, and will never do: https://github.com/python/mypy/issues/848
         if isinstance(result, list):
-            return [list(row) for row in result]
-        elif isinstance(result, Row):
-            return list(result)
-        return result
+            rows: list[Row] = result
+            rows_fields = rows[0].__fields__
+            rows_object = namedtuple("Row", rows_fields)  # type: ignore[misc]
+            return cast(List[tuple], [rows_object(*row) for row in rows])
+        else:
+            row: Row = result
+            row_fields = row.__fields__
+            row_object = namedtuple("Row", row_fields)  # type: ignore[misc]
+            return cast(tuple, row_object(*row))
 
     def bulk_dump(self, table, tmp_file):
         raise NotImplementedError()

airflow/providers/databricks/operators/databricks.py

@@ -521,7 +521,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
         ):
             # If pipeline_id is not provided, we need to fetch it from the pipeline_name
             pipeline_name = self.json["pipeline_task"]["pipeline_name"]
-            self.json["pipeline_task"]["pipeline_id"] = self._hook.get_pipeline_id(pipeline_name)
+            self.json["pipeline_task"]["pipeline_id"] = self._hook.find_pipeline_id_by_name(pipeline_name)
             del self.json["pipeline_task"]["pipeline_name"]
         json_normalised = normalise_json_content(self.json)
         self.run_id = self._hook.submit_run(json_normalised)
@@ -580,26 +580,20 @@ class DatabricksRunNowOperator(BaseOperator):
     For example ::
 
         json = {
-          "job_id": 42,
-          "notebook_params": {
-            "dry-run": "true",
-            "oldest-time-to-consider": "1457570074236"
-          }
+            "job_id": 42,
+            "notebook_params": {"dry-run": "true", "oldest-time-to-consider": "1457570074236"},
         }
 
-        notebook_run = DatabricksRunNowOperator(task_id='notebook_run', json=json)
+        notebook_run = DatabricksRunNowOperator(task_id="notebook_run", json=json)
 
     Another way to accomplish the same thing is to use the named parameters
     of the ``DatabricksRunNowOperator`` directly. Note that there is exactly
     one named parameter for each top level parameter in the ``run-now``
     endpoint. In this method, your code would look like this: ::
 
-        job_id=42
+        job_id = 42
 
-        notebook_params = {
-            "dry-run": "true",
-            "oldest-time-to-consider": "1457570074236"
-        }
+        notebook_params = {"dry-run": "true", "oldest-time-to-consider": "1457570074236"}
 
         python_params = ["douglas adams", "42"]
 
@@ -612,7 +606,7 @@ class DatabricksRunNowOperator(BaseOperator):
             notebook_params=notebook_params,
             python_params=python_params,
             jar_params=jar_params,
-            spark_submit_params=spark_submit_params
+            spark_submit_params=spark_submit_params,
         )
 
     In the case where both the json parameter **AND** the named parameters

airflow/providers/databricks/operators/databricks_sql.py

@@ -113,6 +113,7 @@ class DatabricksSqlOperator(SQLExecuteQueryOperator):
             "catalog": self.catalog,
             "schema": self.schema,
             "caller": "DatabricksSqlOperator",
+            "return_tuple": True,
             **self.client_parameters,
             **self.hook_params,
         }

{apache_airflow_providers_databricks-5.1.0rc2.dist-info → apache_airflow_providers_databricks-6.0.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-databricks
-Version: 5.1.0rc2
+Version: 6.0.0rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,15 +21,15 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: aiohttp>=3.6.3, <4
-Requires-Dist: apache-airflow-providers-common-sql>=1.8.1.dev0
+Requires-Dist: apache-airflow-providers-common-sql>=1.10.0.dev0
 Requires-Dist: apache-airflow>=2.6.0.dev0
 Requires-Dist: databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0
 Requires-Dist: requests>=2.27,<3
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/5.1.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/5.1.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.0.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.0.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -81,7 +81,7 @@ Provides-Extra: sdk
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``5.1.0.rc2``
+Release: ``6.0.0.rc1``
 
 
 `Databricks <https://databricks.com/>`__
@@ -94,7 +94,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/5.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.0.0/>`_.
 
 Installation
 ------------
@@ -112,7 +112,7 @@ Requirements
 PIP package                              Version required
 =======================================  ==========================
 ``apache-airflow``                       ``>=2.6.0``
-``apache-airflow-providers-common-sql``  ``>=1.8.1``
+``apache-airflow-providers-common-sql``  ``>=1.10.0``
 ``requests``                             ``>=2.27,<3``
 ``databricks-sql-connector``             ``>=2.0.0,!=2.9.0,<3.0.0``
 ``aiohttp``                              ``>=3.6.3,<4``
@@ -138,4 +138,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/5.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.0.0/changelog.html>`_.

{apache_airflow_providers_databricks-5.1.0rc2.dist-info → apache_airflow_providers_databricks-6.0.0rc1.dist-info}/RECORD

@@ -1,14 +1,14 @@
 airflow/providers/databricks/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
-airflow/providers/databricks/__init__.py,sha256=JsaG1QZ78eKQBM9TcObkVPwxFuVOGUUkNq8GW0V6rYA,1585
-airflow/providers/databricks/get_provider_info.py,sha256=Fv4b-vDzk-i2J31xahNNm4cvVzVUkz10jGF7iJSecuc,6104
+airflow/providers/databricks/__init__.py,sha256=UcvoLm-UfmmEOWyZjjBQ8rf5a5aUs9Uoge2HmRJKwi0,1585
+airflow/providers/databricks/get_provider_info.py,sha256=tCIb8m8RY9FOo5YKewgLcpntbi4SWCNppvrx3qBycLo,6105
 airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/hooks/databricks.py,sha256=wq7uiSU_WbBY8yrlI2Zm6l12t9YP0MgbBMeHe0zuWe4,22542
+airflow/providers/databricks/hooks/databricks.py,sha256=GeqiBYFsFIwK4vgBTxde-bbdDliWzQAEbvZcShbyuio,22604
 airflow/providers/databricks/hooks/databricks_base.py,sha256=nX-40P0XKZyFyahkbbImvWmoCQ4ic35viAYlFIupUHE,30591
-airflow/providers/databricks/hooks/databricks_sql.py,sha256=PVyN2VYnJR2kmPUJdxeFCChPR-B3R0kpUDDKh4slvOA,10474
+airflow/providers/databricks/hooks/databricks_sql.py,sha256=afzw9l1VNBFvReEheAUSrGgXc7FWePcwuYw9-qMLJUg,12445
 airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/operators/databricks.py,sha256=afG7vCIT8_JwTK7k1gXlgQDTsJj9JichlC2bP9XmHg0,40065
+airflow/providers/databricks/operators/databricks.py,sha256=XmQuoWzesOYpwAtxn3aB-1CihbnT8W67WoxUoAh0Bnc,40012
 airflow/providers/databricks/operators/databricks_repos.py,sha256=NUxa0jvvmK16CDKb-7Tbs3wF9XoFi1AVJlKxlsE3r4k,13092
-airflow/providers/databricks/operators/databricks_sql.py,sha256=UlO9vtAJnThZq0hIvzDYbeCqQWqseeDnA1-UMSPQQnc,16772
+airflow/providers/databricks/operators/databricks_sql.py,sha256=C_XqR5SN_g2t0XXlH248nPAC7BcYB0P1NAsPJT5qcEg,16806
 airflow/providers/databricks/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/databricks/sensors/databricks_partition.py,sha256=TRZVjGEYzIbN4nZDPJEVavXdcegpyj0BVKXINMRFlCM,10605
 airflow/providers/databricks/sensors/databricks_sql.py,sha256=8qSfbzpWgU2_oZU9pS9SA_MSYhyIHgYZjTYfDkDH84Y,5771
@@ -16,7 +16,7 @@ airflow/providers/databricks/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvR
 airflow/providers/databricks/triggers/databricks.py,sha256=Qj9mB0bNYRY_toPEU17gxbxmPkkT3P789kCHu_T64BA,3997
 airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/databricks/utils/databricks.py,sha256=iRzRHvdFETGiFxZccOjfC8NGgDofMfP35Tqp3M5CGr0,2880
-apache_airflow_providers_databricks-5.1.0rc2.dist-info/entry_points.txt,sha256=1WxGXTFDb107eV5Zmrt3p12J4LHYk56-ZKlvpOK7vg4,106
-apache_airflow_providers_databricks-5.1.0rc2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_databricks-5.1.0rc2.dist-info/METADATA,sha256=ehi83rkKMnM6yuXZRDHumEbmc6OYB4rSOGoGvT36M68,6442
-apache_airflow_providers_databricks-5.1.0rc2.dist-info/RECORD,,
+apache_airflow_providers_databricks-6.0.0rc1.dist-info/entry_points.txt,sha256=1WxGXTFDb107eV5Zmrt3p12J4LHYk56-ZKlvpOK7vg4,106
+apache_airflow_providers_databricks-6.0.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_databricks-6.0.0rc1.dist-info/METADATA,sha256=Sd37-KPVkHCMIBl1KPqZAE9xUslg0pKMojaZ6SibGog,6444
+apache_airflow_providers_databricks-6.0.0rc1.dist-info/RECORD,,