apache-airflow-providers-databricks 6.1.0rc1__tar.gz → 6.2.0__tar.gz

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-databricks
-Version: 6.1.0rc1
+Version: 6.2.0
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,16 +20,16 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: aiohttp>=3.6.3, <4
-Requires-Dist: apache-airflow-providers-common-sql>=1.10.0.dev0
-Requires-Dist: apache-airflow>=2.6.0.dev0
+Requires-Dist: aiohttp>=3.9.2, <4
+Requires-Dist: apache-airflow-providers-common-sql>=1.10.0
+Requires-Dist: apache-airflow>=2.6.0
 Requires-Dist: databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0
-Requires-Dist: requests>=2.27,<3
+Requires-Dist: requests>=2.27.0,<3
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -81,7 +81,7 @@ Provides-Extra: sdk
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``6.1.0.rc1``
+Release: ``6.2.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -94,7 +94,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0/>`_.
 
 Installation
 ------------
@@ -113,9 +113,9 @@ PIP package                              Version required
 =======================================  ==========================
 ``apache-airflow``                       ``>=2.6.0``
 ``apache-airflow-providers-common-sql``  ``>=1.10.0``
-``requests``                             ``>=2.27,<3``
+``requests``                             ``>=2.27.0,<3``
 ``databricks-sql-connector``             ``>=2.0.0,!=2.9.0,<3.0.0``
-``aiohttp``                              ``>=3.6.3,<4``
+``aiohttp``                              ``>=3.9.2,<4``
 =======================================  ==========================
 
 Cross provider package dependencies
@@ -138,4 +138,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0/changelog.html>`_.

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``6.1.0.rc1``
+Release: ``6.2.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0/>`_.
 
 Installation
 ------------
@@ -74,9 +74,9 @@ PIP package                              Version required
 =======================================  ==========================
 ``apache-airflow``                       ``>=2.6.0``
 ``apache-airflow-providers-common-sql``  ``>=1.10.0``
-``requests``                             ``>=2.27,<3``
+``requests``                             ``>=2.27.0,<3``
 ``databricks-sql-connector``             ``>=2.0.0,!=2.9.0,<3.0.0``
-``aiohttp``                              ``>=3.6.3,<4``
+``aiohttp``                              ``>=3.9.2,<4``
 =======================================  ==========================
 
 Cross provider package dependencies
@@ -99,4 +99,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0/changelog.html>`_.

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/airflow/providers/databricks/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "6.1.0"
+__version__ = "6.2.0"
 
 try:
     from airflow import __version__ as airflow_version

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/airflow/providers/databricks/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Databricks",
         "description": "`Databricks <https://databricks.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1705912006,
+        "source-date-epoch": 1707636345,
         "versions": [
+            "6.2.0",
             "6.1.0",
             "6.0.0",
             "5.0.1",
@@ -67,9 +68,9 @@ def get_provider_info():
         "dependencies": [
             "apache-airflow>=2.6.0",
             "apache-airflow-providers-common-sql>=1.10.0",
-            "requests>=2.27,<3",
+            "requests>=2.27.0,<3",
             "databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0",
-            "aiohttp>=3.6.3, <4",
+            "aiohttp>=3.9.2, <4",
         ],
         "additional-extras": [
             {

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/airflow/providers/databricks/hooks/databricks_base.py

@@ -103,9 +103,8 @@ class BaseDatabricksHook(BaseHook):
         retry_delay: float = 1.0,
         retry_args: dict[Any, Any] | None = None,
         caller: str = "Unknown",
-        **kwargs,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__()
         self.databricks_conn_id = databricks_conn_id
         self.timeout_seconds = timeout_seconds
         if retry_limit < 1:

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/airflow/providers/databricks/hooks/databricks_sql.py

@@ -282,13 +282,13 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
             rows: list[Row] = result
             if not rows:
                 return []
-            rows_fields = rows[0].__fields__
-            rows_object = namedtuple("Row", rows_fields)  # type: ignore[misc]
+            rows_fields = tuple(rows[0].__fields__)
+            rows_object = namedtuple("Row", rows_fields, rename=True)  # type: ignore
             return cast(List[tuple], [rows_object(*row) for row in rows])
         else:
             row: Row = result
-            row_fields = row.__fields__
-            row_object = namedtuple("Row", row_fields)  # type: ignore[misc]
+            row_fields = tuple(row.__fields__)
+            row_object = namedtuple("Row", row_fields, rename=True)  # type: ignore
             return cast(tuple, row_object(*row))
 
     def bulk_dump(self, table, tmp_file):

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/airflow/providers/databricks/operators/databricks.py

@@ -19,11 +19,12 @@
 from __future__ import annotations
 
 import time
-import warnings
 from functools import cached_property
 from logging import Logger
 from typing import TYPE_CHECKING, Any, Sequence
 
+from deprecated import deprecated
+
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.models import BaseOperator, BaseOperatorLink, XCom
@@ -132,18 +133,24 @@ def _handle_deferrable_databricks_operator_execution(operator, hook, log, contex
     log.info("View run status, Spark UI, and logs at %s", run_page_url)
 
     if operator.wait_for_termination:
-        operator.defer(
-            trigger=DatabricksExecutionTrigger(
-                run_id=operator.run_id,
-                databricks_conn_id=operator.databricks_conn_id,
-                polling_period_seconds=operator.polling_period_seconds,
-                retry_limit=operator.databricks_retry_limit,
-                retry_delay=operator.databricks_retry_delay,
-                retry_args=operator.databricks_retry_args,
-                run_page_url=run_page_url,
-            ),
-            method_name=DEFER_METHOD_NAME,
-        )
+        run_info = hook.get_run(operator.run_id)
+        run_state = RunState(**run_info["state"])
+        if not run_state.is_terminal:
+            operator.defer(
+                trigger=DatabricksExecutionTrigger(
+                    run_id=operator.run_id,
+                    databricks_conn_id=operator.databricks_conn_id,
+                    polling_period_seconds=operator.polling_period_seconds,
+                    retry_limit=operator.databricks_retry_limit,
+                    retry_delay=operator.databricks_retry_delay,
+                    retry_args=operator.databricks_retry_args,
+                    run_page_url=run_page_url,
+                ),
+                method_name=DEFER_METHOD_NAME,
+            )
+        else:
+            if run_state.is_successful:
+                log.info("%s completed successfully.", operator.task_id)
 
 
 def _handle_deferrable_databricks_operator_completion(event: dict, log: Logger) -> None:
@@ -556,17 +563,18 @@ class DatabricksSubmitRunOperator(BaseOperator):
         _handle_deferrable_databricks_operator_completion(event, self.log)
 
 
+@deprecated(
+    reason=(
+        "`DatabricksSubmitRunDeferrableOperator` has been deprecated. "
+        "Please use `airflow.providers.databricks.operators.DatabricksSubmitRunOperator` "
+        "with `deferrable=True` instead."
+    ),
+    category=AirflowProviderDeprecationWarning,
+)
 class DatabricksSubmitRunDeferrableOperator(DatabricksSubmitRunOperator):
     """Deferrable version of ``DatabricksSubmitRunOperator``."""
 
     def __init__(self, *args, **kwargs):
-        warnings.warn(
-            "`DatabricksSubmitRunDeferrableOperator` has been deprecated. "
-            "Please use `airflow.providers.databricks.operators.DatabricksSubmitRunOperator` with "
-            "`deferrable=True` instead.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
         super().__init__(deferrable=True, *args, **kwargs)
 
     def execute(self, context):
@@ -836,15 +844,16 @@ class DatabricksRunNowOperator(BaseOperator):
             self.log.error("Error: Task: %s with invalid run_id was requested to be cancelled.", self.task_id)
 
 
+@deprecated(
+    reason=(
+        "`DatabricksRunNowDeferrableOperator` has been deprecated. "
+        "Please use `airflow.providers.databricks.operators.DatabricksRunNowOperator` "
+        "with `deferrable=True` instead."
+    ),
+    category=AirflowProviderDeprecationWarning,
+)
 class DatabricksRunNowDeferrableOperator(DatabricksRunNowOperator):
     """Deferrable version of ``DatabricksRunNowOperator``."""
 
     def __init__(self, *args, **kwargs):
-        warnings.warn(
-            "`DatabricksRunNowDeferrableOperator` has been deprecated. "
-            "Please use `airflow.providers.databricks.operators.DatabricksRunNowOperator` with "
-            "`deferrable=True` instead.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
         super().__init__(deferrable=True, *args, **kwargs)

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/airflow/providers/databricks/operators/databricks_sql.py

@@ -146,14 +146,14 @@ class DatabricksSqlOperator(SQLExecuteQueryOperator):
                 if write_header:
                     writer.writeheader()
                 for row in last_results:
-                    writer.writerow(row.asDict())
+                    writer.writerow(row._asdict())
         elif self._output_format.lower() == "json":
             with open(self._output_path, "w") as file:
-                file.write(json.dumps([row.asDict() for row in last_results]))
+                file.write(json.dumps([row._asdict() for row in last_results]))
         elif self._output_format.lower() == "jsonl":
             with open(self._output_path, "w") as file:
                 for row in last_results:
-                    file.write(json.dumps(row.asDict()))
+                    file.write(json.dumps(row._asdict()))
                     file.write("\n")
         else:
             raise AirflowException(f"Unsupported output format: '{self._output_format}'")

{apache_airflow_providers_databricks-6.1.0rc1 → apache_airflow_providers_databricks-6.2.0}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-databricks"
-version = "6.1.0.rc1"
+version = "6.2.0"
 description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -55,16 +55,16 @@ classifiers = [
 ]
 requires-python = "~=3.8"
 dependencies = [
-    "aiohttp>=3.6.3, <4",
-    "apache-airflow-providers-common-sql>=1.10.0.dev0",
-    "apache-airflow>=2.6.0.dev0",
+    "aiohttp>=3.9.2, <4",
+    "apache-airflow-providers-common-sql>=1.10.0",
+    "apache-airflow>=2.6.0",
     "databricks-sql-connector>=2.0.0, <3.0.0, !=2.9.0",
-    "requests>=2.27,<3",
+    "requests>=2.27.0,<3",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.1.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.2.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"