apache-airflow-providers-databricks 6.10.0rc1__tar.gz → 6.11.0rc1__tar.gz

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-databricks
-Version: 6.10.0rc1
+Version: 6.11.0rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.8
+Requires-Python: ~=3.9
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,7 +15,6 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -34,8 +33,8 @@ Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -88,7 +87,7 @@ Provides-Extra: sdk
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``6.10.0.rc1``
+Release: ``6.11.0.rc1``
 
 
 `Databricks <https://databricks.com/>`__
@@ -101,7 +100,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0/>`_.
 
 Installation
 ------------
@@ -110,7 +109,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -149,4 +148,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0/changelog.html>`_.

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``6.10.0.rc1``
+Release: ``6.11.0.rc1``
 
 
 `Databricks <https://databricks.com/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0/>`_.
 
 Installation
 ------------
@@ -64,7 +64,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -103,4 +103,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0/changelog.html>`_.

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/airflow/providers/databricks/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "6.10.0"
+__version__ = "6.11.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/airflow/providers/databricks/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Databricks",
         "description": "`Databricks <https://databricks.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1726860611,
+        "source-date-epoch": 1728485027,
         "versions": [
+            "6.11.0",
             "6.10.0",
             "6.9.0",
             "6.8.0",

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/airflow/providers/databricks/hooks/databricks_base.py

@@ -561,6 +561,13 @@ class BaseDatabricksHook(BaseHook):
         try:
             for attempt in self._get_retry_object():
                 with attempt:
+                    self.log.debug(
+                        "Initiating %s request to %s with payload: %s, headers: %s",
+                        method,
+                        url,
+                        json,
+                        headers,
+                    )
                     response = request_func(
                         url,
                         json=json if method in ("POST", "PATCH") else None,
@@ -569,6 +576,8 @@ class BaseDatabricksHook(BaseHook):
                         headers=headers,
                         timeout=self.timeout_seconds,
                     )
+                    self.log.debug("Response Status Code: %s", response.status_code)
+                    self.log.debug("Response text: %s", response.text)
                     response.raise_for_status()
                     return response.json()
         except RetryError:
@@ -615,6 +624,13 @@ class BaseDatabricksHook(BaseHook):
         try:
             async for attempt in self._a_get_retry_object():
                 with attempt:
+                    self.log.debug(
+                        "Initiating %s request to %s with payload: %s, headers: %s",
+                        method,
+                        url,
+                        json,
+                        headers,
+                    )
                     async with request_func(
                         url,
                         json=json,
@@ -622,6 +638,8 @@ class BaseDatabricksHook(BaseHook):
                         headers={**headers, **self.user_agent_header},
                         timeout=self.timeout_seconds,
                     ) as response:
+                        self.log.debug("Response Status Code: %s", response.status_code)
+                        self.log.debug("Response text: %s", response.text)
                         response.raise_for_status()
                         return await response.json()
         except RetryError:

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/airflow/providers/databricks/operators/databricks.py

@@ -1045,7 +1045,15 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
 
     def _get_databricks_task_id(self, task_id: str) -> str:
         """Get the databricks task ID using dag_id and task_id. Removes illegal characters."""
-        return f"{self.dag_id}__{task_id.replace('.', '__')}"
+        task_id = f"{self.dag_id}__{task_id.replace('.', '__')}"
+        if len(task_id) > 100:
+            self.log.warning(
+                "The generated task_key '%s' exceeds 100 characters and will be truncated by the Databricks API. "
+                "This will cause failure when trying to monitor the task. task_key is generated by ",
+                "concatenating dag_id and task_id.",
+                task_id,
+            )
+        return task_id
 
     @property
     def _databricks_workflow_task_group(self) -> DatabricksWorkflowTaskGroup | None:

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/airflow/providers/databricks/operators/databricks_workflow.py

@@ -52,7 +52,7 @@ class WorkflowRunMetadata:
     """
 
     conn_id: str
-    job_id: str
+    job_id: int
     run_id: int
 
 
@@ -116,6 +116,7 @@ class _CreateDatabricksWorkflowOperator(BaseOperator):
         self.notebook_params = notebook_params or {}
         self.tasks_to_convert = tasks_to_convert or []
         self.relevant_upstreams = [task_id]
+        self.workflow_run_metadata: WorkflowRunMetadata | None = None
         super().__init__(task_id=task_id, **kwargs)
 
     def _get_hook(self, caller: str) -> DatabricksHook:
@@ -212,12 +213,36 @@ class _CreateDatabricksWorkflowOperator(BaseOperator):
 
         self._wait_for_job_to_start(run_id)
 
+        self.workflow_run_metadata = WorkflowRunMetadata(
+            self.databricks_conn_id,
+            job_id,
+            run_id,
+        )
+
         return {
             "conn_id": self.databricks_conn_id,
             "job_id": job_id,
             "run_id": run_id,
         }
 
+    def on_kill(self) -> None:
+        if self.workflow_run_metadata:
+            run_id = self.workflow_run_metadata.run_id
+            job_id = self.workflow_run_metadata.job_id
+
+            self._hook.cancel_run(run_id)
+            self.log.info(
+                "Run: %(run_id)s of job_id: %(job_id)s was requested to be cancelled.",
+                {"run_id": run_id, "job_id": job_id},
+            )
+        else:
+            self.log.error(
+                """
+                Error: Workflow Run metadata is not populated, so the run was not canceled. This could be due
+                to the workflow not being started or an error in the workflow creation process.
+                """
+            )
+
 
 class DatabricksWorkflowTaskGroup(TaskGroup):
     """

{apache_airflow_providers_databricks-6.10.0rc1 → apache_airflow_providers_databricks-6.11.0rc1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-databricks"
-version = "6.10.0.rc1"
+version = "6.11.0.rc1"
 description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -47,14 +47,13 @@ classifiers = [
     "Framework :: Apache Airflow",
     "Framework :: Apache Airflow :: Provider",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
 ]
-requires-python = "~=3.8"
+requires-python = "~=3.9"
 dependencies = [
     "aiohttp>=3.9.2, <4",
     "apache-airflow-providers-common-sql>=1.10.0rc0",
@@ -68,8 +67,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.10.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/6.11.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"