apache-airflow-providers-databricks 7.7.1rc1__py3-none-any.whl → 7.7.2rc1__py3-none-any.whl

airflow/providers/databricks/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.7.1"
+__version__ = "7.7.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

airflow/providers/databricks/hooks/databricks.py

@@ -298,6 +298,16 @@ class DatabricksHook(BaseDatabricksHook):
 
         :param json: The data used in the new_settings of the request to the ``reset`` endpoint.
         """
+        access_control_list = json.get("access_control_list", None)
+        if access_control_list:
+            self.log.info(
+                "Updating job permission for Databricks workflow job id %s with access_control_list %s",
+                job_id,
+                access_control_list,
+            )
+            acl_json = {"access_control_list": access_control_list}
+            self.update_job_permission(job_id=int(job_id), json=acl_json)
+
         self._do_api_call(RESET_ENDPOINT, {"job_id": job_id, "new_settings": json})
 
     def update_job(self, job_id: str, json: dict) -> None:

airflow/providers/databricks/hooks/databricks_sql.py

@@ -179,7 +179,6 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
 
         :return: the extracted sqlalchemy.engine.URL object.
         """
-        conn = self.get_conn()
         url_query = {
             "http_path": self._http_path,
             "catalog": self.catalog,
@@ -189,9 +188,8 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         return URL.create(
             drivername="databricks",
             username="token",
-            password=conn.password,
-            host=conn.host,
-            port=conn.port,
+            password=self._get_token(raise_error=True),
+            host=self.host,
             query=url_query,
         )
 

airflow/providers/databricks/operators/databricks.py

@@ -398,10 +398,6 @@ class DatabricksCreateJobsOperator(BaseOperator):
         if job_id is None:
             return self._hook.create_job(self.json)
         self._hook.reset_job(str(job_id), self.json)
-        if (access_control_list := self.json.get("access_control_list")) is not None:
-            acl_json = {"access_control_list": access_control_list}
-            self._hook.update_job_permission(job_id, normalise_json_content(acl_json))
-
         return job_id
 
 

airflow/providers/databricks/plugins/databricks_workflow.py

@@ -27,10 +27,8 @@ from flask_appbuilder import BaseView
 from flask_appbuilder.api import expose
 
 from airflow.exceptions import AirflowException, TaskInstanceNotFound
-from airflow.models import DagBag
-from airflow.models.dag import DAG, clear_task_instances
 from airflow.models.dagrun import DagRun
-from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
+from airflow.models.taskinstance import TaskInstance, TaskInstanceKey, clear_task_instances
 from airflow.plugins_manager import AirflowPlugin
 from airflow.providers.databricks.hooks.databricks import DatabricksHook
 from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink, TaskGroup, XCom
@@ -90,8 +88,15 @@ def get_databricks_task_ids(
 if not AIRFLOW_V_3_0_PLUS:
     from airflow.utils.session import NEW_SESSION, provide_session
 
-    @provide_session
-    def _get_dagrun(dag: DAG, run_id: str, session: Session | None = None) -> DagRun:
+    def _get_dag(dag_id: str, session: Session):
+        from airflow.models.serialized_dag import SerializedDagModel
+
+        dag = SerializedDagModel.get_dag(dag_id, session=session)
+        if not dag:
+            raise AirflowException("Dag not found.")
+        return dag
+
+    def _get_dagrun(dag, run_id: str, session: Session) -> DagRun:
         """
         Retrieve the DagRun object associated with the specified DAG and run_id.
 
@@ -107,10 +112,9 @@ if not AIRFLOW_V_3_0_PLUS:
 
     @provide_session
     def _clear_task_instances(
-        dag_id: str, run_id: str, task_ids: list[str], log: logging.Logger, session: Session | None = None
+        dag_id: str, run_id: str, task_ids: list[str], log: logging.Logger, session: Session = NEW_SESSION
     ) -> None:
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(dag_id)
+        dag = _get_dag(dag_id, session=session)
         log.debug("task_ids %s to clear", str(task_ids))
         dr: DagRun = _get_dagrun(dag, run_id, session=session)
         tis_to_clear = [ti for ti in dr.get_task_instances() if ti.databricks_task_key in task_ids]
@@ -274,13 +278,8 @@ class WorkflowJobRunLink(BaseOperatorLink, LoggingMixin):
             ti = get_task_instance(operator, dttm)
             ti_key = ti.key
         task_group = operator.task_group
-
         if not task_group:
             raise AirflowException("Task group is required for generating Databricks Workflow Job Run Link.")
-
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(ti_key.dag_id)
-        dag.get_task(ti_key.task_id)
         self.log.info("Getting link for task %s", ti_key.task_id)
         if ".launch" not in ti_key.task_id:
             self.log.debug("Finding the launch task for job run metadata %s", ti_key.task_id)
@@ -375,9 +374,12 @@ class WorkflowJobRepairAllFailedLink(BaseOperatorLink, LoggingMixin):
             raise AirflowException("Task group is required for generating repair link.")
         if not task_group.group_id:
             raise AirflowException("Task group ID is required for generating repair link.")
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(ti_key.dag_id)
-        dr = _get_dagrun(dag, ti_key.run_id)
+
+        from airflow.utils.session import create_session
+
+        with create_session() as session:
+            dag = _get_dag(ti_key.dag_id, session=session)
+            dr = _get_dagrun(dag, ti_key.run_id, session=session)
         log.debug("Getting failed and skipped tasks for dag run %s", dr.run_id)
         task_group_sub_tasks = self.get_task_group_children(task_group).items()
         failed_and_skipped_tasks = self._get_failed_and_skipped_tasks(dr)
@@ -435,9 +437,14 @@ class WorkflowJobRepairSingleTaskLink(BaseOperatorLink, LoggingMixin):
             task_group.group_id,
             ti_key.task_id,
         )
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(ti_key.dag_id)
+
+        from airflow.utils.session import create_session
+
+        with create_session() as session:
+            dag = _get_dag(ti_key.dag_id, session=session)
         task = dag.get_task(ti_key.task_id)
+        if TYPE_CHECKING:
+            assert isinstance(task, DatabricksTaskBaseOperator)
 
         if ".launch" not in ti_key.task_id:
             launch_task_id = get_launch_task_id(task_group)

{apache_airflow_providers_databricks-7.7.1rc1.dist-info → apache_airflow_providers_databricks-7.7.2rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.7.1rc1
+Version: 7.7.2rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -23,7 +23,6 @@ Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow>=2.10.0rc1
 Requires-Dist: apache-airflow-providers-common-compat>=1.6.0rc1
 Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1
 Requires-Dist: requests>=2.32.0,<3
 Requires-Dist: databricks-sql-connector>=4.0.0
 Requires-Dist: databricks-sqlalchemy>=1.0.2
@@ -35,17 +34,19 @@ Requires-Dist: pyarrow>=16.1.0; python_version < '3.13'
 Requires-Dist: pyarrow>=18.0.0; python_version >= '3.13'
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: apache-airflow-providers-fab>=2.2.0rc1 ; extra == "fab" and ( python_version < '3.13')
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.1/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.1
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.2/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.2
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: azure-identity
 Provides-Extra: fab
+Provides-Extra: openlineage
 Provides-Extra: sdk
 Provides-Extra: standard
 
@@ -74,9 +75,8 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.7.1``
+Release: ``7.7.2``
 
-Release Date: ``|PypiReleaseDate|``
 
 `Databricks <https://databricks.com/>`__
 
@@ -88,12 +88,12 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.2/>`_.
 
 Installation
 ------------
 
-You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+You can install this package on top of an existing Airflow installation (see ``Requirements`` below
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
@@ -108,7 +108,6 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
-``apache-airflow-providers-openlineage``    ``>=2.3.0``
 ``requests``                                ``>=2.32.0,<3``
 ``databricks-sql-connector``                ``>=4.0.0``
 ``databricks-sqlalchemy``                   ``>=1.0.2``
@@ -143,5 +142,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.2/changelog.html>`_.
 

{apache_airflow_providers_databricks-7.7.1rc1.dist-info → apache_airflow_providers_databricks-7.7.2rc1.dist-info}/RECORD

@@ -1,19 +1,19 @@
 airflow/providers/databricks/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/databricks/__init__.py,sha256=zXiLv28riX2qlvDCrFKrkwUuY5dP9yLF209glzp2_AU,1499
+airflow/providers/databricks/__init__.py,sha256=lYfw4I2LzZTvalJKs-1qVwu0Om8gcTtnYtiRZlSo30A,1499
 airflow/providers/databricks/exceptions.py,sha256=85RklmLOI_PnTzfXNIUd5fAu2aMMUhelwumQAX0wANE,1261
 airflow/providers/databricks/get_provider_info.py,sha256=NZ-rY6k6ctDZN7rDngN7mAzq7RMhLag5NwfnuBNcKuw,5644
 airflow/providers/databricks/version_compat.py,sha256=FPgvVwIHRDhFFg0Ghd4WfCiQt-lI8DXtgv4bHwu7Wx4,2021
 airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/hooks/databricks.py,sha256=-Mrgig6XXQ1Uzk_0gfPlxu93N6aNkfdPiugGTvZfndg,29020
+airflow/providers/databricks/hooks/databricks.py,sha256=rkd1J73Zc8IRnMBapxRHWUvyNAEVyV5j14iBDtRdzFo,29470
 airflow/providers/databricks/hooks/databricks_base.py,sha256=gish0H2rHEzPqI5ZpU3BPFCUaycHMEYGYev0ufJMzzI,35167
-airflow/providers/databricks/hooks/databricks_sql.py,sha256=x-Wgmass2IQRm7wXAqMCmZNUvtb2GAM9_4VTla_fBV4,17765
+airflow/providers/databricks/hooks/databricks_sql.py,sha256=-oGJxteTW1L7L0MLpiNeucWs3q_k2n0Ax2rLSNGc0F8,17726
 airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/operators/databricks.py,sha256=6T5T2YdfX9tKkDZ--5WiiYh08rGPzVaRibpCQAuyvzw,79884
+airflow/providers/databricks/operators/databricks.py,sha256=nWurEENXmey-yWYzseb0lGArz4-Q2S29nGAnNLpytMQ,79642
 airflow/providers/databricks/operators/databricks_repos.py,sha256=NLigItgvQOpxhDhttkU2Jhrcu1gODXQME2i5f8w7gYk,13311
 airflow/providers/databricks/operators/databricks_sql.py,sha256=QmFUM83jY0pvnG4K-iM7Kuc4H48ORIx2jgGoOdAtEJw,21836
 airflow/providers/databricks/operators/databricks_workflow.py,sha256=BAWsfFdEG-7p0_6ykkz-xZX1-vdtHnS8uhwjDFpevyg,15088
 airflow/providers/databricks/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/databricks/plugins/databricks_workflow.py,sha256=EtKDPiCq64b8SlH1pJWjqZSI0Kfr-HmAL4IGWOjuDQY,19817
+airflow/providers/databricks/plugins/databricks_workflow.py,sha256=XTbsGp8C7Y9_Z_At2VghJZts9jPCNagENXD5hJvtwOg,20057
 airflow/providers/databricks/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/databricks/sensors/databricks.py,sha256=AVSqvHDr7iDXL1WZ46MTN3KUnVSIOc_g5JEViA1MeVE,6428
 airflow/providers/databricks/sensors/databricks_partition.py,sha256=1PZo-rdRo6E7yBa30ISFjgQ-iaFdqPYm0gnN5tXgxCU,10205
@@ -24,7 +24,7 @@ airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2
 airflow/providers/databricks/utils/databricks.py,sha256=ecvzZbC4KdXds47VeSayot9EO-RQnTRJTEwKITH7waQ,5117
 airflow/providers/databricks/utils/mixins.py,sha256=WUmkt3AmXalmV6zOUIJZWbTldxYunAZOstddDhKCC94,7407
 airflow/providers/databricks/utils/openlineage.py,sha256=1jT5Woh9YifawdP-VFWsabfF-ecuCjPlzD5P_W4DAhI,15078
-apache_airflow_providers_databricks-7.7.1rc1.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
-apache_airflow_providers_databricks-7.7.1rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_databricks-7.7.1rc1.dist-info/METADATA,sha256=Y2c7WUOaMhwpE39PuDzys59VQbRlhkAutSK7cNh9Uo4,7297
-apache_airflow_providers_databricks-7.7.1rc1.dist-info/RECORD,,
+apache_airflow_providers_databricks-7.7.2rc1.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
+apache_airflow_providers_databricks-7.7.2rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_databricks-7.7.2rc1.dist-info/METADATA,sha256=hpSINe7B6t6yhfwKBxP7oCFPzQGNdaph_S0DZLvzCtE,7256
+apache_airflow_providers_databricks-7.7.2rc1.dist-info/RECORD,,