apache-airflow-providers-databricks 7.7.1rc1__tar.gz → 7.7.2rc1__tar.gz

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.7.1rc1
+Version: 7.7.2rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -23,7 +23,6 @@ Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow>=2.10.0rc1
 Requires-Dist: apache-airflow-providers-common-compat>=1.6.0rc1
 Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1
 Requires-Dist: requests>=2.32.0,<3
 Requires-Dist: databricks-sql-connector>=4.0.0
 Requires-Dist: databricks-sqlalchemy>=1.0.2
@@ -35,17 +34,19 @@ Requires-Dist: pyarrow>=16.1.0; python_version < '3.13'
 Requires-Dist: pyarrow>=18.0.0; python_version >= '3.13'
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: apache-airflow-providers-fab>=2.2.0rc1 ; extra == "fab" and ( python_version < '3.13')
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.1/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.1
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.2/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.2
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: azure-identity
 Provides-Extra: fab
+Provides-Extra: openlineage
 Provides-Extra: sdk
 Provides-Extra: standard
 
@@ -74,9 +75,8 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.7.1``
+Release: ``7.7.2``
 
-Release Date: ``|PypiReleaseDate|``
 
 `Databricks <https://databricks.com/>`__
 
@@ -88,12 +88,12 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.2/>`_.
 
 Installation
 ------------
 
-You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+You can install this package on top of an existing Airflow installation (see ``Requirements`` below
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
@@ -108,7 +108,6 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
-``apache-airflow-providers-openlineage``    ``>=2.3.0``
 ``requests``                                ``>=2.32.0,<3``
 ``databricks-sql-connector``                ``>=4.0.0``
 ``databricks-sqlalchemy``                   ``>=1.0.2``
@@ -143,5 +142,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.2/changelog.html>`_.
 

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/README.rst

@@ -23,9 +23,8 @@
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.7.1``
+Release: ``7.7.2``
 
-Release Date: ``|PypiReleaseDate|``
 
 `Databricks <https://databricks.com/>`__
 
@@ -37,12 +36,12 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.2/>`_.
 
 Installation
 ------------
 
-You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+You can install this package on top of an existing Airflow installation (see ``Requirements`` below
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-databricks``
 
@@ -57,7 +56,6 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
-``apache-airflow-providers-openlineage``    ``>=2.3.0``
 ``requests``                                ``>=2.32.0,<3``
 ``databricks-sql-connector``                ``>=4.0.0``
 ``databricks-sqlalchemy``                   ``>=1.0.2``
@@ -92,4 +90,4 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.7.2/changelog.html>`_.

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/docs/changelog.rst

@@ -26,6 +26,38 @@
 Changelog
 ---------
 
+7.7.2
+.....
+
+
+Release Date: ``|PypiReleaseDate|``
+
+Bug Fixes
+~~~~~~~~~
+
+* ``Fix Databricks sqlalchemy URL construction (#54478)``
+* ``Ensures DatabricksWorkflowOperator updates ACL (if available) when resetting a job. (#47827)``
+
+Misc
+~~~~
+
+* ``Remove airflow.models.DAG (#54383)``
+* ``chore: change openlineage to optional dependency (#54748)``
+* ``Move DagBag to SDK and make it return SDK DAG objects (#53918)``
+
+Doc-only
+~~~~~~~~
+
+* ``Make term Dag consistent in providers docs (#55101)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``Switch pre-commit to prek (#54258)``
+   * ``Add CI support for SQLAlchemy 2.0 (#52233)``
+
+.. Review and move the new changes to one of the sections above:
+   * ``Fix Airflow 2 reference in README/index of providers (#55240)``
+
 7.7.1
 .....
 

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/docs/index.rst

@@ -57,7 +57,7 @@
     :maxdepth: 1
     :caption: Resources
 
-    Example DAGs <https://github.com/apache/airflow/tree/providers-databricks/|version|/providers/databricks/tests/system/databricks>
+    Example Dags <https://github.com/apache/airflow/tree/providers-databricks/|version|/providers/databricks/tests/system/databricks>
     PyPI Repository <https://pypi.org/project/apache-airflow-providers-databricks/>
     Installing from sources <installing-providers-from-sources>
 
@@ -78,9 +78,7 @@ apache-airflow-providers-databricks package
 `Databricks <https://databricks.com/>`__
 
 
-Release: 7.7.1
-
-Release Date: ``|PypiReleaseDate|``
+Release: 7.7.2
 
 Provider package
 ----------------
@@ -91,7 +89,7 @@ All classes for this package are included in the ``airflow.providers.databricks`
 Installation
 ------------
 
-You can install this package on top of an existing Airflow 2 installation via
+You can install this package on top of an existing Airflow installation via
 ``pip install apache-airflow-providers-databricks``.
 For the minimum Airflow version supported, see ``Requirements`` below.
 
@@ -106,7 +104,6 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
-``apache-airflow-providers-openlineage``    ``>=2.3.0``
 ``requests``                                ``>=2.32.0,<3``
 ``databricks-sql-connector``                ``>=4.0.0``
 ``databricks-sqlalchemy``                   ``>=1.0.2``
@@ -146,5 +143,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-databricks 7.7.1 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.1.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.1.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.1.tar.gz.sha512>`__)
-* `The apache-airflow-providers-databricks 7.7.1 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.1-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.1-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.1-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-databricks 7.7.2 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.2.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.2.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.2.tar.gz.sha512>`__)
+* `The apache-airflow-providers-databricks 7.7.2 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.2-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.2-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.7.2-py3-none-any.whl.sha512>`__)

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/docs/operators/task.rst

@@ -22,7 +22,7 @@ DatabricksTaskOperator
 ======================
 
 Use the :class:`~airflow.providers.databricks.operators.databricks.DatabricksTaskOperator` to launch and monitor
-task runs on Databricks as Airflow tasks. This can be used as a standalone operator in a DAG and as well as part of a
+task runs on Databricks as Airflow tasks. This can be used as a standalone operator in a Dag and as well as part of a
 Databricks Workflow by using it as an operator(task) within the
 :class:`~airflow.providers.databricks.operators.databricks_workflow.DatabricksWorkflowTaskGroup`.
 

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/docs/operators/workflow.rst

@@ -28,7 +28,7 @@ Databricks notebook job runs as Airflow tasks. The task group launches a `Databr
 There are a few advantages to defining your Databricks Workflows in Airflow:
 
 =======================================  =============================================  =================================
-Authoring interface                      via Databricks (Web-based with Databricks UI)  via Airflow(Code with Airflow DAG)
+Authoring interface                      via Databricks (Web-based with Databricks UI)  via Airflow(Code with Airflow Dag)
 =======================================  =============================================  =================================
 Workflow compute pricing                 ✅                                             ✅
 Notebook code in source control          ✅                                             ✅
@@ -36,14 +36,14 @@ Workflow structure in source control     ✅
 Retry from beginning                     ✅                                             ✅
 Retry single task                        ✅                                             ✅
 Task groups within Workflows                                                            ✅
-Trigger workflows from other DAGs                                                       ✅
+Trigger workflows from other Dags                                                       ✅
 Workflow-level parameters                                                               ✅
 =======================================  =============================================  =================================
 
 Examples
 --------
 
-Example of what a DAG looks like with a DatabricksWorkflowTaskGroup
+Example of what a Dag looks like with a DatabricksWorkflowTaskGroup
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. exampleinclude:: /../../databricks/tests/system/databricks/example_databricks_workflow.py
     :language: python
@@ -53,13 +53,13 @@ Example of what a DAG looks like with a DatabricksWorkflowTaskGroup
 With this example, Airflow will produce a job named ``<dag_name>.test_workflow_<USER>_<GROUP_ID>`` that will
 run task ``notebook_1`` and then ``notebook_2``. The job will be created in the databricks workspace
 if it does not already exist. If the job already exists, it will be updated to match
-the workflow defined in the DAG.
+the workflow defined in the Dag.
 
 The following image displays the resulting Databricks Workflow in the Airflow UI (based on the above example provided)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. image:: ../img/databricks_workflow_task_group_airflow_graph_view.png
 
-The corresponding Databricks Workflow  in the Databricks UI for the run triggered from the Airflow DAG is depicted below
+The corresponding Databricks Workflow  in the Databricks UI for the run triggered from the Airflow Dag is depicted below
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. image:: ../img/workflow_run_databricks_graph_view.png

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/provider.yaml

@@ -22,12 +22,13 @@ description: |
     `Databricks <https://databricks.com/>`__
 
 state: ready
-source-date-epoch: 1754503130
+source-date-epoch: 1756876778
 # Note that those versions are maintained by release manager - do not update them manually
 # with the exception of case where other provider in sources has >= new provider version.
 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
 # to be done in the same PR
 versions:
+  - 7.7.2
   - 7.7.1
   - 7.7.0
   - 7.6.0

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-databricks"
-version = "7.7.1rc1"
+version = "7.7.2rc1"
 description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -54,13 +54,12 @@ requires-python = ">=3.10"
 
 # The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
-# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# Make sure to run ``prek update-providers-dependencies --all-files``
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.10.0rc1",
     "apache-airflow-providers-common-compat>=1.6.0rc1",
     "apache-airflow-providers-common-sql>=1.27.0rc1",
-    "apache-airflow-providers-openlineage>=2.3.0rc1",
     "requests>=2.32.0,<3",
     "databricks-sql-connector>=4.0.0",
     "databricks-sqlalchemy>=1.0.2",
@@ -88,6 +87,9 @@ dependencies = [
 "standard" = [
     "apache-airflow-providers-standard"
 ]
+"openlineage" = [
+    "apache-airflow-providers-openlineage>=2.3.0rc1"
+]
 
 [dependency-groups]
 dev = [
@@ -131,8 +133,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.1"
-"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.1/changelog.html"
+"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.2"
+"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.7.2/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/src/airflow/providers/databricks/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.7.1"
+__version__ = "7.7.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/src/airflow/providers/databricks/hooks/databricks.py

@@ -298,6 +298,16 @@ class DatabricksHook(BaseDatabricksHook):
 
         :param json: The data used in the new_settings of the request to the ``reset`` endpoint.
         """
+        access_control_list = json.get("access_control_list", None)
+        if access_control_list:
+            self.log.info(
+                "Updating job permission for Databricks workflow job id %s with access_control_list %s",
+                job_id,
+                access_control_list,
+            )
+            acl_json = {"access_control_list": access_control_list}
+            self.update_job_permission(job_id=int(job_id), json=acl_json)
+
         self._do_api_call(RESET_ENDPOINT, {"job_id": job_id, "new_settings": json})
 
     def update_job(self, job_id: str, json: dict) -> None:

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/src/airflow/providers/databricks/hooks/databricks_sql.py

@@ -179,7 +179,6 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
 
         :return: the extracted sqlalchemy.engine.URL object.
         """
-        conn = self.get_conn()
         url_query = {
             "http_path": self._http_path,
             "catalog": self.catalog,
@@ -189,9 +188,8 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         return URL.create(
             drivername="databricks",
             username="token",
-            password=conn.password,
-            host=conn.host,
-            port=conn.port,
+            password=self._get_token(raise_error=True),
+            host=self.host,
             query=url_query,
         )
 

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/src/airflow/providers/databricks/operators/databricks.py

@@ -398,10 +398,6 @@ class DatabricksCreateJobsOperator(BaseOperator):
         if job_id is None:
             return self._hook.create_job(self.json)
         self._hook.reset_job(str(job_id), self.json)
-        if (access_control_list := self.json.get("access_control_list")) is not None:
-            acl_json = {"access_control_list": access_control_list}
-            self._hook.update_job_permission(job_id, normalise_json_content(acl_json))
-
         return job_id
 
 

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/src/airflow/providers/databricks/plugins/databricks_workflow.py

@@ -27,10 +27,8 @@ from flask_appbuilder import BaseView
 from flask_appbuilder.api import expose
 
 from airflow.exceptions import AirflowException, TaskInstanceNotFound
-from airflow.models import DagBag
-from airflow.models.dag import DAG, clear_task_instances
 from airflow.models.dagrun import DagRun
-from airflow.models.taskinstance import TaskInstance, TaskInstanceKey
+from airflow.models.taskinstance import TaskInstance, TaskInstanceKey, clear_task_instances
 from airflow.plugins_manager import AirflowPlugin
 from airflow.providers.databricks.hooks.databricks import DatabricksHook
 from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS, BaseOperatorLink, TaskGroup, XCom
@@ -90,8 +88,15 @@ def get_databricks_task_ids(
 if not AIRFLOW_V_3_0_PLUS:
     from airflow.utils.session import NEW_SESSION, provide_session
 
-    @provide_session
-    def _get_dagrun(dag: DAG, run_id: str, session: Session | None = None) -> DagRun:
+    def _get_dag(dag_id: str, session: Session):
+        from airflow.models.serialized_dag import SerializedDagModel
+
+        dag = SerializedDagModel.get_dag(dag_id, session=session)
+        if not dag:
+            raise AirflowException("Dag not found.")
+        return dag
+
+    def _get_dagrun(dag, run_id: str, session: Session) -> DagRun:
         """
         Retrieve the DagRun object associated with the specified DAG and run_id.
 
@@ -107,10 +112,9 @@ if not AIRFLOW_V_3_0_PLUS:
 
     @provide_session
     def _clear_task_instances(
-        dag_id: str, run_id: str, task_ids: list[str], log: logging.Logger, session: Session | None = None
+        dag_id: str, run_id: str, task_ids: list[str], log: logging.Logger, session: Session = NEW_SESSION
     ) -> None:
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(dag_id)
+        dag = _get_dag(dag_id, session=session)
         log.debug("task_ids %s to clear", str(task_ids))
         dr: DagRun = _get_dagrun(dag, run_id, session=session)
         tis_to_clear = [ti for ti in dr.get_task_instances() if ti.databricks_task_key in task_ids]
@@ -274,13 +278,8 @@ class WorkflowJobRunLink(BaseOperatorLink, LoggingMixin):
             ti = get_task_instance(operator, dttm)
             ti_key = ti.key
         task_group = operator.task_group
-
         if not task_group:
             raise AirflowException("Task group is required for generating Databricks Workflow Job Run Link.")
-
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(ti_key.dag_id)
-        dag.get_task(ti_key.task_id)
         self.log.info("Getting link for task %s", ti_key.task_id)
         if ".launch" not in ti_key.task_id:
             self.log.debug("Finding the launch task for job run metadata %s", ti_key.task_id)
@@ -375,9 +374,12 @@ class WorkflowJobRepairAllFailedLink(BaseOperatorLink, LoggingMixin):
             raise AirflowException("Task group is required for generating repair link.")
         if not task_group.group_id:
             raise AirflowException("Task group ID is required for generating repair link.")
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(ti_key.dag_id)
-        dr = _get_dagrun(dag, ti_key.run_id)
+
+        from airflow.utils.session import create_session
+
+        with create_session() as session:
+            dag = _get_dag(ti_key.dag_id, session=session)
+            dr = _get_dagrun(dag, ti_key.run_id, session=session)
         log.debug("Getting failed and skipped tasks for dag run %s", dr.run_id)
         task_group_sub_tasks = self.get_task_group_children(task_group).items()
         failed_and_skipped_tasks = self._get_failed_and_skipped_tasks(dr)
@@ -435,9 +437,14 @@ class WorkflowJobRepairSingleTaskLink(BaseOperatorLink, LoggingMixin):
             task_group.group_id,
             ti_key.task_id,
         )
-        dag_bag = DagBag(read_dags_from_db=True)
-        dag = dag_bag.get_dag(ti_key.dag_id)
+
+        from airflow.utils.session import create_session
+
+        with create_session() as session:
+            dag = _get_dag(ti_key.dag_id, session=session)
         task = dag.get_task(ti_key.task_id)
+        if TYPE_CHECKING:
+            assert isinstance(task, DatabricksTaskBaseOperator)
 
         if ".launch" not in ti_key.task_id:
             launch_task_id = get_launch_task_id(task_group)

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/tests/unit/databricks/hooks/test_databricks.py

@@ -275,6 +275,13 @@ def list_spark_versions_endpoint(host):
     return f"https://{host}/api/2.0/clusters/spark-versions"
 
 
+def permissions_endpoint(host, job_id):
+    """
+    Utility function to generate the permissions endpoint given the host
+    """
+    return f"https://{host}/api/2.0/permissions/jobs/{job_id}"
+
+
 def create_valid_response_mock(content):
     response = mock.MagicMock()
     response.json.return_value = content
@@ -474,7 +481,7 @@ class TestDatabricksHook:
         )
 
     @mock.patch("airflow.providers.databricks.hooks.databricks_base.requests")
-    def test_reset(self, mock_requests):
+    def test_reset_with_no_acl(self, mock_requests):
         mock_requests.codes.ok = 200
         status_code_mock = mock.PropertyMock(return_value=200)
         type(mock_requests.post.return_value).status_code = status_code_mock
@@ -490,6 +497,40 @@ class TestDatabricksHook:
             timeout=self.hook.timeout_seconds,
         )
 
+    @mock.patch("airflow.providers.databricks.hooks.databricks_base.requests")
+    def test_reset_with_acl(self, mock_requests):
+        mock_requests.codes.ok = 200
+        status_code_mock = mock.PropertyMock(return_value=200)
+        type(mock_requests.post.return_value).status_code = status_code_mock
+        ACCESS_CONTROL_LIST = [{"permission_level": "CAN_MANAGE", "user_name": "test_user"}]
+        json = {
+            "access_control_list": ACCESS_CONTROL_LIST,
+            "name": "test",
+        }
+
+        self.hook.reset_job(JOB_ID, json)
+
+        mock_requests.post.assert_called_once_with(
+            reset_endpoint(HOST),
+            json={
+                "job_id": JOB_ID,
+                "new_settings": json,
+            },
+            params=None,
+            auth=HTTPBasicAuth(LOGIN, PASSWORD),
+            headers=self.hook.user_agent_header,
+            timeout=self.hook.timeout_seconds,
+        )
+
+        mock_requests.patch.assert_called_once_with(
+            permissions_endpoint(HOST, JOB_ID),
+            json={"access_control_list": ACCESS_CONTROL_LIST},
+            params=None,
+            auth=HTTPBasicAuth(LOGIN, PASSWORD),
+            headers=self.hook.user_agent_header,
+            timeout=self.hook.timeout_seconds,
+        )
+
     @mock.patch("airflow.providers.databricks.hooks.databricks_base.requests")
     def test_update(self, mock_requests):
         mock_requests.codes.ok = 200

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/tests/unit/databricks/hooks/test_databricks_sql.py

@@ -39,7 +39,6 @@ TASK_ID = "databricks-sql-operator"
 DEFAULT_CONN_ID = "databricks_default"
 HOST = "xx.cloud.databricks.com"
 HOST_WITH_SCHEME = "https://xx.cloud.databricks.com"
-PORT = 443
 TOKEN = "token"
 HTTP_PATH = "sql/protocolv1/o/1234567890123456/0123-456789-abcd123"
 SCHEMA = "test_schema"
@@ -112,38 +111,36 @@ def mock_timer():
         yield mock_timer
 
 
-def make_mock_connection():
-    return Connection(
-        conn_id=DEFAULT_CONN_ID,
-        conn_type="databricks",
-        host=HOST,
-        port=PORT,
-        login="token",
-        password=TOKEN,
-    )
-
-
-def test_sqlachemy_url_property(mock_get_conn):
-    mock_get_conn.return_value = make_mock_connection()
+def test_sqlachemy_url_property():
     hook = DatabricksSqlHook(
         databricks_conn_id=DEFAULT_CONN_ID, http_path=HTTP_PATH, catalog=CATALOG, schema=SCHEMA
     )
     url = hook.sqlalchemy_url.render_as_string(hide_password=False)
     expected_url = (
-        f"databricks://token:{TOKEN}@{HOST}:{PORT}?"
+        f"databricks://token:{TOKEN}@{HOST}?"
         f"catalog={CATALOG}&http_path={quote_plus(HTTP_PATH)}&schema={SCHEMA}"
     )
     assert url == expected_url
 
 
-def test_get_uri(mock_get_conn):
-    mock_get_conn.return_value = make_mock_connection()
+def test_get_sqlalchemy_engine():
+    hook = DatabricksSqlHook(
+        databricks_conn_id=DEFAULT_CONN_ID, http_path=HTTP_PATH, catalog=CATALOG, schema=SCHEMA
+    )
+    engine = hook.get_sqlalchemy_engine()
+    assert engine.url.render_as_string(hide_password=False) == (
+        f"databricks://token:{TOKEN}@{HOST}?"
+        f"catalog={CATALOG}&http_path={quote_plus(HTTP_PATH)}&schema={SCHEMA}"
+    )
+
+
+def test_get_uri():
     hook = DatabricksSqlHook(
         databricks_conn_id=DEFAULT_CONN_ID, http_path=HTTP_PATH, catalog=CATALOG, schema=SCHEMA
     )
     uri = hook.get_uri()
     expected_uri = (
-        f"databricks://token:{TOKEN}@{HOST}:{PORT}?"
+        f"databricks://token:{TOKEN}@{HOST}?"
         f"catalog={CATALOG}&http_path={quote_plus(HTTP_PATH)}&schema={SCHEMA}"
     )
     assert uri == expected_uri

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/tests/unit/databricks/operators/test_databricks.py

@@ -569,42 +569,6 @@ class TestDatabricksCreateJobsOperator:
         db_mock.reset_job.assert_called_once_with(JOB_ID, expected)
         assert return_result == JOB_ID
 
-    @mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
-    def test_exec_update_job_permission(self, db_mock_class):
-        """
-        Test job permission update.
-        """
-        json = {
-            "name": JOB_NAME,
-            "tags": TAGS,
-            "tasks": TASKS,
-            "job_clusters": JOB_CLUSTERS,
-            "email_notifications": EMAIL_NOTIFICATIONS,
-            "webhook_notifications": WEBHOOK_NOTIFICATIONS,
-            "timeout_seconds": TIMEOUT_SECONDS,
-            "schedule": SCHEDULE,
-            "max_concurrent_runs": MAX_CONCURRENT_RUNS,
-            "git_source": GIT_SOURCE,
-            "access_control_list": ACCESS_CONTROL_LIST,
-        }
-        op = DatabricksCreateJobsOperator(task_id=TASK_ID, json=json)
-        db_mock = db_mock_class.return_value
-        db_mock.find_job_id_by_name.return_value = JOB_ID
-
-        op.execute({})
-
-        expected = utils.normalise_json_content({"access_control_list": ACCESS_CONTROL_LIST})
-
-        db_mock_class.assert_called_once_with(
-            DEFAULT_CONN_ID,
-            retry_limit=op.databricks_retry_limit,
-            retry_delay=op.databricks_retry_delay,
-            retry_args=None,
-            caller="DatabricksCreateJobsOperator",
-        )
-
-        db_mock.update_job_permission.assert_called_once_with(JOB_ID, expected)
-
     @mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
     def test_exec_update_job_permission_with_empty_acl(self, db_mock_class):
         """

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/tests/unit/databricks/operators/test_databricks_workflow.py

@@ -104,7 +104,6 @@ def test_create_or_reset_job_existing(mock_databricks_hook, context, mock_task_g
     operator = _CreateDatabricksWorkflowOperator(task_id="test_task", databricks_conn_id="databricks_default")
     operator.task_group = mock_task_group
     operator._hook.list_jobs.return_value = [{"job_id": 123}]
-    operator._hook.create_job.return_value = 123
 
     job_id = operator._create_or_reset_job(context)
     assert job_id == 123

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/tests/unit/databricks/plugins/test_databricks_workflow.py

@@ -217,7 +217,7 @@ def test_workflow_job_run_link_airflow2():
                 "airflow.providers.databricks.plugins.databricks_workflow.get_xcom_result"
             ) as mock_get_xcom_result:
                 with patch(
-                    "airflow.providers.databricks.plugins.databricks_workflow.DagBag.get_dag"
+                    "airflow.providers.databricks.plugins.databricks_workflow._get_dag"
                 ) as mock_get_dag:
                     mock_connection = Mock()
                     mock_connection.extra_dejson = {"host": "mockhost"}
@@ -448,15 +448,15 @@ class TestDatabricksWorkflowPluginAirflow2:
             with patch(
                 "airflow.providers.databricks.plugins.databricks_workflow.get_xcom_result"
             ) as mock_get_xcom:
-                with patch("airflow.providers.databricks.plugins.databricks_workflow.DagBag") as mock_dag_bag:
+                with patch(
+                    "airflow.providers.databricks.plugins.databricks_workflow._get_dag"
+                ) as mock_get_dag:
                     with patch(
                         "airflow.providers.databricks.plugins.databricks_workflow.DatabricksHook"
                     ) as mock_hook:
                         mock_get_ti.return_value = Mock(key=ti_key)
                         mock_get_xcom.return_value = Mock(conn_id="conn_id", run_id=1, job_id=1)
-                        mock_dag_bag.return_value.get_dag.return_value.get_task.return_value = Mock(
-                            task_id="test_task"
-                        )
+                        mock_get_dag.return_value.get_task.return_value = Mock(task_id="test_task")
 
                         mock_hook_instance = Mock()
                         mock_hook_instance.host = "test-host"

{apache_airflow_providers_databricks-7.7.1rc1 → apache_airflow_providers_databricks-7.7.2rc1}/tests/unit/databricks/sensors/test_databricks.py

@@ -21,6 +21,8 @@ from unittest import mock
 
 import pytest
 
+pytest.importorskip("flask_appbuilder")  # Remove after upgrading to FAB5
+
 from airflow.exceptions import AirflowException, TaskDeferred
 from airflow.providers.databricks.hooks.databricks import SQLStatementState
 from airflow.providers.databricks.sensors.databricks import DatabricksSQLStatementsSensor