apache-airflow-providers-databricks 7.8.0__py3-none-any.whl → 7.8.2rc1__py3-none-any.whl

airflow/providers/databricks/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.8.0"
+__version__ = "7.8.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.11.0"

airflow/providers/databricks/exceptions.py

@@ -21,7 +21,7 @@
 
 from __future__ import annotations
 
-from airflow.exceptions import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException
 
 
 class DatabricksSqlExecutionError(AirflowException):

airflow/providers/databricks/hooks/databricks.py

@@ -20,9 +20,9 @@ Databricks hook.
 
 This hook enable the submitting and running of jobs to the Databricks platform. Internally the
 operators talk to the
-``api/2.1/jobs/run-now``
+``api/2.2/jobs/run-now``
 `endpoint <https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunNow>_`
-or the ``api/2.1/jobs/runs/submit``
+or the ``api/2.2/jobs/runs/submit``
 `endpoint <https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunsSubmit>`_.
 """
 
@@ -34,40 +34,40 @@ from typing import Any
 
 from requests import exceptions as requests_exceptions
 
-from airflow.exceptions import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException
 from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook
 
-GET_CLUSTER_ENDPOINT = ("GET", "2.0/clusters/get")
-RESTART_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/restart")
-START_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/start")
-TERMINATE_CLUSTER_ENDPOINT = ("POST", "2.0/clusters/delete")
+GET_CLUSTER_ENDPOINT = ("GET", "2.2/clusters/get")
+RESTART_CLUSTER_ENDPOINT = ("POST", "2.2/clusters/restart")
+START_CLUSTER_ENDPOINT = ("POST", "2.2/clusters/start")
+TERMINATE_CLUSTER_ENDPOINT = ("POST", "2.2/clusters/delete")
 
-CREATE_ENDPOINT = ("POST", "2.1/jobs/create")
-RESET_ENDPOINT = ("POST", "2.1/jobs/reset")
-UPDATE_ENDPOINT = ("POST", "2.1/jobs/update")
-RUN_NOW_ENDPOINT = ("POST", "2.1/jobs/run-now")
-SUBMIT_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/submit")
-GET_RUN_ENDPOINT = ("GET", "2.1/jobs/runs/get")
-CANCEL_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/cancel")
-DELETE_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/delete")
-REPAIR_RUN_ENDPOINT = ("POST", "2.1/jobs/runs/repair")
-OUTPUT_RUNS_JOB_ENDPOINT = ("GET", "2.1/jobs/runs/get-output")
-CANCEL_ALL_RUNS_ENDPOINT = ("POST", "2.1/jobs/runs/cancel-all")
+CREATE_ENDPOINT = ("POST", "2.2/jobs/create")
+RESET_ENDPOINT = ("POST", "2.2/jobs/reset")
+UPDATE_ENDPOINT = ("POST", "2.2/jobs/update")
+RUN_NOW_ENDPOINT = ("POST", "2.2/jobs/run-now")
+SUBMIT_RUN_ENDPOINT = ("POST", "2.2/jobs/runs/submit")
+GET_RUN_ENDPOINT = ("GET", "2.2/jobs/runs/get")
+CANCEL_RUN_ENDPOINT = ("POST", "2.2/jobs/runs/cancel")
+DELETE_RUN_ENDPOINT = ("POST", "2.2/jobs/runs/delete")
+REPAIR_RUN_ENDPOINT = ("POST", "2.2/jobs/runs/repair")
+OUTPUT_RUNS_JOB_ENDPOINT = ("GET", "2.2/jobs/runs/get-output")
+CANCEL_ALL_RUNS_ENDPOINT = ("POST", "2.2/jobs/runs/cancel-all")
 
-INSTALL_LIBS_ENDPOINT = ("POST", "2.0/libraries/install")
-UNINSTALL_LIBS_ENDPOINT = ("POST", "2.0/libraries/uninstall")
-UPDATE_REPO_ENDPOINT = ("PATCH", "2.0/repos/")
-DELETE_REPO_ENDPOINT = ("DELETE", "2.0/repos/")
-CREATE_REPO_ENDPOINT = ("POST", "2.0/repos")
+INSTALL_LIBS_ENDPOINT = ("POST", "2.2/libraries/install")
+UNINSTALL_LIBS_ENDPOINT = ("POST", "2.2/libraries/uninstall")
+UPDATE_REPO_ENDPOINT = ("PATCH", "2.2/repos/")
+DELETE_REPO_ENDPOINT = ("DELETE", "2.2/repos/")
+CREATE_REPO_ENDPOINT = ("POST", "2.2/repos")
 
-LIST_JOBS_ENDPOINT = ("GET", "2.1/jobs/list")
-LIST_PIPELINES_ENDPOINT = ("GET", "2.0/pipelines")
-LIST_SQL_ENDPOINTS_ENDPOINT = ("GET", "2.0/sql/endpoints")
+LIST_JOBS_ENDPOINT = ("GET", "2.2/jobs/list")
+LIST_PIPELINES_ENDPOINT = ("GET", "2.2/pipelines")
+LIST_SQL_ENDPOINTS_ENDPOINT = ("GET", "2.2/sql/endpoints")
 
-WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "2.0/workspace/get-status")
+WORKSPACE_GET_STATUS_ENDPOINT = ("GET", "2.2/workspace/get-status")
 
-SPARK_VERSIONS_ENDPOINT = ("GET", "2.0/clusters/spark-versions")
-SQL_STATEMENTS_ENDPOINT = "2.0/sql/statements"
+SPARK_VERSIONS_ENDPOINT = ("GET", "2.2/clusters/spark-versions")
+SQL_STATEMENTS_ENDPOINT = "2.2/sql/statements"
 
 
 class RunLifeCycleState(Enum):
@@ -293,7 +293,7 @@ class DatabricksHook(BaseDatabricksHook):
 
     def create_job(self, json: dict) -> int:
         """
-        Call the ``api/2.1/jobs/create`` endpoint.
+        Call the ``api/2.2/jobs/create`` endpoint.
 
         :param json: The data used in the body of the request to the ``create`` endpoint.
         :return: the job_id as an int
@@ -303,7 +303,7 @@ class DatabricksHook(BaseDatabricksHook):
 
     def reset_job(self, job_id: str, json: dict) -> None:
         """
-        Call the ``api/2.1/jobs/reset`` endpoint.
+        Call the ``api/2.2/jobs/reset`` endpoint.
 
         :param json: The data used in the new_settings of the request to the ``reset`` endpoint.
         """
@@ -321,7 +321,7 @@ class DatabricksHook(BaseDatabricksHook):
 
     def update_job(self, job_id: str, json: dict) -> None:
         """
-        Call the ``api/2.1/jobs/update`` endpoint.
+        Call the ``api/2.2/jobs/update`` endpoint.
 
         :param job_id: The id of the job to update.
         :param json: The data used in the new_settings of the request to the ``update`` endpoint.
@@ -330,7 +330,7 @@ class DatabricksHook(BaseDatabricksHook):
 
     def run_now(self, json: dict) -> int:
         """
-        Call the ``api/2.1/jobs/run-now`` endpoint.
+        Call the ``api/2.2/jobs/run-now`` endpoint.
 
         :param json: The data used in the body of the request to the ``run-now`` endpoint.
         :return: the run_id as an int
@@ -340,7 +340,7 @@ class DatabricksHook(BaseDatabricksHook):
 
     def submit_run(self, json: dict) -> int:
         """
-        Call the ``api/2.1/jobs/runs/submit`` endpoint.
+        Call the ``api/2.2/jobs/runs/submit`` endpoint.
 
         :param json: The data used in the body of the request to the ``submit`` endpoint.
         :return: the run_id as an int
@@ -385,9 +385,9 @@ class DatabricksHook(BaseDatabricksHook):
                 all_jobs += [j for j in jobs if j["settings"]["name"] == job_name]
             else:
                 all_jobs += jobs
-            has_more = response.get("has_more", False)
-            if has_more:
-                page_token = response.get("next_page_token", "")
+            # issue-59189: API v2.2 removes "has_more" field
+            page_token = response.get("next_page_token", "")
+            has_more = bool(page_token)
 
         return all_jobs
 
@@ -717,7 +717,7 @@ class DatabricksHook(BaseDatabricksHook):
         """
         Install libraries on the cluster.
 
-        Utility function to call the ``2.0/libraries/install`` endpoint.
+        Utility function to call the ``2.2/libraries/install`` endpoint.
 
         :param json: json dictionary containing cluster_id and an array of library
         """
@@ -727,7 +727,7 @@ class DatabricksHook(BaseDatabricksHook):
         """
         Uninstall libraries on the cluster.
 
-        Utility function to call the ``2.0/libraries/uninstall`` endpoint.
+        Utility function to call the ``2.2/libraries/uninstall`` endpoint.
 
         :param json: json dictionary containing cluster_id and an array of library
         """
@@ -790,7 +790,7 @@ class DatabricksHook(BaseDatabricksHook):
         :param json: payload
         :return: json containing permission specification
         """
-        return self._do_api_call(("PATCH", f"2.0/permissions/jobs/{job_id}"), json)
+        return self._do_api_call(("PATCH", f"2.2/permissions/jobs/{job_id}"), json)
 
     def post_sql_statement(self, json: dict[str, Any]) -> str:
         """

airflow/providers/databricks/hooks/databricks_base.py

@@ -49,7 +49,8 @@ from tenacity import (
 )
 
 from airflow import __version__
-from airflow.exceptions import AirflowException, AirflowOptionalProviderFeatureException
+from airflow.exceptions import AirflowOptionalProviderFeatureException
+from airflow.providers.common.compat.sdk import AirflowException
 from airflow.providers_manager import ProvidersManager
 
 try:

airflow/providers/databricks/hooks/databricks_sql.py

@@ -34,7 +34,7 @@ from databricks import sql
 from databricks.sql.types import Row
 from sqlalchemy.engine import URL
 
-from airflow.exceptions import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException
 from airflow.providers.common.sql.hooks.handlers import return_single_query_results
 from airflow.providers.common.sql.hooks.sql import DbApiHook
 from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout

airflow/providers/databricks/operators/databricks.py

@@ -27,8 +27,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any
 
 from airflow.configuration import conf
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseOperator, BaseOperatorLink, XCom
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, BaseOperatorLink, XCom
 from airflow.providers.databricks.hooks.databricks import (
     DatabricksHook,
     RunLifeCycleState,
@@ -261,7 +260,7 @@ class DatabricksCreateJobsOperator(BaseOperator):
         https://docs.databricks.com/api/workspace/jobs/reset
 
     :param json: A JSON object containing API parameters which will be passed
-        directly to the ``api/2.1/jobs/create`` endpoint. The other named parameters
+        directly to the ``api/2.2/jobs/create`` endpoint. The other named parameters
         (i.e. ``name``, ``tags``, ``tasks``, etc.) to this operator will
         be merged with this json dictionary if they are provided.
         If there are conflicts during the merge, the named parameters will
@@ -392,7 +391,7 @@ class DatabricksCreateJobsOperator(BaseOperator):
 
 class DatabricksSubmitRunOperator(BaseOperator):
     """
-    Submits a Spark job run to Databricks using the api/2.1/jobs/runs/submit API endpoint.
+    Submits a Spark job run to Databricks using the api/2.2/jobs/runs/submit API endpoint.
 
     See: https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunsSubmit
 
@@ -407,7 +406,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
         .. seealso::
             https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunsSubmit
     :param json: A JSON object containing API parameters which will be passed
-        directly to the ``api/2.1/jobs/runs/submit`` endpoint. The other named parameters
+        directly to the ``api/2.2/jobs/runs/submit`` endpoint. The other named parameters
         (i.e. ``spark_jar_task``, ``notebook_task``..) to this operator will
         be merged with this json dictionary if they are provided.
         If there are conflicts during the merge, the named parameters will
@@ -645,14 +644,14 @@ class DatabricksSubmitRunOperator(BaseOperator):
 
 class DatabricksRunNowOperator(BaseOperator):
     """
-    Runs an existing Spark job run to Databricks using the api/2.1/jobs/run-now API endpoint.
+    Runs an existing Spark job run to Databricks using the api/2.2/jobs/run-now API endpoint.
 
     See: https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunNow
 
     There are two ways to instantiate this operator.
 
     In the first way, you can take the JSON payload that you typically use
-    to call the ``api/2.1/jobs/run-now`` endpoint and pass it directly
+    to call the ``api/2.2/jobs/run-now`` endpoint and pass it directly
     to our ``DatabricksRunNowOperator`` through the ``json`` parameter.
     For example ::
 
@@ -730,7 +729,7 @@ class DatabricksRunNowOperator(BaseOperator):
             https://docs.databricks.com/en/workflows/jobs/settings.html#add-parameters-for-all-job-tasks
 
     :param json: A JSON object containing API parameters which will be passed
-        directly to the ``api/2.1/jobs/run-now`` endpoint. The other named parameters
+        directly to the ``api/2.2/jobs/run-now`` endpoint. The other named parameters
         (i.e. ``notebook_params``, ``spark_submit_params``..) to this operator will
         be merged with this json dictionary if they are provided.
         If there are conflicts during the merge, the named parameters will

airflow/providers/databricks/operators/databricks_repos.py

@@ -25,8 +25,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING
 from urllib.parse import urlsplit
 
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperator
 from airflow.providers.databricks.hooks.databricks import DatabricksHook
 
 if TYPE_CHECKING:

airflow/providers/databricks/operators/databricks_sql.py

@@ -27,8 +27,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
 
 from databricks.sql.utils import ParamEscaper
 
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperator
 from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
 from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
 

airflow/providers/databricks/operators/databricks_workflow.py

@@ -25,8 +25,7 @@ from typing import TYPE_CHECKING, Any
 
 from mergedeep import merge
 
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseOperator, TaskGroup
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, TaskGroup
 from airflow.providers.databricks.hooks.databricks import DatabricksHook, RunLifeCycleState
 from airflow.providers.databricks.plugins.databricks_workflow import (
     WorkflowJobRepairAllFailedLink,

airflow/providers/databricks/plugins/databricks_workflow.py

@@ -20,11 +20,13 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any
 from urllib.parse import unquote
 
-from airflow.exceptions import AirflowException, TaskInstanceNotFound
+from sqlalchemy import select
+
+from airflow.exceptions import TaskInstanceNotFound
 from airflow.models.dagrun import DagRun
 from airflow.models.taskinstance import TaskInstance, TaskInstanceKey, clear_task_instances
 from airflow.plugins_manager import AirflowPlugin
-from airflow.providers.common.compat.sdk import BaseOperatorLink, TaskGroup, XCom
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperatorLink, TaskGroup, XCom
 from airflow.providers.databricks.hooks.databricks import DatabricksHook
 from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -143,7 +145,9 @@ if not AIRFLOW_V_3_0_PLUS:
         if not session:
             raise AirflowException("Session not provided.")
 
-        return session.query(DagRun).filter(DagRun.dag_id == dag.dag_id, DagRun.run_id == run_id).one()
+        return session.scalars(
+            select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.run_id == run_id)
+        ).one()
 
     @provide_session
     def _clear_task_instances(
@@ -162,15 +166,13 @@ if not AIRFLOW_V_3_0_PLUS:
             dag_run = DagRun.find(dag_id, execution_date=dttm)[0]  # type: ignore[call-arg]
         else:
             dag_run = DagRun.find(dag_id, logical_date=dttm)[0]
-        ti = (
-            session.query(TaskInstance)
-            .filter(
+        ti = session.scalars(
+            select(TaskInstance).where(
                 TaskInstance.dag_id == dag_id,
                 TaskInstance.run_id == dag_run.run_id,
                 TaskInstance.task_id == operator.task_id,
             )
-            .one_or_none()
-        )
+        ).one_or_none()
         if not ti:
             raise TaskInstanceNotFound("Task instance not found")
         return ti

airflow/providers/databricks/sensors/databricks.py

@@ -23,8 +23,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any
 
 from airflow.configuration import conf
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseSensorOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator
 from airflow.providers.databricks.hooks.databricks import DatabricksHook, SQLStatementState
 from airflow.providers.databricks.operators.databricks import DEFER_METHOD_NAME
 from airflow.providers.databricks.utils.mixins import DatabricksSQLStatementsMixin

airflow/providers/databricks/sensors/databricks_partition.py

@@ -27,8 +27,7 @@ from typing import TYPE_CHECKING, Any
 
 from databricks.sql.utils import ParamEscaper
 
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseSensorOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator
 from airflow.providers.common.sql.hooks.handlers import fetch_all_handler
 from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
 

airflow/providers/databricks/sensors/databricks_sql.py

@@ -24,8 +24,7 @@ from collections.abc import Callable, Iterable, Sequence
 from functools import cached_property
 from typing import TYPE_CHECKING, Any
 
-from airflow.exceptions import AirflowException
-from airflow.providers.common.compat.sdk import BaseSensorOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator
 from airflow.providers.common.sql.hooks.handlers import fetch_all_handler
 from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
 

airflow/providers/databricks/utils/databricks.py

@@ -17,7 +17,7 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.exceptions import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException
 from airflow.providers.databricks.hooks.databricks import DatabricksHook, RunState
 
 

airflow/providers/databricks/utils/mixins.py

@@ -20,18 +20,14 @@ from __future__ import annotations
 
 import time
 from logging import Logger
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Protocol,
-)
+from typing import TYPE_CHECKING, Any, Protocol
 
-from airflow.exceptions import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException
 from airflow.providers.databricks.hooks.databricks import DatabricksHook, SQLStatementState
 from airflow.providers.databricks.triggers.databricks import DatabricksSQLStatementExecutionTrigger
 
 if TYPE_CHECKING:
-    from airflow.utils.context import Context
+    from airflow.sdk import Context
 
 
 class GetHookHasFields(Protocol):

airflow/providers/databricks/utils/openlineage.py

@@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Any
 import requests
 
 from airflow.providers.common.compat.openlineage.check import require_openlineage_version
-from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils import timezone
 
 if TYPE_CHECKING:
@@ -37,60 +36,6 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 
-def _get_logical_date(task_instance):
-    # todo: remove when min airflow version >= 3.0
-    if AIRFLOW_V_3_0_PLUS:
-        dagrun = task_instance.get_template_context()["dag_run"]
-        return dagrun.logical_date or dagrun.run_after
-
-    if hasattr(task_instance, "logical_date"):
-        date = task_instance.logical_date
-    else:
-        date = task_instance.execution_date
-
-    return date
-
-
-def _get_dag_run_clear_number(task_instance):
-    # todo: remove when min airflow version >= 3.0
-    if AIRFLOW_V_3_0_PLUS:
-        dagrun = task_instance.get_template_context()["dag_run"]
-        return dagrun.clear_number
-    return task_instance.dag_run.clear_number
-
-
-# todo: move this run_id logic into OpenLineage's listener to avoid differences
-def _get_ol_run_id(task_instance) -> str:
-    """
-    Get OpenLineage run_id from TaskInstance.
-
-    It's crucial that the task_instance's run_id creation logic matches OpenLineage's listener implementation.
-    Only then can we ensure that the generated run_id aligns with the Airflow task,
-    enabling a proper connection between events.
-    """
-    from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
-
-    # Generate same OL run id as is generated for current task instance
-    return OpenLineageAdapter.build_task_instance_run_id(
-        dag_id=task_instance.dag_id,
-        task_id=task_instance.task_id,
-        logical_date=_get_logical_date(task_instance),
-        try_number=task_instance.try_number,
-        map_index=task_instance.map_index,
-    )
-
-
-# todo: move this run_id logic into OpenLineage's listener to avoid differences
-def _get_ol_dag_run_id(task_instance) -> str:
-    from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
-
-    return OpenLineageAdapter.build_dag_run_id(
-        dag_id=task_instance.dag_id,
-        logical_date=_get_logical_date(task_instance),
-        clear_number=_get_dag_run_clear_number(task_instance),
-    )
-
-
 def _get_parent_run_facet(task_instance):
     """
     Retrieve the ParentRunFacet associated with a specific Airflow task instance.
@@ -101,22 +46,39 @@ def _get_parent_run_facet(task_instance):
     """
     from openlineage.client.facet_v2 import parent_run
 
-    from airflow.providers.openlineage.conf import namespace
+    from airflow.providers.openlineage.plugins.macros import (
+        lineage_job_name,
+        lineage_job_namespace,
+        lineage_root_job_name,
+        lineage_root_run_id,
+        lineage_run_id,
+    )
+
+    parent_run_id = lineage_run_id(task_instance)
+    parent_job_name = lineage_job_name(task_instance)
+    parent_job_namespace = lineage_job_namespace()
+
+    root_parent_run_id = lineage_root_run_id(task_instance)
+    rot_parent_job_name = lineage_root_job_name(task_instance)
+
+    try:  # Added in OL provider 2.9.0, try to use it if possible
+        from airflow.providers.openlineage.plugins.macros import lineage_root_job_namespace
 
-    parent_run_id = _get_ol_run_id(task_instance)
-    root_parent_run_id = _get_ol_dag_run_id(task_instance)
+        root_parent_job_namespace = lineage_root_job_namespace(task_instance)
+    except ImportError:
+        root_parent_job_namespace = lineage_job_namespace()
 
     return parent_run.ParentRunFacet(
         run=parent_run.Run(runId=parent_run_id),
         job=parent_run.Job(
-            namespace=namespace(),
-            name=f"{task_instance.dag_id}.{task_instance.task_id}",
+            namespace=parent_job_namespace,
+            name=parent_job_name,
         ),
         root=parent_run.Root(
             run=parent_run.RootRun(runId=root_parent_run_id),
             job=parent_run.RootJob(
-                name=task_instance.dag_id,
-                namespace=namespace(),
+                name=rot_parent_job_name,
+                namespace=root_parent_job_namespace,
             ),
         ),
     )
@@ -209,7 +171,7 @@ def _create_ol_event_pair(
     return start, end
 
 
-@require_openlineage_version(provider_min_version="2.3.0")
+@require_openlineage_version(provider_min_version="2.5.0")
 def emit_openlineage_events_for_databricks_queries(
     task_instance,
     hook: DatabricksSqlHook | DatabricksHook | None = None,

{apache_airflow_providers_databricks-7.8.0.dist-info → apache_airflow_providers_databricks-7.8.2rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.8.0
+Version: 7.8.2rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,9 +22,9 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
 License-File: LICENSE
 License-File: NOTICE
-Requires-Dist: apache-airflow>=2.11.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.8.0
-Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
+Requires-Dist: apache-airflow>=2.11.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.10.1rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
 Requires-Dist: requests>=2.32.0,<3
 Requires-Dist: databricks-sql-connector>=4.0.0
 Requires-Dist: databricks-sqlalchemy>=1.0.2
@@ -35,13 +35,13 @@ Requires-Dist: pandas>=2.2.3; python_version >="3.13"
 Requires-Dist: pyarrow>=16.1.0; python_version < '3.13'
 Requires-Dist: pyarrow>=18.0.0; python_version >= '3.13'
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
-Requires-Dist: apache-airflow-providers-fab>=2.2.0 ; extra == "fab" and ( python_version < '3.13')
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
+Requires-Dist: apache-airflow-providers-fab>=2.2.0rc1 ; extra == "fab" and ( python_version < '3.13')
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.8.2/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.8.2
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -77,7 +77,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.8.0``
+Release: ``7.8.2``
 
 
 `Databricks <https://databricks.com/>`__
@@ -90,7 +90,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.2/>`_.
 
 Installation
 ------------
@@ -108,7 +108,7 @@ Requirements
 PIP package                                 Version required
 ==========================================  ======================================
 ``apache-airflow``                          ``>=2.11.0``
-``apache-airflow-providers-common-compat``  ``>=1.8.0``
+``apache-airflow-providers-common-compat``  ``>=1.10.1``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
 ``requests``                                ``>=2.32.0,<3``
 ``databricks-sql-connector``                ``>=4.0.0``
@@ -156,5 +156,5 @@ Extra               Dependencies
 ==================  ================================================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.8.2/changelog.html>`_.
 

apache_airflow_providers_databricks-7.8.2rc1.dist-info/RECORD

@@ -0,0 +1,31 @@
+airflow/providers/databricks/__init__.py,sha256=b-DyL0dlIgtgJ_JwUEsrrNbwshoektdumNOnplO1dyI,1499
+airflow/providers/databricks/exceptions.py,sha256=v7TD8auFp9LmyWqRtnXYG8mOit0WE3OuInUNFoC0zTo,1278
+airflow/providers/databricks/get_provider_info.py,sha256=LfK0AwIARVh4tX5146-J2VRZwfe6GP3xjLyltA7X7iU,5738
+airflow/providers/databricks/version_compat.py,sha256=RQbdCueLOaFZWekpQmF0BoAoJInW8EoyvJ3Ah-HbrPo,1577
+airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/databricks/hooks/databricks.py,sha256=6e2qa4jQJub4-Qg3j7GSX7Kt6cWNLa5n-5xqvQJGD0k,29788
+airflow/providers/databricks/hooks/databricks_base.py,sha256=h4dZTVfjnuEznxv2S36IJ3cO8xG1hLuP2-_0HVrv2M8,36911
+airflow/providers/databricks/hooks/databricks_sql.py,sha256=odloOD8Qx9cJ7rozdpEfpR5VSNPNfWH3dw6Yxv6pVCQ,17772
+airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/databricks/operators/databricks.py,sha256=DLLPlSCHBKdNwMemh_gGmzcbBgB-GVr-cYpq5J0DjXU,79286
+airflow/providers/databricks/operators/databricks_repos.py,sha256=jOrYO_tFQJ5JBXeu7Rhrc3pcQJ4qtzSGSjGZ4GffmwU,13125
+airflow/providers/databricks/operators/databricks_sql.py,sha256=m3_5dG3zd1rhWK96YAuBzh7J4Y20B9lLXBUFGtx1V84,21812
+airflow/providers/databricks/operators/databricks_workflow.py,sha256=xqk6kbFcqArHo4w9E0sVGbAkX2tuBqWdtvwiFyc9jzo,14989
+airflow/providers/databricks/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/databricks/plugins/databricks_workflow.py,sha256=ri4yJ19cX8jF2JEuDojNWUgjshYdPKgHYMvlw0jmwow,20063
+airflow/providers/databricks/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/databricks/sensors/databricks.py,sha256=9gFu7Xy8wqaBKPAe94ozpVtruLluLwG07iMP53KAcPo,6232
+airflow/providers/databricks/sensors/databricks_partition.py,sha256=AV7GoAIRnV7NEtbqUxp9WdSeN-LeIc49I3_NaI1cBiY,9910
+airflow/providers/databricks/sensors/databricks_sql.py,sha256=ON3ulhD0I4ukJhKzDYTqw-8ZkdUuED_8QyDZbzFgHko,5603
+airflow/providers/databricks/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/databricks/triggers/databricks.py,sha256=DQbXLw1W_e3Iw-hsDph7vPuHc2caj623V7WmA2_PftM,8672
+airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/databricks/utils/databricks.py,sha256=bnZdjQ1etvAcfgdmb8BR4i1M4YjdcDXxxznVtmur1GM,5134
+airflow/providers/databricks/utils/mixins.py,sha256=XDA9v9BeCgMIznYPpa-X7XIqrD1mJbw4eSQUjvTsQXI,7397
+airflow/providers/databricks/utils/openlineage.py,sha256=naqLzbdBebwDUPvDhhIa5Ey_8SgKkYqdwhzJC_51gFU,13674
+apache_airflow_providers_databricks-7.8.2rc1.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
+apache_airflow_providers_databricks-7.8.2rc1.dist-info/licenses/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+apache_airflow_providers_databricks-7.8.2rc1.dist-info/licenses/NOTICE,sha256=E3-_E02gwwSEFzeeWPKmnIjOoos3hW28CLISV6sYrbQ,168
+apache_airflow_providers_databricks-7.8.2rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_databricks-7.8.2rc1.dist-info/METADATA,sha256=UigXT7Txwtkg9ZSrz4iII-Hw1ruBnYpetJ7PGi8AYws,7784
+apache_airflow_providers_databricks-7.8.2rc1.dist-info/RECORD,,

apache_airflow_providers_databricks-7.8.0.dist-info/RECORD

@@ -1,31 +0,0 @@
-airflow/providers/databricks/__init__.py,sha256=jK9hWZ8jptf_y_7T6PywlGiLdB4zq1EsGECyxMGFi8A,1499
-airflow/providers/databricks/exceptions.py,sha256=85RklmLOI_PnTzfXNIUd5fAu2aMMUhelwumQAX0wANE,1261
-airflow/providers/databricks/get_provider_info.py,sha256=LfK0AwIARVh4tX5146-J2VRZwfe6GP3xjLyltA7X7iU,5738
-airflow/providers/databricks/version_compat.py,sha256=RQbdCueLOaFZWekpQmF0BoAoJInW8EoyvJ3Ah-HbrPo,1577
-airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/hooks/databricks.py,sha256=uOBPUUAEc9eHBdvMgNyJzWivIFCt_GQgR4UlRrRxqgM,29754
-airflow/providers/databricks/hooks/databricks_base.py,sha256=m-m2AKqD3-6mEfvuwgo7Era47zGzsjKbpLTRQNjiUS4,36864
-airflow/providers/databricks/hooks/databricks_sql.py,sha256=xougOWuFgQzhBzFcuYkbX-lo0FpKCQztXoBETJEzesg,17755
-airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/operators/databricks.py,sha256=bVhFNTeGboHkmaJkYDYEyF0V1QUOB_RnsvwaCuEtIew,79316
-airflow/providers/databricks/operators/databricks_repos.py,sha256=VRZye45ZMlDxti6ZJjuouox5umiMoeQ-BKugPpE7jnM,13155
-airflow/providers/databricks/operators/databricks_sql.py,sha256=gwpkr660qpk4dUve98RB-hniaMzuXL6znQZZGilJxi0,21842
-airflow/providers/databricks/operators/databricks_workflow.py,sha256=QLsR0pGLWvvQbutsjj4RWwBE-z6tkWiYLHj6waMv8ZE,15019
-airflow/providers/databricks/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/databricks/plugins/databricks_workflow.py,sha256=5vyG2WNM25ptSv5IwAndUTqKAOmTneOWy_pAtqBKcgc,20020
-airflow/providers/databricks/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/databricks/sensors/databricks.py,sha256=RrjSzncvppdp5U8RYHd975MCIQIb_s1VQoxm9Aqbvac,6262
-airflow/providers/databricks/sensors/databricks_partition.py,sha256=qPDy8oxg-Lo-jnHy1EbxmA5GIjC6t0XnFJ1E3aAmUgg,9940
-airflow/providers/databricks/sensors/databricks_sql.py,sha256=shq7ng4LCiaD4Q7lorm4g1A7aijmq3nVUnCFlYtoI7c,5633
-airflow/providers/databricks/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/databricks/triggers/databricks.py,sha256=DQbXLw1W_e3Iw-hsDph7vPuHc2caj623V7WmA2_PftM,8672
-airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/databricks/utils/databricks.py,sha256=ecvzZbC4KdXds47VeSayot9EO-RQnTRJTEwKITH7waQ,5117
-airflow/providers/databricks/utils/mixins.py,sha256=WUmkt3AmXalmV6zOUIJZWbTldxYunAZOstddDhKCC94,7407
-airflow/providers/databricks/utils/openlineage.py,sha256=1jT5Woh9YifawdP-VFWsabfF-ecuCjPlzD5P_W4DAhI,15078
-apache_airflow_providers_databricks-7.8.0.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
-apache_airflow_providers_databricks-7.8.0.dist-info/licenses/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-apache_airflow_providers_databricks-7.8.0.dist-info/licenses/NOTICE,sha256=E3-_E02gwwSEFzeeWPKmnIjOoos3hW28CLISV6sYrbQ,168
-apache_airflow_providers_databricks-7.8.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_databricks-7.8.0.dist-info/METADATA,sha256=ag4y8wdimVeG_xpU6SGjicRyOsUe66vk2zyUTQ8h9c0,7750
-apache_airflow_providers_databricks-7.8.0.dist-info/RECORD,,