apache-airflow-providers-databricks 7.3.2rc1__py3-none-any.whl → 7.4.0rc1__py3-none-any.whl

airflow/providers/databricks/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.3.2"
+__version__ = "7.4.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.9.0"
+    "2.10.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.9.0+"
+        f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.10.0+"
     )

airflow/providers/databricks/hooks/databricks_sql.py

@@ -35,7 +35,8 @@ from databricks import sql  # type: ignore[attr-defined]
 from databricks.sql.types import Row
 
 from airflow.exceptions import AirflowException
-from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results
+from airflow.providers.common.sql.hooks.handlers import return_single_query_results
+from airflow.providers.common.sql.hooks.sql import DbApiHook
 from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout
 from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook
 
@@ -43,6 +44,8 @@ if TYPE_CHECKING:
     from databricks.sql.client import Connection
 
     from airflow.models.connection import Connection as AirflowConnection
+    from airflow.providers.openlineage.extractors import OperatorLineage
+    from airflow.providers.openlineage.sqlparser import DatabaseInfo
 
 
 LIST_SQL_ENDPOINTS_ENDPOINT = ("GET", "api/2.0/sql/endpoints")
@@ -107,6 +110,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         self.catalog = catalog
         self.schema = schema
         self.additional_params = kwargs
+        self.query_ids: list[str] = []
 
     def _get_extra_config(self) -> dict[str, Any | None]:
         extra_params = copy(self.databricks_conn.extra_dejson)
@@ -226,6 +230,8 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
             it will raise and fail.
         """
         self.descriptions = []
+        self.query_ids = []
+
         if isinstance(sql, str):
             if split_statements:
                 sql_list = [self.strip_sql_string(s) for s in self.split_sql_string(sql)]
@@ -242,6 +248,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         conn = None
         results = []
         for sql_statement in sql_list:
+            self.log.info("Running statement: %s, parameters: %s", sql_statement, parameters)
             # when using AAD tokens, it could expire if previous query run longer than token lifetime
             conn = self.get_conn()
             with closing(conn.cursor()) as cur:
@@ -265,6 +272,10 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
                         if t is not None:
                             t.cancel()
 
+                    if query_id := cur.query_id:
+                        self.log.info("Databricks query id: %s", query_id)
+                        self.query_ids.append(query_id)
+
                     if handler is not None:
                         raw_result = handler(cur)
                         result = self._make_common_data_structure(raw_result)
@@ -307,3 +318,80 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
 
     def bulk_load(self, table, tmp_file):
         raise NotImplementedError()
+
+    def get_openlineage_database_info(self, connection) -> DatabaseInfo:
+        from airflow.providers.openlineage.sqlparser import DatabaseInfo
+
+        return DatabaseInfo(
+            scheme=self.get_openlineage_database_dialect(connection),
+            authority=self.host,
+            database=self.catalog,
+            information_schema_columns=[
+                "table_schema",
+                "table_name",
+                "column_name",
+                "ordinal_position",
+                "data_type",
+                "table_catalog",
+            ],
+            is_information_schema_cross_db=True,
+        )
+
+    def get_openlineage_database_dialect(self, _) -> str:
+        return "databricks"
+
+    def get_openlineage_default_schema(self) -> str | None:
+        return self.schema or "default"
+
+    def get_openlineage_database_specific_lineage(self, task_instance) -> OperatorLineage | None:
+        """
+        Generate OpenLineage metadata for a Databricks task instance based on executed query IDs.
+
+        If a single query ID is present, attach an `ExternalQueryRunFacet` to the lineage metadata.
+        If multiple query IDs are present, emits separate OpenLineage events for each query instead.
+
+        Note that `get_openlineage_database_specific_lineage` is usually called after task's execution,
+        so if multiple query IDs are present, both START and COMPLETE event for each query will be emitted
+        after task's execution. If we are able to query Databricks for query execution metadata,
+        query event times will correspond to actual query's start and finish times.
+
+        Args:
+            task_instance: The Airflow TaskInstance object for which lineage is being collected.
+
+        Returns:
+            An `OperatorLineage` object if a single query ID is found; otherwise `None`.
+        """
+        from airflow.providers.common.compat.openlineage.facet import ExternalQueryRunFacet
+        from airflow.providers.databricks.utils.openlineage import (
+            emit_openlineage_events_for_databricks_queries,
+        )
+        from airflow.providers.openlineage.extractors import OperatorLineage
+        from airflow.providers.openlineage.sqlparser import SQLParser
+
+        if not self.query_ids:
+            self.log.debug("openlineage: no databricks query ids found.")
+            return None
+
+        self.log.debug("openlineage: getting connection to get database info")
+        connection = self.get_connection(self.get_conn_id())
+        namespace = SQLParser.create_namespace(self.get_openlineage_database_info(connection))
+
+        if len(self.query_ids) == 1:
+            self.log.debug("Attaching ExternalQueryRunFacet with single query_id to OpenLineage event.")
+            return OperatorLineage(
+                run_facets={
+                    "externalQuery": ExternalQueryRunFacet(
+                        externalQueryId=self.query_ids[0], source=namespace
+                    )
+                }
+            )
+
+        self.log.info("Multiple query_ids found. Separate OpenLineage event will be emitted for each query.")
+        emit_openlineage_events_for_databricks_queries(
+            query_ids=self.query_ids,
+            query_source_namespace=namespace,
+            task_instance=task_instance,
+            hook=self,
+        )
+
+        return None

airflow/providers/databricks/operators/databricks_sql.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 import csv
 import json
 from collections.abc import Sequence
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, ClassVar
 
 from databricks.sql.utils import ParamEscaper
@@ -106,7 +107,8 @@ class DatabricksSqlOperator(SQLExecuteQueryOperator):
         self.catalog = catalog
         self.schema = schema
 
-    def get_db_hook(self) -> DatabricksSqlHook:
+    @cached_property
+    def _hook(self) -> DatabricksSqlHook:
         hook_params = {
             "http_path": self.http_path,
             "session_configuration": self.session_configuration,
@@ -120,6 +122,9 @@ class DatabricksSqlOperator(SQLExecuteQueryOperator):
         }
         return DatabricksSqlHook(self.databricks_conn_id, **hook_params)
 
+    def get_db_hook(self) -> DatabricksSqlHook:
+        return self._hook
+
     def _should_run_output_processing(self) -> bool:
         return self.do_xcom_push or bool(self._output_path)
 

airflow/providers/databricks/utils/openlineage.py

@@ -0,0 +1,336 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import datetime
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+import requests
+
+from airflow.providers.common.compat.openlineage.check import require_openlineage_version
+from airflow.providers.databricks.version_compat import AIRFLOW_V_3_0_PLUS
+from airflow.utils import timezone
+
+if TYPE_CHECKING:
+    from openlineage.client.event_v2 import RunEvent
+    from openlineage.client.facet_v2 import JobFacet
+
+    from airflow.providers.databricks.hooks.databricks_sql import DatabricksSqlHook
+
+log = logging.getLogger(__name__)
+
+
+def _get_logical_date(task_instance):
+    # todo: remove when min airflow version >= 3.0
+    if AIRFLOW_V_3_0_PLUS:
+        dagrun = task_instance.get_template_context()["dag_run"]
+        return dagrun.logical_date or dagrun.run_after
+
+    if hasattr(task_instance, "logical_date"):
+        date = task_instance.logical_date
+    else:
+        date = task_instance.execution_date
+
+    return date
+
+
+def _get_dag_run_clear_number(task_instance):
+    # todo: remove when min airflow version >= 3.0
+    if AIRFLOW_V_3_0_PLUS:
+        dagrun = task_instance.get_template_context()["dag_run"]
+        return dagrun.clear_number
+    return task_instance.dag_run.clear_number
+
+
+# todo: move this run_id logic into OpenLineage's listener to avoid differences
+def _get_ol_run_id(task_instance) -> str:
+    """
+    Get OpenLineage run_id from TaskInstance.
+
+    It's crucial that the task_instance's run_id creation logic matches OpenLineage's listener implementation.
+    Only then can we ensure that the generated run_id aligns with the Airflow task,
+    enabling a proper connection between events.
+    """
+    from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
+
+    # Generate same OL run id as is generated for current task instance
+    return OpenLineageAdapter.build_task_instance_run_id(
+        dag_id=task_instance.dag_id,
+        task_id=task_instance.task_id,
+        logical_date=_get_logical_date(task_instance),
+        try_number=task_instance.try_number,
+        map_index=task_instance.map_index,
+    )
+
+
+# todo: move this run_id logic into OpenLineage's listener to avoid differences
+def _get_ol_dag_run_id(task_instance) -> str:
+    from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
+
+    return OpenLineageAdapter.build_dag_run_id(
+        dag_id=task_instance.dag_id,
+        logical_date=_get_logical_date(task_instance),
+        clear_number=_get_dag_run_clear_number(task_instance),
+    )
+
+
+def _get_parent_run_facet(task_instance):
+    """
+    Retrieve the ParentRunFacet associated with a specific Airflow task instance.
+
+    This facet helps link OpenLineage events of child jobs - such as queries executed within
+    external systems (e.g., Databricks) by the Airflow task - to the original Airflow task execution.
+    Establishing this connection enables better lineage tracking and observability.
+    """
+    from openlineage.client.facet_v2 import parent_run
+
+    from airflow.providers.openlineage.conf import namespace
+
+    parent_run_id = _get_ol_run_id(task_instance)
+    root_parent_run_id = _get_ol_dag_run_id(task_instance)
+
+    return parent_run.ParentRunFacet(
+        run=parent_run.Run(runId=parent_run_id),
+        job=parent_run.Job(
+            namespace=namespace(),
+            name=f"{task_instance.dag_id}.{task_instance.task_id}",
+        ),
+        root=parent_run.Root(
+            run=parent_run.RootRun(runId=root_parent_run_id),
+            job=parent_run.RootJob(
+                name=task_instance.dag_id,
+                namespace=namespace(),
+            ),
+        ),
+    )
+
+
+def _run_api_call(hook: DatabricksSqlHook, query_ids: list[str]) -> list[dict]:
+    """Retrieve execution details for specific queries from Databricks's query history API."""
+    if not hook._token:
+        # This has logic for token initialization
+        hook.get_conn()
+
+    # https://docs.databricks.com/api/azure/workspace/queryhistory/list
+    try:
+        response = requests.get(
+            url=f"https://{hook.host}/api/2.0/sql/history/queries",
+            headers={"Authorization": f"Bearer {hook._token}"},
+            data=json.dumps({"filter_by": {"statement_ids": query_ids}}),
+            timeout=2,
+        )
+    except Exception as e:
+        log.warning(
+            "OpenLineage could not retrieve Databricks queries details. Error received: `%s`.",
+            e,
+        )
+        return []
+
+    if response.status_code != 200:
+        log.warning(
+            "OpenLineage could not retrieve Databricks queries details. API error received: `%s`: `%s`",
+            response.status_code,
+            response.text,
+        )
+        return []
+
+    return response.json()["res"]
+
+
+def _get_queries_details_from_databricks(
+    hook: DatabricksSqlHook, query_ids: list[str]
+) -> dict[str, dict[str, Any]]:
+    if not query_ids:
+        return {}
+
+    queries_info_from_api = _run_api_call(hook=hook, query_ids=query_ids)
+
+    query_details = {}
+    for query_info in queries_info_from_api:
+        if not query_info.get("query_id"):
+            log.debug("Databricks query ID not found in API response.")
+            continue
+
+        q_start_time = None
+        q_end_time = None
+        if query_info.get("query_start_time_ms") and query_info.get("query_end_time_ms"):
+            q_start_time = datetime.datetime.fromtimestamp(
+                query_info["query_start_time_ms"] / 1000, tz=datetime.timezone.utc
+            )
+            q_end_time = datetime.datetime.fromtimestamp(
+                query_info["query_end_time_ms"] / 1000, tz=datetime.timezone.utc
+            )
+
+        query_details[query_info["query_id"]] = {
+            "status": query_info.get("status"),
+            "start_time": q_start_time,
+            "end_time": q_end_time,
+            "query_text": query_info.get("query_text"),
+            "error_message": query_info.get("error_message"),
+        }
+
+    return query_details
+
+
+def _create_ol_event_pair(
+    job_namespace: str,
+    job_name: str,
+    start_time: datetime.datetime,
+    end_time: datetime.datetime,
+    is_successful: bool,
+    run_facets: dict | None = None,
+    job_facets: dict | None = None,
+) -> tuple[RunEvent, RunEvent]:
+    """Create a pair of OpenLineage RunEvents representing the start and end of a query execution."""
+    from openlineage.client.event_v2 import Job, Run, RunEvent, RunState
+    from openlineage.client.uuid import generate_new_uuid
+
+    run = Run(runId=str(generate_new_uuid()), facets=run_facets or {})
+    job = Job(namespace=job_namespace, name=job_name, facets=job_facets or {})
+
+    start = RunEvent(
+        eventType=RunState.START,
+        eventTime=start_time.isoformat(),
+        run=run,
+        job=job,
+    )
+    end = RunEvent(
+        eventType=RunState.COMPLETE if is_successful else RunState.FAIL,
+        eventTime=end_time.isoformat(),
+        run=run,
+        job=job,
+    )
+    return start, end
+
+
+@require_openlineage_version(provider_min_version="2.3.0")
+def emit_openlineage_events_for_databricks_queries(
+    query_ids: list[str],
+    query_source_namespace: str,
+    task_instance,
+    hook: DatabricksSqlHook | None = None,
+    additional_run_facets: dict | None = None,
+    additional_job_facets: dict | None = None,
+) -> None:
+    """
+    Emit OpenLineage events for executed Databricks queries.
+
+    Metadata retrieval from Databricks is attempted only if a `DatabricksSqlHook` is provided.
+    If metadata is available, execution details such as start time, end time, execution status,
+    error messages, and SQL text are included in the events. If no metadata is found, the function
+    defaults to using the Airflow task instance's state and the current timestamp.
+
+    Note that both START and COMPLETE event for each query will be emitted at the same time.
+    If we are able to query Databricks for query execution metadata, event times
+    will correspond to actual query execution times.
+
+    Args:
+        query_ids: A list of Databricks query IDs to emit events for.
+        query_source_namespace: The namespace to be included in ExternalQueryRunFacet.
+        task_instance: The Airflow task instance that run these queries.
+        hook: A hook instance used to retrieve query metadata if available.
+        additional_run_facets: Additional run facets to include in OpenLineage events.
+        additional_job_facets: Additional job facets to include in OpenLineage events.
+    """
+    from openlineage.client.facet_v2 import job_type_job
+
+    from airflow.providers.common.compat.openlineage.facet import (
+        ErrorMessageRunFacet,
+        ExternalQueryRunFacet,
+        RunFacet,
+        SQLJobFacet,
+    )
+    from airflow.providers.openlineage.conf import namespace
+    from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
+
+    if not query_ids:
+        log.debug("No Databricks query IDs provided; skipping OpenLineage event emission.")
+        return
+
+    query_ids = [q for q in query_ids]  # Make a copy to make sure it does not change
+
+    if hook:
+        log.debug("Retrieving metadata for %s queries from Databricks.", len(query_ids))
+        databricks_metadata = _get_queries_details_from_databricks(hook, query_ids)
+    else:
+        log.debug("DatabricksSqlHook not provided. No extra metadata fill be fetched from Databricks.")
+        databricks_metadata = {}
+
+    # If real metadata is unavailable, we send events with eventTime=now
+    default_event_time = timezone.utcnow()
+    # If no query metadata is provided, we use task_instance's state when checking for success
+    # Adjust state for DBX logic, where "finished" means "success"
+    default_state = task_instance.state.value if hasattr(task_instance, "state") else ""
+    default_state = "finished" if default_state == "success" else default_state
+
+    log.debug("Generating OpenLineage facets")
+    common_run_facets = {"parent": _get_parent_run_facet(task_instance)}
+    common_job_facets: dict[str, JobFacet] = {
+        "jobType": job_type_job.JobTypeJobFacet(
+            jobType="QUERY",
+            integration="DATABRICKS",
+            processingType="BATCH",
+        )
+    }
+    additional_run_facets = additional_run_facets or {}
+    additional_job_facets = additional_job_facets or {}
+
+    events: list[RunEvent] = []
+    for counter, query_id in enumerate(query_ids, 1):
+        query_metadata = databricks_metadata.get(query_id, {})
+        log.debug(
+            "Metadata for query no. %s, (ID `%s`): `%s`",
+            counter,
+            query_id,
+            query_metadata if query_metadata else "not found",
+        )
+
+        query_specific_run_facets: dict[str, RunFacet] = {
+            "externalQuery": ExternalQueryRunFacet(externalQueryId=query_id, source=query_source_namespace)
+        }
+        if query_metadata.get("error_message"):
+            query_specific_run_facets["error"] = ErrorMessageRunFacet(
+                message=query_metadata["error_message"],
+                programmingLanguage="SQL",
+            )
+
+        query_specific_job_facets = {}
+        if query_metadata.get("query_text"):
+            query_specific_job_facets["sql"] = SQLJobFacet(query=query_metadata["query_text"])
+
+        log.debug("Creating OpenLineage event pair for query ID: %s", query_id)
+        event_batch = _create_ol_event_pair(
+            job_namespace=namespace(),
+            job_name=f"{task_instance.dag_id}.{task_instance.task_id}.query.{counter}",
+            start_time=query_metadata.get("start_time", default_event_time),  # type: ignore[arg-type]
+            end_time=query_metadata.get("end_time", default_event_time),  # type: ignore[arg-type]
+            # Only finished status means it completed without failures
+            is_successful=query_metadata.get("status", default_state).lower() == "finished",
+            run_facets={**query_specific_run_facets, **common_run_facets, **additional_run_facets},
+            job_facets={**query_specific_job_facets, **common_job_facets, **additional_job_facets},
+        )
+        events.extend(event_batch)
+
+    log.debug("Generated %s OpenLineage events; emitting now.", len(events))
+    adapter = get_openlineage_listener().adapter
+    for event in events:
+        adapter.emit(event)
+
+    log.info("OpenLineage has successfully finished processing information about Databricks queries.")
+    return

airflow/providers/databricks/version_compat.py

@@ -32,5 +32,4 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
     return airflow_version.major, airflow_version.minor, airflow_version.micro
 
 
-AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
 AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)

{apache_airflow_providers_databricks-7.3.2rc1.dist-info → apache_airflow_providers_databricks-7.4.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.3.2rc1
+Version: 7.4.0rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,27 +20,31 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0rc0
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
+Requires-Dist: apache-airflow>=2.10.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.6.0rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
 Requires-Dist: requests>=2.31.0,<3
 Requires-Dist: databricks-sql-connector>=3.0.0
+Requires-Dist: databricks-sqlalchemy>=1.0.2
 Requires-Dist: aiohttp>=3.9.2, <4
 Requires-Dist: mergedeep>=1.3.4
 Requires-Dist: pandas>=2.1.2,<2.2
 Requires-Dist: pyarrow>=14.0.1
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: apache-airflow-providers-fab ; extra == "fab"
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.4.0/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.4.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: azure-identity
 Provides-Extra: fab
+Provides-Extra: openlineage
 Provides-Extra: sdk
 Provides-Extra: standard
 
@@ -69,7 +73,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.3.2``
+Release: ``7.4.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -82,7 +86,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.4.0/>`_.
 
 Installation
 ------------
@@ -96,18 +100,20 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.9.0``
-``apache-airflow-providers-common-sql``  ``>=1.20.0``
-``requests``                             ``>=2.31.0,<3``
-``databricks-sql-connector``             ``>=3.0.0``
-``aiohttp``                              ``>=3.9.2,<4``
-``mergedeep``                            ``>=1.3.4``
-``pandas``                               ``>=2.1.2,<2.2``
-``pyarrow``                              ``>=14.0.1``
-=======================================  ==================
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.10.0``
+``apache-airflow-providers-common-compat``  ``>=1.6.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
+``requests``                                ``>=2.31.0,<3``
+``databricks-sql-connector``                ``>=3.0.0``
+``databricks-sqlalchemy``                   ``>=1.0.2``
+``aiohttp``                                 ``>=3.9.2,<4``
+``mergedeep``                               ``>=1.3.4``
+``pandas``                                  ``>=2.1.2,<2.2``
+``pyarrow``                                 ``>=14.0.1``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -130,5 +136,5 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.4.0/changelog.html>`_.
 

{apache_airflow_providers_databricks-7.3.2rc1.dist-info → apache_airflow_providers_databricks-7.4.0rc1.dist-info}/RECORD

@@ -1,16 +1,16 @@
 airflow/providers/databricks/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/databricks/__init__.py,sha256=1iYKoYjdiEI3gbSdzOWztaFJUwgD8AKuHKP04iHxV8o,1497
+airflow/providers/databricks/__init__.py,sha256=D4S1V7H6S0R_iJ75oqoynUbmOYkMET00uTSUyPzvkzM,1499
 airflow/providers/databricks/exceptions.py,sha256=85RklmLOI_PnTzfXNIUd5fAu2aMMUhelwumQAX0wANE,1261
 airflow/providers/databricks/get_provider_info.py,sha256=qNMX4Lft-NItPhFewFBSCi8n0_ISid_MQeETKQ67vdo,5573
-airflow/providers/databricks/version_compat.py,sha256=aHg90_DtgoSnQvILFICexMyNlHlALBdaeWqkX3dFDug,1605
+airflow/providers/databricks/version_compat.py,sha256=j5PCtXvZ71aBjixu-EFTNtVDPsngzzs7os0ZQDgFVDk,1536
 airflow/providers/databricks/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/databricks/hooks/databricks.py,sha256=FIoiKWIc9AP3s8Av3Av9yleTg1kI0norwW5CAc6jTQc,28867
 airflow/providers/databricks/hooks/databricks_base.py,sha256=D7-_74QgQaZm1NfHKl_UOXbVAXRo2xjnOx_r1MI-rWI,34871
-airflow/providers/databricks/hooks/databricks_sql.py,sha256=fdxjjeR1u-1dSlbVEBYX0v3XAb8jTT74BVMi3mYY2OE,13092
+airflow/providers/databricks/hooks/databricks_sql.py,sha256=xTdi0JN-ZdsGe2XnCa8yBi-AINZUlyIVlP-5nb2d2T0,16964
 airflow/providers/databricks/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/databricks/operators/databricks.py,sha256=E8fgk3Z67uOTSvWvbF23Miv6EruSGOTdFvHn7pGVWp0,80138
 airflow/providers/databricks/operators/databricks_repos.py,sha256=m_72OnnU9df7UB-8SK2Tp5VjfNyjYeAnil3dCKs9SbA,13282
-airflow/providers/databricks/operators/databricks_sql.py,sha256=thBHpt9_LMLJZ0PN-eLCI3AaT8IFq3NAHLDWDFP-Jiw,17031
+airflow/providers/databricks/operators/databricks_sql.py,sha256=Ycp5mcb3uScQrognB2k8IeSR9oBx-Vnv6NEYGYuE800,17159
 airflow/providers/databricks/operators/databricks_workflow.py,sha256=9WNQR9COa90fbqb9qSzut34K9Z1S_ZdpNHAfIcuH454,14227
 airflow/providers/databricks/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/databricks/plugins/databricks_workflow.py,sha256=1UpsodBLRrTah9zBGBzfM7n1pdkzTo7yilt6QxASspQ,17460
@@ -21,7 +21,8 @@ airflow/providers/databricks/triggers/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvR
 airflow/providers/databricks/triggers/databricks.py,sha256=dSogx6GlcJfZ4CFhtlMeWs9sYFEYthP82S_U8-tM2Tk,9240
 airflow/providers/databricks/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/databricks/utils/databricks.py,sha256=s0qEr_DsFhKW4uUiq2VQbtqcj52isYIplPZsUcxGPrI,2862
-apache_airflow_providers_databricks-7.3.2rc1.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
-apache_airflow_providers_databricks-7.3.2rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_databricks-7.3.2rc1.dist-info/METADATA,sha256=-vW0I-mfTB60WU9-Qk80dGC29S8C97ULEuGW8_GRA1s,6088
-apache_airflow_providers_databricks-7.3.2rc1.dist-info/RECORD,,
+airflow/providers/databricks/utils/openlineage.py,sha256=7fR3CPcOruHapsz1DOZ38QN3ZcAGDADNHPY28CzYCbg,13194
+apache_airflow_providers_databricks-7.4.0rc1.dist-info/entry_points.txt,sha256=hjmZm3ab2cteTR4t9eE28oKixHwNIKtLCThd6sx3XRQ,227
+apache_airflow_providers_databricks-7.4.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_databricks-7.4.0rc1.dist-info/METADATA,sha256=pR6Sl96lEQpHq-x1FYiLA_rNd6-R_e_BsGibC8qrFps,6475
+apache_airflow_providers_databricks-7.4.0rc1.dist-info/RECORD,,