apache-airflow-providers-databricks 7.3.2rc1__tar.gz → 7.4.0rc1__tar.gz

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-databricks
-Version: 7.3.2rc1
+Version: 7.4.0rc1
 Summary: Provider package apache-airflow-providers-databricks for Apache Airflow
 Keywords: airflow-provider,databricks,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,27 +20,31 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0rc0
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
+Requires-Dist: apache-airflow>=2.10.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.6.0rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
 Requires-Dist: requests>=2.31.0,<3
 Requires-Dist: databricks-sql-connector>=3.0.0
+Requires-Dist: databricks-sqlalchemy>=1.0.2
 Requires-Dist: aiohttp>=3.9.2, <4
 Requires-Dist: mergedeep>=1.3.4
 Requires-Dist: pandas>=2.1.2,<2.2
 Requires-Dist: pyarrow>=14.0.1
 Requires-Dist: azure-identity>=1.3.1 ; extra == "azure-identity"
 Requires-Dist: apache-airflow-providers-fab ; extra == "fab"
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: databricks-sdk==0.10.0 ; extra == "sdk"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.4.0/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.4.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: azure-identity
 Provides-Extra: fab
+Provides-Extra: openlineage
 Provides-Extra: sdk
 Provides-Extra: standard
 
@@ -69,7 +73,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.3.2``
+Release: ``7.4.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -82,7 +86,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.4.0/>`_.
 
 Installation
 ------------
@@ -96,18 +100,20 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.9.0``
-``apache-airflow-providers-common-sql``  ``>=1.20.0``
-``requests``                             ``>=2.31.0,<3``
-``databricks-sql-connector``             ``>=3.0.0``
-``aiohttp``                              ``>=3.9.2,<4``
-``mergedeep``                            ``>=1.3.4``
-``pandas``                               ``>=2.1.2,<2.2``
-``pyarrow``                              ``>=14.0.1``
-=======================================  ==================
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.10.0``
+``apache-airflow-providers-common-compat``  ``>=1.6.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
+``requests``                                ``>=2.31.0,<3``
+``databricks-sql-connector``                ``>=3.0.0``
+``databricks-sqlalchemy``                   ``>=1.0.2``
+``aiohttp``                                 ``>=3.9.2,<4``
+``mergedeep``                               ``>=1.3.4``
+``pandas``                                  ``>=2.1.2,<2.2``
+``pyarrow``                                 ``>=14.0.1``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -130,5 +136,5 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.4.0/changelog.html>`_.
 

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-databricks``
 
-Release: ``7.3.2``
+Release: ``7.4.0``
 
 
 `Databricks <https://databricks.com/>`__
@@ -36,7 +36,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro
 are in ``airflow.providers.databricks`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.4.0/>`_.
 
 Installation
 ------------
@@ -50,18 +50,20 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.9.0``
-``apache-airflow-providers-common-sql``  ``>=1.20.0``
-``requests``                             ``>=2.31.0,<3``
-``databricks-sql-connector``             ``>=3.0.0``
-``aiohttp``                              ``>=3.9.2,<4``
-``mergedeep``                            ``>=1.3.4``
-``pandas``                               ``>=2.1.2,<2.2``
-``pyarrow``                              ``>=14.0.1``
-=======================================  ==================
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.10.0``
+``apache-airflow-providers-common-compat``  ``>=1.6.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
+``requests``                                ``>=2.31.0,<3``
+``databricks-sql-connector``                ``>=3.0.0``
+``databricks-sqlalchemy``                   ``>=1.0.2``
+``aiohttp``                                 ``>=3.9.2,<4``
+``mergedeep``                               ``>=1.3.4``
+``pandas``                                  ``>=2.1.2,<2.2``
+``pyarrow``                                 ``>=14.0.1``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -84,4 +86,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.4.0/changelog.html>`_.

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/docs/changelog.rst

@@ -26,6 +26,32 @@
 Changelog
 ---------
 
+7.4.0
+.....
+
+.. note::
+    This release of provider is only available for Airflow 2.10+ as explained in the
+    Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>_.
+
+Features
+~~~~~~~~
+
+* ``Upgrade 'DatabricksSqlHook' to support polars (#50523)``
+* ``feat: Add OpenLineage support for DatabricksSqlHook (#50392)``
+* ``enhance: logs SQL before execution in 'snowflake' and 'databricks_sql' (#48942)``
+
+Misc
+~~~~
+
+* ``Remove AIRFLOW_2_10_PLUS conditions (#49877)``
+* ``Bump min Airflow version in providers to 2.10 (#49843)``
+* ``chore: import paths use the stable functions (#49460)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``Update description of provider.yaml dependencies (#50231)``
+   * ``Avoid committing history for providers (#49907)``
+
 7.3.2
 .....
 

apache_airflow_providers_databricks-7.4.0rc1/docs/commits.rst

@@ -0,0 +1,35 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+    `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+ .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN!
+
+Package apache-airflow-providers-databricks
+------------------------------------------------------
+
+`Databricks <https://databricks.com/>`__
+
+
+This is detailed commit list of changes for versions provider package: ``databricks``.
+For high-level changelog, see :doc:`package information including changelog <index>`.
+
+.. airflow-providers-commits::

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/docs/index.rst

@@ -78,7 +78,7 @@ apache-airflow-providers-databricks package
 `Databricks <https://databricks.com/>`__
 
 
-Release: 7.3.2
+Release: 7.4.0
 
 Provider package
 ----------------
@@ -96,20 +96,22 @@ For the minimum Airflow version supported, see ``Requirements`` below.
 Requirements
 ------------
 
-The minimum Apache Airflow version supported by this provider distribution is ``2.9.0``.
-
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.9.0``
-``apache-airflow-providers-common-sql``  ``>=1.20.0``
-``requests``                             ``>=2.31.0,<3``
-``databricks-sql-connector``             ``>=3.0.0``
-``aiohttp``                              ``>=3.9.2,<4``
-``mergedeep``                            ``>=1.3.4``
-``pandas``                               ``>=2.1.2,<2.2``
-``pyarrow``                              ``>=14.0.1``
-=======================================  ==================
+The minimum Apache Airflow version supported by this provider distribution is ``2.10.0``.
+
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.10.0``
+``apache-airflow-providers-common-compat``  ``>=1.6.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
+``requests``                                ``>=2.31.0,<3``
+``databricks-sql-connector``                ``>=3.0.0``
+``databricks-sqlalchemy``                   ``>=1.0.2``
+``aiohttp``                                 ``>=3.9.2,<4``
+``mergedeep``                               ``>=1.3.4``
+``pandas``                                  ``>=2.1.2,<2.2``
+``pyarrow``                                 ``>=14.0.1``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -137,5 +139,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-databricks 7.3.2 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.3.2.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.3.2.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.3.2.tar.gz.sha512>`__)
-* `The apache-airflow-providers-databricks 7.3.2 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.3.2-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.3.2-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.3.2-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-databricks 7.4.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.4.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.4.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.4.0.tar.gz.sha512>`__)
+* `The apache-airflow-providers-databricks 7.4.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.4.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.4.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_databricks-7.4.0-py3-none-any.whl.sha512>`__)

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/provider.yaml

@@ -22,9 +22,13 @@ description: |
     `Databricks <https://databricks.com/>`__
 
 state: ready
-source-date-epoch: 1744788983
-# note that those versions are maintained by release manager - do not update them manually
+source-date-epoch: 1747132432
+# Note that those versions are maintained by release manager - do not update them manually
+# with the exception of case where other provider in sources has >= new provider version.
+# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
+# to be done in the same PR
 versions:
+  - 7.4.0
   - 7.3.2
   - 7.3.1
   - 7.3.0

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-databricks"
-version = "7.3.2.rc1"
+version = "7.4.0rc1"
 description = "Provider package apache-airflow-providers-databricks for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -57,10 +57,12 @@ requires-python = "~=3.9"
 # Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
-    "apache-airflow>=2.9.0rc0",
-    "apache-airflow-providers-common-sql>=1.20.0rc0",
+    "apache-airflow>=2.10.0rc1",
+    "apache-airflow-providers-common-compat>=1.6.0rc1",
+    "apache-airflow-providers-common-sql>=1.27.0rc1",
     "requests>=2.31.0,<3",
     "databricks-sql-connector>=3.0.0",
+    "databricks-sqlalchemy>=1.0.2",
     "aiohttp>=3.9.2, <4",
     "mergedeep>=1.3.4",
     "pandas>=2.1.2,<2.2",
@@ -83,17 +85,23 @@ dependencies = [
 "standard" = [
     "apache-airflow-providers-standard"
 ]
+"openlineage" = [
+    "apache-airflow-providers-openlineage>=2.3.0rc1"
+]
 
 [dependency-groups]
 dev = [
     "apache-airflow",
     "apache-airflow-task-sdk",
     "apache-airflow-devel-common",
+    "apache-airflow-providers-common-compat",
     "apache-airflow-providers-common-sql",
     "apache-airflow-providers-fab",
+    "apache-airflow-providers-openlineage",
     # Additional devel dependencies (do not remove this line and add extra development dependencies)
     "deltalake>=0.12.0",
     "apache-airflow-providers-microsoft-azure",
+    "apache-airflow-providers-common-sql[pandas,polars]",
 ]
 
 # To build docs:
@@ -122,8 +130,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.3.2/changelog.html"
+"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.4.0"
+"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-databricks/7.4.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/src/airflow/providers/databricks/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "7.3.2"
+__version__ = "7.4.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.9.0"
+    "2.10.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.9.0+"
+        f"The package `apache-airflow-providers-databricks:{__version__}` needs Apache Airflow 2.10.0+"
     )

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/src/airflow/providers/databricks/hooks/databricks_sql.py

@@ -35,7 +35,8 @@ from databricks import sql  # type: ignore[attr-defined]
 from databricks.sql.types import Row
 
 from airflow.exceptions import AirflowException
-from airflow.providers.common.sql.hooks.sql import DbApiHook, return_single_query_results
+from airflow.providers.common.sql.hooks.handlers import return_single_query_results
+from airflow.providers.common.sql.hooks.sql import DbApiHook
 from airflow.providers.databricks.exceptions import DatabricksSqlExecutionError, DatabricksSqlExecutionTimeout
 from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook
 
@@ -43,6 +44,8 @@ if TYPE_CHECKING:
     from databricks.sql.client import Connection
 
     from airflow.models.connection import Connection as AirflowConnection
+    from airflow.providers.openlineage.extractors import OperatorLineage
+    from airflow.providers.openlineage.sqlparser import DatabaseInfo
 
 
 LIST_SQL_ENDPOINTS_ENDPOINT = ("GET", "api/2.0/sql/endpoints")
@@ -107,6 +110,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         self.catalog = catalog
         self.schema = schema
         self.additional_params = kwargs
+        self.query_ids: list[str] = []
 
     def _get_extra_config(self) -> dict[str, Any | None]:
         extra_params = copy(self.databricks_conn.extra_dejson)
@@ -226,6 +230,8 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
             it will raise and fail.
         """
         self.descriptions = []
+        self.query_ids = []
+
         if isinstance(sql, str):
             if split_statements:
                 sql_list = [self.strip_sql_string(s) for s in self.split_sql_string(sql)]
@@ -242,6 +248,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
         conn = None
         results = []
         for sql_statement in sql_list:
+            self.log.info("Running statement: %s, parameters: %s", sql_statement, parameters)
             # when using AAD tokens, it could expire if previous query run longer than token lifetime
             conn = self.get_conn()
             with closing(conn.cursor()) as cur:
@@ -265,6 +272,10 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
                         if t is not None:
                             t.cancel()
 
+                    if query_id := cur.query_id:
+                        self.log.info("Databricks query id: %s", query_id)
+                        self.query_ids.append(query_id)
+
                     if handler is not None:
                         raw_result = handler(cur)
                         result = self._make_common_data_structure(raw_result)
@@ -307,3 +318,80 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
 
     def bulk_load(self, table, tmp_file):
         raise NotImplementedError()
+
+    def get_openlineage_database_info(self, connection) -> DatabaseInfo:
+        from airflow.providers.openlineage.sqlparser import DatabaseInfo
+
+        return DatabaseInfo(
+            scheme=self.get_openlineage_database_dialect(connection),
+            authority=self.host,
+            database=self.catalog,
+            information_schema_columns=[
+                "table_schema",
+                "table_name",
+                "column_name",
+                "ordinal_position",
+                "data_type",
+                "table_catalog",
+            ],
+            is_information_schema_cross_db=True,
+        )
+
+    def get_openlineage_database_dialect(self, _) -> str:
+        return "databricks"
+
+    def get_openlineage_default_schema(self) -> str | None:
+        return self.schema or "default"
+
+    def get_openlineage_database_specific_lineage(self, task_instance) -> OperatorLineage | None:
+        """
+        Generate OpenLineage metadata for a Databricks task instance based on executed query IDs.
+
+        If a single query ID is present, attach an `ExternalQueryRunFacet` to the lineage metadata.
+        If multiple query IDs are present, emits separate OpenLineage events for each query instead.
+
+        Note that `get_openlineage_database_specific_lineage` is usually called after task's execution,
+        so if multiple query IDs are present, both START and COMPLETE event for each query will be emitted
+        after task's execution. If we are able to query Databricks for query execution metadata,
+        query event times will correspond to actual query's start and finish times.
+
+        Args:
+            task_instance: The Airflow TaskInstance object for which lineage is being collected.
+
+        Returns:
+            An `OperatorLineage` object if a single query ID is found; otherwise `None`.
+        """
+        from airflow.providers.common.compat.openlineage.facet import ExternalQueryRunFacet
+        from airflow.providers.databricks.utils.openlineage import (
+            emit_openlineage_events_for_databricks_queries,
+        )
+        from airflow.providers.openlineage.extractors import OperatorLineage
+        from airflow.providers.openlineage.sqlparser import SQLParser
+
+        if not self.query_ids:
+            self.log.debug("openlineage: no databricks query ids found.")
+            return None
+
+        self.log.debug("openlineage: getting connection to get database info")
+        connection = self.get_connection(self.get_conn_id())
+        namespace = SQLParser.create_namespace(self.get_openlineage_database_info(connection))
+
+        if len(self.query_ids) == 1:
+            self.log.debug("Attaching ExternalQueryRunFacet with single query_id to OpenLineage event.")
+            return OperatorLineage(
+                run_facets={
+                    "externalQuery": ExternalQueryRunFacet(
+                        externalQueryId=self.query_ids[0], source=namespace
+                    )
+                }
+            )
+
+        self.log.info("Multiple query_ids found. Separate OpenLineage event will be emitted for each query.")
+        emit_openlineage_events_for_databricks_queries(
+            query_ids=self.query_ids,
+            query_source_namespace=namespace,
+            task_instance=task_instance,
+            hook=self,
+        )
+
+        return None

{apache_airflow_providers_databricks-7.3.2rc1 → apache_airflow_providers_databricks-7.4.0rc1}/src/airflow/providers/databricks/operators/databricks_sql.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 import csv
 import json
 from collections.abc import Sequence
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, ClassVar
 
 from databricks.sql.utils import ParamEscaper
@@ -106,7 +107,8 @@ class DatabricksSqlOperator(SQLExecuteQueryOperator):
         self.catalog = catalog
         self.schema = schema
 
-    def get_db_hook(self) -> DatabricksSqlHook:
+    @cached_property
+    def _hook(self) -> DatabricksSqlHook:
         hook_params = {
             "http_path": self.http_path,
             "session_configuration": self.session_configuration,
@@ -120,6 +122,9 @@ class DatabricksSqlOperator(SQLExecuteQueryOperator):
         }
         return DatabricksSqlHook(self.databricks_conn_id, **hook_params)
 
+    def get_db_hook(self) -> DatabricksSqlHook:
+        return self._hook
+
     def _should_run_output_processing(self) -> bool:
         return self.do_xcom_push or bool(self._output_path)