apache-airflow-providers-dbt-cloud 4.2.1__py3-none-any.whl → 4.3.1rc1__py3-none-any.whl

airflow/providers/dbt/cloud/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "4.2.1"
+__version__ = "4.3.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/dbt/cloud/get_provider_info.py

@@ -27,8 +27,9 @@ def get_provider_info():
         "name": "dbt Cloud",
         "description": "`dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1741508729,
+        "source-date-epoch": 1742980047,
         "versions": [
+            "4.3.1",
             "4.2.1",
             "4.2.0",
             "4.0.0",

airflow/providers/dbt/cloud/operators/dbt.py

@@ -24,7 +24,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from airflow.configuration import conf
-from airflow.models import BaseOperator, XCom
+from airflow.models import BaseOperator
 from airflow.providers.common.compat.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.providers.dbt.cloud.hooks.dbt import (
     DbtCloudHook,
@@ -41,7 +41,9 @@ if TYPE_CHECKING:
 
 if AIRFLOW_V_3_0_PLUS:
     from airflow.sdk import BaseOperatorLink
+    from airflow.sdk.execution_time.xcom import XCom
 else:
+    from airflow.models import XCom  # type: ignore[no-redef]
     from airflow.models.baseoperatorlink import BaseOperatorLink  # type: ignore[no-redef]
 
 
@@ -284,9 +286,13 @@ class DbtCloudRunJobOperator(BaseOperator):
         """
         from airflow.providers.openlineage.extractors import OperatorLineage
 
-        if isinstance(self.run_id, int) and self.wait_for_termination is True:
-            return generate_openlineage_events_from_dbt_cloud_run(operator=self, task_instance=task_instance)
-        return OperatorLineage()
+        if not isinstance(self.run_id, int):
+            self.log.info("Skipping OpenLineage event extraction: `self.run_id` is not set.")
+            return OperatorLineage()
+        if not self.wait_for_termination:
+            self.log.info("Skipping OpenLineage event extraction: `self.wait_for_termination` is False.")
+            return OperatorLineage()
+        return generate_openlineage_events_from_dbt_cloud_run(operator=self, task_instance=task_instance)
 
 
 class DbtCloudGetJobRunArtifactOperator(BaseOperator):

airflow/providers/dbt/cloud/utils/openlineage.py

@@ -17,39 +17,41 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 import re
-from contextlib import suppress
 from typing import TYPE_CHECKING
 
-from packaging.version import parse
-
-from airflow import __version__ as airflow_version
+from airflow.providers.dbt.cloud.version_compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS
 
 if TYPE_CHECKING:
-    from packaging.version import Version
-
     from airflow.models.taskinstance import TaskInstance
     from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator
     from airflow.providers.dbt.cloud.sensors.dbt import DbtCloudJobRunSensor
     from airflow.providers.openlineage.extractors.base import OperatorLineage
 
 
-_AIRFLOW_VERSION: Version = parse(parse(airflow_version).base_version)
+log = logging.getLogger(__name__)
 
 
 def _get_logical_date(task_instance):
     # todo: remove when min airflow version >= 3.0
-    if parse("3") > _AIRFLOW_VERSION:
-        return task_instance.execution_date
-    return task_instance.logical_date
+    if AIRFLOW_V_3_0_PLUS:
+        dagrun = task_instance.get_template_context()["dag_run"]
+        return dagrun.logical_date or dagrun.run_after
+
+    if hasattr(task_instance, "logical_date"):
+        date = task_instance.logical_date
+    else:
+        date = task_instance.execution_date
+
+    return date
 
 
 def _get_try_number(val):
     # todo: remove when min airflow version >= 2.10.0
-    if parse("2.10.0") > _AIRFLOW_VERSION:
-        return val.try_number - 1
-    else:
+    if AIRFLOW_V_2_10_PLUS:
         return val.try_number
+    return val.try_number - 1
 
 
 def generate_openlineage_events_from_dbt_cloud_run(
@@ -87,6 +89,7 @@ def generate_openlineage_events_from_dbt_cloud_run(
     from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
 
     # if no account_id set this will fallback
+    log.debug("Retrieving information about DBT job run.")
     job_run = operator.hook.get_job_run(
         run_id=operator.run_id, account_id=operator.account_id, include_related=["run_steps,job"]
     ).json()["data"]
@@ -98,6 +101,7 @@ def generate_openlineage_events_from_dbt_cloud_run(
     execute_steps = job["execute_steps"]
     run_steps = job_run["run_steps"]
 
+    log.debug("Filtering only DBT invocation steps for further processing.")
     # filter only dbt invocation steps
     steps = []
     for run_step in run_steps:
@@ -110,8 +114,15 @@ def generate_openlineage_events_from_dbt_cloud_run(
 
     # catalog is available only if docs are generated
     catalog = None
-    with suppress(Exception):
+    try:
+        log.debug("Retrieving information about catalog artifact from DBT.")
         catalog = operator.hook.get_job_run_artifact(operator.run_id, path="catalog.json").json()["data"]
+    except Exception:  # type: ignore
+        log.info(
+            "Openlineage could not find DBT catalog artifact, usually available when docs are generated."
+            "Proceeding with metadata extraction. "
+            "If you see error logs above about `HTTP error: Not Found` it's safe to ignore them."
+        )
 
     async def get_artifacts_for_steps(steps, artifacts):
         """Get artifacts for a list of steps concurrently."""
@@ -127,16 +138,37 @@ def generate_openlineage_events_from_dbt_cloud_run(
         return await asyncio.gather(*tasks)
 
     # get artifacts for steps concurrently
+    log.debug("Retrieving information about artifacts for all job steps from DBT.")
     step_artifacts = asyncio.run(
         get_artifacts_for_steps(steps=steps, artifacts=["manifest.json", "run_results.json"])
     )
 
+    log.debug("Preparing OpenLineage parent job information to be included in DBT events.")
+    # generate same run id of current task instance
+    parent_run_id = OpenLineageAdapter.build_task_instance_run_id(
+        dag_id=task_instance.dag_id,
+        task_id=operator.task_id,
+        logical_date=_get_logical_date(task_instance),
+        try_number=_get_try_number(task_instance),
+        map_index=task_instance.map_index,
+    )
+
+    parent_job = ParentRunMetadata(
+        run_id=parent_run_id,
+        job_name=f"{task_instance.dag_id}.{task_instance.task_id}",
+        job_namespace=namespace(),
+    )
+    client = get_openlineage_listener().adapter.get_or_create_openlineage_client()
+
     # process each step in loop, sending generated events in the same order as steps
-    for artifacts in step_artifacts:
+    for counter, artifacts in enumerate(step_artifacts, 1):
+        log.debug("Parsing information about artifact no. %s.", counter)
+
         # process manifest
         manifest = artifacts["manifest.json"]
 
         if not artifacts.get("run_results.json", None):
+            log.debug("No run results found for artifact no. %s. Skipping.", counter)
             continue
 
         processor = DbtCloudArtifactProcessor(
@@ -150,26 +182,14 @@ def generate_openlineage_events_from_dbt_cloud_run(
             catalog=catalog,
         )
 
-        # generate same run id of current task instance
-        parent_run_id = OpenLineageAdapter.build_task_instance_run_id(
-            dag_id=task_instance.dag_id,
-            task_id=operator.task_id,
-            logical_date=_get_logical_date(task_instance),
-            try_number=_get_try_number(task_instance),
-            map_index=task_instance.map_index,
-        )
-
-        parent_job = ParentRunMetadata(
-            run_id=parent_run_id,
-            job_name=f"{task_instance.dag_id}.{task_instance.task_id}",
-            job_namespace=namespace(),
-        )
         processor.dbt_run_metadata = parent_job
 
         events = processor.parse().events()
-
-        client = get_openlineage_listener().adapter.get_or_create_openlineage_client()
+        log.debug("Found %s OpenLineage events for artifact no. %s.", len(events), counter)
 
         for event in events:
             client.emit(event=event)
+        log.debug("Emitted all OpenLineage events for artifact no. %s.", counter)
+
+    log.info("OpenLineage has successfully finished processing information about DBT job run.")
     return OperatorLineage()

airflow/providers/dbt/cloud/version_compat.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
+# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
+# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
+# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
+#
+from __future__ import annotations
+
+
+def get_base_airflow_version_tuple() -> tuple[int, int, int]:
+    from packaging.version import Version
+
+    from airflow import __version__
+
+    airflow_version = Version(__version__)
+    return airflow_version.major, airflow_version.minor, airflow_version.micro
+
+
+AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
+AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)

{apache_airflow_providers_dbt_cloud-4.2.1.dist-info → apache_airflow_providers_dbt_cloud-4.3.1rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-dbt-cloud
-Version: 4.2.1
+Version: 4.3.1rc1
 Summary: Provider package apache-airflow-providers-dbt-cloud for Apache Airflow
 Keywords: airflow-provider,dbt.cloud,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,18 +20,18 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0
+Requires-Dist: apache-airflow>=2.9.0rc0
 Requires-Dist: apache-airflow-providers-http
 Requires-Dist: asgiref>=2.3.0
 Requires-Dist: aiohttp>=3.9.2
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
 Requires-Dist: apache-airflow-providers-openlineage>=1.7.0 ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.2.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.2.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.3.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.3.1
+Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
-Project-URL: Twitter, https://x.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: common-compat
 Provides-Extra: openlineage
@@ -61,7 +61,7 @@ Provides-Extra: openlineage
 
 Package ``apache-airflow-providers-dbt-cloud``
 
-Release: ``4.2.1``
+Release: ``4.3.1``
 
 
 `dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__
@@ -74,7 +74,7 @@ This is a provider package for ``dbt.cloud`` provider. All classes for this prov
 are in ``airflow.providers.dbt.cloud`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.2.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.3.1/>`_.
 
 Installation
 ------------
@@ -101,7 +101,7 @@ Cross provider package dependencies
 -----------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider packages in order to use them.
+You need to install the specified providers in order to use them.
 
 You can install such cross-provider dependencies when installing from PyPI. For example:
 
@@ -119,5 +119,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.2.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.3.1/changelog.html>`_.
 

apache_airflow_providers_dbt_cloud-4.3.1rc1.dist-info/RECORD

@@ -0,0 +1,18 @@
+airflow/providers/dbt/cloud/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+airflow/providers/dbt/cloud/__init__.py,sha256=uuy3l3sjJZJG9-NzfvYGWd1ab5Olcmml4RTNn3DH3-Q,1496
+airflow/providers/dbt/cloud/get_provider_info.py,sha256=lo5ZpzaiVezp6X0U2DqzGfTQBeM6XCiTOAsT-ArEgZg,3776
+airflow/providers/dbt/cloud/version_compat.py,sha256=aHg90_DtgoSnQvILFICexMyNlHlALBdaeWqkX3dFDug,1605
+airflow/providers/dbt/cloud/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/dbt/cloud/hooks/dbt.py,sha256=AxAEyeBewS0UQUtZMHLMbQmDAln3A_x3WdIrtzzK4Pk,34556
+airflow/providers/dbt/cloud/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/dbt/cloud/operators/dbt.py,sha256=dwagI9sMAXmzNRKRUsr7VvKaHJFdKNMj4iaz-Av8W3A,18043
+airflow/providers/dbt/cloud/sensors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/dbt/cloud/sensors/dbt.py,sha256=OGf-VNKcCNCrImrdE8PudmtDak_MKJfPhmaf7_ccOLg,5157
+airflow/providers/dbt/cloud/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/dbt/cloud/triggers/dbt.py,sha256=Oabdc7FcNhCQxkjDC5SqAiYEw4hSZ9mQGZgSt36a1E0,4707
+airflow/providers/dbt/cloud/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/dbt/cloud/utils/openlineage.py,sha256=UueFNMgYw7E63uLZntnjzdEQ0V04vOO0M1YNBs5fF_c,7768
+apache_airflow_providers_dbt_cloud-4.3.1rc1.dist-info/entry_points.txt,sha256=c18L1WEEK18WQeEGrm9kMVqutiYJHiWGH5jU1JqnToE,105
+apache_airflow_providers_dbt_cloud-4.3.1rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+apache_airflow_providers_dbt_cloud-4.3.1rc1.dist-info/METADATA,sha256=pxHD0xtWflLyRPXx2HcJZcycN1r-cIqKl8Z2mbsZ-lY,5695
+apache_airflow_providers_dbt_cloud-4.3.1rc1.dist-info/RECORD,,

apache_airflow_providers_dbt_cloud-4.2.1.dist-info/RECORD

@@ -1,17 +0,0 @@
-airflow/providers/dbt/cloud/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/dbt/cloud/__init__.py,sha256=8bJh6-jVlHon3BqfhpuhOPvcwV1m6rsmzfooP3X2nLc,1496
-airflow/providers/dbt/cloud/get_provider_info.py,sha256=sdFwAL1GPiXWL40JiXqkJZQcgy3nhkl0IZaJJPWlanI,3755
-airflow/providers/dbt/cloud/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/dbt/cloud/hooks/dbt.py,sha256=AxAEyeBewS0UQUtZMHLMbQmDAln3A_x3WdIrtzzK4Pk,34556
-airflow/providers/dbt/cloud/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/dbt/cloud/operators/dbt.py,sha256=mk5-z9r230EWWwDH06KtrjAUyZ5lIFe7qRT6RuhkSIA,17689
-airflow/providers/dbt/cloud/sensors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/dbt/cloud/sensors/dbt.py,sha256=OGf-VNKcCNCrImrdE8PudmtDak_MKJfPhmaf7_ccOLg,5157
-airflow/providers/dbt/cloud/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/dbt/cloud/triggers/dbt.py,sha256=Oabdc7FcNhCQxkjDC5SqAiYEw4hSZ9mQGZgSt36a1E0,4707
-airflow/providers/dbt/cloud/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/dbt/cloud/utils/openlineage.py,sha256=6hGaRpfPE0HJSyEQNo4xt9Pfvdi9GI-mysIb4dBACpk,6611
-apache_airflow_providers_dbt_cloud-4.2.1.dist-info/entry_points.txt,sha256=c18L1WEEK18WQeEGrm9kMVqutiYJHiWGH5jU1JqnToE,105
-apache_airflow_providers_dbt_cloud-4.2.1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
-apache_airflow_providers_dbt_cloud-4.2.1.dist-info/METADATA,sha256=FleQZDY3yK-UJI2XNqppNER_7WNsKbQm6aUkT1dh8Mw,5693
-apache_airflow_providers_dbt_cloud-4.2.1.dist-info/RECORD,,