apache-airflow-providers-dbt-cloud 3.6.1__py3-none-any.whl → 3.7.0rc1__py3-none-any.whl

airflow/providers/dbt/cloud/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "3.6.1"
+__version__ = "3.7.0"
 
 try:
     from airflow import __version__ as airflow_version

airflow/providers/dbt/cloud/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "dbt Cloud",
         "description": "`dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1707636353,
+        "source-date-epoch": 1709555328,
         "versions": [
+            "3.7.0",
             "3.6.1",
             "3.6.0",
             "3.5.1",

airflow/providers/dbt/cloud/hooks/dbt.py

@@ -109,6 +109,7 @@ class DbtCloudJobRunStatus(Enum):
     SUCCESS = 10
     ERROR = 20
     CANCELLED = 30
+    NON_TERMINAL_STATUSES = (QUEUED, STARTING, RUNNING)
     TERMINAL_STATUSES = (SUCCESS, ERROR, CANCELLED)
 
     @classmethod
@@ -460,6 +461,20 @@ class DbtCloudHook(HttpHook):
             paginate=True,
         )
 
+    @fallback_to_default_account
+    def get_job_runs(self, account_id: int | None = None, payload: dict[str, Any] | None = None) -> Response:
+        """
+        Retrieve metadata for a specific run of a dbt Cloud job.
+
+        :param account_id: Optional. The ID of a dbt Cloud account.
+        :param paylod: Optional. Query Parameters
+        :return: The request response.
+        """
+        return self._run_and_get_response(
+            endpoint=f"{account_id}/runs/",
+            payload=payload,
+        )
+
     @fallback_to_default_account
     def get_job_run(
         self, run_id: int, account_id: int | None = None, include_related: list[str] | None = None

airflow/providers/dbt/cloud/operators/dbt.py

@@ -73,6 +73,8 @@ class DbtCloudRunJobOperator(BaseOperator):
         Used only if ``wait_for_termination`` is True. Defaults to 60 seconds.
     :param additional_run_config: Optional. Any additional parameters that should be included in the API
         request when triggering the job.
+    :param reuse_existing_run: Flag to determine whether to reuse existing non terminal job run. If set to
+        true and non terminal job runs found, it use the latest run without triggering a new job run.
     :param deferrable: Run operator in the deferrable mode
     :return: The ID of the triggered dbt Cloud job run.
     """
@@ -102,6 +104,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         timeout: int = 60 * 60 * 24 * 7,
         check_interval: int = 60,
         additional_run_config: dict[str, Any] | None = None,
+        reuse_existing_run: bool = False,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs,
     ) -> None:
@@ -117,6 +120,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         self.check_interval = check_interval
         self.additional_run_config = additional_run_config or {}
         self.run_id: int | None = None
+        self.reuse_existing_run = reuse_existing_run
         self.deferrable = deferrable
 
     def execute(self, context: Context):
@@ -125,16 +129,32 @@ class DbtCloudRunJobOperator(BaseOperator):
                 f"Triggered via Apache Airflow by task {self.task_id!r} in the {self.dag.dag_id} DAG."
             )
 
-        trigger_job_response = self.hook.trigger_job_run(
-            account_id=self.account_id,
-            job_id=self.job_id,
-            cause=self.trigger_reason,
-            steps_override=self.steps_override,
-            schema_override=self.schema_override,
-            additional_run_config=self.additional_run_config,
-        )
-        self.run_id = trigger_job_response.json()["data"]["id"]
-        job_run_url = trigger_job_response.json()["data"]["href"]
+        non_terminal_runs = None
+        if self.reuse_existing_run:
+            non_terminal_runs = self.hook.get_job_runs(
+                account_id=self.account_id,
+                payload={
+                    "job_definition_id": self.job_id,
+                    "status": DbtCloudJobRunStatus.NON_TERMINAL_STATUSES,
+                    "order_by": "-created_at",
+                },
+            ).json()["data"]
+            if non_terminal_runs:
+                self.run_id = non_terminal_runs[0]["id"]
+                job_run_url = non_terminal_runs[0]["href"]
+
+        if not self.reuse_existing_run or not non_terminal_runs:
+            trigger_job_response = self.hook.trigger_job_run(
+                account_id=self.account_id,
+                job_id=self.job_id,
+                cause=self.trigger_reason,
+                steps_override=self.steps_override,
+                schema_override=self.schema_override,
+                additional_run_config=self.additional_run_config,
+            )
+            self.run_id = trigger_job_response.json()["data"]["id"]
+            job_run_url = trigger_job_response.json()["data"]["href"]
+
         # Push the ``job_run_url`` value to XCom regardless of what happens during execution so that the job
         # run can be monitored via the operator link.
         context["ti"].xcom_push(key="job_run_url", value=job_run_url)

{apache_airflow_providers_dbt_cloud-3.6.1.dist-info → apache_airflow_providers_dbt_cloud-3.7.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-dbt-cloud
-Version: 3.6.1
+Version: 3.7.0rc1
 Summary: Provider package apache-airflow-providers-dbt-cloud for Apache Airflow
 Keywords: airflow-provider,dbt.cloud,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,13 +22,13 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: aiohttp>=3.9.2
 Requires-Dist: apache-airflow-providers-http
-Requires-Dist: apache-airflow>=2.6.0
+Requires-Dist: apache-airflow>=2.6.0.dev0
 Requires-Dist: asgiref
 Requires-Dist: apache-airflow-providers-http ; extra == "http"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: openlineage
 
 Package ``apache-airflow-providers-dbt-cloud``
 
-Release: ``3.6.1``
+Release: ``3.7.0.rc1``
 
 
 `dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``dbt.cloud`` provider. All classes for this prov
 are in ``airflow.providers.dbt.cloud`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/>`_.
 
 Installation
 ------------
@@ -137,4 +137,4 @@ Dependent package
 ==============================================================================================================  ===============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/changelog.html>`_.

{apache_airflow_providers_dbt_cloud-3.6.1.dist-info → apache_airflow_providers_dbt_cloud-3.7.0rc1.dist-info}/RECORD

@@ -1,17 +1,17 @@
 airflow/providers/dbt/cloud/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
-airflow/providers/dbt/cloud/__init__.py,sha256=YXfnUFpwHQ2T1PurgtGDEkt3bw19VT3hIeKKByPQkZA,1584
-airflow/providers/dbt/cloud/get_provider_info.py,sha256=ibnxlmMBzvS2gnW2jhh0ghRnUiE2t5yBskE7iZpeSok,3266
+airflow/providers/dbt/cloud/__init__.py,sha256=trB9RLUFqEwMUcliz8Jm6Z2O7yotXITVfVide5sl_28,1584
+airflow/providers/dbt/cloud/get_provider_info.py,sha256=6BIE0tHosNh_5UXJpZ-QDM8tSm2261fxZ3A4EQYvenw,3287
 airflow/providers/dbt/cloud/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/dbt/cloud/hooks/dbt.py,sha256=VIPEcI4h8QCRTV6E7qwtxLi9Xd2yoZ3-5HSX_B2TDe4,25509
+airflow/providers/dbt/cloud/hooks/dbt.py,sha256=okv5AeLO-KI7kVUvTO98_6r-oytl8CqKGDg3uUCwPIo,26082
 airflow/providers/dbt/cloud/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/dbt/cloud/operators/dbt.py,sha256=HK7F8sk3JrLphxm95k_4K-HlrsyHI27kIHJA98qZP5I,14651
+airflow/providers/dbt/cloud/operators/dbt.py,sha256=TW1tpuIWm4EgoEpv4wGtrXcR_Ve32vFQfUbsJzXMseA,15630
 airflow/providers/dbt/cloud/sensors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/dbt/cloud/sensors/dbt.py,sha256=X8Q1YLj76qwj3nAy6a_aF4edID-oJWMfFLVSrrQMZ1k,6860
 airflow/providers/dbt/cloud/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/dbt/cloud/triggers/dbt.py,sha256=NqbxQLEbAHO5ctrxwMHLsJKU_NiqxzBwNyuODC3B2pA,4648
 airflow/providers/dbt/cloud/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/dbt/cloud/utils/openlineage.py,sha256=MTA-FTM3HsnPg5jm5Zf6Nq1PLxfZRYlnHib3dIxX2t0,5644
-apache_airflow_providers_dbt_cloud-3.6.1.dist-info/entry_points.txt,sha256=c18L1WEEK18WQeEGrm9kMVqutiYJHiWGH5jU1JqnToE,105
-apache_airflow_providers_dbt_cloud-3.6.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_dbt_cloud-3.6.1.dist-info/METADATA,sha256=9eHOcujpp8Hv-QgcWrO8ZykZS3LsXxhN5JtAVq5KiFo,6287
-apache_airflow_providers_dbt_cloud-3.6.1.dist-info/RECORD,,
+apache_airflow_providers_dbt_cloud-3.7.0rc1.dist-info/entry_points.txt,sha256=c18L1WEEK18WQeEGrm9kMVqutiYJHiWGH5jU1JqnToE,105
+apache_airflow_providers_dbt_cloud-3.7.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_dbt_cloud-3.7.0rc1.dist-info/METADATA,sha256=A85QiOHG1cNXhpYhnp0T0jSuLmznw8Zf0TEjHFGp1A4,6299
+apache_airflow_providers_dbt_cloud-3.7.0rc1.dist-info/RECORD,,