apache-airflow-providers-dbt-cloud 3.6.1rc1__tar.gz → 3.7.0rc1__tar.gz

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-dbt-cloud
-Version: 3.6.1rc1
+Version: 3.7.0rc1
 Summary: Provider package apache-airflow-providers-dbt-cloud for Apache Airflow
 Keywords: airflow-provider,dbt.cloud,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -27,8 +27,8 @@ Requires-Dist: asgiref
 Requires-Dist: apache-airflow-providers-http ; extra == "http"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: openlineage
 
 Package ``apache-airflow-providers-dbt-cloud``
 
-Release: ``3.6.1.rc1``
+Release: ``3.7.0.rc1``
 
 
 `dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``dbt.cloud`` provider. All classes for this prov
 are in ``airflow.providers.dbt.cloud`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/>`_.
 
 Installation
 ------------
@@ -137,4 +137,4 @@ Dependent package
 ==============================================================================================================  ===============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/changelog.html>`_.

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-dbt-cloud``
 
-Release: ``3.6.1.rc1``
+Release: ``3.7.0.rc1``
 
 
 `dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``dbt.cloud`` provider. All classes for this prov
 are in ``airflow.providers.dbt.cloud`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/>`_.
 
 Installation
 ------------
@@ -99,4 +99,4 @@ Dependent package
 ==============================================================================================================  ===============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/changelog.html>`_.

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/airflow/providers/dbt/cloud/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "3.6.1"
+__version__ = "3.7.0"
 
 try:
     from airflow import __version__ as airflow_version

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/airflow/providers/dbt/cloud/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "dbt Cloud",
         "description": "`dbt Cloud <https://www.getdbt.com/product/dbt-cloud/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1707636353,
+        "source-date-epoch": 1709555328,
         "versions": [
+            "3.7.0",
             "3.6.1",
             "3.6.0",
             "3.5.1",

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/airflow/providers/dbt/cloud/hooks/dbt.py

@@ -109,6 +109,7 @@ class DbtCloudJobRunStatus(Enum):
     SUCCESS = 10
     ERROR = 20
     CANCELLED = 30
+    NON_TERMINAL_STATUSES = (QUEUED, STARTING, RUNNING)
     TERMINAL_STATUSES = (SUCCESS, ERROR, CANCELLED)
 
     @classmethod
@@ -460,6 +461,20 @@ class DbtCloudHook(HttpHook):
             paginate=True,
         )
 
+    @fallback_to_default_account
+    def get_job_runs(self, account_id: int | None = None, payload: dict[str, Any] | None = None) -> Response:
+        """
+        Retrieve metadata for a specific run of a dbt Cloud job.
+
+        :param account_id: Optional. The ID of a dbt Cloud account.
+        :param paylod: Optional. Query Parameters
+        :return: The request response.
+        """
+        return self._run_and_get_response(
+            endpoint=f"{account_id}/runs/",
+            payload=payload,
+        )
+
     @fallback_to_default_account
     def get_job_run(
         self, run_id: int, account_id: int | None = None, include_related: list[str] | None = None

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/airflow/providers/dbt/cloud/operators/dbt.py

@@ -73,6 +73,8 @@ class DbtCloudRunJobOperator(BaseOperator):
         Used only if ``wait_for_termination`` is True. Defaults to 60 seconds.
     :param additional_run_config: Optional. Any additional parameters that should be included in the API
         request when triggering the job.
+    :param reuse_existing_run: Flag to determine whether to reuse existing non terminal job run. If set to
+        true and non terminal job runs found, it use the latest run without triggering a new job run.
     :param deferrable: Run operator in the deferrable mode
     :return: The ID of the triggered dbt Cloud job run.
     """
@@ -102,6 +104,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         timeout: int = 60 * 60 * 24 * 7,
         check_interval: int = 60,
         additional_run_config: dict[str, Any] | None = None,
+        reuse_existing_run: bool = False,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs,
     ) -> None:
@@ -117,6 +120,7 @@ class DbtCloudRunJobOperator(BaseOperator):
         self.check_interval = check_interval
         self.additional_run_config = additional_run_config or {}
         self.run_id: int | None = None
+        self.reuse_existing_run = reuse_existing_run
         self.deferrable = deferrable
 
     def execute(self, context: Context):
@@ -125,16 +129,32 @@ class DbtCloudRunJobOperator(BaseOperator):
                 f"Triggered via Apache Airflow by task {self.task_id!r} in the {self.dag.dag_id} DAG."
             )
 
-        trigger_job_response = self.hook.trigger_job_run(
-            account_id=self.account_id,
-            job_id=self.job_id,
-            cause=self.trigger_reason,
-            steps_override=self.steps_override,
-            schema_override=self.schema_override,
-            additional_run_config=self.additional_run_config,
-        )
-        self.run_id = trigger_job_response.json()["data"]["id"]
-        job_run_url = trigger_job_response.json()["data"]["href"]
+        non_terminal_runs = None
+        if self.reuse_existing_run:
+            non_terminal_runs = self.hook.get_job_runs(
+                account_id=self.account_id,
+                payload={
+                    "job_definition_id": self.job_id,
+                    "status": DbtCloudJobRunStatus.NON_TERMINAL_STATUSES,
+                    "order_by": "-created_at",
+                },
+            ).json()["data"]
+            if non_terminal_runs:
+                self.run_id = non_terminal_runs[0]["id"]
+                job_run_url = non_terminal_runs[0]["href"]
+
+        if not self.reuse_existing_run or not non_terminal_runs:
+            trigger_job_response = self.hook.trigger_job_run(
+                account_id=self.account_id,
+                job_id=self.job_id,
+                cause=self.trigger_reason,
+                steps_override=self.steps_override,
+                schema_override=self.schema_override,
+                additional_run_config=self.additional_run_config,
+            )
+            self.run_id = trigger_job_response.json()["data"]["id"]
+            job_run_url = trigger_job_response.json()["data"]["href"]
+
         # Push the ``job_run_url`` value to XCom regardless of what happens during execution so that the job
         # run can be monitored via the operator link.
         context["ti"].xcom_push(key="job_run_url", value=job_run_url)

{apache_airflow_providers_dbt_cloud-3.6.1rc1 → apache_airflow_providers_dbt_cloud-3.7.0rc1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-dbt-cloud"
-version = "3.6.1.rc1"
+version = "3.7.0.rc1"
 description = "Provider package apache-airflow-providers-dbt-cloud for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -62,8 +62,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.6.1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/3.7.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"