apache-airflow-providers-google 10.2.0rc1__py3-none-any.whl → 10.3.0rc1__py3-none-any.whl

airflow/providers/google/cloud/operators/bigquery_dts.py

@@ -32,6 +32,7 @@ from google.cloud.bigquery_datatransfer_v1 import (
 )
 
 from airflow import AirflowException
+from airflow.configuration import conf
 from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransferServiceHook, get_object_id
 from airflow.providers.google.cloud.links.bigquery_dts import BigQueryDataTransferConfigLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
@@ -279,7 +280,7 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
         metadata: Sequence[tuple[str, str]] = (),
         gcp_conn_id="google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)

airflow/providers/google/cloud/operators/cloud_build.py

@@ -28,6 +28,7 @@ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
 from google.api_core.retry import Retry
 from google.cloud.devtools.cloudbuild_v1.types import Build, BuildTrigger, RepoSource
 
+from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildHook
 from airflow.providers.google.cloud.links.cloud_build import (
@@ -176,7 +177,7 @@ class CloudBuildCreateBuildOperator(GoogleCloudBaseOperator):
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
         poll_interval: float = 4.0,
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         location: str = "global",
         **kwargs,
     ) -> None:

airflow/providers/google/cloud/operators/cloud_composer.py

@@ -27,6 +27,7 @@ from google.cloud.orchestration.airflow.service_v1.types import Environment
 from google.protobuf.field_mask_pb2 import FieldMask
 
 from airflow import AirflowException
+from airflow.configuration import conf
 from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerHook
 from airflow.providers.google.cloud.links.base import BaseGoogleLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
@@ -135,7 +136,7 @@ class CloudComposerCreateEnvironmentOperator(GoogleCloudBaseOperator):
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         pooling_period_seconds: int = 30,
         **kwargs,
     ) -> None:
@@ -264,7 +265,7 @@ class CloudComposerDeleteEnvironmentOperator(GoogleCloudBaseOperator):
         metadata: Sequence[tuple[str, str]] = (),
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         pooling_period_seconds: int = 30,
         **kwargs,
     ) -> None:
@@ -509,7 +510,7 @@ class CloudComposerUpdateEnvironmentOperator(GoogleCloudBaseOperator):
         metadata: Sequence[tuple[str, str]] = (),
         gcp_conn_id: str = "google_cloud_default",
         impersonation_chain: str | Sequence[str] | None = None,
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         pooling_period_seconds: int = 30,
         **kwargs,
     ) -> None:

airflow/providers/google/cloud/operators/cloud_sql.py

@@ -22,12 +22,14 @@ from typing import TYPE_CHECKING, Iterable, Mapping, Sequence
 
 from googleapiclient.errors import HttpError
 
+from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 from airflow.models import Connection
 from airflow.providers.google.cloud.hooks.cloud_sql import CloudSQLDatabaseHook, CloudSQLHook
 from airflow.providers.google.cloud.links.cloud_sql import CloudSQLInstanceDatabaseLink, CloudSQLInstanceLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
+from airflow.providers.google.cloud.triggers.cloud_sql import CloudSQLExportTrigger
 from airflow.providers.google.cloud.utils.field_validator import GcpBodyFieldValidator
 from airflow.providers.google.common.hooks.base_google import get_field
 from airflow.providers.google.common.links.storage import FileDetailsLink
@@ -215,8 +217,7 @@ CLOUD_SQL_DATABASE_PATCH_VALIDATION = [
 
 
 class CloudSQLBaseOperator(GoogleCloudBaseOperator):
-    """
-    Abstract base operator for Google Cloud SQL operators to inherit from.
+    """Abstract base operator for Google Cloud SQL operators.
 
     :param instance: Cloud SQL instance ID. This does not include the project ID.
     :param project_id: Optional, Google Cloud Project ID.  f set to None or missing,
@@ -284,8 +285,8 @@ class CloudSQLBaseOperator(GoogleCloudBaseOperator):
 
 
 class CloudSQLCreateInstanceOperator(CloudSQLBaseOperator):
-    """
-    Creates a new Cloud SQL instance.
+    """Create a new Cloud SQL instance.
+
     If an instance with the same name exists, no action will be taken and
     the operator will succeed.
 
@@ -385,8 +386,7 @@ class CloudSQLCreateInstanceOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLInstancePatchOperator(CloudSQLBaseOperator):
-    """
-    Updates settings of a Cloud SQL instance.
+    """Update settings of a Cloud SQL instance.
 
     Caution: This is a partial update, so only included values for the settings will be
     updated.
@@ -478,8 +478,7 @@ class CloudSQLInstancePatchOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLDeleteInstanceOperator(CloudSQLBaseOperator):
-    """
-    Deletes a Cloud SQL instance.
+    """Delete a Cloud SQL instance.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -525,8 +524,7 @@ class CloudSQLDeleteInstanceOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator):
-    """
-    Clones an instance to a target instance.
+    """Clone an instance to a target instance.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -617,8 +615,7 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator):
-    """
-    Creates a new database inside a Cloud SQL instance.
+    """Create a new database inside a Cloud SQL instance.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -722,9 +719,7 @@ class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLPatchInstanceDatabaseOperator(CloudSQLBaseOperator):
-    """
-    Updates a resource containing information about a database inside a Cloud SQL
-    instance using patch semantics.
+    """Update resource containing information about a database using patch semantics.
 
     See: https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch
 
@@ -827,8 +822,7 @@ class CloudSQLPatchInstanceDatabaseOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLDeleteInstanceDatabaseOperator(CloudSQLBaseOperator):
-    """
-    Deletes a database from a Cloud SQL instance.
+    """Delete a database from a Cloud SQL instance.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -907,9 +901,9 @@ class CloudSQLDeleteInstanceDatabaseOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
-    """
-    Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump
-    or CSV file.
+    """Export data from a Cloud SQL instance to a Cloud Storage bucket.
+
+    The exported format can be a SQL dump or CSV file.
 
     Note: This operator is idempotent. If executed multiple times with the same
     export file URI, the export file in GCS will simply be overridden.
@@ -934,6 +928,9 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
         If set as a sequence, the identities from the list must grant
         Service Account Token Creator IAM role to the directly preceding identity, with first
         account from the list granting this role to the originating account (templated).
+    :param deferrable: Run operator in the deferrable mode.
+    :param poke_interval: (Deferrable mode only) Time (seconds) to wait between calls
+        to check the run status.
     """
 
     # [START gcp_sql_export_template_fields]
@@ -959,10 +956,14 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
         api_version: str = "v1beta4",
         validate_body: bool = True,
         impersonation_chain: str | Sequence[str] | None = None,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+        poke_interval: int = 10,
         **kwargs,
     ) -> None:
         self.body = body
         self.validate_body = validate_body
+        self.deferrable = deferrable
+        self.poke_interval = poke_interval
         super().__init__(
             project_id=project_id,
             instance=instance,
@@ -1002,21 +1003,53 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
             uri=self.body["exportContext"]["uri"][5:],
             project_id=self.project_id or hook.project_id,
         )
-        return hook.export_instance(project_id=self.project_id, instance=self.instance, body=self.body)
+
+        operation_name = hook.export_instance(
+            project_id=self.project_id, instance=self.instance, body=self.body
+        )
+
+        if not self.deferrable:
+            return hook._wait_for_operation_to_complete(
+                project_id=self.project_id, operation_name=operation_name
+            )
+        else:
+            self.defer(
+                trigger=CloudSQLExportTrigger(
+                    operation_name=operation_name,
+                    project_id=self.project_id or hook.project_id,
+                    gcp_conn_id=self.gcp_conn_id,
+                    impersonation_chain=self.impersonation_chain,
+                    poke_interval=self.poke_interval,
+                ),
+                method_name="execute_complete",
+            )
+
+    def execute_complete(self, context, event=None) -> None:
+        """
+        Callback for when the trigger fires - returns immediately.
+        Relies on trigger to throw an exception, otherwise it assumes execution was
+        successful.
+        """
+        if event["status"] == "success":
+            self.log.info("Operation %s completed successfully", event["operation_name"])
+        else:
+            self.log.exception("Unexpected error in the operation.")
+            raise AirflowException(event["message"])
 
 
 class CloudSQLImportInstanceOperator(CloudSQLBaseOperator):
-    """
-    Imports data into a Cloud SQL instance from a SQL dump or CSV file in Cloud Storage.
+    """Import data into a Cloud SQL instance from Cloud Storage.
 
-    CSV IMPORT:
+    CSV IMPORT
+    ``````````
 
     This operator is NOT idempotent for a CSV import. If the same file is imported
     multiple times, the imported data will be duplicated in the database.
     Moreover, if there are any unique constraints the duplicate import may result in an
     error.
 
-    SQL IMPORT:
+    SQL IMPORT
+    ``````````
 
     This operator is idempotent for a SQL import if it was also exported by Cloud SQL.
     The exported SQL contains 'DROP TABLE IF EXISTS' statements for all tables
@@ -1117,9 +1150,10 @@ class CloudSQLImportInstanceOperator(CloudSQLBaseOperator):
 
 
 class CloudSQLExecuteQueryOperator(GoogleCloudBaseOperator):
-    """
-    Performs DML or DDL query on an existing Cloud Sql instance. It optionally uses
-    cloud-sql-proxy to establish secure connection with the database.
+    """Perform DML or DDL query on an existing Cloud Sql instance.
+
+    It optionally uses cloud-sql-proxy to establish secure connection with the
+    database.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:

airflow/providers/google/cloud/operators/dataflow.py

@@ -28,6 +28,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any, Sequence
 
 from airflow import AirflowException
+from airflow.configuration import conf
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType
 from airflow.providers.google.cloud.hooks.dataflow import (
@@ -172,7 +173,9 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
     This class is deprecated.
     Please use `providers.apache.beam.operators.beam.BeamRunJavaPipelineOperator`.
 
-    **Example**: ::
+    Example usage:
+
+    .. code-block:: python
 
         default_args = {
             "owner": "airflow",
@@ -417,7 +420,6 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
                     variables=pipeline_options,
                 )
                 while is_running and self.check_if_running == CheckJobRunning.WaitForRun:
-
                     is_running = self.dataflow_hook.is_job_dataflow_running(
                         name=self.job_name,
                         variables=pipeline_options,
@@ -609,7 +611,7 @@ class DataflowTemplatedJobStartOperator(GoogleCloudBaseOperator):
         cancel_timeout: int | None = 10 * 60,
         wait_until_finished: bool | None = None,
         append_job_name: bool = True,
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -799,7 +801,7 @@ class DataflowStartFlexTemplateOperator(GoogleCloudBaseOperator):
         cancel_timeout: int | None = 10 * 60,
         wait_until_finished: bool | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
-        deferrable: bool = False,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         append_job_name: bool = True,
         *args,
         **kwargs,

airflow/providers/google/cloud/operators/dataform.py

@@ -1029,8 +1029,9 @@ class DataformRemoveDirectoryOperator(GoogleCloudBaseOperator):
 
 
 class DataformInstallNpmPackagesOperator(GoogleCloudBaseOperator):
-    """
-    Installs npm dependencies in the provided workspace. Requires "package.json" to be created in workspace.
+    """Install NPM dependencies in the provided workspace.
+
+    Requires "package.json" to be created in the workspace.
 
     :param project_id: Required. The ID of the Google Cloud project where workspace located.
     :param region: Required. The ID of the Google Cloud region where workspace located.