apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl

airflow/providers/google/cloud/operators/cloud_composer.py

@@ -17,19 +17,23 @@
 # under the License.
 from __future__ import annotations
 
+import shlex
 from typing import TYPE_CHECKING, Sequence
 
 from google.api_core.exceptions import AlreadyExists
 from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
 from google.cloud.orchestration.airflow.service_v1 import ImageVersion
-from google.cloud.orchestration.airflow.service_v1.types import Environment
+from google.cloud.orchestration.airflow.service_v1.types import Environment, ExecuteAirflowCommandResponse
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.cloud_composer import CloudComposerHook
 from airflow.providers.google.cloud.links.base import BaseGoogleLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
-from airflow.providers.google.cloud.triggers.cloud_composer import CloudComposerExecutionTrigger
+from airflow.providers.google.cloud.triggers.cloud_composer import (
+    CloudComposerAirflowCLICommandTrigger,
+    CloudComposerExecutionTrigger,
+)
 from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
 
 if TYPE_CHECKING:
@@ -651,3 +655,144 @@ class CloudComposerListImageVersionsOperator(GoogleCloudBaseOperator):
             metadata=self.metadata,
         )
         return [ImageVersion.to_dict(image) for image in result]
+
+
+class CloudComposerRunAirflowCLICommandOperator(GoogleCloudBaseOperator):
+    """
+    Run Airflow command for provided Composer environment.
+
+    :param project_id: The ID of the Google Cloud project that the service belongs to.
+    :param region: The ID of the Google Cloud region that the service belongs to.
+    :param environment_id: The ID of the Google Cloud environment that the service belongs to.
+    :param command: Airflow command.
+    :param retry: Designation of what errors, if any, should be retried.
+    :param timeout: The timeout for this request.
+    :param metadata: Strings which should be sent along with the request as metadata.
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account (templated).
+    :param deferrable: Run operator in the deferrable mode
+    :param poll_interval: Optional: Control the rate of the poll for the result of deferrable run.
+        By default, the trigger will poll every 10 seconds.
+    """
+
+    template_fields = (
+        "project_id",
+        "region",
+        "environment_id",
+        "command",
+        "impersonation_chain",
+    )
+
+    def __init__(
+        self,
+        *,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        command: str,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+        poll_interval: int = 10,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.project_id = project_id
+        self.region = region
+        self.environment_id = environment_id
+        self.command = command
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self.deferrable = deferrable
+        self.poll_interval = poll_interval
+
+    def execute(self, context: Context):
+        hook = CloudComposerHook(
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.impersonation_chain,
+        )
+
+        self.log.info("Executing the command: [ airflow %s ]...", self.command)
+
+        cmd, subcommand, parameters = self._parse_cmd_to_args(self.command)
+        execution_cmd_info = hook.execute_airflow_command(
+            project_id=self.project_id,
+            region=self.region,
+            environment_id=self.environment_id,
+            command=cmd,
+            subcommand=subcommand,
+            parameters=parameters,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+        execution_cmd_info_dict = ExecuteAirflowCommandResponse.to_dict(execution_cmd_info)
+
+        self.log.info("Command has been started. execution_id=%s", execution_cmd_info_dict["execution_id"])
+
+        if self.deferrable:
+            self.defer(
+                trigger=CloudComposerAirflowCLICommandTrigger(
+                    project_id=self.project_id,
+                    region=self.region,
+                    environment_id=self.environment_id,
+                    execution_cmd_info=execution_cmd_info_dict,
+                    gcp_conn_id=self.gcp_conn_id,
+                    impersonation_chain=self.impersonation_chain,
+                    poll_interval=self.poll_interval,
+                ),
+                method_name=GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME,
+            )
+            return
+
+        result = hook.wait_command_execution_result(
+            project_id=self.project_id,
+            region=self.region,
+            environment_id=self.environment_id,
+            execution_cmd_info=execution_cmd_info_dict,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+            poll_interval=self.poll_interval,
+        )
+        result_str = self._merge_cmd_output_result(result)
+        self.log.info("Command execution result:\n%s", result_str)
+        return result
+
+    def execute_complete(self, context: Context, event: dict) -> dict:
+        if event and event["status"] == "error":
+            raise AirflowException(event["message"])
+        result: dict = event["result"]
+        result_str = self._merge_cmd_output_result(result)
+        self.log.info("Command execution result:\n%s", result_str)
+        return result
+
+    def _parse_cmd_to_args(self, cmd: str) -> tuple:
+        """Parse user command to command, subcommand and parameters."""
+        cmd_dict = shlex.split(cmd)
+        if not cmd_dict:
+            raise AirflowException("The provided command is empty.")
+
+        command = cmd_dict[0] if len(cmd_dict) >= 1 else None
+        subcommand = cmd_dict[1] if len(cmd_dict) >= 2 else None
+        parameters = cmd_dict[2:] if len(cmd_dict) >= 3 else None
+
+        return command, subcommand, parameters
+
+    def _merge_cmd_output_result(self, result) -> str:
+        """Merge output to one string."""
+        result_str = "\n".join(line_dict["content"] for line_dict in result["output"])
+        return result_str

airflow/providers/google/cloud/operators/cloud_memorystore.py

@@ -43,6 +43,7 @@ from airflow.providers.google.cloud.links.cloud_memorystore import (
     RedisInstanceListLink,
 )
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from google.api_core.retry import Retry
@@ -112,7 +113,7 @@ class CloudMemorystoreCreateInstanceOperator(GoogleCloudBaseOperator):
         location: str,
         instance_id: str,
         instance: dict | Instance,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -198,7 +199,7 @@ class CloudMemorystoreDeleteInstanceOperator(GoogleCloudBaseOperator):
         *,
         location: str,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -283,7 +284,7 @@ class CloudMemorystoreExportInstanceOperator(GoogleCloudBaseOperator):
         location: str,
         instance: str,
         output_config: dict | OutputConfig,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -376,7 +377,7 @@ class CloudMemorystoreFailoverInstanceOperator(GoogleCloudBaseOperator):
         location: str,
         instance: str,
         data_protection_mode: FailoverInstanceRequest.DataProtectionMode,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -462,7 +463,7 @@ class CloudMemorystoreGetInstanceOperator(GoogleCloudBaseOperator):
         *,
         location: str,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -556,7 +557,7 @@ class CloudMemorystoreImportOperator(GoogleCloudBaseOperator):
         location: str,
         instance: str,
         input_config: dict | InputConfig,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -646,7 +647,7 @@ class CloudMemorystoreListInstancesOperator(GoogleCloudBaseOperator):
         *,
         location: str,
         page_size: int,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -749,7 +750,7 @@ class CloudMemorystoreUpdateInstanceOperator(GoogleCloudBaseOperator):
         instance: dict | Instance,
         location: str | None = None,
         instance_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -842,7 +843,7 @@ class CloudMemorystoreScaleInstanceOperator(GoogleCloudBaseOperator):
         memory_size_gb: int,
         location: str | None = None,
         instance_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -954,7 +955,7 @@ class CloudMemorystoreCreateInstanceAndImportOperator(GoogleCloudBaseOperator):
         instance_id: str,
         instance: dict | Instance,
         input_config: dict | InputConfig,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1061,7 +1062,7 @@ class CloudMemorystoreExportAndDeleteInstanceOperator(GoogleCloudBaseOperator):
         location: str,
         instance: str,
         output_config: dict | OutputConfig,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1243,7 +1244,7 @@ class CloudMemorystoreMemcachedCreateInstanceOperator(GoogleCloudBaseOperator):
         location: str,
         instance_id: str,
         instance: dict | cloud_memcache.Instance,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1316,7 +1317,7 @@ class CloudMemorystoreMemcachedDeleteInstanceOperator(GoogleCloudBaseOperator):
         self,
         location: str,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1390,7 +1391,7 @@ class CloudMemorystoreMemcachedGetInstanceOperator(GoogleCloudBaseOperator):
         *,
         location: str,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1474,7 +1475,7 @@ class CloudMemorystoreMemcachedListInstancesOperator(GoogleCloudBaseOperator):
         self,
         *,
         location: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -1572,7 +1573,7 @@ class CloudMemorystoreMemcachedUpdateInstanceOperator(GoogleCloudBaseOperator):
         instance: dict | cloud_memcache.Instance,
         location: str | None = None,
         instance_id: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),

airflow/providers/google/cloud/operators/cloud_sql.py

@@ -19,6 +19,7 @@
 
 from __future__ import annotations
 
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence
 
 from googleapiclient.errors import HttpError
@@ -31,7 +32,7 @@ from airflow.providers.google.cloud.links.cloud_sql import CloudSQLInstanceDatab
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.triggers.cloud_sql import CloudSQLExportTrigger
 from airflow.providers.google.cloud.utils.field_validator import GcpBodyFieldValidator
-from airflow.providers.google.common.hooks.base_google import get_field
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, get_field
 from airflow.providers.google.common.links.storage import FileDetailsLink
 
 if TYPE_CHECKING:
@@ -244,7 +245,7 @@ class CloudSQLBaseOperator(GoogleCloudBaseOperator):
         self,
         *,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -337,7 +338,7 @@ class CloudSQLCreateInstanceOperator(CloudSQLBaseOperator):
         *,
         body: dict,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         validate_body: bool = True,
@@ -440,7 +441,7 @@ class CloudSQLInstancePatchOperator(CloudSQLBaseOperator):
         *,
         body: dict,
         instance: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -572,7 +573,7 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator):
         instance: str,
         destination_instance_name: str,
         clone_context: dict | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -663,7 +664,7 @@ class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator):
         *,
         instance: str,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         validate_body: bool = True,
@@ -771,7 +772,7 @@ class CloudSQLPatchInstanceDatabaseOperator(CloudSQLBaseOperator):
         instance: str,
         database: str,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         validate_body: bool = True,
@@ -867,7 +868,7 @@ class CloudSQLDeleteInstanceDatabaseOperator(CloudSQLBaseOperator):
         *,
         instance: str,
         database: str,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         impersonation_chain: str | Sequence[str] | None = None,
@@ -957,7 +958,7 @@ class CloudSQLExportInstanceOperator(CloudSQLBaseOperator):
         *,
         instance: str,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         validate_body: bool = True,
@@ -1104,7 +1105,7 @@ class CloudSQLImportInstanceOperator(CloudSQLBaseOperator):
         *,
         instance: str,
         body: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1beta4",
         validate_body: bool = True,
@@ -1181,10 +1182,35 @@ class CloudSQLExecuteQueryOperator(GoogleCloudBaseOperator):
        details on how to define ``gcpcloudsql://`` connection.
     :param sql_proxy_binary_path: (optional) Path to the cloud-sql-proxy binary.
           is not specified or the binary is not present, it is automatically downloaded.
+    :param ssl_cert: (optional) Path to client certificate to authenticate when SSL is used. Overrides the
+        connection field ``sslcert``.
+    :param ssl_key: (optional) Path to client private key to authenticate when SSL is used. Overrides the
+        connection field ``sslkey``.
+    :param ssl_root_cert: (optional) Path to server's certificate to authenticate when SSL is used. Overrides
+        the connection field ``sslrootcert``.
+    :param ssl_secret_id: (optional) ID of the secret in Google Cloud Secret Manager that stores SSL
+        certificate in the format below:
+
+        {'sslcert': '',
+         'sslkey': '',
+         'sslrootcert': ''}
+
+        Overrides the connection fields ``sslcert``, ``sslkey``, ``sslrootcert``.
+        Note that according to the Secret Manager requirements, the mentioned dict should be saved as a
+        string, and encoded with base64.
+        Note that this parameter is incompatible with parameters ``ssl_cert``, ``ssl_key``, ``ssl_root_cert``.
     """
 
     # [START gcp_sql_query_template_fields]
-    template_fields: Sequence[str] = ("sql", "gcp_cloudsql_conn_id", "gcp_conn_id")
+    template_fields: Sequence[str] = (
+        "sql",
+        "gcp_cloudsql_conn_id",
+        "gcp_conn_id",
+        "ssl_server_cert",
+        "ssl_client_cert",
+        "ssl_client_key",
+        "ssl_secret_id",
+    )
     template_ext: Sequence[str] = (".sql",)
     template_fields_renderers = {"sql": "sql"}
     # [END gcp_sql_query_template_fields]
@@ -1199,6 +1225,10 @@ class CloudSQLExecuteQueryOperator(GoogleCloudBaseOperator):
         gcp_conn_id: str = "google_cloud_default",
         gcp_cloudsql_conn_id: str = "google_cloud_sql_default",
         sql_proxy_binary_path: str | None = None,
+        ssl_server_cert: str | None = None,
+        ssl_client_cert: str | None = None,
+        ssl_client_key: str | None = None,
+        ssl_secret_id: str | None = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -1209,6 +1239,10 @@ class CloudSQLExecuteQueryOperator(GoogleCloudBaseOperator):
         self.parameters = parameters
         self.gcp_connection: Connection | None = None
         self.sql_proxy_binary_path = sql_proxy_binary_path
+        self.ssl_server_cert = ssl_server_cert
+        self.ssl_client_cert = ssl_client_cert
+        self.ssl_client_key = ssl_client_key
+        self.ssl_secret_id = ssl_secret_id
 
     def _execute_query(self, hook: CloudSQLDatabaseHook, database_hook: PostgresHook | MySqlHook) -> None:
         cloud_sql_proxy_runner = None
@@ -1228,12 +1262,8 @@ class CloudSQLExecuteQueryOperator(GoogleCloudBaseOperator):
 
     def execute(self, context: Context):
         self.gcp_connection = BaseHook.get_connection(self.gcp_conn_id)
-        hook = CloudSQLDatabaseHook(
-            gcp_cloudsql_conn_id=self.gcp_cloudsql_conn_id,
-            gcp_conn_id=self.gcp_conn_id,
-            default_gcp_project_id=get_field(self.gcp_connection.extra_dejson, "project"),
-            sql_proxy_binary_path=self.sql_proxy_binary_path,
-        )
+
+        hook = self.hook
         hook.validate_ssl_certs()
         connection = hook.create_connection()
         hook.validate_socket_path_length()
@@ -1242,3 +1272,16 @@ class CloudSQLExecuteQueryOperator(GoogleCloudBaseOperator):
             self._execute_query(hook, database_hook)
         finally:
             hook.cleanup_database_hook()
+
+    @cached_property
+    def hook(self):
+        return CloudSQLDatabaseHook(
+            gcp_cloudsql_conn_id=self.gcp_cloudsql_conn_id,
+            gcp_conn_id=self.gcp_conn_id,
+            default_gcp_project_id=get_field(self.gcp_connection.extra_dejson, "project"),
+            sql_proxy_binary_path=self.sql_proxy_binary_path,
+            ssl_root_cert=self.ssl_server_cert,
+            ssl_cert=self.ssl_client_cert,
+            ssl_key=self.ssl_client_key,
+            ssl_secret_id=self.ssl_secret_id,
+        )

airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py

@@ -28,6 +28,7 @@ from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service import (
     ACCESS_KEY_ID,
     AWS_ACCESS_KEY,
+    AWS_ROLE_ARN,
     AWS_S3_DATA_SOURCE,
     BUCKET_NAME,
     DAY,
@@ -62,6 +63,7 @@ from airflow.providers.google.cloud.links.cloud_storage_transfer import (
 )
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.utils.helpers import normalize_directory_path
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -78,15 +80,23 @@ class TransferJobPreprocessor:
         self.default_schedule = default_schedule
 
     def _inject_aws_credentials(self) -> None:
-        if TRANSFER_SPEC in self.body and AWS_S3_DATA_SOURCE in self.body[TRANSFER_SPEC]:
-            aws_hook = AwsBaseHook(self.aws_conn_id, resource_type="s3")
-            aws_credentials = aws_hook.get_credentials()
-            aws_access_key_id = aws_credentials.access_key  # type: ignore[attr-defined]
-            aws_secret_access_key = aws_credentials.secret_key  # type: ignore[attr-defined]
-            self.body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE][AWS_ACCESS_KEY] = {
-                ACCESS_KEY_ID: aws_access_key_id,
-                SECRET_ACCESS_KEY: aws_secret_access_key,
-            }
+        if TRANSFER_SPEC not in self.body:
+            return
+
+        if AWS_S3_DATA_SOURCE not in self.body[TRANSFER_SPEC]:
+            return
+
+        if AWS_ROLE_ARN in self.body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE]:
+            return
+
+        aws_hook = AwsBaseHook(self.aws_conn_id, resource_type="s3")
+        aws_credentials = aws_hook.get_credentials()
+        aws_access_key_id = aws_credentials.access_key  # type: ignore[attr-defined]
+        aws_secret_access_key = aws_credentials.secret_key  # type: ignore[attr-defined]
+        self.body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE][AWS_ACCESS_KEY] = {
+            ACCESS_KEY_ID: aws_access_key_id,
+            SECRET_ACCESS_KEY: aws_secret_access_key,
+        }
 
     def _reformat_date(self, field_key: str) -> None:
         schedule = self.body[SCHEDULE]
@@ -234,7 +244,7 @@ class CloudDataTransferServiceCreateJobOperator(GoogleCloudBaseOperator):
         aws_conn_id: str | None = "aws_default",
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         google_impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
     ) -> None:
@@ -324,7 +334,7 @@ class CloudDataTransferServiceUpdateJobOperator(GoogleCloudBaseOperator):
         aws_conn_id: str | None = "aws_default",
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         google_impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
     ) -> None:
@@ -407,7 +417,7 @@ class CloudDataTransferServiceDeleteJobOperator(GoogleCloudBaseOperator):
         job_name: str,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         google_impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
     ) -> None:
@@ -467,7 +477,7 @@ class CloudDataTransferServiceGetOperationOperator(GoogleCloudBaseOperator):
     def __init__(
         self,
         *,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         operation_name: str,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
@@ -541,7 +551,7 @@ class CloudDataTransferServiceListOperationsOperator(GoogleCloudBaseOperator):
     def __init__(
         self,
         request_filter: dict,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         api_version: str = "v1",
         google_impersonation_chain: str | Sequence[str] | None = None,
@@ -818,6 +828,9 @@ class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator):
         account from the list granting this role to the originating account (templated).
     :param delete_job_after_completion: If True, delete the job after complete.
         If set to True, 'wait' must be set to True.
+    :param aws_role_arn: Optional AWS role ARN for workload identity federation. This will
+        override the `aws_conn_id` for authentication between GCP and AWS; see
+        https://cloud.google.com/storage-transfer/docs/reference/rest/v1/TransferSpec#AwsS3Data
     """
 
     template_fields: Sequence[str] = (
@@ -829,6 +842,7 @@ class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator):
         "description",
         "object_conditions",
         "google_impersonation_chain",
+        "aws_role_arn",
     )
     ui_color = "#e09411"
 
@@ -839,7 +853,7 @@ class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator):
         gcs_bucket: str,
         s3_path: str | None = None,
         gcs_path: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         aws_conn_id: str | None = "aws_default",
         gcp_conn_id: str = "google_cloud_default",
         description: str | None = None,
@@ -850,6 +864,7 @@ class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator):
         timeout: float | None = None,
         google_impersonation_chain: str | Sequence[str] | None = None,
         delete_job_after_completion: bool = False,
+        aws_role_arn: str | None = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -868,6 +883,7 @@ class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator):
         self.timeout = timeout
         self.google_impersonation_chain = google_impersonation_chain
         self.delete_job_after_completion = delete_job_after_completion
+        self.aws_role_arn = aws_role_arn
         self._validate_inputs()
 
     def _validate_inputs(self) -> None:
@@ -918,6 +934,9 @@ class CloudDataTransferServiceS3ToGCSOperator(GoogleCloudBaseOperator):
         if self.transfer_options is not None:
             body[TRANSFER_SPEC][TRANSFER_OPTIONS] = self.transfer_options  # type: ignore[index]
 
+        if self.aws_role_arn is not None:
+            body[TRANSFER_SPEC][AWS_S3_DATA_SOURCE][AWS_ROLE_ARN] = self.aws_role_arn  # type: ignore[index]
+
         return body
 
 
@@ -1007,7 +1026,7 @@ class CloudDataTransferServiceGCSToGCSOperator(GoogleCloudBaseOperator):
         destination_bucket: str,
         source_path: str | None = None,
         destination_path: str | None = None,
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         gcp_conn_id: str = "google_cloud_default",
         description: str | None = None,
         schedule: dict | None = None,