apache-airflow-providers-google 10.21.1rc2__py3-none-any.whl → 10.22.0rc1__py3-none-any.whl

airflow/providers/google/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.21.1"
+__version__ = "10.22.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.7.0"
+    "2.8.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-google:{__version__}` needs Apache Airflow 2.7.0+"
+        f"The package `apache-airflow-providers-google:{__version__}` needs Apache Airflow 2.8.0+"
     )

airflow/providers/google/cloud/hooks/cloud_run.py

@@ -22,13 +22,19 @@ from typing import TYPE_CHECKING, Any, Iterable, Sequence
 
 from google.cloud.run_v2 import (
     CreateJobRequest,
+    CreateServiceRequest,
     DeleteJobRequest,
+    DeleteServiceRequest,
     GetJobRequest,
+    GetServiceRequest,
     Job,
     JobsAsyncClient,
     JobsClient,
     ListJobsRequest,
     RunJobRequest,
+    Service,
+    ServicesAsyncClient,
+    ServicesClient,
     UpdateJobRequest,
 )
 from google.longrunning import operations_pb2  # type: ignore[attr-defined]
@@ -39,6 +45,7 @@ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
     from google.api_core import operation
+    from google.api_core.operation_async import AsyncOperation
     from google.cloud.run_v2.services.jobs import pagers
 
 
@@ -183,3 +190,120 @@ class CloudRunAsyncHook(GoogleBaseHook):
         return await self.get_conn().get_operation(
             operations_pb2.GetOperationRequest(name=operation_name), timeout=120
         )
+
+
+class CloudRunServiceHook(GoogleBaseHook):
+    """
+    Hook for the Google Cloud Run services.
+
+    :param gcp_conn_id: The connection ID to use when fetching connection info.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account.
+    """
+
+    def __init__(
+        self,
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+    ):
+        self._client: ServicesClient | None = None
+        super().__init__(gcp_conn_id=gcp_conn_id, impersonation_chain=impersonation_chain)
+
+    def get_conn(self):
+        if self._client is None:
+            self._client = ServicesClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
+
+        return self._client
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_service(self, service_name: str, region: str, project_id: str = PROVIDE_PROJECT_ID):
+        get_service_request = GetServiceRequest(
+            name=f"projects/{project_id}/locations/{region}/services/{service_name}"
+        )
+        return self.get_conn().get_service(get_service_request)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_service(
+        self, service_name: str, service: Service | dict, region: str, project_id: str = PROVIDE_PROJECT_ID
+    ) -> Service:
+        if isinstance(service, dict):
+            service = Service(service)
+
+        create_request = CreateServiceRequest(
+            parent=f"projects/{project_id}/locations/{region}",
+            service=service,
+            service_id=service_name,
+        )
+
+        operation = self.get_conn().create_service(create_request)
+        return operation.result()
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_service(self, service_name: str, region: str, project_id: str = PROVIDE_PROJECT_ID) -> Service:
+        delete_request = DeleteServiceRequest(
+            name=f"projects/{project_id}/locations/{region}/services/{service_name}"
+        )
+
+        operation = self.get_conn().delete_service(delete_request)
+        return operation.result()
+
+
+class CloudRunServiceAsyncHook(GoogleBaseHook):
+    """
+    Async hook for the Google Cloud Run services.
+
+    :param gcp_conn_id: The connection ID to use when fetching connection info.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account.
+    """
+
+    def __init__(
+        self,
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+    ):
+        self._client: ServicesClient | None = None
+        super().__init__(gcp_conn_id=gcp_conn_id, impersonation_chain=impersonation_chain)
+
+    def get_conn(self):
+        if self._client is None:
+            self._client = ServicesAsyncClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
+
+        return self._client
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    async def create_service(
+        self, service_name: str, service: Service | dict, region: str, project_id: str = PROVIDE_PROJECT_ID
+    ) -> AsyncOperation:
+        if isinstance(service, dict):
+            service = Service(service)
+
+        create_request = CreateServiceRequest(
+            parent=f"projects/{project_id}/locations/{region}",
+            service=service,
+            service_id=service_name,
+        )
+
+        return await self.get_conn().create_service(create_request)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    async def delete_service(
+        self, service_name: str, region: str, project_id: str = PROVIDE_PROJECT_ID
+    ) -> AsyncOperation:
+        delete_request = DeleteServiceRequest(
+            name=f"projects/{project_id}/locations/{region}/services/{service_name}"
+        )
+
+        return await self.get_conn().delete_service(delete_request)

airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py

@@ -552,6 +552,8 @@ class CloudDataTransferServiceHook(GoogleBaseHook):
 class CloudDataTransferServiceAsyncHook(GoogleBaseAsyncHook):
     """Asynchronous hook for Google Storage Transfer Service."""
 
+    sync_hook_class = CloudDataTransferServiceHook
+
     def __init__(self, project_id: str = PROVIDE_PROJECT_ID, **kwargs: Any) -> None:
         super().__init__(**kwargs)
         self.project_id = project_id

airflow/providers/google/cloud/operators/cloud_run.py

@@ -19,11 +19,13 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Sequence
 
-from google.cloud.run_v2 import Job
+import google.cloud.exceptions
+from google.api_core.exceptions import AlreadyExists
+from google.cloud.run_v2 import Job, Service
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
-from airflow.providers.google.cloud.hooks.cloud_run import CloudRunHook
+from airflow.providers.google.cloud.hooks.cloud_run import CloudRunHook, CloudRunServiceHook
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.triggers.cloud_run import CloudRunJobFinishedTrigger, RunJobStatus
 
@@ -353,3 +355,142 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
         except Exception:
             error = operation.exception(timeout=self.timeout_seconds)
             raise AirflowException(error)
+
+
+class CloudRunCreateServiceOperator(GoogleCloudBaseOperator):
+    """
+    Creates a Service without executing it. Pushes the created service to xcom.
+
+    :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+    :param region: Required. The ID of the Google Cloud region that the service belongs to.
+    :param service_name: Required. The name of the service to create.
+    :param service: The service descriptor containing the configuration of the service to submit.
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account (templated).
+    """
+
+    template_fields = ("project_id", "region", "gcp_conn_id", "impersonation_chain", "service_name")
+
+    def __init__(
+        self,
+        project_id: str,
+        region: str,
+        service_name: str,
+        service: dict | Service,
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.project_id = project_id
+        self.region = region
+        self.service = service
+        self.service_name = service_name
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        missing_fields = [k for k in ["project_id", "region", "service_name"] if not getattr(self, k)]
+        if not self.project_id or not self.region or not self.service_name:
+            raise AirflowException(
+                f"Required parameters are missing: {missing_fields}. These parameters be passed either as "
+                "keyword parameter or as extra field in Airflow connection definition. Both are not set!"
+            )
+
+    def execute(self, context: Context):
+        hook: CloudRunServiceHook = CloudRunServiceHook(
+            gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
+        )
+
+        try:
+            service = hook.create_service(
+                service=self.service,
+                service_name=self.service_name,
+                region=self.region,
+                project_id=self.project_id,
+            )
+        except AlreadyExists:
+            self.log.info(
+                "Already existed Cloud run service, service_name=%s, region=%s",
+                self.service_name,
+                self.region,
+            )
+            return hook.get_service(
+                service_name=self.service_name, region=self.region, project_id=self.project_id
+            )
+        except google.cloud.exceptions.GoogleCloudError as e:
+            self.log.error("An error occurred. Exiting.")
+            raise e
+
+        return Service.to_dict(service)
+
+
+class CloudRunDeleteServiceOperator(GoogleCloudBaseOperator):
+    """
+    Deletes a Service without executing it. Pushes the deleted service to xcom.
+
+    :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+    :param region: Required. The ID of the Google Cloud region that the service belongs to.
+    :param service_name: Required. The name of the service to create.
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account (templated).
+    """
+
+    template_fields = ("project_id", "region", "gcp_conn_id", "impersonation_chain", "service_name")
+
+    def __init__(
+        self,
+        project_id: str,
+        region: str,
+        service_name: str,
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.project_id = project_id
+        self.region = region
+        self.service_name = service_name
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self._validate_inputs()
+
+    def _validate_inputs(self):
+        missing_fields = [k for k in ["project_id", "region", "service_name"] if not getattr(self, k)]
+        if not self.project_id or not self.region or not self.service_name:
+            raise AirflowException(
+                f"Required parameters are missing: {missing_fields}. These parameters be passed either as "
+                "keyword parameter or as extra field in Airflow connection definition. Both are not set!"
+            )
+
+    def execute(self, context: Context):
+        hook: CloudRunServiceHook = CloudRunServiceHook(
+            gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
+        )
+
+        try:
+            service = hook.delete_service(
+                service_name=self.service_name,
+                region=self.region,
+                project_id=self.project_id,
+            )
+        except google.cloud.exceptions.NotFound as e:
+            self.log.error("An error occurred. Not Found.")
+            raise e
+
+        return Service.to_dict(service)

airflow/providers/google/cloud/operators/dataproc.py

@@ -30,6 +30,7 @@ from collections.abc import MutableSequence
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import Enum
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, Sequence
 
 from deprecated import deprecated
@@ -638,7 +639,7 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
         request_id: str | None = None,
         delete_on_error: bool = True,
         use_if_exists: bool = True,
-        retry: AsyncRetry | _MethodDefault = DEFAULT,
+        retry: AsyncRetry | _MethodDefault | Retry = DEFAULT,
         timeout: float = 1 * 60 * 60,
         metadata: Sequence[tuple[str, str]] = (),
         gcp_conn_id: str = "google_cloud_default",
@@ -1184,7 +1185,7 @@ class _DataprocStartStopClusterBaseOperator(GoogleCloudBaseOperator):
         project_id: str = PROVIDE_PROJECT_ID,
         cluster_uuid: str | None = None,
         request_id: str | None = None,
-        retry: AsyncRetry | _MethodDefault = DEFAULT,
+        retry: AsyncRetry | _MethodDefault | Retry = DEFAULT,
         timeout: float = 1 * 60 * 60,
         metadata: Sequence[tuple[str, str]] = (),
         gcp_conn_id: str = "google_cloud_default",
@@ -2712,7 +2713,7 @@ class DataprocUpdateClusterOperator(GoogleCloudBaseOperator):
         region: str,
         request_id: str | None = None,
         project_id: str = PROVIDE_PROJECT_ID,
-        retry: AsyncRetry | _MethodDefault = DEFAULT,
+        retry: AsyncRetry | _MethodDefault | Retry = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
         gcp_conn_id: str = "google_cloud_default",
@@ -2985,10 +2986,10 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
     def __init__(
         self,
         *,
-        region: str | None = None,
+        region: str,
         project_id: str = PROVIDE_PROJECT_ID,
         batch: dict | Batch,
-        batch_id: str,
+        batch_id: str | None = None,
         request_id: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
@@ -3021,20 +3022,20 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
         self.polling_interval_seconds = polling_interval_seconds
 
     def execute(self, context: Context):
-        hook = DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
-        # batch_id might not be set and will be generated
-        if self.batch_id:
-            link = DATAPROC_BATCH_LINK.format(
-                region=self.region, project_id=self.project_id, batch_id=self.batch_id
+        if self.asynchronous and self.deferrable:
+            raise AirflowException(
+                "Both asynchronous and deferrable parameters were passed. Please, provide only one."
             )
-            self.log.info("Creating batch %s", self.batch_id)
-            self.log.info("Once started, the batch job will be available at %s", link)
+
+        batch_id: str = ""
+        if self.batch_id:
+            batch_id = self.batch_id
+            self.log.info("Starting batch %s", batch_id)
         else:
-            self.log.info("Starting batch job. The batch ID will be generated since it was not provided.")
-        if self.region is None:
-            raise AirflowException("Region should be set here")
+            self.log.info("Starting batch. The batch ID will be generated since it was not provided.")
+
         try:
-            self.operation = hook.create_batch(
+            self.operation = self.hook.create_batch(
                 region=self.region,
                 project_id=self.project_id,
                 batch=self.batch,
@@ -3044,85 +3045,62 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
                 timeout=self.timeout,
                 metadata=self.metadata,
             )
-            if self.operation is None:
-                raise RuntimeError("The operation should be set here!")
-
-            if not self.deferrable:
-                if not self.asynchronous:
-                    result = hook.wait_for_operation(
-                        timeout=self.timeout, result_retry=self.result_retry, operation=self.operation
-                    )
-                    self.log.info("Batch %s created", self.batch_id)
-
-                else:
-                    DataprocBatchLink.persist(
-                        context=context,
-                        operator=self,
-                        project_id=self.project_id,
-                        region=self.region,
-                        batch_id=self.batch_id,
-                    )
-                    return self.operation.operation.name
-
-            else:
-                # processing ends in execute_complete
-                self.defer(
-                    trigger=DataprocBatchTrigger(
-                        batch_id=self.batch_id,
-                        project_id=self.project_id,
-                        region=self.region,
-                        gcp_conn_id=self.gcp_conn_id,
-                        impersonation_chain=self.impersonation_chain,
-                        polling_interval_seconds=self.polling_interval_seconds,
-                    ),
-                    method_name="execute_complete",
-                )
-
         except AlreadyExists:
-            self.log.info("Batch with given id already exists")
-            # This is only likely to happen if batch_id was provided
-            # Could be running if Airflow was restarted after task started
-            # poll until a final state is reached
-
-            self.log.info("Attaching to the job %s if it is still running.", self.batch_id)
+            self.log.info("Batch with given id already exists.")
+            self.log.info("Attaching to the job %s if it is still running.", batch_id)
+        else:
+            batch_id = self.operation.metadata.batch.split("/")[-1]
+            self.log.info("The batch %s was created.", batch_id)
 
-            # deferrable handling of a batch_id that already exists - processing ends in execute_complete
-            if self.deferrable:
-                self.defer(
-                    trigger=DataprocBatchTrigger(
-                        batch_id=self.batch_id,
-                        project_id=self.project_id,
-                        region=self.region,
-                        gcp_conn_id=self.gcp_conn_id,
-                        impersonation_chain=self.impersonation_chain,
-                        polling_interval_seconds=self.polling_interval_seconds,
-                    ),
-                    method_name="execute_complete",
-                )
+        DataprocBatchLink.persist(
+            context=context,
+            operator=self,
+            project_id=self.project_id,
+            region=self.region,
+            batch_id=batch_id,
+        )
 
-            # non-deferrable handling of a batch_id that already exists
-            result = hook.wait_for_batch(
-                batch_id=self.batch_id,
+        if self.asynchronous:
+            batch = self.hook.get_batch(
+                batch_id=batch_id,
                 region=self.region,
                 project_id=self.project_id,
                 retry=self.retry,
                 timeout=self.timeout,
                 metadata=self.metadata,
-                wait_check_interval=self.polling_interval_seconds,
             )
-        batch_id = self.batch_id or result.name.split("/")[-1]
+            self.log.info("The batch %s was created asynchronously. Exiting.", batch_id)
+            return Batch.to_dict(batch)
 
-        self.handle_batch_status(context, result.state, batch_id)
-        project_id = self.project_id or hook.project_id
-        if project_id:
-            DataprocBatchLink.persist(
-                context=context,
-                operator=self,
-                project_id=project_id,
-                region=self.region,
-                batch_id=batch_id,
+        if self.deferrable:
+            self.defer(
+                trigger=DataprocBatchTrigger(
+                    batch_id=batch_id,
+                    project_id=self.project_id,
+                    region=self.region,
+                    gcp_conn_id=self.gcp_conn_id,
+                    impersonation_chain=self.impersonation_chain,
+                    polling_interval_seconds=self.polling_interval_seconds,
+                ),
+                method_name="execute_complete",
             )
-        return Batch.to_dict(result)
+
+        self.log.info("Waiting for the completion of batch job %s", batch_id)
+        batch = self.hook.wait_for_batch(
+            batch_id=batch_id,
+            region=self.region,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+
+        self.handle_batch_status(context, batch.state, batch_id, batch.state_message)
+        return Batch.to_dict(batch)
+
+    @cached_property
+    def hook(self) -> DataprocHook:
+        return DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
 
     def execute_complete(self, context, event=None) -> None:
         """
@@ -3135,23 +3113,27 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
             raise AirflowException("Batch failed.")
         state = event["batch_state"]
         batch_id = event["batch_id"]
-        self.handle_batch_status(context, state, batch_id)
+        self.handle_batch_status(context, state, batch_id, state_message=event["batch_state_message"])
 
     def on_kill(self):
         if self.operation:
             self.operation.cancel()
 
-    def handle_batch_status(self, context: Context, state: Batch.State, batch_id: str) -> None:
+    def handle_batch_status(
+        self, context: Context, state: Batch.State, batch_id: str, state_message: str | None = None
+    ) -> None:
         # The existing batch may be a number of states other than 'SUCCEEDED'\
         # wait_for_operation doesn't fail if the job is cancelled, so we will check for it here which also
         # finds a cancelling|canceled|unspecified job from wait_for_batch or the deferred trigger
         link = DATAPROC_BATCH_LINK.format(region=self.region, project_id=self.project_id, batch_id=batch_id)
         if state == Batch.State.FAILED:
-            raise AirflowException("Batch job %s failed.  Driver Logs: %s", batch_id, link)
+            raise AirflowException(
+                f"Batch job {batch_id} failed with error: {state_message}\nDriver Logs: {link}"
+            )
         if state in (Batch.State.CANCELLED, Batch.State.CANCELLING):
-            raise AirflowException("Batch job %s was cancelled. Driver logs: %s", batch_id, link)
+            raise AirflowException(f"Batch job {batch_id} was cancelled. Driver logs: {link}")
         if state == Batch.State.STATE_UNSPECIFIED:
-            raise AirflowException("Batch job %s unspecified. Driver logs: %s", batch_id, link)
+            raise AirflowException(f"Batch job {batch_id} unspecified. Driver logs: {link}")
         self.log.info("Batch job %s completed. Driver logs: %s", batch_id, link)
 
 

airflow/providers/google/cloud/triggers/dataproc.py

@@ -371,7 +371,10 @@ class DataprocBatchTrigger(DataprocBaseTrigger):
             self.log.info("Current state is %s", state)
             self.log.info("Sleeping for %s seconds.", self.polling_interval_seconds)
             await asyncio.sleep(self.polling_interval_seconds)
-        yield TriggerEvent({"batch_id": self.batch_id, "batch_state": state})
+
+        yield TriggerEvent(
+            {"batch_id": self.batch_id, "batch_state": state, "batch_state_message": batch.state_message}
+        )
 
 
 class DataprocDeleteClusterTrigger(DataprocBaseTrigger):

airflow/providers/google/common/hooks/base_google.py

@@ -19,6 +19,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import datetime
 import functools
 import json
@@ -36,7 +37,7 @@ import requests
 import tenacity
 from asgiref.sync import sync_to_async
 from deprecated import deprecated
-from gcloud.aio.auth.token import Token
+from gcloud.aio.auth.token import Token, TokenResponse
 from google.api_core.exceptions import Forbidden, ResourceExhausted, TooManyRequests
 from google.auth import _cloud_sdk, compute_engine  # type: ignore[attr-defined]
 from google.auth.environment_vars import CLOUD_SDK_CONFIG_DIR, CREDENTIALS
@@ -745,17 +746,41 @@ class _CredentialsToken(Token):
     async def get_project(self) -> str | None:
         return self.project
 
-    async def acquire_access_token(self, timeout: int = 10) -> None:
+    async def refresh(self, *, timeout: int) -> TokenResponse:
         await sync_to_async(self.credentials.refresh)(google.auth.transport.requests.Request())
 
         self.access_token = cast(str, self.credentials.token)
         self.access_token_duration = 3600
-        # access_token_acquired_at is specific to gcloud-aio's Token. On subsequent calls of `get` it will be used
-        # with `datetime.datetime.utcnow()`. Therefore we have to use an offset-naive datetime.
-        # https://github.com/talkiq/gcloud-aio/blob/f1132b005ba35d8059229a9ca88b90f31f77456d/auth/gcloud/aio/auth/token.py#L204
-        self.access_token_acquired_at = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)
+        self.access_token_acquired_at = self._now()
+        return TokenResponse(value=self.access_token, expires_in=self.access_token_duration)
+
+    async def acquire_access_token(self, timeout: int = 10) -> None:
+        await self.refresh(timeout=timeout)
         self.acquiring = None
 
+    async def ensure_token(self) -> None:
+        if self.acquiring and not self.acquiring.done():
+            await self.acquiring
+            return
+
+        if self.access_token:
+            delta = (self._now() - self.access_token_acquired_at).total_seconds()
+            if delta <= self.access_token_duration / 2:
+                return
+
+        self.acquiring = asyncio.ensure_future(  # pylint: disable=used-before-assignment
+            self.acquire_access_token()
+        )
+        await self.acquiring
+
+    @staticmethod
+    def _now():
+        # access_token_acquired_at is specific to gcloud-aio's Token.
+        # On subsequent calls of `get` it will be used with `datetime.datetime.utcnow()`.
+        # Therefore we have to use an offset-naive datetime.
+        # https://github.com/talkiq/gcloud-aio/blob/f1132b005ba35d8059229a9ca88b90f31f77456d/auth/gcloud/aio/auth/token.py#L204
+        return datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)
+
 
 class GoogleBaseAsyncHook(BaseHook):
     """GoogleBaseAsyncHook inherits from BaseHook class, run on the trigger worker."""

airflow/providers/google/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Google",
         "description": "Google services including:\n\n  - `Google Ads <https://ads.google.com/>`__\n  - `Google Cloud (GCP) <https://cloud.google.com/>`__\n  - `Google Firebase <https://firebase.google.com/>`__\n  - `Google LevelDB <https://github.com/google/leveldb/>`__\n  - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n  - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
         "state": "ready",
-        "source-date-epoch": 1722145669,
+        "source-date-epoch": 1723970253,
         "versions": [
+            "10.22.0",
             "10.21.1",
             "10.21.0",
             "10.20.0",
@@ -91,12 +92,12 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.7.0",
+            "apache-airflow>=2.8.0",
             "apache-airflow-providers-common-compat>=1.1.0",
             "apache-airflow-providers-common-sql>=1.7.2",
             "asgiref>=3.5.2",
             "dill>=0.2.3",
-            "gcloud-aio-auth>=4.0.0,<5.0.0",
+            "gcloud-aio-auth>=5.2.0",
             "gcloud-aio-bigquery>=6.1.2",
             "gcloud-aio-storage>=9.0.0",
             "gcsfs>=2023.10.0",

{apache_airflow_providers_google-10.21.1rc2.dist-info → apache_airflow_providers_google-10.22.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-google
-Version: 10.21.1rc2
+Version: 10.22.0rc1
 Summary: Provider package apache-airflow-providers-google for Apache Airflow
 Keywords: airflow-provider,google,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -24,10 +24,10 @@ Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyOpenSSL>=23.0.0
 Requires-Dist: apache-airflow-providers-common-compat>=1.1.0rc0
 Requires-Dist: apache-airflow-providers-common-sql>=1.7.2rc0
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow>=2.8.0rc0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: dill>=0.2.3
-Requires-Dist: gcloud-aio-auth>=4.0.0,<5.0.0
+Requires-Dist: gcloud-aio-auth>=5.2.0
 Requires-Dist: gcloud-aio-bigquery>=6.1.2
 Requires-Dist: gcloud-aio-storage>=9.0.0
 Requires-Dist: gcsfs>=2023.10.0
@@ -107,8 +107,8 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Requires-Dist: apache-airflow-providers-trino ; extra == "trino"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/10.21.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/10.21.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/10.22.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/10.22.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -177,7 +177,7 @@ Provides-Extra: trino
 
 Package ``apache-airflow-providers-google``
 
-Release: ``10.21.1.rc2``
+Release: ``10.22.0.rc1``
 
 
 Google services including:
@@ -197,7 +197,7 @@ This is a provider package for ``google`` provider. All classes for this provide
 are in ``airflow.providers.google`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/10.21.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/10.22.0/>`_.
 
 Installation
 ------------
@@ -214,12 +214,12 @@ Requirements
 ==========================================  =========================================
 PIP package                                 Version required
 ==========================================  =========================================
-``apache-airflow``                          ``>=2.7.0``
+``apache-airflow``                          ``>=2.8.0``
 ``apache-airflow-providers-common-compat``  ``>=1.1.0``
 ``apache-airflow-providers-common-sql``     ``>=1.7.2``
 ``asgiref``                                 ``>=3.5.2``
 ``dill``                                    ``>=0.2.3``
-``gcloud-aio-auth``                         ``>=4.0.0,<5.0.0``
+``gcloud-aio-auth``                         ``>=5.2.0``
 ``gcloud-aio-bigquery``                     ``>=6.1.2``
 ``gcloud-aio-storage``                      ``>=9.0.0``
 ``gcsfs``                                   ``>=2023.10.0``
@@ -318,4 +318,4 @@ Dependent package
 ========================================================================================================================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/10.21.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/10.22.0/changelog.html>`_.

{apache_airflow_providers_google-10.21.1rc2.dist-info → apache_airflow_providers_google-10.22.0rc1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 airflow/providers/google/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/google/__init__.py,sha256=GHoF0_Dab1MavJJls-ic8buuO7ZKBAUC98uTOEffKNU,1495
-airflow/providers/google/get_provider_info.py,sha256=nRFCOZSoLWDT95bnj9UFHtI8j071DErhIZUnZ4gHpdA,82075
+airflow/providers/google/__init__.py,sha256=PhPTVkzX5GfOr8JhhgFot6m9omJHWWwVtvCzjGchv-E,1495
+airflow/providers/google/get_provider_info.py,sha256=I7lLiO0j0cHpbSM5oUVi2s_Rmqio5FYpPF0NP3XZZow,82091
 airflow/providers/google/go_module_utils.py,sha256=XVM-IGME6CPgJA8fgDgkusFc4fz3lEghZaZ4elBkv7s,1780
 airflow/providers/google/ads/.gitignore,sha256=z_qaKzblF2LuVvP-06iDord9JBeyzIlNeJ4bx3LbtGc,167
 airflow/providers/google/ads/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -30,9 +30,9 @@ airflow/providers/google/cloud/hooks/cloud_batch.py,sha256=FjpR_Av7z8oMnB4Q7S-aP
 airflow/providers/google/cloud/hooks/cloud_build.py,sha256=jqjvBxP1T4u-h7lDcmADgluYzH85Y-Kz6QhV2NzKmi8,28324
 airflow/providers/google/cloud/hooks/cloud_composer.py,sha256=_GRmI4CHry6_J8-ABccsSy-xpQJ5hmHxzu9je4UGCSg,28716
 airflow/providers/google/cloud/hooks/cloud_memorystore.py,sha256=4hUbVhN-AphoOjQ5NB2Ge4Z2Ay6vd5gnyvlxE3ffzWQ,40442
-airflow/providers/google/cloud/hooks/cloud_run.py,sha256=VfcDtF1APloCiVn6Fu6vEiXrRvAFaPnkndq_I8bfIlk,7362
+airflow/providers/google/cloud/hooks/cloud_run.py,sha256=EEHZ84yRcX8QBfXyPqerC73fhWXqdAFV3VCNUSaLdto,12449
 airflow/providers/google/cloud/hooks/cloud_sql.py,sha256=H0tWvU3KOCKwJ2whdxU7hOO8OMgPwysKirsQUooU1jQ,53442
-airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py,sha256=hn0M_lJ8l_FdSeax6UeLmynLXtYvTPDUIjDNQ5mARyk,22963
+airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py,sha256=BI1HEa0P5_7tDUfoMtE0swREsCO5TTyIXJShVhgmtkM,23015
 airflow/providers/google/cloud/hooks/compute.py,sha256=sBpi7oZjJwV4JPkWm3iKw23l7cAjHDET2MU9S8FOc58,40673
 airflow/providers/google/cloud/hooks/compute_ssh.py,sha256=GTSiuhDFpeN-7n8ggrF-XBaPQ2hfk80tWChGImYGpTo,15689
 airflow/providers/google/cloud/hooks/datacatalog.py,sha256=KoOcXUE-YhCR81wgHNJQ_YwTr5LYYyoDXTFJWZz_mds,54292
@@ -127,7 +127,7 @@ airflow/providers/google/cloud/operators/cloud_batch.py,sha256=uH4II9f-dUtC7LskX
 airflow/providers/google/cloud/operators/cloud_build.py,sha256=7L3mBxls1r628o1XyC1vMOnRThWgfRx1x1z5zWXjx70,48481
 airflow/providers/google/cloud/operators/cloud_composer.py,sha256=MAmqS0FkbKfSk3-1lUyC5F3-76H5eVUBugmMeioEFFk,32679
 airflow/providers/google/cloud/operators/cloud_memorystore.py,sha256=0I4nmUIRsA2lcJlKh2uTnty9ONH40sTxAshsrVN81dY,70989
-airflow/providers/google/cloud/operators/cloud_run.py,sha256=CJRZ10SOPcP7azb_l1VqIK91vY_ED0uhRfGtXbWcpdQ,15606
+airflow/providers/google/cloud/operators/cloud_run.py,sha256=SXpCgomERnXdyKymRC-djbTfY_Mscv-94h12uT8RRBQ,21872
 airflow/providers/google/cloud/operators/cloud_sql.py,sha256=VA_RRg_Zv3zo4cKmpEfhCIpMo5c6IIdDdkj7YkeiZLs,53374
 airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py,sha256=YQsVg8pDegEDvsWsZCbGrSqCln3iQyLQErZS_XZTnBo,48066
 airflow/providers/google/cloud/operators/compute.py,sha256=lFGCacevkKJvAszJhfSLAOfetlsbYrCoImTeWXS5bqw,74607
@@ -138,7 +138,7 @@ airflow/providers/google/cloud/operators/datafusion.py,sha256=NZoR65aChdkPUG8bxE
 airflow/providers/google/cloud/operators/datapipeline.py,sha256=3Zk_v_wlCKRuqx5yeajNWGIJN09xGNWJAN0qwsWE56o,2403
 airflow/providers/google/cloud/operators/dataplex.py,sha256=IgGwt95uW72IeLi1oHpGk8V0fKyt9apsc4kUpBz_7YQ,91195
 airflow/providers/google/cloud/operators/dataprep.py,sha256=jTDDgRccd2zIUqGzJebZpbNTJsFdRi5RnMtldXHqiMs,10477
-airflow/providers/google/cloud/operators/dataproc.py,sha256=K_77TrB5P4Wr0R6KZ-yerO-pbF54rPGBZ1RfRVrkWDA,153070
+airflow/providers/google/cloud/operators/dataproc.py,sha256=JtfSWwNeK4xnKP3QlieiR_nP3LEs-ospZFiSB8YLJ-8,151792
 airflow/providers/google/cloud/operators/dataproc_metastore.py,sha256=mJOqDv4GEqQ7tx32ar-mMsPhIjYC_B1AZyiVDZBKOio,50402
 airflow/providers/google/cloud/operators/datastore.py,sha256=di00jFy3Z1v0GcmcQ0df8NJ32yxcseOqWuojC4TKdmY,24927
 airflow/providers/google/cloud/operators/dlp.py,sha256=SQCGml0RIKl0UrvXHIUiOskg5ayTj4F5_4k4rztClvM,120742
@@ -233,7 +233,7 @@ airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py,sha256
 airflow/providers/google/cloud/triggers/dataflow.py,sha256=evg6qbg-LNTYI4714xCzdsp6aLEFUqn9-8Svmxv6JwQ,28640
 airflow/providers/google/cloud/triggers/datafusion.py,sha256=blqNx4hLHRrLp-FQMCNR3yWmAZ2hCTfql2eyf5XY0P0,5985
 airflow/providers/google/cloud/triggers/dataplex.py,sha256=fEHbvNYgP-6htNkp9b7nmvea_r3Z6CIMfIh57NaWxFA,8334
-airflow/providers/google/cloud/triggers/dataproc.py,sha256=5eCSAkjg6FDL56P_7gvXu_fL0OWHnBrXQc5yg5agifg,24304
+airflow/providers/google/cloud/triggers/dataproc.py,sha256=xK2tZdS2gZTAqgQEQ9kQ9Uq571uV7OmfKBoF8Q46a48,24371
 airflow/providers/google/cloud/triggers/gcs.py,sha256=pMjeNOkWHkOyiAxeK-JoyDInUf2VNtefOZxp8K-aNjw,18973
 airflow/providers/google/cloud/triggers/kubernetes_engine.py,sha256=OH0KLkSGAJRKYgJ0n_6KNBUgh0oMO1T0RAs3jbhw1rQ,12440
 airflow/providers/google/cloud/triggers/mlengine.py,sha256=qpOa9Gz8FmHDxXvPWrXO3M7snGbRTq92gy6kGafCUiY,5265
@@ -257,7 +257,7 @@ airflow/providers/google/common/consts.py,sha256=KKjQX0FO6HP4MuzS8DDfWPoQNk0mhdO
 airflow/providers/google/common/auth_backend/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/google/common/auth_backend/google_openid.py,sha256=HvUAoHfpuK8gZiss0DCgB7R3wk2c1aDjAhwc7GknV4c,4525
 airflow/providers/google/common/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/google/common/hooks/base_google.py,sha256=JdR5UrmXiDVKrxmX4zXRhzxppbXVyX8VpeO3vqwuEvk,33305
+airflow/providers/google/common/hooks/base_google.py,sha256=87Pge2hmzCPxfxp6NCFM3MJFM-D1AJ-duJRoipuPvRA,34122
 airflow/providers/google/common/hooks/discovery_api.py,sha256=dHA3AHwsY7ETWM1fRoNknZZgz4cMnWw4q0PTQt0TiNA,6767
 airflow/providers/google/common/links/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/google/common/links/storage.py,sha256=24tb1xQ00gbGyOu26wryNkTYXlk_9H-ieUra9WmuPIY,2265
@@ -310,7 +310,7 @@ airflow/providers/google/suite/transfers/gcs_to_gdrive.py,sha256=CxtVhp3wlEOBtjR
 airflow/providers/google/suite/transfers/gcs_to_sheets.py,sha256=4nwXWkTySeBXNuThPxzO7uww_hH6PthpppTeuShn27Q,4363
 airflow/providers/google/suite/transfers/local_to_drive.py,sha256=ZSK0b1Rd6x_xsP2DVcUzeYu3qoo9Bsp3VmnKyBsFRH8,6105
 airflow/providers/google/suite/transfers/sql_to_sheets.py,sha256=sORkYSUDArRPnvi8WCiXP7YIXtpAgpEPhf8cqgpu644,5220
-apache_airflow_providers_google-10.21.1rc2.dist-info/entry_points.txt,sha256=Ay1Uo7uHxdXCxWew3CyBHumZ44Ld-iR7AcSR2fY-PLw,102
-apache_airflow_providers_google-10.21.1rc2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_google-10.21.1rc2.dist-info/METADATA,sha256=DJ9x5xHT1nCE-iVrDsWHBoIBVAEtVRJb3LZQSS-aOg0,17137
-apache_airflow_providers_google-10.21.1rc2.dist-info/RECORD,,
+apache_airflow_providers_google-10.22.0rc1.dist-info/entry_points.txt,sha256=Ay1Uo7uHxdXCxWew3CyBHumZ44Ld-iR7AcSR2fY-PLw,102
+apache_airflow_providers_google-10.22.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_google-10.22.0rc1.dist-info/METADATA,sha256=mNFv94dJnExaIHI-wLhXlM5tfV8KQTKxlYRiyCrU-m8,17123
+apache_airflow_providers_google-10.22.0rc1.dist-info/RECORD,,