apache-airflow-providers-google 10.12.0rc1__py3-none-any.whl → 10.13.0rc2__py3-none-any.whl

airflow/providers/google/cloud/operators/gcs.py

@@ -313,6 +313,7 @@ class GCSDeleteObjectsOperator(GoogleCloudBaseOperator):
             )
             raise ValueError(err_message)
 
+        self._objects: list[str] = []
         super().__init__(**kwargs)
 
     def execute(self, context: Context) -> None:
@@ -322,13 +323,47 @@ class GCSDeleteObjectsOperator(GoogleCloudBaseOperator):
         )
 
         if self.objects is not None:
-            objects = self.objects
+            self._objects = self.objects
         else:
-            objects = hook.list(bucket_name=self.bucket_name, prefix=self.prefix)
-        self.log.info("Deleting %s objects from %s", len(objects), self.bucket_name)
-        for object_name in objects:
+            self._objects = hook.list(bucket_name=self.bucket_name, prefix=self.prefix)
+        self.log.info("Deleting %s objects from %s", len(self._objects), self.bucket_name)
+        for object_name in self._objects:
             hook.delete(bucket_name=self.bucket_name, object_name=object_name)
 
+    def get_openlineage_facets_on_complete(self, task_instance):
+        """Implementing on_complete as execute() resolves object names."""
+        from openlineage.client.facet import (
+            LifecycleStateChange,
+            LifecycleStateChangeDatasetFacet,
+            LifecycleStateChangeDatasetFacetPreviousIdentifier,
+        )
+        from openlineage.client.run import Dataset
+
+        from airflow.providers.openlineage.extractors import OperatorLineage
+
+        if not self._objects:
+            return OperatorLineage()
+
+        bucket_url = f"gs://{self.bucket_name}"
+        input_datasets = [
+            Dataset(
+                namespace=bucket_url,
+                name=object_name,
+                facets={
+                    "lifecycleStateChange": LifecycleStateChangeDatasetFacet(
+                        lifecycleStateChange=LifecycleStateChange.DROP.value,
+                        previousIdentifier=LifecycleStateChangeDatasetFacetPreviousIdentifier(
+                            namespace=bucket_url,
+                            name=object_name,
+                        ),
+                    )
+                },
+            )
+            for object_name in self._objects
+        ]
+
+        return OperatorLineage(inputs=input_datasets)
+
 
 class GCSBucketCreateAclEntryOperator(GoogleCloudBaseOperator):
     """
@@ -596,6 +631,22 @@ class GCSFileTransformOperator(GoogleCloudBaseOperator):
                 filename=destination_file.name,
             )
 
+    def get_openlineage_facets_on_start(self):
+        from openlineage.client.run import Dataset
+
+        from airflow.providers.openlineage.extractors import OperatorLineage
+
+        input_dataset = Dataset(
+            namespace=f"gs://{self.source_bucket}",
+            name=self.source_object,
+        )
+        output_dataset = Dataset(
+            namespace=f"gs://{self.destination_bucket}",
+            name=self.destination_object,
+        )
+
+        return OperatorLineage(inputs=[input_dataset], outputs=[output_dataset])
+
 
 class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
     """
@@ -722,6 +773,9 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
         self.upload_continue_on_fail = upload_continue_on_fail
         self.upload_num_attempts = upload_num_attempts
 
+        self._source_object_names: list[str] = []
+        self._destination_object_names: list[str] = []
+
     def execute(self, context: Context) -> list[str]:
         # Define intervals and prefixes.
         try:
@@ -773,7 +827,7 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
         )
 
         # Fetch list of files.
-        blobs_to_transform = source_hook.list_by_timespan(
+        self._source_object_names = source_hook.list_by_timespan(
             bucket_name=self.source_bucket,
             prefix=source_prefix_interp,
             timespan_start=timespan_start,
@@ -785,7 +839,7 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
             temp_output_dir_path = Path(temp_output_dir)
 
             # TODO: download in parallel.
-            for blob_to_transform in blobs_to_transform:
+            for blob_to_transform in self._source_object_names:
                 destination_file = temp_input_dir_path / blob_to_transform
                 destination_file.parent.mkdir(parents=True, exist_ok=True)
                 try:
@@ -822,8 +876,6 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
 
             self.log.info("Transformation succeeded. Output temporarily located at %s", temp_output_dir_path)
 
-            files_uploaded = []
-
             # TODO: upload in parallel.
             for upload_file in temp_output_dir_path.glob("**/*"):
                 if upload_file.is_dir():
@@ -844,12 +896,35 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator):
                         chunk_size=self.chunk_size,
                         num_max_attempts=self.upload_num_attempts,
                     )
-                    files_uploaded.append(str(upload_file_name))
+                    self._destination_object_names.append(str(upload_file_name))
                 except GoogleCloudError:
                     if not self.upload_continue_on_fail:
                         raise
 
-            return files_uploaded
+            return self._destination_object_names
+
+    def get_openlineage_facets_on_complete(self, task_instance):
+        """Implementing on_complete as execute() resolves object names."""
+        from openlineage.client.run import Dataset
+
+        from airflow.providers.openlineage.extractors import OperatorLineage
+
+        input_datasets = [
+            Dataset(
+                namespace=f"gs://{self.source_bucket}",
+                name=object_name,
+            )
+            for object_name in self._source_object_names
+        ]
+        output_datasets = [
+            Dataset(
+                namespace=f"gs://{self.destination_bucket}",
+                name=object_name,
+            )
+            for object_name in self._destination_object_names
+        ]
+
+        return OperatorLineage(inputs=input_datasets, outputs=output_datasets)
 
 
 class GCSDeleteBucketOperator(GoogleCloudBaseOperator):

airflow/providers/google/cloud/secrets/secret_manager.py

@@ -20,12 +20,16 @@ from __future__ import annotations
 import logging
 import re
 import warnings
+from typing import Sequence
 
 from google.auth.exceptions import DefaultCredentialsError
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient
-from airflow.providers.google.cloud.utils.credentials_provider import get_credentials_and_project_id
+from airflow.providers.google.cloud.utils.credentials_provider import (
+    _get_target_principal_and_delegates,
+    get_credentials_and_project_id,
+)
 from airflow.secrets import BaseSecretsBackend
 from airflow.utils.log.logging_mixin import LoggingMixin
 
@@ -76,6 +80,14 @@ class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
     :param project_id: Project ID to read the secrets from. If not passed, the project ID from credentials
         will be used.
     :param sep: Separator used to concatenate connections_prefix and conn_id. Default: "-"
+    :param impersonation_chain: Optional service account to impersonate using
+        short-term credentials, or chained list of accounts required to get the
+        access token of the last account in the list, which will be impersonated
+        in the request. If set as a string, the account must grant the
+        originating account the Service Account Token Creator IAM role. If set
+        as a sequence, the identities from the list must grant Service Account
+        Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account.
     """
 
     def __init__(
@@ -89,6 +101,7 @@ class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
         gcp_scopes: str | None = None,
         project_id: str | None = None,
         sep: str = "-",
+        impersonation_chain: str | Sequence[str] | None = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -103,11 +116,19 @@ class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
                     f"follows that pattern {SECRET_ID_PATTERN}"
                 )
         try:
+            if impersonation_chain:
+                target_principal, delegates = _get_target_principal_and_delegates(impersonation_chain)
+            else:
+                target_principal = None
+                delegates = None
+
             self.credentials, self.project_id = get_credentials_and_project_id(
                 keyfile_dict=gcp_keyfile_dict,
                 key_path=gcp_key_path,
                 credential_config_file=gcp_credential_config_file,
                 scopes=gcp_scopes,
+                target_principal=target_principal,
+                delegates=delegates,
             )
         except (DefaultCredentialsError, FileNotFoundError):
             log.exception(

airflow/providers/google/cloud/sensors/cloud_composer.py

@@ -19,9 +19,10 @@
 
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING, Any, Sequence
 
-from airflow.exceptions import AirflowException, AirflowSkipException
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException
 from airflow.providers.google.cloud.triggers.cloud_composer import CloudComposerExecutionTrigger
 from airflow.sensors.base import BaseSensorOperator
 
@@ -33,6 +34,11 @@ class CloudComposerEnvironmentSensor(BaseSensorOperator):
     """
     Check the status of the Cloud Composer Environment task.
 
+    This Sensor is deprecated. You can achieve the same functionality by using Cloud Composer Operators
+    CloudComposerCreateEnvironmentOperator, CloudComposerDeleteEnvironmentOperator and
+    CloudComposerUpdateEnvironmentOperator in  deferrable or non-deferrable mode, since every operator
+    gives user a possibility to wait (asynchronously or synchronously) until Operation will be finished.
+
     :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
     :param region: Required. The ID of the Google Cloud region that the service belongs to.
     :param operation_name: The name of the operation resource
@@ -59,6 +65,13 @@ class CloudComposerEnvironmentSensor(BaseSensorOperator):
         pooling_period_seconds: int = 30,
         **kwargs,
     ):
+        warnings.warn(
+            f"The `{self.__class__.__name__}` operator is deprecated. You can achieve the same functionality "
+            f"by using operators in deferrable or non-deferrable mode, since every operator for Cloud "
+            f"Composer will wait for the operation to complete.",
+            AirflowProviderDeprecationWarning,
+            stacklevel=2,
+        )
         super().__init__(**kwargs)
         self.project_id = project_id
         self.region = region

airflow/providers/google/cloud/sensors/dataplex.py

@@ -259,3 +259,121 @@ class DataplexDataQualityJobStatusSensor(BaseSensorOperator):
                     raise AirflowSkipException(message)
                 raise AirflowDataQualityScanException(message)
         return job_status == DataScanJob.State.SUCCEEDED
+
+
+class DataplexDataProfileJobStatusSensor(BaseSensorOperator):
+    """
+    Check the status of the Dataplex DataProfile job.
+
+    :param project_id: Required. The ID of the Google Cloud project that the task belongs to.
+    :param region: Required. The ID of the Google Cloud region that the task belongs to.
+    :param data_scan_id: Required. Data Quality scan identifier.
+    :param job_id: Required. Job ID.
+    :param api_version: The version of the api that will be requested for example 'v3'.
+    :param retry: A retry object used  to retry requests. If `None` is specified, requests
+        will not be retried.
+    :param metadata: Additional metadata that is provided to the method.
+    :param gcp_conn_id: The connection ID to use when fetching connection info.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account (templated).
+    :param result_timeout: Value in seconds for which operator will wait for the Data Quality scan result.
+        Throws exception if there is no result found after specified amount of seconds.
+
+    :return: Boolean indicating if the job run has reached the ``DataScanJob.State.SUCCEEDED``.
+    """
+
+    template_fields = ["job_id"]
+
+    def __init__(
+        self,
+        project_id: str,
+        region: str,
+        data_scan_id: str,
+        job_id: str,
+        api_version: str = "v1",
+        retry: Retry | _MethodDefault = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        result_timeout: float = 60.0 * 10,
+        start_sensor_time: float | None = None,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.project_id = project_id
+        self.region = region
+        self.data_scan_id = data_scan_id
+        self.job_id = job_id
+        self.api_version = api_version
+        self.retry = retry
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self.result_timeout = result_timeout
+        self.start_sensor_time = start_sensor_time
+
+    def _duration(self):
+        if not self.start_sensor_time:
+            self.start_sensor_time = time.monotonic()
+        return time.monotonic() - self.start_sensor_time
+
+    def poke(self, context: Context) -> bool:
+        self.log.info("Waiting for job %s to be %s", self.job_id, DataScanJob.State.SUCCEEDED)
+        if self.result_timeout:
+            duration = self._duration()
+            if duration > self.result_timeout:
+                # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+                message = (
+                    f"Timeout: Data Profile scan {self.job_id} is not ready after {self.result_timeout}s"
+                )
+                if self.soft_fail:
+                    raise AirflowSkipException(message)
+                raise AirflowDataQualityScanResultTimeoutException(message)
+
+        hook = DataplexHook(
+            gcp_conn_id=self.gcp_conn_id,
+            api_version=self.api_version,
+            impersonation_chain=self.impersonation_chain,
+        )
+
+        try:
+            job = hook.get_data_scan_job(
+                project_id=self.project_id,
+                region=self.region,
+                data_scan_id=self.data_scan_id,
+                job_id=self.job_id,
+                timeout=self.timeout,
+                retry=self.retry,
+                metadata=self.metadata,
+            )
+        except GoogleAPICallError as e:
+            # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+            message = f"Error occurred when trying to retrieve Data Profile scan job: {self.data_scan_id}"
+            if self.soft_fail:
+                raise AirflowSkipException(message, e)
+            raise AirflowException(message, e)
+
+        job_status = job.state
+        self.log.info(
+            "Current status of the Dataplex Data Profile scan job %s => %s", self.job_id, job_status
+        )
+        if job_status == DataScanJob.State.FAILED:
+            # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+            message = f"Data Profile scan job failed: {self.job_id}"
+            if self.soft_fail:
+                raise AirflowSkipException(message)
+            raise AirflowException(message)
+        if job_status == DataScanJob.State.CANCELLED:
+            # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+            message = f"Data Profile scan job cancelled: {self.job_id}"
+            if self.soft_fail:
+                raise AirflowSkipException(message)
+            raise AirflowException(message)
+        return job_status == DataScanJob.State.SUCCEEDED

airflow/providers/google/cloud/triggers/cloud_run.py

@@ -102,21 +102,21 @@ class CloudRunJobFinishedTrigger(BaseTrigger):
         while timeout is None or timeout > 0:
             operation: operations_pb2.Operation = await hook.get_operation(self.operation_name)
             if operation.done:
-                # An operation can only have one of those two combinations: if it is succeeded, then
-                # the response field will be populated, else, then the error field will be.
-                if operation.response is not None:
+                # An operation can only have one of those two combinations: if it is failed, then
+                # the error field will be populated, else, then the response field will be.
+                if operation.error.SerializeToString():
                     yield TriggerEvent(
                         {
-                            "status": RunJobStatus.SUCCESS,
+                            "status": RunJobStatus.FAIL.value,
+                            "operation_error_code": operation.error.code,
+                            "operation_error_message": operation.error.message,
                             "job_name": self.job_name,
                         }
                     )
                 else:
                     yield TriggerEvent(
                         {
-                            "status": RunJobStatus.FAIL,
-                            "operation_error_code": operation.error.code,
-                            "operation_error_message": operation.error.message,
+                            "status": RunJobStatus.SUCCESS.value,
                             "job_name": self.job_name,
                         }
                     )

airflow/providers/google/cloud/triggers/dataplex.py

@@ -107,3 +107,85 @@ class DataplexDataQualityJobTrigger(BaseTrigger):
     def _convert_to_dict(self, job: DataScanJob) -> dict:
         """Returns a representation of a DataScanJob instance as a dict."""
         return DataScanJob.to_dict(job)
+
+
+class DataplexDataProfileJobTrigger(BaseTrigger):
+    """
+    DataplexDataProfileJobTrigger runs on the trigger worker and waits for the job to be `SUCCEEDED` state.
+
+    :param job_id: Optional. The ID of a Dataplex job.
+    :param data_scan_id: Required. DataScan identifier.
+    :param project_id: Google Cloud Project where the job is running.
+    :param region: The ID of the Google Cloud region that the job belongs to.
+    :param gcp_conn_id: Optional, the connection ID used to connect to Google Cloud Platform.
+    :param impersonation_chain: Optional service account to impersonate using short-term
+        credentials, or chained list of accounts required to get the access_token
+        of the last account in the list, which will be impersonated in the request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding identity, with first
+        account from the list granting this role to the originating account (templated).
+    :param polling_interval_seconds: polling period in seconds to check for the status.
+    """
+
+    def __init__(
+        self,
+        job_id: str | None,
+        data_scan_id: str,
+        project_id: str | None,
+        region: str,
+        gcp_conn_id: str = "google_cloud_default",
+        polling_interval_seconds: int = 10,
+        impersonation_chain: str | Sequence[str] | None = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.job_id = job_id
+        self.data_scan_id = data_scan_id
+        self.project_id = project_id
+        self.region = region
+        self.gcp_conn_id = gcp_conn_id
+        self.polling_interval_seconds = polling_interval_seconds
+        self.impersonation_chain = impersonation_chain
+
+    def serialize(self):
+        return (
+            "airflow.providers.google.cloud.triggers.dataplex.DataplexDataProfileJobTrigger",
+            {
+                "job_id": self.job_id,
+                "data_scan_id": self.data_scan_id,
+                "project_id": self.project_id,
+                "region": self.region,
+                "gcp_conn_id": self.gcp_conn_id,
+                "impersonation_chain": self.impersonation_chain,
+                "polling_interval_seconds": self.polling_interval_seconds,
+            },
+        )
+
+    async def run(self) -> AsyncIterator[TriggerEvent]:
+        hook = DataplexAsyncHook(
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.impersonation_chain,
+        )
+        while True:
+            job = await hook.get_data_scan_job(
+                project_id=self.project_id,
+                region=self.region,
+                job_id=self.job_id,
+                data_scan_id=self.data_scan_id,
+            )
+            state = job.state
+            if state in (DataScanJob.State.FAILED, DataScanJob.State.SUCCEEDED, DataScanJob.State.CANCELLED):
+                break
+            self.log.info(
+                "Current state is: %s, sleeping for %s seconds.",
+                DataScanJob.State(state).name,
+                self.polling_interval_seconds,
+            )
+            await asyncio.sleep(self.polling_interval_seconds)
+        yield TriggerEvent({"job_id": self.job_id, "job_state": state, "job": self._convert_to_dict(job)})
+
+    def _convert_to_dict(self, job: DataScanJob) -> dict:
+        """Returns a representation of a DataScanJob instance as a dict."""
+        return DataScanJob.to_dict(job)

airflow/providers/google/cloud/triggers/dataproc.py

@@ -25,7 +25,6 @@ from typing import Any, AsyncIterator, Sequence
 from google.api_core.exceptions import NotFound
 from google.cloud.dataproc_v1 import Batch, ClusterStatus, JobStatus
 
-from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.dataproc import DataprocAsyncHook
 from airflow.triggers.base import BaseTrigger, TriggerEvent
 
@@ -98,12 +97,10 @@ class DataprocSubmitTrigger(DataprocBaseTrigger):
             )
             state = job.status.state
             self.log.info("Dataproc job: %s is in state: %s", self.job_id, state)
-            if state in (JobStatus.State.DONE, JobStatus.State.CANCELLED):
+            if state in (JobStatus.State.DONE, JobStatus.State.CANCELLED, JobStatus.State.ERROR):
                 break
-            elif state == JobStatus.State.ERROR:
-                raise AirflowException(f"Dataproc job execution failed {self.job_id}")
             await asyncio.sleep(self.polling_interval_seconds)
-        yield TriggerEvent({"job_id": self.job_id, "job_state": state})
+        yield TriggerEvent({"job_id": self.job_id, "job_state": state, "job": job})
 
 
 class DataprocClusterTrigger(DataprocBaseTrigger):

airflow/providers/google/common/hooks/base_google.py

@@ -188,8 +188,8 @@ class GoogleBaseHook(BaseHook):
     conn_type = "google_cloud_platform"
     hook_name = "Google Cloud"
 
-    @staticmethod
-    def get_connection_form_widgets() -> dict[str, Any]:
+    @classmethod
+    def get_connection_form_widgets(cls) -> dict[str, Any]:
         """Returns connection widgets to add to connection form."""
         from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
         from flask_babel import lazy_gettext
@@ -221,8 +221,8 @@ class GoogleBaseHook(BaseHook):
             ),
         }
 
-    @staticmethod
-    def get_ui_field_behaviour() -> dict[str, Any]:
+    @classmethod
+    def get_ui_field_behaviour(cls) -> dict[str, Any]:
         """Returns custom field behaviour."""
         return {
             "hidden_fields": ["host", "schema", "login", "password", "port", "extra"],
@@ -267,6 +267,8 @@ class GoogleBaseHook(BaseHook):
 
         if not self.impersonation_chain:
             self.impersonation_chain = self._get_field("impersonation_chain", None)
+            if isinstance(self.impersonation_chain, str) and "," in self.impersonation_chain:
+                self.impersonation_chain = [s.strip() for s in self.impersonation_chain.split(",")]
 
         target_principal, delegates = _get_target_principal_and_delegates(self.impersonation_chain)
 

airflow/providers/google/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Google",
         "description": "Google services including:\n\n  - `Google Ads <https://ads.google.com/>`__\n  - `Google Cloud (GCP) <https://cloud.google.com/>`__\n  - `Google Firebase <https://firebase.google.com/>`__\n  - `Google LevelDB <https://github.com/google/leveldb/>`__\n  - `Google Marketing Platform <https://marketingplatform.google.com/>`__\n  - `Google Workspace <https://workspace.google.com/>`__ (formerly Google Suite)\n",
         "suspended": False,
-        "source-date-epoch": 1700827465,
+        "source-date-epoch": 1701983383,
         "versions": [
+            "10.13.0",
             "10.12.0",
             "10.11.1",
             "10.11.0",
@@ -80,30 +81,30 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.5.0",
+            "apache-airflow>=2.6.0",
             "apache-airflow-providers-common-sql>=1.7.2",
             "asgiref>=3.5.2",
             "gcloud-aio-auth>=4.0.0,<5.0.0",
             "gcloud-aio-bigquery>=6.1.2",
             "gcloud-aio-storage",
-            "gcsfs>=2023.9.2",
+            "gcsfs>=2023.10.0",
             "google-ads>=22.1.0",
             "google-api-core>=2.11.0",
             "google-api-python-client>=1.6.0",
             "google-auth>=1.0.0",
             "google-auth-httplib2>=0.0.1",
             "google-cloud-aiplatform>=1.22.1",
-            "google-cloud-automl>=2.11.0",
-            "google-cloud-bigquery-datatransfer>=3.11.0",
+            "google-cloud-automl>=2.12.0",
+            "google-cloud-bigquery-datatransfer>=3.13.0",
             "google-cloud-bigtable>=2.17.0",
-            "google-cloud-build>=3.13.0",
+            "google-cloud-build>=3.22.0",
             "google-cloud-compute>=1.10.0",
             "google-cloud-container>=2.17.4",
             "google-cloud-datacatalog>=3.11.1",
-            "google-cloud-dataflow-client>=0.8.2",
+            "google-cloud-dataflow-client>=0.8.6",
             "google-cloud-dataform>=0.5.0",
-            "google-cloud-dataplex>=1.4.2",
-            "google-cloud-dataproc>=5.5.0",
+            "google-cloud-dataplex>=1.10.0",
+            "google-cloud-dataproc>=5.8.0",
             "google-cloud-dataproc-metastore>=1.12.0",
             "google-cloud-dlp>=3.12.0",
             "google-cloud-kms>=2.15.0",
@@ -111,7 +112,7 @@ def get_provider_info():
             "google-cloud-logging>=3.5.0",
             "google-cloud-memcache>=1.7.0",
             "google-cloud-monitoring>=2.14.1",
-            "google-cloud-orchestration-airflow>=1.7.0",
+            "google-cloud-orchestration-airflow>=1.10.0",
             "google-cloud-os-login>=2.9.1",
             "google-cloud-pubsub>=2.15.0",
             "google-cloud-redis>=2.12.0",