apache-airflow-providers-google 10.23.0rc1__tar.gz → 10.24.0__tar.gz

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-google
-Version: 10.23.0rc1
+Version: 10.24.0
 Summary: Provider package apache-airflow-providers-google for Apache Airflow
 Keywords: airflow-provider,google,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.8
+Requires-Python: ~=3.9
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,16 +15,15 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyOpenSSL>=23.0.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.1.0rc0
-Requires-Dist: apache-airflow-providers-common-sql>=1.7.2rc0
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.7.2
+Requires-Dist: apache-airflow>=2.8.0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: dill>=0.2.3
 Requires-Dist: gcloud-aio-auth>=5.2.0
@@ -87,20 +86,20 @@ Requires-Dist: python-slugify>=7.0.0
 Requires-Dist: sqlalchemy-bigquery>=1.2.1
 Requires-Dist: sqlalchemy-spanner>=1.6.2
 Requires-Dist: tenacity>=8.1.0
-Requires-Dist: apache-airflow-providers-amazon>=2.6.0rc0 ; extra == "amazon"
+Requires-Dist: apache-airflow-providers-amazon>=2.6.0 ; extra == "amazon"
 Requires-Dist: apache-airflow-providers-apache-beam ; extra == "apache.beam"
 Requires-Dist: apache-beam[gcp] ; extra == "apache.beam"
 Requires-Dist: apache-airflow-providers-apache-cassandra ; extra == "apache.cassandra"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf.kubernetes"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf.kubernetes"
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
-Requires-Dist: apache-airflow-providers-facebook>=2.2.0rc0 ; extra == "facebook"
+Requires-Dist: apache-airflow-providers-facebook>=2.2.0 ; extra == "facebook"
 Requires-Dist: plyvel ; extra == "leveldb"
 Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft.azure"
 Requires-Dist: apache-airflow-providers-microsoft-mssql ; extra == "microsoft.mssql"
 Requires-Dist: apache-airflow-providers-mysql ; extra == "mysql"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
-Requires-Dist: apache-airflow-providers-oracle>=3.1.0rc0 ; extra == "oracle"
+Requires-Dist: apache-airflow-providers-oracle>=3.1.0 ; extra == "oracle"
 Requires-Dist: apache-airflow-providers-postgres ; extra == "postgres"
 Requires-Dist: apache-airflow-providers-presto ; extra == "presto"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
@@ -108,8 +107,8 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Requires-Dist: apache-airflow-providers-trino ; extra == "trino"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/10.23.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/10.23.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-google/10.24.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-google/10.24.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -178,7 +177,7 @@ Provides-Extra: trino
 
 Package ``apache-airflow-providers-google``
 
-Release: ``10.23.0.rc1``
+Release: ``10.24.0``
 
 
 Google services including:
@@ -198,7 +197,7 @@ This is a provider package for ``google`` provider. All classes for this provide
 are in ``airflow.providers.google`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/10.23.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/10.24.0/>`_.
 
 Installation
 ------------
@@ -207,7 +206,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-google``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -320,4 +319,4 @@ Dependent package
 ========================================================================================================================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/10.23.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/10.24.0/changelog.html>`_.

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-google``
 
-Release: ``10.23.0.rc1``
+Release: ``10.24.0``
 
 
 Google services including:
@@ -62,7 +62,7 @@ This is a provider package for ``google`` provider. All classes for this provide
 are in ``airflow.providers.google`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/10.23.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-google/10.24.0/>`_.
 
 Installation
 ------------
@@ -71,7 +71,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-google``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -184,4 +184,4 @@ Dependent package
 ========================================================================================================================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/10.23.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-google/10.24.0/changelog.html>`_.

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.23.0"
+__version__ = "10.24.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/hooks/bigquery.py

@@ -2418,10 +2418,6 @@ class BigQueryHook(GoogleBaseHook, DbApiHook):
                 f"{var_print(var_name)}Expect format of (<project.|<project:)<dataset>.<table>, "
                 f"got {table_input}"
             )
-
-        # Exclude partition from the table name
-        table_id = table_id.split("$")[0]
-
         if project_id is None:
             if var_name is not None:
                 self.log.info(

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/hooks/dataflow.py

@@ -1124,18 +1124,17 @@ class DataflowHook(GoogleBaseHook):
 
         return safe_job_name
 
-    @_fallback_to_location_from_variables
     @_fallback_to_project_id_from_variables
     @GoogleBaseHook.fallback_to_default_project_id
     def is_job_dataflow_running(
         self,
         name: str,
         project_id: str,
-        location: str = DEFAULT_DATAFLOW_LOCATION,
+        location: str | None = None,
         variables: dict | None = None,
     ) -> bool:
         """
-        Check if jos is still running in dataflow.
+        Check if job is still running in dataflow.
 
         :param name: The name of the job.
         :param project_id: Optional, the Google Cloud project ID in which to start a job.
@@ -1145,11 +1144,21 @@ class DataflowHook(GoogleBaseHook):
         """
         if variables:
             warnings.warn(
-                "The variables parameter has been deprecated. You should pass location using "
-                "the location parameter.",
+                "The variables parameter has been deprecated. You should pass project_id using "
+                "the project_id parameter.",
                 AirflowProviderDeprecationWarning,
                 stacklevel=4,
             )
+
+        if location is None:
+            location = DEFAULT_DATAFLOW_LOCATION
+            warnings.warn(
+                "The location argument will be become mandatory in future versions, "
+                f"currently, it defaults to {DEFAULT_DATAFLOW_LOCATION}, please set the location explicitly.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=4,
+            )
+
         jobs_controller = _DataflowJobsController(
             dataflow=self.get_conn(),
             project_number=project_id,

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/hooks/dataproc.py

@@ -59,6 +59,10 @@ if TYPE_CHECKING:
     from google.type.interval_pb2 import Interval
 
 
+class DataprocResourceIsNotReadyError(AirflowException):
+    """Raise when resource is not ready for create Dataproc cluster."""
+
+
 class DataProcJobBuilder:
     """A helper class for building Dataproc job."""
 
@@ -281,6 +285,8 @@ class DataprocHook(GoogleBaseHook):
             return operation.result(timeout=timeout, retry=result_retry)
         except Exception:
             error = operation.exception(timeout=timeout)
+            if self.check_error_for_resource_is_not_ready_msg(error.message):
+                raise DataprocResourceIsNotReadyError(error.message)
             raise AirflowException(error)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -1192,6 +1198,11 @@ class DataprocHook(GoogleBaseHook):
 
         return result
 
+    def check_error_for_resource_is_not_ready_msg(self, error_msg: str) -> bool:
+        """Check that reason of error is resource is not ready."""
+        key_words = ["The resource", "is not ready"]
+        return all([word in error_msg for word in key_words])
+
 
 class DataprocAsyncHook(GoogleBaseHook):
     """

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/hooks/gcs.py

@@ -59,6 +59,7 @@ if TYPE_CHECKING:
 
     from aiohttp import ClientSession
     from google.api_core.retry import Retry
+    from google.cloud.storage.blob import Blob
 
 
 RT = TypeVar("RT")
@@ -597,11 +598,7 @@ class GCSHook(GoogleBaseHook):
         :param object_name: The name of the blob to get updated time from the Google cloud
             storage bucket.
         """
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
-        if blob is None:
-            raise ValueError(f"Object ({object_name}) not found in Bucket ({bucket_name})")
+        blob = self._get_blob(bucket_name, object_name)
         return blob.updated
 
     def is_updated_after(self, bucket_name: str, object_name: str, ts: datetime) -> bool:
@@ -957,19 +954,35 @@ class GCSHook(GoogleBaseHook):
                 break
         return ids
 
-    def get_size(self, bucket_name: str, object_name: str) -> int:
+    def _get_blob(self, bucket_name: str, object_name: str) -> Blob:
         """
-        Get the size of a file in Google Cloud Storage.
+        Get a blob object in Google Cloud Storage.
 
         :param bucket_name: The Google Cloud Storage bucket where the blob_name is.
         :param object_name: The name of the object to check in the Google
             cloud storage bucket_name.
 
         """
-        self.log.info("Checking the file size of object: %s in bucket_name: %s", object_name, bucket_name)
         client = self.get_conn()
         bucket = client.bucket(bucket_name)
         blob = bucket.get_blob(blob_name=object_name)
+
+        if blob is None:
+            raise ValueError(f"Object ({object_name}) not found in Bucket ({bucket_name})")
+
+        return blob
+
+    def get_size(self, bucket_name: str, object_name: str) -> int:
+        """
+        Get the size of a file in Google Cloud Storage.
+
+        :param bucket_name: The Google Cloud Storage bucket where the blob_name is.
+        :param object_name: The name of the object to check in the Google
+            cloud storage bucket_name.
+
+        """
+        self.log.info("Checking the file size of object: %s in bucket_name: %s", object_name, bucket_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_size = blob.size
         self.log.info("The file size of %s is %s bytes.", object_name, blob_size)
         return blob_size
@@ -987,9 +1000,7 @@ class GCSHook(GoogleBaseHook):
             object_name,
             bucket_name,
         )
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_crc32c = blob.crc32c
         self.log.info("The crc32c checksum of %s is %s", object_name, blob_crc32c)
         return blob_crc32c
@@ -1003,9 +1014,7 @@ class GCSHook(GoogleBaseHook):
             storage bucket_name.
         """
         self.log.info("Retrieving the MD5 hash of object: %s in bucket: %s", object_name, bucket_name)
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_md5hash = blob.md5_hash
         self.log.info("The md5Hash of %s is %s", object_name, blob_md5hash)
         return blob_md5hash
@@ -1019,11 +1028,7 @@ class GCSHook(GoogleBaseHook):
         :return: The metadata associated with the object
         """
         self.log.info("Retrieving the metadata dict of object (%s) in bucket (%s)", object_name, bucket_name)
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
-        if blob is None:
-            raise ValueError("Object (%s) not found in bucket (%s)", object_name, bucket_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_metadata = blob.metadata
         if blob_metadata:
             self.log.info("Retrieved metadata of object (%s) with %s fields", object_name, len(blob_metadata))

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/log/gcs_task_handler.py

@@ -129,12 +129,7 @@ class GCSTaskHandler(FileTaskHandler, LoggingMixin):
         )
 
     def set_context(self, ti: TaskInstance, *, identifier: str | None = None) -> None:
-        # todo: remove-at-min-airflow-version-2.8
-        #   after Airflow 2.8 can always pass `identifier`
-        if getattr(super(), "supports_task_context_logging", False):
-            super().set_context(ti, identifier=identifier)
-        else:
-            super().set_context(ti)
+        super().set_context(ti, identifier=identifier)
         # Log relative path is used to construct local and remote
         # log path to upload log files into GCS and read from the
         # remote location.

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/operators/automl.py

@@ -34,7 +34,7 @@ from google.cloud.automl_v1beta1 import (
     TableSpec,
 )
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
 from airflow.providers.google.cloud.hooks.vertex_ai.prediction_service import PredictionServiceHook
 from airflow.providers.google.cloud.links.translate import (
@@ -45,6 +45,7 @@ from airflow.providers.google.cloud.links.translate import (
     TranslationLegacyModelTrainLink,
 )
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
+from airflow.providers.google.common.deprecated import deprecated
 from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 
 if TYPE_CHECKING:
@@ -338,6 +339,11 @@ class AutoMLPredictOperator(GoogleCloudBaseOperator):
         return PredictResponse.to_dict(result)
 
 
+@deprecated(
+    planned_removal_date="January 01, 2025",
+    use_instead="airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job",
+    category=AirflowProviderDeprecationWarning,
+)
 class AutoMLBatchPredictOperator(GoogleCloudBaseOperator):
     """
     Perform a batch prediction on Google Cloud AutoML.

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/operators/dataflow.py

@@ -432,11 +432,13 @@ class DataflowCreateJavaJobOperator(GoogleCloudBaseOperator):
                 is_running = self.dataflow_hook.is_job_dataflow_running(
                     name=self.job_name,
                     variables=pipeline_options,
+                    location=self.location,
                 )
                 while is_running and self.check_if_running == CheckJobRunning.WaitForRun:
                     is_running = self.dataflow_hook.is_job_dataflow_running(
                         name=self.job_name,
                         variables=pipeline_options,
+                        location=self.location,
                     )
             if not is_running:
                 pipeline_options["jobName"] = job_name

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/operators/dataproc.py

@@ -40,7 +40,11 @@ from google.cloud.dataproc_v1 import Batch, Cluster, ClusterStatus, JobStatus
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
-from airflow.providers.google.cloud.hooks.dataproc import DataprocHook, DataProcJobBuilder
+from airflow.providers.google.cloud.hooks.dataproc import (
+    DataprocHook,
+    DataProcJobBuilder,
+    DataprocResourceIsNotReadyError,
+)
 from airflow.providers.google.cloud.hooks.gcs import GCSHook
 from airflow.providers.google.cloud.links.dataproc import (
     DATAPROC_BATCH_LINK,
@@ -593,6 +597,8 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
     :param delete_on_error: If true the cluster will be deleted if created with ERROR state. Default
         value is true.
     :param use_if_exists: If true use existing cluster
+    :param num_retries_if_resource_is_not_ready: Optional. The number of retry for cluster creation request
+        when resource is not ready error appears.
     :param request_id: Optional. A unique id used to identify the request. If the server receives two
         ``DeleteClusterRequest`` requests with the same id, then the second request will be ignored and the
         first ``google.longrunning.Operation`` created and stored in the backend is returned.
@@ -639,6 +645,7 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
         request_id: str | None = None,
         delete_on_error: bool = True,
         use_if_exists: bool = True,
+        num_retries_if_resource_is_not_ready: int = 0,
         retry: AsyncRetry | _MethodDefault | Retry = DEFAULT,
         timeout: float = 1 * 60 * 60,
         metadata: Sequence[tuple[str, str]] = (),
@@ -695,6 +702,7 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
         self.virtual_cluster_config = virtual_cluster_config
         self.deferrable = deferrable
         self.polling_interval_seconds = polling_interval_seconds
+        self.num_retries_if_resource_is_not_ready = num_retries_if_resource_is_not_ready
 
     def _create_cluster(self, hook: DataprocHook):
         return hook.create_cluster(
@@ -729,20 +737,26 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
             return
         self.log.info("Cluster is in ERROR state")
         self.log.info("Gathering diagnostic information.")
-        operation = hook.diagnose_cluster(
-            region=self.region, cluster_name=self.cluster_name, project_id=self.project_id
-        )
-        operation.result()
-        gcs_uri = str(operation.operation.response.value)
-        self.log.info("Diagnostic information for cluster %s available at: %s", self.cluster_name, gcs_uri)
-
-        if self.delete_on_error:
-            self._delete_cluster(hook)
-            # The delete op is asynchronous and can cause further failure if the cluster finishes
-            # deleting between catching AlreadyExists and checking state
-            self._wait_for_cluster_in_deleting_state(hook)
-            raise AirflowException("Cluster was created in an ERROR state then deleted.")
-        raise AirflowException("Cluster was created but is in ERROR state")
+        try:
+            operation = hook.diagnose_cluster(
+                region=self.region, cluster_name=self.cluster_name, project_id=self.project_id
+            )
+            operation.result()
+            gcs_uri = str(operation.operation.response.value)
+            self.log.info(
+                "Diagnostic information for cluster %s available at: %s", self.cluster_name, gcs_uri
+            )
+        except Exception as diagnose_error:
+            self.log.info("Some error occurred when trying to diagnose cluster.")
+            self.log.exception(diagnose_error)
+        finally:
+            if self.delete_on_error:
+                self._delete_cluster(hook)
+                # The delete op is asynchronous and can cause further failure if the cluster finishes
+                # deleting between catching AlreadyExists and checking state
+                self._wait_for_cluster_in_deleting_state(hook)
+                raise AirflowException("Cluster was created in an ERROR state then deleted.")
+            raise AirflowException("Cluster was created but is in ERROR state")
 
     def _wait_for_cluster_in_deleting_state(self, hook: DataprocHook) -> None:
         time_left = self.timeout
@@ -780,6 +794,16 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
         )
         return hook.wait_for_operation(timeout=self.timeout, result_retry=self.retry, operation=op)
 
+    def _retry_cluster_creation(self, hook: DataprocHook):
+        self.log.info("Retrying creation process for Cluster %s", self.cluster_name)
+        self._delete_cluster(hook)
+        self._wait_for_cluster_in_deleting_state(hook)
+        self.log.info("Starting a new creation for Cluster %s", self.cluster_name)
+        operation = self._create_cluster(hook)
+        cluster = hook.wait_for_operation(timeout=self.timeout, result_retry=self.retry, operation=operation)
+        self.log.info("Cluster created.")
+        return Cluster.to_dict(cluster)
+
     def execute(self, context: Context) -> dict:
         self.log.info("Creating cluster: %s", self.cluster_name)
         hook = DataprocHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
@@ -829,6 +853,25 @@ class DataprocCreateClusterOperator(GoogleCloudBaseOperator):
                 raise
             self.log.info("Cluster already exists.")
             cluster = self._get_cluster(hook)
+        except DataprocResourceIsNotReadyError as resource_not_ready_error:
+            if self.num_retries_if_resource_is_not_ready:
+                attempt = self.num_retries_if_resource_is_not_ready
+                while attempt > 0:
+                    attempt -= 1
+                    try:
+                        cluster = self._retry_cluster_creation(hook)
+                    except DataprocResourceIsNotReadyError:
+                        continue
+                    else:
+                        return cluster
+                self.log.info(
+                    "Retrying Cluster %s creation because of resource not ready was unsuccessful.",
+                    self.cluster_name,
+                )
+            if self.delete_on_error:
+                self._delete_cluster(hook)
+                self._wait_for_cluster_in_deleting_state(hook)
+            raise resource_not_ready_error
         except AirflowException as ae:
             # There still could be a cluster created here in an ERROR state which
             # should be deleted immediately rather than consuming another retry attempt
@@ -2948,6 +2991,8 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
     :param request_id: Optional. A unique id used to identify the request. If the server receives two
         ``CreateBatchRequest`` requests with the same id, then the second request will be ignored and
         the first ``google.longrunning.Operation`` created and stored in the backend is returned.
+    :param num_retries_if_resource_is_not_ready: Optional. The number of retry for cluster creation request
+        when resource is not ready error appears.
     :param retry: A retry object used to retry requests. If ``None`` is specified, requests will not be
         retried.
     :param result_retry: Result retry object used to retry requests. Is used to decrease delay between
@@ -2988,6 +3033,7 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
         batch: dict | Batch,
         batch_id: str | None = None,
         request_id: str | None = None,
+        num_retries_if_resource_is_not_ready: int = 0,
         retry: Retry | _MethodDefault = DEFAULT,
         timeout: float | None = None,
         metadata: Sequence[tuple[str, str]] = (),
@@ -3007,6 +3053,7 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
         self.batch = batch
         self.batch_id = batch_id
         self.request_id = request_id
+        self.num_retries_if_resource_is_not_ready = num_retries_if_resource_is_not_ready
         self.retry = retry
         self.result_retry = result_retry
         self.timeout = timeout
@@ -3028,6 +3075,14 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
         if self.batch_id:
             batch_id = self.batch_id
             self.log.info("Starting batch %s", batch_id)
+            # Persist the link earlier so users can observe the progress
+            DataprocBatchLink.persist(
+                context=context,
+                operator=self,
+                project_id=self.project_id,
+                region=self.region,
+                batch_id=self.batch_id,
+            )
         else:
             self.log.info("Starting batch. The batch ID will be generated since it was not provided.")
 
@@ -3091,6 +3146,15 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
             timeout=self.timeout,
             metadata=self.metadata,
         )
+        if self.num_retries_if_resource_is_not_ready and self.hook.check_error_for_resource_is_not_ready_msg(
+            batch.state_message
+        ):
+            attempt = self.num_retries_if_resource_is_not_ready
+            while attempt > 0:
+                attempt -= 1
+                batch, batch_id = self.retry_batch_creation(batch_id)
+                if not self.hook.check_error_for_resource_is_not_ready_msg(batch.state_message):
+                    break
 
         self.handle_batch_status(context, batch.state, batch_id, batch.state_message)
         return Batch.to_dict(batch)
@@ -3133,6 +3197,50 @@ class DataprocCreateBatchOperator(GoogleCloudBaseOperator):
             raise AirflowException(f"Batch job {batch_id} unspecified. Driver logs: {link}")
         self.log.info("Batch job %s completed. Driver logs: %s", batch_id, link)
 
+    def retry_batch_creation(
+        self,
+        previous_batch_id: str,
+    ):
+        self.log.info("Retrying creation process for batch_id %s", self.batch_id)
+        self.log.info("Deleting previous failed Batch")
+        self.hook.delete_batch(
+            batch_id=previous_batch_id,
+            region=self.region,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+        self.log.info("Starting a new creation for batch_id %s", self.batch_id)
+        try:
+            self.operation = self.hook.create_batch(
+                region=self.region,
+                project_id=self.project_id,
+                batch=self.batch,
+                batch_id=self.batch_id,
+                request_id=self.request_id,
+                retry=self.retry,
+                timeout=self.timeout,
+                metadata=self.metadata,
+            )
+        except AlreadyExists:
+            self.log.info("Batch with given id already exists.")
+            self.log.info("Attaching to the job %s if it is still running.", self.batch_id)
+        else:
+            batch_id = self.operation.metadata.batch.split("/")[-1]
+            self.log.info("The batch %s was created.", batch_id)
+
+        self.log.info("Waiting for the completion of batch job %s", batch_id)
+        batch = self.hook.wait_for_batch(
+            batch_id=batch_id,
+            region=self.region,
+            project_id=self.project_id,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+        return batch, batch_id
+
 
 class DataprocDeleteBatchOperator(GoogleCloudBaseOperator):
     """

{apache_airflow_providers_google-10.23.0rc1 → apache_airflow_providers_google-10.24.0}/airflow/providers/google/cloud/operators/speech_to_text.py

@@ -113,15 +113,14 @@ class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
             gcp_conn_id=self.gcp_conn_id,
             impersonation_chain=self.impersonation_chain,
         )
-
-        FileDetailsLink.persist(
-            context=context,
-            task_instance=self,
-            # Slice from: "gs://{BUCKET_NAME}/{FILE_NAME}" to: "{BUCKET_NAME}/{FILE_NAME}"
-            uri=self.audio["uri"][5:],
-            project_id=self.project_id or hook.project_id,
-        )
-
+        if self.audio.uri:
+            FileDetailsLink.persist(
+                context=context,
+                task_instance=self,
+                # Slice from: "gs://{BUCKET_NAME}/{FILE_NAME}" to: "{BUCKET_NAME}/{FILE_NAME}"
+                uri=self.audio.uri[5:],
+                project_id=self.project_id or hook.project_id,
+            )
         response = hook.recognize_speech(
             config=self.config, audio=self.audio, retry=self.retry, timeout=self.timeout
         )