apache-airflow-providers-google 15.1.0__py3-none-any.whl → 16.0.0__py3-none-any.whl

airflow/providers/google/cloud/hooks/vertex_ai/ray.py

@@ -0,0 +1,223 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""This module contains a Google Cloud Vertex AI hook."""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any
+
+import vertex_ray
+from google._upb._message import ScalarMapContainer  # type: ignore[attr-defined]
+from google.cloud import aiplatform
+from google.cloud.aiplatform.vertex_ray.util import resources
+from google.cloud.aiplatform_v1 import (
+    PersistentResourceServiceClient,
+)
+from proto.marshal.collections.repeated import Repeated
+
+from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
+
+
+class RayHook(GoogleBaseHook):
+    """Hook for Google Cloud Vertex AI Ray APIs."""
+
+    def extract_cluster_id(self, cluster_path) -> str:
+        """Extract cluster_id from cluster_path."""
+        cluster_id = PersistentResourceServiceClient.parse_persistent_resource_path(cluster_path)[
+            "persistent_resource"
+        ]
+        return cluster_id
+
+    def serialize_cluster_obj(self, cluster_obj: resources.Cluster) -> dict:
+        """Serialize Cluster dataclass to dict."""
+
+        def __encode_value(value: Any) -> Any:
+            if isinstance(value, (list, Repeated)):
+                return [__encode_value(nested_value) for nested_value in value]
+            if isinstance(value, ScalarMapContainer):
+                return {key: __encode_value(nested_value) for key, nested_value in dict(value).items()}
+            if dataclasses.is_dataclass(value):
+                return dataclasses.asdict(value)
+            return value
+
+        return {
+            field.name: __encode_value(getattr(cluster_obj, field.name))
+            for field in dataclasses.fields(cluster_obj)
+        }
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_ray_cluster(
+        self,
+        project_id: str,
+        location: str,
+        head_node_type: resources.Resources = resources.Resources(),
+        python_version: str = "3.10",
+        ray_version: str = "2.33",
+        network: str | None = None,
+        service_account: str | None = None,
+        cluster_name: str | None = None,
+        worker_node_types: list[resources.Resources] | None = None,
+        custom_images: resources.NodeImages | None = None,
+        enable_metrics_collection: bool = True,
+        enable_logging: bool = True,
+        psc_interface_config: resources.PscIConfig | None = None,
+        reserved_ip_ranges: list[str] | None = None,
+        labels: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Create a Ray cluster on the Vertex AI.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param head_node_type: The head node resource. Resources.node_count must be 1. If not set, default
+            value of Resources() class will be used.
+        :param python_version: Python version for the ray cluster.
+        :param ray_version: Ray version for the ray cluster. Default is 2.33.0.
+        :param network: Virtual private cloud (VPC) network. For Ray Client, VPC peering is required to
+            connect to the Ray Cluster managed in the Vertex API service. For Ray Job API, VPC network is not
+            required because Ray Cluster connection can be accessed through dashboard address.
+        :param service_account: Service account to be used for running Ray programs on the cluster.
+        :param cluster_name: This value may be up to 63 characters, and valid characters are `[a-z0-9_-]`.
+            The first character cannot be a number or hyphen.
+        :param worker_node_types: The list of Resources of the worker nodes. The same Resources object should
+            not appear multiple times in the list.
+        :param custom_images: The NodeImages which specifies head node and worker nodes images. All the
+            workers will share the same image. If each Resource has a specific custom image, use
+            `Resources.custom_image` for head/worker_node_type(s). Note that configuring
+            `Resources.custom_image` will override `custom_images` here. Allowlist only.
+        :param enable_metrics_collection: Enable Ray metrics collection for visualization.
+        :param enable_logging: Enable exporting Ray logs to Cloud Logging.
+        :param psc_interface_config: PSC-I config.
+        :param reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC network that can
+            be used for this cluster. If set, we will deploy the cluster within the provided IP ranges.
+            Otherwise, the cluster is deployed to any IP ranges under the provided VPC network.
+            Example: ["vertex-ai-ip-range"].
+        :param labels: The labels with user-defined metadata to organize Ray cluster.
+            Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain
+            lowercase letters, numeric characters, underscores and dashes. International characters are allowed.
+            See https://goo.gl/xmQnxf for more information and examples of labels.
+        """
+        aiplatform.init(project=project_id, location=location, credentials=self.get_credentials())
+        cluster_path = vertex_ray.create_ray_cluster(
+            head_node_type=head_node_type,
+            python_version=python_version,
+            ray_version=ray_version,
+            network=network,
+            service_account=service_account,
+            cluster_name=cluster_name,
+            worker_node_types=worker_node_types,
+            custom_images=custom_images,
+            enable_metrics_collection=enable_metrics_collection,
+            enable_logging=enable_logging,
+            psc_interface_config=psc_interface_config,
+            reserved_ip_ranges=reserved_ip_ranges,
+            labels=labels,
+        )
+        return cluster_path
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def list_ray_clusters(
+        self,
+        project_id: str,
+        location: str,
+    ) -> list[resources.Cluster]:
+        """
+        List Ray clusters under the currently authenticated project.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        """
+        aiplatform.init(project=project_id, location=location, credentials=self.get_credentials())
+        ray_clusters = vertex_ray.list_ray_clusters()
+        return ray_clusters
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_ray_cluster(
+        self,
+        project_id: str,
+        location: str,
+        cluster_id: str,
+    ) -> resources.Cluster:
+        """
+        Get Ray cluster.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param cluster_id: Cluster resource ID.
+        """
+        aiplatform.init(project=project_id, location=location, credentials=self.get_credentials())
+        ray_cluster_name = PersistentResourceServiceClient.persistent_resource_path(
+            project=project_id,
+            location=location,
+            persistent_resource=cluster_id,
+        )
+        ray_cluster = vertex_ray.get_ray_cluster(
+            cluster_resource_name=ray_cluster_name,
+        )
+        return ray_cluster
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def update_ray_cluster(
+        self,
+        project_id: str,
+        location: str,
+        cluster_id: str,
+        worker_node_types: list[resources.Resources],
+    ) -> str:
+        """
+        Update Ray cluster (currently support resizing node counts for worker nodes).
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param cluster_id: Cluster resource ID.
+        :param worker_node_types: The list of Resources of the resized worker nodes. The same Resources
+            object should not appear multiple times in the list.
+        """
+        aiplatform.init(project=project_id, location=location, credentials=self.get_credentials())
+        ray_cluster_name = PersistentResourceServiceClient.persistent_resource_path(
+            project=project_id,
+            location=location,
+            persistent_resource=cluster_id,
+        )
+        updated_ray_cluster_name = vertex_ray.update_ray_cluster(
+            cluster_resource_name=ray_cluster_name, worker_node_types=worker_node_types
+        )
+        return updated_ray_cluster_name
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_ray_cluster(
+        self,
+        project_id: str,
+        location: str,
+        cluster_id: str,
+    ) -> None:
+        """
+        Delete Ray cluster.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param cluster_id: Cluster resource ID.
+        """
+        aiplatform.init(project=project_id, location=location, credentials=self.get_credentials())
+        ray_cluster_name = PersistentResourceServiceClient.persistent_resource_path(
+            project=project_id,
+            location=location,
+            persistent_resource=cluster_id,
+        )
+        vertex_ray.delete_ray_cluster(cluster_resource_name=ray_cluster_name)

airflow/providers/google/cloud/links/cloud_run.py

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.providers.google.cloud.links.base import BaseGoogleLink
+from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
+
+if TYPE_CHECKING:
+    from airflow.models import BaseOperator
+    from airflow.models.taskinstancekey import TaskInstanceKey
+    from airflow.utils.context import Context
+
+if AIRFLOW_V_3_0_PLUS:
+    from airflow.sdk.execution_time.xcom import XCom
+else:
+    from airflow.models.xcom import XCom  # type: ignore[no-redef]
+
+
+class CloudRunJobLoggingLink(BaseGoogleLink):
+    """Helper class for constructing Cloud Run Job Logging link."""
+
+    name = "Cloud Run Job Logging"
+    key = "log_uri"
+
+    @staticmethod
+    def persist(
+        context: Context,
+        task_instance: BaseOperator,
+        log_uri: str,
+    ):
+        task_instance.xcom_push(
+            context,
+            key=CloudRunJobLoggingLink.key,
+            value=log_uri,
+        )
+
+    def get_link(
+        self,
+        operator: BaseOperator,
+        *,
+        ti_key: TaskInstanceKey,
+    ) -> str:
+        return XCom.get_value(key=self.key, ti_key=ti_key)

airflow/providers/google/cloud/links/vertex_ai.py

@@ -54,6 +54,10 @@ VERTEX_AI_PIPELINE_JOB_LINK = (
     VERTEX_AI_BASE_LINK + "/locations/{region}/pipelines/runs/{pipeline_id}?project={project_id}"
 )
 VERTEX_AI_PIPELINE_JOB_LIST_LINK = VERTEX_AI_BASE_LINK + "/pipelines/runs?project={project_id}"
+VERTEX_AI_RAY_CLUSTER_LINK = (
+    VERTEX_AI_BASE_LINK + "/locations/{location}/ray-clusters/{cluster_id}?project={project_id}"
+)
+VERTEX_AI_RAY_CLUSTER_LIST_LINK = VERTEX_AI_BASE_LINK + "/ray?project={project_id}"
 
 
 class VertexAIModelLink(BaseGoogleLink):
@@ -369,3 +373,48 @@ class VertexAIPipelineJobListLink(BaseGoogleLink):
                 "project_id": task_instance.project_id,
             },
         )
+
+
+class VertexAIRayClusterLink(BaseGoogleLink):
+    """Helper class for constructing Vertex AI Ray Cluster link."""
+
+    name = "Ray Cluster"
+    key = "ray_cluster_conf"
+    format_str = VERTEX_AI_RAY_CLUSTER_LINK
+
+    @staticmethod
+    def persist(
+        context: Context,
+        task_instance,
+        cluster_id: str,
+    ):
+        task_instance.xcom_push(
+            context=context,
+            key=VertexAIRayClusterLink.key,
+            value={
+                "location": task_instance.location,
+                "cluster_id": cluster_id,
+                "project_id": task_instance.project_id,
+            },
+        )
+
+
+class VertexAIRayClusterListLink(BaseGoogleLink):
+    """Helper class for constructing Vertex AI Ray Cluster List link."""
+
+    name = "Ray Cluster List"
+    key = "ray_cluster_list_conf"
+    format_str = VERTEX_AI_RAY_CLUSTER_LIST_LINK
+
+    @staticmethod
+    def persist(
+        context: Context,
+        task_instance,
+    ):
+        task_instance.xcom_push(
+            context=context,
+            key=VertexAIRayClusterListLink.key,
+            value={
+                "project_id": task_instance.project_id,
+            },
+        )

airflow/providers/google/cloud/log/gcs_task_handler.py

@@ -61,13 +61,15 @@ class GCSRemoteLogIO(LoggingMixin):  # noqa: D101
     remote_base: str
     base_log_folder: Path = attrs.field(converter=Path)
     delete_local_copy: bool
+    project_id: str | None = None
 
-    gcp_key_path: str | None
-    gcp_keyfile_dict: dict | None
-    scopes: Collection[str] | None
-    project_id: str
+    gcp_key_path: str | None = None
+    gcp_keyfile_dict: dict | None = None
+    scopes: Collection[str] | None = _DEFAULT_SCOPESS
 
-    def upload(self, path: os.PathLike, ti: RuntimeTI):
+    processors = ()
+
+    def upload(self, path: os.PathLike | str, ti: RuntimeTI):
         """Upload the given log path to the remote storage."""
         path = Path(path)
         if path.is_absolute():

airflow/providers/google/cloud/operators/bigquery.py

@@ -93,16 +93,32 @@ class IfExistAction(enum.Enum):
     SKIP = "skip"
 
 
+class _BigQueryHookWithFlexibleProjectId(BigQueryHook):
+    @property
+    def project_id(self) -> str:
+        _, project_id = self.get_credentials_and_project_id()
+        return project_id or PROVIDE_PROJECT_ID
+
+    @project_id.setter
+    def project_id(self, value: str) -> None:
+        cached_creds, _ = self.get_credentials_and_project_id()
+        self._cached_project_id = value or PROVIDE_PROJECT_ID
+        self._cached_credntials = cached_creds
+
+
 class _BigQueryDbHookMixin:
-    def get_db_hook(self: BigQueryCheckOperator) -> BigQueryHook:  # type:ignore[misc]
+    def get_db_hook(self: BigQueryCheckOperator) -> _BigQueryHookWithFlexibleProjectId:  # type:ignore[misc]
         """Get BigQuery DB Hook."""
-        return BigQueryHook(
+        hook = _BigQueryHookWithFlexibleProjectId(
             gcp_conn_id=self.gcp_conn_id,
             use_legacy_sql=self.use_legacy_sql,
             location=self.location,
             impersonation_chain=self.impersonation_chain,
             labels=self.labels,
         )
+        if self.project_id:
+            hook.project_id = self.project_id
+        return hook
 
 
 class _BigQueryOperatorsEncryptionConfigurationMixin:
@@ -190,6 +206,7 @@ class BigQueryCheckOperator(
         https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs.
         For example, [{ 'name': 'corpus', 'parameterType': { 'type': 'STRING' },
         'parameterValue': { 'value': 'romeoandjuliet' } }]. (templated)
+    :param project_id: Google Cloud Project where the job is running
     """
 
     template_fields: Sequence[str] = (
@@ -208,6 +225,7 @@ class BigQueryCheckOperator(
         *,
         sql: str,
         gcp_conn_id: str = "google_cloud_default",
+        project_id: str = PROVIDE_PROJECT_ID,
         use_legacy_sql: bool = True,
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
@@ -228,6 +246,7 @@ class BigQueryCheckOperator(
         self.deferrable = deferrable
         self.poll_interval = poll_interval
         self.query_params = query_params
+        self.project_id = project_id
 
     def _submit_job(
         self,
@@ -243,7 +262,7 @@ class BigQueryCheckOperator(
 
         return hook.insert_job(
             configuration=configuration,
-            project_id=hook.project_id,
+            project_id=self.project_id,
             location=self.location,
             job_id=job_id,
             nowait=True,
@@ -257,6 +276,8 @@ class BigQueryCheckOperator(
                 gcp_conn_id=self.gcp_conn_id,
                 impersonation_chain=self.impersonation_chain,
             )
+            if self.project_id is None:
+                self.project_id = hook.project_id
             job = self._submit_job(hook, job_id="")
             context["ti"].xcom_push(key="job_id", value=job.job_id)
             if job.running():
@@ -265,7 +286,7 @@ class BigQueryCheckOperator(
                     trigger=BigQueryCheckTrigger(
                         conn_id=self.gcp_conn_id,
                         job_id=job.job_id,
-                        project_id=hook.project_id,
+                        project_id=self.project_id,
                         location=self.location or hook.location,
                         poll_interval=self.poll_interval,
                         impersonation_chain=self.impersonation_chain,
@@ -342,6 +363,7 @@ class BigQueryValueCheckOperator(
     :param deferrable: Run operator in the deferrable mode.
     :param poll_interval: (Deferrable mode only) polling period in seconds to
         check for the status of job.
+    :param project_id: Google Cloud Project where the job is running
     """
 
     template_fields: Sequence[str] = (
@@ -363,6 +385,7 @@ class BigQueryValueCheckOperator(
         tolerance: Any = None,
         encryption_configuration: dict | None = None,
         gcp_conn_id: str = "google_cloud_default",
+        project_id: str = PROVIDE_PROJECT_ID,
         use_legacy_sql: bool = True,
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
@@ -380,6 +403,7 @@ class BigQueryValueCheckOperator(
         self.labels = labels
         self.deferrable = deferrable
         self.poll_interval = poll_interval
+        self.project_id = project_id
 
     def _submit_job(
         self,
@@ -398,7 +422,7 @@ class BigQueryValueCheckOperator(
 
         return hook.insert_job(
             configuration=configuration,
-            project_id=hook.project_id,
+            project_id=self.project_id,
             location=self.location,
             job_id=job_id,
             nowait=True,
@@ -409,7 +433,8 @@ class BigQueryValueCheckOperator(
             super().execute(context=context)
         else:
             hook = BigQueryHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
-
+            if self.project_id is None:
+                self.project_id = hook.project_id
             job = self._submit_job(hook, job_id="")
             context["ti"].xcom_push(key="job_id", value=job.job_id)
             if job.running():
@@ -418,7 +443,7 @@ class BigQueryValueCheckOperator(
                     trigger=BigQueryValueCheckTrigger(
                         conn_id=self.gcp_conn_id,
                         job_id=job.job_id,
-                        project_id=hook.project_id,
+                        project_id=self.project_id,
                         location=self.location or hook.location,
                         sql=self.sql,
                         pass_value=self.pass_value,
@@ -575,6 +600,9 @@ class BigQueryIntervalCheckOperator(
             hook = BigQueryHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
             self.log.info("Using ratio formula: %s", self.ratio_formula)
 
+            if self.project_id is None:
+                self.project_id = hook.project_id
+
             self.log.info("Executing SQL check: %s", self.sql1)
             job_1 = self._submit_job(hook, sql=self.sql1, job_id="")
             context["ti"].xcom_push(key="job_id", value=job_1.job_id)
@@ -587,7 +615,7 @@ class BigQueryIntervalCheckOperator(
                     conn_id=self.gcp_conn_id,
                     first_job_id=job_1.job_id,
                     second_job_id=job_2.job_id,
-                    project_id=hook.project_id,
+                    project_id=self.project_id,
                     table=self.table,
                     location=self.location or hook.location,
                     metrics_thresholds=self.metrics_thresholds,
@@ -654,6 +682,7 @@ class BigQueryColumnCheckOperator(
         Service Account Token Creator IAM role to the directly preceding identity, with first
         account from the list granting this role to the originating account (templated).
     :param labels: a dictionary containing labels for the table, passed to BigQuery
+    :param project_id: Google Cloud Project where the job is running
     """
 
     template_fields: Sequence[str] = tuple(set(SQLColumnCheckOperator.template_fields) | {"gcp_conn_id"})
@@ -670,6 +699,7 @@ class BigQueryColumnCheckOperator(
         accept_none: bool = True,
         encryption_configuration: dict | None = None,
         gcp_conn_id: str = "google_cloud_default",
+        project_id: str = PROVIDE_PROJECT_ID,
         use_legacy_sql: bool = True,
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
@@ -695,6 +725,7 @@ class BigQueryColumnCheckOperator(
         self.location = location
         self.impersonation_chain = impersonation_chain
         self.labels = labels
+        self.project_id = project_id
 
     def _submit_job(
         self,
@@ -706,7 +737,7 @@ class BigQueryColumnCheckOperator(
         self.include_encryption_configuration(configuration, "query")
         return hook.insert_job(
             configuration=configuration,
-            project_id=hook.project_id,
+            project_id=self.project_id,
             location=self.location,
             job_id=job_id,
             nowait=False,
@@ -715,6 +746,9 @@ class BigQueryColumnCheckOperator(
     def execute(self, context=None):
         """Perform checks on the given columns."""
         hook = self.get_db_hook()
+
+        if self.project_id is None:
+            self.project_id = hook.project_id
         failed_tests = []
 
         job = self._submit_job(hook, job_id="")
@@ -786,6 +820,7 @@ class BigQueryTableCheckOperator(
         account from the list granting this role to the originating account (templated).
     :param labels: a dictionary containing labels for the table, passed to BigQuery
     :param encryption_configuration: (Optional) Custom encryption configuration (e.g., Cloud KMS keys).
+    :param project_id: Google Cloud Project where the job is running
 
         .. code-block:: python
 
@@ -805,6 +840,7 @@ class BigQueryTableCheckOperator(
         checks: dict,
         partition_clause: str | None = None,
         gcp_conn_id: str = "google_cloud_default",
+        project_id: str = PROVIDE_PROJECT_ID,
         use_legacy_sql: bool = True,
         location: str | None = None,
         impersonation_chain: str | Sequence[str] | None = None,
@@ -819,6 +855,7 @@ class BigQueryTableCheckOperator(
         self.impersonation_chain = impersonation_chain
         self.labels = labels
         self.encryption_configuration = encryption_configuration
+        self.project_id = project_id
 
     def _submit_job(
         self,
@@ -832,7 +869,7 @@ class BigQueryTableCheckOperator(
 
         return hook.insert_job(
             configuration=configuration,
-            project_id=hook.project_id,
+            project_id=self.project_id,
             location=self.location,
             job_id=job_id,
             nowait=False,
@@ -841,6 +878,8 @@ class BigQueryTableCheckOperator(
     def execute(self, context=None):
         """Execute the given checks on the table."""
         hook = self.get_db_hook()
+        if self.project_id is None:
+            self.project_id = hook.project_id
         job = self._submit_job(hook, job_id="")
         context["ti"].xcom_push(key="job_id", value=job.job_id)
         records = job.result().to_dataframe()

airflow/providers/google/cloud/operators/cloud_run.py

@@ -27,6 +27,7 @@ from google.cloud.run_v2 import Job, Service
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.cloud_run import CloudRunHook, CloudRunServiceHook
+from airflow.providers.google.cloud.links.cloud_run import CloudRunJobLoggingLink
 from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
 from airflow.providers.google.cloud.triggers.cloud_run import CloudRunJobFinishedTrigger, RunJobStatus
 
@@ -248,7 +249,7 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
 
     :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
     :param region: Required. The ID of the Google Cloud region that the service belongs to.
-    :param job_name: Required. The name of the job to update.
+    :param job_name: Required. The name of the job to execute.
     :param overrides: Optional map of override values.
     :param gcp_conn_id: The connection ID used to connect to Google Cloud.
     :param polling_period_seconds: Optional. Control the rate of the poll for the result of deferrable run.
@@ -265,7 +266,17 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
     :param deferrable: Run the operator in deferrable mode.
     """
 
-    template_fields = ("project_id", "region", "gcp_conn_id", "impersonation_chain", "job_name", "overrides")
+    operator_extra_links = (CloudRunJobLoggingLink(),)
+    template_fields = (
+        "project_id",
+        "region",
+        "gcp_conn_id",
+        "impersonation_chain",
+        "job_name",
+        "overrides",
+        "polling_period_seconds",
+        "timeout_seconds",
+    )
 
     def __init__(
         self,
@@ -303,6 +314,13 @@ class CloudRunExecuteJobOperator(GoogleCloudBaseOperator):
         if self.operation is None:
             raise AirflowException("Operation is None")
 
+        if self.operation.metadata.log_uri:
+            CloudRunJobLoggingLink.persist(
+                context=context,
+                task_instance=self,
+                log_uri=self.operation.metadata.log_uri,
+            )
+
         if not self.deferrable:
             result: Execution = self._wait_for_operation(self.operation)
             self._fail_if_execution_failed(result)

airflow/providers/google/cloud/operators/gcs.py

@@ -204,6 +204,7 @@ class GCSListObjectsOperator(GoogleCloudBaseOperator):
         "bucket",
         "prefix",
         "delimiter",
+        "match_glob",
         "impersonation_chain",
     )