apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl

airflow/providers/google/cloud/hooks/cloud_sql.py

@@ -19,6 +19,7 @@
 
 from __future__ import annotations
 
+import base64
 import errno
 import json
 import os
@@ -34,7 +35,7 @@ import uuid
 from inspect import signature
 from pathlib import Path
 from subprocess import PIPE, Popen
-from tempfile import gettempdir
+from tempfile import NamedTemporaryFile, _TemporaryFileWrapper, gettempdir
 from typing import TYPE_CHECKING, Any, Sequence
 from urllib.parse import quote_plus
 
@@ -49,12 +50,21 @@ from googleapiclient.errors import HttpError
 from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 from airflow.models import Connection
-from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook, get_field
+from airflow.providers.google.cloud.hooks.secret_manager import (
+    GoogleCloudSecretManagerHook,
+)
+from airflow.providers.google.common.hooks.base_google import (
+    PROVIDE_PROJECT_ID,
+    GoogleBaseAsyncHook,
+    GoogleBaseHook,
+    get_field,
+)
 from airflow.providers.mysql.hooks.mysql import MySqlHook
 from airflow.providers.postgres.hooks.postgres import PostgresHook
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
+    from google.cloud.secretmanager_v1 import AccessSecretVersionResponse
     from requests import Session
 
 UNIX_PATH_MAX = 108
@@ -377,6 +387,29 @@ class CloudSQLHook(GoogleBaseHook):
         except HttpError as ex:
             raise AirflowException(f"Cloning of instance {instance} failed: {ex.content}")
 
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_ssl_certificate(self, instance: str, body: dict, project_id: str):
+        """
+        Create SSL certificate for a Cloud SQL instance.
+
+        :param instance: Cloud SQL instance ID. This does not include the project ID.
+        :param body: The request body, as described in
+            https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/sslCerts/insert#SslCertsInsertRequest
+        :param project_id: Project ID of the project that contains the instance. If set
+            to None or missing, the default project_id from the Google Cloud connection is used.
+        :return: SslCert insert response. For more details see:
+            https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/sslCerts/insert#response-body
+        """
+        response = (
+            self.get_conn()
+            .sslCerts()
+            .insert(project=project_id, instance=instance, body=body)
+            .execute(num_retries=self.num_retries)
+        )
+        operation_name = response.get("operation", {}).get("name", {})
+        self._wait_for_operation_to_complete(project_id=project_id, operation_name=operation_name)
+        return response
+
     @GoogleBaseHook.fallback_to_default_project_id
     def _wait_for_operation_to_complete(
         self, project_id: str, operation_name: str, time_to_sleep: int = TIME_TO_SLEEP_IN_SECONDS
@@ -482,7 +515,7 @@ class CloudSqlProxyRunner(LoggingMixin):
         path_prefix: str,
         instance_specification: str,
         gcp_conn_id: str = "google_cloud_default",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         sql_proxy_version: str | None = None,
         sql_proxy_binary_path: str | None = None,
     ) -> None:
@@ -758,7 +791,24 @@ class CloudSQLDatabaseHook(BaseHook):
     :param gcp_conn_id: The connection ID used to connect to Google Cloud for
         cloud-sql-proxy authentication.
     :param default_gcp_project_id: Default project id used if project_id not specified
-           in the connection URL
+        in the connection URL
+    :param ssl_cert: Optional. Path to client certificate to authenticate when SSL is used. Overrides the
+        connection field ``sslcert``.
+    :param ssl_key: Optional. Path to client private key to authenticate when SSL is used. Overrides the
+        connection field ``sslkey``.
+    :param ssl_root_cert: Optional. Path to server's certificate to authenticate when SSL is used. Overrides
+        the connection field ``sslrootcert``.
+    :param ssl_secret_id: Optional. ID of the secret in Google Cloud Secret Manager that stores SSL
+        certificate in the format below:
+
+        {'sslcert': '',
+         'sslkey': '',
+         'sslrootcert': ''}
+
+        Overrides the connection fields ``sslcert``, ``sslkey``, ``sslrootcert``.
+        Note that according to the Secret Manager requirements, the mentioned dict should be saved as a
+        string, and encoded with base64.
+        Note that this parameter is incompatible with parameters ``ssl_cert``, ``ssl_key``, ``ssl_root_cert``.
     """
 
     conn_name_attr = "gcp_cloudsql_conn_id"
@@ -770,12 +820,18 @@ class CloudSQLDatabaseHook(BaseHook):
         self,
         gcp_cloudsql_conn_id: str = "google_cloud_sql_default",
         gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
         default_gcp_project_id: str | None = None,
         sql_proxy_binary_path: str | None = None,
+        ssl_cert: str | None = None,
+        ssl_key: str | None = None,
+        ssl_root_cert: str | None = None,
+        ssl_secret_id: str | None = None,
     ) -> None:
         super().__init__()
         self.gcp_conn_id = gcp_conn_id
         self.gcp_cloudsql_conn_id = gcp_cloudsql_conn_id
+        self.impersonation_chain = impersonation_chain
         self.cloudsql_connection = self.get_connection(self.gcp_cloudsql_conn_id)
         self.extras = self.cloudsql_connection.extra_dejson
         self.project_id = self.extras.get("project_id", default_gcp_project_id)
@@ -792,9 +848,11 @@ class CloudSQLDatabaseHook(BaseHook):
         self.password = self.cloudsql_connection.password
         self.public_ip = self.cloudsql_connection.host
         self.public_port = self.cloudsql_connection.port
-        self.sslcert = self.extras.get("sslcert")
-        self.sslkey = self.extras.get("sslkey")
-        self.sslrootcert = self.extras.get("sslrootcert")
+        self.ssl_cert = ssl_cert
+        self.ssl_key = ssl_key
+        self.ssl_root_cert = ssl_root_cert
+        self.ssl_secret_id = ssl_secret_id
+        self._ssl_cert_temp_files: dict[str, _TemporaryFileWrapper] = {}
         # Port and socket path and db_hook are automatically generated
         self.sql_proxy_tcp_port = None
         self.sql_proxy_unique_path: str | None = None
@@ -805,6 +863,84 @@ class CloudSQLDatabaseHook(BaseHook):
         self.db_conn_id = str(uuid.uuid1())
         self._validate_inputs()
 
+    @property
+    def sslcert(self) -> str | None:
+        return self._get_ssl_temporary_file_path(cert_name="sslcert", cert_path=self.ssl_cert)
+
+    @property
+    def sslkey(self) -> str | None:
+        return self._get_ssl_temporary_file_path(cert_name="sslkey", cert_path=self.ssl_key)
+
+    @property
+    def sslrootcert(self) -> str | None:
+        return self._get_ssl_temporary_file_path(cert_name="sslrootcert", cert_path=self.ssl_root_cert)
+
+    def _get_ssl_temporary_file_path(self, cert_name: str, cert_path: str | None) -> str | None:
+        cert_value = self._get_cert_from_secret(cert_name)
+        original_cert_path = cert_path or self.extras.get(cert_name)
+        if cert_value or original_cert_path:
+            if cert_name not in self._ssl_cert_temp_files:
+                return self._set_temporary_ssl_file(
+                    cert_name=cert_name, cert_path=original_cert_path, cert_value=cert_value
+                )
+            return self._ssl_cert_temp_files[cert_name].name
+        return None
+
+    def _get_cert_from_secret(self, cert_name: str) -> str | None:
+        if not self.ssl_secret_id:
+            return None
+
+        secret_hook = GoogleCloudSecretManagerHook(
+            gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
+        )
+        secret: AccessSecretVersionResponse = secret_hook.access_secret(
+            project_id=self.project_id,
+            secret_id=self.ssl_secret_id,
+        )
+        secret_data = json.loads(base64.b64decode(secret.payload.data))
+        if cert_name in secret_data:
+            return secret_data[cert_name]
+        else:
+            raise AirflowException(
+                "Invalid secret format. Expected dictionary with keys: `sslcert`, `sslkey`, `sslrootcert`"
+            )
+
+    def _set_temporary_ssl_file(
+        self, cert_name: str, cert_path: str | None = None, cert_value: str | None = None
+    ) -> str | None:
+        """Save the certificate as a temporary file.
+
+        This method was implemented in order to overcome psql connection error caused by excessive file
+        permissions: "private key file "..." has group or world access; file must have permissions
+        u=rw (0600) or less if owned by the current user, or permissions u=rw,g=r (0640) or less if owned
+        by root". NamedTemporaryFile enforces using exactly one of create/read/write/append mode so the
+        created file obtains least required permissions "-rw-------" that satisfies the rules.
+
+        :param cert_name: Required. Name of the certificate (one of sslcert, sslkey, sslrootcert).
+        :param cert_path: Optional. Path to the certificate.
+        :param cert_value: Optional. The certificate content.
+
+        :returns: The path to the temporary certificate file.
+        """
+        if all([cert_path, cert_value]):
+            raise AirflowException(
+                "Both parameters were specified: `cert_path`, `cert_value`. Please use only one of them."
+            )
+        if not any([cert_path, cert_value]):
+            self.log.info("Neither cert path and cert value provided. Nothing to save.")
+            return None
+
+        _temp_file = NamedTemporaryFile(mode="w+b", prefix="/tmp/certs/")
+        if cert_path:
+            with open(cert_path, "rb") as cert_file:
+                _temp_file.write(cert_file.read())
+        elif cert_value:
+            _temp_file.write(cert_value.encode("ascii"))
+        _temp_file.flush()
+        self._ssl_cert_temp_files[cert_name] = _temp_file
+        self.log.info("Copied the certificate '%s' into a temporary file '%s'", cert_name, _temp_file.name)
+        return _temp_file.name
+
     @staticmethod
     def _get_bool(val: Any) -> bool:
         if val == "False" or val is False:
@@ -836,6 +972,17 @@ class CloudSQLDatabaseHook(BaseHook):
                 " SSL is not needed as Cloud SQL Proxy "
                 "provides encryption on its own"
             )
+        if any([self.ssl_key, self.ssl_cert, self.ssl_root_cert]) and self.ssl_secret_id:
+            raise AirflowException(
+                "Invalid SSL settings. Please use either all of parameters ['ssl_cert', 'ssl_cert', "
+                "'ssl_root_cert'] or a single parameter 'ssl_secret_id'."
+            )
+        if any([self.ssl_key, self.ssl_cert, self.ssl_root_cert]):
+            field_names = ["ssl_key", "ssl_cert", "ssl_root_cert"]
+            if missed_values := [field for field in field_names if not getattr(self, field)]:
+                s = "s are" if len(missed_values) > 1 else "is"
+                missed_values_str = ", ".join(f for f in missed_values)
+                raise AirflowException(f"Invalid SSL settings. Parameter{s} missing: {missed_values_str}")
 
     def validate_ssl_certs(self) -> None:
         """

airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py

@@ -46,7 +46,11 @@ from googleapiclient.errors import HttpError
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.google.common.consts import CLIENT_INFO
-from airflow.providers.google.common.hooks.base_google import GoogleBaseAsyncHook, GoogleBaseHook
+from airflow.providers.google.common.hooks.base_google import (
+    PROVIDE_PROJECT_ID,
+    GoogleBaseAsyncHook,
+    GoogleBaseHook,
+)
 
 if TYPE_CHECKING:
     from google.cloud.storage_transfer_v1.services.storage_transfer_service.pagers import (
@@ -84,6 +88,7 @@ ALREADY_EXISTING_IN_SINK = "overwriteObjectsAlreadyExistingInSink"
 AWS_ACCESS_KEY = "awsAccessKey"
 AWS_SECRET_ACCESS_KEY = "secretAccessKey"
 AWS_S3_DATA_SOURCE = "awsS3DataSource"
+AWS_ROLE_ARN = "roleArn"
 BODY = "body"
 BUCKET_NAME = "bucketName"
 COUNTERS = "counters"
@@ -504,7 +509,7 @@ class CloudDataTransferServiceHook(GoogleBaseHook):
 class CloudDataTransferServiceAsyncHook(GoogleBaseAsyncHook):
     """Asynchronous hook for Google Storage Transfer Service."""
 
-    def __init__(self, project_id: str | None = None, **kwargs: Any) -> None:
+    def __init__(self, project_id: str = PROVIDE_PROJECT_ID, **kwargs: Any) -> None:
         super().__init__(**kwargs)
         self.project_id = project_id
         self._client: StorageTransferServiceAsyncClient | None = None

airflow/providers/google/cloud/hooks/compute_ssh.py

@@ -29,6 +29,7 @@ from paramiko.ssh_exception import SSHException
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.compute import ComputeEngineHook
 from airflow.providers.google.cloud.hooks.os_login import OSLoginHook
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
 from airflow.providers.ssh.hooks.ssh import SSHHook
 from airflow.utils.types import NOTSET, ArgNotSet
 
@@ -109,7 +110,7 @@ class ComputeEngineSSHHook(SSHHook):
         instance_name: str | None = None,
         zone: str | None = None,
         user: str | None = "root",
-        project_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
         hostname: str | None = None,
         use_internal_ip: bool = False,
         use_iap_tunnel: bool = False,

airflow/providers/google/cloud/hooks/dataflow.py

@@ -31,9 +31,23 @@ from copy import deepcopy
 from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
 
 from deprecated import deprecated
-from google.cloud.dataflow_v1beta3 import GetJobRequest, Job, JobState, JobsV1Beta3AsyncClient, JobView
+from google.cloud.dataflow_v1beta3 import (
+    GetJobRequest,
+    Job,
+    JobState,
+    JobsV1Beta3AsyncClient,
+    JobView,
+    ListJobMessagesRequest,
+    MessagesV1Beta3AsyncClient,
+    MetricsV1Beta3AsyncClient,
+)
+from google.cloud.dataflow_v1beta3.types import (
+    GetJobMetricsRequest,
+    JobMessageImportance,
+    JobMetrics,
+)
 from google.cloud.dataflow_v1beta3.types.jobs import ListJobsRequest
-from googleapiclient.discovery import build
+from googleapiclient.discovery import Resource, build
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType, beam_options_to_args
@@ -47,6 +61,8 @@ from airflow.utils.timeout import timeout
 
 if TYPE_CHECKING:
     from google.cloud.dataflow_v1beta3.services.jobs_v1_beta3.pagers import ListJobsAsyncPager
+    from google.cloud.dataflow_v1beta3.services.messages_v1_beta3.pagers import ListJobMessagesAsyncPager
+    from google.protobuf.timestamp_pb2 import Timestamp
 
 
 # This is the default location
@@ -561,7 +577,7 @@ class DataflowHook(GoogleBaseHook):
             impersonation_chain=impersonation_chain,
         )
 
-    def get_conn(self) -> build:
+    def get_conn(self) -> Resource:
         """Return a Google Cloud Dataflow service object."""
         http_authorized = self._authorize()
         return build("dataflow", "v1b3", http=http_authorized, cache_discovery=False)
@@ -641,9 +657,9 @@ class DataflowHook(GoogleBaseHook):
         on_new_job_callback: Callable[[dict], None] | None = None,
         location: str = DEFAULT_DATAFLOW_LOCATION,
         environment: dict | None = None,
-    ) -> dict:
+    ) -> dict[str, str]:
         """
-        Start Dataflow template job.
+        Launch a Dataflow job with a Classic Template and wait for its completion.
 
         :param job_name: The name of the job.
         :param variables: Map of job runtime environment options.
@@ -676,26 +692,14 @@ class DataflowHook(GoogleBaseHook):
             environment=environment,
         )
 
-        service = self.get_conn()
-
-        request = (
-            service.projects()
-            .locations()
-            .templates()
-            .launch(
-                projectId=project_id,
-                location=location,
-                gcsPath=dataflow_template,
-                body={
-                    "jobName": name,
-                    "parameters": parameters,
-                    "environment": environment,
-                },
-            )
+        job: dict[str, str] = self.send_launch_template_request(
+            project_id=project_id,
+            location=location,
+            gcs_path=dataflow_template,
+            job_name=name,
+            parameters=parameters,
+            environment=environment,
         )
-        response = request.execute(num_retries=self.num_retries)
-
-        job = response["job"]
 
         if on_new_job_id_callback:
             warnings.warn(
@@ -703,7 +707,7 @@ class DataflowHook(GoogleBaseHook):
                 AirflowProviderDeprecationWarning,
                 stacklevel=3,
             )
-            on_new_job_id_callback(job.get("id"))
+            on_new_job_id_callback(job["id"])
 
         if on_new_job_callback:
             on_new_job_callback(job)
@@ -722,7 +726,62 @@ class DataflowHook(GoogleBaseHook):
             expected_terminal_state=self.expected_terminal_state,
         )
         jobs_controller.wait_for_done()
-        return response["job"]
+        return job
+
+    @_fallback_to_location_from_variables
+    @_fallback_to_project_id_from_variables
+    @GoogleBaseHook.fallback_to_default_project_id
+    def launch_job_with_template(
+        self,
+        *,
+        job_name: str,
+        variables: dict,
+        parameters: dict,
+        dataflow_template: str,
+        project_id: str,
+        append_job_name: bool = True,
+        location: str = DEFAULT_DATAFLOW_LOCATION,
+        environment: dict | None = None,
+    ) -> dict[str, str]:
+        """
+        Launch a Dataflow job with a Classic Template and exit without waiting for its completion.
+
+        :param job_name: The name of the job.
+        :param variables: Map of job runtime environment options.
+            It will update environment argument if passed.
+
+            .. seealso::
+                For more information on possible configurations, look at the API documentation
+                `https://cloud.google.com/dataflow/pipelines/specifying-exec-params
+                <https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment>`__
+
+        :param parameters: Parameters for the template
+        :param dataflow_template: GCS path to the template.
+        :param project_id: Optional, the Google Cloud project ID in which to start a job.
+            If set to None or missing, the default project_id from the Google Cloud connection is used.
+        :param append_job_name: True if unique suffix has to be appended to job name.
+        :param location: Job location.
+
+            .. seealso::
+                For more information on possible configurations, look at the API documentation
+                `https://cloud.google.com/dataflow/pipelines/specifying-exec-params
+                <https://cloud.google.com/dataflow/docs/reference/rest/v1b3/RuntimeEnvironment>`__
+        :return: the Dataflow job response
+        """
+        name = self.build_dataflow_job_name(job_name, append_job_name)
+        environment = self._update_environment(
+            variables=variables,
+            environment=environment,
+        )
+        job: dict[str, str] = self.send_launch_template_request(
+            project_id=project_id,
+            location=location,
+            gcs_path=dataflow_template,
+            job_name=name,
+            parameters=parameters,
+            environment=environment,
+        )
+        return job
 
     def _update_environment(self, variables: dict, environment: dict | None = None) -> dict:
         environment = environment or {}
@@ -758,6 +817,35 @@ class DataflowHook(GoogleBaseHook):
 
         return environment
 
+    def send_launch_template_request(
+        self,
+        *,
+        project_id: str,
+        location: str,
+        gcs_path: str,
+        job_name: str,
+        parameters: dict,
+        environment: dict,
+    ) -> dict[str, str]:
+        service: Resource = self.get_conn()
+        request = (
+            service.projects()
+            .locations()
+            .templates()
+            .launch(
+                projectId=project_id,
+                location=location,
+                gcsPath=gcs_path,
+                body={
+                    "jobName": job_name,
+                    "parameters": parameters,
+                    "environment": environment,
+                },
+            )
+        )
+        response: dict = request.execute(num_retries=self.num_retries)
+        return response["job"]
+
     @GoogleBaseHook.fallback_to_default_project_id
     def start_flex_template(
         self,
@@ -766,9 +854,9 @@ class DataflowHook(GoogleBaseHook):
         project_id: str,
         on_new_job_id_callback: Callable[[str], None] | None = None,
         on_new_job_callback: Callable[[dict], None] | None = None,
-    ) -> dict:
+    ) -> dict[str, str]:
         """
-        Start flex templates with the Dataflow pipeline.
+        Launch a Dataflow job with a Flex Template and wait for its completion.
 
         :param body: The request body. See:
             https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.locations.flexTemplates/launch#request-body
@@ -779,15 +867,16 @@ class DataflowHook(GoogleBaseHook):
         :param on_new_job_callback: A callback that is called when a Job is detected.
         :return: the Job
         """
-        service = self.get_conn()
+        service: Resource = self.get_conn()
         request = (
             service.projects()
             .locations()
             .flexTemplates()
             .launch(projectId=project_id, body=body, location=location)
         )
-        response = request.execute(num_retries=self.num_retries)
+        response: dict = request.execute(num_retries=self.num_retries)
         job = response["job"]
+        job_id: str = job["id"]
 
         if on_new_job_id_callback:
             warnings.warn(
@@ -795,7 +884,7 @@ class DataflowHook(GoogleBaseHook):
                 AirflowProviderDeprecationWarning,
                 stacklevel=3,
             )
-            on_new_job_id_callback(job.get("id"))
+            on_new_job_id_callback(job_id)
 
         if on_new_job_callback:
             on_new_job_callback(job)
@@ -803,7 +892,7 @@ class DataflowHook(GoogleBaseHook):
         jobs_controller = _DataflowJobsController(
             dataflow=self.get_conn(),
             project_number=project_id,
-            job_id=job.get("id"),
+            job_id=job_id,
             location=location,
             poll_sleep=self.poll_sleep,
             num_retries=self.num_retries,
@@ -814,6 +903,42 @@ class DataflowHook(GoogleBaseHook):
 
         return jobs_controller.get_jobs(refresh=True)[0]
 
+    @GoogleBaseHook.fallback_to_default_project_id
+    def launch_job_with_flex_template(
+        self,
+        body: dict,
+        location: str,
+        project_id: str,
+    ) -> dict[str, str]:
+        """
+        Launch a Dataflow Job with a Flex Template and exit without waiting for the job completion.
+
+        :param body: The request body. See:
+            https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.locations.flexTemplates/launch#request-body
+        :param location: The location of the Dataflow job (for example europe-west1)
+        :param project_id: The ID of the GCP project that owns the job.
+            If set to ``None`` or missing, the default project_id from the GCP connection is used.
+        :return: a Dataflow job response
+        """
+        service: Resource = self.get_conn()
+        request = (
+            service.projects()
+            .locations()
+            .flexTemplates()
+            .launch(projectId=project_id, body=body, location=location)
+        )
+        response: dict = request.execute(num_retries=self.num_retries)
+        return response["job"]
+
+    @staticmethod
+    def extract_job_id(job: dict) -> str:
+        try:
+            return job["id"]
+        except KeyError:
+            raise AirflowException(
+                "While reading job object after template execution error occurred. Job object has no id."
+            )
+
     @_fallback_to_location_from_variables
     @_fallback_to_project_id_from_variables
     @GoogleBaseHook.fallback_to_default_project_id
@@ -1353,3 +1478,92 @@ class AsyncDataflowHook(GoogleBaseAsyncHook):
         )
         page_result: ListJobsAsyncPager = await client.list_jobs(request=request)
         return page_result
+
+    async def list_job_messages(
+        self,
+        job_id: str,
+        project_id: str | None = PROVIDE_PROJECT_ID,
+        minimum_importance: int = JobMessageImportance.JOB_MESSAGE_BASIC,
+        page_size: int | None = None,
+        page_token: str | None = None,
+        start_time: Timestamp | None = None,
+        end_time: Timestamp | None = None,
+        location: str | None = DEFAULT_DATAFLOW_LOCATION,
+    ) -> ListJobMessagesAsyncPager:
+        """
+        Return ListJobMessagesAsyncPager object from MessagesV1Beta3AsyncClient.
+
+        This method wraps around a similar method of MessagesV1Beta3AsyncClient. ListJobMessagesAsyncPager can be iterated
+        over to extract messages associated with a specific Job ID.
+
+        For more details see the MessagesV1Beta3AsyncClient method description at:
+        https://cloud.google.com/python/docs/reference/dataflow/latest/google.cloud.dataflow_v1beta3.services.messages_v1_beta3.MessagesV1Beta3AsyncClient
+
+        :param job_id: ID of the Dataflow job to get messages about.
+        :param project_id: Optional. The Google Cloud project ID in which to start a job.
+            If set to None or missing, the default project_id from the Google Cloud connection is used.
+        :param minimum_importance: Optional. Filter to only get messages with importance >= level.
+            For more details see the description at:
+            https://cloud.google.com/python/docs/reference/dataflow/latest/google.cloud.dataflow_v1beta3.types.JobMessageImportance
+        :param page_size: Optional. If specified, determines the maximum number of messages to return.
+            If unspecified, the service may choose an appropriate default, or may return an arbitrarily large number of results.
+        :param page_token: Optional. If supplied, this should be the value of next_page_token returned by an earlier call.
+            This will cause the next page of results to be returned.
+        :param start_time: Optional. If specified, return only messages with timestamps >= start_time.
+            The default is the job creation time (i.e. beginning of messages).
+        :param end_time: Optional. If specified, return only messages with timestamps < end_time. The default is the current time.
+        :param location: Optional. The [regional endpoint] (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that contains
+            the job specified by job_id.
+        """
+        project_id = project_id or (await self.get_project_id())
+        client = await self.initialize_client(MessagesV1Beta3AsyncClient)
+        request = ListJobMessagesRequest(
+            {
+                "project_id": project_id,
+                "job_id": job_id,
+                "minimum_importance": minimum_importance,
+                "page_size": page_size,
+                "page_token": page_token,
+                "start_time": start_time,
+                "end_time": end_time,
+                "location": location,
+            }
+        )
+        page_results: ListJobMessagesAsyncPager = await client.list_job_messages(request=request)
+        return page_results
+
+    async def get_job_metrics(
+        self,
+        job_id: str,
+        project_id: str | None = PROVIDE_PROJECT_ID,
+        start_time: Timestamp | None = None,
+        location: str | None = DEFAULT_DATAFLOW_LOCATION,
+    ) -> JobMetrics:
+        """
+        Return JobMetrics object from MetricsV1Beta3AsyncClient.
+
+        This method wraps around a similar method of MetricsV1Beta3AsyncClient.
+
+        For more details see the MetricsV1Beta3AsyncClient method description at:
+        https://cloud.google.com/python/docs/reference/dataflow/latest/google.cloud.dataflow_v1beta3.services.metrics_v1_beta3.MetricsV1Beta3AsyncClient
+
+        :param job_id: ID of the Dataflow job to get metrics for.
+        :param project_id: Optional. The Google Cloud project ID in which to start a job.
+            If set to None or missing, the default project_id from the Google Cloud connection is used.
+        :param start_time: Optional. Return only metric data that has changed since this time.
+            Default is to return all information about all metrics for the job.
+        :param location: Optional. The [regional endpoint] (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that contains
+            the job specified by job_id.
+        """
+        project_id = project_id or (await self.get_project_id())
+        client: MetricsV1Beta3AsyncClient = await self.initialize_client(MetricsV1Beta3AsyncClient)
+        request = GetJobMetricsRequest(
+            {
+                "project_id": project_id,
+                "job_id": job_id,
+                "start_time": start_time,
+                "location": location,
+            }
+        )
+        job_metrics: JobMetrics = await client.get_job_metrics(request=request)
+        return job_metrics