apache-airflow-providers-google 8.12.0rc1__py3-none-any.whl → 9.0.0rc2__py3-none-any.whl

airflow/providers/google_vendor/googleads/interceptors/interceptor.py

@@ -0,0 +1,252 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A mixin class to store shared functionality for all the gRPC Interceptors.
+
+This mixin class centralizes sets of functionality that are common across all
+Interceptors, including retrieving data from gRPC metadata and initializing
+instances of grpc.ClientCallDetails.
+"""
+
+from dataclasses import dataclass
+from importlib import import_module
+from typing import AnyStr, Optional, Sequence, Tuple
+import json
+
+from google.protobuf.message import DecodeError
+from grpc import ClientCallDetails, StatusCode, CallCredentials
+
+from airflow.providers.google_vendor.googleads.errors import GoogleAdsException
+
+_REQUEST_ID_KEY = "request-id"
+# Codes that are retried upon by google.api_core.
+_RETRY_STATUS_CODES = (StatusCode.INTERNAL, StatusCode.RESOURCE_EXHAUSTED)
+
+
+class Interceptor:
+    _SENSITIVE_INFO_MASK = "REDACTED"
+
+    @dataclass
+    class _ClientCallDetails(ClientCallDetails):
+        """Wrapper class for initializing a new ClientCallDetails instance."""
+
+        method: str
+        timeout: Optional[float]
+        metadata: Optional[Sequence[Tuple[str, AnyStr]]]
+        credentials: Optional[CallCredentials]
+
+    @classmethod
+    def get_request_id_from_metadata(cls, trailing_metadata):
+        """Gets the request ID for the Google Ads API request.
+
+        Args:
+            trailing_metadata: a tuple of metadatum from the service response.
+
+        Returns:
+            A str request ID associated with the Google Ads API request, or None
+            if it doesn't exist.
+        """
+        for kv in trailing_metadata:
+            if kv[0] == _REQUEST_ID_KEY:
+                return kv[1]  # Return the found request ID.
+
+        return None
+
+    @classmethod
+    def parse_metadata_to_json(cls, metadata):
+        """Parses metadata from gRPC request and response messages to a JSON str.
+
+        Obscures the value for "developer-token".
+
+        Args:
+            metadata: a tuple of metadatum.
+
+        Returns:
+            A str of metadata formatted as JSON key/value pairs.
+        """
+        metadata_dict = {}
+
+        if metadata is None:
+            return "{}"
+
+        for datum in metadata:
+            key = datum[0]
+            if key == "developer-token":
+                metadata_dict[key] = cls._SENSITIVE_INFO_MASK
+            else:
+                value = datum[1]
+                metadata_dict[key] = value
+
+        return cls.format_json_object(metadata_dict)
+
+    @classmethod
+    def format_json_object(cls, obj):
+        """Parses a serializable object into a consistently formatted JSON string.
+
+        Returns:
+            A str of formatted JSON serialized from the given object.
+
+        Args:
+            obj: an object or dict.
+
+        Returns:
+            A str of metadata formatted as JSON key/value pairs.
+        """
+
+        def default_serializer(value):
+            if isinstance(value, bytes):
+                return value.decode(errors="ignore")
+            else:
+                return None
+
+        return str(
+            json.dumps(
+                obj,
+                indent=2,
+                sort_keys=True,
+                ensure_ascii=False,
+                default=default_serializer,
+                separators=(",", ": "),
+            )
+        )
+
+    @classmethod
+    def get_trailing_metadata_from_interceptor_exception(cls, exception):
+        """Retrieves trailing metadata from an exception object.
+
+        Args:
+            exception: an instance of grpc.Call.
+
+        Returns:
+            A tuple of trailing metadata key value pairs.
+        """
+        try:
+            # GoogleAdsFailure exceptions will contain trailing metadata on the
+            # error attribute.
+            return exception.error.trailing_metadata()
+        except AttributeError:
+            try:
+                # Transport failures, i.e. issues at the gRPC layer, will contain
+                # trailing metadata on the exception itself.
+                return exception.trailing_metadata()
+            except AttributeError:
+                # if trailing metadata is not found in either location then
+                # return an empty tuple
+                return tuple()
+
+    @classmethod
+    def get_client_call_details_instance(
+        cls, method, timeout, metadata, credentials=None
+    ):
+        """Initializes an instance of the ClientCallDetails with the given data.
+
+        Args:
+            method: A str of the service method being invoked.
+            timeout: A float of the request timeout
+            metadata: A list of metadata tuples
+            credentials: An optional grpc.CallCredentials instance for the RPC
+
+        Returns:
+            An instance of _ClientCallDetails that wraps grpc.ClientCallDetails.
+        """
+        return cls._ClientCallDetails(method, timeout, metadata, credentials)
+
+    def __init__(self, api_version):
+        self._error_protos = None
+        self._failure_key = (
+            f"google.ads.googleads.{api_version}.errors.googleadsfailure-bin"
+        )
+        self._api_version = api_version
+
+    def _get_error_from_response(self, response):
+        """Attempts to wrap failed responses as GoogleAdsException instances.
+
+        Handles failed gRPC responses of by attempting to convert them
+        to a more readable GoogleAdsException. Certain types of exceptions are
+        not converted; if the object's trailing metadata does not indicate that
+        it is a GoogleAdsException, or if it falls under a certain category of
+        status code, (INTERNAL or RESOURCE_EXHAUSTED). See documentation for
+        more information about gRPC status codes:
+        https://github.com/grpc/grpc/blob/master/doc/statuscodes.md
+
+        Args:
+            response: a grpc.Call/grpc.Future instance.
+
+        Returns:
+            GoogleAdsException: If the exception's trailing metadata
+                indicates that it is a GoogleAdsException.
+            RpcError: If the exception's is a gRPC exception but the trailing
+                metadata is empty or is not indicative of a GoogleAdsException,
+                or if the exception has a status code of INTERNAL or
+                RESOURCE_EXHAUSTED.
+            Exception: If not a GoogleAdsException or RpcException the error
+                will be raised as-is.
+        """
+        status_code = response.code()
+        response_exception = response.exception()
+
+        if status_code not in _RETRY_STATUS_CODES:
+            trailing_metadata = response.trailing_metadata()
+            google_ads_failure = self._get_google_ads_failure(trailing_metadata)
+
+            if google_ads_failure:
+                request_id = self.get_request_id_from_metadata(
+                    trailing_metadata
+                )
+
+                # If exception is a GoogleAdsFailure then it gets wrapped in a
+                # library-specific Error type for easy handling. These errors
+                # originate from the Google Ads API and are often caused by
+                # invalid requests.
+                return GoogleAdsException(
+                    response_exception, response, google_ads_failure, request_id
+                )
+            else:
+                # Raise the original exception if not a GoogleAdsFailure. This
+                # type of error is generally caused by problems at the request
+                # level, such as when an invalid endpoint is given.
+                return response_exception
+        else:
+            # Raise the original exception if error has status code
+            # INTERNAL or RESOURCE_EXHAUSTED, meaning that
+            return response_exception
+
+    def _get_google_ads_failure(self, trailing_metadata):
+        """Gets the Google Ads failure details if they exist.
+
+        Args:
+            trailing_metadata: a tuple of metadatum from the service response.
+
+        Returns:
+            A GoogleAdsFailure that describes how a GoogleAds API call failed.
+            Returns None if either the trailing metadata of the request did not
+            return the failure details, or if the GoogleAdsFailure fails to
+            parse.
+        """
+        if trailing_metadata is not None:
+            for kv in trailing_metadata:
+                if kv[0] == self._failure_key:
+                    try:
+                        if not self._error_protos:
+                            self._error_protos = import_module(
+                                f"airflow.providers.google_vendor.googleads.{self._api_version}.errors."
+                                "types.errors"
+                            )
+
+                        return self._error_protos.GoogleAdsFailure.deserialize(
+                            kv[1]
+                        )
+                    except DecodeError:
+                        return None
+
+        return None

airflow/providers/google_vendor/googleads/interceptors/logging_interceptor.py

@@ -0,0 +1,391 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A gRPC Interceptor that is responsible for logging requests and responses.
+
+This class is initialized in the GoogleAdsClient and passed into a grpc
+intercept_channel whenever a new service is initialized. It intercepts requests
+and responses, parses them into a human readable structure and logs them using
+the passed in logger instance.
+"""
+
+from copy import deepcopy
+from types import SimpleNamespace
+import json
+import logging
+
+from grpc import UnaryUnaryClientInterceptor, UnaryStreamClientInterceptor
+
+from airflow.providers.google_vendor.googleads.interceptors import Interceptor, mask_message
+from types import SimpleNamespace
+
+
+class LoggingInterceptor(
+    Interceptor, UnaryUnaryClientInterceptor, UnaryStreamClientInterceptor
+):
+    """An interceptor that logs rpc requests and responses."""
+
+    _FULL_REQUEST_LOG_LINE = (
+        "Request\n-------\nMethod: {}\nHost: {}\n"
+        "Headers: {}\nRequest: {}\n\nResponse\n-------\n"
+        "Headers: {}\nResponse: {}\n"
+    )
+    _FULL_FAULT_LOG_LINE = (
+        "Request\n-------\nMethod: {}\nHost: {}\n"
+        "Headers: {}\nRequest: {}\n\nResponse\n-------\n"
+        "Headers: {}\nFault: {}\n"
+    )
+    _SUMMARY_LOG_LINE = (
+        "Request made: ClientCustomerId: {}, Host: {}, "
+        "Method: {}, RequestId: {}, IsFault: {}, "
+        "FaultMessage: {}"
+    )
+
+    def __init__(self, logger, api_version, endpoint=None):
+        """Initializer for the LoggingInterceptor.
+
+        Args:
+            logger: An instance of logging.Logger.
+            api_version: a str of the API version of the request.
+            endpoint: a str specifying the endpoint for requests.
+        """
+        super().__init__(api_version)
+        self.endpoint = endpoint
+        self.logger = logger
+        self._cache = None
+
+    def _get_trailing_metadata(self, response):
+        """Retrieves trailing metadata from a response object.
+
+        If the exception is a GoogleAdsException the trailing metadata will be
+        on its error object, otherwise it will be on the response object.
+
+        Returns:
+            A tuple of metadatum representing response header key value pairs.
+
+        Args:
+            response: A grpc.Call/grpc.Future instance.
+        """
+        try:
+            trailing_metadata = response.trailing_metadata()
+
+            if not trailing_metadata:
+                return self.get_trailing_metadata_from_interceptor_exception(
+                    response.exception()
+                )
+
+            return trailing_metadata
+        except AttributeError:
+            return self.get_trailing_metadata_from_interceptor_exception(
+                response.exception()
+            )
+
+    def _get_initial_metadata(self, client_call_details):
+        """Retrieves the initial metadata from client_call_details.
+
+        Returns an empty tuple if metadata isn't present on the
+        client_call_details object.
+
+        Returns:
+            A tuple of metadatum representing request header key value pairs.
+
+        Args:
+            client_call_details: An instance of grpc.ClientCallDetails.
+        """
+        return getattr(client_call_details, "metadata", tuple())
+
+    def _get_call_method(self, client_call_details):
+        """Retrieves the call method from client_call_details.
+
+        Returns None if the method is not present on the client_call_details
+        object.
+
+        Returns:
+            A str with the call method or None if it isn't present.
+
+        Args:
+            client_call_details: An instance of grpc.ClientCallDetails.
+        """
+        return getattr(client_call_details, "method", None)
+
+    def _get_customer_id(self, request):
+        """Retrieves the customer_id from the grpc request.
+
+        Returns None if a customer_id is not present on the request object.
+
+        Returns:
+            A str with the customer id from the request or None if it isn't
+            present.
+
+        Args:
+            request: An instance of a request proto message.
+        """
+        if hasattr(request, "customer_id"):
+            return getattr(request, "customer_id")
+        elif hasattr(request, "resource_name"):
+            resource_name = getattr(request, "resource_name")
+            segments = resource_name.split("/")
+            if segments[0] == "customers":
+                return segments[1]
+        else:
+            return None
+
+    def _parse_exception_to_str(self, exception):
+        """Parses response exception object to str for logging.
+
+        Returns:
+            A str representing a exception from the API.
+
+        Args:
+            exception: A grpc.Call instance.
+        """
+        try:
+            # If the exception is from the Google Ads API then the failure
+            # attribute will be an instance of GoogleAdsFailure. We convert it
+            # to str here so the return type is consistent, rather than casting
+            # by concatenating in the logs.
+            return str(exception.failure)
+        except AttributeError:
+            try:
+                # if exception.failure isn't present then it's likely this is a
+                # transport error with a .debug_error_string method and the
+                # returned JSON string will need to be formatted.
+                return self.format_json_object(
+                    json.loads(exception.debug_error_string())
+                )
+            except (AttributeError, ValueError):
+                # if both attempts to retrieve serializable error data fail
+                # then simply return an empty JSON string
+                return "{}"
+
+    def _get_fault_message(self, exception):
+        """Retrieves a fault/error message from an exception object.
+
+        Returns None if no error message can be found on the exception.
+
+        Returns:
+            A str with an error message or None if one cannot be found.
+
+        Args:
+            response: A grpc.Call/grpc.Future instance.
+            exception: A grpc.Call instance.
+        """
+        try:
+            return exception.failure.errors[0].message
+        except AttributeError:
+            try:
+                return exception.details()
+            except AttributeError:
+                return None
+
+    def log_successful_request(
+        self,
+        method,
+        customer_id,
+        metadata_json,
+        request_id,
+        request,
+        trailing_metadata_json,
+        response,
+    ):
+        """Handles logging of a successful request.
+
+        Args:
+            method: The method of the request.
+            customer_id: The customer ID associated with the request.
+            metadata_json: A JSON str of initial_metadata.
+            request_id: A unique ID for the request provided in the response.
+            request: An instance of a request proto message.
+            trailing_metadata_json: A JSON str of trailing_metadata.
+            response: A grpc.Call/grpc.Future instance.
+        """
+        result = self.retrieve_and_mask_result(response)
+
+        # Check log level here to avoid calling .format() unless necessary.
+        if self.logger.isEnabledFor(logging.DEBUG):
+            self.logger.debug(
+                self._FULL_REQUEST_LOG_LINE.format(
+                    method,
+                    self.endpoint,
+                    metadata_json,
+                    request,
+                    trailing_metadata_json,
+                    result,
+                )
+            )
+
+        if self.logger.isEnabledFor(logging.INFO):
+            self.logger.info(
+                self._SUMMARY_LOG_LINE.format(
+                    customer_id, self.endpoint, method, request_id, False, None
+                )
+            )
+
+    def log_failed_request(
+        self,
+        method,
+        customer_id,
+        metadata_json,
+        request_id,
+        request,
+        trailing_metadata_json,
+        response,
+    ):
+        """Handles logging of a failed request.
+
+        Args:
+            method: The method of the request.
+            customer_id: The customer ID associated with the request.
+            metadata_json: A JSON str of initial_metadata.
+            request_id: A unique ID for the request provided in the response.
+            request: An instance of a request proto message.
+            trailing_metadata_json: A JSON str of trailing_metadata.
+            response: A JSON str of the response message.
+        """
+        exception = self._get_error_from_response(response)
+        exception_str = self._parse_exception_to_str(exception)
+        fault_message = self._get_fault_message(exception)
+
+        if self.logger.isEnabledFor(logging.INFO):
+            self.logger.info(
+                self._FULL_FAULT_LOG_LINE.format(
+                    method,
+                    self.endpoint,
+                    metadata_json,
+                    request,
+                    trailing_metadata_json,
+                    exception_str,
+                )
+            )
+
+        self.logger.warning(
+            self._SUMMARY_LOG_LINE.format(
+                customer_id,
+                self.endpoint,
+                method,
+                request_id,
+                True,
+                fault_message,
+            )
+        )
+
+    def log_request(self, client_call_details, request, response):
+        """Handles logging all requests.
+
+        Args:
+            client_call_details: An instance of grpc.ClientCallDetails.
+            request: An instance of a request proto message.
+            response: A grpc.Call/grpc.Future instance.
+        """
+        method = self._get_call_method(client_call_details)
+        customer_id = self._get_customer_id(request)
+        initial_metadata = self._get_initial_metadata(client_call_details)
+        initial_metadata_json = self.parse_metadata_to_json(initial_metadata)
+        trailing_metadata = self._get_trailing_metadata(response)
+        request_id = self.get_request_id_from_metadata(trailing_metadata)
+        trailing_metadata_json = self.parse_metadata_to_json(trailing_metadata)
+        request = mask_message(request, self._SENSITIVE_INFO_MASK)
+
+        if response.exception():
+            self.log_failed_request(
+                method,
+                customer_id,
+                initial_metadata_json,
+                request_id,
+                request,
+                trailing_metadata_json,
+                response,
+            )
+        else:
+            self.log_successful_request(
+                method,
+                customer_id,
+                initial_metadata_json,
+                request_id,
+                request,
+                trailing_metadata_json,
+                response,
+            )
+
+    def intercept_unary_unary(self, continuation, client_call_details, request):
+        """Intercepts and logs API interactions.
+
+        Overrides abstract method defined in grpc.UnaryUnaryClientInterceptor.
+
+        Args:
+            continuation: a function to continue the request process.
+            client_call_details: a grpc._interceptor._ClientCallDetails
+                instance containing request metadata.
+            request: a SearchGoogleAdsRequest or SearchGoogleAdsStreamRequest
+                message class instance.
+
+        Returns:
+            A grpc.Call/grpc.Future instance representing a service response.
+        """
+        response = continuation(client_call_details, request)
+
+        if self.logger.isEnabledFor(logging.WARNING):
+            self.log_request(client_call_details, request, response)
+
+        return response
+
+    def intercept_unary_stream(
+        self, continuation, client_call_details, request
+    ):
+        """Intercepts and logs API interactions for Unary-Stream requests.
+
+        Overrides abstract method defined in grpc.UnaryStreamClientInterceptor.
+
+        Args:
+            continuation: a function to continue the request process.
+            client_call_details: a grpc._interceptor._ClientCallDetails
+                instance containing request metadata.
+            request: a SearchGoogleAdsRequest or SearchGoogleAdsStreamRequest
+                message class instance.
+
+        Returns:
+            A grpc.Call/grpc.Future instance representing a service response.
+        """
+
+        def on_rpc_complete(response_future):
+            if self.logger.isEnabledFor(logging.WARNING):
+                self.log_request(client_call_details, request, response_future)
+
+        response = continuation(client_call_details, request)
+
+        response.add_done_callback(on_rpc_complete)
+        self._cache = response.get_cache()
+        return response
+
+    def retrieve_and_mask_result(self, response):
+        """If the cache is populated, mask the cached stream response object.
+        Otherwise, mask the non-streaming response result.
+
+        Args:
+            response: A grpc.Call/grpc.Future instance.
+
+        Returns:
+            A masked response message.
+        """
+        if self._cache and self._cache.initial_response_object is not None:
+            # Get the first streaming response object from the cache.
+            result = self._cache.initial_response_object
+        else:
+            result = response.result()
+        # Mask the response object if debug level logging is enabled.
+        if type(self.logger) == logging.Logger and self.logger.isEnabledFor(
+            logging.DEBUG
+        ):
+            result = mask_message(result, self._SENSITIVE_INFO_MASK)
+
+        return result