ibm-cloud-sdk-core 3.16.0__py3-none-any.whl → 3.21.0__py3-none-any.whl

ibm_cloud_sdk_core/authenticators/vpc_instance_authenticator.py

@@ -1,6 +1,6 @@
 # coding: utf-8
 
-# Copyright 2021 IBM All Rights Reserved.
+# Copyright 2021, 2024 IBM All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,9 +18,12 @@ from typing import Optional
 
 from requests import Request
 
+from ibm_cloud_sdk_core.logger import get_logger
 from ..token_managers.vpc_instance_token_manager import VPCInstanceTokenManager
 from .authenticator import Authenticator
 
+logger = get_logger()
+
 
 class VPCInstanceAuthenticator(Authenticator):
     """VPCInstanceAuthenticator implements an authentication scheme in which it
@@ -53,16 +56,15 @@ class VPCInstanceAuthenticator(Authenticator):
 
     DEFAULT_IMS_ENDPOINT = 'http://169.254.169.254'
 
-    def __init__(self,
-                 iam_profile_crn: Optional[str] = None,
-                 iam_profile_id: Optional[str] = None,
-                 url: Optional[str] = None) -> None:
-
+    def __init__(
+        self, iam_profile_crn: Optional[str] = None, iam_profile_id: Optional[str] = None, url: Optional[str] = None
+    ) -> None:
         if not url:
             url = self.DEFAULT_IMS_ENDPOINT
 
         self.token_manager = VPCInstanceTokenManager(
-            url=url, iam_profile_crn=iam_profile_crn, iam_profile_id=iam_profile_id)
+            url=url, iam_profile_crn=iam_profile_crn, iam_profile_id=iam_profile_id
+        )
 
         self.validate()
 
@@ -74,8 +76,7 @@ class VPCInstanceAuthenticator(Authenticator):
         super().validate()
 
         if self.token_manager.iam_profile_crn and self.token_manager.iam_profile_id:
-            raise ValueError(
-                'At most one of "iam_profile_id" or "iam_profile_crn" may be specified.')
+            raise ValueError('At most one of "iam_profile_id" or "iam_profile_crn" may be specified.')
 
     def authenticate(self, req: Request) -> None:
         """Adds IAM authentication information to the request.
@@ -85,13 +86,13 @@ class VPCInstanceAuthenticator(Authenticator):
             Authorization: Bearer <bearer-token>
 
         Args:
-            req: The request to add IAM authentication information too. Must contain a key to a dictionary
+            req: The request to add IAM authentication information to. Must contain a key to a dictionary
             called headers.
         """
         headers = req.get('headers')
         bearer_token = self.token_manager.get_token()
         headers['Authorization'] = 'Bearer {0}'.format(bearer_token)
-
+        logger.debug('Authenticated outbound request (type=%s)', self.authentication_type())
 
     def set_iam_profile_crn(self, iam_profile_crn: str) -> None:
         """Sets CRN of the IAM profile.

ibm_cloud_sdk_core/base_service.py

@@ -1,6 +1,6 @@
 # coding: utf-8
 
-# Copyright 2019 IBM All Rights Reserved.
+# Copyright 2019, 2024 IBM All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,36 +15,44 @@
 # limitations under the License.
 
 import gzip
-import json as json_import
+import io
 import logging
-import platform
+import json as json_import
 from http.cookiejar import CookieJar
+from http import client
 from os.path import basename
 from typing import Dict, List, Optional, Tuple, Union
 from urllib3.util.retry import Retry
 
 import requests
 from requests.structures import CaseInsensitiveDict
+from requests.exceptions import JSONDecodeError
 
 from ibm_cloud_sdk_core.authenticators import Authenticator
 from .api_exception import ApiException
 from .detailed_response import DetailedResponse
+from .http_adapter import SSLHTTPAdapter
 from .token_managers.token_manager import TokenManager
-from .utils import (has_bad_first_or_last_char, remove_null_values,
-                    cleanup_values, read_external_sources, strip_extra_slashes,
-                    SSLHTTPAdapter)
-from .version import __version__
-
-# Uncomment this to enable http debugging
-# import http.client as http_client
-# http_client.HTTPConnection.debuglevel = 1
-
-
-logger = logging.getLogger(__name__)
-
-
-#pylint: disable=too-many-instance-attributes
-#pylint: disable=too-many-locals
+from .utils import (
+    has_bad_first_or_last_char,
+    is_json_mimetype,
+    remove_null_values,
+    cleanup_values,
+    read_external_sources,
+    strip_extra_slashes,
+    GzipStream,
+)
+from .private_helpers import _build_user_agent
+from .logger import (
+    get_logger,
+    LoggingFilter,
+)
+
+logger = get_logger()
+
+
+# pylint: disable=too-many-instance-attributes
+# pylint: disable=too-many-locals
 class BaseService:
     """Common functionality shared by generated service classes.
 
@@ -72,18 +80,22 @@ class BaseService:
     Raises:
         ValueError: If Authenticator is not provided or invalid type.
     """
-    SDK_NAME = 'ibm-python-sdk-core'
-    ERROR_MSG_DISABLE_SSL = 'The connection failed because the SSL certificate is not valid. To use a self-signed '\
-                            'certificate, disable verification of the server\'s SSL certificate by invoking the '\
-                            'set_disable_ssl_verification(True) on your service instance and/ or use the '\
-                            'disable_ssl_verification option of the authenticator.'
-
-    def __init__(self,
-                 *,
-                 service_url: str = None,
-                 authenticator: Authenticator = None,
-                 disable_ssl_verification: bool = False,
-                 enable_gzip_compression: bool = False) -> None:
+
+    ERROR_MSG_DISABLE_SSL = (
+        'The connection failed because the SSL certificate is not valid. To use a self-signed '
+        'certificate, disable verification of the server\'s SSL certificate by invoking the '
+        'set_disable_ssl_verification(True) on your service instance and/ or use the '
+        'disable_ssl_verification option of the authenticator.'
+    )
+
+    def __init__(
+        self,
+        *,
+        service_url: str = None,
+        authenticator: Optional[Authenticator] = None,
+        disable_ssl_verification: bool = False,
+        enable_gzip_compression: bool = False,
+    ) -> None:
         self.set_service_url(service_url)
         self.http_client = requests.Session()
         self.http_config = {}
@@ -92,9 +104,9 @@ class BaseService:
         self.disable_ssl_verification = disable_ssl_verification
         self.default_headers = None
         self.enable_gzip_compression = enable_gzip_compression
-        self._set_user_agent_header(self._build_user_agent())
+        self._set_user_agent_header(_build_user_agent())
         self.retry_config = None
-        self.http_adapter = SSLHTTPAdapter()
+        self.http_adapter = SSLHTTPAdapter(_disable_ssl_verification=self.disable_ssl_verification)
         if not self.authenticator:
             raise ValueError('authenticator must be provided')
         if not isinstance(self.authenticator, Authenticator):
@@ -102,50 +114,48 @@ class BaseService:
 
         self.http_client.mount('http://', self.http_adapter)
         self.http_client.mount('https://', self.http_adapter)
-
-    def enable_retries(self, max_retries: int = 4, retry_interval: float = 1.0) -> None:
+        # If debug logging is requested, then trigger HTTP message logging as well.
+        if logger.isEnabledFor(logging.DEBUG):
+            client.HTTPConnection.debuglevel = 1
+            # Replace the `print` function in the HTTPClient module to
+            # use the debug logger instead of the bare Python print.
+            client.print = lambda *args: logger.debug(LoggingFilter.filter_message(" ".join(args)))
+
+    def enable_retries(self, max_retries: int = 4, retry_interval: float = 30.0) -> None:
         """Enable automatic retries on the underlying http client used by the BaseService instance.
 
         Args:
           max_retries: the maximum number of retries to attempt for a failed retryable request
-          retry_interval: the default wait time (in seconds) to use for the first retry attempt.
+          retry_interval: the maximum wait time (in seconds) to use for retry attempts.
             In general, if a response includes the Retry-After header, that will be used for
             the wait time associated with the retry attempt.  If the Retry-After header is not
-            present, then the wait time is based on the retry_interval and retry attempt number:
-               wait_time = retry_interval * (2 ^ (n-1)), where n is the retry attempt number
+            present, then the wait time is based on an exponential backoff policy with a maximum
+            backoff time of "retry_interval".
         """
         self.retry_config = Retry(
             total=max_retries,
-            backoff_factor=retry_interval,
+            backoff_factor=1.0,
+            backoff_max=retry_interval,
             # List of HTTP status codes to retry on in addition to Timeout/Connection Errors
             status_forcelist=[429, 500, 502, 503, 504],
             # List of HTTP methods to retry on
             # Omitting this will default to all methods except POST
-            allowed_methods=['HEAD', 'GET', 'PUT',
-                             'DELETE', 'OPTIONS', 'TRACE', 'POST']
+            allowed_methods=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'],
+        )
+        self.http_adapter = SSLHTTPAdapter(
+            max_retries=self.retry_config, _disable_ssl_verification=self.disable_ssl_verification
         )
-        self.http_adapter = SSLHTTPAdapter(max_retries=self.retry_config)
         self.http_client.mount('http://', self.http_adapter)
         self.http_client.mount('https://', self.http_adapter)
+        logger.debug('Enabled retries; max_retries=%d, max_retry_interval=%f', max_retries, retry_interval)
 
     def disable_retries(self):
         """Remove retry config from http_adapter"""
         self.retry_config = None
-        self.http_adapter = SSLHTTPAdapter()
+        self.http_adapter = SSLHTTPAdapter(_disable_ssl_verification=self.disable_ssl_verification)
         self.http_client.mount('http://', self.http_adapter)
         self.http_client.mount('https://', self.http_adapter)
-
-    @staticmethod
-    def _get_system_info() -> str:
-        return '{0} {1} {2}'.format(
-            platform.system(),  # OS
-            platform.release(),  # OS version
-            platform.python_version()  # Python version
-        )
-
-    def _build_user_agent(self) -> str:
-        return '{0}-{1} {2}'.format(self.SDK_NAME, __version__,
-                                    self._get_system_info())
+        logger.debug('Disabled retries')
 
     def configure_service(self, service_name: str) -> None:
         """Look for external configuration of a service. Set service properties.
@@ -164,27 +174,27 @@ class BaseService:
         if not isinstance(service_name, str):
             raise ValueError('Service_name must be of type string.')
 
+        logger.debug('Configuring BaseService instance with service name: %s', service_name)
+
         config = read_external_sources(service_name)
         if config.get('URL'):
             self.set_service_url(config.get('URL'))
         if config.get('DISABLE_SSL'):
-            self.set_disable_ssl_verification(
-                config.get('DISABLE_SSL').lower() == 'true')
+            self.set_disable_ssl_verification(config.get('DISABLE_SSL').lower() == 'true')
         if config.get('ENABLE_GZIP'):
-            self.set_enable_gzip_compression(
-                config.get('ENABLE_GZIP').lower() == 'true')
+            self.set_enable_gzip_compression(config.get('ENABLE_GZIP').lower() == 'true')
         if config.get('ENABLE_RETRIES'):
             if config.get('ENABLE_RETRIES').lower() == 'true':
                 kwargs = {}
                 if config.get('MAX_RETRIES'):
                     kwargs["max_retries"] = int(config.get('MAX_RETRIES'))
                 if config.get('RETRY_INTERVAL'):
-                    kwargs["retry_interval"] = float(
-                        config.get('RETRY_INTERVAL'))
+                    kwargs["retry_interval"] = float(config.get('RETRY_INTERVAL'))
                 self.enable_retries(**kwargs)
 
     def _set_user_agent_header(self, user_agent_string: str) -> None:
         self.user_agent_header = {'User-Agent': user_agent_string}
+        logger.debug('Set User-Agent: %s', user_agent_string)
 
     def set_http_config(self, http_config: dict) -> None:
         """Sets the http config dictionary.
@@ -199,10 +209,11 @@ class BaseService:
         """
         if isinstance(http_config, dict):
             self.http_config = http_config
-            if (self.authenticator
-                    and hasattr(self.authenticator, 'token_manager')
-                    and isinstance(self.authenticator.token_manager,
-                                   TokenManager)):
+            if (
+                self.authenticator
+                and hasattr(self.authenticator, 'token_manager')
+                and isinstance(self.authenticator.token_manager, TokenManager)
+            ):
                 self.authenticator.token_manager.http_config = http_config
         else:
             raise TypeError("http_config parameter must be a dictionary")
@@ -214,8 +225,19 @@ class BaseService:
         Keyword Arguments:
             status: set to true to disable ssl verification (default: {False})
         """
+        if self.disable_ssl_verification == status:
+            # Do nothing if the state doesn't change.
+            return
+
         self.disable_ssl_verification = status
 
+        self.http_adapter = SSLHTTPAdapter(
+            max_retries=self.retry_config, _disable_ssl_verification=self.disable_ssl_verification
+        )
+        self.http_client.mount('http://', self.http_adapter)
+        self.http_client.mount('https://', self.http_adapter)
+        logger.debug('Disabled SSL verification in HTTP client')
+
     def set_service_url(self, service_url: str) -> None:
         """Set the url the service will make HTTP requests too.
 
@@ -233,6 +255,7 @@ class BaseService:
         if service_url is not None:
             service_url = service_url.rstrip('/')
         self.service_url = service_url
+        logger.debug('Set service URL: %s', service_url)
 
     def get_http_client(self) -> requests.sessions.Session:
         """Get the http client session currently used by the service.
@@ -251,8 +274,7 @@ class BaseService:
         if isinstance(http_client, requests.sessions.Session):
             self.http_client = http_client
         else:
-            raise TypeError(
-                "http_client parameter must be a requests.sessions.Session")
+            raise TypeError("http_client parameter must be a requests.sessions.Session")
 
     def get_authenticator(self) -> Authenticator:
         """Get the authenticator currently used by the service.
@@ -296,7 +318,7 @@ class BaseService:
         # Check to see if the caller specified the 'stream' argument.
         stream_response = kwargs.get('stream') or False
 
-        # Remove the keys we set manually, don't let the user to overwrite these.
+        # Remove the keys we set manually, don't let the user overwrite these.
         reserved_keys = ['method', 'url', 'headers', 'params', 'cookies']
         silent_keys = ['headers']
         for key in reserved_keys:
@@ -305,35 +327,44 @@ class BaseService:
                 if key not in silent_keys:
                     logger.warning('"%s" has been removed from the request', key)
         try:
-            response = self.http_client.request(**request,
-                                                cookies=self.jar,
-                                                **kwargs)
+            logger.debug('Sending HTTP request message')
 
+            response = self.http_client.request(**request, cookies=self.jar, **kwargs)
+
+            logger.debug('Received HTTP response message, status code %d', response.status_code)
+
+            # Process a "success" response.
             if 200 <= response.status_code <= 299:
                 if response.status_code == 204 or request['method'] == 'HEAD':
-                    # There is no body content for a HEAD request or a 204 response
+                    # There is no body content for a HEAD response or a 204 response.
                     result = None
                 elif stream_response:
                     result = response
                 elif not response.text:
                     result = None
-                else:
+                elif is_json_mimetype(response.headers.get('Content-Type')):
+                    # If this is a JSON response, then try to unmarshal it.
                     try:
-                        result = response.json()
-                    except:
-                        result = response
-                return DetailedResponse(response=result,
-                                        headers=response.headers,
-                                        status_code=response.status_code)
+                        result = response.json(strict=False)
+                    except JSONDecodeError as err:
+                        raise ApiException(
+                            code=response.status_code,
+                            http_response=response,
+                            message='Error processing the HTTP response',
+                        ) from err
+                else:
+                    # Non-JSON response, just use response body as-is.
+                    result = response
+
+                return DetailedResponse(response=result, headers=response.headers, status_code=response.status_code)
 
+            # Received error status code from server, raise an APIException.
             raise ApiException(response.status_code, http_response=response)
         except requests.exceptions.SSLError:
             logger.exception(self.ERROR_MSG_DISABLE_SSL)
             raise
 
-    def set_enable_gzip_compression(self,
-                                    should_enable_compression: bool = False
-                                    ) -> None:
+    def set_enable_gzip_compression(self, should_enable_compression: bool = False) -> None:
         """Set value to enable gzip compression on request bodies"""
         self.enable_gzip_compression = should_enable_compression
 
@@ -341,18 +372,17 @@ class BaseService:
         """Get value for enabling gzip compression on request bodies"""
         return self.enable_gzip_compression
 
-    def prepare_request(self,
-                        method: str,
-                        url: str,
-                        *,
-                        headers: Optional[dict] = None,
-                        params: Optional[dict] = None,
-                        data: Optional[Union[str, dict]] = None,
-                        files: Optional[Union[Dict[str, Tuple[str]],
-                                              List[Tuple[str,
-                                                         Tuple[str,
-                                                               ...]]]]] = None,
-                        **kwargs) -> dict:
+    def prepare_request(
+        self,
+        method: str,
+        url: str,
+        *,
+        headers: Optional[dict] = None,
+        params: Optional[dict] = None,
+        data: Optional[Union[str, dict]] = None,
+        files: Optional[Union[Dict[str, Tuple[str]], List[Tuple[str, Tuple[str, ...]]]]] = None,
+        **kwargs,
+    ) -> dict:
         """Build a dict that represents an HTTP service request.
 
         Clean up headers, add default http configuration, convert data
@@ -363,8 +393,10 @@ class BaseService:
             url: The origin + pathname according to WHATWG spec.
 
         Keyword Arguments:
-            headers: Headers of the request.
-            params: Querystring data to be appended to the url.
+            headers: A dictionary containing the headers to be included in the request.
+                    Entries with a value of None will be ignored (excluded).
+            params: A dictionary containing the query parameters to be included in the request.
+                    Entries with a value of None will be ignored (excluded).
             data: The request body. Converted to json if a dict.
             files: 'files' can be a dictionary (i.e { '<part-name>': (<tuple>)}),
                 or a list of tuples [ (<part-name>, (<tuple>))... ]
@@ -409,14 +441,16 @@ class BaseService:
         self.authenticator.authenticate(request)
 
         # Compress the request body if applicable
-        if (self.get_enable_gzip_compression()
-                and 'content-encoding' not in headers
-                and request['data'] is not None):
+        if self.get_enable_gzip_compression() and 'content-encoding' not in headers and request['data'] is not None:
             headers['content-encoding'] = 'gzip'
-            uncompressed_data = request['data']
-            request_body = gzip.compress(uncompressed_data)
-            request['data'] = request_body
             request['headers'] = headers
+            # If the provided data is a file-like object, we create `GzipStream` which will handle
+            # the compression on-the-fly when the requests package starts reading its content.
+            # This helps avoid OOM errors when the opened file is too big.
+            # In any other cases, we use the in memory compression directly from
+            # the `gzip` package for backward compatibility.
+            raw_data = request['data']
+            request['data'] = GzipStream(raw_data) if isinstance(raw_data, io.IOBase) else gzip.compress(raw_data)
 
         # Next, we need to process the 'files' argument to try to fill in
         # any missing filenames where possible.
@@ -431,14 +465,14 @@ class BaseService:
                 files = files.items()
             # Next, fill in any missing filenames from file tuples.
             for part_name, file_tuple in files:
-                if file_tuple and len(
-                        file_tuple) == 3 and file_tuple[0] is None:
+                if file_tuple and len(file_tuple) == 3 and file_tuple[0] is None:
                     file = file_tuple[1]
                     if file and hasattr(file, 'name'):
                         filename = basename(file.name)
                         file_tuple = (filename, file_tuple[1], file_tuple[2])
                 new_files.append((part_name, file_tuple))
         request['files'] = new_files
+        logger.debug('Prepared request [%s %s]', request['method'], request['url'])
         return request
 
     @staticmethod

ibm_cloud_sdk_core/detailed_response.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import json
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 import requests
 
@@ -26,44 +26,50 @@ class DetailedResponse:
     Keyword Args:
         response: The response to the service request, defaults to None.
         headers: The headers of the response, defaults to None.
-        status_code: The status code of there response, defaults to None.
+        status_code: The status code of the response, defaults to None.
 
     Attributes:
-        response (requests.Response): The response to the service request.
+        result (dict, requests.Response, None): The response to the service request.
         headers (dict): The headers of the response.
         status_code (int): The status code of the response.
 
     """
-    def __init__(self,
-                 *,
-                 response: Optional[requests.Response] = None,
-                 headers: Optional[Dict[str, str]] = None,
-                 status_code: Optional[int] = None) -> None:
+
+    def __init__(
+        self,
+        *,
+        response: Optional[Union[dict, requests.Response]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        status_code: Optional[int] = None,
+    ) -> None:
         self.result = response
         self.headers = headers
         self.status_code = status_code
 
-    def get_result(self) -> requests.Response:
-        """Get the HTTP response of the service request.
+    def get_result(self) -> Optional[Union[dict, requests.Response]]:
+        """Get the response returned by the service request.
 
         Returns:
-            The response to the service request
+            The response to the service request. This could be one of the following:
+            1. a dict that represents an instance of a response model
+            2. a requests.Response instance if the operation returns a streamed response
+            3. None if the server returned no response body
         """
         return self.result
 
-    def get_headers(self) -> dict:
+    def get_headers(self) -> Optional[dict]:
         """The HTTP response headers of the service request.
 
         Returns:
-            A dictionary of response headers
+            A dictionary of response headers or None if no headers are present.
         """
         return self.headers
 
-    def get_status_code(self) -> int:
+    def get_status_code(self) -> Union[int, None]:
         """The HTTP status code of the service request.
 
         Returns:
-            The status code of the service request.
+            The status code associated with the service request.
         """
         return self.status_code