ibm-cloud-sdk-core 3.15.0__py3-none-any.whl → 3.20.6__py3-none-any.whl

ibm_cloud_sdk_core/base_service.py

@@ -1,6 +1,6 @@
 # coding: utf-8
 
-# Copyright 2019 IBM All Rights Reserved.
+# Copyright 2019, 2024 IBM All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,25 +15,33 @@
 # limitations under the License.
 
 import gzip
+import io
 import json as json_import
 import logging
-import platform
 from http.cookiejar import CookieJar
 from os.path import basename
 from typing import Dict, List, Optional, Tuple, Union
 from urllib3.util.retry import Retry
 
 import requests
-from requests.adapters import HTTPAdapter
 from requests.structures import CaseInsensitiveDict
+from requests.exceptions import JSONDecodeError
 
 from ibm_cloud_sdk_core.authenticators import Authenticator
 from .api_exception import ApiException
 from .detailed_response import DetailedResponse
+from .http_adapter import SSLHTTPAdapter
 from .token_managers.token_manager import TokenManager
-from .utils import (has_bad_first_or_last_char, remove_null_values,
-                    cleanup_values, read_external_sources, strip_extra_slashes)
-from .version import __version__
+from .utils import (
+    has_bad_first_or_last_char,
+    is_json_mimetype,
+    remove_null_values,
+    cleanup_values,
+    read_external_sources,
+    strip_extra_slashes,
+    GzipStream,
+)
+from .private_helpers import _build_user_agent
 
 # Uncomment this to enable http debugging
 # import http.client as http_client
@@ -43,8 +51,8 @@ from .version import __version__
 logger = logging.getLogger(__name__)
 
 
-#pylint: disable=too-many-instance-attributes
-#pylint: disable=too-many-locals
+# pylint: disable=too-many-instance-attributes
+# pylint: disable=too-many-locals
 class BaseService:
     """Common functionality shared by generated service classes.
 
@@ -72,18 +80,22 @@ class BaseService:
     Raises:
         ValueError: If Authenticator is not provided or invalid type.
     """
-    SDK_NAME = 'ibm-python-sdk-core'
-    ERROR_MSG_DISABLE_SSL = 'The connection failed because the SSL certificate is not valid. To use a self-signed '\
-                            'certificate, disable verification of the server\'s SSL certificate by invoking the '\
-                            'set_disable_ssl_verification(True) on your service instance and/ or use the '\
-                            'disable_ssl_verification option of the authenticator.'
-
-    def __init__(self,
-                 *,
-                 service_url: str = None,
-                 authenticator: Authenticator = None,
-                 disable_ssl_verification: bool = False,
-                 enable_gzip_compression: bool = False) -> None:
+
+    ERROR_MSG_DISABLE_SSL = (
+        'The connection failed because the SSL certificate is not valid. To use a self-signed '
+        'certificate, disable verification of the server\'s SSL certificate by invoking the '
+        'set_disable_ssl_verification(True) on your service instance and/ or use the '
+        'disable_ssl_verification option of the authenticator.'
+    )
+
+    def __init__(
+        self,
+        *,
+        service_url: str = None,
+        authenticator: Authenticator = None,
+        disable_ssl_verification: bool = False,
+        enable_gzip_compression: bool = False,
+    ) -> None:
         self.set_service_url(service_url)
         self.http_client = requests.Session()
         self.http_config = {}
@@ -92,58 +104,51 @@ class BaseService:
         self.disable_ssl_verification = disable_ssl_verification
         self.default_headers = None
         self.enable_gzip_compression = enable_gzip_compression
-        self._set_user_agent_header(self._build_user_agent())
+        self._set_user_agent_header(_build_user_agent())
         self.retry_config = None
-        self.http_adapter = HTTPAdapter()
+        self.http_adapter = SSLHTTPAdapter(_disable_ssl_verification=self.disable_ssl_verification)
         if not self.authenticator:
             raise ValueError('authenticator must be provided')
         if not isinstance(self.authenticator, Authenticator):
             raise ValueError('authenticator should be of type Authenticator')
 
-    def enable_retries(self, max_retries: int = 4, retry_interval: float = 1.0) -> None:
+        self.http_client.mount('http://', self.http_adapter)
+        self.http_client.mount('https://', self.http_adapter)
+
+    def enable_retries(self, max_retries: int = 4, retry_interval: float = 30.0) -> None:
         """Enable automatic retries on the underlying http client used by the BaseService instance.
 
         Args:
           max_retries: the maximum number of retries to attempt for a failed retryable request
-          retry_interval: the default wait time (in seconds) to use for the first retry attempt.
+          retry_interval: the maximum wait time (in seconds) to use for retry attempts.
             In general, if a response includes the Retry-After header, that will be used for
             the wait time associated with the retry attempt.  If the Retry-After header is not
-            present, then the wait time is based on the retry_interval and retry attempt number:
-               wait_time = retry_interval * (2 ^ (n-1)), where n is the retry attempt number
+            present, then the wait time is based on an exponential backoff policy with a maximum
+            backoff time of "retry_interval".
         """
         self.retry_config = Retry(
             total=max_retries,
-            backoff_factor=retry_interval,
+            backoff_factor=1.0,
+            backoff_max=retry_interval,
             # List of HTTP status codes to retry on in addition to Timeout/Connection Errors
             status_forcelist=[429, 500, 502, 503, 504],
             # List of HTTP methods to retry on
             # Omitting this will default to all methods except POST
-            allowed_methods=['HEAD', 'GET', 'PUT',
-                             'DELETE', 'OPTIONS', 'TRACE', 'POST']
+            allowed_methods=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'],
+        )
+        self.http_adapter = SSLHTTPAdapter(
+            max_retries=self.retry_config, _disable_ssl_verification=self.disable_ssl_verification
         )
-        self.http_adapter = HTTPAdapter(max_retries=self.retry_config)
         self.http_client.mount('http://', self.http_adapter)
         self.http_client.mount('https://', self.http_adapter)
 
     def disable_retries(self):
         """Remove retry config from http_adapter"""
         self.retry_config = None
-        self.http_adapter = HTTPAdapter()
+        self.http_adapter = SSLHTTPAdapter(_disable_ssl_verification=self.disable_ssl_verification)
         self.http_client.mount('http://', self.http_adapter)
         self.http_client.mount('https://', self.http_adapter)
 
-    @staticmethod
-    def _get_system_info() -> str:
-        return '{0} {1} {2}'.format(
-            platform.system(),  # OS
-            platform.release(),  # OS version
-            platform.python_version()  # Python version
-        )
-
-    def _build_user_agent(self) -> str:
-        return '{0}-{1} {2}'.format(self.SDK_NAME, __version__,
-                                    self._get_system_info())
-
     def configure_service(self, service_name: str) -> None:
         """Look for external configuration of a service. Set service properties.
 
@@ -165,19 +170,16 @@ class BaseService:
         if config.get('URL'):
             self.set_service_url(config.get('URL'))
         if config.get('DISABLE_SSL'):
-            self.set_disable_ssl_verification(
-                config.get('DISABLE_SSL').lower() == 'true')
+            self.set_disable_ssl_verification(config.get('DISABLE_SSL').lower() == 'true')
         if config.get('ENABLE_GZIP'):
-            self.set_enable_gzip_compression(
-                config.get('ENABLE_GZIP').lower() == 'true')
+            self.set_enable_gzip_compression(config.get('ENABLE_GZIP').lower() == 'true')
         if config.get('ENABLE_RETRIES'):
             if config.get('ENABLE_RETRIES').lower() == 'true':
                 kwargs = {}
                 if config.get('MAX_RETRIES'):
                     kwargs["max_retries"] = int(config.get('MAX_RETRIES'))
                 if config.get('RETRY_INTERVAL'):
-                    kwargs["retry_interval"] = float(
-                        config.get('RETRY_INTERVAL'))
+                    kwargs["retry_interval"] = float(config.get('RETRY_INTERVAL'))
                 self.enable_retries(**kwargs)
 
     def _set_user_agent_header(self, user_agent_string: str) -> None:
@@ -196,10 +198,11 @@ class BaseService:
         """
         if isinstance(http_config, dict):
             self.http_config = http_config
-            if (self.authenticator
-                    and hasattr(self.authenticator, 'token_manager')
-                    and isinstance(self.authenticator.token_manager,
-                                   TokenManager)):
+            if (
+                self.authenticator
+                and hasattr(self.authenticator, 'token_manager')
+                and isinstance(self.authenticator.token_manager, TokenManager)
+            ):
                 self.authenticator.token_manager.http_config = http_config
         else:
             raise TypeError("http_config parameter must be a dictionary")
@@ -211,8 +214,18 @@ class BaseService:
         Keyword Arguments:
             status: set to true to disable ssl verification (default: {False})
         """
+        if self.disable_ssl_verification == status:
+            # Do nothing if the state doesn't change.
+            return
+
         self.disable_ssl_verification = status
 
+        self.http_adapter = SSLHTTPAdapter(
+            max_retries=self.retry_config, _disable_ssl_verification=self.disable_ssl_verification
+        )
+        self.http_client.mount('http://', self.http_adapter)
+        self.http_client.mount('https://', self.http_adapter)
+
     def set_service_url(self, service_url: str) -> None:
         """Set the url the service will make HTTP requests too.
 
@@ -248,8 +261,7 @@ class BaseService:
         if isinstance(http_client, requests.sessions.Session):
             self.http_client = http_client
         else:
-            raise TypeError(
-                "http_client parameter must be a requests.sessions.Session")
+            raise TypeError("http_client parameter must be a requests.sessions.Session")
 
     def get_authenticator(self) -> Authenticator:
         """Get the authenticator currently used by the service.
@@ -295,40 +307,47 @@ class BaseService:
 
         # Remove the keys we set manually, don't let the user to overwrite these.
         reserved_keys = ['method', 'url', 'headers', 'params', 'cookies']
+        silent_keys = ['headers']
         for key in reserved_keys:
             if key in kwargs:
                 del kwargs[key]
-                logger.warning('"%s" has been removed from the request', key)
+                if key not in silent_keys:
+                    logger.warning('"%s" has been removed from the request', key)
         try:
-            response = self.http_client.request(**request,
-                                                cookies=self.jar,
-                                                **kwargs)
+            response = self.http_client.request(**request, cookies=self.jar, **kwargs)
 
+            # Process a "success" response.
             if 200 <= response.status_code <= 299:
                 if response.status_code == 204 or request['method'] == 'HEAD':
-                    # There is no body content for a HEAD request or a 204 response
+                    # There is no body content for a HEAD response or a 204 response.
                     result = None
                 elif stream_response:
                     result = response
                 elif not response.text:
                     result = None
-                else:
+                elif is_json_mimetype(response.headers.get('Content-Type')):
+                    # If this is a JSON response, then try to unmarshal it.
                     try:
-                        result = response.json()
-                    except:
-                        result = response
-                return DetailedResponse(response=result,
-                                        headers=response.headers,
-                                        status_code=response.status_code)
+                        result = response.json(strict=False)
+                    except JSONDecodeError as err:
+                        raise ApiException(
+                            code=response.status_code,
+                            http_response=response,
+                            message='Error processing the HTTP response',
+                        ) from err
+                else:
+                    # Non-JSON response, just use response body as-is.
+                    result = response
+
+                return DetailedResponse(response=result, headers=response.headers, status_code=response.status_code)
 
+            # Received error status code from server, raise an APIException.
             raise ApiException(response.status_code, http_response=response)
         except requests.exceptions.SSLError:
             logger.exception(self.ERROR_MSG_DISABLE_SSL)
             raise
 
-    def set_enable_gzip_compression(self,
-                                    should_enable_compression: bool = False
-                                    ) -> None:
+    def set_enable_gzip_compression(self, should_enable_compression: bool = False) -> None:
         """Set value to enable gzip compression on request bodies"""
         self.enable_gzip_compression = should_enable_compression
 
@@ -336,18 +355,17 @@ class BaseService:
         """Get value for enabling gzip compression on request bodies"""
         return self.enable_gzip_compression
 
-    def prepare_request(self,
-                        method: str,
-                        url: str,
-                        *,
-                        headers: Optional[dict] = None,
-                        params: Optional[dict] = None,
-                        data: Optional[Union[str, dict]] = None,
-                        files: Optional[Union[Dict[str, Tuple[str]],
-                                              List[Tuple[str,
-                                                         Tuple[str,
-                                                               ...]]]]] = None,
-                        **kwargs) -> dict:
+    def prepare_request(
+        self,
+        method: str,
+        url: str,
+        *,
+        headers: Optional[dict] = None,
+        params: Optional[dict] = None,
+        data: Optional[Union[str, dict]] = None,
+        files: Optional[Union[Dict[str, Tuple[str]], List[Tuple[str, Tuple[str, ...]]]]] = None,
+        **kwargs,
+    ) -> dict:
         """Build a dict that represents an HTTP service request.
 
         Clean up headers, add default http configuration, convert data
@@ -358,8 +376,10 @@ class BaseService:
             url: The origin + pathname according to WHATWG spec.
 
         Keyword Arguments:
-            headers: Headers of the request.
-            params: Querystring data to be appended to the url.
+            headers: A dictionary containing the headers to be included in the request.
+                    Entries with a value of None will be ignored (excluded).
+            params: A dictionary containing the query parameters to be included in the request.
+                    Entries with a value of None will be ignored (excluded).
             data: The request body. Converted to json if a dict.
             files: 'files' can be a dictionary (i.e { '<part-name>': (<tuple>)}),
                 or a list of tuples [ (<part-name>, (<tuple>))... ]
@@ -404,14 +424,16 @@ class BaseService:
         self.authenticator.authenticate(request)
 
         # Compress the request body if applicable
-        if (self.get_enable_gzip_compression()
-                and 'content-encoding' not in headers
-                and request['data'] is not None):
+        if self.get_enable_gzip_compression() and 'content-encoding' not in headers and request['data'] is not None:
             headers['content-encoding'] = 'gzip'
-            uncompressed_data = request['data']
-            request_body = gzip.compress(uncompressed_data)
-            request['data'] = request_body
             request['headers'] = headers
+            # If the provided data is a file-like object, we create `GzipStream` which will handle
+            # the compression on-the-fly when the requests package starts reading its content.
+            # This helps avoid OOM errors when the opened file is too big.
+            # In any other cases, we use the in memory compression directly from
+            # the `gzip` package for backward compatibility.
+            raw_data = request['data']
+            request['data'] = GzipStream(raw_data) if isinstance(raw_data, io.IOBase) else gzip.compress(raw_data)
 
         # Next, we need to process the 'files' argument to try to fill in
         # any missing filenames where possible.
@@ -426,8 +448,7 @@ class BaseService:
                 files = files.items()
             # Next, fill in any missing filenames from file tuples.
             for part_name, file_tuple in files:
-                if file_tuple and len(
-                        file_tuple) == 3 and file_tuple[0] is None:
+                if file_tuple and len(file_tuple) == 3 and file_tuple[0] is None:
                     file = file_tuple[1]
                     if file and hasattr(file, 'name'):
                         filename = basename(file.name)

ibm_cloud_sdk_core/detailed_response.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import json
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 import requests
 
@@ -26,44 +26,50 @@ class DetailedResponse:
     Keyword Args:
         response: The response to the service request, defaults to None.
         headers: The headers of the response, defaults to None.
-        status_code: The status code of there response, defaults to None.
+        status_code: The status code of the response, defaults to None.
 
     Attributes:
-        response (requests.Response): The response to the service request.
+        result (dict, requests.Response, None): The response to the service request.
         headers (dict): The headers of the response.
         status_code (int): The status code of the response.
 
     """
-    def __init__(self,
-                 *,
-                 response: Optional[requests.Response] = None,
-                 headers: Optional[Dict[str, str]] = None,
-                 status_code: Optional[int] = None) -> None:
+
+    def __init__(
+        self,
+        *,
+        response: Optional[Union[dict, requests.Response]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        status_code: Optional[int] = None,
+    ) -> None:
         self.result = response
         self.headers = headers
         self.status_code = status_code
 
-    def get_result(self) -> requests.Response:
-        """Get the HTTP response of the service request.
+    def get_result(self) -> Optional[Union[dict, requests.Response]]:
+        """Get the response returned by the service request.
 
         Returns:
-            The response to the service request
+            The response to the service request. This could be one of the following:
+            1. a dict that represents an instance of a response model
+            2. a requests.Response instance if the operation returns a streamed response
+            3. None if the server returned no response body
         """
         return self.result
 
-    def get_headers(self) -> dict:
+    def get_headers(self) -> Optional[dict]:
         """The HTTP response headers of the service request.
 
         Returns:
-            A dictionary of response headers
+            A dictionary of response headers or None if no headers are present.
         """
         return self.headers
 
-    def get_status_code(self) -> int:
+    def get_status_code(self) -> Union[int, None]:
         """The HTTP status code of the service request.
 
         Returns:
-            The status code of the service request.
+            The status code associated with the service request.
         """
         return self.status_code
 

ibm_cloud_sdk_core/get_authenticator.py

@@ -14,9 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .authenticators import (Authenticator, BasicAuthenticator, BearerTokenAuthenticator, ContainerAuthenticator,
-                             CloudPakForDataAuthenticator, IAMAuthenticator, NoAuthAuthenticator,
-                             VPCInstanceAuthenticator)
+from .authenticators import (
+    Authenticator,
+    BasicAuthenticator,
+    BearerTokenAuthenticator,
+    ContainerAuthenticator,
+    CloudPakForDataAuthenticator,
+    IAMAuthenticator,
+    NoAuthAuthenticator,
+    VPCInstanceAuthenticator,
+    MCSPAuthenticator,
+)
 from .utils import read_external_sources
 
 
@@ -59,12 +67,9 @@ def __construct_authenticator(config: dict) -> Authenticator:
     authenticator = None
 
     if auth_type == Authenticator.AUTHTYPE_BASIC.lower():
-        authenticator = BasicAuthenticator(
-            username=config.get('USERNAME'),
-            password=config.get('PASSWORD'))
+        authenticator = BasicAuthenticator(username=config.get('USERNAME'), password=config.get('PASSWORD'))
     elif auth_type == Authenticator.AUTHTYPE_BEARERTOKEN.lower():
-        authenticator = BearerTokenAuthenticator(
-            bearer_token=config.get('BEARER_TOKEN'))
+        authenticator = BearerTokenAuthenticator(bearer_token=config.get('BEARER_TOKEN'))
     elif auth_type == Authenticator.AUTHTYPE_CONTAINER.lower():
         authenticator = ContainerAuthenticator(
             cr_token_filename=config.get('CR_TOKEN_FILENAME'),
@@ -73,30 +78,37 @@ def __construct_authenticator(config: dict) -> Authenticator:
             url=config.get('AUTH_URL'),
             client_id=config.get('CLIENT_ID'),
             client_secret=config.get('CLIENT_SECRET'),
-            disable_ssl_verification=config.get(
-                'AUTH_DISABLE_SSL', 'false').lower() == 'true',
-            scope=config.get('SCOPE'))
+            disable_ssl_verification=config.get('AUTH_DISABLE_SSL', 'false').lower() == 'true',
+            scope=config.get('SCOPE'),
+        )
     elif auth_type == Authenticator.AUTHTYPE_CP4D.lower():
         authenticator = CloudPakForDataAuthenticator(
             username=config.get('USERNAME'),
             password=config.get('PASSWORD'),
             url=config.get('AUTH_URL'),
             apikey=config.get('APIKEY'),
-            disable_ssl_verification=config.get('AUTH_DISABLE_SSL', 'false').lower() == 'true')
+            disable_ssl_verification=config.get('AUTH_DISABLE_SSL', 'false').lower() == 'true',
+        )
     elif auth_type == Authenticator.AUTHTYPE_IAM.lower() and config.get('APIKEY'):
         authenticator = IAMAuthenticator(
             apikey=config.get('APIKEY'),
             url=config.get('AUTH_URL'),
             client_id=config.get('CLIENT_ID'),
             client_secret=config.get('CLIENT_SECRET'),
-            disable_ssl_verification=config.get(
-                'AUTH_DISABLE_SSL', 'false').lower() == 'true',
-            scope=config.get('SCOPE'))
+            disable_ssl_verification=config.get('AUTH_DISABLE_SSL', 'false').lower() == 'true',
+            scope=config.get('SCOPE'),
+        )
     elif auth_type == Authenticator.AUTHTYPE_VPC.lower():
         authenticator = VPCInstanceAuthenticator(
             iam_profile_crn=config.get('IAM_PROFILE_CRN'),
             iam_profile_id=config.get('IAM_PROFILE_ID'),
-            url=config.get('AUTH_URL'))
+            url=config.get('AUTH_URL'),
+        )
+    elif auth_type == Authenticator.AUTHTYPE_MCSP.lower():
+        authenticator = MCSPAuthenticator(
+            apikey=config.get('APIKEY'),
+            url=config.get('AUTH_URL'),
+        )
     elif auth_type == Authenticator.AUTHTYPE_NOAUTH.lower():
         authenticator = NoAuthAuthenticator()
 

ibm_cloud_sdk_core/http_adapter.py

@@ -0,0 +1,28 @@
+import ssl
+
+from requests import certs
+from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK
+from urllib3.util.ssl_ import create_urllib3_context
+
+
+class SSLHTTPAdapter(HTTPAdapter):
+    """Wraps the original HTTP adapter and adds additional SSL context."""
+
+    def __init__(self, *args, **kwargs):
+        self._disable_ssl_verification = kwargs.pop('_disable_ssl_verification', None)
+
+        super().__init__(*args, **kwargs)
+
+    def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
+        """Create and use custom SSL configuration."""
+
+        ssl_context = create_urllib3_context()
+        # NOTE: https://github.com/psf/requests/pull/6731/files#r1622893724
+        ssl_context.load_verify_locations(certs.where())
+        ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
+
+        if self._disable_ssl_verification:
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        super().init_poolmanager(connections, maxsize, block, ssl_context=ssl_context, **pool_kwargs)

ibm_cloud_sdk_core/private_helpers.py

@@ -0,0 +1,34 @@
+# coding: utf-8
+
+# Copyright 2024 IBM All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# from ibm_cloud_sdk_core.authenticators import Authenticator
+
+import platform
+from .version import __version__
+
+SDK_NAME = 'ibm-python-sdk-core'
+
+
+def _get_system_info() -> str:
+    return 'os.name={0} os.version={1} python.version={2}'.format(
+        platform.system(), platform.release(), platform.python_version()
+    )
+
+
+def _build_user_agent(component: str = None) -> str:
+    sub_component = ""
+    if component is not None:
+        sub_component = '/{0}'.format(component)
+    return '{0}{1}-{2} {3}'.format(SDK_NAME, sub_component, __version__, _get_system_info())