osdu-api 1.0.3__py3-none-any.whl

osdu_api/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2020 Google LLC
+#  Copyright © 2020 Amazon Web Services
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+

osdu_api/auth/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2020 Google LLC
+#  Copyright 2020 EPAM Systems
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+

osdu_api/auth/authorization.py

@@ -0,0 +1,174 @@
+#  Copyright 2020 Google LLC
+#  Copyright 2020 EPAM Systems
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import logging
+from abc import ABC, abstractmethod
+from functools import partial
+from http import HTTPStatus
+from typing import Any, Callable, Optional, Union
+
+import requests
+
+from osdu_api.exceptions.exceptions import TokenRefresherNotPresentError
+
+logger = logging.getLogger()
+
+
+class TokenRefresher(ABC):
+
+    def __init__(self):
+        self._access_token = ""
+
+    @abstractmethod
+    def refresh_token(self) -> str:
+        """
+        Implement logics of refreshing token here.
+        """
+        pass
+
+    def authorize(self):
+        self._access_token = self.refresh_token()
+
+    @property
+    def access_token(self) -> str:
+        return self._access_token
+
+    @property
+    def authorization_header(self) -> dict:
+        """
+        Must return  authorization header for updating headers dict.
+        E.g. return {"Authorization": "Bearer <access_token>"}
+        """
+        return {"Authorization": f"Bearer {self.access_token}"}
+
+
+def make_callable_request(obj: Union[object, None], request_function: Callable, headers: dict,
+                          *args, **kwargs) -> Callable:
+    """
+    Create send_request_with_auth function.
+    """
+    if obj:  # if wrapped function is an object's method
+        callable_request = partial(
+            request_function, obj, headers, *args, **kwargs)
+    else:
+        callable_request = partial(request_function, headers, *args, **kwargs)
+    return callable_request
+
+
+def _validate_headers_type(headers: Any):
+    if not isinstance(headers, dict):
+        logger.error(f"Got headers {headers}")
+        raise TypeError(
+            f"Request's headers type expected to be 'dict'. Got {dict}")
+
+
+def _validate_response_type(response: requests.Response, request_function: Callable):
+    if not isinstance(response, requests.Response):
+        logger.error(f"Function or method {request_function}"
+                     f" must return values of type 'requests.Response'. "
+                     f"Got {type(response)} instead")
+        raise TypeError
+
+
+def _validate_token_refresher_type(token_refresher: TokenRefresher):
+    if not isinstance(token_refresher, TokenRefresher):
+        raise TypeError(
+            f"Token refresher must be of type {TokenRefresher}. Got {type(token_refresher)}"
+        )
+
+
+def _get_object_token_refresher(
+        obj: object
+) -> TokenRefresher:
+    """
+    Check if token refresher passed into decorator or specified in object's as 'token_refresher'
+    property.
+    """
+    try:
+        obj.__getattribute__("token_refresher")
+    except AttributeError:
+        raise TokenRefresherNotPresentError("Token refresher must be passed into decorator or "
+                                            "set as object's 'refresh_token' attribute.")
+    else:
+        token_refresher = obj.token_refresher # type: ignore
+        return token_refresher
+
+
+def send_request_with_auth_header(token_refresher: TokenRefresher, *args,
+                                  **kwargs) -> requests.Response:
+    """
+    Send request with authorization token. If response status is in HTTPStatus.UNAUTHORIZED or
+    HTTPStatus.FORBIDDEN, then refresh token and send request once again.
+    """
+    obj = kwargs.pop("obj", None)
+    request_function = kwargs.pop("request_function")
+    headers = kwargs.pop("headers")
+    _validate_headers_type(headers)
+    headers.update(token_refresher.authorization_header) # type: ignore
+
+    send_request_with_auth = make_callable_request(
+        obj, request_function, headers, *args, **kwargs)
+    response = send_request_with_auth()
+    _validate_response_type(response, request_function)
+
+    if not response.ok:
+        if response.status_code in (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN):
+            token_refresher.authorize()
+            headers.update(token_refresher.authorization_header) # type: ignore
+            send_request_with_auth = make_callable_request(obj,
+                                                           request_function,
+                                                           headers,
+                                                           *args, **kwargs)
+            response = send_request_with_auth()
+    return response
+
+
+def authorize(token_refresher: Optional[TokenRefresher] = None) -> Callable:
+    """
+    Wrap a request function and check response. If response's error status code
+    is about Authorization, refresh token and invoke this function once again.
+    Expects function:
+    If response is not ok and not about Authorization, then raises HTTPError
+    request_func(header: dict, *args, **kwargs) -> requests.Response
+    Or method:
+    request_method(self, header: dict, *args, **kwargs) -> requests.Response
+    """
+
+    def refresh_token_wrapper(request_function: Callable) -> Callable:
+        is_method = len(request_function.__qualname__.split(".")) > 1
+        if is_method:
+            def _wrapper(obj: object, headers: dict, *args, **kwargs) -> requests.Response:
+                _token_refresher = _get_object_token_refresher(obj)
+                _validate_token_refresher_type(_token_refresher)
+                return send_request_with_auth_header(_token_refresher,
+                                                     request_function=request_function,
+                                                     obj=obj,
+                                                     headers=headers,
+                                                     *args,
+                                                     **kwargs)
+        else:
+            def _wrapper(headers: dict, *args, **kwargs) -> requests.Response:
+                if not isinstance(token_refresher, TokenRefresher):
+                    raise TokenRefresherNotPresentError(
+                        "Token refresher isn't provided or has wrong type in 'authorize' decorator."
+                    )
+                _validate_token_refresher_type(token_refresher)
+                return send_request_with_auth_header(token_refresher,
+                                                     request_function=request_function,
+                                                     headers=headers,
+                                                     *args, **kwargs)
+        return _wrapper
+
+    return refresh_token_wrapper

osdu_api/auth/refresh_token.py

@@ -0,0 +1,62 @@
+#  Copyright 2023 Google LLC
+#  Copyright 2023 EPAM Systems
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Auth and refresh token utility functions."""
+from typing import Optional
+
+from tenacity import retry, stop_after_attempt
+
+from osdu_api.auth.authorization import TokenRefresher
+from osdu_api.providers import credentials
+from osdu_api.providers.types import BaseCredentials
+
+RETRIES = 3
+
+
+class BaseTokenRefresher(TokenRefresher):
+    """Base Token refresher, that works with Credentials and has methods to refresh access tokens"""
+
+    def __init__(self, creds: Optional[BaseCredentials] = None):
+        super().__init__()
+        self._credentials = creds or credentials.get_credentials()
+
+    @retry(stop=stop_after_attempt(RETRIES))
+    def refresh_token(self) -> str:
+        """Refresh the token and cache token using airflow variables.
+
+        :return: The refreshed token
+        :rtype: str
+        """
+        self._credentials.refresh_token()
+        self._access_token = self._credentials.access_token
+        return self._access_token
+
+    @property
+    def access_token(self) -> str:
+        """The access token.
+
+        :return: The access token
+        :rtype: str
+        """
+        return self._access_token
+
+    @property
+    def authorization_header(self) -> dict:
+        """Authorization header with bearer token.
+
+        :return: Auth header as dict
+        :rtype: dict
+        """
+        return {"Authorization": f"Bearer {self.access_token}"}

osdu_api/clients/__init__.py

@@ -0,0 +1,15 @@
+# Copyright © 2020 Amazon Web Services
+# Copyright 2022 Google LLC
+# Copyright 2022 EPAM Systems
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

osdu_api/clients/base_client.py

@@ -0,0 +1,389 @@
+# Copyright © 2020 Amazon Web Services
+# Copyright 2022 Google LLC
+# Copyright 2022 EPAM Systems
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import configparser
+import importlib
+import logging
+import os
+
+import httpx
+import requests
+from typing import Any, Optional
+
+import tenacity
+from httpx import Response
+
+from osdu_api.auth.authorization import authorize, TokenRefresher
+from osdu_api.configuration.base_config_manager import BaseConfigManager
+from osdu_api.configuration.config_manager import DefaultConfigManager
+from osdu_api.constants import RETRY_MIN_WAIT, RETRY_MAX_WAIT, RETRY_MULTIPLIER, \
+    RETRY_STOP_AFTER_ATTEMPT
+from osdu_api.exceptions.exceptions import MakeRequestError
+from osdu_api.model.http_method import HttpMethod
+from osdu_api.utils.request import response_not_ok
+
+
+class BaseClient:
+    """
+    Base client that is meant to be extended by service specific clients
+    """
+
+    def __init__(
+        self,
+        config_manager: Optional[BaseConfigManager] = None,
+        provider: Optional[str] = None,
+        data_partition_id: Optional[str] = None,
+        token_refresher: Optional[TokenRefresher] = None,
+        logger: Optional[Any] = None,
+        user_id: Optional[str] = None,
+        async_client: httpx.AsyncClient = None
+    ):
+        """
+        Base client gets initialized with configuration values and a bearer token
+        based on provider-specific logic
+        """
+        self._ssl_verify = True
+
+        self._parse_config(config_manager, provider, data_partition_id)
+        self.user_id = user_id
+
+        self.unauth_retries = 0
+        if self.use_service_principal:
+            self._refresh_service_principal_token()
+
+        self.logger = logging.getLogger(__name__) if logger is None else logger
+
+        self.token_refresher = token_refresher
+        self.async_client = async_client
+
+    def _parse_config(
+        self,
+        config_manager: Optional[BaseConfigManager] = None,
+        provider: Optional[str] = None,
+        data_partition_id: Optional[str] = None
+    ):
+        """
+        Parse config.
+
+        :param config_manager: ConfigManager to get configs, defaults to None
+        :type config_manager: BaseConfigManager, optional
+        """
+        if os.getenv('OSDU_API_DISABLE_SSL_VERIFICAITON') == 'iknowthemaninthemiddle':
+            self._ssl_verify = False
+
+        self.config_manager = config_manager or DefaultConfigManager()
+
+        # self.use_service_principal is used by AWS only, so other CSPs can set this attribute to False in any way.
+        try:
+            self.provider = provider or self.config_manager.get('provider', 'name')
+            self.use_service_principal = self.config_manager.getbool('environment',
+                                                                     'use_service_principal', False)
+            if self.use_service_principal:
+                self.service_principal_module_name = self.config_manager.get('provider',
+                                                                             'service_principal_module_name')
+        except configparser.Error:
+            self.use_service_principal = False
+
+        if data_partition_id is None:
+            self.data_partition_id = self.config_manager.get(
+                'environment', 'data_partition_id')
+        else:
+            self.data_partition_id = data_partition_id
+
+
+    def _refresh_service_principal_token(self):
+        """
+        The path to the logic to get a valid bearer token is dynamically injected based on
+        what provider and entitlements module name is provided in the configuration yaml
+        """
+        entitlements_client = importlib.import_module(
+            'osdu_api.providers.%s.%s' % (self.provider, self.service_principal_module_name))
+        self.service_principal_token = entitlements_client.get_service_principal_token()
+
+    def _send_request(self, method: HttpMethod, url: str, data: str, headers: dict,
+                      params: dict) -> requests.Response:
+        """
+        Send request to OSDU
+
+        :param method: HTTP method
+        :type method: HttpMethod
+        :param url: service's URL
+        :type url: str
+        :param data: request's data
+        :type data: str
+        :param headers: request's headers
+        :type headers: dict
+        :param params: params
+        :type params: dict
+        :return: response from OSDU service
+        :rtype: requests.Response
+        """
+        if method == HttpMethod.GET:
+            response = requests.get(url=url, params=params, headers=headers,
+                                    verify=self._ssl_verify)
+        elif method == HttpMethod.DELETE:
+            response = requests.delete(url=url, params=params, headers=headers,
+                                       verify=self._ssl_verify)
+        elif method == HttpMethod.POST:
+            response = requests.post(url=url, params=params, data=data, headers=headers,
+                                     verify=self._ssl_verify)
+        elif method == HttpMethod.PUT:
+            response = requests.put(url=url, params=params, data=data, headers=headers,
+                                    verify=self._ssl_verify)
+        return response
+
+    async def _send_async_request(self, method: HttpMethod, url: str, data, headers: dict,
+                                  params: dict) -> requests.Response:
+        if method == HttpMethod.GET:
+            response = await self.async_client.get(url=url, params=params, headers=headers)
+        elif method == HttpMethod.DELETE:
+            response = await self.async_client.delete(url=url, params=params, headers=headers)
+        elif method == HttpMethod.POST:
+            response = await self.async_client.post(url=url, params=params, data=data,
+                                                    headers=headers)
+        elif method == HttpMethod.PUT:
+            response = await self.async_client.put(url=url, params=params, data=data,
+                                                   headers=headers)
+        return response
+
+    def _send_request_with_principle_token(
+        self,
+        method: HttpMethod,
+        url: str,
+        data: str,
+        headers: dict,
+        params: dict,
+    ) -> requests.Response:
+        bearer_token = self.service_principal_token
+        if bearer_token is not None and 'Bearer ' not in bearer_token:
+            bearer_token = 'Bearer ' + bearer_token
+
+        headers["Authorization"] = bearer_token
+
+        response = self._send_request(method, url, data, headers, params)
+
+        if (response.status_code == 401 or response.status_code == 403) and self.unauth_retries < 1:
+            self.unauth_retries += 1
+            self._refresh_service_principal_token()
+            self._send_request_with_principle_token(method, url, data, headers, params)
+
+        self.unauth_retries = 0
+        return response
+
+    async def _send_async_request_with_principle_token(
+        self,
+        method: HttpMethod,
+        url: str,
+        data: str,
+        headers: dict,
+        params: dict,
+    ) -> requests.Response:
+        bearer_token = self.service_principal_token
+        if bearer_token is not None and 'Bearer ' not in bearer_token:
+            bearer_token = 'Bearer ' + bearer_token
+
+        headers["Authorization"] = bearer_token
+
+        response = await self._send_async_request(method, url, data, headers, params)
+
+        if (response.status_code == 401 or response.status_code == 403) and self.unauth_retries < 1:
+            self.unauth_retries += 1
+            self._refresh_service_principal_token()
+            await self._send_async_request_with_principle_token(method, url, data, headers, params)
+
+        self.unauth_retries = 0
+        return response
+
+    def _send_request_with_bearer_token(
+        self,
+        method: HttpMethod,
+        url: str,
+        data: str,
+        headers: dict,
+        params: dict,
+        bearer_token: str
+    ) -> requests.Response:
+        """
+        Send request with bearer_token provided by SDK user.
+
+        :param method: HTTP method
+        :type method: HttpMethod
+        :param url: service's URL
+        :type url: str
+        :param data: request's data
+        :type data: str
+        :param headers: request's headers
+        :type headers: dict
+        :param params: params
+        :type params: dict
+        :param bearer_token: bearer_token
+        :type params: str
+        :return: response from OSDU service
+        :rtype: requests.Response
+        """
+
+        if bearer_token is not None and 'Bearer ' not in bearer_token:
+            bearer_token = 'Bearer ' + bearer_token
+        headers["Authorization"] = bearer_token
+
+        response = self._send_request(method, url, data, headers, params)
+
+        if not response.ok:
+            response.raise_for_status()
+
+        return response
+
+    async def _send_async_request_with_bearer_token(
+        self,
+        method: HttpMethod,
+        url: str,
+        data: str,
+        headers: dict,
+        params: dict,
+        bearer_token: str
+    ) -> Response:
+
+        if bearer_token is not None and 'Bearer ' not in bearer_token:
+            bearer_token = 'Bearer ' + bearer_token
+        headers["Authorization"] = bearer_token
+
+        response: Response = await self._send_async_request(method, url, data, headers, params)
+
+        if response.status_code != 200:
+            response.raise_for_status()
+
+        return response
+
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(RETRY_STOP_AFTER_ATTEMPT),
+        wait=tenacity.wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN_WAIT,
+                                       max=RETRY_MAX_WAIT),
+        retry=(tenacity.retry_if_result(response_not_ok) | tenacity.retry_if_exception_type(
+            Exception)),
+        retry_error_callback=lambda retry_state: retry_state.outcome.result()
+    )
+    @authorize()
+    def _send_request_with_token_refresher(
+        self,
+        headers: dict,
+        method: HttpMethod,
+        url: str,
+        data: str,
+        params: dict
+    ) -> requests.Response:
+        return self._send_request(method, url, data, headers, params)
+
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(RETRY_STOP_AFTER_ATTEMPT),
+        wait=tenacity.wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN_WAIT,
+                                       max=RETRY_MAX_WAIT),
+        retry=(tenacity.retry_if_result(response_not_ok) | tenacity.retry_if_exception_type(
+            Exception)),
+        retry_error_callback=lambda retry_state: retry_state.outcome.result()
+    )
+    @authorize()
+    async def _send_async_request_with_token_refresher(
+        self,
+        headers: dict,
+        method: HttpMethod,
+        url: str,
+        data: str,
+        params: dict
+    ) -> requests.Response:
+        return await self._send_async_request(method, url, data, headers, params)
+
+    def make_request(
+        self,
+        method: HttpMethod,
+        url: str,
+        data: Any = '',
+        add_headers: Optional[dict] = None,
+        params: Optional[dict] = None,
+        bearer_token: Optional[str] = None,
+        no_auth: bool = False
+    ) -> requests.Response:
+        """
+        Makes a request using python's built in requests library. Takes additional headers if
+        necessary
+        """
+        add_headers = add_headers or {}
+        params = params or {}
+
+        headers = self.prepare_headers(add_headers, self.data_partition_id, self.user_id)
+
+        if no_auth:
+            response = self._send_request(method, url, data, headers, params)
+        elif bearer_token:
+            response = self._send_request_with_bearer_token(method, url, data, headers, params,
+                                                            bearer_token)
+        elif self.token_refresher:
+            # _send_request_with_token_refresher has other method signature to work with @authorize decorator
+            response = self._send_request_with_token_refresher(headers, method, url, data, params)
+        elif self.use_service_principal:
+            response = self._send_request_with_principle_token(method, url, data, headers, params)
+        else:
+            raise MakeRequestError("There is no strategy to get Bearer token.")
+        return response
+
+    async def make_async_request(
+        self,
+        method: HttpMethod,
+        url: str,
+        data: Any = '',
+        add_headers: Optional[dict] = None,
+        params: Optional[dict] = None,
+        bearer_token: Optional[str] = None,
+        no_auth: bool = False
+    ) -> requests.Response:
+
+        add_headers = add_headers or {}
+        params = params or {}
+
+        headers = self.prepare_headers(add_headers, self.data_partition_id, self.user_id)
+
+        if no_auth:
+            response = await self._send_async_request(method, url, data, headers, params)
+        elif bearer_token:
+            response = await self._send_async_request_with_bearer_token(method, url, data, headers,
+                                                                  params,
+                                                                  bearer_token)
+        elif self.token_refresher:
+            response = await self._send_async_request_with_token_refresher(headers, method, url, data,
+                                                                     params)
+        elif self.use_service_principal:
+            response = await self._send_async_request_with_principle_token(method, url, data, headers,
+                                                                     params)
+        else:
+            raise MakeRequestError("There is no strategy to get Bearer token.")
+        return response
+
+    @staticmethod
+    def prepare_headers(add_headers, data_partition, user_id):
+        headers = {
+            'content-type': 'application/json',
+            'data-partition-id': data_partition,
+        }
+        if user_id is not None:
+            # TODO: come up with the single header for all CSP
+            if os.getenv("ENTITLEMENTS_IMPERSONATION") in ("True", "true"):
+                # it's for GC and Baremetal implementations
+                headers["on-behalf-of"] = user_id
+            else:
+                # it's for Azure implementation
+                headers['x-on-behalf-of'] = user_id
+        for key, value in add_headers.items():
+            headers[key] = value
+        return headers

osdu_api/clients/crs_catalog/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 Geosiris
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.