cloe-nessy 0.2.9__py3-none-any.whl

cloe_nessy/clients/__init__.py

@@ -0,0 +1,5 @@
+from .api_client import APIClient
+
+__all__ = [
+    "APIClient",
+]

cloe_nessy/clients/api_client/__init__.py

@@ -0,0 +1,3 @@
+from .api_client import APIClient
+
+__all__ = ["APIClient"]

cloe_nessy/clients/api_client/api_client.py

@@ -0,0 +1,188 @@
+from http import HTTPStatus
+from time import sleep
+from typing import Any
+from urllib.parse import urljoin
+
+import requests
+from requests.auth import AuthBase
+
+from .api_response import APIResponse
+from .exceptions import APIClientConnectionError, APIClientError, APIClientHTTPError, APIClientTimeoutError
+
+
+class APIClient:
+    """A standardized client for the interaction with APIs.
+
+    This class handles the communication with an API, including retries for specific status codes.
+
+    Attributes:
+        RETRY_CODES: List of HTTP status codes that should trigger a retry.
+        MAX_SLEEP_TIME: Maximum time to wait between retries, in seconds.
+        base_url: The base URL for the API.
+        session: The session object for making requests.
+    """
+
+    RETRY_CODES: list[int] = [
+        HTTPStatus.TOO_MANY_REQUESTS,
+        HTTPStatus.SERVICE_UNAVAILABLE,
+        HTTPStatus.GATEWAY_TIMEOUT,
+    ]
+
+    MAX_SLEEP_TIME: int = 180  # seconds
+
+    def __init__(
+        self,
+        base_url: str,
+        auth: AuthBase | None = None,
+        default_headers: dict[str, str] | None = None,
+    ):
+        """Initializes the APIClient object.
+
+        Args:
+            base_url: The base URL for the API.
+            auth: The authentication method for the API.
+            default_headers: Default headers to include in requests.
+        """
+        if not base_url.endswith("/"):
+            base_url += "/"
+        self.base_url = base_url
+        self.session = requests.Session()
+        if default_headers:
+            self.session.headers.update(default_headers)
+        self.session.auth = auth
+
+    def _make_request(
+        self,
+        method: str,
+        endpoint: str,
+        timeout: int = 30,
+        params: dict[str, str] | None = None,
+        data: dict[str, str] | None = None,
+        json: dict[str, str] | None = None,
+        headers: dict[str, str] | None = None,
+        max_retries: int = 0,
+    ) -> APIResponse:
+        """Makes a request to the API endpoint.
+
+        Args:
+            method: The HTTP method to use for the request.
+            endpoint: The endpoint to send the request to.
+            timeout: The timeout for the request in seconds.
+            params: The query parameters for the request.
+            data: The form data to include in the request.
+            json: The JSON data to include in the request.
+            headers: The headers to include in the request.
+            max_retries: The maximum number of retries for the request.
+
+        Returns:
+            APIResponse: The response from the API.
+
+        Raises:
+            APIClientError: If the request fails.
+        """
+        url = urljoin(self.base_url, endpoint.strip("/"))
+        params = params or {}
+        data = data or {}
+        json = json or {}
+        headers = headers or {}
+
+        for attempt in range(max_retries + 1):
+            try:
+                response = self.session.request(
+                    method=method,
+                    url=url,
+                    timeout=timeout,
+                    params=params,
+                    data=data,
+                    json=json,
+                    headers=headers,
+                )
+                if response.status_code not in APIClient.RETRY_CODES:
+                    response.raise_for_status()
+                    return APIResponse(response)
+            except requests.exceptions.HTTPError as err:
+                raise APIClientHTTPError(f"HTTP error occurred: {err}") from err
+            except requests.exceptions.ConnectionError as err:
+                if attempt < max_retries:
+                    sleep_time = min(2**attempt, APIClient.MAX_SLEEP_TIME)
+                    sleep(sleep_time)
+                    continue
+                raise APIClientConnectionError(f"Connection error occurred: {err}") from err
+            except requests.exceptions.Timeout as err:
+                raise APIClientTimeoutError(f"Timeout error occurred: {err}") from err
+            except requests.exceptions.RequestException as err:
+                raise APIClientError(f"An error occurred: {err}") from err
+        raise APIClientError(f"The maximum configured retries of [ '{max_retries}' ] have been exceeded")
+
+    def get(self, endpoint: str, **kwargs: Any) -> APIResponse:
+        """Sends a GET request to the specified endpoint.
+
+        Args:
+            endpoint: The endpoint to send the request to.
+            **kwargs: Additional arguments to pass to the request.
+
+        Returns:
+            APIResponse: The response from the API.
+        """
+        return self._make_request(method="GET", endpoint=endpoint, **kwargs)
+
+    def post(self, endpoint: str, **kwargs: Any) -> APIResponse:
+        """Sends a POST request to the specified endpoint.
+
+        Args:
+            endpoint: The endpoint to send the request to.
+            **kwargs: Additional arguments to pass to the request.
+
+        Returns:
+            APIResponse: The response from the API.
+        """
+        return self._make_request(method="POST", endpoint=endpoint, **kwargs)
+
+    def put(self, endpoint: str, **kwargs: Any) -> APIResponse:
+        """Sends a PUT request to the specified endpoint.
+
+        Args:
+            endpoint: The endpoint to send the request to.
+            **kwargs: Additional arguments to pass to the request.
+
+        Returns:
+            APIResponse: The response from the API.
+        """
+        return self._make_request(method="PUT", endpoint=endpoint, **kwargs)
+
+    def delete(self, endpoint: str, **kwargs: Any) -> APIResponse:
+        """Sends a DELETE request to the specified endpoint.
+
+        Args:
+            endpoint: The endpoint to send the request to.
+            **kwargs: Additional arguments to pass to the request.
+
+        Returns:
+            APIResponse: The response from the API.
+        """
+        return self._make_request(method="DELETE", endpoint=endpoint, **kwargs)
+
+    def patch(self, endpoint: str, **kwargs: Any) -> APIResponse:
+        """Sends a PATCH request to the specified endpoint.
+
+        Args:
+            endpoint: The endpoint to send the request to.
+            **kwargs: Additional arguments to pass to the request.
+
+        Returns:
+            APIResponse: The response from the API.
+        """
+        return self._make_request(method="PATCH", endpoint=endpoint, **kwargs)
+
+    def request(self, method: str, endpoint: str, **kwargs: Any) -> APIResponse:
+        """Sends a request to the specified endpoint with the specified method.
+
+        Args:
+            method: The HTTP method to use for the request.
+            endpoint: The endpoint to send the request to.
+            **kwargs: Additional arguments to pass to the request.
+
+        Returns:
+            APIResponse: The response from the API.
+        """
+        return self._make_request(method=method, endpoint=endpoint, **kwargs)

cloe_nessy/clients/api_client/api_response.py

@@ -0,0 +1,72 @@
+from typing import Any
+
+import requests
+
+from .exceptions import APIClientError
+
+
+class APIResponse:
+    """An abstracted response to implement parsing.
+
+    This class provides methods to parse the response from an API request.
+
+    Attributes:
+        response: The original response object.
+        headers: The headers of the response.
+        status_code: The status code of the response.
+        content_type: The content type of the response.
+    """
+
+    def __init__(self, response: requests.Response):
+        """Initializes the APIResponse object.
+
+        Args:
+            response: The response object from an API request.
+        """
+        self.response = response
+        self.headers = self.response.headers
+        self.status_code = self.response.status_code
+        self.url = self.response.url
+        self.reason = self.response.reason
+        self.elapsed = self.response.elapsed
+        self.content_type = self.headers.get("Content-Type", "").lower()
+
+    def to_dict(self, key: str | None = None) -> dict[str, Any]:
+        """Parses the values from the response into a dictionary.
+
+        Args:
+            key: The key to return from the dictionary. If specified, the method
+                will return the value associated with this key from the parsed dictionary.
+
+        Returns:
+            The response parsed to a dictionary. If a key is specified,
+                the method returns the value associated with this key.
+
+        Raises:
+            KeyError: If the specified key is not found in the response.
+            ValueError: If there is an error parsing the JSON response.
+            Exception: For any other unexpected errors.
+        """
+        dict_response = {}
+        try:
+            if "application/json" in self.content_type:
+                dict_response = self.response.json()
+            else:
+                # Handling of other response types can be added below.
+                dict_response = {"value": self.response.text}
+
+            if key:
+                dict_response = dict_response[key]
+        except KeyError as err:
+            raise KeyError(
+                f"The key '{err.args[0]}' was not found in the response. Status code: {self.status_code}, Headers: {self.headers}, Response: {dict_response}"
+            ) from err
+        except ValueError as err:
+            raise ValueError(
+                f"Error parsing JSON response: {err}. Status code: {self.status_code}, Headers: {self.headers}, Response content: {self.response.text}"
+            ) from err
+        except Exception as err:
+            raise APIClientError(
+                f"An unexpected error occurred: {err}. Status code: {self.status_code}, Headers: {self.headers}, Response content: {self.response.text}"
+            ) from err
+        return dict_response

cloe_nessy/clients/api_client/auth.py

@@ -0,0 +1,178 @@
+import os
+import time
+from typing import Any
+
+from azure.core.credentials import TokenCredential
+from azure.identity import ClientSecretCredential
+from requests import PreparedRequest
+from requests.auth import AuthBase
+
+from ...session import SessionManager
+
+
+class AzureCredentialAuth(AuthBase):
+    """This Auth can be used with requests and an Azure Credential."""
+
+    def __init__(
+        self,
+        scope: str,
+        credential: TokenCredential | ClientSecretCredential | None = None,
+        client_id: str | None = None,
+        client_secret: str | None = None,
+        tenant_id: str | None = None,
+    ):
+        """Initializes the AzureCredentialAuth with an Azure credential.
+
+        The client can either be initialized with a TokenCredential object or with the client_id, client_secret, and tenant_id via an ClientSecretCredential.
+
+        Args:
+            scope: The scope for the token. E.g., the client ID of the Azure AD application.
+            credential: The Azure credential object.
+            client_id: The client ID for the Azure AD application.
+            client_secret: The client secret for the Azure AD application.
+            tenant_id: The tenant ID for the Azure AD application.
+        """
+        if credential is None:
+            if client_id is None or client_secret is None or tenant_id is None:
+                raise ValueError("Either a credential or client_id, client_secret, and tenant_id must be provided.")
+            credential = ClientSecretCredential(
+                tenant_id=tenant_id,
+                client_id=client_id,
+                client_secret=client_secret,
+            )
+        self.credential = credential
+        self.scope = scope
+        self._token = None
+
+    @property
+    def token(self):
+        """Get a valid token using the TokenCredential."""
+        if self._token is None or self._token.expires_on < (int(time.time()) + 5):
+            self._token = self.credential.get_token(self.scope)
+        return self._token.token
+
+    def __call__(self, r: PreparedRequest) -> PreparedRequest:
+        """Appends an Authorization header to the request using the provided Azure credential.
+
+        Args:
+            r (PreparedRequest): The request that needs to be sent.
+
+        Returns:
+            PreparedRequest: The same request object but with an added Authorization header.
+        """
+        r.headers["Authorization"] = f"Bearer {self.token}"
+        return r
+
+
+class SecretScopeAuth(AuthBase):
+    """This Auth pulls Secrets from a Secret Scope."""
+
+    def __init__(self, header_template: dict[str, str], secret_scope: str):
+        """Initializes the SecretScopeAuth with a header template, secret scope, and secret key.
+
+        Args:
+            header_template: The template for the header that will use the secret.
+                                   secret names are defined as placeholders in curly braces.
+            secret_scope: The secret scope from where the secrets will be retrieved.
+
+        Example:
+        ```python
+        header_template = {
+            "jfrog-user-key": "jfrog-user",
+            "jfrog-password-key": "jfrog-secret",
+        }
+        auth = SecretScopeAuth(header_template, "my_secret_scope")
+        # given, that 'jfrog-user' and 'jfrog-secret' are secrets in 'my_secret_scope'
+        ```
+        """
+        self.header_template = header_template
+        self.secret_scope = secret_scope
+
+    def __call__(self, r: PreparedRequest) -> PreparedRequest:
+        """The header is constructed using the template and the secret retrieved from the secret scope.
+
+        Args:
+            r: The request that needs to be sent.
+
+        Returns:
+            PreparedRequest: The same request object, but with an added header. The header
+                             is constructed using the template and the secret retrieved from
+                             the secret scope.
+        """
+        utils = SessionManager.get_utils()
+        auth_header = {key: utils.secrets.get(self.secret_scope, ref) for key, ref in self.header_template.items()}
+        r.headers.update(auth_header)
+        return r
+
+
+class ChainedAuth(AuthBase):
+    """This Auth can be used to chain multiple Auths."""
+
+    def __init__(self, *args: Any):
+        """Initializes the ChainedAuth.
+
+        Args:
+            *args: One or more Auth objects that are chained to
+                              construct the auth header.
+
+        Example:
+        ```python
+        auth_1 = SecretScopeAuth({"secret": "key"}, "my_secret_scope")
+        auth_2 = SecretScopeAuth({"secret": "key"}, "my_other_secret_scope")
+        chained_auth = ChainedAuth(auth_1, auth_2)
+        ```
+        """
+        self.auths = list(args)
+
+    def __call__(self, r: PreparedRequest) -> PreparedRequest:
+        """The header is constructed using the template and the secret retrieved from the secret scope.
+
+        Args:
+            r: The request that needs to be sent.
+
+        Returns:
+            PreparedRequest: The same request object, but with an added header. The header
+                                is constructed using the template and the secret retrieved from
+                                the secret scope.
+        """
+        for auth in self.auths:
+            r = auth(r)
+        return r
+
+
+class EnvVariableAuth(AuthBase):
+    """This Auth can be used to create an auth header from environment variables."""
+
+    def __init__(self, header_template: dict[str, str]):
+        """Initializes the EnvVariableAuth with a header template.
+
+        Args:
+            header_template: The template for the header that will use the environment variables.
+                                   variable names are defined as placeholders.
+
+        Example:
+        ```python
+        header_template = {
+            "user": "USER_NAME",
+            "password": "USER_SECRET",
+        }
+        auth = EnvVariableAuth(header_template)
+        # given, that "USER_NAME" and "USER_SECRET" are environment variables
+        ```
+        """
+        self.header_template = header_template
+
+    def __call__(self, r: PreparedRequest) -> PreparedRequest:
+        """The header is constructed using the template and the secret retrieved from the secret scope.
+
+        Args:
+            r: The request that needs to be sent.
+
+        Returns:
+            PreparedRequest: The same request object, but with an added header. The header
+                             is constructed using the template and the secret retrieved from
+                             environment variables.
+        """
+        auth_header = {key: os.environ.get(value, "") for key, value in self.header_template.items()}
+        r.headers.update(auth_header)
+        return r

cloe_nessy/clients/api_client/exceptions.py

@@ -0,0 +1,22 @@
+class APIClientError(Exception):
+    """Base class for API client exceptions."""
+
+    pass
+
+
+class APIClientHTTPError(APIClientError):
+    """Exception raised for HTTP errors."""
+
+    pass
+
+
+class APIClientConnectionError(APIClientError):
+    """Exception raised for connection errors."""
+
+    pass
+
+
+class APIClientTimeoutError(APIClientError):
+    """Exception raised for timeout errors."""
+
+    pass

cloe_nessy/file_utilities/__init__.py

@@ -0,0 +1,3 @@
+from .get_file_paths import get_file_paths
+
+__all__ = ["get_file_paths"]

cloe_nessy/file_utilities/exceptions.py

@@ -0,0 +1,4 @@
+class FileUtilitiesError(Exception):
+    """Base class for file utility exceptions."""
+
+    pass

cloe_nessy/file_utilities/factory.py

@@ -0,0 +1,42 @@
+from .location_types import LocationType
+from .strategies.local_strategy import LocalDirectoryStrategy
+from .strategies.onelake_strategy import OneLakeStrategy
+from .strategies.utils_strategy import UtilsStrategy
+
+
+class FileRetrievalFactory:
+    """Factory for creating file retrieval strategies based on location type.
+
+    This factory class is responsible for returning the appropriate strategy
+    implementation for retrieving files based on the specified location type.
+    """
+
+    _strategy_map = {
+        LocationType.LOCAL: LocalDirectoryStrategy,
+        LocationType.VOLUME: LocalDirectoryStrategy,
+        LocationType.ABFS: UtilsStrategy,
+        LocationType.S3: UtilsStrategy,
+        LocationType.ONELAKE: OneLakeStrategy,
+    }
+
+    @staticmethod
+    def get_strategy(location_type: LocationType) -> LocalDirectoryStrategy | OneLakeStrategy | UtilsStrategy:
+        """Returns the appropriate file retrieval strategy for the given location type.
+
+        Depending on the provided location type, this method returns an instance
+        of either `LocalDirectoryStrategy` or `UtilsStrategy`. If the
+        location type is not recognized, a `ValueError` is raised.
+
+        Args:
+            location_type: The location type for which to get the retrieval strategy.
+
+        Returns:
+            FileRetrievalStrategy: An instance of the appropriate file retrieval strategy.
+
+        Raises:
+            ValueError: If the provided location type is unknown or unsupported.
+        """
+        strategy_class = FileRetrievalFactory._strategy_map.get(location_type)
+        if not strategy_class:
+            raise ValueError(f"Unknown location type: {location_type}")
+        return strategy_class()  # type: ignore

cloe_nessy/file_utilities/get_file_paths.py

@@ -0,0 +1,72 @@
+import os
+
+from ..logging.logger_mixin import LoggerMixin
+from .factory import FileRetrievalFactory
+from .location_types import LocationType
+
+
+def get_file_paths(location: str, file_name_pattern: str | None = None, search_subdirs: bool = True) -> list[str]:
+    """Retrieves file paths from a specified location based on the provided criteria.
+
+    This function determines the type of location (e.g., local directory, blob storage),
+    retrieves the appropriate file retrieval strategy using a factory, and then uses
+    that strategy to get a list of file paths that match the given file_name_pattern and search options.
+
+    Args:
+        location: The location to search for files. This could be a path to a local directory or a URI for blob storage.
+        file_name_pattern: The file file_name_pattern to filter by as string. None retrieves all files regardless of file_name_pattern.
+        search_subdirs: Whether to include files from subdirectories in the search.
+
+    Returns:
+        A list of file paths that match the specified criteria. The paths are returned as strings.
+
+    Raises:
+        ValueError: If the `location` argument is empty or None.
+        FileUtilitiesError: If an error occurs while determining the location type, retrieving the strategy, or getting file paths.
+    """
+    logger = LoggerMixin().get_console_logger()
+    if not location:
+        raise ValueError("location is required")
+
+    logger.debug("location", location)
+    logger.debug("Getting location type")
+    location_type = get_location_type(location=location)
+    logger.debug("location_type", location_type)
+    strategy = FileRetrievalFactory.get_strategy(location_type)
+    logger.debug("strategy", strategy)
+    logger.info(
+        f"Retrieving file paths from location [  '{location}'  ] with strategy [  '{strategy.__class__.__name__}'  ]"
+    )
+    paths = strategy.get_file_paths(location, file_name_pattern, search_subdirs)
+    logger.debug("paths:", paths)
+    return paths
+
+
+def get_location_type(location: str) -> LocationType:
+    """Get the location type based on the given location string.
+
+    Args:
+        location: The location string to check.
+
+    Returns:
+        LocationType: The determined location type.
+    """
+    location_mapping = {
+        "abfss://": LocationType.ABFS,
+        "/Volumes/": LocationType.VOLUME,
+        "s3://": LocationType.S3,
+        "/lakehouse/default/": LocationType.ONELAKE,
+    }
+
+    for prefix, loc_type in location_mapping.items():
+        if location.startswith(prefix):
+            return loc_type
+
+    if os.path.isdir(location):
+        return LocationType.LOCAL
+
+    raise NotImplementedError(
+        f"Could not determine Location type of location [  '{location}'  ]."
+        f"Ensure that the provided path is valid."
+        f"Available Location type implementations are: [  {', '.join(LocationType.list())}  ].",
+    )

cloe_nessy/file_utilities/location_types.py

@@ -0,0 +1,29 @@
+from enum import Enum
+
+
+class LocationType(Enum):
+    """Enum representing different types of locations.
+
+    Attributes:
+        LOCAL: Represents a local location.
+        VOLUME: Represents a volume location.
+        ABFS: Represents an Azure Blob File System (ABFS) location.
+        ONELAKE: Represents a OneLake location.
+    """
+
+    LOCAL = "local"
+    VOLUME = "volumes"
+    ABFS = "abfs"
+    S3 = "s3"
+    ONELAKE = "onelake"
+
+    @staticmethod
+    def list() -> list[str]:
+        """Returns a list of all location type values.
+
+        This method provides a list of strings, each representing a location type.
+
+        Returns:
+            list of str: A list of all the values of the LocationType enum.
+        """
+        return list(map(lambda location: location.value, LocationType))