frogml-core 0.0.1__py3-none-any.whl

frogml/inner/runtime_di/__init__.py

@@ -0,0 +1,13 @@
+from frogml.inner.runtime_di.containers import ContainerLock, FrogmlRuntimeContainer
+
+
+def wire_runtime():
+    container = FrogmlRuntimeContainer()
+    from frogml.model import decorators
+
+    container.wire(
+        packages=[
+            decorators,
+        ]
+    )
+    return container

frogml/inner/runtime_di/containers.py

@@ -0,0 +1,17 @@
+from dependency_injector import containers, providers
+
+from frogml.model.decorators.impl.api_implementation import create_decorator_function
+from frogml.model.decorators.impl.timer_implementation import create_frogml_timer
+
+
+class FrogmlRuntimeContainer(containers.DeclarativeContainer):
+    """
+    Frogml Core's Runtime Dependency Injection Container
+    """
+
+    api_decorator_function_creator = providers.Object(create_decorator_function)
+    timer_function_creator = providers.Object(create_frogml_timer)
+
+
+class ContainerLock:
+    locked: bool = False

frogml/inner/singleton_meta.py

@@ -0,0 +1,18 @@
+class SingletonMeta(type):
+    """
+    The Singleton class can be implemented in different ways in Python. Some
+    possible methods include: base class, decorator, metaclass. We will use the
+    metaclass because it is best suited for this purpose.
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        """
+        Possible changes to the value of the `__init__` argument do not affect
+        the returned instance.
+        """
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]

frogml/inner/tool/__init__.py

@@ -0,0 +1,2 @@
+from .auth import FrogMLAuthClient
+from .grpc.grpc_auth import FrogMLAuthMetadataPlugin

frogml/inner/tool/auth.py

@@ -0,0 +1,70 @@
+from typing import Dict, Optional, Union, cast
+
+import requests
+from frogml_storage._utils import BearerAuth
+from frogml_storage.authentication._authentication_utils import get_credentials
+from frogml_storage.authentication.models._auth_config import AuthConfig
+from requests.auth import AuthBase, HTTPBasicAuth
+
+from frogml.exceptions import FrogmlLoginException
+
+
+class FrogMLAuthClient:
+    def __init__(self, auth_config: Optional[AuthConfig] = None):
+        self.auth_config: Optional[AuthConfig] = auth_config
+        self._token: Optional[str] = None
+        self._tenant_id: Optional[str] = None
+
+    def get_token(self) -> str:
+        if not self._token:
+            self.login()
+
+        return cast(str, self._token)
+
+    def get_tenant_id(self) -> str:
+        if not self._tenant_id:
+            self.login()
+
+        return cast(str, self._tenant_id)
+
+    def login(self):
+        artifactory_url, auth = get_credentials(self.auth_config)
+
+        if isinstance(auth, BearerAuth):  # For BearerAuth
+            self._token = auth.token
+
+        self.__get_tenant_id(artifactory_url, auth)
+
+    def __get_tenant_id(self, artifactory_url: str, auth: AuthBase):
+        login_exception = FrogmlLoginException(
+            "Failed to authenticate with JFrog. Please check your credentials"
+        )
+
+        # Remove '/artifactory/' from the URL
+        if "/artifactory" in artifactory_url:
+            base_url = artifactory_url.replace("/artifactory", "/ui")
+        else:
+            # Remove trailing slash if it exists and append /ui
+            base_url = artifactory_url.rstrip("/") + "/ui"
+
+        url = f"{base_url}/api/v1/system/auth/screen/footer"
+        kwargs: Dict[str, Union[HTTPBasicAuth, str]] = {}
+
+        if isinstance(auth, HTTPBasicAuth):
+            kwargs["auth"] = auth
+        elif isinstance(auth, BearerAuth):
+            kwargs["headers"] = {"Authorization": f"Bearer {auth.token}"}
+        else:
+            raise login_exception
+
+        try:
+            response = requests.get(url, timeout=60, **kwargs)
+            response.raise_for_status()  # Raises an HTTPError for bad responses
+            response_data = response.json()
+
+            if "serverId" not in response_data:
+                raise login_exception
+            self._tenant_id = response_data["serverId"]
+
+        except (requests.exceptions.RequestException, ValueError) as exc:
+            raise login_exception from exc

frogml/inner/tool/grpc/grpc_auth.py

@@ -0,0 +1,42 @@
+from typing import TYPE_CHECKING, Callable, Optional, Tuple
+
+import grpc
+
+from frogml.inner.const import FrogMLConstants
+
+if TYPE_CHECKING:
+    from frogml.inner.tool.auth import FrogMLAuthClient
+
+_SIGNATURE_HEADER_KEY = "authorization"
+
+
+class FrogMLAuthMetadataPlugin(grpc.AuthMetadataPlugin):
+    def __init__(self):
+        self._auth_client: Optional["FrogMLAuthClient"] = None
+
+    def __call__(
+        self,
+        context: grpc.AuthMetadataContext,
+        callback: Callable[[Tuple[Tuple[str, str], Tuple[str, str]], None], None],
+    ):
+        """Implements authentication by passing metadata to a callback.
+
+        Args:
+            context: An AuthMetadataContext providing information on the RPC that
+                the plugin is being called to authenticate.
+            callback: A callback that accepts a tuple of metadata key/value pairs and a None
+                parameter.
+        """
+        # Get token from FrogML client
+        if not self._auth_client:
+            from frogml.inner.tool.auth import FrogMLAuthClient
+
+            self._auth_client = FrogMLAuthClient()
+
+        token = self._auth_client.get_token()
+        jfrog_tenant_id = self._auth_client.get_tenant_id()
+        metadata = (
+            (_SIGNATURE_HEADER_KEY, f"Bearer {token}"),
+            (FrogMLConstants.JFROG_TENANT_HEADER_KEY.lower(), jfrog_tenant_id),
+        )
+        callback(metadata, None)

frogml/inner/tool/grpc/grpc_tools.py

@@ -0,0 +1,209 @@
+import logging
+import time
+from abc import ABC, abstractmethod
+from random import randint
+from typing import Callable, Optional, Tuple
+
+import grpc
+
+from frogml.exceptions import FrogmlException
+from frogml.inner.tool.grpc.grpc_auth import FrogMLAuthMetadataPlugin
+
+logger = logging.getLogger()
+
+
+def create_grpc_channel(
+    url: str,
+    enable_ssl: bool = True,
+    enable_auth: bool = True,
+    auth_metadata_plugin: grpc.AuthMetadataPlugin = None,
+    timeout: int = 100,
+    options=None,
+    backoff_options=None,
+    max_attempts=4,
+    status_for_retry=(grpc.StatusCode.UNAVAILABLE,),
+    attempt=0,
+) -> grpc.Channel:
+    """
+    Create a gRPC channel
+    Args:
+        url: gRPC URL to connect to
+        enable_ssl: Enable TLS/SSL, optionally provide a server side certificate
+        enable_auth: Enable user auth
+        auth_metadata_plugin: Metadata plugin to use to sign requests, only used
+            with "enable auth" when SSL/TLS is enabled
+        timeout: Connection timeout to server
+        options:  An optional list of key-value pairs (channel_arguments in gRPC Core runtime) to configure the channel.
+        backoff_options: dictionary - init_backoff_ms: default 50, max_backoff_ms: default 500, multiplier: default 2
+        max_attempts: max number of retry attempts
+        attempt: current retry attempts
+        status_for_retry: grpc statuses to retry upon
+    Returns: Returns a grpc.Channel
+    """
+    if backoff_options is None:
+        backoff_options = {}
+
+    if not url:
+        raise FrogmlException(
+            "Unable to create gRPC channel. URL has not been defined."
+        )
+
+    if enable_ssl or url.endswith(":443"):
+        credentials: grpc.ChannelCredentials = grpc.ssl_channel_credentials()
+
+        if enable_auth:
+            if auth_metadata_plugin is None:
+                auth_metadata_plugin = FrogMLAuthMetadataPlugin()
+
+            credentials = grpc.composite_channel_credentials(
+                credentials, grpc.metadata_call_credentials(auth_metadata_plugin)
+            )
+
+        channel: grpc.Channel = grpc.secure_channel(
+            url, credentials=credentials, options=options
+        )
+    else:
+        channel = grpc.insecure_channel(url, options=options)
+    try:
+        interceptors: Tuple[RetryOnRpcErrorClientInterceptor] = (
+            RetryOnRpcErrorClientInterceptor(
+                max_attempts=max_attempts,
+                sleeping_policy=ExponentialBackoff(**backoff_options),
+                status_for_retry=status_for_retry,
+            ),
+        )
+
+        intercept_channel: grpc.Channel = grpc.intercept_channel(channel, *interceptors)
+        try:
+            grpc.channel_ready_future(intercept_channel).result(timeout=timeout)
+        except (grpc.FutureTimeoutError, grpc.RpcError, Exception) as e:
+            logger.debug(
+                f"Received error: {repr(e)} attempt #{attempt + 1} of {max_attempts}"
+            )
+            if attempt < max_attempts:
+                return create_grpc_channel(
+                    url,
+                    enable_ssl,
+                    enable_auth,
+                    auth_metadata_plugin,
+                    timeout,
+                    options,
+                    backoff_options,
+                    max_attempts,
+                    status_for_retry,
+                    attempt + 1,
+                )
+            else:
+                raise e
+
+        return intercept_channel
+    except grpc.FutureTimeoutError as e:
+        raise FrogmlException(
+            f"Connection timed out while attempting to connect to {url}, with: {repr(e)}"
+        ) from e
+
+
+def create_grpc_channel_or_none(
+    url: str,
+    enable_ssl: bool = True,
+    enable_auth: bool = True,
+    auth_metadata_plugin: grpc.AuthMetadataPlugin = None,
+    timeout: int = 30,
+    options=None,
+    backoff_options=None,
+    max_attempts=2,
+    status_for_retry=(grpc.StatusCode.UNAVAILABLE,),
+    attempt=0,
+) -> Callable[[Optional[str], Optional[bool]], Optional[grpc.Channel]]:
+    final_backoff_options = backoff_options if backoff_options else {}
+
+    def deferred_channel(
+        url_overwrite: Optional[str] = None, ssl_overwrite: Optional[bool] = None
+    ):
+        try:
+            return create_grpc_channel(
+                url_overwrite if url_overwrite else url,
+                ssl_overwrite if ssl_overwrite else enable_ssl,
+                enable_auth,
+                auth_metadata_plugin,
+                timeout,
+                options,
+                final_backoff_options,
+                max_attempts,
+                status_for_retry,
+                attempt,
+            )
+        except Exception as e:
+            logger.debug(f"create_grpc_channel error: {repr(e)}")
+            return None
+
+    return deferred_channel
+
+
+class SleepingPolicy(ABC):
+    @abstractmethod
+    def sleep(self, try_i: int):
+        """
+        How long to sleep in milliseconds.
+        :param try_i: the number of retry (starting from zero)
+        """
+        if try_i < 0:
+            raise ValueError("Number of retries must be non-negative.")
+
+
+class ExponentialBackoff(SleepingPolicy):
+    def __init__(
+        self,
+        *,
+        init_backoff_ms: int = 50,
+        max_backoff_ms: int = 5000,
+        multiplier: int = 2,
+    ):
+        self.init_backoff = init_backoff_ms
+        self.max_backoff = max_backoff_ms
+        self.multiplier = multiplier
+
+    def sleep(self, try_i: int):
+        sleep_time = min(self.init_backoff * self.multiplier**try_i, self.max_backoff)
+        sleep_ms = sleep_time + randint(0, self.init_backoff)  # nosec B311
+        logger.debug("ExponentialBackoff - Sleeping between retries")
+        time.sleep(sleep_ms / 1000)
+
+
+class RetryOnRpcErrorClientInterceptor(grpc.UnaryUnaryClientInterceptor):
+    def __init__(
+        self,
+        *,
+        max_attempts: int,
+        sleeping_policy: SleepingPolicy,
+        status_for_retry: Optional[Tuple[grpc.StatusCode]] = None,
+    ):
+        self.max_attempts = max_attempts
+        self.sleeping_policy = sleeping_policy
+        self.status_for_retry = status_for_retry
+
+    def _intercept_call(self, continuation, client_call_details, request_or_iterator):
+        for try_i in range(self.max_attempts):
+            response = continuation(client_call_details, request_or_iterator)
+
+            if isinstance(response, grpc.RpcError):
+                # Return if it was last attempt
+                if try_i == (self.max_attempts - 1):
+                    return response
+
+                # If status code is not in retryable status codes
+                if (
+                    self.status_for_retry
+                    and response.code() not in self.status_for_retry
+                ):
+                    return response
+                logger.debug(
+                    f"Retry GRPC call attempt #{try_i} after status {response.code()}"
+                )
+                logger.debug(f"Client call details: {client_call_details}")
+                self.sleeping_policy.sleep(try_i)
+            else:
+                return response
+
+    def intercept_unary_unary(self, continuation, client_call_details, request):
+        return self._intercept_call(continuation, client_call_details, request)

frogml/inner/tool/grpc/grpc_try_wrapping.py

@@ -0,0 +1,16 @@
+import grpc
+
+from frogml.exceptions import FrogmlException
+
+
+def grpc_try_catch_wrapper(exception_message):
+    def decorator(function):
+        def _inner_wrapper(*args, **kwargs):
+            try:
+                return function(*args, **kwargs)
+            except grpc.RpcError as e:
+                raise FrogmlException(exception_message + f". Error is: {e.details()}.")
+
+        return _inner_wrapper
+
+    return decorator

frogml/inner/tool/protobuf_factory.py

@@ -0,0 +1,45 @@
+from frogml.exceptions import FrogmlException
+
+
+def protobuf_factory(
+    protobuf_class, exclude_fields=None, include_only=None, mapping=None
+):
+    def get_final_field_names(cls):
+        fields = dict(cls.__dataclass_fields__)
+        all_field_names = set(fields.keys())
+        excluded_fields = set(exclude_fields) if exclude_fields else set()
+        included_fields = set(include_only) if include_only else all_field_names
+        unrecognized_fields = included_fields.difference(all_field_names)
+        if unrecognized_fields:
+            raise FrogmlException(
+                f"Unknown fields were included: {list(unrecognized_fields)}"
+            )
+        return included_fields.difference(excluded_fields)
+
+    def protofy_decorator(cls):
+        if not hasattr(cls, "__dataclass_fields__"):
+            raise FrogmlException("Protofy must receive a dataclass")
+
+        fields_mapping = mapping if mapping else {}
+        final_field_names = get_final_field_names(cls)
+
+        def to_proto(self) -> protobuf_class:
+            final_fields = {}
+            for field_name in final_field_names:
+                field_val = self.__getattribute__(field_name)
+                field_name = (
+                    fields_mapping[field_name]
+                    if field_name in fields_mapping
+                    else field_name
+                )
+                final_fields[field_name] = (
+                    field_val.to_proto()
+                    if hasattr(field_val, "to_proto")
+                    else field_val
+                )
+            return protobuf_class(**final_fields)
+
+        setattr(cls, "to_proto", to_proto)
+        return cls
+
+    return protofy_decorator

frogml/inner/tool/retry_utils.py

@@ -0,0 +1,23 @@
+import time
+
+
+def retry(
+    f,
+    args=None,
+    kwargs=None,
+    exceptions=Exception,
+    attempts=-1,
+    delay=0,
+):
+    _attempts = attempts
+    args = args if args else list()
+    kwargs = kwargs if kwargs else dict()
+    while _attempts:
+        try:
+            return f(*args, **kwargs)
+        except exceptions:
+            _attempts -= 1
+            if not _attempts:
+                raise
+
+            time.sleep(delay)

frogml/inner/tool/run_config/__init__.py

@@ -0,0 +1,12 @@
+from .base import FrogmlConfigBase, YamlConfigMixin
+from .utils import (
+    ConfigCliMap,
+    validate_bool,
+    validate_enum,
+    validate_float,
+    validate_int,
+    validate_list_of_strings,
+    validate_positive_int,
+    validate_purchase_option,
+    validate_string,
+)

frogml/inner/tool/run_config/base.py

@@ -0,0 +1,127 @@
+from abc import ABC, abstractmethod
+from logging import Logger
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from marshmallow import RAISE, Schema
+from marshmallow_dataclass import class_schema
+from yaml import SafeLoader, dump, load
+
+from frogml.exceptions import FrogmlException, LoadConfigurationException
+
+from .utils import ConfigCliMap, rgetattr, rsetattr
+
+ERROR_OPENING_YAML_FILE_MSG = "Error opening the config file"
+ERROR_PARSING_YAML_FILE_MSG = "Error parsing the config file's yaml"
+ERROR_IN_CONFIG_MSG = "Error loading the config"
+
+
+class FrogmlConfigBase(ABC):
+    """Base frogml config object."""
+
+    @property
+    @abstractmethod
+    def _config_mapping(self) -> List[ConfigCliMap]:
+        """Config mapping, Return a list of ConfigCliMap object in order to create the mapping.
+
+        Returns:
+            list: List of ConfigCliMap to apply.
+        """
+        pass
+
+    def merge_cli_argument(
+        self, sections: Tuple[str, ...] = (), **kwargs: Dict[str, Any]
+    ):
+        """Merge and validate cli arguments by supplied mapping.
+
+        Args:
+            sections: Sections to validate.
+            **kwargs: argument from cli.
+
+        Raises:
+            FrogmlException: In case that the argument is not valid.
+        """
+        for prop_map in self._config_mapping:
+            value = kwargs.get(prop_map.key)
+            if value is not None:
+                if isinstance(value, (list, tuple)):
+                    new_value = list(rgetattr(self, prop_map.prop))
+                    new_value.extend(value)
+                    value = new_value
+                rsetattr(self, prop_map.prop, value)
+            if (
+                not sections
+                or any(
+                    list(
+                        map(
+                            lambda section, _prop_map=prop_map: _prop_map.prop.startswith(
+                                section
+                            ),
+                            sections,
+                        )
+                    )
+                )
+                or "." not in prop_map.prop
+            ):
+                config_value = rgetattr(self, prop_map.prop)
+                if not prop_map.validation_func(config_value, prop_map.is_required):
+                    raise FrogmlException(
+                        f"{prop_map.key} argument contain invalid argument: "
+                        f"{value or config_value}"
+                    )
+        self._post_merge_cli()
+
+    @abstractmethod
+    def _post_merge_cli(self):
+        """Actions to perform after merging cli argument in to properties"""
+        pass
+
+
+class YamlConfigMixin(object):
+    @classmethod
+    def from_yaml(
+        cls, yaml_path: str, unknown: str = RAISE, logger: Optional[Logger] = None
+    ) -> Any:
+        """Create instance of class from yaml and class scheme.
+
+        Args:
+            yaml_path (str): Yaml path.
+
+        Returns:
+            object: Instance of created class.
+        """
+        if not yaml_path:
+            return cls()
+        Schema.Meta.unknown = unknown
+        schema = class_schema(cls)
+        try:
+            yaml_content = Path(yaml_path).read_text()
+        except Exception as e:
+            if logger:
+                logger.error(f"Error opening the config file {yaml_path}")
+            raise LoadConfigurationException(ERROR_OPENING_YAML_FILE_MSG) from e
+
+        try:
+            yaml_parsed = load(stream=yaml_content, Loader=SafeLoader)
+        except Exception as e:
+            if logger:
+                logger.exception(ERROR_PARSING_YAML_FILE_MSG, e)
+            raise LoadConfigurationException(ERROR_PARSING_YAML_FILE_MSG) from e
+
+        try:
+            return schema(unknown=unknown).load(yaml_parsed)
+        except Exception as e:
+            if logger:
+                logger.exception(ERROR_IN_CONFIG_MSG, e)
+            raise LoadConfigurationException(ERROR_IN_CONFIG_MSG) from e
+
+    def to_yaml(self) -> str:
+        """Convert class by scheme to yaml.
+
+        Returns:
+            str: Class as yaml string by scheme.
+        """
+        loaded_type = type(self)
+        schema = class_schema(loaded_type)
+
+        return dump(schema().dump(self))