frogml-core 0.0.116__py3-none-any.whl → 0.0.117__py3-none-any.whl

frogml_core/inner/tool/run_config/base.py

@@ -1,127 +0,0 @@
-from abc import ABC, abstractmethod
-from logging import Logger
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-from marshmallow import RAISE, Schema
-from marshmallow_dataclass import class_schema
-from yaml import SafeLoader, dump, load
-
-from frogml_core.exceptions import FrogmlException, LoadConfigurationException
-
-from .utils import ConfigCliMap, rgetattr, rsetattr
-
-ERROR_OPENING_YAML_FILE_MSG = "Error opening the config file"
-ERROR_PARSING_YAML_FILE_MSG = "Error parsing the config file's yaml"
-ERROR_IN_CONFIG_MSG = "Error loading the config"
-
-
-class FrogmlConfigBase(ABC):
-    """Base frogml config object."""
-
-    @property
-    @abstractmethod
-    def _config_mapping(self) -> List[ConfigCliMap]:
-        """Config mapping, Return a list of ConfigCliMap object in order to create the mapping.
-
-        Returns:
-            list: List of ConfigCliMap to apply.
-        """
-        pass
-
-    def merge_cli_argument(
-        self, sections: Tuple[str, ...] = (), **kwargs: Dict[str, Any]
-    ):
-        """Merge and validate cli arguments by supplied mapping.
-
-        Args:
-            sections: Sections to validate.
-            **kwargs: argument from cli.
-
-        Raises:
-            FrogmlException: In case that the argument is not valid.
-        """
-        for prop_map in self._config_mapping:
-            value = kwargs.get(prop_map.key)
-            if value is not None:
-                if isinstance(value, (list, tuple)):
-                    new_value = list(rgetattr(self, prop_map.prop))
-                    new_value.extend(value)
-                    value = new_value
-                rsetattr(self, prop_map.prop, value)
-            if (
-                not sections
-                or any(
-                    list(
-                        map(
-                            lambda section, _prop_map=prop_map: _prop_map.prop.startswith(
-                                section
-                            ),
-                            sections,
-                        )
-                    )
-                )
-                or "." not in prop_map.prop
-            ):
-                config_value = rgetattr(self, prop_map.prop)
-                if not prop_map.validation_func(config_value, prop_map.is_required):
-                    raise FrogmlException(
-                        f"{prop_map.key} argument contain invalid argument: "
-                        f"{value or config_value}"
-                    )
-        self._post_merge_cli()
-
-    @abstractmethod
-    def _post_merge_cli(self):
-        """Actions to perform after merging cli argument in to properties"""
-        pass
-
-
-class YamlConfigMixin(object):
-    @classmethod
-    def from_yaml(
-        cls, yaml_path: str, unknown: str = RAISE, logger: Optional[Logger] = None
-    ) -> Any:
-        """Create instance of class from yaml and class scheme.
-
-        Args:
-            yaml_path (str): Yaml path.
-
-        Returns:
-            object: Instance of created class.
-        """
-        if not yaml_path:
-            return cls()
-        Schema.Meta.unknown = unknown
-        schema = class_schema(cls)
-        try:
-            yaml_content = Path(yaml_path).read_text()
-        except Exception as e:
-            if logger:
-                logger.error(f"Error opening the config file {yaml_path}")
-            raise LoadConfigurationException(ERROR_OPENING_YAML_FILE_MSG) from e
-
-        try:
-            yaml_parsed = load(stream=yaml_content, Loader=SafeLoader)
-        except Exception as e:
-            if logger:
-                logger.exception(ERROR_PARSING_YAML_FILE_MSG, e)
-            raise LoadConfigurationException(ERROR_PARSING_YAML_FILE_MSG) from e
-
-        try:
-            return schema(unknown=unknown).load(yaml_parsed)
-        except Exception as e:
-            if logger:
-                logger.exception(ERROR_IN_CONFIG_MSG, e)
-            raise LoadConfigurationException(ERROR_IN_CONFIG_MSG) from e
-
-    def to_yaml(self) -> str:
-        """Convert class by scheme to yaml.
-
-        Returns:
-            str: Class as yaml string by scheme.
-        """
-        loaded_type = type(self)
-        schema = class_schema(loaded_type)
-
-        return dump(schema().dump(self))

frogml_core/model/_entity_extraction.py

@@ -1,116 +0,0 @@
-import dataclasses
-from typing import Dict, List, cast
-
-from frogml_core.exceptions import FrogmlException
-from frogml_core.feature_store._common.feature_set_utils import (
-    FeatureSetInfo,
-    get_entity_type,
-    get_typed_feature,
-)
-from frogml_core.model.fs_info_mapping_retriever import retrieve_fs_mapping
-from frogml_core.model.schema_entities import BaseFeature, Entity, FeatureStoreInput
-from frogml_core.model.utils.feature_utils import (
-    discard_env_from_name,
-    extract_env_name,
-    extract_featureset_name,
-    validate_and_sanitize_features_name,
-)
-
-
-def normalize_features(
-    features: List[BaseFeature],
-    env_to_feature_sets_infos: Dict[str, Dict[str, FeatureSetInfo]],
-) -> List[BaseFeature]:
-    """
-    Adding the relevant entity to the feature set features
-    Args:
-        features: list of features
-        env_to_feature_sets_infos: dict of environment name to feature set name to it's entity
-    return normalize features - the features with the entities
-    :param
-    """
-    new_normalize_features = []
-    for feature in features:
-        if isinstance(feature, FeatureStoreInput):
-            feature_set_env = extract_env_name(feature.name)
-            feature_set_name = extract_featureset_name(feature.name)
-            feature_set_to_feature_set_info = env_to_feature_sets_infos.get(
-                feature_set_env
-            )
-            if feature_set_to_feature_set_info is None:
-                raise FrogmlException(f"Environment '{feature_set_env}' does not exist")
-            feature_set_info = feature_set_to_feature_set_info.get(feature_set_name)
-            if feature_set_info is None:
-                raise FrogmlException(
-                    f"Featureset '{feature_set_name}' does not exist in env '{feature_set_env}'"  # noqa
-                )
-            entity_spec = feature_set_info.entity_spec
-            retrieved_entity = Entity(
-                name=entity_spec.keys[
-                    0
-                ],  # currently support only one entity key per feature set
-                type=get_entity_type(entity_spec.value_type),
-            )
-            if (
-                feature.entity
-                and retrieved_entity.name.lower() != feature.entity.name.lower()
-            ):
-                raise FrogmlException(
-                    f"Explicitly supplied with an invalid entity: {feature.entity}, "
-                    f"actual: {retrieved_entity}"
-                )
-            feature.entity = retrieved_entity
-            feature.entity.name = feature.entity.name.lower()
-            feature_type = feature_set_info.feature_set_type
-            feature_version = feature_set_info.feature_version
-            feature_without_env = cast(
-                FeatureStoreInput,
-                dataclasses.replace(feature, name=discard_env_from_name(feature.name)),
-            )
-            feature = get_typed_feature(
-                feature_without_env, feature_type, feature_version
-            )
-        new_normalize_features.append(feature)
-    return new_normalize_features
-
-
-def adding_entities_to_schema(entities: List[Entity], features: List[BaseFeature]):
-    """
-    Adding the new entities for feature store feature to the entities list from schema
-    Args:
-        entities: list of entities
-        features: list of features
-    Returns: the entities with the new entities from the feature store features
-    """
-    entity_name_to_entity = {entity.name: entity for entity in entities}
-    for feature in features:
-        if isinstance(feature, FeatureStoreInput):
-            entity_name_to_entity[feature.entity.name] = feature.entity
-
-    for entity in entity_name_to_entity.values():
-        entity.name = entity.name.lower()
-
-    return list(set(entity_name_to_entity.values()))
-
-
-def normalize_schema(features: List[BaseFeature], entities: List[Entity]):
-    """
-    validate and sanitize the feature names and entities
-    Normalize schema - will add entity to each features in schema if not exists
-    Arg:
-        model: the model with the required schema
-    Return:
-        updated model
-    """
-    sanitized_features = validate_and_sanitize_features_name(features)
-    fs_info_cache = retrieve_fs_mapping(sanitized_features)
-
-    normalized_features = normalize_features(sanitized_features, fs_info_cache)
-    normalized_entities = adding_entities_to_schema(entities, sanitized_features)
-
-    return normalized_features, normalized_entities
-
-
-def enrich_schema(features: List[BaseFeature], entities: List[Entity]):
-    normalized_features, normalized_entities = normalize_schema(features, entities)
-    return normalized_features, normalized_entities

frogml_core/model/adapters/input_adapters/numpy_input_adapter.py

@@ -1,100 +0,0 @@
-import json
-import typing as t
-
-from frogml_core.exceptions import FrogmlException
-
-from .base_input_adapter import BaseInputAdapter
-
-try:
-    import numpy as np
-except ImportError:
-    pass
-    # Numpy is supported either by supplying it as a dependency (or sub-dependency)
-    # in your Python project, or during the runtime
-
-
-def _is_matched_shape(
-    left: t.Optional[t.Tuple[int, ...]],
-    right: t.Optional[t.Tuple[int, ...]],
-) -> bool:  # pragma: no cover
-    if (left is None) or (right is None):
-        return False
-
-    if len(left) != len(right):
-        return False
-
-    for i, j in zip(left, right):
-        if i == -1 or j == -1:
-            continue
-        if i == j:
-            continue
-        return False
-    return True
-
-
-class NumpyInputAdapter(BaseInputAdapter):
-    def __init__(
-        self,
-        dtype: t.Optional[t.Union[str]] = None,
-        enforce_dtype: bool = False,
-        shape: t.Optional[t.Tuple[int, ...]] = None,
-        enforce_shape: bool = False,
-    ):
-        if dtype is not None and not isinstance(dtype, np.dtype):
-            # Convert from primitive type or type string, e.g.:
-            # np.dtype(float)
-            # np.dtype("float64")
-            try:
-                dtype = np.dtype(dtype)
-            except TypeError as e:
-                raise FrogmlException(f'NumpyNdarray: Invalid dtype "{dtype}": {e}')
-
-        self._dtype = dtype
-        self._shape = shape
-        self._enforce_dtype = enforce_dtype
-        self._enforce_shape = enforce_shape
-
-    def _verify_ndarray(
-        self,
-        obj,
-        exception_cls: t.Type[Exception] = FrogmlException,
-    ):
-        if self._dtype is not None and self._dtype != obj.dtype:
-            # ‘same_kind’ means only safe casts or casts within a kind, like float64
-            # to float32, are allowed.
-            if np.can_cast(obj.dtype, self._dtype, casting="same_kind"):
-                obj = obj.astype(self._dtype, casting="same_kind")  # type: ignore
-            else:
-                msg = (
-                    f'{self.__class__.__name__}: Expecting ndarray of dtype "{self._dtype}", but "{obj.dtype}" was '
-                    f"received. "
-                )
-                if self._enforce_dtype:
-                    raise exception_cls(msg)
-                else:
-                    print(msg)
-
-        if self._shape is not None and not _is_matched_shape(self._shape, obj.shape):
-            msg = (
-                f'{self.__class__.__name__}: Expecting ndarray of shape "{self._shape}", but "{obj.shape}" was '
-                f"received. "
-            )
-            if self._enforce_shape:
-                raise exception_cls(msg)
-            try:
-                obj = obj.reshape(self._shape)
-            except ValueError as e:
-                print(f"{msg} Failed to reshape: {e}.")
-
-        return obj
-
-    def extract_user_func_arg(self, data: bytes):
-        import numpy as np
-
-        data = json.loads(data)
-        try:
-            res = np.array(data, dtype=self._dtype)  # type: ignore[arg-type]
-        except ValueError:
-            res = np.array(data)  # type: ignore[arg-type]
-        res = self._verify_ndarray(res, FrogmlException)
-        return [res]

frogml_core/model/adapters/output_adapters/numpy_output_adapter.py

@@ -1,32 +0,0 @@
-import io
-
-from frogml_core.exceptions import FrogmlException
-
-try:
-    import numpy as np
-except ImportError:
-    pass
-    # Numpy is supported either by supplying it as a dependency (or sub-dependency)
-    # in your Python project, or during the runtime
-
-from .base_output_adapter import BaseOutputAdapter
-
-
-class NumpyOutputAdapter(BaseOutputAdapter):
-    @staticmethod
-    def http_content_type():
-        return {"Content-Type": "application/x-npy"}
-
-    @staticmethod
-    def pack_user_func_return_value(return_result) -> bytes:
-        if not isinstance(return_result, np.ndarray):
-            raise FrogmlException(
-                "XNpyOutputAdapter can only work with numpy.ndarray type"
-            )
-        try:
-            results_buffer = io.BytesIO()
-            # Disallow pickle: Changed in version 1.16.3: in response to CVE-2019-6446.
-            np.save(results_buffer, return_result, allow_pickle=False)
-            return results_buffer.getvalue()
-        except Exception as e:
-            raise ValueError(f"Failed to dump numpy array. Error is {e}")

frogml_core/model/analytics_logging.py

@@ -1,14 +0,0 @@
-from frogml_core.inner.singleton_meta import SingletonMeta
-
-
-class FrogMLAnalyticsLogger(metaclass=SingletonMeta):
-    """
-    Base class for Frogml analytics logger
-    """
-
-    def log(self, log):
-        """
-        Log data to Frogml's data lake
-        :param log: data to be persisted
-        """
-        pass

frogml_core/model/base.py

@@ -1,86 +0,0 @@
-from abc import abstractmethod
-
-from frogml_core.model.schema import ModelSchema
-
-
-class BaseModel:
-    """
-    Base class for all Frogml based models.
-    """
-
-    @abstractmethod
-    def build(self):
-        """
-        Responsible for loading the model. This method is invoked during build time (frogml build command)
-
-        Example usage:
-
-        >>> def build(self):
-        >>>     ...
-        >>>     train_pool = Pool(X_train, y_train, cat_features=categorical_features_indices)
-        >>>     validate_pool = Pool(X_validation, y_validation, cat_features=categorical_features_indices)
-        >>>     self.catboost.fit(train_pool, eval_set=validate_pool)
-
-        :return:
-        """
-        raise ValueError("Please implement build method")
-
-    @abstractmethod
-    def predict(self, df):
-        """
-        Invoked on every API inference request.
-        :param df: the inference vector, as a pandas dataframe
-
-        Example usage:
-
-        >>> def predict(self, df) -> pd.DataFrame:
-        >>>     return pd.DataFrame(self.catboost.predict(df), columns=['churn'])
-
-        :return: model output (inference results), as a pandas dataframe
-        """
-        raise ValueError("Please implement predict method")
-
-    def initialize_model(self):
-        """
-        Invoked when a model is loaded at serving time. Called once per model instance initialization. Can be used for
-        loading and storing values that should only be available in a serving setting or loading pretrained models. Optional method
-
-        Example usage:
-
-        >>> def initialize_model(self):
-        >>>     with open('model.pkl', 'rb') as infile:
-        >>>         self.model = pickle.load(infile)
-        """
-        pass
-
-    def schema(self) -> ModelSchema:
-        """
-        Specification of the model inputs and outputs. Optional method
-
-        Example usage:
-
-        >>> from frogml_core.model.schema import ModelSchema, ExplicitFeature, InferenceOutput
-        >>>
-        >>>
-        >>> def schema(self) -> ModelSchema:
-        >>>     model_schema = ModelSchema(
-        >>>     features=[
-        >>>         ExplicitFeature(name="State", type=str),
-        >>>     ],
-        >>>     predictions=[
-        >>>         InferenceOutput(name="score", type=float)
-        >>>     ])
-        >>>     return model_schema
-
-        :return: a model schema specification
-        """
-        pass
-
-    def _add_mixin(self, mixin):
-        """
-        Adds a mixin to the model. This is used to add functionality to the model without modifying the model itself.
-        For Frogml internal use only. Anything added here might be overwritten during the build process.
-        :param mixin: The mixin to add to the model
-        :return: None
-        """
-        self.__class__ = type(self.__class__.__name__, (mixin, self.__class__), {})

frogml_core/model/decorators/api.py

@@ -1,66 +0,0 @@
-from typing import Callable
-
-from dependency_injector.wiring import Provide, inject
-
-from frogml_core.exceptions import FrogmlException
-from frogml_core.inner.runtime_di.containers import FrogmlRuntimeContainer
-from frogml_core.model.adapters.input_adapters.base_input_adapter import (
-    BaseInputAdapter,
-)
-from frogml_core.model.adapters.input_adapters.dataframe_input_adapter import (
-    DataFrameInputAdapter,
-)
-from frogml_core.model.adapters.output_adapters.base_output_adapter import (
-    BaseOutputAdapter,
-)
-from frogml_core.model.adapters.output_adapters.dataframe_output_adapter import (
-    DataFrameOutputAdapter,
-)
-
-API_NOT_CONFIGURED_ERROR_MESSAGE = (
-    "There has been an error configuring the Frogml model. When testing locally, "
-    "please use 'run_local' as described in our docs: "
-    "https://docs.qwak.com/docs/testing-locally#/running-models-locally"
-)
-
-
-@inject
-def api_decorator(
-    analytics: bool = True,
-    analytics_sample_ratio: float = 1.0,
-    analytics_exclude_columns: list = [],
-    feature_extraction: bool = False,
-    input_adapter: BaseInputAdapter = DataFrameInputAdapter(),
-    output_adapter: BaseOutputAdapter = DataFrameOutputAdapter(),
-    api_decorator_function_creator=Provide[
-        FrogmlRuntimeContainer.api_decorator_function_creator
-    ],
-) -> Callable:
-    if callable(analytics):
-        raise TypeError(
-            """
-        You forgot to call the `@api` decorator.
-
-        Correct way -
-        @api()
-        def function():
-            pass
-
-        Wrong way -
-        @api
-        def function():
-            pass
-        """
-        )
-    try:
-        return api_decorator_function_creator(
-            analytics,
-            feature_extraction,
-            input_adapter,
-            output_adapter,
-            analytics_sample_ratio,
-            analytics_exclude_columns,
-        )
-    except TypeError as e:
-        if "__call__() takes 1" in str(e):
-            raise FrogmlException(API_NOT_CONFIGURED_ERROR_MESSAGE)

frogml_core/model/decorators/impl/api_implementation.py

@@ -1,29 +0,0 @@
-from functools import wraps
-from typing import Any, Callable
-
-from frogml_core.model.adapters import BaseInputAdapter, BaseOutputAdapter
-from frogml_core.model.utils.extract_wrapped_function import extract_wrapped
-
-
-def create_decorator_function(
-    analytics: bool,
-    feature_extraction: bool,
-    input_adapter: BaseInputAdapter,
-    output_adapter: BaseOutputAdapter,
-    analytics_sample_ratio: float,
-    analytics_exclude_columns: list,
-) -> Callable:
-    def api_decorator_function_load_logic(func: Callable[..., Any]):
-        func = extract_wrapped(func)
-        setattr(func, "_input_adapter", input_adapter)
-        setattr(func, "_output_adapter", output_adapter)
-        setattr(func, "_fs_extraction", feature_extraction)
-
-        @wraps(func)
-        def api_decorator_function_execution_logic(*args, **kwargs) -> Any:
-            output = func(*args, **kwargs)
-            return output
-
-        return api_decorator_function_execution_logic
-
-    return api_decorator_function_load_logic

frogml_core/model/decorators/timer.py

@@ -1,20 +0,0 @@
-import contextlib
-
-from dependency_injector.wiring import Provide, inject
-
-from frogml_core.inner.runtime_di.containers import FrogmlRuntimeContainer
-
-
-@inject
-@contextlib.contextmanager
-def frogml_timer(
-    name, create_timer_function=Provide[FrogmlRuntimeContainer.timer_function_creator]
-):
-    """
-    The Frogml timer is a utility for measuring the runtime of ML models, specifically for visibility and latency evaluation.
-    It provides a convenient way to measure the time it takes for models to perform predictions
-    and to provide a breakdown of the different parts in the production process
-    Args:
-        name (str): The Frogml timer name. Users may define multiple impl with different names
-    """
-    return create_timer_function(name)

frogml_core/model/fs_info_mapping_retriever.py

@@ -1,63 +0,0 @@
-from typing import Dict, List
-
-from frogml_core.clients.administration.eco_system.eco_system_utils import (
-    EcosystemUtils,
-)
-from frogml_core.clients.feature_store import FeatureRegistryClient
-from frogml_core.feature_store._common.feature_set_utils import (
-    FeatureSetInfo,
-    get_env_to_featuresets_mapping,
-    get_feature_set_info,
-)
-from frogml_core.model.schema_entities import BaseFeature, FeatureStoreInput
-from frogml_core.model.utils.feature_utils import (
-    extract_env_name,
-    extract_featureset_name,
-)
-
-
-def retrieve_fs_mapping(
-    features: List[BaseFeature],
-) -> Dict[str, Dict[str, FeatureSetInfo]]:
-    """
-    Retrieve the feature set info mapping.
-    If one of the features is a FeatureStoreInput and is in a different environment from the current one, populate it using a single API call.
-    Otherwise, retrieve each feature set info individually.
-    The reasoning behind this logic is that if all required features are in the same environment, it's more efficient to retrieve only the necessary feature set, rather than all feature sets.
-    * Assuming the feature name is fully qualified with the environment name
-    :param features:
-    :return: dict of environment name to dict of feature set name to FeatureSetInfo
-    """
-    current_env_name = EcosystemUtils().get_current_environment_name()
-    features_manager_client = FeatureRegistryClient()
-
-    if any(
-        [
-            feature
-            for feature in features
-            if isinstance(feature, FeatureStoreInput)
-            and extract_env_name(feature.name) != current_env_name
-        ]
-    ):
-        return get_env_to_featuresets_mapping(features_manager_client)
-    else:
-        return _retrieve_fs_info_one_by_one(
-            features_manager_client, current_env_name, features
-        )
-
-
-def _retrieve_fs_info_one_by_one(
-    features_manager_client: FeatureRegistryClient,
-    current_env_name: str,
-    features: List[BaseFeature],
-) -> Dict[str, Dict[str, FeatureSetInfo]]:
-    fs_info_cache: Dict[str, FeatureSetInfo] = dict()
-
-    for feature in features:
-        if isinstance(feature, FeatureStoreInput):
-            feature_set_name = extract_featureset_name(feature.name).lower()
-            if feature_set_name not in fs_info_cache:
-                fs_info_cache[feature_set_name] = get_feature_set_info(
-                    features_manager_client, feature_set_name
-                )
-    return {current_env_name: fs_info_cache}