frogml-core 0.0.116__py3-none-any.whl → 0.0.118__py3-none-any.whl

sdk/model/_entity_extraction.py

@@ -0,0 +1,116 @@
+import dataclasses
+from typing import Dict, List, cast
+
+from frogml.core.exceptions import FrogmlException
+from frogml.feature_store._common.feature_set_utils import (
+    FeatureSetInfo,
+    get_entity_type,
+    get_typed_feature,
+)
+from frogml.sdk.model.fs_info_mapping_retriever import retrieve_fs_mapping
+from frogml.sdk.model.schema_entities import BaseFeature, Entity, FeatureStoreInput
+from frogml.sdk.model.utils.feature_utils import (
+    discard_env_from_name,
+    extract_env_name,
+    extract_featureset_name,
+    validate_and_sanitize_features_name,
+)
+
+
+def normalize_features(
+    features: List[BaseFeature],
+    env_to_feature_sets_infos: Dict[str, Dict[str, FeatureSetInfo]],
+) -> List[BaseFeature]:
+    """
+    Adding the relevant entity to the feature set features
+    Args:
+        features: list of features
+        env_to_feature_sets_infos: dict of environment name to feature set name to it's entity
+    return normalize features - the features with the entities
+    :param
+    """
+    new_normalize_features = []
+    for feature in features:
+        if isinstance(feature, FeatureStoreInput):
+            feature_set_env = extract_env_name(feature.name)
+            feature_set_name = extract_featureset_name(feature.name)
+            feature_set_to_feature_set_info = env_to_feature_sets_infos.get(
+                feature_set_env
+            )
+            if feature_set_to_feature_set_info is None:
+                raise FrogmlException(f"Environment '{feature_set_env}' does not exist")
+            feature_set_info = feature_set_to_feature_set_info.get(feature_set_name)
+            if feature_set_info is None:
+                raise FrogmlException(
+                    f"Featureset '{feature_set_name}' does not exist in env '{feature_set_env}'"  # noqa
+                )
+            entity_spec = feature_set_info.entity_spec
+            retrieved_entity = Entity(
+                name=entity_spec.keys[
+                    0
+                ],  # currently support only one entity key per feature set
+                type=get_entity_type(entity_spec.value_type),
+            )
+            if (
+                feature.entity
+                and retrieved_entity.name.lower() != feature.entity.name.lower()
+            ):
+                raise FrogmlException(
+                    f"Explicitly supplied with an invalid entity: {feature.entity}, "
+                    f"actual: {retrieved_entity}"
+                )
+            feature.entity = retrieved_entity
+            feature.entity.name = feature.entity.name.lower()
+            feature_type = feature_set_info.feature_set_type
+            feature_version = feature_set_info.feature_version
+            feature_without_env = cast(
+                FeatureStoreInput,
+                dataclasses.replace(feature, name=discard_env_from_name(feature.name)),
+            )
+            feature = get_typed_feature(
+                feature_without_env, feature_type, feature_version
+            )
+        new_normalize_features.append(feature)
+    return new_normalize_features
+
+
+def adding_entities_to_schema(entities: List[Entity], features: List[BaseFeature]):
+    """
+    Adding the new entities for feature store feature to the entities list from schema
+    Args:
+        entities: list of entities
+        features: list of features
+    Returns: the entities with the new entities from the feature store features
+    """
+    entity_name_to_entity = {entity.name: entity for entity in entities}
+    for feature in features:
+        if isinstance(feature, FeatureStoreInput):
+            entity_name_to_entity[feature.entity.name] = feature.entity
+
+    for entity in entity_name_to_entity.values():
+        entity.name = entity.name.lower()
+
+    return list(set(entity_name_to_entity.values()))
+
+
+def normalize_schema(features: List[BaseFeature], entities: List[Entity]):
+    """
+    validate and sanitize the feature names and entities
+    Normalize schema - will add entity to each features in schema if not exists
+    Arg:
+        model: the model with the required schema
+    Return:
+        updated model
+    """
+    sanitized_features = validate_and_sanitize_features_name(features)
+    fs_info_cache = retrieve_fs_mapping(sanitized_features)
+
+    normalized_features = normalize_features(sanitized_features, fs_info_cache)
+    normalized_entities = adding_entities_to_schema(entities, sanitized_features)
+
+    return normalized_features, normalized_entities
+
+
+def enrich_schema(features: List[BaseFeature], entities: List[Entity]):
+    normalized_features, normalized_entities = normalize_schema(features, entities)
+    return normalized_features, normalized_entities

sdk/model/adapters/input_adapters/numpy_input_adapter.py

@@ -0,0 +1,100 @@
+import json
+import typing as t
+
+from frogml.core.exceptions import FrogmlException
+
+from .base_input_adapter import BaseInputAdapter
+
+try:
+    import numpy as np
+except ImportError:
+    pass
+    # Numpy is supported either by supplying it as a dependency (or sub-dependency)
+    # in your Python project, or during the runtime
+
+
+def _is_matched_shape(
+    left: t.Optional[t.Tuple[int, ...]],
+    right: t.Optional[t.Tuple[int, ...]],
+) -> bool:  # pragma: no cover
+    if (left is None) or (right is None):
+        return False
+
+    if len(left) != len(right):
+        return False
+
+    for i, j in zip(left, right):
+        if i == -1 or j == -1:
+            continue
+        if i == j:
+            continue
+        return False
+    return True
+
+
+class NumpyInputAdapter(BaseInputAdapter):
+    def __init__(
+        self,
+        dtype: t.Optional[t.Union[str]] = None,
+        enforce_dtype: bool = False,
+        shape: t.Optional[t.Tuple[int, ...]] = None,
+        enforce_shape: bool = False,
+    ):
+        if dtype is not None and not isinstance(dtype, np.dtype):
+            # Convert from primitive type or type string, e.g.:
+            # np.dtype(float)
+            # np.dtype("float64")
+            try:
+                dtype = np.dtype(dtype)
+            except TypeError as e:
+                raise FrogmlException(f'NumpyNdarray: Invalid dtype "{dtype}": {e}')
+
+        self._dtype = dtype
+        self._shape = shape
+        self._enforce_dtype = enforce_dtype
+        self._enforce_shape = enforce_shape
+
+    def _verify_ndarray(
+        self,
+        obj,
+        exception_cls: t.Type[Exception] = FrogmlException,
+    ):
+        if self._dtype is not None and self._dtype != obj.dtype:
+            # ‘same_kind’ means only safe casts or casts within a kind, like float64
+            # to float32, are allowed.
+            if np.can_cast(obj.dtype, self._dtype, casting="same_kind"):
+                obj = obj.astype(self._dtype, casting="same_kind")  # type: ignore
+            else:
+                msg = (
+                    f'{self.__class__.__name__}: Expecting ndarray of dtype "{self._dtype}", but "{obj.dtype}" was '
+                    f"received. "
+                )
+                if self._enforce_dtype:
+                    raise exception_cls(msg)
+                else:
+                    print(msg)
+
+        if self._shape is not None and not _is_matched_shape(self._shape, obj.shape):
+            msg = (
+                f'{self.__class__.__name__}: Expecting ndarray of shape "{self._shape}", but "{obj.shape}" was '
+                f"received. "
+            )
+            if self._enforce_shape:
+                raise exception_cls(msg)
+            try:
+                obj = obj.reshape(self._shape)
+            except ValueError as e:
+                print(f"{msg} Failed to reshape: {e}.")
+
+        return obj
+
+    def extract_user_func_arg(self, data: bytes):
+        import numpy as np
+
+        data = json.loads(data)
+        try:
+            res = np.array(data, dtype=self._dtype)  # type: ignore[arg-type]
+        except ValueError:
+            res = np.array(data)  # type: ignore[arg-type]
+        res = self._verify_ndarray(res, FrogmlException)
+        return [res]

sdk/model/adapters/output_adapters/numpy_output_adapter.py

@@ -0,0 +1,32 @@
+import io
+
+from frogml.core.exceptions import FrogmlException
+
+try:
+    import numpy as np
+except ImportError:
+    pass
+    # Numpy is supported either by supplying it as a dependency (or sub-dependency)
+    # in your Python project, or during the runtime
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class NumpyOutputAdapter(BaseOutputAdapter):
+    @staticmethod
+    def http_content_type():
+        return {"Content-Type": "application/x-npy"}
+
+    @staticmethod
+    def pack_user_func_return_value(return_result) -> bytes:
+        if not isinstance(return_result, np.ndarray):
+            raise FrogmlException(
+                "XNpyOutputAdapter can only work with numpy.ndarray type"
+            )
+        try:
+            results_buffer = io.BytesIO()
+            # Disallow pickle: Changed in version 1.16.3: in response to CVE-2019-6446.
+            np.save(results_buffer, return_result, allow_pickle=False)
+            return results_buffer.getvalue()
+        except Exception as e:
+            raise ValueError(f"Failed to dump numpy array. Error is {e}")

sdk/model/analytics_logging.py

@@ -0,0 +1,14 @@
+from frogml.core.inner.singleton_meta import SingletonMeta
+
+
+class FrogMLAnalyticsLogger(metaclass=SingletonMeta):
+    """
+    Base class for Frogml analytics logger
+    """
+
+    def log(self, log):
+        """
+        Log data to Frogml's data lake
+        :param log: data to be persisted
+        """
+        pass

sdk/model/base.py

@@ -0,0 +1,86 @@
+from abc import abstractmethod
+
+from frogml.sdk.model.schema import ModelSchema
+
+
+class BaseModel:
+    """
+    Base class for all Frogml based models.
+    """
+
+    @abstractmethod
+    def build(self):
+        """
+        Responsible for loading the model. This method is invoked during build time (frogml build command)
+
+        Example usage:
+
+        >>> def build(self):
+        >>>     ...
+        >>>     train_pool = Pool(X_train, y_train, cat_features=categorical_features_indices)
+        >>>     validate_pool = Pool(X_validation, y_validation, cat_features=categorical_features_indices)
+        >>>     self.catboost.fit(train_pool, eval_set=validate_pool)
+
+        :return:
+        """
+        raise ValueError("Please implement build method")
+
+    @abstractmethod
+    def predict(self, df):
+        """
+        Invoked on every API inference request.
+        :param df: the inference vector, as a pandas dataframe
+
+        Example usage:
+
+        >>> def predict(self, df) -> pd.DataFrame:
+        >>>     return pd.DataFrame(self.catboost.predict(df), columns=['churn'])
+
+        :return: model output (inference results), as a pandas dataframe
+        """
+        raise ValueError("Please implement predict method")
+
+    def initialize_model(self):
+        """
+        Invoked when a model is loaded at serving time. Called once per model instance initialization. Can be used for
+        loading and storing values that should only be available in a serving setting or loading pretrained models. Optional method
+
+        Example usage:
+
+        >>> def initialize_model(self):
+        >>>     with open('model.pkl', 'rb') as infile:
+        >>>         self.model = pickle.load(infile)
+        """
+        pass
+
+    def schema(self) -> ModelSchema:
+        """
+        Specification of the model inputs and outputs. Optional method
+
+        Example usage:
+
+        >>> from frogml.core import ModelSchema, ExplicitFeature, InferenceOutput
+        >>>
+        >>>
+        >>> def schema(self) -> ModelSchema:
+        >>>     model_schema = ModelSchema(
+        >>>     features=[
+        >>>         ExplicitFeature(name="State", type=str),
+        >>>     ],
+        >>>     predictions=[
+        >>>         InferenceOutput(name="score", type=float)
+        >>>     ])
+        >>>     return model_schema
+
+        :return: a model schema specification
+        """
+        pass
+
+    def _add_mixin(self, mixin):
+        """
+        Adds a mixin to the model. This is used to add functionality to the model without modifying the model itself.
+        For Frogml internal use only. Anything added here might be overwritten during the build process.
+        :param mixin: The mixin to add to the model
+        :return: None
+        """
+        self.__class__ = type(self.__class__.__name__, (mixin, self.__class__), {})

sdk/model/decorators/api.py

@@ -0,0 +1,66 @@
+from typing import Callable
+
+from dependency_injector.wiring import Provide, inject
+
+from frogml.core.exceptions import FrogmlException
+from frogml.sdk.inner.runtime_di.containers import FrogmlRuntimeContainer
+from frogml.sdk.model.adapters.input_adapters.base_input_adapter import (
+    BaseInputAdapter,
+)
+from frogml.sdk.model.adapters import (
+    DataFrameInputAdapter,
+)
+from frogml.sdk.model.adapters.output_adapters.base_output_adapter import (
+    BaseOutputAdapter,
+)
+from frogml.sdk.model.adapters import (
+    DataFrameOutputAdapter,
+)
+
+API_NOT_CONFIGURED_ERROR_MESSAGE = (
+    "There has been an error configuring the Frogml model. When testing locally, "
+    "please use 'run_local' as described in our docs: "
+    "https://docs.qwak.com/docs/testing-locally#/running-models-locally"
+)
+
+
+@inject
+def api_decorator(
+    analytics: bool = True,
+    analytics_sample_ratio: float = 1.0,
+    analytics_exclude_columns: list = [],
+    feature_extraction: bool = False,
+    input_adapter: BaseInputAdapter = DataFrameInputAdapter(),
+    output_adapter: BaseOutputAdapter = DataFrameOutputAdapter(),
+    api_decorator_function_creator=Provide[
+        FrogmlRuntimeContainer.api_decorator_function_creator
+    ],
+) -> Callable:
+    if callable(analytics):
+        raise TypeError(
+            """
+        You forgot to call the `@api` decorator.
+
+        Correct way -
+        @api()
+        def function():
+            pass
+
+        Wrong way -
+        @api
+        def function():
+            pass
+        """
+        )
+    try:
+        return api_decorator_function_creator(
+            analytics,
+            feature_extraction,
+            input_adapter,
+            output_adapter,
+            analytics_sample_ratio,
+            analytics_exclude_columns,
+        )
+    except TypeError as e:
+        if "__call__() takes 1" in str(e):
+            raise FrogmlException(API_NOT_CONFIGURED_ERROR_MESSAGE)

sdk/model/decorators/impl/api_implementation.py

@@ -0,0 +1,29 @@
+from functools import wraps
+from typing import Any, Callable
+
+from frogml.sdk.model.adapters import BaseInputAdapter, BaseOutputAdapter
+from frogml.sdk.model.utils.extract_wrapped_function import extract_wrapped
+
+
+def create_decorator_function(
+    analytics: bool,
+    feature_extraction: bool,
+    input_adapter: BaseInputAdapter,
+    output_adapter: BaseOutputAdapter,
+    analytics_sample_ratio: float,
+    analytics_exclude_columns: list,
+) -> Callable:
+    def api_decorator_function_load_logic(func: Callable[..., Any]):
+        func = extract_wrapped(func)
+        setattr(func, "_input_adapter", input_adapter)
+        setattr(func, "_output_adapter", output_adapter)
+        setattr(func, "_fs_extraction", feature_extraction)
+
+        @wraps(func)
+        def api_decorator_function_execution_logic(*args, **kwargs) -> Any:
+            output = func(*args, **kwargs)
+            return output
+
+        return api_decorator_function_execution_logic
+
+    return api_decorator_function_load_logic

sdk/model/decorators/timer.py

@@ -0,0 +1,20 @@
+import contextlib
+
+from dependency_injector.wiring import Provide, inject
+
+from frogml.sdk.inner.runtime_di.containers import FrogmlRuntimeContainer
+
+
+@inject
+@contextlib.contextmanager
+def frogml_timer(
+    name, create_timer_function=Provide[FrogmlRuntimeContainer.timer_function_creator]
+):
+    """
+    The Frogml timer is a utility for measuring the runtime of ML models, specifically for visibility and latency evaluation.
+    It provides a convenient way to measure the time it takes for models to perform predictions
+    and to provide a breakdown of the different parts in the production process
+    Args:
+        name (str): The Frogml timer name. Users may define multiple impl with different names
+    """
+    return create_timer_function(name)

sdk/model/fs_info_mapping_retriever.py

@@ -0,0 +1,63 @@
+from typing import Dict, List
+
+from frogml.core.clients.administration.eco_system.eco_system_utils import (
+    EcosystemUtils,
+)
+from frogml.core.clients.feature_store import FeatureRegistryClient
+from frogml.feature_store._common.feature_set_utils import (
+    FeatureSetInfo,
+    get_env_to_featuresets_mapping,
+    get_feature_set_info,
+)
+from frogml.sdk.model.schema_entities import BaseFeature, FeatureStoreInput
+from frogml.sdk.model.utils.feature_utils import (
+    extract_env_name,
+    extract_featureset_name,
+)
+
+
+def retrieve_fs_mapping(
+    features: List[BaseFeature],
+) -> Dict[str, Dict[str, FeatureSetInfo]]:
+    """
+    Retrieve the feature set info mapping.
+    If one of the features is a FeatureStoreInput and is in a different environment from the current one, populate it using a single API call.
+    Otherwise, retrieve each feature set info individually.
+    The reasoning behind this logic is that if all required features are in the same environment, it's more efficient to retrieve only the necessary feature set, rather than all feature sets.
+    * Assuming the feature name is fully qualified with the environment name
+    :param features:
+    :return: dict of environment name to dict of feature set name to FeatureSetInfo
+    """
+    current_env_name = EcosystemUtils().get_current_environment_name()
+    features_manager_client = FeatureRegistryClient()
+
+    if any(
+        [
+            feature
+            for feature in features
+            if isinstance(feature, FeatureStoreInput)
+            and extract_env_name(feature.name) != current_env_name
+        ]
+    ):
+        return get_env_to_featuresets_mapping(features_manager_client)
+    else:
+        return _retrieve_fs_info_one_by_one(
+            features_manager_client, current_env_name, features
+        )
+
+
+def _retrieve_fs_info_one_by_one(
+    features_manager_client: FeatureRegistryClient,
+    current_env_name: str,
+    features: List[BaseFeature],
+) -> Dict[str, Dict[str, FeatureSetInfo]]:
+    fs_info_cache: Dict[str, FeatureSetInfo] = dict()
+
+    for feature in features:
+        if isinstance(feature, FeatureStoreInput):
+            feature_set_name = extract_featureset_name(feature.name).lower()
+            if feature_set_name not in fs_info_cache:
+                fs_info_cache[feature_set_name] = get_feature_set_info(
+                    features_manager_client, feature_set_name
+                )
+    return {current_env_name: fs_info_cache}

sdk/model/model_version_tracking.py

@@ -0,0 +1,57 @@
+from typing import Dict, Union
+
+from frogml.core.inner.singleton_meta import SingletonMeta
+
+
+class FrogmlModelVersionTracking(metaclass=SingletonMeta):
+    def __init__(self):
+        self.metric: Dict[str, Union[int, str]] = {}
+        self.param: Dict[str, Union[int, str]] = {}
+
+    def log_param(self, params: Dict[str, Union[int, str]]):
+        self.param = {**self.param, **params}
+
+    def log_params(self, params: Dict[str, Union[int, str]]):
+        self.log_param(params)
+
+    def log_metric(self, metrics: Dict[str, Union[int, str]]):
+        self.metric = {**self.metric, **metrics}
+
+    def log_metrics(self, metrics: Dict[str, Union[int, str]]):
+        self.log_metric(metrics)
+
+    def get_metrics(self) -> Dict[str, Union[int, str]]:
+        return self.metric
+
+    def get_params(self) -> Dict[str, Union[int, str]]:
+        return self.param
+
+
+def log_metric(metrics):
+    client = FrogmlModelVersionTracking()
+    client.log_metric(metrics)
+
+
+def log_param(params):
+    client = FrogmlModelVersionTracking()
+    client.log_param(params)
+
+
+def log_metrics(metrics):
+    client = FrogmlModelVersionTracking()
+    client.log_metric(metrics)
+
+
+def log_params(params):
+    client = FrogmlModelVersionTracking()
+    client.log_param(params)
+
+
+def get_params():
+    client = FrogmlModelVersionTracking()
+    return client.get_params()
+
+
+def get_metrics():
+    client = FrogmlModelVersionTracking()
+    return client.get_metrics()

sdk/model/schema.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass
+from typing import List
+
+from frogml._proto.qwak.build.v1.build_pb2 import ModelSchema as ProtoModelSchema
+from frogml.sdk.model._entity_extraction import enrich_schema
+from frogml.sdk.model.schema_entities import (
+    BaseFeature,
+    Entity,
+    ExplicitFeature,
+    FeatureStoreInput,
+    InferenceOutput,
+)
+
+__all__ = ["ModelSchema", "ExplicitFeature", "InferenceOutput", "FeatureStoreInput"]
+
+
+@dataclass
+class ModelSchema:
+    """
+    The ModelSchema class provides a structured representation of a Machine Learning (ML) model's schema.
+     It captures the details such as input features, output predictions, and entities, providing a standard way to
+    communicate these details between different components of an ML system.
+    """
+
+    def __init__(
+        self,
+        entities: List[Entity] = None,
+        inputs: List[BaseFeature] = None,
+        outputs: List[InferenceOutput] = None,
+    ):
+        """
+        Initializes the ModelSchema object with entities, inputs, and outputs.
+
+        Args:
+            entities (List[Entity], optional): A list of Entity objects. Defaults to None.
+            inputs (List[BaseFeature], optional): A list of BaseFeature objects. Defaults to None.
+            outputs (List[InferenceOutput], optional): A list of InferenceOutput objects. Defaults to None.
+
+        Raises:
+            TypeError: If entities, inputs, or outputs are not provided in the correct format.
+        """
+        raw_entities = entities if entities else []
+        raw_inputs = inputs if inputs else []
+        self._inputs, self._entities = enrich_schema(raw_inputs, raw_entities)
+        self._outputs = outputs if outputs else []
+
+    def to_proto(self):
+        return ProtoModelSchema(
+            entities=[entity.to_proto() for entity in self._entities],
+            features=[feature.to_proto() for feature in self._inputs],
+            inference_output=[inference.to_proto() for inference in self._outputs],
+        )