frogml-core 0.0.1__py3-none-any.whl

frogml/model/_entity_extraction.py

@@ -0,0 +1,116 @@
+import dataclasses
+from typing import Dict, List, cast
+
+from frogml.exceptions import FrogmlException
+from frogml.feature_store._common.feature_set_utils import (
+    FeatureSetInfo,
+    get_entity_type,
+    get_typed_feature,
+)
+from frogml.model.fs_info_mapping_retriever import retrieve_fs_mapping
+from frogml.model.schema_entities import BaseFeature, Entity, FeatureStoreInput
+from frogml.model.utils.feature_utils import (
+    discard_env_from_name,
+    extract_env_name,
+    extract_featureset_name,
+    validate_and_sanitize_features_name,
+)
+
+
+def normalize_features(
+    features: List[BaseFeature],
+    env_to_feature_sets_infos: Dict[str, Dict[str, FeatureSetInfo]],
+) -> List[BaseFeature]:
+    """
+    Adding the relevant entity to the feature set features
+    Args:
+        features: list of features
+        env_to_feature_sets_infos: dict of environment name to feature set name to it's entity
+    return normalize features - the features with the entities
+    :param
+    """
+    new_normalize_features = []
+    for feature in features:
+        if isinstance(feature, FeatureStoreInput):
+            feature_set_env = extract_env_name(feature.name)
+            feature_set_name = extract_featureset_name(feature.name)
+            feature_set_to_feature_set_info = env_to_feature_sets_infos.get(
+                feature_set_env
+            )
+            if feature_set_to_feature_set_info is None:
+                raise FrogmlException(f"Environment '{feature_set_env}' does not exist")
+            feature_set_info = feature_set_to_feature_set_info.get(feature_set_name)
+            if feature_set_info is None:
+                raise FrogmlException(
+                    f"Featureset '{feature_set_name}' does not exist in env '{feature_set_env}'"  # noqa
+                )
+            entity_spec = feature_set_info.entity_spec
+            retrieved_entity = Entity(
+                name=entity_spec.keys[
+                    0
+                ],  # currently support only one entity key per feature set
+                type=get_entity_type(entity_spec.value_type),
+            )
+            if (
+                feature.entity
+                and retrieved_entity.name.lower() != feature.entity.name.lower()
+            ):
+                raise FrogmlException(
+                    f"Explicitly supplied with an invalid entity: {feature.entity}, "
+                    f"actual: {retrieved_entity}"
+                )
+            feature.entity = retrieved_entity
+            feature.entity.name = feature.entity.name.lower()
+            feature_type = feature_set_info.feature_set_type
+            feature_version = feature_set_info.feature_version
+            feature_without_env = cast(
+                FeatureStoreInput,
+                dataclasses.replace(feature, name=discard_env_from_name(feature.name)),
+            )
+            feature = get_typed_feature(
+                feature_without_env, feature_type, feature_version
+            )
+        new_normalize_features.append(feature)
+    return new_normalize_features
+
+
+def adding_entities_to_schema(entities: List[Entity], features: List[BaseFeature]):
+    """
+    Adding the new entities for feature store feature to the entities list from schema
+    Args:
+        entities: list of entities
+        features: list of features
+    Returns: the entities with the new entities from the feature store features
+    """
+    entity_name_to_entity = {entity.name: entity for entity in entities}
+    for feature in features:
+        if isinstance(feature, FeatureStoreInput):
+            entity_name_to_entity[feature.entity.name] = feature.entity
+
+    for entity in entity_name_to_entity.values():
+        entity.name = entity.name.lower()
+
+    return list(set(entity_name_to_entity.values()))
+
+
+def normalize_schema(features: List[BaseFeature], entities: List[Entity]):
+    """
+    validate and sanitize the feature names and entities
+    Normalize schema - will add entity to each features in schema if not exists
+    Arg:
+        model: the model with the required schema
+    Return:
+        updated model
+    """
+    sanitized_features = validate_and_sanitize_features_name(features)
+    fs_info_cache = retrieve_fs_mapping(sanitized_features)
+
+    normalized_features = normalize_features(sanitized_features, fs_info_cache)
+    normalized_entities = adding_entities_to_schema(entities, sanitized_features)
+
+    return normalized_features, normalized_entities
+
+
+def enrich_schema(features: List[BaseFeature], entities: List[Entity]):
+    normalized_features, normalized_entities = normalize_schema(features, entities)
+    return normalized_features, normalized_entities

frogml/model/adapters/__init__.py

@@ -0,0 +1,39 @@
+from .input_adapters.base_input_adapter import BaseInputAdapter
+from .input_adapters.dataframe_input_adapter import DataFrameInputAdapter
+from .input_adapters.file_input_adapter import FileInputAdapter
+from .input_adapters.image_input_adapter import ImageInputAdapter
+from .input_adapters.json_input_adapter import JsonInputAdapter
+from .input_adapters.multi_input_adapter import MultiInputAdapter
+from .input_adapters.numpy_input_adapter import NumpyInputAdapter
+from .input_adapters.proto_input_adapter import ProtoInputAdapter
+from .input_adapters.string_input_adapter import StringInputAdapter
+from .input_adapters.tf_tensor_input_adapter import TfTensorInputAdapter
+from .output_adapters.base_output_adapter import BaseOutputAdapter
+from .output_adapters.dataframe_output_adapter import DataFrameOutputAdapter
+from .output_adapters.default_output_adapter import DefaultOutputAdapter
+from .output_adapters.json_output_adapter import JsonOutputAdapter
+from .output_adapters.numpy_output_adapter import NumpyOutputAdapter
+from .output_adapters.proto_output_adapter import ProtoOutputAdapter
+from .output_adapters.qwak_with_default_fallback import AutodetectOutputAdapter
+from .output_adapters.tf_tensor_output_adapter import TfTensorOutputAdapter
+
+__all__ = [
+    "BaseInputAdapter",
+    "DataFrameInputAdapter",
+    "FileInputAdapter",
+    "ImageInputAdapter",
+    "JsonInputAdapter",
+    "MultiInputAdapter",
+    "NumpyInputAdapter",
+    "ProtoInputAdapter",
+    "StringInputAdapter",
+    "TfTensorInputAdapter",
+    "BaseOutputAdapter",
+    "DataFrameOutputAdapter",
+    "DefaultOutputAdapter",
+    "JsonOutputAdapter",
+    "NumpyOutputAdapter",
+    "ProtoOutputAdapter",
+    "AutodetectOutputAdapter",
+    "TfTensorOutputAdapter",
+]

frogml/model/adapters/input_adapters/base_input_adapter.py

@@ -0,0 +1,8 @@
+from abc import ABC, abstractmethod
+from typing import Any, Sequence
+
+
+class BaseInputAdapter(ABC):
+    @abstractmethod
+    def extract_user_func_arg(self, data: str) -> Sequence[Any]:
+        pass

frogml/model/adapters/input_adapters/dataframe_input_adapter.py

@@ -0,0 +1,11 @@
+from typing import Any, Optional, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+
+
+class DataFrameInputAdapter(BaseInputAdapter):
+    def __init__(self, input_orient: Optional[str] = None):
+        self.input_orient = input_orient
+
+    def extract_user_func_arg(self, data: str) -> Sequence[Any]:
+        pass

frogml/model/adapters/input_adapters/file_input_adapter.py

@@ -0,0 +1,8 @@
+from typing import Any, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+
+
+class FileInputAdapter(BaseInputAdapter):
+    def extract_user_func_arg(self, data: Any) -> Sequence[Any]:
+        pass

frogml/model/adapters/input_adapters/image_input_adapter.py

@@ -0,0 +1,8 @@
+from typing import Any, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+
+
+class ImageInputAdapter(BaseInputAdapter):
+    def extract_user_func_arg(self, data: Any) -> Sequence[Any]:
+        pass

frogml/model/adapters/input_adapters/json_input_adapter.py

@@ -0,0 +1,8 @@
+from typing import Any, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+
+
+class JsonInputAdapter(BaseInputAdapter):
+    def extract_user_func_arg(self, data: str) -> Sequence[Any]:
+        pass

frogml/model/adapters/input_adapters/multi_input_adapter.py

@@ -0,0 +1,66 @@
+from typing import Any, List, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+from .proto_input_adapter import ProtoInputAdapter
+
+try:
+    from bentoml.adapters import JsonInput
+    from bentoml.types import InferenceTask
+except ImportError:
+    pass
+    # bentoml is supported either by supplying it as a dependency (or sub-dependency)
+    # in your Python project, or during the runtime
+
+
+class MultiInputAdapter(BaseInputAdapter):
+    def __init__(self, adapters: List[BaseInputAdapter]):
+        self.adapters = adapters
+
+    def extract_user_func_arg(self, data: bytes, headers=None) -> Sequence[Any]:
+        task = InferenceTask(data=data)
+        if headers:
+            adapter_name = headers.get("Adapter-Name")
+            if adapter_name:
+                for inner_adapter in self.adapters:
+                    if isinstance(inner_adapter, ProtoInputAdapter):
+                        if (
+                            inner_adapter.message.DESCRIPTOR.name == adapter_name
+                            or inner_adapter.message.DESCRIPTOR.full_name
+                            == adapter_name
+                        ):
+                            extracted_data = inner_adapter.extract_user_func_arg(data)
+                            if isinstance(extracted_data, tuple):
+                                extracted_data = extracted_data[0]
+                            if extracted_data is None:
+                                raise ValueError("Parser returned None")
+                            return extracted_data
+        for inner_adapter in self.adapters:
+            try:
+                if isinstance(inner_adapter, ProtoInputAdapter):
+                    adapter_impl = inner_adapter
+                    extracted_data = inner_adapter.extract_user_func_arg(data)
+                else:
+                    if isinstance(inner_adapter, type):
+                        adapter_impl = self.mappings.get(inner_adapter)
+                    else:
+                        adapter_impl = self.mappings.get(inner_adapter.__class__)
+
+                    extracted_data = adapter_impl().extract_user_func_args([task])
+
+                if isinstance(extracted_data, tuple):
+                    extracted_data = extracted_data[0]
+                if extracted_data is None:
+                    raise ValueError("Parser returned None")
+
+                if adapter_impl == JsonInput and extracted_data == [] and len(data) > 3:
+                    # JsonParser return an empty array when it cannot parse the input.
+                    # This if raises an exception when the returned [] is caused by an error
+                    # (the input isn't an empty array)
+                    raise ValueError(
+                        "JsonParser returned an empty array when the data is too long for an empty array"
+                    )
+                return extracted_data
+            except Exception as e:
+                print(f"{inner_adapter} skipped an adapter because of: {e}")
+
+        raise ValueError(f"Cannot parse with given adapters: {self.adapters}")

frogml/model/adapters/input_adapters/numpy_input_adapter.py

@@ -0,0 +1,100 @@
+import json
+import typing as t
+
+from frogml.exceptions import FrogmlException
+
+from .base_input_adapter import BaseInputAdapter
+
+try:
+    import numpy as np
+except ImportError:
+    pass
+    # Numpy is supported either by supplying it as a dependency (or sub-dependency)
+    # in your Python project, or during the runtime
+
+
+def _is_matched_shape(
+    left: t.Optional[t.Tuple[int, ...]],
+    right: t.Optional[t.Tuple[int, ...]],
+) -> bool:  # pragma: no cover
+    if (left is None) or (right is None):
+        return False
+
+    if len(left) != len(right):
+        return False
+
+    for i, j in zip(left, right):
+        if i == -1 or j == -1:
+            continue
+        if i == j:
+            continue
+        return False
+    return True
+
+
+class NumpyInputAdapter(BaseInputAdapter):
+    def __init__(
+        self,
+        dtype: t.Optional[t.Union[str]] = None,
+        enforce_dtype: bool = False,
+        shape: t.Optional[t.Tuple[int, ...]] = None,
+        enforce_shape: bool = False,
+    ):
+        if dtype is not None and not isinstance(dtype, np.dtype):
+            # Convert from primitive type or type string, e.g.:
+            # np.dtype(float)
+            # np.dtype("float64")
+            try:
+                dtype = np.dtype(dtype)
+            except TypeError as e:
+                raise FrogmlException(f'NumpyNdarray: Invalid dtype "{dtype}": {e}')
+
+        self._dtype = dtype
+        self._shape = shape
+        self._enforce_dtype = enforce_dtype
+        self._enforce_shape = enforce_shape
+
+    def _verify_ndarray(
+        self,
+        obj,
+        exception_cls: t.Type[Exception] = FrogmlException,
+    ):
+        if self._dtype is not None and self._dtype != obj.dtype:
+            # ‘same_kind’ means only safe casts or casts within a kind, like float64
+            # to float32, are allowed.
+            if np.can_cast(obj.dtype, self._dtype, casting="same_kind"):
+                obj = obj.astype(self._dtype, casting="same_kind")  # type: ignore
+            else:
+                msg = (
+                    f'{self.__class__.__name__}: Expecting ndarray of dtype "{self._dtype}", but "{obj.dtype}" was '
+                    f"received. "
+                )
+                if self._enforce_dtype:
+                    raise exception_cls(msg)
+                else:
+                    print(msg)
+
+        if self._shape is not None and not _is_matched_shape(self._shape, obj.shape):
+            msg = (
+                f'{self.__class__.__name__}: Expecting ndarray of shape "{self._shape}", but "{obj.shape}" was '
+                f"received. "
+            )
+            if self._enforce_shape:
+                raise exception_cls(msg)
+            try:
+                obj = obj.reshape(self._shape)
+            except ValueError as e:
+                print(f"{msg} Failed to reshape: {e}.")
+
+        return obj
+
+    def extract_user_func_arg(self, data: bytes):
+        import numpy as np
+
+        data = json.loads(data)
+        try:
+            res = np.array(data, dtype=self._dtype)  # type: ignore[arg-type]
+        except ValueError:
+            res = np.array(data)  # type: ignore[arg-type]
+        res = self._verify_ndarray(res, FrogmlException)
+        return [res]

frogml/model/adapters/input_adapters/proto_input_adapter.py

@@ -0,0 +1,32 @@
+from enum import Enum
+from typing import TypeVar
+
+from .base_input_adapter import BaseInputAdapter
+
+T = TypeVar("T")
+
+
+class ProtobufFlavor(Enum):
+    GOOGLE_PROTO = 1
+    BETTERPROTO = 2
+
+
+class ProtoInputAdapter(BaseInputAdapter):
+    def __init__(
+        self, message: T, flavor: ProtobufFlavor = ProtobufFlavor.GOOGLE_PROTO
+    ):
+        self.message: T = message
+        self.flavor = flavor
+
+    def extract_user_func_arg(self, data: bytes) -> T:
+        try:
+            model_input = self.message()
+            if self.flavor is ProtobufFlavor.GOOGLE_PROTO:
+                model_input.ParseFromString(data)
+
+            if self.flavor is ProtobufFlavor.BETTERPROTO:
+                model_input.parse(data)
+
+            return model_input
+        except Exception as e:
+            raise ValueError(f"Failed to serialize Proto message. Error is {e}")

frogml/model/adapters/input_adapters/string_input_adapter.py

@@ -0,0 +1,8 @@
+from typing import Any, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+
+
+class StringInputAdapter(BaseInputAdapter):
+    def extract_user_func_arg(self, data: str) -> Sequence[Any]:
+        pass

frogml/model/adapters/input_adapters/tf_tensor_input_adapter.py

@@ -0,0 +1,8 @@
+from typing import Any, Sequence
+
+from .base_input_adapter import BaseInputAdapter
+
+
+class TfTensorInputAdapter(BaseInputAdapter):
+    def extract_user_func_arg(self, data: str) -> Sequence[Any]:
+        pass

frogml/model/adapters/output_adapters/base_output_adapter.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseOutputAdapter(ABC):
+    @staticmethod
+    def http_content_type():
+        return {"Content-Type": "text/plain; charset=utf-8"}
+
+    @staticmethod
+    @abstractmethod
+    def pack_user_func_return_value(return_result: Any) -> Any:
+        pass

frogml/model/adapters/output_adapters/dataframe_output_adapter.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class DataFrameOutputAdapter(BaseOutputAdapter):
+    def __init__(self, output_orient: str = "records"):
+        self.output_orient = output_orient
+
+    @staticmethod
+    def pack_user_func_return_value(return_result: Any) -> str:
+        pass

frogml/model/adapters/output_adapters/default_output_adapter.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class DefaultOutputAdapter(BaseOutputAdapter):
+    @staticmethod
+    def pack_user_func_return_value(return_result: Any) -> str:
+        pass

frogml/model/adapters/output_adapters/json_output_adapter.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class JsonOutputAdapter(BaseOutputAdapter):
+    @staticmethod
+    def pack_user_func_return_value(return_result: Any) -> str:
+        pass

frogml/model/adapters/output_adapters/numpy_output_adapter.py

@@ -0,0 +1,32 @@
+import io
+
+from frogml.exceptions import FrogmlException
+
+try:
+    import numpy as np
+except ImportError:
+    pass
+    # Numpy is supported either by supplying it as a dependency (or sub-dependency)
+    # in your Python project, or during the runtime
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class NumpyOutputAdapter(BaseOutputAdapter):
+    @staticmethod
+    def http_content_type():
+        return {"Content-Type": "application/x-npy"}
+
+    @staticmethod
+    def pack_user_func_return_value(return_result) -> bytes:
+        if not isinstance(return_result, np.ndarray):
+            raise FrogmlException(
+                "XNpyOutputAdapter can only work with numpy.ndarray type"
+            )
+        try:
+            results_buffer = io.BytesIO()
+            # Disallow pickle: Changed in version 1.16.3: in response to CVE-2019-6446.
+            np.save(results_buffer, return_result, allow_pickle=False)
+            return results_buffer.getvalue()
+        except Exception as e:
+            raise ValueError(f"Failed to dump numpy array. Error is {e}")

frogml/model/adapters/output_adapters/proto_output_adapter.py

@@ -0,0 +1,16 @@
+from google.protobuf.message import Message
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class ProtoOutputAdapter(BaseOutputAdapter):
+    @staticmethod
+    def http_content_type():
+        return {"Content-Type": "application/octet-stream"}
+
+    @staticmethod
+    def pack_user_func_return_value(return_result: Message) -> bytes:
+        try:
+            return return_result.SerializeToString()
+        except Exception as e:
+            raise ValueError(f"Failed to serialize proto message. Error is {e}")

frogml/model/adapters/output_adapters/qwak_with_default_fallback.py

@@ -0,0 +1,5 @@
+from .proto_output_adapter import ProtoOutputAdapter
+
+
+class AutodetectOutputAdapter(ProtoOutputAdapter):
+    pass

frogml/model/adapters/output_adapters/tf_tensor_output_adapter.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from .base_output_adapter import BaseOutputAdapter
+
+
+class TfTensorOutputAdapter(BaseOutputAdapter):
+    @staticmethod
+    def pack_user_func_return_value(return_result: Any) -> str:
+        pass

frogml/model/analytics_logging.py

@@ -0,0 +1,14 @@
+from frogml.inner.singleton_meta import SingletonMeta
+
+
+class QwakAnalyticsLogger(metaclass=SingletonMeta):
+    """
+    Base class for Frogml analytics logger
+    """
+
+    def log(self, log):
+        """
+        Log data to Frogml's data lake
+        :param log: data to be persisted
+        """
+        pass

frogml/model/base.py

@@ -0,0 +1,86 @@
+from abc import abstractmethod
+
+from frogml.model.schema import ModelSchema
+
+
+class BaseModel:
+    """
+    Base class for all Frogml based models.
+    """
+
+    @abstractmethod
+    def build(self):
+        """
+        Responsible for loading the model. This method is invoked during build time (frogml build command)
+
+        Example usage:
+
+        >>> def build(self):
+        >>>     ...
+        >>>     train_pool = Pool(X_train, y_train, cat_features=categorical_features_indices)
+        >>>     validate_pool = Pool(X_validation, y_validation, cat_features=categorical_features_indices)
+        >>>     self.catboost.fit(train_pool, eval_set=validate_pool)
+
+        :return:
+        """
+        raise ValueError("Please implement build method")
+
+    @abstractmethod
+    def predict(self, df):
+        """
+        Invoked on every API inference request.
+        :param df: the inference vector, as a pandas dataframe
+
+        Example usage:
+
+        >>> def predict(self, df) -> pd.DataFrame:
+        >>>     return pd.DataFrame(self.catboost.predict(df), columns=['churn'])
+
+        :return: model output (inference results), as a pandas dataframe
+        """
+        raise ValueError("Please implement predict method")
+
+    def initialize_model(self):
+        """
+        Invoked when a model is loaded at serving time. Called once per model instance initialization. Can be used for
+        loading and storing values that should only be available in a serving setting or loading pretrained models. Optional method
+
+        Example usage:
+
+        >>> def initialize_model(self):
+        >>>     with open('model.pkl', 'rb') as infile:
+        >>>         self.model = pickle.load(infile)
+        """
+        pass
+
+    def schema(self) -> ModelSchema:
+        """
+        Specification of the model inputs and outputs. Optional method
+
+        Example usage:
+
+        >>> from frogml.model.schema import ModelSchema, ExplicitFeature, InferenceOutput
+        >>>
+        >>>
+        >>> def schema(self) -> ModelSchema:
+        >>>     model_schema = ModelSchema(
+        >>>     features=[
+        >>>         ExplicitFeature(name="State", type=str),
+        >>>     ],
+        >>>     predictions=[
+        >>>         InferenceOutput(name="score", type=float)
+        >>>     ])
+        >>>     return model_schema
+
+        :return: a model schema specification
+        """
+        pass
+
+    def _add_mixin(self, mixin):
+        """
+        Adds a mixin to the model. This is used to add functionality to the model without modifying the model itself.
+        For Frogml internal use only. Anything added here might be overwritten during the build process.
+        :param mixin: The mixin to add to the model
+        :return: None
+        """
+        self.__class__ = type(self.__class__.__name__, (mixin, self.__class__), {})