frogml-core 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

frogml/model/tools/adapters/input.py

@@ -1,66 +0,0 @@
-from typing import Any, Union
-
-from frogml.exceptions import FrogmlHTTPException
-from frogml.model.adapters import (
-    DataFrameInputAdapter,
-    FileInputAdapter,
-    ImageInputAdapter,
-    JsonInputAdapter,
-    MultiInputAdapter,
-    NumpyInputAdapter,
-    ProtoInputAdapter,
-    StringInputAdapter,
-    TfTensorInputAdapter,
-)
-from frogml.model.base import BaseModel
-
-from .input_adapters.dataframe_input import DataframeInput
-from .input_adapters.file_input import FileInput
-from .input_adapters.image_input import ImageInput
-from .input_adapters.json_input import JsonInput
-from .input_adapters.string_input import StringInput
-from .input_adapters.tf_tensor_input import TfTensorInput
-
-
-def get_input_adapter(
-    model: BaseModel,
-) -> Union[JsonInput, DataframeInput, StringInput]:
-    adapter = getattr(model.predict, "_input_adapter", "")
-
-    class QwakInput(StringInput):
-        def extract_user_func_args(self, data: str) -> Any:
-            try:
-                return adapter.extract_user_func_arg(data)
-            except Exception as e:
-                raise FrogmlHTTPException(
-                    status_code=500,
-                    message=f"Failed to deserialize input message. Error is: {e}. For more information "
-                    f"please check model logs.",
-                )
-
-    class MultiFormatInput(QwakInput):
-        pass
-
-    mapping = {
-        JsonInputAdapter: JsonInput,
-        DataFrameInputAdapter: DataframeInput,
-        StringInputAdapter: StringInput,
-        ProtoInputAdapter: QwakInput,
-        NumpyInputAdapter: QwakInput,
-        MultiInputAdapter: MultiFormatInput,
-        TfTensorInputAdapter: TfTensorInput,
-        ImageInputAdapter: ImageInput,
-        FileInputAdapter: FileInput,
-    }
-
-    if adapter:
-        adapter.mappings = mapping
-        for qwak_impl, runtime_impl in mapping.items():
-            if isinstance(adapter, qwak_impl):
-                if isinstance(adapter, DataFrameInputAdapter) and hasattr(
-                    adapter, "input_orient"
-                ):
-                    return runtime_impl(input_orient=adapter.input_orient)
-                return runtime_impl()
-
-    return DataframeInput()

frogml/model/tools/adapters/input_adapters/dataframe_input.py

@@ -1,48 +0,0 @@
-import json
-from typing import List, Optional, Union
-
-from frogml.exceptions import FrogmlHTTPException
-
-from .string_input import StringInput
-
-try:
-    import pandas
-    from bentoml.adapters import DataframeInput as BentoMLDataframeInput
-except ImportError:
-    pass
-
-
-class DataframeInput(StringInput):
-    def __init__(
-        self,
-        input_orient: Optional[str] = None,
-        **base_kwargs,
-    ):
-        self.input_orient = input_orient
-        super().__init__(**base_kwargs)
-
-        # Verify pandas imported properly and retry import if it has failed initially
-        try:
-            import pandas  # noqa: F401
-        except ImportError:
-            raise ImportError("Pandas package is required to use DataframeInput.")
-
-    def extract_user_func_args(
-        self,
-        data: Union[
-            str, List
-        ],  # it's a list of InferenceTasks, but we don't want to fail when bentoml is not installed
-    ) -> "pandas.DataFrame":
-        try:
-            if isinstance(data, list):
-                adapter = BentoMLDataframeInput(orient=self.input_orient)
-                # because parsing is deferred to bentoml, it will also set the batch property of a task
-                (df,) = adapter.extract_user_func_args(data)
-                return df
-            else:
-                return pandas.DataFrame.from_dict(json.loads(data), orient="columns")
-        except Exception as e:
-            raise FrogmlHTTPException(
-                status_code=400,
-                message=f"Error loading DataFrame input: {e}.",
-            )

frogml/model/tools/adapters/input_adapters/image_input.py

@@ -1,45 +0,0 @@
-import os
-from typing import BinaryIO
-
-from frogml.exceptions import FrogmlHTTPException
-
-from .file_input import FileInput
-
-
-class ImageInput(FileInput):
-    def __init__(
-        self,
-        **base_kwargs,
-    ):
-        try:
-            import imageio  # noqa: F401
-        except ImportError:
-            raise ImportError("imageio package is required to use ImageInput adapter")
-
-        super().__init__(**base_kwargs)
-        self.accept_image_formats = {".jpg", ".png", ".jpeg", ".tiff", ".webp", ".bmp"}
-
-    def extract_user_func_args(self, data: BinaryIO) -> "numpy.ndarray":  # noqa F821
-        try:
-            import imageio
-        except ImportError:
-            raise ImportError("imageio package is required to use ImageInput adapter")
-
-        if getattr(data, "name", None) and not self.check_file_extension(
-            data.name, self.accept_image_formats
-        ):
-            raise FrogmlHTTPException(
-                400,
-                message=f"Current service only accepts {self.accept_image_formats} formats",
-            )
-        try:
-            return imageio.v2.imread(data, mode="RGB")
-        except ValueError as e:
-            raise FrogmlHTTPException(400, message=str(e))
-
-    @staticmethod
-    def check_file_extension(file_name, supported_types):
-        if not file_name:
-            return False
-        _, extension = os.path.splitext(file_name)
-        return extension.lower() in (supported_types or [])

frogml/model/tools/adapters/input_adapters/json_input.py

@@ -1,20 +0,0 @@
-import json
-import traceback
-from typing import Dict, List, Union
-
-from frogml.exceptions import FrogmlHTTPException
-
-from .string_input import StringInput
-
-
-class JsonInput(StringInput):
-    def extract_user_func_args(
-        self, data: str
-    ) -> Union[bool, None, Dict, List, int, float, str]:
-        try:
-            return json.loads(data)
-        except json.JSONDecodeError:
-            raise FrogmlHTTPException(400, message="Not a valid JSON format")
-        except Exception:
-            err = traceback.format_exc()
-            raise FrogmlHTTPException(500, message=f"Internal Server Error: {err}")

frogml/model/tools/adapters/input_adapters/tf_tensor_input.py

@@ -1,44 +0,0 @@
-import base64
-import json
-import traceback
-
-from frogml.exceptions import FrogmlHTTPException
-from frogml.model.tools.adapters.encoders import TF_B64_KEY
-
-from .string_input import StringInput
-
-
-def b64_hook(o):
-    if isinstance(o, dict) and TF_B64_KEY in o:
-        return base64.b64decode(o[TF_B64_KEY])
-    return o
-
-
-class TfTensorInput(StringInput):
-    def extract_user_func_args(self, data: str) -> "tensorflow.Tensor":  # noqa F401
-        try:
-            import tensorflow as tf
-        except ImportError:
-            raise ImportError(
-                "tensorflow package is required to use TfTensorInput adapter"
-            )
-
-        try:
-            parsed_json = json.loads(data, object_hook=b64_hook)
-            if parsed_json.get("instances") is None:
-                raise FrogmlHTTPException(
-                    status_code=400,
-                    message="input format is not implemented",
-                )
-            else:
-                instances = parsed_json.get("instances")
-                return tf.constant(instances)
-        except json.JSONDecodeError:
-            raise FrogmlHTTPException(
-                status_code=400, message="Not a valid JSON format"
-            )
-        except Exception:
-            err = traceback.format_exc()
-            raise FrogmlHTTPException(
-                status_code=500, message=f"Internal Server Error: {err}"
-            )

frogml/model/tools/adapters/output.py

@@ -1,75 +0,0 @@
-from typing import Any
-
-from frogml.exceptions import FrogmlException, FrogmlHTTPException
-from frogml.model.adapters import (
-    AutodetectOutputAdapter,
-    DataFrameOutputAdapter,
-    DefaultOutputAdapter,
-    JsonOutputAdapter,
-    NumpyOutputAdapter,
-    ProtoOutputAdapter,
-    TfTensorOutputAdapter,
-)
-from frogml.model.base import BaseModel
-
-from .output_adapters.dataframe_output import DataFrameOutput
-from .output_adapters.default_output import DefaultOutput
-from .output_adapters.json_output import JsonOutput
-from .output_adapters.tf_tensor_output import TfTensorOutput
-
-
-def get_output_adapter(
-    model: BaseModel,
-):
-    adapter = getattr(model.predict, "_output_adapter", "")
-
-    class QwakOutput(JsonOutput):
-        def __init__(self, **kwargs):
-            super().__init__(**kwargs)
-
-        def pack_user_func_return_value(self, return_result: Any) -> Any:
-            try:
-                return adapter.pack_user_func_return_value(return_result)
-            except Exception as e:
-                raise FrogmlHTTPException(
-                    message=f"Failed to serialize model output. Error is: {e}. For more "
-                    f"information please check model logs.",
-                    status_code=500,
-                )
-
-    class QwakOutputWithDefaultFallbackOutput(QwakOutput):
-        def __init__(self, **kwargs):
-            super().__init__(**kwargs)
-
-        def pack_user_func_return_value(self, return_result: Any) -> Any:
-            from google.protobuf.message import Message
-
-            first_result = (
-                return_result[0]
-                if type(return_result) == list and len(return_result) > 0
-                else return_result
-            )
-            if issubclass(type(first_result), Message):
-                response = QwakOutput().pack_user_func_return_value(return_result)
-                return response
-            else:
-                return DefaultOutput().pack_user_func_return_value(return_result)
-
-    mapping = {
-        JsonOutputAdapter: JsonOutput,
-        DataFrameOutputAdapter: DataFrameOutput,
-        DefaultOutputAdapter: DefaultOutput,
-        AutodetectOutputAdapter: QwakOutputWithDefaultFallbackOutput,
-        ProtoOutputAdapter: QwakOutput,
-        NumpyOutputAdapter: QwakOutput,
-        TfTensorOutputAdapter: TfTensorOutput,
-    }
-    for qwak_impl, runtime_impl in mapping.items():
-        if isinstance(adapter, qwak_impl):
-            if isinstance(adapter, DataFrameOutputAdapter) and hasattr(
-                adapter, "output_orient"
-            ):
-                return runtime_impl(output_orient=adapter.output_orient)
-            return runtime_impl()
-
-    raise FrogmlException("No Adapter selected for output")

frogml/model/tools/adapters/output_adapters/dataframe_output.py

@@ -1,30 +0,0 @@
-import json
-
-from frogml.exceptions import FrogmlHTTPException
-
-from .json_output import JsonOutput
-
-
-def df_to_json(result, output_orient: str):
-    import pandas as pd
-
-    if isinstance(result, pd.DataFrame):
-        return result.to_json(orient=output_orient)
-
-    if isinstance(result, pd.Series):
-        return pd.DataFrame(result).to_json(orient=output_orient)
-    return json.dumps(result)
-
-
-class DataFrameOutput(JsonOutput):
-    def __init__(self, output_orient: str = "records"):
-        self.output_orient = output_orient
-
-    def pack_user_func_return_value(
-        self,
-        return_result,
-    ) -> str:
-        try:
-            return df_to_json(return_result, self.output_orient)
-        except Exception as e:
-            raise FrogmlHTTPException(message=str(e), status_code=500)

frogml/model/tools/adapters/output_adapters/json_output.py

@@ -1,17 +0,0 @@
-import json
-from typing import Any
-
-from frogml.exceptions import FrogmlHTTPException
-from frogml.model.tools.adapters.encoders import NumpyJsonEncoder
-
-from .base_output import BaseOutputAdapter
-
-
-class JsonOutput(BaseOutputAdapter):
-    def pack_user_func_return_value(self, return_result: Any) -> str:
-        try:
-            return json.dumps(return_result, cls=NumpyJsonEncoder, ensure_ascii=False)
-        except AssertionError as e:
-            FrogmlHTTPException(400, message=str(e))
-        except Exception as e:
-            FrogmlHTTPException(500, message=str(e))

frogml/model/tools/adapters/output_adapters/tf_tensor_output.py

@@ -1,41 +0,0 @@
-import json
-
-from frogml.exceptions import FrogmlHTTPException
-from frogml.model.tools.adapters.encoders import TfTensorJsonEncoder
-
-from .json_output import JsonOutput
-
-
-def tf_to_numpy(tensor):
-    """
-    Tensor -> ndarray
-    List[Tensor] -> tuple[ndarray]
-    """
-    try:
-        import numpy as np  # noqa F401
-        import tensorflow as tf
-    except ImportError:
-        raise ImportError(
-            "Tensorflow and numpy packages are required to use TfTensorOutput"
-        )
-
-    if isinstance(tensor, (list, tuple)):
-        return tuple(tf_to_numpy(t) for t in tensor)
-
-    if tf.__version__.startswith("1."):
-        with tf.compat.v1.Session():
-            return tensor.numpy()
-    else:
-        return tensor.numpy()
-
-
-class TfTensorOutput(JsonOutput):
-    def pack_user_func_return_value(
-        self,
-        return_result,
-    ) -> str:
-        result = tf_to_numpy(return_result)
-        try:
-            return json.dumps(result, cls=TfTensorJsonEncoder)
-        except Exception as e:
-            raise FrogmlHTTPException(message=str(e), status_code=500)

frogml/model/tools/run_model_locally.py

@@ -1,68 +0,0 @@
-import inspect
-import os
-from contextlib import contextmanager
-from typing import Union
-
-from frogml.feature_store.online.client import OnlineClient
-from frogml.model.base import BaseModel
-
-from .adapters.input import get_input_adapter
-from .adapters.output import get_output_adapter
-
-_FS_EXTRACTION = "_fs_extraction"
-_DF = "df"
-_EXTRACTED_DF = "extracted_df"
-
-
-def run_local(model: BaseModel, payload: Union[str, bytes]):
-    """
-    Invokes the .build, .initialize_model, and in order and then invokes the input adapter, calls the predict method
-    and invokes the output adapter.
-    :param model: the model to run locally
-    :param payload: a string or bytes with the input data
-    :return: a string or bytes with the model output
-    """
-    if not (hasattr(model, "initialized") and model.initialized):
-        with _set_local_mode():
-            model.build()
-            model.initialize_model()
-            setattr(model, "initialized", True)
-    input_adapter = get_input_adapter(model)
-    output_adapter = get_output_adapter(model)
-    input_data = input_adapter.extract_user_func_args(payload)
-
-    if getattr(model.predict, _FS_EXTRACTION, False):
-        args_list = list(inspect.signature(model.predict).parameters.keys())
-        if (_DF or _EXTRACTED_DF) not in args_list:
-            raise ValueError(
-                """
-                Missing 'extracted_df' or 'df' arguments in function invocation, even though 'feature_extraction' flag is true.
-                manner:
-                >>> @api(feature_extraction=True)
-                >>> def predict(df: pd.DataFrame, extracted_df: pd.DataFrame):
-                >>>    ...
-                >>>    ...
-                """
-            )
-
-        extracted_df = _extract_online_features(model, input_data)
-        output_data = model.predict(input_data, extracted_df)
-    else:
-        output_data = model.predict(input_data)
-
-    return output_adapter.pack_user_func_return_value(output_data)
-
-
-def _extract_online_features(model, df):
-    return OnlineClient().get_feature_values(
-        schema=model.schema(), df=df, model_name="Test Model"
-    )
-
-
-@contextmanager
-def _set_local_mode():
-    os.environ["QWAK_IS_RUN_LOCAL"] = "true"
-    try:
-        yield
-    finally:
-        del os.environ["QWAK_IS_RUN_LOCAL"]

frogml/model/utils/feature_utils.py

@@ -1,69 +0,0 @@
-import dataclasses
-from typing import List, cast
-
-from frogml.clients.administration.eco_system.eco_system_utils import EcosystemUtils
-from frogml.exceptions import FrogmlException
-from frogml.model.schema_entities import BaseFeature, FeatureStoreInput
-
-
-def validate_and_sanitize_feature_name(feature_name: str, current_env_name: str) -> str:
-    qualifiers = feature_name.split(".")
-    if len(qualifiers) <= 1:
-        raise FrogmlException(
-            f"Feature name '{feature_name}' must adhere to the <featureset_name>.<feature_name> or <env_name>.<featureset_name>.<feature_name> convention"
-        )
-    elif len(qualifiers) == 2:  # env name is not included in the feature name
-        feature_name = f"{current_env_name}.{feature_name}"
-    return feature_name.lower()
-
-
-def validate_and_sanitize_features_name(
-    features: List[BaseFeature],
-) -> List[BaseFeature]:
-    """
-    Verify that the features (from type FeatureStoreInput) names are valid and adhere to convention <featureset_name>.<feature_name> or <env_name>.<featureset_name>.<feature_name>
-    Add the current env qualifier if env is missing and transform name to lowercase
-    :param features:
-    :return: sanitized features with lower case (to support case-insensitive) fully qualified feature names
-    :exception QwakException: if the feature name does not adhere to the convention
-    """
-    ecosystem_utils = EcosystemUtils()
-    current_env_name = ecosystem_utils.get_current_environment_name()
-    return [
-        (
-            cast(
-                FeatureStoreInput,
-                dataclasses.replace(
-                    feature,
-                    name=validate_and_sanitize_feature_name(
-                        feature.name, current_env_name
-                    ),
-                ),
-            )
-            if isinstance(feature, FeatureStoreInput)
-            else feature
-        )
-        for feature in features
-    ]
-
-
-def extract_env_name(feature_name: str) -> str:
-    """
-    extracts env_name from featureset name
-    assuming feature name is fully qualified (passed sanitization)
-    :param feature_name:
-    :return: env_name
-    """
-    qualifiers = feature_name.split(".")
-    return ".".join(qualifiers[:-2])  # Everything except the last two parts
-
-
-def extract_featureset_name(feature_name: str) -> str:
-    qualifiers = feature_name.split(".")
-    return qualifiers[-2]
-
-
-def discard_env_from_name(
-    feature_name: str,
-) -> str:
-    return ".".join(feature_name.split(".")[-2:])

frogml/model_loggers/artifact_logger.py

@@ -1,95 +0,0 @@
-import urllib.request
-from typing import Optional
-
-from _frogml_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
-from frogml.clients.build_orchestrator.client import BuildOrchestratorClient
-from frogml.clients.file_versioning.client import FileVersioningManagementClient
-from frogml.exceptions import FrogmlException
-from frogml.inner.model_loggers_utils import (
-    fetch_build_id,
-    upload_data,
-    validate_model,
-    validate_tag,
-)
-
-
-def log_file(
-    from_path: str,
-    tag: str = "artifact",
-    model_id: Optional[str] = None,
-    build_id: Optional[str] = None,
-) -> None:
-    """
-    Log a file by a given tag
-
-    Args:
-        from_path: file path to log
-        tag: tag to save the file with
-        model_id: optional model id to save data with - if not given found from environment
-        build_id: optional build id - if not given found from environment.
-    """
-    if not validate_tag(tag):
-        raise FrogmlException(
-            "Tag should contain only letters, numbers, underscore or hyphen"
-        )
-
-    model_id = validate_model(model_id)
-    if not build_id:
-        # Checking if called inside a model - then build id saved as environment variable or stays
-        build_id = fetch_build_id()
-
-    upload_url_response = BuildOrchestratorClient().get_build_versioning_upload_url(
-        build_id=build_id,
-        model_id=model_id,
-        tag=tag,
-        tag_type=BuildVersioningTagsType.FILE_TAG_TYPE,
-    )
-
-    FileVersioningManagementClient().register_file_tag(
-        model_id, tag, from_path, build_id
-    )
-
-    with open(from_path, mode="rb") as f:
-        upload_data(
-            upload_url_response.upload_url,
-            f.read(),
-            upload_url_response.authentication_detail,
-        )
-
-
-def load_file(
-    to_path: str,
-    tag: str = "artifact",
-    model_id: Optional[str] = None,
-    build_id: Optional[str] = None,
-) -> str:
-    """
-    Load a file by a given tag
-
-    Args:
-        to_path: the local path the downloaded file will be written to
-        tag: load the artifact with this given tag
-        model_id: optional model id to save data with - if not given found from environment
-        build_id: optional build id - if not given found from environment.
-
-    Returns:
-        the path to the newly created data file
-    """
-    if not validate_tag(tag):
-        raise FrogmlException(
-            "Tag should contain only letters, numbers, underscore or hyphen"
-        )
-
-    model_id = validate_model(model_id)
-    download_url_response = BuildOrchestratorClient().get_build_versioning_download_url(
-        build_id=build_id, model_id=model_id, tag=tag
-    )
-
-    try:
-        filename, headers = urllib.request.urlretrieve(  # nosec B310
-            url=download_url_response.download_url, filename=to_path
-        )
-
-        return filename
-    except Exception as error:
-        raise FrogmlException(f"Unable to load save artifact locally: {str(error)}")