frogml-core 0.0.116__py3-none-any.whl → 0.0.118__py3-none-any.whl

frogml_core/model/tools/run_model_locally.py

@@ -1,68 +0,0 @@
-import inspect
-import os
-from contextlib import contextmanager
-from typing import Union
-
-from frogml_core.feature_store.online.client import OnlineClient
-from frogml_core.model.base import BaseModel
-
-from .adapters.input import get_input_adapter
-from .adapters.output import get_output_adapter
-
-_FS_EXTRACTION = "_fs_extraction"
-_DF = "df"
-_EXTRACTED_DF = "extracted_df"
-
-
-def run_local(model: BaseModel, payload: Union[str, bytes]):
-    """
-    Invokes the .build, .initialize_model, and in order and then invokes the input adapter, calls the predict method
-    and invokes the output adapter.
-    :param model: the model to run locally
-    :param payload: a string or bytes with the input data
-    :return: a string or bytes with the model output
-    """
-    if not (hasattr(model, "initialized") and model.initialized):
-        with _set_local_mode():
-            model.build()
-            model.initialize_model()
-            setattr(model, "initialized", True)
-    input_adapter = get_input_adapter(model)
-    output_adapter = get_output_adapter(model)
-    input_data = input_adapter.extract_user_func_args(payload)
-
-    if getattr(model.predict, _FS_EXTRACTION, False):
-        args_list = list(inspect.signature(model.predict).parameters.keys())
-        if (_DF or _EXTRACTED_DF) not in args_list:
-            raise ValueError(
-                """
-                Missing 'extracted_df' or 'df' arguments in function invocation, even though 'feature_extraction' flag is true.
-                manner:
-                >>> @api(feature_extraction=True)
-                >>> def predict(df: pd.DataFrame, extracted_df: pd.DataFrame):
-                >>>    ...
-                >>>    ...
-                """
-            )
-
-        extracted_df = _extract_online_features(model, input_data)
-        output_data = model.predict(input_data, extracted_df)
-    else:
-        output_data = model.predict(input_data)
-
-    return output_adapter.pack_user_func_return_value(output_data)
-
-
-def _extract_online_features(model, df):
-    return OnlineClient().get_feature_values(
-        schema=model.schema(), df=df, model_name="Test Model"
-    )
-
-
-@contextmanager
-def _set_local_mode():
-    os.environ["FROGML_IS_RUN_LOCAL"] = "true"
-    try:
-        yield
-    finally:
-        del os.environ["FROGML_IS_RUN_LOCAL"]

frogml_core/model/utils/feature_utils.py

@@ -1,71 +0,0 @@
-import dataclasses
-from typing import List, cast
-
-from frogml_core.clients.administration.eco_system.eco_system_utils import (
-    EcosystemUtils,
-)
-from frogml_core.exceptions import FrogmlException
-from frogml_core.model.schema_entities import BaseFeature, FeatureStoreInput
-
-
-def validate_and_sanitize_feature_name(feature_name: str, current_env_name: str) -> str:
-    qualifiers = feature_name.split(".")
-    if len(qualifiers) <= 1:
-        raise FrogmlException(
-            f"Feature name '{feature_name}' must adhere to the <featureset_name>.<feature_name> or <env_name>.<featureset_name>.<feature_name> convention"
-        )
-    elif len(qualifiers) == 2:  # env name is not included in the feature name
-        feature_name = f"{current_env_name}.{feature_name}"
-    return feature_name.lower()
-
-
-def validate_and_sanitize_features_name(
-    features: List[BaseFeature],
-) -> List[BaseFeature]:
-    """
-    Verify that the features (from type FeatureStoreInput) names are valid and adhere to convention <featureset_name>.<feature_name> or <env_name>.<featureset_name>.<feature_name>
-    Add the current env qualifier if env is missing and transform name to lowercase
-    :param features:
-    :return: sanitized features with lower case (to support case-insensitive) fully qualified feature names
-    :exception FrogmlException: if the feature name does not adhere to the convention
-    """
-    ecosystem_utils = EcosystemUtils()
-    current_env_name = ecosystem_utils.get_current_environment_name()
-    return [
-        (
-            cast(
-                FeatureStoreInput,
-                dataclasses.replace(
-                    feature,
-                    name=validate_and_sanitize_feature_name(
-                        feature.name, current_env_name
-                    ),
-                ),
-            )
-            if isinstance(feature, FeatureStoreInput)
-            else feature
-        )
-        for feature in features
-    ]
-
-
-def extract_env_name(feature_name: str) -> str:
-    """
-    extracts env_name from featureset name
-    assuming feature name is fully qualified (passed sanitization)
-    :param feature_name:
-    :return: env_name
-    """
-    qualifiers = feature_name.split(".")
-    return ".".join(qualifiers[:-2])  # Everything except the last two parts
-
-
-def extract_featureset_name(feature_name: str) -> str:
-    qualifiers = feature_name.split(".")
-    return qualifiers[-2]
-
-
-def discard_env_from_name(
-    feature_name: str,
-) -> str:
-    return ".".join(feature_name.split(".")[-2:])

frogml_core/model_loggers/artifact_logger.py

@@ -1,128 +0,0 @@
-import logging
-from datetime import timedelta
-from typing import Optional
-import requests
-from requests import Response
-
-from frogml_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
-from frogml_core.clients.build_orchestrator.client import BuildOrchestratorClient
-from frogml_core.clients.file_versioning.client import FileVersioningManagementClient
-from frogml_core.exceptions import FrogmlException
-from frogml_core.inner.model_loggers_utils import (
-    fetch_build_id,
-    upload_data,
-    validate_model,
-    validate_tag,
-)
-from frogml_proto.qwak.builds.builds_orchestrator_service_pb2 import (
-    GetBuildVersioningUploadURLResponse,
-    GetBuildVersioningDownloadURLResponse,
-)
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-MAX_CHUNK_SIZE = 8 * 1_024
-
-
-def log_file(
-    from_path: str,
-    tag: str = "artifact",
-    model_id: Optional[str] = None,
-    build_id: Optional[str] = None,
-) -> None:
-    """
-    Log a file by a given tag
-
-    Args:
-        from_path: file path to log
-        tag: tag to save the file with
-        model_id: optional model id to save data with - if not given found from environment
-        build_id: optional build id - if not given found from environment.
-    """
-    if not validate_tag(tag):
-        raise FrogmlException(
-            "Tag should contain only letters, numbers, underscore or hyphen"
-        )
-
-    model_id = validate_model(model_id)
-    if not build_id:
-        # Checking if called inside a model - then build id saved as environment variable or stays
-        build_id = fetch_build_id()
-
-    upload_url_response: (
-        GetBuildVersioningUploadURLResponse
-    ) = BuildOrchestratorClient().get_build_versioning_upload_url(
-        build_id=build_id,
-        model_id=model_id,
-        tag=tag,
-        tag_type=BuildVersioningTagsType.FILE_TAG_TYPE,
-    )
-
-    FileVersioningManagementClient().register_file_tag(
-        model_id, tag, from_path, build_id
-    )
-
-    with open(from_path, mode="rb") as f:
-        upload_data(
-            upload_url_response.upload_url,
-            f.read(),
-            upload_url_response.headers,
-        )
-
-
-def load_file(
-    to_path: str,
-    tag: str = "artifact",
-    model_id: Optional[str] = None,
-    build_id: Optional[str] = None,
-) -> str:
-    """
-    Load a file by a given tag
-
-    Args:
-        to_path: the local path the downloaded file will be written to
-        tag: load the artifact with this given tag
-        model_id: optional model id to save data with - if not given found from environment
-        build_id: optional build id - if not given found from environment.
-
-    Returns:
-        the path to the newly created data file
-    """
-    logger.info(f"Loading file to {to_path}")
-    if not validate_tag(tag):
-        raise FrogmlException(
-            "Tag should contain only letters, numbers, underscore or hyphen"
-        )
-
-    model_id = validate_model(model_id)
-    download_url_response: (
-        GetBuildVersioningDownloadURLResponse
-    ) = BuildOrchestratorClient().get_build_versioning_download_url(
-        build_id=build_id,
-        model_id=model_id,
-        tag=tag,
-        tag_type=BuildVersioningTagsType.FILE_TAG_TYPE,
-    )
-
-    try:
-        response: Response = requests.get(
-            download_url_response.download_url,
-            headers=download_url_response.headers,
-            stream=True,
-            timeout=(
-                timedelta(seconds=10).total_seconds(),  # timeout to connect
-                timedelta(minutes=20).total_seconds(),  # timeout to read
-            ),
-        )
-        logger.info(f"Downloading file finished with status {response.status_code}")
-        response.raise_for_status()
-
-        with open(to_path, "wb") as f:
-            for chunk in response.iter_content(chunk_size=MAX_CHUNK_SIZE):
-                if chunk:
-                    f.write(chunk)
-
-        return to_path
-    except Exception as error:
-        raise FrogmlException(f"Unable to load save artifact locally: {str(error)}")

frogml_core/model_loggers/data_logger.py

@@ -1,177 +0,0 @@
-import gzip
-import os
-from io import StringIO
-from pathlib import Path
-from typing import TYPE_CHECKING, Optional
-
-from frogml_proto.qwak.builds.build_pb2 import (
-    CsvFormat,
-    DataColumnDefinition,
-    DataColumnType,
-    DataFormat,
-    DataTableDefinition,
-)
-from frogml_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
-from frogml_proto.qwak.builds.builds_orchestrator_service_pb2 import (
-    GetBuildVersioningUploadURLResponse,
-)
-from frogml_core.clients.build_orchestrator.client import BuildOrchestratorClient
-from frogml_core.clients.data_versioning.client import DataVersioningManagementClient
-from frogml_core.exceptions import FrogmlException
-from frogml_core.inner.model_loggers_utils import (
-    fetch_build_id,
-    upload_data,
-    validate_model,
-    validate_tag,
-)
-
-if TYPE_CHECKING:
-    try:
-        import pandas as pd
-    except ImportError:
-        pass
-
-
-def log_data(
-    dataframe: "pd.DataFrame",
-    tag: str,
-    model_id: Optional[str] = None,
-    build_id: Optional[str] = None,
-    encoding: str = "utf-8",
-) -> None:
-    """
-    Log data by specific parameters
-
-    Args:
-        dataframe: data to log
-        tag: tag to save data with
-        model_id: optional model id to save data with - if not given found from environment
-        build_id: optional build id - if not given found from environment.
-        encoding: Encoding for the dataframe - utf-8 is default.
-    """
-    if os.getenv("FROGML_IS_RUN_LOCAL"):
-        return
-
-    if not validate_tag(tag):
-        raise FrogmlException(
-            "Tag should contain only letters, numbers, underscore or hyphen"
-        )
-
-    data_file_extension = "data.csv.gz"
-    data_tag = Path(tag) / data_file_extension
-    model_id = validate_model(model_id)
-
-    if not build_id:
-        # If called inside a model - build id saved as environment variable
-        build_id = fetch_build_id()
-
-    client = BuildOrchestratorClient()
-    upload_url_response: GetBuildVersioningUploadURLResponse = (
-        client.get_build_versioning_upload_url(
-            build_id=build_id,
-            model_id=model_id,
-            tag=str(data_tag),
-            tag_type=BuildVersioningTagsType.DATA_TAG_TYPE,
-        )
-    )
-
-    string_buffer = StringIO()
-    dataframe.to_csv(string_buffer, index=False, escapechar="\\", encoding=encoding)
-    upload_data(
-        upload_url=upload_url_response.upload_url,
-        data=gzip.compress(bytes(string_buffer.getvalue(), "utf-8")),
-        content_type="text/plain",
-        headers=upload_url_response.headers,
-    )
-
-    dataframe_definition = DataTableDefinition(
-        columns=[
-            DataColumnDefinition(name=str(name), type=map_column_type(str(col_type)))
-            for name, col_type in dataframe.dtypes.to_dict().items()
-        ],
-        data_format=DataFormat(
-            csv=CsvFormat(delimiter=",", escape_char="\\", quote_char='"')
-        ),
-    )
-
-    client.define_build_data_table(
-        build_id=build_id, model_id=model_id, tag=tag, table=dataframe_definition
-    )
-
-    DataVersioningManagementClient().register_data_tag(
-        model_id=model_id, build_id=build_id, tag=tag, extension=data_file_extension
-    )
-
-
-def load_data(
-    tag: str,
-    model_id: Optional[str] = None,
-    build_id: Optional[str] = None,
-    encoding: str = "utf-8",
-    compressed: bool = True,
-) -> "pd.DataFrame":
-    """
-    Load data by specific parameters
-
-    Args:
-        tag: tag to load data from
-        model_id: model id to load data with - if not given found from environment
-        build_id: optional build id - if not given found from environment.
-        encoding: Encoding for the dataframe - utf-8 is default.
-        compressed: read a compressed dataframe. Since Frogml version 0.9.39 all logged data frames are compressed by
-         default. In order to read a dataframe persisted with an SDK version prior to 0.9.39, add compressed=False.
-
-    Returns:
-        wanted dataframe
-    """
-    if not validate_tag(tag):
-        raise FrogmlException(
-            "Tag should contain only letters, numbers, underscore or hyphen"
-        )
-
-    file_extension = "csv.gz" if compressed else "csv"
-    data_tag = f"{tag}/data.{file_extension}"
-
-    model_id = validate_model(model_id)
-
-    client = BuildOrchestratorClient()
-    download_url_response = client.get_build_versioning_download_url(
-        build_id=build_id, model_id=model_id, tag=data_tag
-    )
-
-    additional_args = {"compression": "gzip"} if compressed else {}
-    try:
-        import pandas as pd
-    except ImportError:
-        raise FrogmlException(
-            "Missing Pandas dependency required for logging a dataframe"
-        )
-    try:
-        return pd.read_csv(
-            download_url_response.download_url, encoding=encoding, **additional_args
-        )
-    except Exception as error:
-        raise FrogmlException("Unable to load data: %s" % str(error))
-
-
-def map_column_type(type_name: str) -> DataColumnType:
-    data_type = type_mapper.get(type_name, DataColumnType.INVALID_COLUMN_TYPE)
-    if data_type == DataColumnType.INVALID_COLUMN_TYPE:
-        raise FrogmlException(
-            f"Failed to map data type {type_name} to Frogml type, which is required for table registration."
-            f" Supported types are: {list(type_mapper.keys())}"
-        )
-
-    return data_type
-
-
-type_mapper = {
-    "object": DataColumnType.OBJECT,
-    "uint8": DataColumnType.INT,
-    "int64": DataColumnType.INT,
-    "float64": DataColumnType.FLOAT,
-    "datetime64": DataColumnType.DATETIME,
-    "datetime64[ns]": DataColumnType.DATETIME,
-    "datetime64[ns, UTC]": DataColumnType.DATETIME,
-    "bool": DataColumnType.BOOLEAN,
-}

frogml_core/model_loggers/model_logger.py

@@ -1,30 +0,0 @@
-import os
-import pickle  # nosec B403
-import tempfile
-
-from frogml_core.model.base import BaseModel
-
-from .artifact_logger import load_file, log_file
-
-MODEL_TAG = "frogml_model"
-FILE_NAME = "frogml_model.pkl"
-
-
-def log_model(build_id: str, model: BaseModel) -> None:
-    with open(FILE_NAME, "wb") as handle:
-        pickle.dump(model, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-    log_file(from_path=FILE_NAME, tag=MODEL_TAG, build_id=build_id, model_id=None)
-
-
-def load_model(model_id: str, build_id: str) -> BaseModel:
-    with tempfile.TemporaryDirectory() as td:
-        temp_file_path = os.path.join(td, FILE_NAME)
-        load_file(
-            to_path=temp_file_path, tag=MODEL_TAG, build_id=build_id, model_id=model_id
-        )
-
-        with open(temp_file_path, "rb") as handle:
-            model = pickle.load(handle)  # nosec B301
-
-    return model