frogml-core 0.0.1__py3-none-any.whl

frogml/model_loggers/data_logger.py

@@ -0,0 +1,172 @@
+import gzip
+import os
+from io import StringIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from _frogml_proto.qwak.builds.build_pb2 import (
+    CsvFormat,
+    DataColumnDefinition,
+    DataColumnType,
+    DataFormat,
+    DataTableDefinition,
+)
+from _frogml_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
+from frogml.clients.build_orchestrator.client import BuildOrchestratorClient
+from frogml.clients.data_versioning.client import DataVersioningManagementClient
+from frogml.exceptions import FrogmlException
+from frogml.inner.model_loggers_utils import (
+    fetch_build_id,
+    upload_data,
+    validate_model,
+    validate_tag,
+)
+
+if TYPE_CHECKING:
+    try:
+        import pandas as pd
+    except ImportError:
+        pass
+
+
+def log_data(
+    dataframe: "pd.DataFrame",
+    tag: str,
+    model_id: Optional[str] = None,
+    build_id: Optional[str] = None,
+    encoding: str = "utf-8",
+) -> None:
+    """
+    Log data by specific parameters
+
+    Args:
+        dataframe: data to log
+        tag: tag to save data with
+        model_id: optional model id to save data with - if not given found from environment
+        build_id: optional build id - if not given found from environment.
+        encoding: Encoding for the dataframe - utf-8 is default.
+    """
+    if os.getenv("FROGML_IS_RUN_LOCAL"):
+        return
+
+    if not validate_tag(tag):
+        raise FrogmlException(
+            "Tag should contain only letters, numbers, underscore or hyphen"
+        )
+
+    data_file_extension = "data.csv.gz"
+    data_tag = Path(tag) / data_file_extension
+    model_id = validate_model(model_id)
+
+    if not build_id:
+        # If called inside a model - build id saved as environment variable
+        build_id = fetch_build_id()
+
+    client = BuildOrchestratorClient()
+    upload_url_response = client.get_build_versioning_upload_url(
+        build_id=build_id,
+        model_id=model_id,
+        tag=str(data_tag),
+        tag_type=BuildVersioningTagsType.DATA_TAG_TYPE,
+    )
+
+    string_buffer = StringIO()
+    dataframe.to_csv(string_buffer, index=False, escapechar="\\", encoding=encoding)
+    upload_data(
+        upload_url=upload_url_response.upload_url,
+        data=gzip.compress(bytes(string_buffer.getvalue(), "utf-8")),
+        content_type="text/plain",
+        authentication_details=upload_url_response.authentication_detail,
+    )
+
+    dataframe_definition = DataTableDefinition(
+        columns=[
+            DataColumnDefinition(name=str(name), type=map_column_type(str(col_type)))
+            for name, col_type in dataframe.dtypes.to_dict().items()
+        ],
+        data_format=DataFormat(
+            csv=CsvFormat(delimiter=",", escape_char="\\", quote_char='"')
+        ),
+    )
+
+    client.define_build_data_table(
+        build_id=build_id, model_id=model_id, tag=tag, table=dataframe_definition
+    )
+
+    DataVersioningManagementClient().register_data_tag(
+        model_id=model_id, build_id=build_id, tag=tag, extension=data_file_extension
+    )
+
+
+def load_data(
+    tag: str,
+    model_id: Optional[str] = None,
+    build_id: Optional[str] = None,
+    encoding: str = "utf-8",
+    compressed: bool = True,
+) -> "pd.DataFrame":
+    """
+    Load data by specific parameters
+
+    Args:
+        tag: tag to load data from
+        model_id: model id to load data with - if not given found from environment
+        build_id: optional build id - if not given found from environment.
+        encoding: Encoding for the dataframe - utf-8 is default.
+        compressed: read a compressed dataframe. Since Frogml version 0.9.39 all logged data frames are compressed by
+         default. In order to read a dataframe persisted with an SDK version prior to 0.9.39, add compressed=False.
+
+    Returns:
+        wanted dataframe
+    """
+    if not validate_tag(tag):
+        raise FrogmlException(
+            "Tag should contain only letters, numbers, underscore or hyphen"
+        )
+
+    file_extension = "csv.gz" if compressed else "csv"
+    data_tag = f"{tag}/data.{file_extension}"
+
+    model_id = validate_model(model_id)
+
+    client = BuildOrchestratorClient()
+    download_url_response = client.get_build_versioning_download_url(
+        build_id=build_id, model_id=model_id, tag=data_tag
+    )
+
+    additional_args = {"compression": "gzip"} if compressed else {}
+    try:
+        import pandas as pd
+    except ImportError:
+        raise FrogmlException(
+            "Missing Pandas dependency required for logging a dataframe"
+        )
+    try:
+        return pd.read_csv(
+            download_url_response.download_url, encoding=encoding, **additional_args
+        )
+    except Exception as error:
+        raise FrogmlException("Unable to load data: %s" % str(error))
+
+
+def map_column_type(type_name: str) -> DataColumnType:
+    data_type = type_mapper.get(type_name, DataColumnType.INVALID_COLUMN_TYPE)
+    if data_type == DataColumnType.INVALID_COLUMN_TYPE:
+        raise FrogmlException(
+            f"Failed to map data type {type_name} to Frogml type, which is required for table registration."
+            f" Supported types are: {list(type_mapper.keys())}"
+        )
+
+    return data_type
+
+
+type_mapper = {
+    "object": DataColumnType.OBJECT,
+    "uint8": DataColumnType.INT,
+    "int64": DataColumnType.INT,
+    "float64": DataColumnType.FLOAT,
+    "datetime64": DataColumnType.DATETIME,
+    "datetime64[ns]": DataColumnType.DATETIME,
+    "datetime64[ns, UTC]": DataColumnType.DATETIME,
+    "bool": DataColumnType.BOOLEAN,
+}

frogml/model_loggers/model_logger.py

@@ -0,0 +1,30 @@
+import os
+import pickle  # nosec B403
+import tempfile
+
+from frogml.model.base import BaseModel
+
+from .artifact_logger import load_file, log_file
+
+MODEL_TAG = "frogml_model"
+FILE_NAME = "frogml_model.pkl"
+
+
+def log_model(build_id: str, model: BaseModel) -> None:
+    with open(FILE_NAME, "wb") as handle:
+        pickle.dump(model, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+    log_file(from_path=FILE_NAME, tag=MODEL_TAG, build_id=build_id, model_id=None)
+
+
+def load_model(model_id: str, build_id: str) -> BaseModel:
+    with tempfile.TemporaryDirectory() as td:
+        temp_file_path = os.path.join(td, FILE_NAME)
+        load_file(
+            to_path=temp_file_path, tag=MODEL_TAG, build_id=build_id, model_id=model_id
+        )
+
+        with open(temp_file_path, "rb") as handle:
+            model = pickle.load(handle)  # nosec B301
+
+    return model

frogml/testing/fixtures.py

@@ -0,0 +1,7 @@
+def real_time_client():
+    """
+    This is a placeholder for a real time client mock. It is used to test the real time client during the build process.
+    During the build process a fixture with this name will be available for testing purposes against the created model.
+    :return: RealTimeClient
+    """
+    pass

frogml/tools/logger/__init__.py

@@ -0,0 +1,3 @@
+from .logger import get_frogml_logger, setup_frogml_logger
+
+__all__ = ["setup_frogml_logger", "get_frogml_logger"]

frogml/tools/logger/logger.py

@@ -0,0 +1,287 @@
+import logging
+import logging.config
+import os
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+from frogml.exceptions import FrogmlException
+
+DEFAULT_LOGGER_NAME = "frogml"
+REMOTE_LOGGER_NAME = "frogml_remote"
+MODEL_LOGGER_NAME = "frogml_model"
+FEATURE_STORE_LOGGER_NAME = "feature_store"
+BUILD_LOCAL_LOGGER_NAME = "build_local"
+DOCKER_INTERNAL_LOGGER_NAME = "docker_internal"
+
+ENVIRON_LOGGER_TYPE = "LOGGER_TYPE"
+
+BUILD_LOCAL_FILE_HANDLER_NAME = "build_log_file_handler"
+FILE_HANDLER_NAME = "file_handler"
+CONSOLE_HANDLER_NAME = "console"
+REMOTE_CONSOLE_HANDLER_NAME = "remote_console"
+
+DEFINED_LOGGER_NAMES = {
+    DEFAULT_LOGGER_NAME,
+    REMOTE_LOGGER_NAME,
+    MODEL_LOGGER_NAME,
+    FEATURE_STORE_LOGGER_NAME,
+    BUILD_LOCAL_LOGGER_NAME,
+    DOCKER_INTERNAL_LOGGER_NAME,
+}
+
+VERBOSITY_LEVEL_MAPPING = {0: logging.CRITICAL, 1: logging.INFO, 2: logging.DEBUG}
+REVERSED_VERBOSITY_LEVEL_MAPPING = {
+    value: key for key, value in VERBOSITY_LEVEL_MAPPING.items()
+}
+
+AIRFLOW_ENV_FLAG = "AIRFLOW__LOGGING__REMOTE_LOGGING"
+LOGGING_CONFIG_FILE_NAME = "logging.yml"
+HANDLERS_KEY = "handlers"
+FILE_HANDLER_KEY = "file_handler"
+FILE_NAME = "filename"
+DISABLE_EXISTING_LOGGERS_KEY = "disable_existing_loggers"
+LOGFILE_DEFAULT_NAME = "frogml.log"
+
+LOGGER_CONFIGURATION_EXCEPTION_MESSAGE = (
+    "Error in Logging Configuration. Error message: {e}"
+)
+LOGGER_NOT_FOUND_MESSAGE = (
+    "Failed to get requested logger name {logger_name}. Using default logger"
+)
+ANOTHER_LOGGER_FOUND_MESSAGE = (
+    "Another logger is already enabled, not changing configuration"
+)
+LOGGER_HANDLER_ADDITION_NOT_FOUND_MESSAGE = (
+    "Tried to set orphan loggers handlers with a non-existing logger name handlers"
+)
+DEFAULT_LOGGER_PATH = Path.home() / ".frogml" / "log" / LOGFILE_DEFAULT_NAME
+
+
+def setup_frogml_logger(
+    logs_folder: str = None,
+    log_file_name: str = LOGFILE_DEFAULT_NAME,
+    logger_name_handler_addition: str = None,
+    disable_existing_loggers: bool = False,
+):
+    """Setup frogml logger:
+            1. Rotating file in $HOME/.frogml/log/sdk.log (10MB * 5 files) (DEBUG level)
+            2. Stdout logger with colored logs. (INFO level)
+
+    Args:
+        logs_folder: Folder where logs will be stored
+        log_file_name: The name of the log file. Default: frogml.log
+        logger_name_handler_addition:
+            Logger name which the handlers of will be appended to all loggers which have handlers
+            Overriding stdout stream handler if exists
+        disable_existing_loggers: disables all existing loggers
+
+    Raises:
+        FrogmlException: If loading logging.yml fails or the preparation of the logging environment raises an exception
+
+    Notes:
+        1. https://docs.python.org/3/library/logging.html#logging-levels
+    """
+
+    logs_folder = logs_folder if logs_folder else DEFAULT_LOGGER_PATH
+    config_file = Path(__file__).parent / LOGGING_CONFIG_FILE_NAME
+    if not non_frogml_logger_enabled():
+        # Creating log directory
+        log_path = _create_logs_path(logs_folder)
+        _load_config(log_file_name, config_file, disable_existing_loggers, log_path)
+        logger = get_frogml_logger()
+
+        if logger_name_handler_addition:
+            if logger_name_handler_addition in logging.Logger.manager.loggerDict:
+                _add_logger_handlers(logger_name_handler_addition)
+            else:
+                logger.warning(LOGGER_HANDLER_ADDITION_NOT_FOUND_MESSAGE)
+
+    else:
+        logger = logging.getLogger(__file__)
+        logger.info(ANOTHER_LOGGER_FOUND_MESSAGE)
+
+
+def _create_logs_path(logs_folder: str) -> Path:
+    log_path = Path(logs_folder)
+    log_path.mkdir(parents=True, exist_ok=True)
+    return log_path
+
+
+def _load_config(
+    log_file_name: str,
+    config_file: Path,
+    disable_existing_loggers: bool,
+    log_path: Path,
+) -> None:
+    with config_file.open(mode="rt") as f:
+        try:
+            # Load logger configuration
+            config = yaml.safe_load(f.read())
+            config[HANDLERS_KEY][FILE_HANDLER_KEY][FILE_NAME] = str(
+                log_path / log_file_name
+            )
+            config[HANDLERS_KEY][BUILD_LOCAL_FILE_HANDLER_NAME][FILE_NAME] = str(
+                log_path / log_file_name
+            )
+            config[DISABLE_EXISTING_LOGGERS_KEY] = disable_existing_loggers
+
+            logging.config.dictConfig(config)
+
+        except Exception as e:
+            raise FrogmlException(LOGGER_CONFIGURATION_EXCEPTION_MESSAGE.format(e=e))
+
+
+def _add_logger_handlers(logger_name: str) -> None:
+    """
+    Add a specific logger handlers to all loggers
+    Override loggers StreamHandler handlers if the input logger has a StreamHandler
+
+    :param logger_name: logger name which consists of the handlers we wish to set
+    """
+
+    requested_logger_handlers = logging.getLogger(logger_name).handlers
+    replace_stdout_handler = any(
+        [
+            handler
+            for handler in requested_logger_handlers
+            if isinstance(handler, logging.StreamHandler)
+        ]
+    )
+
+    for existing_logger in [
+        logger
+        for logger in logging.Logger.manager.loggerDict.values()
+        if not isinstance(logger, logging.PlaceHolder)
+    ]:
+        if existing_logger.handlers:
+            if replace_stdout_handler:
+                existing_handlers = list(
+                    filter(
+                        lambda h: not isinstance(h, logging.StreamHandler),
+                        existing_logger.handlers,
+                    )
+                )
+            else:
+                existing_handlers = existing_logger.handlers
+            existing_logger.handlers = existing_handlers + requested_logger_handlers
+
+
+def get_frogml_logger(
+    logger_name: Optional[str] = None,
+    fallback_logger_name: Optional[str] = DEFAULT_LOGGER_NAME,
+) -> logging.Logger:
+    """Get frogml logger (Singleton)
+    :param logger_name: logger name to get
+    :param fallback_logger_name: fallback logger name to get if logger_name is not defined
+
+    Returns:
+        logging.Logger: Frogml logger.
+    """
+    if not logger_name:
+        logger_name = get_frogml_logger_name(fallback_logger_name)
+
+    if (logger_name not in DEFINED_LOGGER_NAMES) and not non_frogml_logger_enabled():
+        print(LOGGER_NOT_FOUND_MESSAGE.format(logger_name=logger_name))
+
+    return logging.getLogger(logger_name)
+
+
+def get_frogml_logger_name(fallback_logger_name: str) -> str:
+    return os.getenv(ENVIRON_LOGGER_TYPE, fallback_logger_name)
+
+
+def non_frogml_logger_enabled() -> bool:
+    return os.getenv(AIRFLOW_ENV_FLAG) is not None
+
+
+def copy_file_handler_from_existing(
+    handler: RotatingFileHandler, log_file: Path
+) -> RotatingFileHandler:
+    return RotatingFileHandler(
+        log_file,
+        mode=handler.mode,
+        maxBytes=int(handler.maxBytes),
+        backupCount=int(handler.backupCount),
+        encoding=handler.encoding,
+        delay=handler.delay,
+    )
+
+
+def set_frogml_logger_stdout_verbosity_level(verbose: int, format: str = "text"):
+    """Set frogml stdout to verbose (a.k.a DEBUG level)
+
+    Args:
+        verbose: Log verbosity level - 0: WARNING, 1:INFO, 2: DEBUG
+
+
+    Notes:
+        1. https://docs.python.org/3/library/logging.html#logging-levels
+    """
+    if format == "json":
+        verbose = 0
+    logger: logging.Logger = get_frogml_logger()
+    logger.setLevel(VERBOSITY_LEVEL_MAPPING[verbose])
+    for handler in logger.handlers:
+        if isinstance(handler, logging.StreamHandler):
+            handler.setLevel(VERBOSITY_LEVEL_MAPPING[verbose])
+
+
+def get_frogml_logger_verbosity_level() -> Optional[int]:
+    """Get current Frogml logger level.
+
+    Returns:
+        int: Frogml logger level 10 < level < 50.
+
+    Notes:
+        1. https://docs.python.org/3/library/logging.html#logging-levels
+        2. when we update the log level through set_frogml_logger_stdout_verbosity_level we update all handler levels
+           thus returning the first stream handler should be correct
+    """
+
+    logger: logging.Logger = get_frogml_logger()
+
+    for handler in logger.handlers:
+        if isinstance(handler, logging.StreamHandler):
+            return logging.getLevelName(handler.level)
+
+
+def set_handler_verbosity(logger: logging.Logger, handler_name: str, log_level: int):
+    handler = get_handler_from_logger(logger, handler_name)
+    handler.setLevel(log_level)
+
+
+def get_handler_from_logger(
+    logger: logging.Logger, handler_name: str
+) -> logging.Handler:
+    matching_handlers = list(
+        filter(lambda h: h.get_name() == handler_name, logger.handlers)
+    )
+    if len(matching_handlers) == 0 and not non_frogml_logger_enabled:
+        raise FrogmlException(
+            f"Error in setting log file. Error message: handler of name {handler_name} was not found in logger"
+        )
+    elif len(matching_handlers) > 1:
+        raise FrogmlException(
+            f"Error in setting log file. Error message: handler of name {handler_name} was found more than once "
+            f"in logger"
+        )
+
+    return matching_handlers[0]
+
+
+def set_file_handler_log_file(
+    logger: logging.Logger, handler_name: str, log_file: Path
+):
+    existing_handler = get_handler_from_logger(logger, handler_name)
+    if type(existing_handler) != RotatingFileHandler:
+        raise FrogmlException(
+            f"Error in setting log file. Error message: handler of name {handler_name} is not a file logger handler"
+        )
+    replacement_handler: RotatingFileHandler = copy_file_handler_from_existing(
+        existing_handler, log_file
+    )
+    logger.removeHandler(existing_handler)
+    logger.addHandler(replacement_handler)

frogml/tools/logger/logging.yml

@@ -0,0 +1,79 @@
+version: 1
+disable_existing_loggers: false
+
+formatters:
+  brief:
+    format: '%(message)s'
+  semi_precise:
+    format: "%(asctime)s [%(levelname)-5.5s] %(message)s"
+    datefmt: '%Y-%m-%d %H:%M:%S'
+  precise:
+    format: '%(asctime)s %(levelname)-8s %(module)s - %(lineno)d - %(message)s'
+    datefmt: '%Y-%m-%d %H:%M:%S'
+  remote:
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
+    format: '%(asctime)s %(process)d %(levelname)s %(module)s %(lineno)d %(message)s'
+    datefmt: '%Y-%m-%dT%H:%M:%S'
+  json:
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
+    format: '%(asctime)s %(process)d %(levelname)s %(module)s %(lineno)d %(message)s'
+    datefmt: '%Y-%m-%dT%H:%M:%S'
+
+handlers:
+  console:
+    class : logging.StreamHandler
+    formatter: brief
+    level   : INFO
+    stream  : ext://sys.stdout
+  file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: precise
+    filename: ""
+    maxBytes: 10485760 # 10MB
+    backupCount: 5
+    encoding: utf8
+  remote_console:
+    class: logging.StreamHandler
+    formatter: remote
+    level: DEBUG
+    stream: ext://sys.stdout
+  model:
+    class: logging.StreamHandler
+    formatter: json
+    level: INFO
+    stream: ext://sys.stdout
+  build_log_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: semi_precise
+    filename: ""
+    maxBytes: 10485760 # 10MB
+    backupCount: 5
+    encoding: utf8
+
+loggers:
+  qwak:
+    level: DEBUG
+    handlers: [console, file_handler]
+    propagate: false
+  qwak_remote:
+    level: DEBUG
+    handlers: [remote_console]
+    propagate: false
+  qwak_model:
+    level: DEBUG
+    handlers: [model]
+    propagate: false
+  feature_store:
+    level: DEBUG
+    handlers: [model]
+    propagate: false
+  build_local:
+    level: DEBUG
+    handlers: [build_log_file_handler, console]
+    propagate: false
+  docker_internal:
+    level: DEBUG
+    handlers: [console]
+    propagate: false

frogml/utils/datetime_utils.py

@@ -0,0 +1,18 @@
+from datetime import datetime, timezone
+from typing import Optional
+
+from google.protobuf.timestamp_pb2 import Timestamp as ProtoTimestamp
+
+
+def datetime_to_pts(dtime: Optional[datetime]) -> Optional[ProtoTimestamp]:
+    """
+    converts a python datetime to Protobuf Timestamp
+    @param dtime: python datetime.datetime
+    @return: if the input is None, returns None. else converts to proto timestamp in utc timezone
+    """
+    if not dtime:
+        return None
+    res: ProtoTimestamp = ProtoTimestamp()
+    res.FromDatetime(dtime.replace(tzinfo=timezone.utc))
+
+    return res

frogml/utils/dict_utils.py

@@ -0,0 +1,12 @@
+from copy import deepcopy
+from typing import Any, Dict
+
+
+def remove_none_value_keys(dict: Dict[Any, Any]):
+    copy = deepcopy(dict)
+    keys_to_remove = [k for k, v in dict.items() if v is None]
+
+    for k in keys_to_remove:
+        del copy[k]
+
+    return copy

frogml/vector_store/__init__.py

@@ -0,0 +1,4 @@
+from .client import VectorStoreClient
+from .collection import Collection
+
+__all__ = ["VectorStoreClient", "Collection"]