frogml-cli 0.0.1__py3-none-any.whl

frogml_cli/inner/tools/cli_tools.py

@@ -0,0 +1,179 @@
+import functools
+import time
+import warnings
+from collections import namedtuple
+from typing import Any, Callable, Dict, List, Tuple, Union
+
+import click
+from click import Context, OptionParser
+from frogml.core.exceptions import FrogmlException
+from frogml.core.inner.di_configuration import UserAccountConfiguration
+
+from frogml_cli.inner.tools.tracking import log_event
+
+
+def get_user_identifiers(account: UserAccountConfiguration):
+    try:
+        user_id = account.get_user_config().token
+        identifier_cls = namedtuple("Identifier", ["user_id"])
+        return identifier_cls(user_id)
+
+    except FrogmlException:
+        # User might not be registered or have a valid API key
+        return None
+
+
+def usage_statistics_wrapper(func: Callable):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        ctx = click.get_current_context()
+        start_time = time.time()
+
+        account = UserAccountConfiguration()
+        user_identifiers = get_user_identifiers(account)
+
+        # If we could not get user identifiers for some reason (for example - user not logged in) - do not track
+        if not user_identifiers:
+            return func(*args, **kwargs)
+
+        event_properties = {
+            "command_group": ctx.command_path,
+            "commands": ctx.command.name,
+            "duration": time.time() - start_time,
+        }
+        try:
+            return_value = func(*args, **kwargs)
+
+            event_properties["response"] = return_value
+            event_properties["status"] = 0
+            log_event(
+                event_properties,
+                user_id=user_identifiers.user_id,
+            )
+
+            return return_value
+
+        except BaseException as e:
+            command_status = 1
+            if type(e) == KeyboardInterrupt:
+                command_status = 1
+
+            event_properties["error_type"] = type(e).__name__
+            event_properties["error_message"] = str(e)
+            event_properties["status"] = command_status
+
+            log_event(
+                event_properties,
+                user_id=user_identifiers.user_id,
+            )
+            raise
+
+    return wrapper
+
+
+class DeprecatedOption(click.Option):
+    def __init__(self, *args, **kwargs):
+        self.deprecated = kwargs.pop("deprecated", ())
+        self.preferred = kwargs.pop("preferred", args[0][-1])
+        super(DeprecatedOption, self).__init__(*args, **kwargs)
+
+
+class FrogMLCommand(click.Command):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def make_parser(self, ctx: Context) -> OptionParser:
+        parser = super(FrogMLCommand, self).make_parser(ctx)
+
+        # get the parser options
+        options = set(parser._short_opt.values())
+        options |= set(parser._long_opt.values())
+
+        for option in options:
+            if not isinstance(option.obj, DeprecatedOption):
+                continue
+
+            def make_process(an_option):
+                """Construct a closure to the parser option processor"""
+
+                orig_process = an_option.process
+                deprecated = getattr(an_option.obj, "deprecated", None)
+                preferred = getattr(an_option.obj, "preferred", None)
+
+                if not deprecated:
+                    f"Expected `deprecated` value for `{an_option.obj.name}`"
+
+                def process(value, state):
+                    """The function above us on the stack used 'opt' to
+                    pick option from a dict, see if it is deprecated"""
+
+                    # reach up the stack and get 'opt'
+                    import inspect
+
+                    frame = inspect.currentframe()
+                    try:
+                        opt = frame.f_back.f_locals.get("opt")
+                    finally:
+                        del frame
+
+                    if opt in deprecated:
+                        warnings.warn(
+                            f"{opt} has been deprecated, use {preferred}."
+                            f" {opt} will be removed in future releases.",
+                            DeprecationWarning,
+                        )
+
+                    return orig_process(value, state)
+
+                return process
+
+            option.process = make_process(option)
+
+        return parser
+
+    @usage_statistics_wrapper
+    def invoke(self, ctx: click.Context) -> Any:
+        return super().invoke(ctx)
+
+
+def ask_yesno(
+    question: str, force: bool, print_callback: Callable[[str], None] = print
+) -> bool:
+    """
+    Helper to get yes / no answer from user.
+
+    Args:
+        question: yes/no question to show the user
+        force: automatically returns True if force is True
+        print_callback: function to use for printing
+
+    Returns:
+        True/False by user input (unless forced and then always True)
+    """
+
+    print_callback(f"{question}")
+    if force:
+        print_callback("Forced yes")
+        return True
+    return click.confirm("continue?")
+
+
+def dictify_params(env_vars: Union[Dict[str, str], List[str], Tuple[str]]):
+    result = dict()
+
+    if env_vars is None:
+        return result
+
+    if isinstance(env_vars, dict):
+        return env_vars
+
+    if isinstance(env_vars, (list, tuple)):
+        for env_var in env_vars:
+            if "=" not in env_var:
+                raise FrogmlException(
+                    f'The environment variable definition passed {env_var} is invalid. Format is "KEY=VALUE"'
+                )
+            split_param = env_var.split("=")
+            result[split_param[0]] = split_param[1]
+
+    return result

frogml_cli/inner/tools/config_handler.py

@@ -0,0 +1,29 @@
+from typing import Any, Tuple, Union
+
+from frogml.core.clients.build_orchestrator import BuildOrchestratorClient
+from frogml.core.clients.instance_template.client import (
+    InstanceTemplateManagementClient,
+)
+from frogml.core.inner.build_config.build_config_v1 import BuildConfigV1
+from frogml.core.inner.tool.run_config import FrogmlConfigBase, YamlConfigMixin
+
+
+def config_handler(
+    config: Union[FrogmlConfigBase, YamlConfigMixin, Any],
+    from_file: str,
+    out_conf: bool,
+    sections: Tuple[str, ...] = (),
+    **kwargs,
+) -> Any:
+    conf: Union[FrogmlConfigBase, YamlConfigMixin] = config.from_yaml(from_file)
+    conf.merge_cli_argument(sections=sections, **kwargs)
+
+    if isinstance(conf, BuildConfigV1):
+        conf.fetch_base_docker_image_name(
+            BuildOrchestratorClient(), InstanceTemplateManagementClient()
+        )
+
+    if out_conf:
+        print(conf.to_yaml())
+
+    return conf

frogml_cli/inner/tools/dataclasses_utils.py

@@ -0,0 +1,21 @@
+import dataclasses
+from typing import Any, Dict, List, Type, Union
+
+
+def create_dataclass_instance(dataclass_type: Type, input_dict: Dict[str, Any]) -> Any:
+    fields = {f.name: f.type for f in dataclasses.fields(dataclass_type)}
+
+    def construct_dataclass(
+        dc_type: Union[Type, List], value: Any
+    ) -> Union[Type, List]:
+        if dataclasses.is_dataclass(dc_type):
+            return create_dataclass_instance(dc_type, value)
+        elif getattr(dc_type, "__origin__", None) is list:  # This is a List
+            element_type = dc_type.__args__[0]
+            return [construct_dataclass(element_type, v) for v in value]
+        else:
+            return value
+
+    return dataclass_type(
+        **{k: construct_dataclass(fields[k], v) for k, v in input_dict.items()}
+    )

frogml_cli/inner/tools/logger/__init__.py

@@ -0,0 +1,3 @@
+from .logger import setup_frogml_logger
+
+__all__ = ["setup_frogml_logger"]

frogml_cli/inner/tools/logger/logger.py

@@ -0,0 +1,278 @@
+import logging
+import logging.config
+import os
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Optional
+
+import yaml
+from frogml.core.exceptions import FrogmlException
+
+DEFAULT_LOGGER_NAME = "frogml"
+REMOTE_LOGGER_NAME = "frogml_remote"
+MODEL_LOGGER_NAME = "frogml_model"
+FEATURE_STORE_LOGGER_NAME = "feature_store"
+BUILD_LOCAL_LOGGER_NAME = "build_local"
+DOCKER_INTERNAL_LOGGER_NAME = "docker_internal"
+
+ENVIRON_LOGGER_TYPE = "LOGGER_TYPE"
+
+BUILD_LOCAL_FILE_HANDLER_NAME = "build_log_file_handler"
+FILE_HANDLER_NAME = "file_handler"
+CONSOLE_HANDLER_NAME = "console"
+REMOTE_CONSOLE_HANDLER_NAME = "remote_console"
+
+DEFINED_LOGGER_NAMES = {
+    DEFAULT_LOGGER_NAME,
+    REMOTE_LOGGER_NAME,
+    MODEL_LOGGER_NAME,
+    FEATURE_STORE_LOGGER_NAME,
+    BUILD_LOCAL_LOGGER_NAME,
+    DOCKER_INTERNAL_LOGGER_NAME,
+}
+
+VERBOSITY_LEVEL_MAPPING = {0: logging.CRITICAL, 1: logging.INFO, 2: logging.DEBUG}
+REVERSED_VERBOSITY_LEVEL_MAPPING = {
+    value: key for key, value in VERBOSITY_LEVEL_MAPPING.items()
+}
+
+AIRFLOW_ENV_FLAG = "AIRFLOW__LOGGING__REMOTE_LOGGING"
+
+
+def setup_frogml_logger(
+    default_level: int = logging.INFO,
+    logger_name_handler_addition: str = None,
+    disable_existing_loggers: bool = False,
+):
+    """Setup frogml logger:
+            1. Rotating file in $HOME/.frogml/log/sdk.log (10MB * 5 files) (DEBUG level)
+            2. Stdout logger with colored logs. (INFO level)
+
+    Args:
+        default_level: Default logging level in case of failure
+        logger_name_handler_addition:
+            Logger name which the handlers of will be appended to all loggers which have handlers
+            Overriding stdout stream handler if exists
+        disable_existing_loggers: disables all existing loggers
+
+    Raises:
+        FrogmlException: If loading logging.yml fails or the preparation of the logging environment raises an exceptions
+
+    Notes:
+        1. https://docs.python.org/3/library/logging.html#logging-levels
+    """
+
+    config_file = Path(__file__).parent / "logging.yml"
+    if config_file.exists() and not non_frogml_logger_enabled():
+        with config_file.open(mode="rt") as f:
+            try:
+                # Creating log directory
+                log_path = Path(__get_config_folder())
+                log_path.mkdir(parents=True, exist_ok=True)
+                # Load logger configuration
+                config = yaml.safe_load(f.read())
+                config["handlers"][FILE_HANDLER_NAME]["filename"] = str(
+                    log_path / "sdk.log"
+                )
+                config["handlers"][BUILD_LOCAL_FILE_HANDLER_NAME]["filename"] = str(
+                    log_path / "sdk.log"
+                )
+                config["disable_existing_loggers"] = disable_existing_loggers
+
+                logging.config.dictConfig(config)
+
+                if logger_name_handler_addition:
+                    if (
+                        logger_name_handler_addition
+                        in logging.Logger.manager.loggerDict
+                    ):
+                        _add_logger_handlers(logger_name_handler_addition)
+                    else:
+                        print(
+                            "Tried to set orphan loggers handlers with a non-existing logger name handlers'"
+                        )
+
+            except Exception as e:
+                raise FrogmlException(
+                    f"Error in Logging Configuration. Error message: {e}"
+                )
+    else:
+        logging.basicConfig(level=default_level)
+        print("Failed to load configuration file. Using default configs")
+
+
+def __get_config_folder() -> str:
+    frogml_home_dir: str = (
+        os.getenv("FROGML_HOME")
+        if os.getenv("FROGML_HOME") is not None
+        else f"{str(Path.home())}"
+    )
+
+    return f"{frogml_home_dir}/.frogml"
+
+
+def _add_logger_handlers(logger_name):
+    """
+    Add a specific logger handlers to all loggers
+    Override loggers StreamHandler handlers if the input logger has a StreamHandler
+
+    Args:
+        logger_name: logger name which consists of the handlers we wish to set
+    """
+
+    logger_handlers = logging.getLogger(logger_name).handlers
+    base_contains_stdout = any(
+        [
+            handler
+            for handler in logger_handlers
+            if type(handler) is logging.StreamHandler
+        ]
+    )
+
+    for logger in [
+        logger
+        for logger in logging.Logger.manager.loggerDict.values()
+        if not isinstance(logger, logging.PlaceHolder)
+    ]:
+        if logger.handlers:
+            logger.handlers = [
+                handler
+                for handler in logger.handlers
+                if base_contains_stdout and type(handler) is not logging.StreamHandler
+            ] + logger_handlers
+
+
+def get_frogml_logger(
+    logger_name: Optional[str] = None, fallback_logger_name: Optional[str] = None
+) -> logging.Logger:
+    """Get frogml logger (Singleton)
+
+    Returns:
+        logging.Logger: Frogml logger.
+    """
+    if not logger_name:
+        logger_name = get_frogml_logger_name(fallback_logger_name)
+
+    if (logger_name not in DEFINED_LOGGER_NAMES) and not non_frogml_logger_enabled:
+        print("Failed to get requested logger name. Using default logger")
+
+    return logging.getLogger(logger_name)
+
+
+def set_frogml_logger_stdout_verbosity_level(verbose: int, format: str = "text"):
+    """Set frogml stdout to verbose (a.k.a DEBUG level)
+
+    Args:
+        verbose: Log verbosity level - 0: WARNING, 1:INFO, 2: DEBUG
+
+    Notes:
+        1. https://docs.python.org/3/library/logging.html#logging-levels
+    """
+    if format == "json":
+        verbose = 0
+    logger: logging.Logger = get_frogml_logger()
+    logger.setLevel(VERBOSITY_LEVEL_MAPPING[verbose])
+    for handler in logger.handlers:
+        if isinstance(handler, logging.StreamHandler):
+            handler.setLevel(VERBOSITY_LEVEL_MAPPING[verbose])
+
+
+def get_frogml_logger_verbosity_level() -> Optional[int]:
+    """Get current Frogml logger level.
+
+    Returns:
+        int: Frogml logger level 10 < level < 50.
+
+    Notes:
+        1. https://docs.python.org/3/library/logging.html#logging-levels
+        2. when we update the log level through set_frogml_logger_stdout_verbosity_level we update all handler levels
+           thus returning the first stream handler should be correct
+    """
+
+    logger: logging.Logger = get_frogml_logger()
+
+    for handler in logger.handlers:
+        if isinstance(handler, logging.StreamHandler):
+            return logging.getLevelName(handler.level)
+
+
+def set_frogml_logger_propagate(prop: bool):
+    """Set frogml logger propagation
+
+    Args:
+        prop: True if propagate else False
+
+    """
+    get_frogml_logger().propagate = prop
+
+
+def translate_to_frogml_verbosity_level(loglevel: int) -> Optional[int]:
+    """Get frogml equivalent of the logging verbosity levels
+
+    Args:
+        loglevel: logging log level int
+    """
+    return REVERSED_VERBOSITY_LEVEL_MAPPING.get(loglevel, None)
+
+
+def get_frogml_logger_name(fallback_logger_name: Optional[str] = None):
+    if fallback_logger_name:
+        return os.getenv(ENVIRON_LOGGER_TYPE, fallback_logger_name)
+    else:
+        return os.getenv(ENVIRON_LOGGER_TYPE, DEFAULT_LOGGER_NAME)
+
+
+def get_handler_from_logger(
+    logger: logging.Logger, handler_name: str
+) -> logging.Handler:
+    matching_handlers = list(
+        filter(lambda h: h.get_name() == handler_name, logger.handlers)
+    )
+    if len(matching_handlers) == 0 and not non_frogml_logger_enabled:
+        raise FrogmlException(
+            f"Error in setting log file. Error message: handler of name {handler_name} was not found in logger"
+        )
+    elif len(matching_handlers) > 1:
+        raise FrogmlException(
+            f"Error in setting log file. Error message: handler of name {handler_name} was found more than once "
+            f"in logger"
+        )
+
+    return matching_handlers[0]
+
+
+def set_file_handler_log_file(
+    logger: logging.Logger, handler_name: str, log_file: Path
+):
+    existing_handler = get_handler_from_logger(logger, handler_name)
+    if type(existing_handler) != RotatingFileHandler:
+        raise FrogmlException(
+            f"Error in setting log file. Error message: handler of name {handler_name} is not a file logger handler"
+        )
+    replacement_handler: RotatingFileHandler = copy_file_handler_from_existing(
+        existing_handler, log_file
+    )
+    logger.removeHandler(existing_handler)
+    logger.addHandler(replacement_handler)
+
+
+def copy_file_handler_from_existing(
+    handler: RotatingFileHandler, log_file: Path
+) -> RotatingFileHandler:
+    return RotatingFileHandler(
+        log_file,
+        mode=handler.mode,
+        maxBytes=int(handler.maxBytes),
+        backupCount=int(handler.backupCount),
+        encoding=handler.encoding,
+        delay=handler.delay,
+    )
+
+
+def set_handler_verbosity(logger: logging.Logger, handler_name: str, log_level: int):
+    handler = get_handler_from_logger(logger, handler_name)
+    handler.setLevel(log_level)
+
+
+def non_frogml_logger_enabled():
+    return os.getenv(AIRFLOW_ENV_FLAG) is not None

frogml_cli/inner/tools/logger/logging.yml

@@ -0,0 +1,79 @@
+version: 1
+disable_existing_loggers: false
+
+formatters:
+  brief:
+    format: '%(message)s'
+  semi_precise:
+    format: "%(asctime)s [%(levelname)-5.5s] %(message)s"
+    datefmt: '%Y-%m-%d %H:%M:%S'
+  precise:
+    format: '%(asctime)s %(levelname)-8s %(module)s - %(lineno)d - %(message)s'
+    datefmt: '%Y-%m-%d %H:%M:%S'
+  remote:
+    class: pythonjsonlogger.json.JsonFormatter
+    format: '%(asctime)s %(process)d %(levelname)s %(module)s %(lineno)d %(message)s'
+    datefmt: '%Y-%m-%dT%H:%M:%S'
+  json:
+    class: pythonjsonlogger.json.JsonFormatter
+    format: '%(asctime)s %(process)d %(levelname)s %(module)s %(lineno)d %(message)s'
+    datefmt: '%Y-%m-%dT%H:%M:%S'
+
+handlers:
+  console:
+    class : logging.StreamHandler
+    formatter: brief
+    level   : INFO
+    stream  : ext://sys.stdout
+  file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: precise
+    filename: ""
+    maxBytes: 10485760 # 10MB
+    backupCount: 5
+    encoding: utf8
+  remote_console:
+    class: logging.StreamHandler
+    formatter: remote
+    level: DEBUG
+    stream: ext://sys.stdout
+  model:
+    class: logging.StreamHandler
+    formatter: json
+    level: INFO
+    stream: ext://sys.stdout
+  build_log_file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: semi_precise
+    filename: ""
+    maxBytes: 10485760 # 10MB
+    backupCount: 5
+    encoding: utf8
+
+loggers:
+  frogml:
+    level: DEBUG
+    handlers: [console, file_handler]
+    propagate: false
+  frogml_remote:
+    level: DEBUG
+    handlers: [remote_console]
+    propagate: false
+  frogml_model:
+    level: DEBUG
+    handlers: [model]
+    propagate: false
+  feature_store:
+    level: DEBUG
+    handlers: [model]
+    propagate: false
+  build_local:
+    level: DEBUG
+    handlers: [build_log_file_handler, console]
+    propagate: false
+  docker_internal:
+    level: DEBUG
+    handlers: [console]
+    propagate: false

frogml_cli/inner/tools/tracking.py

@@ -0,0 +1,47 @@
+import logging
+import platform
+import sys
+
+import requests
+
+from frogml_cli import __version__ as cli_version
+
+logger = logging.getLogger(__name__)
+
+
+def _amplitude_url():
+    return "https://api2.amplitude.com/2/httpapi"
+
+
+def _platform_info():
+    return platform.platform()
+
+
+def _python_version():
+    return "{major}.{minor}.{micro}".format(
+        major=sys.version_info.major,
+        minor=sys.version_info.minor,
+        micro=sys.version_info.micro,
+    )
+
+
+def log_event(properties, user_id):
+    properties["sdk_version"] = cli_version
+    properties["python_version"] = _python_version()
+    properties["platform_info"] = _platform_info()
+
+    event = [
+        {
+            "event_type": "sdk-event",
+            "user_id": user_id,
+            "event_properties": properties,
+            "ip": "$remote",
+        }
+    ]
+
+    event_data = {"events": event}
+
+    try:
+        return requests.post(_amplitude_url(), json=event_data, timeout=1)
+    except Exception as e:
+        logger.debug(str(e))

frogml_cli/main.py

@@ -0,0 +1,9 @@
+from frogml_cli.cli import create_frogml_cli
+
+
+def frogml_cli():
+    return create_frogml_cli()()
+
+
+if __name__ == "__main__":
+    frogml_cli()()

frogml_cli/tools/colors.py

@@ -0,0 +1,13 @@
+class Color:
+    PURPLE = "\033[95m"
+    CYAN = "\033[96m"
+    WHITE = "\033[97m"
+    GREY = "\033[98m"
+    DARKCYAN = "\033[36m"
+    BLUE = "\033[94m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    END = "\033[0m"

frogml_cli/tools/const.py

@@ -0,0 +1,3 @@
+from frogml._proto.qwak.batch_job.v1.batch_job_resources_pb2 import GpuType
+
+GPU_TYPES = list(set(GpuType.DESCRIPTOR.values_by_name) - {"INVALID_GPU"})