frogml-cli 0.0.1__py3-none-any.whl

frogml_cli/commands/models/build/_logic/util/step_decorator.py

@@ -0,0 +1,60 @@
+import os
+import shutil
+from functools import wraps
+from typing import List
+
+from frogml.core.exceptions import QuietError
+from frogml.core.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml.core.inner.build_logic.interface.step_inteface import Step
+
+
+def step_exception_handler_decorator(step: Step):
+    execute_func = step.execute
+
+    @wraps(execute_func)
+    def inner_function(*args, **kwargs):
+        try:
+            execute_func(*args, **kwargs)
+        except BaseException as e:
+            if not isinstance(e, QuietError):
+                build_logger: BuildLogger = step.build_logger
+                build_logger.error(
+                    "Build failed with Exception. See the stack trace above."
+                )
+                cleaning_up_after_build(step)
+            raise e
+
+    return inner_function
+
+
+def build_failure_handler():
+    def _exception_handler(func):
+        @wraps(func)
+        def inner_function(*args, **kwargs):
+            try:
+                func(*args, **kwargs)
+            except BaseException as e:
+                if not isinstance(e, QuietError):
+                    build_logger: BuildLogger = args[0].build_logger
+                    build_logger.error(
+                        "Build failed with Exception. See the stack trace above."
+                    )
+                    cleaning_up_after_build(args[0])
+                raise e
+        return inner_function
+
+    return _exception_handler
+
+
+def cleaning_up_after_build(step: Step):
+    if os.getenv("QWAK_DEBUG") != "true":
+        step.build_logger.debug("Removing Qwak temp artifacts directory")
+        shutil.rmtree(step.context.host_temp_local_build_dir, ignore_errors=True)
+
+
+def add_decorator_to_steps(phase_steps: List[Step]):
+    for step in phase_steps:
+        if not hasattr(step.execute, '__wrapped__'):
+            step_exception_handler_decorator(step)
+
+    return phase_steps

frogml_cli/commands/models/build/_logic/util/text.py

@@ -0,0 +1,9 @@
+from re import sub
+
+
+def snake_case(s):
+    return "_".join(
+        sub(
+            "([A-Z][a-z]+)", r" \1", sub("([A-Z]+)", r" \1", s.replace("-", " "))
+        ).split()
+    ).lower()

frogml_cli/commands/models/build/_logic/wait_until_finished.py

@@ -0,0 +1,27 @@
+from logging import Logger
+import time
+from typing import List
+from frogml._proto.qwak.builds.builds_pb2 import BuildStatus
+from frogml.core.clients.build_orchestrator import BuildOrchestratorClient
+
+
+def __get_current_status(build_id) -> BuildStatus:
+    return BuildOrchestratorClient().get_build(build_id).build.build_status
+
+
+def __end_state_statuses() -> List[BuildStatus.ValueType]:
+    return [BuildStatus.FAILED, BuildStatus.SUCCESSFUL, BuildStatus.REMOTE_BUILD_CANCELLED, BuildStatus.REMOTE_BUILD_TIMED_OUT]
+
+
+def wait_until_finished(build_id, log: Logger, pool_interval_seconds=10) -> None:
+    status = __get_current_status(build_id)
+    log.info(f"Waiting for build {build_id} to finish. Aborting this process will not stop the build!")
+    log.debug(f"Current status of build {build_id}: {BuildStatus.DESCRIPTOR.values_by_number[status].name}")
+    while status not in __end_state_statuses():
+        time.sleep(pool_interval_seconds)
+        status = __get_current_status(build_id)
+        log.debug(f"Current status of build {build_id}: {BuildStatus.DESCRIPTOR.values_by_number[status].name}")
+
+
+def is_final_status_successful(build_id) -> bool:
+    return __get_current_status(build_id) == BuildStatus.SUCCESSFUL

frogml_cli/commands/models/build/ui.py

@@ -0,0 +1,337 @@
+import click
+from frogml.core.inner.build_config.build_config_v1 import (
+    BuildConfigV1,
+    BuildProperties,
+    DockerConf,
+    ModelUri,
+    RemoteBuildResources,
+)
+from frogml.core.inner.build_logic.execute_build_pipeline import execute_build_pipeline
+from frogml.core.inner.build_logic.run_handlers.programmatic_phase_run_handler import (
+    ProgrammaticPhaseRunHandler,
+)
+
+from frogml_cli.commands.models.build._logic.build_steps import create_pipeline
+from frogml_cli.commands.models.build._logic.wait_until_finished import wait_until_finished, is_final_status_successful
+from frogml_cli.commands.models.build._logic.client_logs.cli_phase_run_handler import (
+    CLIPhaseRunHandler,
+)
+from frogml_cli.commands.models.build._logic.client_logs.logger import get_build_logger
+from frogml_cli.commands.models.build._logic.client_logs.utils import zip_logs
+from frogml_cli.inner.tools.cli_tools import FrogMLCommand
+from frogml_cli.inner.tools.config_handler import config_handler
+from frogml_cli.tools.const import GPU_TYPES
+
+
+@click.command("build", cls=FrogMLCommand)
+@click.option(
+    "--model-id",
+    metavar="NAME",
+    required=False,
+    help="Model ID to assign the build for",
+)
+@click.option(
+    "--main-dir",
+    metavar="NAME",
+    help=f"Model main directory name, [Default: {ModelUri.main_dir}]",
+)
+@click.option(
+    "-P",
+    "--param-list",
+    required=False,
+    metavar="NAME=VALUE",
+    multiple=True,
+    help="A parameter for the build, of the form -P name=value. the params will be saved and can be viewed later",
+)
+@click.option(
+    "-E",
+    "--env-vars",
+    required=False,
+    metavar="NAME=VALUE",
+    multiple=True,
+    help="A parameter for the build, of the form -E name=value",
+)
+@click.option(
+    "-T",
+    "--tags",
+    required=False,
+    multiple=True,
+    help="A tag for the model build",
+)
+@click.option(
+    "--git-credentials",
+    required=False,
+    metavar="USERNAME:ACCESS_TOKEN",
+    help="Access credentials for private repositories listed in the python dependencies file",
+)
+@click.option(
+    "--git-branch",
+    metavar="NAME",
+    required=False,
+    help=f"Branch to use for git repo model code if defined."
+    f"\n[Default: {ModelUri.git_branch}]",
+)
+@click.option(
+    "--git-credentials-secret",
+    metavar="NAME",
+    required=False,
+    help="Predefined Frogml secret secret name, that contains access credentials to private repositories"
+    + "Secrets should be of the form USERNAME:ACCESS_TOKEN. For info regarding defining Frogml Secrets using the"
+    + "`frogml secret` command",
+)
+@click.option(
+    "--cpus",
+    metavar="NAME",
+    required=False,
+    help="Number of cpus to use on the remote build. [Default (If GPU not configured): 2] "
+    "(DO NOT CONFIGURE GPU AND CPU TOGETHER)",
+    type=click.FLOAT,
+)
+@click.option(
+    "--memory",
+    metavar="NAME",
+    required=False,
+    help="Memory to use on the remote build. [Default (If GPU not configured): 4Gi] "
+    "(DO NOT CONFIGURE GPU AND CPU TOGETHER)",
+)
+@click.option(
+    "--gpu-type",
+    metavar="NAME",
+    required=False,
+    help=f"Type of GPU to use on the remote build ({', '.join([x for x in GPU_TYPES])})."
+    f"\n[Default: {RemoteBuildResources.gpu_type}]"
+    "(DO NOT CONFIGURE GPU AND CPU TOGETHER)",
+    type=click.STRING,
+)
+@click.option(
+    "--gpu-amount",
+    metavar="NAME",
+    required=False,
+    type=int,
+    help=f"Amount of GPU's to use on the remote build."
+    f"\n[Default: {RemoteBuildResources.gpu_amount}] "
+    "(DO NOT CONFIGURE GPU AND CPU TOGETHER)",
+)
+@click.option(
+    "--gpu-compatible",
+    help=f"Whether to build an image that is compatible to be deployd on a GPU instance."
+    f"\n[Default: {BuildProperties.gpu_compatible}] ",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--iam-role-arn",
+    required=False,
+    type=str,
+    help="Custom IAM Role ARN for AWS based builds",
+)
+@click.option(
+    "--service-account-key-secret-name",
+    type=str,
+    help="Custom service account for GCP",
+)
+@click.option(
+    "--cache/--no-cache",
+    default=None,
+    help="Disable docker build cache. [Default: Cache enabled]",
+)
+@click.option(
+    "-v",
+    "--verbose",
+    count=True,
+    default=None,
+    help="Log verbosity level - v: INFO, vv: DEBUG [default: WARNING], Default ERROR",
+)
+@click.option(
+    "--base-image",
+    help="Used for customizing the docker container image built for train, build and deploy."
+    "Docker images should be based on Frogml images, The entrypoint or cmd of the docker "
+    "image should not be changed."
+    f"\n[Default: {DockerConf.base_image}]",
+    required=False,
+)
+@click.option(
+    "-f",
+    "--from-file",
+    help="Build by run_config file, Command arguments will overwrite any run_config.",
+    required=False,
+    type=click.Path(exists=True, resolve_path=True, dir_okay=False),
+)
+@click.option(
+    "--out-conf",
+    help="Extract models build conf from command arguments, the command will not run it wil only output valid yaml "
+    "structure",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--json-logs",
+    help="Output logs as json for easier parsing",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--programmatic",
+    help="Run the _logic without the UI and receive the build id and any exception as return values",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--validate-build-artifact/--no-validate-build-artifact",
+    help="Skip validate build artifact step",
+    default=None,
+)
+@click.option(
+    "--tests/--no-tests",
+    help="Skip tests step",
+    default=None,
+)
+@click.option(
+    "--dependency-file-path",
+    help="Custom dependency file path",
+    default=None,
+)
+@click.option(
+    "--validate-build-artifact-timeout",
+    help="Timeout in seconds for the validation step",
+    default=120,
+)
+@click.option(
+    "--dependency-required-folders",
+    help="Folder to be copied into the build. In order to copy several folders, use this flag several times",
+    default=None,
+    required=False,
+    multiple=True,
+)
+@click.option(
+    "--deploy",
+    help="Whether you want to deploy the build if it finishes successfully. "
+    "Choosing this will follow the build process in the terminal and will trigger a deployment when the "
+    "build finishes.",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--instance",
+    required=False,
+    type=str,
+    help="The instance size to build on - 'small', 'medium', 'large', etc...",
+    default=None,
+)
+@click.option(
+    "--deployment-instance",
+    required=False,
+    type=str,
+    help="The instance size to deploy on - 'small', 'medium', 'large', etc...",
+    default=None,
+)
+@click.option(
+    "--purchase-option",
+    required=False,
+    type=str,
+    help="The type of the instance to build on - 'ondemand' or 'spot'",
+    default=None,
+)
+@click.option(
+    "--id-only",
+    help="Receiving only the build id and any exception as return values (Depends on "
+    "--programmatic in order to avoid UI output too)",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--sync",
+    help="Waits until the build is finished (successfully or not) and return the build status",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--git-secret-ssh",
+    metavar="NAME",
+    required=False,
+    help="[REMOTE BUILD] Predefined Frogml ssh secret name, that contains access credentials to private repositories"
+    + "secret ssh key should contain the private ssh key."
+    + "`frogml secret` command",
+)
+@click.option(
+    "--push/--no-push",
+    help="Whether to push the build image to the registry (default is True)",
+    default=True,
+    is_flag=True,
+)
+@click.option(
+    "--provision-instance-timeout",
+    help="Timeout in minutes for the provision instance step",
+    default=120,
+)
+@click.option(
+    "-N",
+    "--name",
+    "build_name",
+    required=False,
+    help="The build's name",
+)
+@click.argument("uri", required=False)
+def models_build(**kwargs):
+    return build(**kwargs)
+
+
+def build(
+    from_file: str,
+    out_conf: bool,
+    json_logs: bool,
+    programmatic: bool,
+    **kwargs,
+):
+    # If FROGML_DEBUG=true is set then the artifacts will not be deleted, all intermediate files located in ~/.frogml/builds
+    # Including all intermediate images
+    config: BuildConfigV1 = config_handler(
+        config=BuildConfigV1,
+        from_file=from_file,
+        out_conf=out_conf,
+        **kwargs,
+    )
+    if out_conf:
+        return
+    else:
+        id_only = kwargs.get("id_only")
+        is_sync = kwargs.get("sync")
+        pipeline, success_msg = create_pipeline(config)
+        with get_build_logger(config=config, json_logs=json_logs) as (
+            logger,
+            log_path,
+        ):
+            if programmatic:
+                build_runner = ProgrammaticPhaseRunHandler(
+                    logger, config.verbose, json_logs
+                )
+            else:
+                build_runner = CLIPhaseRunHandler(
+                    logger, log_path, config.verbose, json_logs
+                )
+
+            execute_build_pipeline(
+                pipeline,
+                build_runner,
+            )
+
+            if is_sync:
+                wait_until_finished(pipeline.context.build_id, logger)
+                if not is_final_status_successful(pipeline.context.build_id):
+                    click.echo(f"Build {pipeline.context.build_id} failed")
+                    exit(1)
+
+            if id_only:
+                print(pipeline.context.build_id)
+            else:
+                print(
+                    success_msg.format(
+                        base_url=pipeline.context.platform_url,
+                        build_id=pipeline.context.build_id,
+                        model_id=pipeline.context.model_id,
+                        project_uuid=pipeline.context.project_uuid,
+                    )
+                )
+            zip_logs(log_path=log_path, build_id=pipeline.context.build_id)
+
+            return pipeline.context.build_id

frogml_cli/commands/models/builds/builds_commands_group.py

@@ -0,0 +1,16 @@
+import click
+
+from frogml_cli.commands.models.builds.cancel.ui import cancel_build
+from frogml_cli.commands.models.builds.logs.ui import build_logs
+from frogml_cli.commands.models.builds.status.ui import get_build_status
+
+
+@click.group(name="builds", help="Ongoing builds")
+def builds_commands_group():
+    # Click commands group injection
+    pass
+
+
+builds_commands_group.add_command(cancel_build)
+builds_commands_group.add_command(get_build_status)
+builds_commands_group.add_command(build_logs)

frogml_cli/commands/models/builds/cancel/_logic.py

@@ -0,0 +1,5 @@
+from frogml.core.clients.build_orchestrator import BuildOrchestratorClient
+
+
+def execute_cancel_build(build_id):
+    return BuildOrchestratorClient().cancel_build_model(build_id=build_id)

frogml_cli/commands/models/builds/cancel/ui.py

@@ -0,0 +1,15 @@
+import click
+from frogml.core.tools.logger.logger import get_frogml_logger
+
+from frogml_cli.commands.models.builds.cancel._logic import execute_cancel_build
+from frogml_cli.inner.tools.cli_tools import FrogMLCommand
+
+logger = get_frogml_logger()
+
+
+@click.command("cancel", cls=FrogMLCommand)
+@click.argument("build_id")
+def cancel_build(build_id, **kwargs):
+    logger.info(f"Attempting to cancel remote build with build id [{build_id}]")
+    execute_cancel_build(build_id=build_id)
+    logger.info("Successfully canceled remote build")

frogml_cli/commands/models/builds/logs/ui.py

@@ -0,0 +1,35 @@
+import click
+
+from frogml_cli.inner.tools.cli_tools import FrogMLCommand
+from frogml_cli.tools.log_handling import FrogmlLogHandling
+
+
+@click.command("logs", cls=FrogMLCommand)
+@click.option("-f", "--follow", is_flag=True, default=False, help="Follow log tail")
+@click.option(
+    "-b", "--build-id", required=True, help="Runtime model Build ID to show logs of"
+)
+@click.option(
+    "-s",
+    "--since",
+    required=False,
+    metavar="X [TYPE] ago",
+    help=f"Get logs from X1 [type] X2 [type]... Xn [type] ago. "
+    f"(x = INT; type = {FrogmlLogHandling.TIME_UNITS}) i.e. [1 day 2 hours ago]",
+)
+@click.option(
+    "-n",
+    "--number-of-results",
+    required=False,
+    type=int,
+    help="Maximum number of results per query",
+)
+@click.option(
+    "-g", "--grep", required=False, help="Filter by log content contains [expression]"
+)
+def build_logs(
+    follow=True, build_id=None, since=None, number_of_results=None, grep=None, **kwargs
+):
+    FrogmlLogHandling().get_logs(
+        follow, since, number_of_results, grep, {"build_id": build_id}, "build"
+    )

frogml_cli/commands/models/builds/status/_logic.py

@@ -0,0 +1,6 @@
+from frogml._proto.qwak.builds.builds_pb2 import BuildStatus
+from frogml.core.clients.build_orchestrator import BuildOrchestratorClient
+
+
+def execute_get_build_status(build_id) -> BuildStatus:
+    return BuildOrchestratorClient().get_build(build_id).build.build_status

frogml_cli/commands/models/builds/status/ui.py

@@ -0,0 +1,39 @@
+import json
+
+import click
+from frogml._proto.qwak.builds.builds_pb2 import BuildStatus
+from frogml.core.tools.logger.logger import get_frogml_logger
+
+from frogml_cli.commands.models.builds.status._logic import execute_get_build_status
+from frogml_cli.inner.tools.cli_tools import FrogMLCommand
+
+logger = get_frogml_logger()
+
+
+@click.command("status", cls=FrogMLCommand)
+@click.argument("build_id")
+@click.option(
+    "--format",
+    default="text",
+    show_default=True,
+    type=click.Choice(["text", "json"], case_sensitive=True),
+    metavar="FORMAT",
+    required=False,
+    help="The formatting style for commands output (choose from text, json)",
+)
+def get_build_status(build_id, format, **kwargs):
+    if format == "text":
+        logger.info(f"Getting build status for build id [{build_id}]")
+    build_status = execute_get_build_status(build_id)
+    if format == "text":
+        logger.info(f"Build status: {BuildStatus.Name(build_status)}")
+    elif format == "json":
+        print(
+            json.dumps(
+                {
+                    "build_id": build_id,
+                    "build_status": BuildStatus.Name(build_status),
+                }
+            )
+        )
+    return BuildStatus.Name(build_status)

frogml_cli/commands/models/create/_logic.py

@@ -0,0 +1,40 @@
+import logging
+
+from frogml._proto.qwak.model_group.model_group_pb2 import ModelGroupBriefInfoResponse
+from frogml._proto.qwak.models.models_pb2 import CreateModelResponse
+from frogml.core.clients.model_group_management import ModelGroupManagementClient
+from frogml.core.clients.model_management import ModelsManagementClient
+from frogml.core.exceptions import FrogmlException
+
+from frogml_cli.exceptions import FrogmlCommandException
+
+logger = logging.getLogger(__name__)
+
+
+def execute_model_create(
+    model_name: str,
+    model_description: str,
+    jfrog_project_key: str,
+) -> CreateModelResponse:
+    if not jfrog_project_key:
+        raise FrogmlCommandException("You must supply project key")
+    try:
+        response: ModelGroupBriefInfoResponse = (
+            ModelGroupManagementClient().create_if_not_exists_model_group(
+                project_key=jfrog_project_key
+            )
+        )
+        logger.info(
+            "Retrieved model group id %s that associated with project %s",
+            response.model_group_id,
+            jfrog_project_key,
+        )
+        return ModelsManagementClient().create_model(
+            project_id=response.model_group_id,
+            model_name=model_name,
+            model_description=model_description,
+            jfrog_project_key=jfrog_project_key,
+        )
+    except FrogmlException as e:
+        logger.error("Error occurred during model creation due to %s", e.message)
+        raise FrogmlCommandException(e)

frogml_cli/commands/models/create/ui.py

@@ -0,0 +1,46 @@
+import click
+
+from frogml_cli.commands.models.create._logic import execute_model_create
+from frogml_cli.commands.ui_tools import output_as_json
+from frogml_cli.inner.tools.cli_tools import FrogMLCommand
+from frogml_cli.tools.colors import Color
+
+
+@click.command("create", cls=FrogMLCommand)
+@click.argument("name", metavar="name", required=True)
+@click.option(
+    "--project-key",
+    metavar="ID",
+    required=False,
+    help="An existing jfrog project key",
+)
+@click.option(
+    "--description",
+    metavar="DESCRIPTION",
+    required=False,
+    help="Model description",
+)
+@click.option(
+    "--format",
+    default="text",
+    show_default=True,
+    type=click.Choice(["text", "json"], case_sensitive=True),
+    metavar="FORMAT",
+    required=False,
+    help="The formatting style for commands output (choose from text, json)",
+)
+def model_create(
+    name,
+    project_key: str,
+    description,
+    format,
+    **kwargs,
+):
+    try:
+        response = execute_model_create(name, description, project_key)
+        if format == "json":
+            output_as_json(response)
+        else:
+            print(f"Model created\nmodel id : {response.model_id}")
+    except Exception as e:
+        print(f"{Color.RED}Error creating model: {e}{Color.RED}")

frogml_cli/commands/models/delete/_logic.py

@@ -0,0 +1,18 @@
+from frogml._proto.qwak.projects.projects_pb2 import GetProjectResponse
+from frogml.core.clients.model_management import ModelsManagementClient
+from frogml.core.clients.project.client import ProjectsManagementClient
+from frogml.core.exceptions import FrogmlException
+
+
+def execute_model_delete(project_key: str, model_id: str):
+    project_response: GetProjectResponse = ProjectsManagementClient().get_project(
+        project_name=project_key
+    )
+    is_model_exists: bool = any(
+        m.model_id == model_id for m in project_response.project.models
+    )
+    if not is_model_exists:
+        raise FrogmlException(f"No such model {model_id} for project {project_key}")
+
+    project_id: str = project_response.project.spec.project_id
+    return ModelsManagementClient().delete_model(project_id, model_id)