frogml-cli 0.0.1__py3-none-any.whl

frogml_cli/commands/models/deployments/deploy/_logic/deployment_size_mapper.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from frogml._proto.qwak.deployment.deployment_pb2 import DeploymentSize, MemoryUnit
+from frogml._proto.qwak.user_application.common.v0.resources_pb2 import (
+    ClientPodComputeResources,
+    CpuResources,
+    GpuResources,
+    PodComputeResourceTemplateSpec,
+)
+from frogml.core.clients.administration.eco_system.client import EcosystemClient
+from frogml.core.clients.instance_template.client import (
+    InstanceTemplateManagementClient,
+)
+from frogml.core.exceptions import FrogmlException
+from frogml.core.inner.instance_template.verify_template_id import verify_template_id
+from frogml.core.inner.provider import Provider
+
+from frogml_cli.commands.models.deployments.deploy._logic.deploy_config import (
+    DeployConfig,
+)
+
+
+def deployment_size_from_deploy_config(
+    deploy_config: DeployConfig,
+    instance_template_client: InstanceTemplateManagementClient,
+    ecosystem_client: EcosystemClient,
+) -> DeploymentSize:
+    if deploy_config.resources.instance_size:
+        deploy_config.resources.instance_size = (
+            deploy_config.resources.instance_size.lower()
+        )
+        account_details = ecosystem_client.get_account_details()
+        if deploy_config.environments:
+            environments_config = list(
+                ecosystem_client.get_environments_names_to_details(
+                    deploy_config.environments
+                ).values()
+            )
+        else:
+            environments_config = [
+                account_details.environment_by_id[
+                    account_details.default_environment_id
+                ]
+            ]
+        for environment_config in environments_config:
+            provider = __get_provider(environment_config)
+            try:
+                verify_template_id(
+                    deploy_config.resources.instance_size,
+                    instance_template_client,
+                    provider=provider,
+                )
+            except FrogmlException as e:
+                raise FrogmlException(
+                    f"Error with template {deploy_config.resources.instance_size} for environment {environment_config.name}: {e.message}"
+                )
+
+        return DeploymentSize(
+            number_of_pods=deploy_config.resources.pods,
+            client_pod_compute_resources=ClientPodComputeResources(
+                template_spec=PodComputeResourceTemplateSpec(
+                    template_id=deploy_config.resources.instance_size
+                )
+            ),
+        )
+    elif deploy_config.resources.gpu_type:
+        return DeploymentSize(
+            number_of_pods=deploy_config.resources.pods,
+            client_pod_compute_resources=ClientPodComputeResources(
+                gpu_resources=GpuResources(
+                    gpu_type=deploy_config.resources.gpu_type,
+                    gpu_amount=deploy_config.resources.gpu_amount,
+                )
+            ),
+        )
+    else:
+        return DeploymentSize(
+            number_of_pods=deploy_config.resources.pods,
+            client_pod_compute_resources=ClientPodComputeResources(
+                cpu_resources=CpuResources(
+                    cpu=deploy_config.resources.cpus,
+                    memory_amount=deploy_config.resources.memory,
+                    memory_units=MemoryUnit.MIB,
+                )
+            ),
+        )
+
+
+def __get_provider(environment_config):
+    provider = None
+    cloud_type = environment_config.configuration.cloud_configuration.WhichOneof(
+        "configuration"
+    )
+    if cloud_type == "aws_cloud_configuration":
+        provider = Provider.AWS
+    elif cloud_type == "gcp_cloud_configuration":
+        provider = Provider.GCP
+    return provider

frogml_cli/commands/models/deployments/deploy/_logic/get_latest_successful_build.py

@@ -0,0 +1,28 @@
+from frogml._proto.qwak.builds.builds_pb2 import SUCCESSFUL
+from frogml.core.clients.build_orchestrator import BuildOrchestratorClient
+from frogml.core.clients.model_management.client import ModelsManagementClient
+from frogml.core.exceptions import FrogmlException
+
+
+def get_latest_successful_build_from_model(model_id: str) -> str:
+    """Get the latest successful build from model
+
+    Args:
+        model_id: Model id to search build for.
+
+    Returns:
+        str: Latest build id.
+
+    Raises:
+        FrogmlException: if no successful build exists in model.
+    """
+    model = ModelsManagementClient().get_model(model_id)
+    model_uuid = model.uuid
+    all_builds = BuildOrchestratorClient().list_builds(model_uuid).build
+    builds = [build for build in all_builds if build.build_status == SUCCESSFUL]
+    builds.sort(key=lambda r: r.audit.created_at.seconds)
+
+    if not builds:
+        raise FrogmlException(f"Unable to find successful build for model {model_id}")
+
+    return builds[-1].buildId

frogml_cli/commands/models/deployments/deploy/_logic/local_deployment.py

@@ -0,0 +1,193 @@
+import base64
+
+from frogml._proto.qwak.ecosystem.v0.ecosystem_runtime_service_pb2 import (
+    GetCloudCredentialsParameters,
+    GetCloudCredentialsRequest,
+    PermissionSet,
+    PullModelsContainerRegistry,
+)
+from frogml.core.clients.administration.eco_system.client import EcosystemClient
+from frogml.core.clients.build_orchestrator import BuildOrchestratorClient
+from frogml.core.exceptions import FrogmlException
+from google.protobuf.duration_pb2 import Duration
+from rich.console import Console
+from rich.layout import Layout
+from rich.panel import Panel
+from rich.progress import Progress
+
+from frogml_cli.commands.models.deployments.deploy._logic.deploy_config import (
+    DeployConfig,
+)
+
+tasks = {}
+
+
+def local_deploy(config: DeployConfig):
+    try:
+        import docker
+    except ImportError:
+        raise FrogmlException(
+            "Error: 'docker' package is required to for local model deployment."
+        )
+
+    console = Console()
+    console.print(Panel(f"Deploying model {config.model_id} locally", style="green"))
+
+    client = docker.from_env()
+    build_image = _get_build_image(config)
+
+    if not client.images.list(name=build_image):
+        console.print(
+            "Pulling serving image from Frogml's remote repository. Might take a few minutes...",
+            style="yellow",
+        )
+        _docker_login(client)
+        _image_pull(client, build_image)
+
+    try:
+        container = start_local_frogml_container(build_image, client, config)
+        container.reload()
+        host_port = container.ports.get("5000/tcp")[0]["HostPort"]
+
+        layout = example_usage_layout(host_port)
+        console.print(layout)
+
+        for line in container.logs(stream=True):
+            console.print(str(line.strip().decode()), style="blue")
+
+    except KeyboardInterrupt:
+        console.print(Panel("Stopping container...", style="red"))
+        container.stop()
+        container.remove()
+
+
+def example_usage_layout(host_port: str):
+    layout = Layout(size=6)
+    layout.split_row(
+        Layout(name="left"),
+        Layout(name="right"),
+    )
+    layout["left"].update(
+        Panel(
+            f"[bold]cURL usage:[bold]\n\n"
+            f"curl --location\n"
+            f"--request POST http://localhost:{host_port}/predict\n"
+            f"--header 'Content-Type: application/json'\n"
+            f"--data '...'",
+            style="green",
+        )
+    )
+    layout["right"].update(
+        Panel(
+            f"[bold]Python client usage:[bold]\n\n"
+            f"from frogml_inference import RealtimeClient\n\nfeature_vector=[...]\n"
+            f"client = RealTimeClient(model_api='http://localhost:{host_port}/predict')\n"
+            f"client.predict(feature_vector)",
+            style="green",
+        )
+    )
+    return layout
+
+
+def start_local_frogml_container(build_image: str, docker_client, config: DeployConfig):
+    return docker_client.containers.run(
+        build_image,
+        command=["bentoml", "serve-gunicorn", "./"],
+        entrypoint="./docker-entrypoint.sh",
+        environment=[
+            "BENTOML_GUNICORN_WORKERS=2",
+            "QWAK_DEBUG_MODE=True",
+            f"QWAK_MODEL_ID={config.model_id}",
+            "BENTOML_DO_NOT_TRACK=True",
+            "PYTHONPATH=/qwak/model_dir:/qwak/model_dir/main:$PYTHONPATH",
+            f"QWAK_BUILD_ID={config.build_id}",
+            "BUNDLE_PATH=/home/bentoml/bundle",
+            "BENTOML_HOME=/home/bentoml/",
+        ],
+        stream=True,
+        detach=True,
+        publish_all_ports=True,
+    )
+
+
+def _get_build_image(config: DeployConfig):
+    return (
+        BuildOrchestratorClient().get_build(config.build_id).build.build_destined_image
+    )
+
+
+def _show_progress(line, progress):
+    if line["status"] == "Downloading":
+        idx = f'[red][Download {line["id"]}]'
+    elif line["status"] == "Extracting":
+        idx = f'[green][Extract {line["id"]}]'
+    else:
+        # skip other statuses
+        return
+
+    if idx not in tasks.keys():
+        tasks[idx] = progress.add_task(f"{idx}", total=line["progressDetail"]["total"])
+    else:
+        progress.update(tasks[idx], completed=line["progressDetail"]["current"])
+
+
+def _image_pull(docker_client, image_name: str):
+    with Progress() as progress:
+        resp = docker_client.api.pull(image_name, stream=True, decode=True)
+        for line in resp:
+            _show_progress(line, progress)
+
+
+def _docker_login(docker_client):
+    try:
+        import boto3
+    except ImportError:
+        raise FrogmlException(
+            "Error: The 'boto3' package is necessary for local model deployment. Install it directly or by running"
+            " 'pip install frogml-cli[local-deployment]'"
+        )
+
+    # TODO: block if not an AWS based container registry
+    credentials = _get_aws_credentials()
+
+    aws_credentials = credentials.cloud_credentials.aws_temporary_credentials
+    ecr_client = boto3.Session(
+        aws_access_key_id=aws_credentials.access_key_id,
+        aws_secret_access_key=aws_credentials.secret_access_key,
+        aws_session_token=aws_credentials.session_token,
+        region_name=aws_credentials.region,
+    ).client("ecr")
+
+    ecr_credentials = ecr_client.get_authorization_token()["authorizationData"][0]
+
+    try:
+        docker_client.login(
+            username="AWS",
+            registry=ecr_credentials["proxyEndpoint"],
+            password=base64.b64decode(ecr_credentials["authorizationToken"])
+            .replace(b"AWS:", b"")
+            .decode("utf-8"),
+        )
+
+    except Exception as e:
+        raise FrogmlException(f"Failed to login to Qwak's container registry: {e}")
+
+
+def _get_aws_credentials():
+    try:
+        eco_client = EcosystemClient()
+        credentials = eco_client.get_cloud_credentials(
+            request=GetCloudCredentialsRequest(
+                parameters=GetCloudCredentialsParameters(
+                    duration=Duration(seconds=60 * 60, nanos=0),  # 6 hours
+                    permission_set=PermissionSet(
+                        pull_models_container_registry=PullModelsContainerRegistry()
+                    ),
+                )
+            )
+        )
+        return credentials
+    except Exception as e:
+        raise FrogmlException(
+            f"Failed to get credentials to pull image from Frogml's container registry: {e}."
+        )

frogml_cli/commands/models/deployments/deploy/batch/_logic/advanced_deployment_mapper.py

@@ -0,0 +1,15 @@
+from frogml._proto.qwak.deployment.deployment_pb2 import AdvancedDeploymentOptions
+
+from frogml_cli.commands.models.deployments.deploy._logic.deploy_config import (
+    DeployConfig,
+)
+
+
+def batch_advanced_deployment_options_from_deploy_config(
+    deploy_config: DeployConfig,
+) -> AdvancedDeploymentOptions:
+    return AdvancedDeploymentOptions(
+        custom_iam_role_arn=deploy_config.advanced_options.iam_role_arn,
+        purchase_option=deploy_config.advanced_options.purchase_option,
+        service_account_key_secret_name=deploy_config.advanced_options.service_account_key_secret_name,
+    )

frogml_cli/commands/models/deployments/deploy/batch/_logic/deploy_executor.py

@@ -0,0 +1,24 @@
+from frogml._proto.qwak.deployment.deployment_pb2 import KubeDeploymentType
+from frogml._proto.qwak.deployment.deployment_service_pb2 import DeployModelResponse
+
+from frogml_cli.commands.models.deployments.deploy._logic.base_deploy_executor import (
+    BaseDeployExecutor,
+)
+from frogml_cli.commands.models.deployments.deploy._logic.deployment_message_helpers import (
+    get_env_to_deployment_message,
+)
+
+
+class BatchDeployExecutor(BaseDeployExecutor):
+    def deploy(self) -> DeployModelResponse:
+        env_deployment_messages = get_env_to_deployment_message(
+            self.config,
+            KubeDeploymentType.BATCH,
+            self.ecosystem_client,
+            self.instance_template_client,
+        )
+        return self.deploy_client.deploy_model(
+            model_id=self.config.model_id,
+            build_id=self.config.build_id,
+            env_deployment_messages=env_deployment_messages,
+        )

frogml_cli/commands/models/deployments/deploy/batch/ui.py

@@ -0,0 +1,119 @@
+import click
+from frogml.core.tools.logger.logger import set_frogml_logger_stdout_verbosity_level
+
+from frogml_cli.commands.models.deployments.deploy._logic.deploy_config import (
+    DeployConfig,
+    PurchaseOption,
+)
+from frogml_cli.commands.models.deployments.deploy._logic.deployment_response_handler import (
+    client_deployment,
+)
+from frogml_cli.commands.models.deployments.deploy.batch._logic.deploy_executor import (
+    BatchDeployExecutor,
+)
+from frogml_cli.inner.tools.cli_tools import DeprecatedOption, FrogMLCommand
+from frogml_cli.inner.tools.config_handler import config_handler
+from frogml_cli.tools.const import GPU_TYPES
+
+
+@click.command("batch", help="Deploy batch model.", cls=FrogMLCommand)
+@click.option(
+    "--build-id",
+    help="Build ID, If not specified latest successful build will deployed.",
+    type=str,
+)
+@click.option("--model-id", help="Model ID", type=str, required=True)
+@click.option(
+    "--pods",
+    "--replicas",
+    type=int,
+    help="Number of pods to deploy.",
+    deprecated=["--pods"],
+    preferred="--replicas",
+    cls=DeprecatedOption,
+)
+@click.option(
+    "--cpus",
+    type=float,
+    help="Number of CPU cores, can be fraction of a core (e.g. 0.4).",
+)
+@click.option(
+    "--memory",
+    type=int,
+    help="Memory amount in Mib (e.g. 256).",
+)
+@click.option(
+    "--gpu-type",
+    metavar="NAME",
+    required=False,
+    help=f"Type of GPU to use on the deployments ({', '.join([x for x in GPU_TYPES])}).",
+    type=click.STRING,
+)
+@click.option(
+    "--gpu-amount",
+    metavar="NAME",
+    required=False,
+    type=int,
+    help="Amount of GPU to use on the deployments.",
+)
+@click.option(
+    "--iam-role-arn",
+    type=str,
+    help="Custom IAM Role ARN.",
+)
+@click.option(
+    "--service-account-key-secret-name",
+    type=str,
+    help="Custom service account for Gcp.",
+)
+@click.option(
+    "--sync",
+    is_flag=True,
+    default=False,
+    help="Waiting for deployments to be ready",
+)
+@click.option(
+    "-v",
+    "--verbose",
+    count=True,
+    help="Log verbosity level - v: INFO, vv: DEBUG [default: WARNING]",
+)
+@click.option(
+    "-f",
+    "--from-file",
+    help="Deploy by run_config file, Command arguments will overwrite any run_config.",
+    required=False,
+    type=click.Path(exists=True, resolve_path=True, dir_okay=False),
+)
+@click.option(
+    "--out-conf",
+    help="Extract deploy conf from commands arguments, the commands will not run it wil only output valid yaml "
+    "structure",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--purchase-option",
+    required=False,
+    type=click.Choice([i.value for i in PurchaseOption], case_sensitive=False),
+    help="Indicate the instance deployments type, whether it is spot/ondemand.  Default is spot",
+)
+@click.option(
+    "--instance",
+    required=False,
+    type=str,
+    help="The instance size to deploy on - 'small', 'medium', 'large', etc...",
+    default=None,
+)
+def batch(verbose: bool, from_file: str, out_conf: bool, sync: bool, **kwargs):
+    set_frogml_logger_stdout_verbosity_level(verbose + 1)
+    config: DeployConfig = config_handler(
+        config=DeployConfig,
+        from_file=from_file,
+        out_conf=out_conf,
+        sections=("batch",),
+        **kwargs,
+    )
+    if not out_conf:
+        deploy_executor = BatchDeployExecutor(config)
+        client_deployment(deploy=deploy_executor, sync=sync)

frogml_cli/commands/models/deployments/deploy/deploy_commands_group.py

@@ -0,0 +1,19 @@
+from click import group
+
+from frogml_cli.commands.models.deployments.deploy.batch.ui import batch
+from frogml_cli.commands.models.deployments.deploy.realtime.ui import realtime
+from frogml_cli.commands.models.deployments.deploy.streaming.ui import stream
+
+
+@group(
+    name="deploy",
+    help="Model deployments",
+)
+def deploy_group():
+    # Click group injection
+    pass
+
+
+deploy_group.add_command(realtime, "realtime")
+deploy_group.add_command(stream, "stream")
+deploy_group.add_command(batch, "batch")

frogml_cli/commands/models/deployments/deploy/realtime/_logic/advanced_deployment_mapper.py

@@ -0,0 +1,21 @@
+from frogml._proto.qwak.deployment.deployment_pb2 import AdvancedDeploymentOptions
+from google.protobuf.wrappers_pb2 import BoolValue
+
+from frogml_cli.commands.models.deployments.deploy._logic.deploy_config import (
+    DeployConfig,
+)
+
+
+def realtime_advanced_deployment_options_from_deploy_config(
+    deploy_config: DeployConfig,
+) -> AdvancedDeploymentOptions:
+    return AdvancedDeploymentOptions(
+        number_of_http_server_workers=deploy_config.realtime.workers,
+        http_request_timeout_ms=deploy_config.realtime.timeout,
+        daemon_mode=BoolValue(value=deploy_config.realtime.daemon_mode),
+        max_batch_size=deploy_config.realtime.max_batch_size,
+        custom_iam_role_arn=deploy_config.advanced_options.iam_role_arn,
+        purchase_option=deploy_config.advanced_options.purchase_option,
+        deployment_process_timeout_limit=deploy_config.realtime.deployment_timeout,
+        service_account_key_secret_name=deploy_config.advanced_options.service_account_key_secret_name,
+    )

frogml_cli/commands/models/deployments/deploy/realtime/_logic/deploy_executor.py

@@ -0,0 +1,24 @@
+from frogml._proto.qwak.deployment.deployment_pb2 import KubeDeploymentType
+from frogml._proto.qwak.deployment.deployment_service_pb2 import DeployModelResponse
+
+from frogml_cli.commands.models.deployments.deploy._logic.base_deploy_executor import (
+    BaseDeployExecutor,
+)
+from frogml_cli.commands.models.deployments.deploy._logic.deployment_message_helpers import (
+    get_env_to_deployment_message,
+)
+
+
+class RealtimeDeployExecutor(BaseDeployExecutor):
+    def deploy(self) -> DeployModelResponse:
+        env_deployment_messages = get_env_to_deployment_message(
+            self.config,
+            KubeDeploymentType.ONLINE,
+            self.ecosystem_client,
+            self.instance_template_client,
+        )
+        return self.deploy_client.deploy_model(
+            model_id=self.config.model_id,
+            build_id=self.config.build_id,
+            env_deployment_messages=env_deployment_messages,
+        )

frogml_cli/commands/models/deployments/deploy/realtime/_logic/serving_strategy_mapper.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import Dict, List
+
+from frogml._proto.qwak.auto_scaling.v1.auto_scaling_pb2 import AutoScalingConfig
+from frogml._proto.qwak.deployment.deployment_pb2 import (
+    RealTimeConfig,
+    ServingStrategy,
+    TrafficConfig,
+)
+from frogml._proto.qwak.ecosystem.v0.ecosystem_pb2 import UserContextEnvironmentDetails
+from frogml.core.exceptions import FrogmlException
+
+from frogml_cli.commands.audience._logic.config.v1.audience_config import AudienceConfig
+from frogml_cli.commands.models.deployments.deploy._logic.deploy_config import (
+    DeployConfig,
+)
+
+
+def create_realtime_serving_strategy(
+    auto_scaling: AutoScalingConfig,
+    audiences: List[AudienceConfig],
+    fallback_variation: str,
+    variation_name: str,
+    variation_protected_state: bool = False,
+) -> ServingStrategy:
+    return ServingStrategy(
+        realtime_config=(
+            RealTimeConfig(
+                auto_scaling_config=auto_scaling,
+                traffic_config=TrafficConfig(
+                    selected_variation_name=variation_name,
+                    audience_routes_entries=[
+                        audience.to_audience_route_entry(index)
+                        for index, audience in enumerate(audiences)
+                    ],
+                    fallback_variation=fallback_variation,
+                    selected_variation_protect_state=variation_protected_state,
+                ),
+            )
+        )
+    )
+
+
+def create_realtime_serving_strategy_from_deploy_config(
+    deploy_config: DeployConfig,
+    environment_name_to_config: Dict[str, UserContextEnvironmentDetails],
+) -> Dict[str, ServingStrategy]:
+    serving_strategies = {}
+    errors = []
+    variation_name = deploy_config.realtime.variation_name or "default"
+    variation_protected_state = deploy_config.realtime.variation_protected_state
+    fallback_variation = deploy_config.realtime.fallback_variation
+    auto_scaling = (
+        deploy_config.auto_scaling.to_autoscaling_api()
+        if deploy_config.auto_scaling
+        else None
+    )
+    for env_name, env_config in environment_name_to_config.items():
+        try:
+            serving_strategies[env_config.id] = create_realtime_serving_strategy(
+                auto_scaling,
+                deploy_config.realtime.audiences,
+                fallback_variation,
+                variation_name,
+                variation_protected_state,
+            )
+
+        except FrogmlException as e:
+            errors.append(e.message)
+
+    if errors:
+        raise FrogmlException("\n".join(errors))
+
+    return serving_strategies