frogml-core 0.0.1__py3-none-any.whl

frogml/inner/build_config/build_config_v1.py

@@ -0,0 +1,313 @@
+from dataclasses import dataclass, field
+from os.path import expandvars
+from typing import Dict, List, Optional, Tuple, Union
+
+import grpc
+
+from _frogml_proto.qwak.builds.build_pb2 import BaseDockerImageType
+from _frogml_proto.qwak.builds.builds_pb2 import (
+    BuildConfiguration,
+    BuildEnvironment,
+    BuildModelUri,
+    BuildPropertiesProto,
+    CondaBuild,
+    DockerBuild,
+    PoetryBuild,
+    PythonBuild,
+    RemoteBuild,
+)
+from _frogml_proto.qwak.builds.builds_pb2 import (
+    RemoteBuildResources as RemoteBuildResourcesProto,
+)
+from _frogml_proto.qwak.builds.builds_pb2 import Step, VirtualEnvironmentBuild
+from _frogml_proto.qwak.instance_template.instance_template_pb2 import InstanceType
+from frogml.clients.build_orchestrator import BuildOrchestratorClient
+from frogml.clients.instance_template.client import InstanceTemplateManagementClient
+from frogml.exceptions import FrogmlException
+from frogml.inner.tool.protobuf_factory import protobuf_factory
+from frogml.inner.tool.run_config import (
+    ConfigCliMap,
+    FrogmlConfigBase,
+    YamlConfigMixin,
+    validate_bool,
+    validate_float,
+    validate_int,
+    validate_list_of_strings,
+    validate_string,
+)
+from frogml.inner.tool.run_config.utils import (
+    validate_positive_int,
+    validate_purchase_option,
+)
+from frogml.model.base import BaseModel
+
+CONFIG_MAPPING: List[ConfigCliMap] = [
+    # ConfigV1
+    ConfigCliMap(
+        "validate_build_artifact", "step.validate_build_artifact", validate_bool, True
+    ),
+    ConfigCliMap(
+        "validate_build_artifact_timeout",
+        "step.validate_build_artifact_timeout",
+        validate_positive_int,
+        True,
+    ),
+    ConfigCliMap("tests", "step.tests", validate_bool, True),
+    # BuildProperties
+    ConfigCliMap("model_id", "build_properties.model_id", validate_string, True),
+    ConfigCliMap("build_id", "build_properties.build_id", validate_string, False),
+    ConfigCliMap(
+        "gpu_compatible", "build_properties.gpu_compatible", validate_bool, True
+    ),
+    ConfigCliMap("branch", "build_properties.branch", validate_string),
+    ConfigCliMap("tags", "build_properties.tags", validate_list_of_strings),
+    ConfigCliMap("build_name", "build_properties.build_name", validate_string),
+    # ModelUri
+    ConfigCliMap("uri", "build_properties.model_uri.uri", validate_string, False),
+    ConfigCliMap(
+        "git_branch", "build_properties.model_uri.git_branch", validate_string
+    ),
+    ConfigCliMap("main_dir", "build_properties.model_uri.main_dir", validate_string),
+    ConfigCliMap(
+        "git_credentials", "build_properties.model_uri.git_credentials", validate_string
+    ),
+    ConfigCliMap(
+        "git_secret_ssh", "build_properties.model_uri.git_secret_ssh", validate_string
+    ),
+    ConfigCliMap(
+        "git_credentials_secret",
+        "build_properties.model_uri.git_credentials_secret",
+        validate_string,
+    ),
+    ConfigCliMap(
+        "dependency_required_folders",
+        "build_properties.model_uri.dependency_required_folders",
+        validate_list_of_strings,
+    ),
+    # BuildEnv
+    # PythonEnv
+    ConfigCliMap(
+        "dependency_file_path",
+        "build_env.python_env.dependency_file_path",
+        validate_string,
+    ),
+    # DockerConf
+    ConfigCliMap("base_image", "build_env.docker.base_image", validate_string),
+    ConfigCliMap("param_list", "build_env.docker.params", validate_list_of_strings),
+    ConfigCliMap("env_vars", "build_env.docker.env_vars", validate_list_of_strings),
+    ConfigCliMap("cache", "build_env.docker.cache", validate_bool),
+    ConfigCliMap("push", "build_env.docker.push", validate_bool),
+    ConfigCliMap(
+        "iam_role_arn", "build_env.docker.assumed_iam_role_arn", validate_string
+    ),
+    ConfigCliMap(
+        "service_account_key_secret_name",
+        "build_env.docker.service_account_key_secret_name",
+        validate_string,
+    ),
+    # RemoteConf
+    ConfigCliMap("cpus", "build_env.remote.resources.cpus", validate_float),
+    ConfigCliMap("memory", "build_env.remote.resources.memory", validate_string),
+    ConfigCliMap("gpu_type", "build_env.remote.resources.gpu_type", validate_string),
+    ConfigCliMap(
+        "gpu_amount", "build_env.remote.resources.gpu_amount", validate_positive_int
+    ),
+    ConfigCliMap("instance", "build_env.remote.resources.instance", validate_string),
+    # Verbosity level
+    ConfigCliMap("verbose", "verbose", validate_int),
+    ConfigCliMap("deploy", "deploy", validate_bool),
+    ConfigCliMap("deployment_instance", "deployment_instance", validate_string),
+    ConfigCliMap("purchase_option", "purchase_option", validate_purchase_option),
+    ConfigCliMap(
+        "provision_instance_timeout",
+        "provision_instance_timeout",
+        validate_positive_int,
+    ),
+]
+
+
+@protobuf_factory(Step)
+@dataclass
+class Step:
+    validate_build_artifact: bool = field(default=True)
+    validate_build_artifact_timeout: int = field(default=120)
+    tests: bool = field(default=True)
+
+
+@protobuf_factory(
+    BuildModelUri,
+    exclude_fields=[
+        "git_credentials",
+        "git_credentials_secret",
+        "dependency_required_folders",
+    ],
+)
+@dataclass
+class ModelUri:
+    uri: Optional[str] = field(default=None)
+    git_branch: Optional[str] = field(default="master")
+    main_dir: Optional[str] = field(default="main")
+    dependency_required_folders: List[str] = field(default_factory=list)
+    git_credentials: Optional[str] = field(default=None)
+    git_credentials_secret: Optional[str] = field(default=None)
+    git_secret_ssh: Optional[str] = field(default=None)
+
+
+@protobuf_factory(BuildPropertiesProto)
+@dataclass
+class BuildProperties:
+    build_id: Optional[str] = field(default=None)
+    gpu_compatible: bool = field(default=False)
+    branch: Optional[str] = field(default="main")
+    model_id: str = field(default="")
+    tags: List[str] = field(default_factory=list)
+    model_uri: ModelUri = field(default_factory=ModelUri)
+    build_name: Optional[str] = field(default="")
+
+
+@protobuf_factory(DockerBuild)
+@dataclass
+class DockerConf:
+    base_image: Optional[str] = field(default=None)
+    env_vars: Optional[Union[List[str], Dict[str, str], Tuple]] = field(
+        default_factory=list
+    )
+    assumed_iam_role_arn: str = field(default=None)
+    service_account_key_secret_name: str = field(default=None)
+    params: Optional[List[str]] = field(default_factory=list)
+    no_cache: bool = field(default=False)
+    cache: bool = field(default=True)
+    push: bool = field(default=True)
+
+    def __post_init__(self):
+        if isinstance(self.env_vars, (list, tuple)):
+            self.env_vars = [expandvars(var) for var in self.env_vars]
+
+
+@protobuf_factory(VirtualEnvironmentBuild)
+@dataclass
+class VirtualenvType:
+    python_version: Optional[str] = field(default=None)
+    requirements_txt: Optional[str] = field(default=None)
+
+
+@protobuf_factory(PoetryBuild)
+@dataclass
+class PoetryType:
+    python_version: Optional[str] = field(default=None)
+    lock_file: Optional[str] = field(default=None)
+
+
+@protobuf_factory(CondaBuild)
+@dataclass
+class CondaType:
+    conda_file: Optional[str] = field(default=None)
+
+
+@protobuf_factory(
+    PythonBuild, exclude_fields=["git_credentials_secret", "git_credentials"]
+)
+@dataclass
+class PythonEnv:
+    git_credentials: Optional[str] = field(default=None)
+    git_credentials_secret: Optional[str] = field(default=None)
+    poetry: PoetryType = field(default=None)
+    conda: CondaType = field(default=None)
+    virtualenv: VirtualenvType = field(default=None)
+    dependency_file_path: Optional[str] = field(default=None)
+
+    def __post_init__(self):
+        if not (self.poetry or self.conda or self.virtualenv):
+            self.conda = CondaType("conda.yml")
+
+
+@protobuf_factory(RemoteBuildResourcesProto)
+@dataclass
+class RemoteBuildResources:
+    cpus: Optional[float] = field(default=None)
+    memory: Optional[str] = field(default=None)
+    gpu_type: Optional[str] = field(default=None)
+    gpu_amount: Optional[int] = field(default=None)
+    instance: Optional[str] = field(default=None)
+
+
+@protobuf_factory(RemoteBuild)
+@dataclass
+class RemoteConf:
+    is_remote: bool = field(default=True)
+    resources: RemoteBuildResources = field(default_factory=RemoteBuildResources)
+
+
+@protobuf_factory(BuildEnvironment)
+@dataclass
+class BuildEnv:
+    docker: DockerConf = field(default_factory=DockerConf)
+    python_env: PythonEnv = field(default_factory=PythonEnv)
+    remote: RemoteConf = field(default_factory=RemoteConf)
+
+
+@protobuf_factory(BuildConfiguration, exclude_fields=["deploy"])
+@dataclass
+class BuildConfigV1(YamlConfigMixin, FrogmlConfigBase):
+    @property
+    def _config_mapping(self) -> List[ConfigCliMap]:
+        return CONFIG_MAPPING
+
+    build_properties: BuildProperties = field(default_factory=BuildProperties)
+    build_env: Optional[BuildEnv] = field(default_factory=BuildEnv)
+    pre_build: Optional[BuildEnv] = field(default=None)
+    post_build: Optional[BuildEnv] = field(default=None)
+    step: Step = field(default_factory=Step)
+    deploy: bool = field(default=False)
+    deployment_instance: Optional[str] = field(default=None)
+    purchase_option: Optional[str] = field(default=None)
+    provision_instance_timeout: Optional[int] = field(default=None)
+    verbose: int = field(default=0)
+    pre_built_model: Optional[BaseModel] = field(default=None, init=False)
+
+    def _post_merge_cli(self):
+        """Implementing post merge actions."""
+        self.build_properties.model_id = self.build_properties.model_id.lower()
+        self.build_properties.tags = list(self.build_properties.tags)
+        self.build_properties.model_uri.dependency_required_folders = list(
+            self.build_properties.model_uri.dependency_required_folders
+        )
+        self.build_env.docker.env_vars = list(self.build_env.docker.env_vars)
+        self.build_env.docker.params = list(self.build_env.docker.params)
+
+    def fetch_base_docker_image_name(
+        self,
+        build_orchestrator: BuildOrchestratorClient,
+        instance_template: InstanceTemplateManagementClient,
+    ):
+        if not self.build_env.docker.base_image:
+            is_gpu_used = (
+                self.build_env.remote.resources.gpu_type
+                and self.build_env.remote.resources.gpu_amount
+                and self.build_env.remote.resources.gpu_amount > 0
+            )
+
+            if self.build_env.remote.resources.instance and not is_gpu_used:
+                is_gpu_used = self.deduce_gpu_from_template(instance_template)
+
+            if is_gpu_used or self.build_properties.gpu_compatible:
+                result = build_orchestrator.fetch_base_docker_image_name(
+                    BaseDockerImageType.GPU
+                )
+                self.build_env.docker.base_image = result.base_docker_image_name
+            else:
+                result = build_orchestrator.fetch_base_docker_image_name(
+                    BaseDockerImageType.CPU
+                )
+                self.build_env.docker.base_image = result.base_docker_image_name
+
+    def deduce_gpu_from_template(self, instance_template):
+        try:
+            instance_spec = instance_template.get_instance_template(
+                self.build_env.remote.resources.instance
+            )
+            return instance_spec.instance_type == InstanceType.INSTANCE_TYPE_GPU
+
+        except FrogmlException as e:
+            if not e.status_code or e.status_code != grpc.StatusCode.NOT_FOUND:
+                raise e

frogml/inner/build_logic/build_loggers/trigger_build_logger.py

@@ -0,0 +1,50 @@
+import logging
+from logging import Logger
+
+from frogml.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml.inner.build_logic.interface.build_phase import BuildPhase
+
+PREFIX_FORMAT = "{prefix} - "
+EXCEPTION_FORMAT = """Message: {line}
+Exception: {e}
+"""
+
+
+class TriggerBuildLogger(BuildLogger):
+    def __init__(
+        self,
+        logger: Logger,
+        prefix: str,
+        build_phase: BuildPhase,
+        verbose: int = 0,
+        json_logs: bool = False,
+    ) -> None:
+        self.logger = logging.LoggerAdapter(
+            logger,
+            {
+                "phase": build_phase.description,
+                "phase_id": build_phase.phase_id,
+            },
+        )
+        self.prefix = PREFIX_FORMAT.format(prefix=prefix) if prefix else ""
+        self.spinner = None
+        self.verbose = verbose
+        self.json_logs = json_logs
+
+    def exception(self, line: str, e: BaseException) -> None:
+        self.logger.error(
+            EXCEPTION_FORMAT.format(line=line, e=e),
+            exc_info=False,
+        )
+
+    def error(self, line: str) -> None:
+        self.logger.error(f"{self.prefix}{line}")
+
+    def warning(self, line: str) -> None:
+        self.logger.warning(f"{self.prefix}{line}")
+
+    def info(self, line: str) -> None:
+        self.logger.info(f"{self.prefix}{line}")
+
+    def debug(self, line: str) -> None:
+        self.logger.debug(f"{self.prefix}{line}")

frogml/inner/build_logic/constants/dependencies.py

@@ -0,0 +1,5 @@
+PIP_DEPENDENCY_FILE = "requirements.txt"
+CONDA_YML_DEPENDENCY_FILE = "conda.yml"
+CONDA_YAML_DEPENDENCY_FILE = "conda.yaml"
+POETRY_DEPENDENCY_FILE = "pyproject.toml"
+POETRY_LOCK_DEPENDENCY_FILE = "poetry.lock"

frogml/inner/build_logic/constants/host_resource.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+HOST_FROGML_HIDDEN_FOLDER = Path.home() / ".frogml"
+HOST_TEMP_BUILD_DIR = HOST_FROGML_HIDDEN_FOLDER / "builds"

frogml/inner/build_logic/constants/messages.py

@@ -0,0 +1,3 @@
+FAILED_CONTACT_FROGML_SUPPORT_PROGRAMMATIC = """
+Build ID \033[4m{build_id}\033[0m failed!!
+"""

frogml/inner/build_logic/constants/temp_dir.py

@@ -0,0 +1 @@
+TEMP_LOCAL_MODEL_DIR = "model_code"

frogml/inner/build_logic/constants/upload_tag.py

@@ -0,0 +1,7 @@
+MODEL_CODE_TAG = "code.zip"
+SKINNY_MODEL_CODE_TAG = "skinny_code.zip"
+FROGML_SDK_VERSION_TAG = "frogml_sdk_version"
+FROGML_RUNTIME_WHEEL_TAG = "frogml_runtime_wheel"
+FROGML_CORE_WHEEL_TAG = "frogml_core_wheel"
+BUILD_CONFIG_TAG = "frogml_build_config"
+FROGML_BUILT_MODEL_TAG = "frogml_built_model"

frogml/inner/build_logic/dependency_manager_type.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class DependencyManagerType(Enum):
+    UNKNOWN = 0
+    PIP = 1
+    POETRY = 2
+    CONDA = 3

frogml/inner/build_logic/execute_build_pipeline.py

@@ -0,0 +1,64 @@
+from frogml.exceptions import (
+    FrogmlGeneralBuildException,
+    FrogmlRemoteBuildFailedException,
+)
+from frogml.inner.build_logic.interface.phase_run_handler import PhaseRunHandler
+from frogml.inner.build_logic.interface.time_source import (
+    Stopwatch,
+    SystemClockTimeSource,
+    TimeSource,
+)
+from frogml.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+
+
+def execute_build_pipeline(
+    pipeline: PhasesPipeline,
+    phase_run_handler: PhaseRunHandler,
+):
+    time_source = SystemClockTimeSource()
+    for phase in pipeline.phases:
+        with phase_run_handler.handle_current_phase(phase):
+            phase_run(phase, phase_run_handler, time_source)
+
+
+def phase_run(
+    phase: PhasesPipeline,
+    phase_run_handler: PhaseRunHandler,
+    time_source: TimeSource,
+):
+    build_id = phase.context.build_id
+    current_step_phase = None
+    stop_watch = Stopwatch(time_source)
+    try:
+        for step in phase.steps:
+            current_step_phase = step.build_phase
+            step.set_logger(phase_run_handler.build_logger)
+            phase_run_handler.handle_phase_in_progress(current_step_phase)
+
+            step.execute()
+
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_phase_finished_successfully(
+            current_step_phase, phase_duration
+        )
+    except FrogmlGeneralBuildException as e:
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_contact_support_error(
+            build_id, current_step_phase, e, phase_duration
+        )
+    except FrogmlRemoteBuildFailedException as e:
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_remote_build_error(
+            build_id, current_step_phase, e, phase_duration
+        )
+    except KeyboardInterrupt:
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_keyboard_interrupt(
+            build_id, current_step_phase, phase_duration
+        )
+    except BaseException as e:
+        phase_run_handler.build_logger.exception("Failed", e)
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_pipeline_exception(
+            build_id, current_step_phase, e, phase_duration
+        )

frogml/inner/build_logic/interface/build_logger_interface.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class BuildLogger(ABC):
+    verbose: int
+    json_logs: bool
+
+    @abstractmethod
+    def error(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def exception(self, line: str, e: BaseException) -> None:
+        pass
+
+    @abstractmethod
+    def warning(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def info(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def debug(self, line: str) -> None:
+        pass

frogml/inner/build_logic/interface/build_phase.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+
+class BuildPhase:
+    def __init__(
+        self,
+        phase_id: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ):
+        self.phase_id = phase_id
+        self.name = name
+        self.description = description
+        if not self.name:
+            self.name = phase_id.replace("_", " ").capitalize()
+        if not self.description:
+            self.description = phase_id.replace("_", " ").capitalize()
+
+    def __eq__(self, other):
+        return (
+            self.phase_id == other.phase_id
+            and self.name == other.name
+            and self.description == other.description
+        )

frogml/inner/build_logic/interface/context_interface.py

@@ -0,0 +1,56 @@
+from abc import ABCMeta
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from frogml.clients.administration.eco_system.client import EcosystemClient
+from frogml.clients.build_orchestrator import BuildOrchestratorClient
+from frogml.clients.instance_template.client import InstanceTemplateManagementClient
+from frogml.clients.model_management import ModelsManagementClient
+from frogml.inner.build_logic.dependency_manager_type import DependencyManagerType
+
+
+@dataclass
+class Context(metaclass=ABCMeta):
+    # Clients
+    client_builds_orchestrator: BuildOrchestratorClient = field(
+        default_factory=BuildOrchestratorClient
+    )
+    client_models_management: ModelsManagementClient = field(
+        default_factory=ModelsManagementClient
+    )
+    client_instance_template: InstanceTemplateManagementClient = field(
+        default_factory=InstanceTemplateManagementClient
+    )
+    client_ecosystem: EcosystemClient = field(default_factory=EcosystemClient)
+
+    # Pre fetch validation
+    build_id: str = field(default="")
+    model_id: str = field(default="")
+    build_name: str = field(default="")
+    project_uuid: str = field(default="")
+    host_temp_local_build_dir: Path = field(default=None)
+    model_uri: Path = field(default=None)
+    git_credentials: str = field(default="")
+    git_ssh_key: str = field(default="")
+
+    # Fetch model
+    git_commit_id: Optional[str] = field(default=None)
+
+    # Post fetch validation
+    dependency_manager_type: DependencyManagerType = field(
+        default=DependencyManagerType.UNKNOWN
+    )
+    model_relative_dependency_file: Path = field(default=None)
+    model_relative_dependency_lock_file: Path = field(default=None)
+
+    # Upload model
+    model_code_remote_url: Optional[str] = field(default=None)
+    frogml_sdk_version: Optional[str] = field(default=None)
+
+    # Image
+    base_image: Optional[str] = field(default=None)
+
+    # Upload Custom Wheels
+    custom_runtime_wheel: Optional[Path] = field(default=None)
+    custom_core_wheel: Optional[Path] = field(default=None)

frogml/inner/build_logic/interface/phase_run_handler.py

@@ -0,0 +1,71 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from frogml.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml.inner.build_logic.interface.build_phase import BuildPhase
+from frogml.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+
+
+class PhaseRunHandler(ABC):
+    current_phase: PhasesPipeline
+    build_logger: Optional[BuildLogger]
+
+    @abstractmethod
+    def handle_phase_in_progress(self, build_phase: BuildPhase):
+        pass
+
+    @abstractmethod
+    def handle_phase_finished_successfully(
+        self, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        pass
+
+    @abstractmethod
+    def handle_contact_support_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_remote_build_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_keyboard_interrupt(
+        self, build_id: str, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        pass
+
+    @abstractmethod
+    def handle_pipeline_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_pipeline_quiet_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_current_phase(self, phase: PhasesPipeline):
+        pass

frogml/inner/build_logic/interface/step_inteface.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from frogml.inner.build_config.build_config_v1 import BuildConfigV1
+from frogml.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml.inner.build_logic.interface.build_phase import BuildPhase
+from frogml.inner.build_logic.interface.context_interface import Context
+
+
+class Step(ABC):
+    context: Context
+    config: BuildConfigV1
+    build_logger: BuildLogger
+    build_phase: BuildPhase
+
+    @abstractmethod
+    def description(self) -> str:
+        pass
+
+    @abstractmethod
+    def execute(self) -> None:
+        pass
+
+    def set_logger(self, build_logger: BuildLogger) -> None:
+        self.build_logger = build_logger