frogml-core 0.0.1__py3-none-any.whl

frogml/inner/build_logic/phases/phase_010_fetch_model/post_fetch_validation_step.py

@@ -0,0 +1,139 @@
+import importlib.util
+import os
+import shutil
+import subprocess  # nosec
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from frogml.exceptions import FrogmlSuggestionException
+from frogml.inner.build_logic.constants.temp_dir import TEMP_LOCAL_MODEL_DIR
+from frogml.inner.build_logic.dependency_manager_type import DependencyManagerType
+from frogml.inner.build_logic.interface.step_inteface import Step
+from frogml.inner.build_logic.tools.dependencies_tools import find_dependency_files
+
+TEMP_LOCAL_EDITABLE_FOLDER = "editable"
+
+
+@dataclass
+class DependencyFileObject:
+    dep_file_name: List[str]
+    lock_file_name: str = field(default="")
+
+
+DEPS_MANAGER_FILE_MAP = {
+    DependencyManagerType.PIP: DependencyFileObject(dep_file_name=["requirements.txt"]),
+    DependencyManagerType.POETRY: DependencyFileObject(
+        dep_file_name=["pyproject.toml"], lock_file_name="poetry.lock"
+    ),
+    DependencyManagerType.CONDA: DependencyFileObject(
+        dep_file_name=["conda.yml", "conda.yaml"]
+    ),
+}
+
+
+class PostFetchValidationStep(Step):
+    STEP_DESCRIPTION = "Post model fetch validation"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        self.validate_dependencies()
+        self.configure_base_docker_image()
+        self.create_development_wheels()
+
+    def validate_dependencies(self):
+        if (
+            not Path(self.config.build_properties.model_uri.uri).is_dir()
+            or self.config.build_env.python_env.dependency_file_path
+        ):
+            self.build_logger.debug("Validating dependency file exists")
+            model_uri, main_dir = (
+                self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR,
+                self.config.build_properties.model_uri.main_dir,
+            )
+            (
+                self.context.dependency_manager_type,
+                self.context.model_relative_dependency_file,
+                self.context.model_relative_dependency_lock_file,
+            ) = find_dependency_files(model_uri, main_dir, self.build_logger)
+
+            if (
+                self.context.dependency_manager_type
+                and self.context.model_relative_dependency_file
+            ):
+                return
+
+            self.build_logger.error("Dependency file wasn't found, failing...")
+            raise FrogmlSuggestionException(
+                message="Dependency file isn't found",
+                suggestion="Make sure your model include one of dependencies manager: pip/poetry/conda",
+            )
+
+    def configure_base_docker_image(self):
+        base_image = self.config.build_env.docker.base_image
+        if (not base_image) and self.config.build_env.remote.resources.gpu_type:
+            base_image = "frogml/frogml:gpu-py39"  # todo check gpu
+        self.context.base_image = base_image
+
+    def create_development_wheels(self):
+        if not os.getenv("FROGML_DEBUG"):
+            return
+
+        (module_location,) = importlib.util.find_spec(
+            "frogml_sdk"
+        ).submodule_search_locations
+        source_dir = Path(module_location).parent
+        pyproject_toml = source_dir / "pyproject.toml"
+
+        if pyproject_toml.is_file():
+            self.build_logger.info(
+                "Detected non-PyPI-released frogml-sdk installed, creating local frogml-runtime "
+                "and frogml-core wheel files to pass to build process"
+            )
+            runtime_dir = source_dir.parent / "frogml-runtime"
+            core_dir = source_dir.parent / "frogml-core"
+            target_path = self.context.host_temp_local_build_dir
+            self.context.custom_runtime_wheel = self._create_wheel(
+                runtime_dir, target_path
+            )
+            self.context.custom_core_wheel = self._create_wheel(core_dir, target_path)
+
+    def _create_wheel(self, package_dir, target_path):
+        dist_dir = package_dir / "dist"
+        shutil.rmtree(dist_dir, ignore_errors=True)
+        output = subprocess.check_output(["make", "install"], cwd=package_dir)  # nosec
+        for line in output.decode().split("\n"):
+            self.build_logger.debug(f">>> {line}")
+        output = subprocess.check_output(["poetry", "build"], cwd=package_dir)  # nosec
+        for line in output.decode().split("\n"):
+            self.build_logger.debug(f">>> {line}")
+        wheel_file = next(dist_dir.glob("*.whl"), None)
+        editable_folder = target_path / TEMP_LOCAL_EDITABLE_FOLDER
+        editable_folder.mkdir(exist_ok=True)
+        copied_wheel_file = shutil.move(wheel_file, editable_folder / wheel_file.name)
+        shutil.rmtree(dist_dir, ignore_errors=True)
+        self.build_logger.info(f"Created wheel for {package_dir} successfully")
+        return copied_wheel_file
+
+
+def find_file_location(model_uri, main_dir, filename) -> Path:
+    file_locations: List[Path] = [
+        model_uri / filename,
+        model_uri / main_dir / filename,
+    ]
+    for file in file_locations:
+        if file.is_file():
+            return file
+
+
+def get_possible_dependency_lock_paths(dependency_path: Path):
+    paths = []
+    for _, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        if dependency_file_object.lock_file_name:
+            lock_file_path = (
+                dependency_path.parent / dependency_file_object.lock_file_name
+            )
+            paths.append(lock_file_path)
+    return paths

frogml/inner/build_logic/phases/phase_010_fetch_model/pre_fetch_validation_step.py

@@ -0,0 +1,263 @@
+import uuid
+from pathlib import Path
+
+import grpc
+
+from frogml.clients.secret_service import SecretServiceClient
+from frogml.exceptions import FrogmlException, FrogmlSuggestionException
+from frogml.inner.build_logic.constants.host_resource import HOST_TEMP_BUILD_DIR
+from frogml.inner.build_logic.interface.step_inteface import Step
+from frogml.inner.build_logic.tools.dependencies_tools import find_dependency_files
+from frogml.inner.build_logic.tools.text import snake_case
+from frogml.inner.instance_template.verify_template_id import verify_template_id
+from frogml.inner.provider import Provider
+from frogml.model.base import BaseModel
+
+
+class PreFetchValidationStep(Step):
+    STEP_DESCRIPTION = "Pre model fetch validation"
+    DEFAULT_CPUS = 2
+    DEFAULT_MEMORY = "4Gi"
+    VALIDATION_FAILURE_CPU_AND_GPU_MESSAGE = (
+        "CPU and GPU configured together, Invalid configuration."
+    )
+    VALIDATION_FAILURE_CPU_AND_GPU_SUGGESTION = "Please configure only CPU or GPU"
+    VALIDATION_FAILURE_CPU_AND_INSTANCE_MESSAGE = (
+        "CPU and instance configured together, Invalid configuration."
+    )
+    VALIDATION_FAILURE_CPU_AND_INSTANCE_SUGGESTION = (
+        "Please configure only CPU or instance"
+    )
+    VALIDATION_FAILURE_GPU_AND_INSTANCE_MESSAGE = (
+        "GPU and instance configured together, Invalid configuration."
+    )
+    VALIDATION_FAILURE_GPU_AND_INSTANCE_SUGGESTION = (
+        "Please configure only GPU or instance"
+    )
+    EXISTING_MODEL_PASSED_MSG = "Build process was provided with pre built model"
+    INVALID_FROGML_MODEL_MSG_FORMAT = (
+        "The provided frogml model is not of {expected_class} but of {passed_class}"
+    )
+    SERVICE_ACCOUNT_NOT_FOUND = (
+        "GCP service account key secret '{secret_name}' wasn't found"
+    )
+    SERVICE_ACCOUNT_NOT_FOUND_SUGGESTION = "Make sure your GCP service account key secret is configured correctly in the environment"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        self.validate_build_properties()
+        self.create_build_id()
+        self.collect_build_name()
+        self.collect_model_id()
+        self.create_build_dir()
+        self.collect_git_credentials()
+        self.validate_dependencies()
+        self.validate_resources()
+        self.validate_deployment_resources()
+        self.validate_frogml_model()
+        self.validate_gcp_service_account_secret()
+
+    def validate_build_properties(self):
+        if not self.config.build_properties.model_uri.uri:
+            error_message = "Model uri wasn't set"
+            self.build_logger.error(f"{error_message}, failing...")
+
+            raise FrogmlSuggestionException(
+                message=error_message,
+                suggestion="Make sure your build properties object contains model uri argument",
+            )
+        if (
+            self.config.build_properties.model_uri.git_credentials_secret
+            and self.config.build_properties.model_uri.git_secret_ssh
+        ):
+            error_message = (
+                "Only one of git credentials secret or git secret ssh can be configured"
+            )
+            raise FrogmlSuggestionException(
+                message=error_message,
+                suggestion="Make sure your git credentials or git secret ssh are configured correctly and only one of "
+                "them is configured",
+            )
+
+    def collect_model_id(self):
+        model_id = self.config.build_properties.model_id
+        model = self.context.client_models_management.get_model(
+            model_id=model_id,
+            exception_on_missing=False,
+        )
+        if not model:
+            suggestion = f"Create model {model_id} or check model ID spelling"
+            snake_case_model_id = snake_case(model_id)
+            if self.context.client_models_management.is_model_exists(
+                snake_case_model_id
+            ):
+                suggestion = f"Try using model ID {snake_case_model_id} instead"
+            raise FrogmlSuggestionException(
+                message=f"Model ID {model_id} isn't found",
+                suggestion=suggestion,
+            )
+        self.context.project_uuid = model.project_id
+        self.context.model_uuid = model.uuid
+        self.context.model_id = model_id
+
+    def create_build_id(self):
+        if not self.context.build_id:
+            self.context.build_id = str(uuid.uuid4())
+            self.build_logger.info(f"Generated build ID - {self.context.build_id}")
+        else:
+            self.build_logger.info(f"Using given build ID - {self.context.build_id}")
+
+    def collect_build_name(self):
+        build_name: str = self.config.build_properties.build_name
+
+        if build_name:
+            self.build_logger.info(f"Using given build name - {build_name}")
+            self.context.build_name = build_name
+
+    def create_build_dir(self):
+        build_dir = HOST_TEMP_BUILD_DIR / self.context.model_id / self.context.build_id
+        build_dir.mkdir(exist_ok=True, parents=True)
+        self.context.host_temp_local_build_dir = build_dir
+        self.build_logger.debug(f"Build directory created - {build_dir}")
+
+    def collect_git_credentials(self):
+        if self.config.build_properties.model_uri.git_credentials:
+            self.context.git_credentials = (
+                self.config.build_properties.model_uri.git_credentials
+            )
+        elif self.config.build_properties.model_uri.git_credentials_secret:
+            self.context.git_credentials = SecretServiceClient().get_secret(
+                self.config.build_properties.model_uri.git_credentials_secret
+            )
+        elif self.config.build_properties.model_uri.git_secret_ssh:
+            self.context.git_ssh_key = SecretServiceClient().get_secret(
+                self.config.build_properties.model_uri.git_secret_ssh
+            )
+        else:
+            self.build_logger.debug("Git credentials isn't configured")
+
+    def validate_dependencies(self):
+        if (
+            Path(self.config.build_properties.model_uri.uri).is_dir()
+            and not self.config.build_env.python_env.dependency_file_path
+        ):
+            model_uri, main_dir = (
+                Path(self.config.build_properties.model_uri.uri),
+                self.config.build_properties.model_uri.main_dir,
+            )
+            (
+                self.context.dependency_manager_type,
+                self.context.model_relative_dependency_file,
+                self.context.model_relative_dependency_lock_file,
+            ) = find_dependency_files(model_uri, main_dir, self.build_logger)
+
+            if (
+                self.context.dependency_manager_type
+                and self.context.model_relative_dependency_file
+            ):
+                return
+
+            self.build_logger.error("Dependency file wasn't found, failing...")
+            raise FrogmlSuggestionException(
+                message="Dependency file isn't found",
+                suggestion="Make sure your model include one of dependencies manager: pip/poetry/conda",
+            )
+
+    def validate_resources(self):
+        gpu_configured = (
+            self.config.build_env.remote.resources.gpu_type
+            or self.config.build_env.remote.resources.gpu_amount
+        )
+        cpu_configured = (
+            self.config.build_env.remote.resources.cpus
+            or self.config.build_env.remote.resources.memory
+        )
+        instance_configured = self.config.build_env.remote.resources.instance
+
+        if cpu_configured and gpu_configured:
+            raise FrogmlSuggestionException(
+                message=self.VALIDATION_FAILURE_CPU_AND_GPU_MESSAGE,
+                suggestion=self.VALIDATION_FAILURE_CPU_AND_GPU_SUGGESTION,
+            )
+        if cpu_configured and instance_configured:
+            raise FrogmlSuggestionException(
+                message=self.VALIDATION_FAILURE_CPU_AND_INSTANCE_MESSAGE,
+                suggestion=self.VALIDATION_FAILURE_CPU_AND_INSTANCE_SUGGESTION,
+            )
+        if gpu_configured and instance_configured:
+            raise FrogmlSuggestionException(
+                message=self.VALIDATION_FAILURE_GPU_AND_INSTANCE_MESSAGE,
+                suggestion=self.VALIDATION_FAILURE_GPU_AND_INSTANCE_SUGGESTION,
+            )
+
+        if instance_configured:
+            provider = self.__get_provider()
+            verify_template_id(
+                self.config.build_env.remote.resources.instance,
+                self.context.client_instance_template,
+                provider=provider,
+            )
+
+        if not (cpu_configured or gpu_configured or instance_configured):
+            self.config.build_env.remote.resources.cpus = self.DEFAULT_CPUS
+            self.config.build_env.remote.resources.memory = self.DEFAULT_MEMORY
+
+    def __get_provider(self):
+        user_context = self.context.client_ecosystem.get_authenticated_user_context()
+        provider = None
+        if (
+            user_context.user.environment_details.configuration.cloud_configuration.WhichOneof(
+                "configuration"
+            )
+            == "aws_cloud_configuration"
+        ):
+            provider = Provider.AWS
+        elif (
+            user_context.user.environment_details.configuration.cloud_configuration.WhichOneof(
+                "configuration"
+            )
+            == "gcp_cloud_configuration"
+        ):
+            provider = Provider.GCP
+        return provider
+
+    def validate_deployment_resources(self):
+        if self.config.deploy and self.config.deployment_instance:
+            provider = self.__get_provider()
+            verify_template_id(
+                self.config.deployment_instance,
+                self.context.client_instance_template,
+                provider=provider,
+            )
+
+    def validate_frogml_model(self):
+        if self.config.pre_built_model:
+            self.build_logger.debug(self.EXISTING_MODEL_PASSED_MSG)
+            if not isinstance(self.config.pre_built_model, BaseModel):
+                raise FrogmlException(
+                    self.INVALID_FROGML_MODEL_MSG_FORMAT.format(
+                        expected_class=BaseModel.__name__,
+                        passed_class=self.config.pre_built_model.__class__,
+                    )
+                )
+
+    def validate_gcp_service_account_secret(self):
+        if self.config.build_env.docker.service_account_key_secret_name:
+            self.build_logger.debug("Checking GCP service account key secret")
+            try:
+                SecretServiceClient().get_secret(
+                    self.config.build_env.docker.service_account_key_secret_name
+                )
+            except FrogmlException as e:
+                if e.status_code == grpc.StatusCode.NOT_FOUND:
+                    raise FrogmlSuggestionException(
+                        message=self.SERVICE_ACCOUNT_NOT_FOUND.format(
+                            secret_name=self.config.build_env.docker.service_account_key_secret_name
+                        ),
+                        suggestion=self.SERVICE_ACCOUNT_NOT_FOUND_SUGGESTION,
+                    )
+                raise e
+
+            self.build_logger.debug("service account key secret found")

frogml/inner/build_logic/phases/phase_010_fetch_model/set_version_step.py

@@ -0,0 +1,33 @@
+import importlib
+
+from frogml import __version__ as frogml_core_version
+from frogml.inner.build_logic.interface.step_inteface import Step
+
+
+class SetVersionStep(Step):
+    STEP_DESCRIPTION = "Getting SDK Version"
+    SDK_VERSION_NOT_AVAILABLE_MSG_FORMAT = (
+        "Sdk version not available, using core version {frogml_core_version}"
+    )
+    SDK_VERSION_FOUND_MSG_FORMAT = "Found sdk version {frogml_sdk_version}"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        try:
+            self.build_logger.debug("Getting sdk version")
+            frogml_sdk_version = importlib.import_module("frogml_sdk").__version__
+            self.context.frogml_sdk_version = frogml_sdk_version
+            self.build_logger.debug(
+                self.SDK_VERSION_FOUND_MSG_FORMAT.format(
+                    frogml_sdk_version=frogml_sdk_version
+                )
+            )
+        except ImportError:
+            self.build_logger.debug(
+                self.SDK_VERSION_NOT_AVAILABLE_MSG_FORMAT.format(
+                    frogml_core_version=frogml_core_version
+                )
+            )
+            self.context.frogml_sdk_version = frogml_core_version

frogml/inner/build_logic/phases/phase_020_remote_register_frogml_build/cleanup_step.py

@@ -0,0 +1,20 @@
+import os
+
+from frogml.inner.build_logic.interface.step_inteface import Step
+from frogml.inner.build_logic.tools.files import cleaning_up_after_build
+
+
+class CleanupStep(Step):
+    STEP_DESCRIPTION = "Clean build artifacts"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        self.build_logger.debug(
+            "Cleanup environment - Deleting file and intermediate images"
+        )
+        if os.getenv("POD_NAME"):
+            self.build_logger.debug("Skipping cleanup - Remote executor is temporary")
+        else:
+            cleaning_up_after_build(self)

frogml/inner/build_logic/phases/phase_020_remote_register_frogml_build/start_remote_build_step.py

@@ -0,0 +1,42 @@
+from copy import deepcopy
+
+from _frogml_proto.qwak.builds.build_url_pb2 import BuildVersioningTagsType
+from frogml.inner.build_logic.constants.upload_tag import (
+    BUILD_CONFIG_TAG,
+    FROGML_SDK_VERSION_TAG,
+    MODEL_CODE_TAG,
+)
+from frogml.inner.build_logic.interface.step_inteface import Step
+
+
+class StartRemoteBuildStep(Step):
+    STEP_DESCRIPTION = "Start remote build"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        self.build_logger.info(f"Start remote build - {self.context.build_id}")
+        config_copy = deepcopy(self.config)
+        config_copy.build_properties.build_id = self.context.build_id
+        config_copy.build_properties.build_name = self.context.build_name
+        self.context.client_builds_orchestrator.build_model(
+            build_conf=config_copy,
+            build_v1_flag=False,
+            build_config_url=self.get_download_url(BUILD_CONFIG_TAG),
+            qwak_sdk_version_url=self.get_download_url(FROGML_SDK_VERSION_TAG),
+            resolved_model_url=self.get_download_url(MODEL_CODE_TAG),
+            git_commit_id=self.context.git_commit_id,
+            sdk_version=self.context.qwak_sdk_version,
+        )
+        self.build_logger.info("Remote build started successfully")
+
+    def get_download_url(self, tag: str) -> str:
+        return (
+            self.context.client_builds_orchestrator.get_build_versioning_download_url(
+                model_id=self.config.build_properties.model_id,
+                build_id=self.context.build_id,
+                tag=tag,
+                tag_type=BuildVersioningTagsType.FILE_TAG_TYPE,
+            ).download_url
+        )