qwak-sdk 0.1.0__py3-none-any.whl → 0.2.20rc0__py3-none-any.whl

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_strategy_manager/strategy/git/git_strategy.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import shutil
+import tempfile
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+from qwak.exceptions import QwakException
+
+from qwak_sdk.exceptions import QwakSuggestionException
+from qwak_sdk.tools.files import copytree
+
+if TYPE_CHECKING:
+    from qwak_sdk.commands.models.build import Notifier
+
+try:
+    from git import Repo
+    from git.exc import GitError
+except ImportError:
+    pass
+
+
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class GitStrategy(Strategy):
+    def fetch(
+        self,
+        src: Union[str, Path],
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        custom_dependencies_path: str,
+        main_dir: str,
+        **kwargs,
+    ) -> str:
+        self.notifier.debug(f"Fetching Model code from git repository -  {src}")
+        with tempfile.TemporaryDirectory() as clone_dest:
+            git_url, git_subdirectory = split_git_url_and_subdirectory(uri=src)
+            self.notifier.debug(
+                f"Fetching model code from git {git_url}, Model located in {git_subdirectory}"
+            )
+            git_commit_id = self.clone_and_checkout_repo(
+                clone_dest=clone_dest,
+                git_url=git_url,
+                git_branch=git_branch,
+                git_credentials=git_credentials,
+            )
+            self.copy_model(
+                clone_dest=clone_dest,
+                git_subdirectory=git_subdirectory,
+                model_dest=dest,
+                custom_dependencies_path=custom_dependencies_path,
+                main_dir=main_dir,
+            )
+
+        return git_commit_id
+
+    def clone_and_checkout_repo(
+        self,
+        clone_dest: str,
+        git_url: str,
+        git_branch: str,
+        git_credentials: str,
+    ) -> str:
+        git_remote_url = get_git_remote_uri(
+            self.notifier,
+            url=git_url,
+            git_credentials=git_credentials,
+        )
+        git_commit_id = clone_and_checkout_repo(
+            git_repo_url=git_remote_url, to_path=clone_dest, git_branch=git_branch
+        )
+
+        return git_commit_id
+
+    def copy_model(
+        self,
+        clone_dest: str,
+        git_subdirectory: str,
+        model_dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+    ):
+        ignore_patterns, patterns_for_printing = get_ignore_pattern(
+            Path(clone_dest) / git_subdirectory, main_dir, self.notifier
+        )
+        self.notifier.debug(
+            f"Will ignore the following files: {patterns_for_printing}."
+        )
+
+        # Copy model
+        copytree(
+            src=Path(clone_dest) / git_subdirectory,
+            dst=model_dest,
+            dirs_exist_ok=True,
+            ignore=ignore_patterns,
+        )
+        self.notifier.debug("Model copied from git repository")
+
+        # Copy custom dependencies path
+        if custom_dependencies_path:
+            git_custom_dependencies_path = Path(clone_dest) / Path(
+                custom_dependencies_path
+            )
+            if git_custom_dependencies_path.is_file():
+                shutil.copy(
+                    src=git_custom_dependencies_path,
+                    dst=Path(model_dest) / git_custom_dependencies_path.name,
+                )
+                self.notifier.debug("Custom dependency file copied from git repository")
+
+
+def split_git_url_and_subdirectory(uri):
+    subdirectory = ""
+    parsed_uri = uri
+    if "#" in uri:
+        subdirectory = uri[uri.find("#") + 1 :]
+        parsed_uri = uri[: uri.find("#")]
+    if subdirectory and "." in subdirectory:
+        raise QwakException("'.' is not allowed in project subdirectory paths.")
+    return parsed_uri, subdirectory
+
+
+def get_git_remote_uri(notifier: Notifier, url: str, git_credentials: str) -> str:
+    if git_credentials:
+        url_start_index = url.find("://") + 3
+        url = f"{url[:url_start_index]}{git_credentials}@{url[url_start_index:]}"
+    else:
+        notifier.warning("Git credentials secret has been provided")
+
+    return url
+
+
+def clone_and_checkout_repo(git_repo_url: str, to_path: str, git_branch: str) -> str:
+    try:
+        repo = Repo.clone_from(url=git_repo_url, to_path=to_path)
+        repo.submodule_update(init=True, recursive=True)
+        if git_branch in repo.branches:
+            repo.git.checkout(git_branch)
+        else:
+            repo.git.checkout("-b", git_branch, f"{repo.remote()}/{git_branch}")
+        return repo.head.object.hexsha
+    except GitError as e:
+        raise QwakSuggestionException(
+            src_exception=e,
+            message="Unable to clone git repository",
+            suggestion="Please check you git credentials",
+        )

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_strategy_manager/strategy/strategy.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import shutil
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple, Union
+
+_IGNORED_PATTERNS = [r"\..*", r"__pycache__"]
+QWAK_IGNORE_FILE_NAME = ".qwakignore"
+
+
+if TYPE_CHECKING:
+    from qwak_sdk.commands.models.build import Notifier
+
+
+def load_patterns_from_ignore_file(notifier: Notifier, ignore_file_path: Path):
+    ignore_file_path = Path(ignore_file_path)
+    if ignore_file_path.is_file():
+        notifier.info(
+            f"Found a Qwak ignore file {ignore_file_path} - will ignore listed patterns"
+        )
+
+        with ignore_file_path.open("r") as ignore_file:
+            patterns_to_ignore = [
+                pattern.strip() for pattern in ignore_file.readlines()
+            ]
+            notifier.debug(
+                f"Patterns from Qwak ignore file detected - {str(patterns_to_ignore)}"
+            )
+            return patterns_to_ignore
+
+    notifier.debug("no Qwak ignore file was found, skipping")
+    return []
+
+
+def get_ignore_pattern(
+    src: str, main_dir: str, notifier: Notifier
+) -> Tuple[Callable[[Any, list[str]], set[str]], list[str]]:
+    if (Path(src) / main_dir / QWAK_IGNORE_FILE_NAME).is_file():
+        ignore_file_path = Path(src) / main_dir / QWAK_IGNORE_FILE_NAME
+    else:
+        ignore_file_path = Path(src) / QWAK_IGNORE_FILE_NAME
+
+    ignored_patterns = (
+        load_patterns_from_ignore_file(
+            notifier=notifier, ignore_file_path=ignore_file_path
+        )
+        + _IGNORED_PATTERNS
+    )
+
+    return shutil.ignore_patterns(*ignored_patterns), ignored_patterns
+
+
+class Strategy(ABC):
+    def __init__(self, notifier: Notifier):
+        self.notifier = notifier
+
+    @abstractmethod
+    def fetch(
+        self,
+        src: Union[str, Path],
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        model_id: str,
+        build_id: str,
+        custom_dependency_path: Optional[str],
+    ) -> Optional[str]:
+        pass

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_strategy_manager/strategy/zip/zip_strategy.py

@@ -0,0 +1,64 @@
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Optional, Union
+from zipfile import ZipFile
+
+from qwak_sdk.exceptions import QwakSuggestionException
+from qwak_sdk.tools.files import copytree
+
+from ...common import get_git_commit_id
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class ZipStrategy(Strategy):
+    def fetch(
+        self,
+        src: Union[str, Path],
+        dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+        **kwargs,
+    ) -> Optional[str]:
+        self.notifier.info(f"Fetching Model code from local zip file -  {src}")
+        try:
+            with tempfile.TemporaryDirectory() as temp_extraction_target:
+                with ZipFile(src) as zip_file:
+                    zip_file.extractall(path=temp_extraction_target)
+
+                    git_commit_id = get_git_commit_id(
+                        temp_extraction_target, self.notifier
+                    )
+
+                    ignore_patterns, patterns_for_printing = get_ignore_pattern(
+                        temp_extraction_target,
+                        main_dir,
+                        self.notifier,
+                    )
+                    self.notifier.info(
+                        f"Will ignore the following files: {patterns_for_printing}."
+                    )
+
+                    copytree(
+                        src=temp_extraction_target,
+                        dst=dest,
+                        ignore=ignore_patterns,
+                    )
+
+                    # Copy custom dependencies path
+                    if (
+                        custom_dependencies_path
+                        and Path(custom_dependencies_path).is_file()
+                    ):
+                        shutil.copy(
+                            src=custom_dependencies_path,
+                            dst=Path(dest) / Path(custom_dependencies_path).name,
+                        )
+
+                return git_commit_id
+        except Exception as e:
+            raise QwakSuggestionException(
+                message="Unable to unzip zipped file",
+                src_exception=e,
+                suggestion=f"Please make sure that {src} has read permissions",
+            )

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/post_fetch_validation_step.py

@@ -0,0 +1,117 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from qwak_sdk.commands.models.build._logic.constant.temp_dir import TEMP_LOCAL_MODEL_DIR
+from qwak_sdk.commands.models.build._logic.context import DependencyManagerType
+from qwak_sdk.commands.models.build._logic.interface.step_inteface import Step
+from qwak_sdk.commands.models.build._logic.util.step_decorator import (
+    build_failure_handler,
+)
+from qwak_sdk.exceptions import QwakSuggestionException
+
+
+@dataclass
+class DependencyFileObject:
+    dep_file_name: List[str]
+    lock_file_name: str = field(default="")
+
+
+DEPS_MANAGER_FILE_MAP = {
+    DependencyManagerType.PIP: DependencyFileObject(dep_file_name=["requirements.txt"]),
+    DependencyManagerType.POETRY: DependencyFileObject(
+        dep_file_name=["pyproject.toml"], lock_file_name="poetry.lock"
+    ),
+    DependencyManagerType.CONDA: DependencyFileObject(
+        dep_file_name=["conda.yml", "conda.yaml"]
+    ),
+}
+
+
+class PostFetchValidationStep(Step):
+    def description(self) -> str:
+        return "Post model fetch validation"
+
+    @build_failure_handler()
+    def execute(self) -> None:
+        self.validate_dependencies()
+        self.configure_base_docker_image()
+
+    def validate_dependencies(self):
+        if (
+            not Path(self.config.build_properties.model_uri.uri).is_dir()
+            or self.config.build_env.python_env.dependency_file_path
+        ):
+            self.notifier.debug("Validating dependency file exists")
+            model_uri, main_dir = (
+                self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR,
+                self.config.build_properties.model_uri.main_dir,
+            )
+            (
+                self.context.dependency_manager_type,
+                self.context.model_relative_dependency_file,
+                self.context.model_relative_dependency_lock_file,
+            ) = find_dependency_files(model_uri, main_dir, self.notifier)
+
+            if (
+                self.context.dependency_manager_type
+                and self.context.model_relative_dependency_file
+            ):
+                return
+
+            self.notifier.error("Dependency file wasn't found, failing...")
+            raise QwakSuggestionException(
+                message="Dependency file isn't found",
+                suggestion="Make sure your model include one of dependencies manager: pip/poetry/conda",
+            )
+
+    def configure_base_docker_image(self):
+        base_image = self.config.build_env.docker.base_image
+        if (not base_image) and self.config.build_env.remote.resources.gpu_type:
+            base_image = "qwakai/qwak:gpu-py39"
+        self.context.base_image = base_image
+
+
+def find_file_location(model_uri, main_dir, filename) -> Path:
+    file_locations: List[Path] = [
+        model_uri / filename,
+        model_uri / main_dir / filename,
+    ]
+    for file in file_locations:
+        if file.is_file():
+            return file
+
+
+def find_dependency_files(model_uri, main_dir, notifier):
+    dependency_manager_type = None
+    model_relative_dependency_file = None
+    model_relative_dependency_lock_file = None
+    for dep_type, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        for filename in dependency_file_object.dep_file_name:
+            dep_file_path = find_file_location(model_uri, main_dir, filename)
+            if dep_file_path:
+                notifier.info(
+                    f"Found dependency type: {dep_type.name} by file: {dep_file_path}"
+                )
+                dependency_manager_type = dep_type
+                model_relative_dependency_file = dep_file_path.relative_to(model_uri)
+                if dependency_file_object.lock_file_name:
+                    dep_lock_file_path = find_file_location(
+                        model_uri,
+                        main_dir,
+                        dependency_file_object.lock_file_name,
+                    )
+                    if dep_lock_file_path:
+                        notifier.info(
+                            f"Found dependency lock file {dep_lock_file_path}"
+                        )
+                        model_relative_dependency_lock_file = (
+                            dep_lock_file_path.relative_to(model_uri)
+                        )
+                break
+
+    return (
+        dependency_manager_type,
+        model_relative_dependency_file,
+        model_relative_dependency_lock_file,
+    )

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/pre_fetch_validation_step.py

@@ -0,0 +1,135 @@
+import uuid
+from pathlib import Path
+
+from qwak.clients.secret_service import SecretServiceClient
+
+from qwak_sdk.commands.models.build._logic.constant.host_resource import (
+    HOST_TEMP_BUILD_DIR,
+)
+from qwak_sdk.commands.models.build._logic.interface.step_inteface import Step
+from qwak_sdk.commands.models.build._logic.phase.a_fetch_model_code.post_fetch_validation_step import (
+    find_dependency_files,
+)
+from qwak_sdk.commands.models.build._logic.util.step_decorator import (
+    build_failure_handler,
+)
+from qwak_sdk.commands.models.build._logic.util.text import snake_case
+from qwak_sdk.exceptions import QwakSuggestionException
+
+
+class PreFetchValidationStep(Step):
+    DEFAULT_CPUS = 2
+    DEFAULT_MEMORY = "4Gi"
+
+    def description(self) -> str:
+        return "Pre model fetch validation"
+
+    def execute(self) -> None:
+        self.validate_build_properties()
+        self.create_build_id()
+        self.collect_model_id()
+        self.create_build_dir()
+        self.collect_git_credentials()
+        self.validate_dependencies()
+        self.validate_cpu_gpu_resources()
+
+    def validate_build_properties(self):
+        if not self.config.build_properties.model_uri.uri:
+            error_message = "Model uri wasn't set"
+            self.notifier.error(f"{error_message}, failing...")
+
+            raise QwakSuggestionException(
+                message=error_message,
+                suggestion="Make sure your build properties object contains model uri argument",
+            )
+
+    @build_failure_handler()
+    def collect_model_id(self):
+        model_id = self.config.build_properties.model_id
+        model = self.context.client_models_management.get_model(
+            model_id=model_id,
+            exception_on_missing=False,
+        )
+        if not model:
+            suggestion = f"Create model {model_id} or check model ID spelling"
+            snake_case_model_id = snake_case(model_id)
+            if self.context.client_models_management.is_model_exists(
+                snake_case_model_id
+            ):
+                suggestion = f"Try using model ID {snake_case_model_id} instead"
+            raise QwakSuggestionException(
+                message=f"Model ID {model_id} isn't found",
+                suggestion=suggestion,
+            )
+        self.context.project_uuid = model.project_id
+        self.context.model_uuid = model.uuid
+        self.context.model_id = model_id
+
+    def create_build_id(self):
+        self.context.build_id = str(uuid.uuid4())
+        self.notifier.info(f"Generated build ID - {self.context.build_id}")
+
+    def create_build_dir(self):
+        build_dir = HOST_TEMP_BUILD_DIR / self.context.model_id / self.context.build_id
+        build_dir.mkdir(exist_ok=True, parents=True)
+        self.context.host_temp_local_build_dir = build_dir
+        self.notifier.debug(f"Build directory created - {build_dir}")
+
+    def collect_git_credentials(self):
+        if self.config.build_properties.model_uri.git_credentials:
+            self.context.git_credentials = (
+                self.config.build_properties.model_uri.git_credentials
+            )
+        elif self.config.build_properties.model_uri.git_credentials_secret:
+            self.context.git_credentials = SecretServiceClient().get_secret(
+                self.config.build_properties.model_uri.git_credentials_secret
+            )
+        else:
+            self.notifier.debug("Git credentials isn't configured")
+
+    def validate_dependencies(self):
+        if (
+            Path(self.config.build_properties.model_uri.uri).is_dir()
+            and not self.config.build_env.python_env.dependency_file_path
+        ):
+            model_uri, main_dir = (
+                Path(self.config.build_properties.model_uri.uri),
+                self.config.build_properties.model_uri.main_dir,
+            )
+            (
+                self.context.dependency_manager_type,
+                self.context.model_relative_dependency_file,
+                self.context.model_relative_dependency_lock_file,
+            ) = find_dependency_files(model_uri, main_dir, self.notifier)
+
+            if (
+                self.context.dependency_manager_type
+                and self.context.model_relative_dependency_file
+            ):
+                return
+
+            self.notifier.error("Dependency file wasn't found, failing...")
+            raise QwakSuggestionException(
+                message="Dependency file isn't found",
+                suggestion="Make sure your model include one of dependencies manager: pip/poetry/conda",
+            )
+
+    def validate_cpu_gpu_resources(self):
+        gpu_configured = (
+            self.config.build_env.remote.resources.gpu_type
+            or self.config.build_env.remote.resources.gpu_amount
+        )
+        cpu_configured = (
+            self.config.build_env.remote.resources.cpus
+            or self.config.build_env.remote.resources.memory
+        )
+
+        if cpu_configured and gpu_configured:
+            raise QwakSuggestionException(
+                message="CPU and GPU configured together, Invalid configuration.",
+                suggestion="Please configure only CPU or GPU",
+            )
+
+        if not (cpu_configured or gpu_configured):
+            self.config.build_env.remote.resources.cpus = self.DEFAULT_CPUS
+            self.config.build_env.remote.resources.memory = self.DEFAULT_MEMORY

qwak_sdk/commands/models/build/_logic/phase/b_remote_register_qwak_build/__init__.py

@@ -0,0 +1,11 @@
+from .cleanup_step import CleanupStep
+from .start_remote_build_step import StartRemoteBuildStep
+from .upload_step import UploadStep
+
+
+def get_remote_register_qwak_build_steps():
+    return [
+        UploadStep(),
+        StartRemoteBuildStep(),
+        CleanupStep(),
+    ]

qwak_sdk/commands/models/build/_logic/phase/b_remote_register_qwak_build/cleanup_step.py

@@ -0,0 +1,20 @@
+import os
+
+from qwak_sdk.commands.models.build._logic.interface.step_inteface import Step
+from qwak_sdk.commands.models.build._logic.util.step_decorator import (
+    cleaning_up_after_build,
+)
+
+
+class CleanupStep(Step):
+    def description(self) -> str:
+        return "Clean build artifacts"
+
+    def execute(self) -> None:
+        self.notifier.debug(
+            "Cleanup environment - Deleting file and intermediate images"
+        )
+        if os.getenv("POD_NAME"):
+            self.notifier.debug("Skipping cleanup - Remote executor is temporary")
+        else:
+            cleaning_up_after_build(self)

qwak_sdk/commands/models/build/_logic/phase/b_remote_register_qwak_build/start_remote_build_step.py

@@ -0,0 +1,42 @@
+from copy import deepcopy
+
+from qwak_sdk import __version__ as qwak_sdk_version
+from qwak_sdk.commands.models.build._logic.constant.upload_tag import (
+    BUILD_CONFIG_TAG,
+    MODEL_CODE_TAG,
+    QWAK_SDK_VERSION_TAG,
+)
+from qwak_sdk.commands.models.build._logic.interface.step_inteface import Step
+from qwak_sdk.commands.models.build._logic.util.step_decorator import (
+    build_failure_handler,
+)
+
+
+class StartRemoteBuildStep(Step):
+    def description(self) -> str:
+        return "Start remote build"
+
+    @build_failure_handler()
+    def execute(self) -> None:
+        self.notifier.info(f"Start remote build - {self.context.build_id}")
+        config_copy = deepcopy(self.config)
+        config_copy.build_properties.build_id = self.context.build_id
+        self.context.client_builds_orchestrator.build_model(
+            build_conf=config_copy,
+            build_v1_flag=False,
+            build_config_url=self.get_download_url(BUILD_CONFIG_TAG),
+            qwak_sdk_version_url=self.get_download_url(QWAK_SDK_VERSION_TAG),
+            resolved_model_url=self.get_download_url(MODEL_CODE_TAG),
+            git_commit_id=self.context.git_commit_id,
+            sdk_version=qwak_sdk_version,
+        )
+        self.notifier.info("Remote build started successfully")
+
+    def get_download_url(self, tag: str) -> str:
+        return (
+            self.context.client_builds_orchestrator.get_build_versioning_download_url(
+                model_id=self.config.build_properties.model_id,
+                build_id=self.context.build_id,
+                tag=tag,
+            ).download_url
+        )