qwak-sdk 0.1.0__py3-none-any.whl → 0.2.21rc0__py3-none-any.whl

qwak_sdk/commands/models/build/_logic/config/config_v1.py

@@ -0,0 +1,253 @@
+from dataclasses import dataclass, field
+from os.path import expandvars
+from typing import Dict, List, Optional, Tuple, Union
+
+from _qwak_proto.qwak.builds.builds_pb2 import (
+    BuildConfiguration,
+    BuildEnvironment,
+    BuildModelUri,
+    BuildPropertiesProto,
+    CondaBuild,
+    DockerBuild,
+    LocalBuild,
+    PoetryBuild,
+    PythonBuild,
+    RemoteBuild,
+    RemoteBuildResources,
+    Step,
+    VirtualEnvironmentBuild,
+)
+from qwak.inner.tool.run_config import (
+    QwakConfigBase,
+    YamlConfigMixin,
+    validate_bool,
+    validate_float,
+    validate_int,
+    validate_list_of_strings,
+    validate_string,
+)
+
+from qwak_sdk.commands.models.build._logic.util.protobuf_factory import protobuf_factory
+from qwak_sdk.common.run_config import ConfigCliMap
+
+CONFIG_MAPPING: List[ConfigCliMap] = [
+    # ConfigV1
+    ConfigCliMap(
+        "validate_build_artifact", "step.validate_build_artifact", validate_bool, True
+    ),
+    ConfigCliMap(
+        "validate_build_artifact_timeout",
+        "step.validate_build_artifact_timeout",
+        validate_int,
+        True,
+    ),
+    ConfigCliMap("tests", "step.tests", validate_bool, True),
+    # BuildProperties
+    ConfigCliMap("model_id", "build_properties.model_id", validate_string, True),
+    ConfigCliMap("build_id", "build_properties.build_id", validate_string, False),
+    ConfigCliMap(
+        "gpu_compatible", "build_properties.gpu_compatible", validate_bool, False
+    ),
+    ConfigCliMap("branch", "build_properties.branch", validate_string),
+    ConfigCliMap("tags", "build_properties.tags", validate_list_of_strings),
+    # ModelUri
+    ConfigCliMap("uri", "build_properties.model_uri.uri", validate_string, False),
+    ConfigCliMap(
+        "git_branch", "build_properties.model_uri.git_branch", validate_string
+    ),
+    ConfigCliMap("main_dir", "build_properties.model_uri.main_dir", validate_string),
+    ConfigCliMap(
+        "git_credentials", "build_properties.model_uri.git_credentials", validate_string
+    ),
+    ConfigCliMap(
+        "git_credentials_secret",
+        "build_properties.model_uri.git_credentials_secret",
+        validate_string,
+    ),
+    ConfigCliMap(
+        "dependency_required_folders",
+        "build_properties.model_uri.dependency_required_folders",
+        validate_list_of_strings,
+    ),
+    # BuildEnv
+    # PythonEnv
+    ConfigCliMap(
+        "dependency_file_path",
+        "build_env.python_env.dependency_file_path",
+        validate_string,
+    ),
+    # DockerConf
+    ConfigCliMap("base_image", "build_env.docker.base_image", validate_string),
+    ConfigCliMap("param_list", "build_env.docker.params", validate_list_of_strings),
+    ConfigCliMap("env_vars", "build_env.docker.env_vars", validate_list_of_strings),
+    ConfigCliMap("cache", "build_env.docker.cache", validate_bool),
+    ConfigCliMap(
+        "iam_role_arn", "build_env.docker.assumed_iam_role_arn", validate_string
+    ),
+    # LocalConf
+    ConfigCliMap("push", "build_env.docker.push", validate_bool),
+    ConfigCliMap("aws_profile", "build_env.local.aws_profile", validate_string),
+    # RemoteConf
+    ConfigCliMap("remote", "build_env.remote.is_remote", validate_bool),
+    ConfigCliMap("cpus", "build_env.remote.resources.cpus", validate_float),
+    ConfigCliMap("memory", "build_env.remote.resources.memory", validate_string),
+    ConfigCliMap("gpu_type", "build_env.remote.resources.gpu_type", validate_string),
+    ConfigCliMap("gpu_amount", "build_env.remote.resources.gpu_amount", validate_int),
+    # Verbosity level
+    ConfigCliMap("verbose", "verbose", validate_int),
+    ConfigCliMap("deploy", "deploy", validate_bool),
+]
+
+
+@protobuf_factory(BuildConfiguration, exclude_fields=["deploy"])
+@dataclass
+class ConfigV1(YamlConfigMixin, QwakConfigBase):
+    @property
+    def _config_mapping(self) -> List[ConfigCliMap]:
+        return CONFIG_MAPPING
+
+    @protobuf_factory(Step)
+    @dataclass
+    class Step:
+        validate_build_artifact: bool = field(default=True)
+        validate_build_artifact_timeout: int = field(default=120)
+        tests: bool = field(default=True)
+
+    @protobuf_factory(BuildPropertiesProto, exclude_fields=["gpu_compatible"])
+    @dataclass
+    class BuildProperties:
+        @protobuf_factory(
+            BuildModelUri,
+            exclude_fields=[
+                "git_credentials",
+                "git_credentials_secret",
+                "dependency_required_folders",
+            ],
+        )
+        @dataclass
+        class ModelUri:
+            uri: Optional[str] = field(default=None)
+            git_branch: Optional[str] = field(default="master")
+            main_dir: Optional[str] = field(default="main")
+            dependency_required_folders: List[str] = field(default_factory=list)
+            git_credentials: Optional[str] = field(default=None)
+            git_credentials_secret: Optional[str] = field(default=None)
+
+        build_id: Optional[str] = field(default=None)
+        gpu_compatible: bool = field(default=False)
+        branch: Optional[str] = field(default="main")
+        model_id: str = field(default="")
+        tags: List[str] = field(default_factory=list)
+        model_uri: ModelUri = field(default_factory=ModelUri)
+
+    @protobuf_factory(BuildEnvironment)
+    @dataclass
+    class BuildEnv:
+        @protobuf_factory(DockerBuild)
+        @dataclass
+        class DockerConf:
+            base_image: Optional[str] = field(
+                default="public.ecr.aws/w8k8y6b6/qwak-base:0.0.18-cpu"
+            )
+            env_vars: Optional[Union[List[str], Dict[str, str], Tuple]] = field(
+                default_factory=list
+            )
+            assumed_iam_role_arn: str = field(default=None)
+            params: Optional[List[str]] = field(default_factory=list)
+            no_cache: bool = field(default=False)
+            cache: bool = field(default=True)
+            push: bool = field(default=True)
+
+        @protobuf_factory(
+            PythonBuild, exclude_fields=["git_credentials_secret", "git_credentials"]
+        )
+        @dataclass
+        class PythonEnv:
+            @protobuf_factory(PoetryBuild)
+            @dataclass
+            class PoetryType:
+                python_version: Optional[str] = field(default=None)
+                lock_file: Optional[str] = field(default=None)
+
+            @protobuf_factory(CondaBuild)
+            @dataclass
+            class CondaType:
+                conda_file: Optional[str] = field(default=None)
+
+            @protobuf_factory(VirtualEnvironmentBuild)
+            @dataclass
+            class VirtualenvType:
+                python_version: Optional[str] = field(default=None)
+                requirements_txt: Optional[str] = field(default=None)
+
+            git_credentials: Optional[str] = field(default=None)
+            git_credentials_secret: Optional[str] = field(default=None)
+            poetry: PoetryType = field(default=None)
+            conda: CondaType = field(default=None)
+            virtualenv: VirtualenvType = field(default=None)
+            qwak_sdk_extra_index_url: str = field(default=None)
+            dependency_file_path: Optional[str] = field(default=None)
+
+            def __post_init__(self):
+                if not (self.poetry or self.conda or self.virtualenv):
+                    self.conda = self.CondaType("conda.yml")
+
+        @protobuf_factory(LocalBuild)
+        @dataclass
+        class LocalConf:
+            no_push: bool = field(default=False)  # Deprecated
+            aws_profile: str = field(default=None)
+
+        @protobuf_factory(RemoteBuild)
+        @dataclass
+        class RemoteConf:
+            @protobuf_factory(RemoteBuildResources)
+            @dataclass
+            class RemoteBuildResources:
+                cpus: Optional[float] = field(default=None)
+                memory: Optional[str] = field(default=None)
+                gpu_type: Optional[str] = field(default=None)
+                gpu_amount: Optional[int] = field(default=None)
+
+            is_remote: bool = field(default=True)
+            resources: RemoteBuildResources = field(
+                default_factory=RemoteBuildResources
+            )
+
+        docker: DockerConf = field(default_factory=DockerConf)
+        python_env: PythonEnv = field(default_factory=PythonEnv)
+        local: LocalConf = field(default_factory=LocalConf)
+        remote: RemoteConf = field(default_factory=RemoteConf)
+
+    build_properties: BuildProperties = field(default_factory=BuildProperties)
+    build_env: Optional[BuildEnv] = field(default_factory=BuildEnv)
+    pre_build: Optional[BuildEnv] = field(default=None)
+    post_build: Optional[BuildEnv] = field(default=None)
+    step: Step = field(default_factory=Step)
+    deploy: bool = field(default=False)
+    verbose: int = field(default=0)
+
+    def __post_init__(self):
+        if (
+            self.build_env.remote.resources.gpu_type
+            and self.build_env.docker.base_image
+            == ConfigV1.BuildEnv.DockerConf.base_image
+            or self.build_properties.gpu_compatible
+        ):
+            self.build_env.docker.base_image = (
+                "public.ecr.aws/w8k8y6b6/qwak-base:0.0.6-gpu"
+            )
+        if isinstance(self.build_env.docker.env_vars, (list, tuple)):
+            self.build_env.docker.env_vars = [
+                expandvars(var) for var in self.build_env.docker.env_vars
+            ]
+
+    def _post_merge_cli(self):
+        """Implementing post merge actions."""
+        self.build_properties.model_id = self.build_properties.model_id.lower()
+        self.build_properties.tags = list(self.build_properties.tags)
+        self.build_properties.model_uri.dependency_required_folders = list(
+            self.build_properties.model_uri.dependency_required_folders
+        )
+        self.build_env.docker.env_vars = list(self.build_env.docker.env_vars)
+        self.build_env.docker.params = list(self.build_env.docker.params)

qwak_sdk/commands/models/build/_logic/constant/host_resource.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+HOST_QWAK_HIDDEN_FOLDER = Path.home() / ".qwak"
+HOST_TEMP_BUILD_DIR = HOST_QWAK_HIDDEN_FOLDER / "builds"

qwak_sdk/commands/models/build/_logic/constant/step_description.py

@@ -0,0 +1,29 @@
+from enum import Enum
+
+FETCHING_MODEL_CODE_PHASE_DESCRIPTION = "Fetching Model Code"
+REGISTER_QWAK_BUILD_PHASE_DESCRIPTION = "Registering Qwak Build"
+DEPLOY_PHASE_DESCRIPTION = "Deploying"
+
+
+class BuildPhase(Enum):
+    FETCHING_MODEL_CODE = 1
+    REGISTERING_QWAK_BUILD = 2
+    DEPLOY_PHASE = 3
+
+
+class PhaseDetails:
+    def __init__(self, build_phase: BuildPhase):
+        self.phase = build_phase
+
+    def get_id(self):
+        return str(self.phase.name)
+
+    def get_description(self):
+        if self.phase == BuildPhase.FETCHING_MODEL_CODE:
+            return FETCHING_MODEL_CODE_PHASE_DESCRIPTION
+        elif self.phase == BuildPhase.REGISTERING_QWAK_BUILD:
+            return REGISTER_QWAK_BUILD_PHASE_DESCRIPTION
+        elif self.phase == BuildPhase.DEPLOY_PHASE:
+            return DEPLOY_PHASE_DESCRIPTION
+        else:
+            raise ValueError("Invalid PhaseId")

qwak_sdk/commands/models/build/_logic/constant/temp_dir.py

@@ -0,0 +1,2 @@
+TEMP_LOCAL_MODEL_DIR = "model_code"
+TEMP_LOCAL_EDITABLE_SDK = "editable"

qwak_sdk/commands/models/build/_logic/constant/upload_tag.py

@@ -0,0 +1,5 @@
+MODEL_CODE_TAG = "code.zip"
+SKINNY_MODEL_CODE_TAG = "skinny_code.zip"
+QWAK_SDK_VERSION_TAG = "qwak_sdk_version"
+QWAK_SDK_WHEEL_TAG = "qwak_sdk_wheel"
+BUILD_CONFIG_TAG = "qwak_build_config"

qwak_sdk/commands/models/build/_logic/context.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+from qwak.clients.build_management import BuildsManagementClient
+from qwak.clients.build_orchestrator import BuildOrchestratorClient
+from qwak.clients.model_management import ModelsManagementClient
+from qwak.inner.di_configuration import UserAccountConfiguration
+from qwak.inner.di_configuration.account import UserAccount
+
+
+class DependencyManagerType(Enum):
+    UNKNOWN = 0
+    PIP = 1
+    POETRY = 2
+    CONDA = 3
+
+
+@dataclass
+class Context:
+    # Clients
+    client_builds_orchestrator: BuildOrchestratorClient = field(
+        default_factory=BuildOrchestratorClient
+    )
+    client_build_management: BuildsManagementClient = field(
+        default_factory=BuildsManagementClient
+    )
+    client_models_management: ModelsManagementClient = field(
+        default_factory=ModelsManagementClient
+    )
+
+    # General
+    user_account: UserAccount = field(
+        default_factory=UserAccountConfiguration().get_user_config
+    )
+
+    # Pre fetch validation
+    build_id: str = field(default="")
+    model_id: str = field(default="")
+    project_uuid: str = field(default="")
+    host_temp_local_build_dir: Path = field(default=None)
+    model_uri: Path = field(default=None)
+    git_credentials: str = field(default="")
+
+    # Fetch model
+    git_commit_id: Optional[str] = field(default=None)
+
+    # Post fetch validation
+    dependency_manager_type: DependencyManagerType = field(
+        default=DependencyManagerType.UNKNOWN
+    )
+    model_relative_dependency_file: Path = field(default=None)
+    model_relative_dependency_lock_file: Path = field(default=None)
+
+    # Upload model
+    model_code_remote_url: Optional[str] = field(default=None)
+
+    # Image
+    base_image: Optional[str] = field(default=None)

qwak_sdk/commands/models/build/_logic/interface/notifier_interface.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class Notifier(ABC):
+    @abstractmethod
+    def error(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def exception(self, line: str, e: BaseException) -> None:
+        pass
+
+    @abstractmethod
+    def warning(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def info(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def debug(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def spinner_text(self, line: str) -> None:
+        pass

qwak_sdk/commands/models/build/_logic/interface/step_inteface.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+from qwak_sdk.commands.models.build._logic.config.config_v1 import ConfigV1
+from qwak_sdk.commands.models.build._logic.constant.step_description import BuildPhase
+from qwak_sdk.commands.models.build._logic.context import Context
+
+if TYPE_CHECKING:
+    from qwak_sdk.commands.models.build import Notifier
+
+
+class Step(ABC):
+    context: Context
+    config: ConfigV1
+    notifier: Notifier
+    build_phase: BuildPhase
+
+    @abstractmethod
+    def description(self) -> str:
+        pass
+
+    @abstractmethod
+    def execute(self) -> None:
+        pass
+
+    def set_notifier(self, notifier: Notifier) -> None:
+        self.notifier = notifier

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/__init__.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from ...interface.step_inteface import Step
+from .fetch_model_step.fetch_model_step import FetchModelStep
+from .post_fetch_validation_step import PostFetchValidationStep
+from .pre_fetch_validation_step import PreFetchValidationStep
+
+
+def get_fetch_model_code_steps() -> List[Step]:
+    return [
+        PreFetchValidationStep(),
+        FetchModelStep(),
+        PostFetchValidationStep(),
+    ]

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_model_step.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from qwak_sdk.commands.models.build._logic.constant.temp_dir import TEMP_LOCAL_MODEL_DIR
+from qwak_sdk.commands.models.build._logic.interface.step_inteface import Step
+from qwak_sdk.commands.models.build._logic.phase.a_fetch_model_code.fetch_model_step.fetch_strategy_manager.fetch_strategy_manager import (
+    FetchStrategyManager,
+)
+from qwak_sdk.commands.models.build._logic.util.step_decorator import (
+    build_failure_handler,
+)
+
+
+class FetchModelStep(Step):
+    def description(self) -> str:
+        return "Fetch model code"
+
+    @build_failure_handler()
+    def execute(self) -> None:
+        fetch_strategy_manager = FetchStrategyManager(
+            uri=self.config.build_properties.model_uri.uri,
+            notifier=self.notifier,
+        )
+        self.notifier.debug("Fetching model code")
+        git_commit_id = fetch_strategy_manager.fetch(
+            dest=self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR,
+            git_branch=self.config.build_properties.model_uri.git_branch,
+            git_credentials=self.context.git_credentials,
+            model_id=self.config.build_properties.model_id,
+            build_id=self.context.build_id,
+            custom_dependencies_path=self.config.build_env.python_env.dependency_file_path,
+            main_dir=self.config.build_properties.model_uri.main_dir,
+            dependency_path=self.context.model_relative_dependency_file,
+            dependency_required_folders=self.config.build_properties.model_uri.dependency_required_folders,
+        )
+        if git_commit_id:
+            self.context.git_commit_id = git_commit_id
+            self.notifier.debug(f"Git commit ID identified - {git_commit_id}")
+
+        self.notifier.debug(
+            f"Model code stored in {self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR}"
+        )
+        self.notifier.info("Successfully fetched model code")

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_strategy_manager/common.py

@@ -0,0 +1,33 @@
+import re
+from pathlib import Path
+from typing import Union
+
+_GIT_URI_REGEX = re.compile(r"^[^/]*:")
+_FILE_URI_REGEX = re.compile(r"^file://.+")
+_ZIP_URI_REGEX = re.compile(r".+\.zip")
+
+
+def is_zip_uri(uri: Union[str, Path]) -> bool:
+    return uri and _ZIP_URI_REGEX.match(str(uri)) is not None
+
+
+def is_local_dir(uri: Union[str, Path]) -> bool:
+    return uri and not _GIT_URI_REGEX.match(str(uri)) and Path(uri).is_dir()
+
+
+def is_git_uri(uri: Union[str, Path]) -> bool:
+    return uri and _GIT_URI_REGEX.match(str(uri)) is not None
+
+
+def get_git_commit_id(path: Union[str, Path], notifier) -> str:
+    try:
+        import git
+        from git import InvalidGitRepositoryError
+
+        repo = git.Repo(path=path, search_parent_directories=True)
+        return repo.head.object.hexsha
+    except InvalidGitRepositoryError:
+        return ""
+    except Exception as e:
+        notifier.warning(f"Failed to get git commit with error: {e}")
+        return ""

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_strategy_manager/fetch_strategy_manager.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List, Optional
+
+from qwak_sdk.exceptions import QwakSuggestionException
+
+from .common import is_git_uri, is_local_dir, is_zip_uri
+from .strategy.folder.folder_strategy import FolderStrategy
+from .strategy.git.git_strategy import GitStrategy
+from .strategy.strategy import Strategy
+from .strategy.zip.zip_strategy import ZipStrategy
+
+URI_CHECK_STRATEGY_MAPPING = {
+    is_local_dir: FolderStrategy,
+    is_git_uri: GitStrategy,
+    is_zip_uri: ZipStrategy,
+}
+
+if TYPE_CHECKING:
+    from qwak_sdk.commands.models.build import Notifier
+
+
+class FetchStrategyManager:
+    def __init__(self, uri: str, notifier: Notifier) -> None:
+        self._strategy: Optional[Strategy] = None
+        self._uri = uri
+        for check, clazz in URI_CHECK_STRATEGY_MAPPING.items():
+            if check(uri):
+                self._strategy = clazz(notifier)
+
+        if self._strategy is None:
+            raise QwakSuggestionException(
+                message=f"Model URI {uri} is no valid.",
+                suggestion=f"Please make sure that model path: {uri} is a zip file/git uri/local folder.",
+            )
+
+    def fetch(
+        self,
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        model_id: str,
+        build_id: str,
+        custom_dependencies_path: str,
+        main_dir: str,
+        dependency_path: str,
+        dependency_required_folders: List[str],
+    ) -> str:
+        return self._strategy.fetch(
+            src=self._uri,
+            dest=dest,
+            git_branch=git_branch,
+            git_credentials=git_credentials,
+            model_id=model_id,
+            build_id=build_id,
+            custom_dependencies_path=custom_dependencies_path,
+            main_dir=main_dir,
+            dependency_path=dependency_path,
+            dependency_required_folders=dependency_required_folders,
+        )

qwak_sdk/commands/models/build/_logic/phase/a_fetch_model_code/fetch_model_step/fetch_strategy_manager/strategy/folder/folder_strategy.py

@@ -0,0 +1,73 @@
+import shutil
+from pathlib import Path
+from typing import List, Optional
+
+from qwak_sdk.exceptions import QwakSuggestionException
+from qwak_sdk.tools.files import copytree
+
+from ...common import get_git_commit_id
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class FolderStrategy(Strategy):
+    def fetch(
+        self,
+        src: str,
+        dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+        dependency_path: str,
+        dependency_required_folders: List[str],
+        **kwargs,
+    ) -> Optional[str]:
+        try:
+            self.notifier.debug(f"Fetching Model code from local directory -  {src}")
+
+            ignore_patterns, patterns_for_printing = get_ignore_pattern(
+                src, main_dir, self.notifier
+            )
+            self.notifier.debug(
+                f"Will ignore the following files: {patterns_for_printing}."
+            )
+
+            copytree(
+                src=Path(src) / main_dir,
+                dst=Path(dest) / main_dir,
+                ignore=ignore_patterns,
+            )
+            for folder in dependency_required_folders:
+                if (Path(src) / folder).exists():
+                    copytree(
+                        src=Path(src) / folder,
+                        dst=Path(dest) / folder,
+                        ignore=ignore_patterns,
+                    )
+                else:
+                    self.notifier.warning(
+                        'Folder "{}" does not exist. Skipping it.'.format(folder)
+                    )
+            if (Path(src) / "tests").exists():
+                copytree(
+                    src=Path(src) / "tests",
+                    dst=Path(dest) / "tests",
+                    ignore=ignore_patterns,
+                )
+            if dependency_path:
+                shutil.copy(
+                    src=Path(src) / dependency_path, dst=Path(dest) / dependency_path
+                )
+
+            # Copy custom dependencies path
+            if custom_dependencies_path and Path(custom_dependencies_path).is_file():
+                shutil.copy(
+                    src=custom_dependencies_path,
+                    dst=Path(dest) / Path(custom_dependencies_path).name,
+                )
+            # Get git commit id if exists
+            return get_git_commit_id(src, self.notifier)
+        except Exception as e:
+            raise QwakSuggestionException(
+                message="Unable to copy model.",
+                src_exception=e,
+                suggestion=f"Please make sure that {src} has read permissions.",
+            )