qwak-core 0.1.37__py3-none-any.whl → 0.5.19__py3-none-any.whl

qwak/inner/build_logic/execute_build_pipeline.py

@@ -0,0 +1,62 @@
+from qwak.exceptions import QwakGeneralBuildException
+from qwak.exceptions import QwakRemoteBuildFailedException
+from qwak.inner.build_logic.interface.phase_run_handler import PhaseRunHandler
+from qwak.inner.build_logic.interface.time_source import (
+    SystemClockTimeSource,
+    TimeSource,
+    Stopwatch,
+)
+from qwak.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+
+
+def execute_build_pipeline(
+    pipeline: PhasesPipeline,
+    phase_run_handler: PhaseRunHandler,
+):
+    time_source = SystemClockTimeSource()
+    for phase in pipeline.phases:
+        with phase_run_handler.handle_current_phase(phase):
+            phase_run(phase, phase_run_handler, time_source)
+
+
+def phase_run(
+    phase: PhasesPipeline,
+    phase_run_handler: PhaseRunHandler,
+    time_source: TimeSource,
+):
+    build_id = phase.context.build_id
+    current_step_phase = None
+    stop_watch = Stopwatch(time_source)
+    try:
+        for step in phase.steps:
+            current_step_phase = step.build_phase
+            step.set_logger(phase_run_handler.build_logger)
+            phase_run_handler.handle_phase_in_progress(current_step_phase)
+
+            step.execute()
+
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_phase_finished_successfully(
+            current_step_phase, phase_duration
+        )
+    except QwakGeneralBuildException as e:
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_contact_support_error(
+            build_id, current_step_phase, e, phase_duration
+        )
+    except QwakRemoteBuildFailedException as e:
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_remote_build_error(
+            build_id, current_step_phase, e, phase_duration
+        )
+    except KeyboardInterrupt:
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_keyboard_interrupt(
+            build_id, current_step_phase, phase_duration
+        )
+    except BaseException as e:
+        phase_run_handler.build_logger.exception("Failed", e)
+        phase_duration = stop_watch.elapsed_time_in_seconds()
+        phase_run_handler.handle_pipeline_exception(
+            build_id, current_step_phase, e, phase_duration
+        )

qwak/inner/build_logic/interface/build_logger_interface.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class BuildLogger(ABC):
+    verbose: int
+    json_logs: bool
+
+    @abstractmethod
+    def error(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def exception(self, line: str, e: BaseException) -> None:
+        pass
+
+    @abstractmethod
+    def warning(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def info(self, line: str) -> None:
+        pass
+
+    @abstractmethod
+    def debug(self, line: str) -> None:
+        pass

qwak/inner/build_logic/interface/build_phase.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+
+class BuildPhase:
+    def __init__(
+        self,
+        phase_id: str,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+    ):
+        self.phase_id = phase_id
+        self.name = name
+        self.description = description
+        if not self.name:
+            self.name = phase_id.replace("_", " ").capitalize()
+        if not self.description:
+            self.description = phase_id.replace("_", " ").capitalize()
+
+    def __eq__(self, other):
+        return (
+            self.phase_id == other.phase_id
+            and self.name == other.name
+            and self.description == other.description
+        )

qwak/inner/build_logic/interface/context_interface.py

@@ -0,0 +1,66 @@
+from abc import ABCMeta
+from dataclasses import field, dataclass
+from pathlib import Path
+from typing import Optional
+
+from qwak.clients.administration.eco_system.client import EcosystemClient
+from qwak.clients.build_orchestrator import BuildOrchestratorClient
+from qwak.clients.instance_template.client import InstanceTemplateManagementClient
+from qwak.clients.model_management import ModelsManagementClient
+from qwak.inner.build_logic.dependency_manager_type import DependencyManagerType
+from qwak.inner.di_configuration.account import UserAccount, UserAccountConfiguration
+
+
+@dataclass
+class Context(metaclass=ABCMeta):
+    # Clients
+    client_builds_orchestrator: BuildOrchestratorClient = field(
+        default_factory=BuildOrchestratorClient
+    )
+    client_models_management: ModelsManagementClient = field(
+        default_factory=ModelsManagementClient
+    )
+    client_instance_template: InstanceTemplateManagementClient = field(
+        default_factory=InstanceTemplateManagementClient
+    )
+    client_ecosystem: EcosystemClient = field(default_factory=EcosystemClient)
+
+    # General
+    user_account: UserAccount = field(
+        default_factory=UserAccountConfiguration().get_user_config
+    )
+
+    # Pre fetch validation
+    build_id: str = field(default="")
+    model_id: str = field(default="")
+    build_name: str = field(default="")
+    project_uuid: str = field(default="")
+    host_temp_local_build_dir: Path = field(default=None)
+    model_uri: Path = field(default=None)
+    git_credentials: str = field(default="")
+    git_ssh_key: str = field(default="")
+
+    # Fetch model
+    git_commit_id: Optional[str] = field(default=None)
+
+    # Post fetch validation
+    dependency_manager_type: DependencyManagerType = field(
+        default=DependencyManagerType.UNKNOWN
+    )
+    model_relative_dependency_file: Path = field(default=None)
+    model_relative_dependency_lock_file: Path = field(default=None)
+
+    # Upload model
+    model_code_remote_url: Optional[str] = field(default=None)
+    qwak_sdk_version: Optional[str] = field(default=None)
+
+    # Image
+    base_image: Optional[str] = field(default=None)
+
+    # Upload Custom Wheels
+    custom_runtime_wheel: Optional[Path] = field(default=None)
+    custom_core_wheel: Optional[Path] = field(default=None)
+
+    platform_url: Optional[str] = field(
+        default_factory=UserAccountConfiguration().retrieve_platform_url
+    )

qwak/inner/build_logic/interface/phase_run_handler.py

@@ -0,0 +1,71 @@
+from abc import abstractmethod, ABC
+from typing import Optional
+
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+from qwak.inner.build_logic.interface.build_phase import BuildPhase
+from qwak.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+
+
+class PhaseRunHandler(ABC):
+    current_phase: PhasesPipeline
+    build_logger: Optional[BuildLogger]
+
+    @abstractmethod
+    def handle_phase_in_progress(self, build_phase: BuildPhase):
+        pass
+
+    @abstractmethod
+    def handle_phase_finished_successfully(
+        self, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        pass
+
+    @abstractmethod
+    def handle_contact_support_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_remote_build_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_keyboard_interrupt(
+        self, build_id: str, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        pass
+
+    @abstractmethod
+    def handle_pipeline_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_pipeline_quiet_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        pass
+
+    @abstractmethod
+    def handle_current_phase(self, phase: PhasesPipeline):
+        pass

qwak/inner/build_logic/interface/step_inteface.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from qwak.inner.build_config.build_config_v1 import BuildConfigV1
+from qwak.inner.build_logic.interface.build_phase import BuildPhase
+from qwak.inner.build_logic.interface.context_interface import Context
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+
+class Step(ABC):
+    context: Context
+    config: BuildConfigV1
+    build_logger: BuildLogger
+    build_phase: BuildPhase
+
+    @abstractmethod
+    def description(self) -> str:
+        pass
+
+    @abstractmethod
+    def execute(self) -> None:
+        pass
+
+    def set_logger(self, build_logger: BuildLogger) -> None:
+        self.build_logger = build_logger

qwak/inner/build_logic/interface/time_source.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from time import time
+
+
+class TimeSource(ABC):
+    @abstractmethod
+    def current_epoch_time(self) -> int:
+        pass
+
+
+class Stopwatch:
+    def __init__(self, time_source: TimeSource):
+        self.time_source = time_source
+        self.start_time = self.time_source.current_epoch_time()
+
+    def elapsed_time_in_seconds(self) -> int:
+        end_time = self.time_source.current_epoch_time()
+        return end_time - self.start_time
+
+
+class SystemClockTimeSource(TimeSource):
+    def current_epoch_time(self) -> int:
+        return int(time())

qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_model_step.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from qwak.inner.build_logic.constants.temp_dir import TEMP_LOCAL_MODEL_DIR
+
+from qwak.inner.build_logic.interface.step_inteface import Step
+from .fetch_strategy_manager.fetch_strategy_manager import (
+    FetchStrategyManager,
+)
+
+
+class FetchModelStep(Step):
+    STEP_DESCRIPTION = "Fetch model code"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        fetch_strategy_manager = FetchStrategyManager(
+            uri=self.config.build_properties.model_uri.uri,
+            build_logger=self.build_logger,
+        )
+        self.build_logger.debug("Fetching model code")
+        git_commit_id = fetch_strategy_manager.fetch(
+            dest=self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR,
+            git_branch=self.config.build_properties.model_uri.git_branch,
+            git_credentials=self.context.git_credentials,
+            model_id=self.config.build_properties.model_id,
+            build_id=self.context.build_id,
+            custom_dependencies_path=self.config.build_env.python_env.dependency_file_path,
+            main_dir=self.config.build_properties.model_uri.main_dir,
+            dependency_path=self.context.model_relative_dependency_file,
+            lock_dependency_path=self.context.model_relative_dependency_lock_file,
+            dependency_required_folders=self.config.build_properties.model_uri.dependency_required_folders,
+            git_ssh_key=self.context.git_ssh_key,
+        )
+        if git_commit_id:
+            self.context.git_commit_id = git_commit_id
+            self.build_logger.debug(f"Git commit ID identified - {git_commit_id}")
+
+        self.build_logger.debug(
+            f"Model code stored in {self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR}"
+        )
+        self.build_logger.info("Successfully fetched model code")

qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/common.py

@@ -0,0 +1,32 @@
+import re
+from pathlib import Path
+from typing import Union
+
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+_GIT_URI_REGEX = re.compile(r"^[^/]*:")
+_FILE_URI_REGEX = re.compile(r"^file://.+")
+_ZIP_URI_REGEX = re.compile(r".+\.zip")
+
+
+def is_zip_uri(uri: Union[str, Path]) -> bool:
+    return uri and _ZIP_URI_REGEX.match(str(uri)) is not None
+
+
+def is_local_dir(uri: Union[str, Path]) -> bool:
+    return uri and not _GIT_URI_REGEX.match(str(uri)) and Path(uri).is_dir()
+
+
+def is_git_uri(uri: Union[str, Path]) -> bool:
+    return uri and _GIT_URI_REGEX.match(str(uri)) is not None
+
+
+def get_git_commit_id(path: Union[str, Path], build_logger: BuildLogger) -> str:
+    try:
+        import git
+
+        repo = git.Repo(path=path, search_parent_directories=True)
+        return repo.head.object.hexsha
+    except Exception as e:
+        build_logger.warning(f"Failed to get git commit with error: {e}")
+        return ""

qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/fetch_strategy_manager.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import List, Optional
+
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+from qwak.exceptions import QwakSuggestionException
+from .common import is_git_uri, is_local_dir, is_zip_uri
+from .strategy.folder.folder_strategy import FolderStrategy
+from .strategy.git.git_strategy import GitStrategy
+from .strategy.strategy import Strategy
+from .strategy.zip.zip_strategy import ZipStrategy
+
+URI_CHECK_STRATEGY_MAPPING = {
+    is_local_dir: FolderStrategy,
+    is_git_uri: GitStrategy,
+    is_zip_uri: ZipStrategy,
+}
+
+
+class FetchStrategyManager:
+    def __init__(self, uri: str, build_logger: BuildLogger) -> None:
+        self._strategy: Optional[Strategy] = None
+        self._uri = uri
+        for check, clazz in URI_CHECK_STRATEGY_MAPPING.items():
+            if check(uri):
+                self._strategy = clazz(build_logger)
+
+        if self._strategy is None:
+            raise QwakSuggestionException(
+                message=f"Model URI {uri} is no valid.",
+                suggestion=f"Please make sure that model path: {uri} is a zip file/git uri/local folder.",
+            )
+
+    def fetch(
+        self,
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        model_id: str,
+        build_id: str,
+        custom_dependencies_path: str,
+        main_dir: str,
+        dependency_path: str,
+        lock_dependency_path: str,
+        dependency_required_folders: List[str],
+        git_ssh_key: str,
+    ) -> str:
+        return self._strategy.fetch(
+            src=self._uri,
+            dest=dest,
+            git_branch=git_branch,
+            git_credentials=git_credentials,
+            model_id=model_id,
+            build_id=build_id,
+            custom_dependencies_path=custom_dependencies_path,
+            main_dir=main_dir,
+            dependency_path=dependency_path,
+            lock_dependency_path=lock_dependency_path,
+            dependency_required_folders=dependency_required_folders,
+            git_ssh_key=git_ssh_key,
+        )

qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/folder/folder_strategy.py

@@ -0,0 +1,131 @@
+import shutil
+import re
+from pathlib import Path
+from typing import List, Optional
+
+from qwak.exceptions import QwakSuggestionException
+from qwak.inner.build_logic.tools.files import (
+    copytree,
+    get_possible_dependency_lock_paths,
+)
+
+from ...common import get_git_commit_id
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class FolderStrategy(Strategy):
+    def fetch(
+        self,
+        src: str,
+        dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+        dependency_path: str,
+        lock_dependency_path: str,
+        dependency_required_folders: List[str],
+        **kwargs,
+    ) -> Optional[str]:
+        try:
+            self.build_logger.debug(
+                f"Fetching Model code from local directory -  {src}"
+            )
+
+            ignore_patterns, patterns_for_printing = get_ignore_pattern(
+                src, main_dir, self.build_logger
+            )
+            self.build_logger.debug(
+                f"Will ignore the following files: {patterns_for_printing}."
+            )
+
+            copytree(
+                src=Path(src) / main_dir,
+                dst=Path(dest) / main_dir,
+                ignore=ignore_patterns,
+            )
+
+            self._copy_dependency_required_folders(
+                dependency_required_folders, src, dest, ignore_patterns
+            )
+
+            if (Path(src) / "tests").exists():
+                copytree(
+                    src=Path(src) / "tests",
+                    dst=Path(dest) / "tests",
+                    ignore=ignore_patterns,
+                )
+            if dependency_path:
+                shutil.copy(
+                    src=Path(src) / dependency_path, dst=Path(dest) / dependency_path
+                )
+            if lock_dependency_path:
+                if (Path(src) / lock_dependency_path).is_file():
+                    shutil.copy(
+                        src=Path(src) / lock_dependency_path,
+                        dst=Path(dest) / lock_dependency_path,
+                    )
+                else:
+                    self.build_logger.warning(
+                        "No lock dependency file found in model directory."
+                    )
+
+            self._copy_custom_dependencies(custom_dependencies_path, dest)
+            # Get git commit id if exists
+            return get_git_commit_id(src, self.build_logger)
+        except Exception as e:
+            if isinstance(e, QwakSuggestionException):
+                raise e  # Propagate suggestions without changes
+            main_dir_path = Path(src) / main_dir
+            if not main_dir_path.exists():
+                message = f"""Seems like you're running the 'qwak models build' command from the wrong location.
+Please make sure you run the command at the parent directory of your '{main_dir}' folder"""
+            else:
+                message = (f"Please make sure that {src} has read permissions.",)
+
+            raise QwakSuggestionException(
+                message="Unable to copy model",
+                src_exception=e,
+                suggestion=message,
+            )
+
+    def _copy_dependency_required_folders(
+        self, dependency_required_folders, src, dest, ignore_patterns
+    ):
+        for folder in dependency_required_folders:
+            destination_folder = folder
+            while destination_folder.startswith(".."):
+                destination_folder = re.sub(r"^\.\./", "", destination_folder)
+
+            if (Path(dest) / destination_folder).exists():
+                raise QwakSuggestionException(
+                    message="Unable to copy model",
+                    suggestion=f"It's not possible to copy directory `{folder}` into `{destination_folder}` because the target `{destination_folder}` already exist. Mixing files from multiple sources is not allowed.",
+                )
+            if (Path(src) / folder).exists():
+                copytree(
+                    src=Path(src) / folder,
+                    dst=Path(dest) / destination_folder,
+                    ignore=ignore_patterns,
+                )
+            else:
+                self.build_logger.warning(
+                    'Folder "{}" does not exist. Skipping it.'.format(folder)
+                )
+
+    def _copy_custom_dependencies(self, custom_dependencies_path, dest):
+        if custom_dependencies_path and Path(custom_dependencies_path).is_file():
+            shutil.copy(
+                src=custom_dependencies_path,
+                dst=Path(dest) / Path(custom_dependencies_path).name,
+            )
+            possible_dependency_lock_paths = get_possible_dependency_lock_paths(
+                Path(custom_dependencies_path)
+            )
+            for path in possible_dependency_lock_paths:
+                if path.is_file():
+                    self.build_logger.info(
+                        "Found dependency lock file: {}".format(path)
+                    )
+                    shutil.copy(
+                        src=path,
+                        dst=Path(dest) / path.name,
+                    )