frogml-core 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

frogml_core/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/fetch_strategy_manager.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import List, Optional
+
+from frogml_core.exceptions import FrogmlSuggestionException
+from frogml_core.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+from .common import is_git_uri, is_local_dir, is_zip_uri
+from .strategy.folder.folder_strategy import FolderStrategy
+from .strategy.git.git_strategy import GitStrategy
+from .strategy.strategy import Strategy
+from .strategy.zip.zip_strategy import ZipStrategy
+
+URI_CHECK_STRATEGY_MAPPING = {
+    is_local_dir: FolderStrategy,
+    is_git_uri: GitStrategy,
+    is_zip_uri: ZipStrategy,
+}
+
+
+class FetchStrategyManager:
+    def __init__(self, uri: str, build_logger: BuildLogger) -> None:
+        self._strategy: Optional[Strategy] = None
+        self._uri = uri
+        for check, clazz in URI_CHECK_STRATEGY_MAPPING.items():
+            if check(uri):
+                self._strategy = clazz(build_logger)
+
+        if self._strategy is None:
+            raise FrogmlSuggestionException(
+                message=f"Model URI {uri} is no valid.",
+                suggestion=f"Please make sure that model path: {uri} is a zip file/git uri/local folder.",
+            )
+
+    def fetch(
+        self,
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        model_id: str,
+        build_id: str,
+        custom_dependencies_path: str,
+        main_dir: str,
+        dependency_path: str,
+        lock_dependency_path: str,
+        dependency_required_folders: List[str],
+        git_ssh_key: str,
+    ) -> str:
+        return self._strategy.fetch(
+            src=self._uri,
+            dest=dest,
+            git_branch=git_branch,
+            git_credentials=git_credentials,
+            model_id=model_id,
+            build_id=build_id,
+            custom_dependencies_path=custom_dependencies_path,
+            main_dir=main_dir,
+            dependency_path=dependency_path,
+            lock_dependency_path=lock_dependency_path,
+            dependency_required_folders=dependency_required_folders,
+            git_ssh_key=git_ssh_key,
+        )

frogml_core/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/folder/folder_strategy.py

@@ -0,0 +1,131 @@
+import re
+import shutil
+from pathlib import Path
+from typing import List, Optional
+
+from frogml_core.exceptions import FrogmlSuggestionException
+from frogml_core.inner.build_logic.tools.files import (
+    copytree,
+    get_possible_dependency_lock_paths,
+)
+
+from ...common import get_git_commit_id
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class FolderStrategy(Strategy):
+    def fetch(
+        self,
+        src: str,
+        dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+        dependency_path: str,
+        lock_dependency_path: str,
+        dependency_required_folders: List[str],
+        **kwargs,
+    ) -> Optional[str]:
+        try:
+            self.build_logger.debug(
+                f"Fetching Model code from local directory -  {src}"
+            )
+
+            ignore_patterns, patterns_for_printing = get_ignore_pattern(
+                src, main_dir, self.build_logger
+            )
+            self.build_logger.debug(
+                f"Will ignore the following files: {patterns_for_printing}."
+            )
+
+            copytree(
+                src=Path(src) / main_dir,
+                dst=Path(dest) / main_dir,
+                ignore=ignore_patterns,
+            )
+
+            self._copy_dependency_required_folders(
+                dependency_required_folders, src, dest, ignore_patterns
+            )
+
+            if (Path(src) / "tests").exists():
+                copytree(
+                    src=Path(src) / "tests",
+                    dst=Path(dest) / "tests",
+                    ignore=ignore_patterns,
+                )
+            if dependency_path:
+                shutil.copy(
+                    src=Path(src) / dependency_path, dst=Path(dest) / dependency_path
+                )
+            if lock_dependency_path:
+                if (Path(src) / lock_dependency_path).is_file():
+                    shutil.copy(
+                        src=Path(src) / lock_dependency_path,
+                        dst=Path(dest) / lock_dependency_path,
+                    )
+                else:
+                    self.build_logger.warning(
+                        "No lock dependency file found in model directory."
+                    )
+
+            self._copy_custom_dependencies(custom_dependencies_path, dest)
+            # Get git commit id if exists
+            return get_git_commit_id(src, self.build_logger)
+        except Exception as e:
+            if isinstance(e, FrogmlSuggestionException):
+                raise e  # Propagate suggestions without changes
+            main_dir_path = Path(src) / main_dir
+            if not main_dir_path.exists():
+                message = f"""Seems like you're running the 'frogml models build' command from the wrong location.
+Please make sure you run the command at the parent directory of your '{main_dir}' folder"""
+            else:
+                message = (f"Please make sure that {src} has read permissions.",)
+
+            raise FrogmlSuggestionException(
+                message="Unable to copy model",
+                src_exception=e,
+                suggestion=message,
+            )
+
+    def _copy_dependency_required_folders(
+        self, dependency_required_folders, src, dest, ignore_patterns
+    ):
+        for folder in dependency_required_folders:
+            destination_folder = folder
+            while destination_folder.startswith(".."):
+                destination_folder = re.sub(r"^\.\./", "", destination_folder)
+
+            if (Path(dest) / destination_folder).exists():
+                raise FrogmlSuggestionException(
+                    message="Unable to copy model",
+                    suggestion=f"It's not possible to copy directory `{folder}` into `{destination_folder}` because the target `{destination_folder}` already exist. Mixing files from multiple sources is not allowed.",
+                )
+            if (Path(src) / folder).exists():
+                copytree(
+                    src=Path(src) / folder,
+                    dst=Path(dest) / destination_folder,
+                    ignore=ignore_patterns,
+                )
+            else:
+                self.build_logger.warning(
+                    'Folder "{}" does not exist. Skipping it.'.format(folder)
+                )
+
+    def _copy_custom_dependencies(self, custom_dependencies_path, dest):
+        if custom_dependencies_path and Path(custom_dependencies_path).is_file():
+            shutil.copy(
+                src=custom_dependencies_path,
+                dst=Path(dest) / Path(custom_dependencies_path).name,
+            )
+            possible_dependency_lock_paths = get_possible_dependency_lock_paths(
+                Path(custom_dependencies_path)
+            )
+            for path in possible_dependency_lock_paths:
+                if path.is_file():
+                    self.build_logger.info(
+                        "Found dependency lock file: {}".format(path)
+                    )
+                    shutil.copy(
+                        src=path,
+                        dst=Path(dest) / path.name,
+                    )

frogml_core/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/git/git_strategy.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Optional, Union
+
+from frogml_core.exceptions import FrogmlException, FrogmlSuggestionException
+from frogml_core.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml_core.inner.build_logic.tools.files import copytree
+
+try:
+    from git import Repo
+    from git.exc import GitError
+except ImportError:
+    pass
+
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class GitStrategy(Strategy):
+    def fetch(
+        self,
+        src: Union[str, Path],
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        custom_dependencies_path: str,
+        main_dir: str,
+        git_ssh_key: str,
+        **kwargs,
+    ) -> str:
+        self.build_logger.debug(f"Fetching Model code from git repository -  {src}")
+        with tempfile.TemporaryDirectory() as clone_dest:
+            with ssh_key_context(git_ssh_key):
+                git_url, git_subdirectory = split_git_url_and_subdirectory(uri=src)
+                self.build_logger.debug(
+                    f"Fetching model code from git {git_url}, Model located in {git_subdirectory}"
+                )
+                git_commit_id = self.clone_and_checkout_repo(
+                    clone_dest=clone_dest,
+                    git_url=git_url,
+                    git_branch=git_branch,
+                    git_credentials=git_credentials,
+                )
+                self.copy_model(
+                    clone_dest=clone_dest,
+                    git_subdirectory=git_subdirectory,
+                    model_dest=dest,
+                    custom_dependencies_path=custom_dependencies_path,
+                    main_dir=main_dir,
+                )
+
+        return git_commit_id
+
+    def clone_and_checkout_repo(
+        self,
+        clone_dest: str,
+        git_url: str,
+        git_branch: str,
+        git_credentials: str,
+    ) -> str:
+        git_remote_url = get_git_remote_uri(
+            self.build_logger,
+            url=git_url,
+            git_credentials=git_credentials,
+        )
+        git_commit_id = clone_and_checkout_repo(
+            git_repo_url=git_remote_url, to_path=clone_dest, git_branch=git_branch
+        )
+
+        return git_commit_id
+
+    def copy_model(
+        self,
+        clone_dest: str,
+        git_subdirectory: str,
+        model_dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+    ):
+        ignore_patterns, patterns_for_printing = get_ignore_pattern(
+            Path(clone_dest) / git_subdirectory, main_dir, self.build_logger
+        )
+        self.build_logger.debug(
+            f"Will ignore the following files: {patterns_for_printing}."
+        )
+
+        # Copy model
+        copytree(
+            src=Path(clone_dest) / git_subdirectory,
+            dst=model_dest,
+            dirs_exist_ok=True,
+            ignore=ignore_patterns,
+        )
+        self.build_logger.debug("Model copied from git repository")
+
+        # Copy custom dependencies path
+        if custom_dependencies_path:
+            git_custom_dependencies_path = Path(clone_dest) / Path(
+                custom_dependencies_path
+            )
+            if git_custom_dependencies_path.is_file():
+                shutil.copy(
+                    src=git_custom_dependencies_path,
+                    dst=Path(model_dest) / git_custom_dependencies_path.name,
+                )
+                self.build_logger.debug(
+                    "Custom dependency file copied from git repository"
+                )
+
+
+def split_git_url_and_subdirectory(uri):
+    subdirectory = ""
+    parsed_uri = uri
+    if "#" in uri:
+        subdirectory = uri[uri.find("#") + 1 :]
+        parsed_uri = uri[: uri.find("#")]
+    if subdirectory and "." in subdirectory:
+        raise FrogmlException("'.' is not allowed in project subdirectory paths.")
+    return parsed_uri, subdirectory
+
+
+def get_git_remote_uri(
+    build_logger: BuildLogger, url: str, git_credentials: str
+) -> str:
+    if git_credentials:
+        url_start_index = url.find("://") + 3
+        url = f"{url[:url_start_index]}{git_credentials}@{url[url_start_index:]}"
+    else:
+        build_logger.warning("Git credentials secret has been provided")
+
+    return url
+
+
+@contextmanager
+def ssh_key_context(git_ssh_key: str):
+    ssh_key_path = None
+    if git_ssh_key:
+        ssh_key_path = make_ssh_key_file(git_ssh_key)
+        add_ssh_file_to_env(ssh_key_path)
+    try:
+        yield
+    finally:
+        if ssh_key_path:
+            try:
+                os.remove(ssh_key_path)
+            except OSError as e:
+                FrogmlException(f"Failed to delete ssh key temp key file: {e}")
+
+
+def make_ssh_key_file(git_ssh_key: str) -> str:
+    with tempfile.NamedTemporaryFile(
+        delete=False, mode="w", encoding="utf-8"
+    ) as temp_file:
+        temp_file.write(git_ssh_key + "\n")
+    temp_file_path = temp_file.name
+    return temp_file_path
+
+
+def add_ssh_file_to_env(ssh_key_file_path: str) -> None:
+    os.environ[
+        "GIT_SSH_COMMAND"
+    ] = f"ssh -i {ssh_key_file_path} -o StrictHostKeyChecking=no"
+    os.environ["GIT_SSH"] = f"ssh -i {ssh_key_file_path} -o StrictHostKeyChecking=no"
+
+
+def clone_and_checkout_repo(git_repo_url: str, to_path: str, git_branch: str) -> str:
+    try:
+        repo = Repo.clone_from(url=git_repo_url, to_path=to_path)
+        repo.submodule_update(init=True, recursive=True)
+        if git_branch in repo.branches:
+            repo.git.checkout(git_branch)
+        else:
+            repo.git.checkout("-b", git_branch, f"{repo.remote()}/{git_branch}")
+        return repo.head.object.hexsha
+    except GitError as e:
+        raise FrogmlSuggestionException(
+            src_exception=e,
+            message="Unable to clone git repository",
+            suggestion="Please check you git credentials",
+        )

frogml_core/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/strategy.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import shutil
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Callable, Optional, Tuple, Union
+
+from frogml_core.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml_core.inner.build_logic.tools.ignore_files import (
+    load_patterns_from_ignore_file,
+)
+
+_IGNORED_PATTERNS = [r"\..*", r"__pycache__"]
+QWAK_IGNORE_FILE_NAME = ".qwakignore"
+
+
+def get_ignore_pattern(
+    src: str, main_dir: str, build_logger: BuildLogger
+) -> Tuple[Callable[[Any, list[str]], set[str]], list[str]]:
+    if (Path(src) / main_dir / QWAK_IGNORE_FILE_NAME).is_file():
+        ignore_file_path = Path(src) / main_dir / QWAK_IGNORE_FILE_NAME
+    else:
+        ignore_file_path = Path(src) / QWAK_IGNORE_FILE_NAME
+
+    ignored_patterns = (
+        load_patterns_from_ignore_file(
+            build_logger=build_logger, ignore_file_path=ignore_file_path
+        )
+        + _IGNORED_PATTERNS
+    )
+
+    return shutil.ignore_patterns(*ignored_patterns), ignored_patterns
+
+
+class Strategy(ABC):
+    def __init__(self, build_logger: BuildLogger):
+        self.build_logger = build_logger
+
+    @abstractmethod
+    def fetch(
+        self,
+        src: Union[str, Path],
+        dest: str,
+        git_branch: str,
+        git_credentials: str,
+        model_id: str,
+        build_id: str,
+        custom_dependency_path: Optional[str],
+    ) -> Optional[str]:
+        pass

frogml_core/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/zip/zip_strategy.py

@@ -0,0 +1,64 @@
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Optional, Union
+from zipfile import ZipFile
+
+from frogml_core.exceptions import FrogmlSuggestionException
+from frogml_core.inner.build_logic.tools.files import copytree
+
+from ...common import get_git_commit_id
+from ..strategy import Strategy, get_ignore_pattern
+
+
+class ZipStrategy(Strategy):
+    def fetch(
+        self,
+        src: Union[str, Path],
+        dest: str,
+        custom_dependencies_path: Optional[str],
+        main_dir: str,
+        **kwargs,
+    ) -> Optional[str]:
+        self.build_logger.info(f"Fetching Model code from local zip file -  {src}")
+        try:
+            with tempfile.TemporaryDirectory() as temp_extraction_target:
+                with ZipFile(src) as zip_file:
+                    zip_file.extractall(path=temp_extraction_target)
+
+                    git_commit_id = get_git_commit_id(
+                        temp_extraction_target, self.build_logger
+                    )
+
+                    ignore_patterns, patterns_for_printing = get_ignore_pattern(
+                        temp_extraction_target,
+                        main_dir,
+                        self.build_logger,
+                    )
+                    self.build_logger.info(
+                        f"Will ignore the following files: {patterns_for_printing}."
+                    )
+
+                    copytree(
+                        src=temp_extraction_target,
+                        dst=dest,
+                        ignore=ignore_patterns,
+                    )
+
+                    # Copy custom dependencies path
+                    if (
+                        custom_dependencies_path
+                        and Path(custom_dependencies_path).is_file()
+                    ):
+                        shutil.copy(
+                            src=custom_dependencies_path,
+                            dst=Path(dest) / Path(custom_dependencies_path).name,
+                        )
+
+                return git_commit_id
+        except Exception as e:
+            raise FrogmlSuggestionException(
+                message="Unable to unzip zipped file",
+                src_exception=e,
+                suggestion=f"Please make sure that {src} has read permissions",
+            )

frogml_core/inner/build_logic/phases/phase_010_fetch_model/post_fetch_validation_step.py

@@ -0,0 +1,139 @@
+import importlib.util
+import os
+import shutil
+import subprocess  # nosec
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from frogml_core.exceptions import FrogmlSuggestionException
+from frogml_core.inner.build_logic.constants.temp_dir import TEMP_LOCAL_MODEL_DIR
+from frogml_core.inner.build_logic.dependency_manager_type import DependencyManagerType
+from frogml_core.inner.build_logic.interface.step_inteface import Step
+from frogml_core.inner.build_logic.tools.dependencies_tools import find_dependency_files
+
+TEMP_LOCAL_EDITABLE_FOLDER = "editable"
+
+
+@dataclass
+class DependencyFileObject:
+    dep_file_name: List[str]
+    lock_file_name: str = field(default="")
+
+
+DEPS_MANAGER_FILE_MAP = {
+    DependencyManagerType.PIP: DependencyFileObject(dep_file_name=["requirements.txt"]),
+    DependencyManagerType.POETRY: DependencyFileObject(
+        dep_file_name=["pyproject.toml"], lock_file_name="poetry.lock"
+    ),
+    DependencyManagerType.CONDA: DependencyFileObject(
+        dep_file_name=["conda.yml", "conda.yaml"]
+    ),
+}
+
+
+class PostFetchValidationStep(Step):
+    STEP_DESCRIPTION = "Post model fetch validation"
+
+    def description(self) -> str:
+        return self.STEP_DESCRIPTION
+
+    def execute(self) -> None:
+        self.validate_dependencies()
+        self.configure_base_docker_image()
+        self.create_development_wheels()
+
+    def validate_dependencies(self):
+        if (
+            not Path(self.config.build_properties.model_uri.uri).is_dir()
+            or self.config.build_env.python_env.dependency_file_path
+        ):
+            self.build_logger.debug("Validating dependency file exists")
+            model_uri, main_dir = (
+                self.context.host_temp_local_build_dir / TEMP_LOCAL_MODEL_DIR,
+                self.config.build_properties.model_uri.main_dir,
+            )
+            (
+                self.context.dependency_manager_type,
+                self.context.model_relative_dependency_file,
+                self.context.model_relative_dependency_lock_file,
+            ) = find_dependency_files(model_uri, main_dir, self.build_logger)
+
+            if (
+                self.context.dependency_manager_type
+                and self.context.model_relative_dependency_file
+            ):
+                return
+
+            self.build_logger.error("Dependency file wasn't found, failing...")
+            raise FrogmlSuggestionException(
+                message="Dependency file isn't found",
+                suggestion="Make sure your model include one of dependencies manager: pip/poetry/conda",
+            )
+
+    def configure_base_docker_image(self):
+        base_image = self.config.build_env.docker.base_image
+        if (not base_image) and self.config.build_env.remote.resources.gpu_type:
+            base_image = "frogml/frogml:gpu-py39"  # todo check gpu
+        self.context.base_image = base_image
+
+    def create_development_wheels(self):
+        if not os.getenv("FROGML_DEBUG"):
+            return
+
+        (module_location,) = importlib.util.find_spec(
+            "frogml_sdk"
+        ).submodule_search_locations
+        source_dir = Path(module_location).parent
+        pyproject_toml = source_dir / "pyproject.toml"
+
+        if pyproject_toml.is_file():
+            self.build_logger.info(
+                "Detected non-PyPI-released frogml-sdk installed, creating local frogml-runtime "
+                "and frogml-core wheel files to pass to build process"
+            )
+            runtime_dir = source_dir.parent / "frogml-runtime"
+            core_dir = source_dir.parent / "frogml-core"
+            target_path = self.context.host_temp_local_build_dir
+            self.context.custom_runtime_wheel = self._create_wheel(
+                runtime_dir, target_path
+            )
+            self.context.custom_core_wheel = self._create_wheel(core_dir, target_path)
+
+    def _create_wheel(self, package_dir, target_path):
+        dist_dir = package_dir / "dist"
+        shutil.rmtree(dist_dir, ignore_errors=True)
+        output = subprocess.check_output(["make", "install"], cwd=package_dir)  # nosec
+        for line in output.decode().split("\n"):
+            self.build_logger.debug(f">>> {line}")
+        output = subprocess.check_output(["poetry", "build"], cwd=package_dir)  # nosec
+        for line in output.decode().split("\n"):
+            self.build_logger.debug(f">>> {line}")
+        wheel_file = next(dist_dir.glob("*.whl"), None)
+        editable_folder = target_path / TEMP_LOCAL_EDITABLE_FOLDER
+        editable_folder.mkdir(exist_ok=True)
+        copied_wheel_file = shutil.move(wheel_file, editable_folder / wheel_file.name)
+        shutil.rmtree(dist_dir, ignore_errors=True)
+        self.build_logger.info(f"Created wheel for {package_dir} successfully")
+        return copied_wheel_file
+
+
+def find_file_location(model_uri, main_dir, filename) -> Path:
+    file_locations: List[Path] = [
+        model_uri / filename,
+        model_uri / main_dir / filename,
+    ]
+    for file in file_locations:
+        if file.is_file():
+            return file
+
+
+def get_possible_dependency_lock_paths(dependency_path: Path):
+    paths = []
+    for _, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        if dependency_file_object.lock_file_name:
+            lock_file_path = (
+                dependency_path.parent / dependency_file_object.lock_file_name
+            )
+            paths.append(lock_file_path)
+    return paths