frogml-core 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

frogml_core/inner/build_logic/run_handlers/programmatic_phase_run_handler.py

@@ -0,0 +1,101 @@
+from contextlib import contextmanager
+from logging import Logger
+
+from frogml_core.exceptions import FrogmlException
+from frogml_core.inner.build_logic.build_loggers.trigger_build_logger import (
+    TriggerBuildLogger,
+)
+from frogml_core.inner.build_logic.constants.messages import (
+    FAILED_CONTACT_FROGML_SUPPORT_PROGRAMMATIC,
+)
+from frogml_core.inner.build_logic.interface.build_phase import BuildPhase
+from frogml_core.inner.build_logic.interface.phase_run_handler import PhaseRunHandler
+from frogml_core.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+
+
+class ProgrammaticPhaseRunHandler(PhaseRunHandler):
+    BUILD_IN_PROGRESS_FORMAT = "Build phase in progress: {}"
+    BUILD_FINISHED_FORMAT = "Phase successfully finished: {} after {} seconds"
+    BUILD_FAILURE_FORMAT = "Build phase failed: {} after {} seconds"
+
+    def __init__(self, python_logger: Logger, verbose: int, json_logs: bool):
+        self.build_logger = None
+        self.python_logger = python_logger
+        self.json_logs = json_logs
+        self.verbose = verbose
+
+    @contextmanager
+    def handle_current_phase(self, phase: PhasesPipeline):
+        self.build_logger = TriggerBuildLogger(
+            self.python_logger,
+            prefix="" if self.json_logs else phase.build_phase.description,
+            build_phase=phase.build_phase,
+        )
+        yield
+
+    def handle_phase_in_progress(self, build_phase: BuildPhase):
+        logger = self.build_logger or self.python_logger
+        logger.debug(f"Build phase in progress: {build_phase.name}")
+
+    def handle_phase_finished_successfully(
+        self, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        logger = self.build_logger or self.python_logger
+        logger.debug(
+            f"Phase successfully finished: {build_phase.name} after {duration_in_seconds} seconds"
+        )
+
+    def _report_failure(self, build_phase: BuildPhase, duration_in_seconds: int):
+        logger = self.build_logger or self.python_logger
+        logger.debug(
+            self.BUILD_FAILURE_FORMAT.format(build_phase.name, duration_in_seconds)
+        )
+
+    def handle_contact_support_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        print(
+            FAILED_CONTACT_FROGML_SUPPORT_PROGRAMMATIC.format(
+                build_id=build_id,
+            )
+        )
+        self._report_failure(build_phase, duration_in_seconds)
+        raise FrogmlException(str(ex))
+
+    def handle_keyboard_interrupt(
+        self, build_id: str, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        self._report_failure(build_phase, duration_in_seconds)
+        raise FrogmlException("KeyboardInterrupt")
+
+    def handle_pipeline_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        self._report_failure(build_phase, duration_in_seconds)
+        raise FrogmlException(str(ex))
+
+    def handle_pipeline_quiet_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        self._report_failure(build_phase, duration_in_seconds)
+
+    def handle_remote_build_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        self.handle_pipeline_exception(build_id, build_phase, ex, duration_in_seconds)

frogml_core/inner/build_logic/tools/dependencies_tools.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from frogml_core.inner.build_logic.dependency_manager_type import DependencyManagerType
+from frogml_core.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+
+@dataclass
+class DependencyFileObject:
+    dep_file_name: List[str]
+    lock_file_name: str = field(default="")
+
+
+DEPS_MANAGER_FILE_MAP = {
+    DependencyManagerType.PIP: DependencyFileObject(dep_file_name=["requirements.txt"]),
+    DependencyManagerType.POETRY: DependencyFileObject(
+        dep_file_name=["pyproject.toml"], lock_file_name="poetry.lock"
+    ),
+    DependencyManagerType.CONDA: DependencyFileObject(
+        dep_file_name=["conda.yml", "conda.yaml"]
+    ),
+}
+
+
+def find_dependency_files(model_uri, main_dir, build_logger: BuildLogger):
+    dependency_manager_type = None
+    model_relative_dependency_file = None
+    model_relative_dependency_lock_file = None
+    for dep_type, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        for filename in dependency_file_object.dep_file_name:
+            dep_file_path = find_file_location(model_uri, main_dir, filename)
+            if dep_file_path:
+                build_logger.info(
+                    f"Found dependency type: {dep_type.name} by file: {dep_file_path}"
+                )
+                dependency_manager_type = dep_type
+                model_relative_dependency_file = dep_file_path.relative_to(model_uri)
+                if dependency_file_object.lock_file_name:
+                    dep_lock_file_path = find_file_location(
+                        model_uri,
+                        main_dir,
+                        dependency_file_object.lock_file_name,
+                    )
+                    if dep_lock_file_path:
+                        build_logger.info(
+                            f"Found dependency lock file {dep_lock_file_path}"
+                        )
+                        model_relative_dependency_lock_file = (
+                            dep_lock_file_path.relative_to(model_uri)
+                        )
+                break
+
+    return (
+        dependency_manager_type,
+        model_relative_dependency_file,
+        model_relative_dependency_lock_file,
+    )
+
+
+def find_file_location(model_uri, main_dir, filename) -> Path:
+    file_locations: List[Path] = [
+        model_uri / filename,
+        model_uri / main_dir / filename,
+    ]
+    for file in file_locations:
+        if file.is_file():
+            return file

frogml_core/inner/build_logic/tools/files.py

@@ -0,0 +1,237 @@
+import fnmatch
+import os
+import re
+import shutil
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+from frogml_core.exceptions import FrogmlGeneralBuildException
+from frogml_core.inner.build_logic.constants.temp_dir import TEMP_LOCAL_MODEL_DIR
+from frogml_core.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml_core.inner.build_logic.interface.step_inteface import Step
+from frogml_core.inner.build_logic.tools.dependencies_tools import DEPS_MANAGER_FILE_MAP
+
+# Hidden directories setting
+HIDDEN_FILES_PREFIX = "."
+HIDDEN_DIRS_TO_INCLUDE = [".dvc"]
+IGNORED_PATTERNS_FOR_UPLOAD = [r"\..*", r"__pycache__"]
+FROGML_IGNORE_FILE_NAME = ".frogmlignore"
+
+
+def copytree(
+    src,
+    dst,
+    symlinks=False,
+    ignore=None,
+    ignore_dangling_symlinks=False,
+    dirs_exist_ok=False,
+):
+    names = os.listdir(src)
+    if ignore is not None:
+        ignored_names = ignore(src, names)
+    else:
+        ignored_names = set()
+
+    os.makedirs(dst, exist_ok=dirs_exist_ok)
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if os.path.islink(srcname):
+                linkto = os.readlink(srcname)
+                if symlinks:
+                    # We can't just leave it to `copy_function` because legacy
+                    # code with a custom `copy_function` may rely on copytree
+                    # doing the right thing.
+                    os.symlink(linkto, dstname)
+                    shutil.copystat(srcname, dstname, follow_symlinks=not symlinks)
+                else:
+                    # ignore dangling symlink if the flag is on
+                    if not os.path.exists(linkto) and ignore_dangling_symlinks:
+                        continue
+                    # otherwise let the copy occurs. copy2 will raise an error
+                    if os.path.isdir(srcname):
+                        copytree(srcname, dstname, symlinks, ignore)
+                    else:
+                        shutil.copy2(srcname, dstname)
+            elif os.path.isdir(srcname):
+                copytree(srcname, dstname, symlinks, ignore)
+            else:
+                # Will raise a SpecialFileError for unsupported file types
+                shutil.copy2(srcname, dstname)
+        # catch the Error from the recursive copytree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        shutil.copystat(src, dst)
+    except OSError as why:
+        # Copying file access times may fail on Windows
+        if getattr(why, "winerror", None) is None:
+            errors.append((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+    return dst
+
+
+def get_possible_dependency_lock_paths(dependency_path: Path):
+    paths = []
+    for _, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        if dependency_file_object.lock_file_name:
+            lock_file_path = (
+                dependency_path.parent / dependency_file_object.lock_file_name
+            )
+            paths.append(lock_file_path)
+    return paths
+
+
+def get_files_to_ignore(directory: Path, patterns: Iterable[str] = ()):
+    def ignore_hidden(file: Path, exclusions: List[str]):
+        name = os.path.basename(os.path.abspath(file))
+        is_hidden = name.startswith(HIDDEN_FILES_PREFIX) and (
+            name != FROGML_IGNORE_FILE_NAME and name not in exclusions
+        )
+        return is_hidden
+
+    def is_ignore_by_pattern(file: Path):
+        return (
+            len(
+                [
+                    pattern
+                    for pattern in patterns
+                    if re.search(fnmatch.translate(pattern), str(file))
+                ]
+            )
+            != 0
+        )
+
+    return [
+        file.name
+        for file in Path(directory).rglob("*")
+        if is_ignore_by_pattern(file)
+        or ignore_hidden(file, exclusions=HIDDEN_DIRS_TO_INCLUDE)
+    ]
+
+
+def _replace_large_files_with_too_large_file_message(
+    filtered_model: Path, max_bytes: Optional[int]
+):
+    def does_exceed_size(file: Path):
+        file_size = file.lstat().st_size
+        return file_size > max_bytes, file.lstat().st_size
+
+    if max_bytes is None:
+        return
+
+    for root, dirs, files in os.walk(filtered_model):
+        for file in files:
+            file_path = Path(os.path.join(root, file))
+            replace_content, file_size = does_exceed_size(file_path)
+            if replace_content:
+                Path(file_path).write_text(
+                    f"File is too big to display. Size: {file_size} bytes"
+                )
+
+
+def zip_model(
+    build_dir: Path,
+    dependency_file: Path,
+    dirs_to_include: List[str],
+    zip_name: str,
+    ignored_patterns: Iterable[str],
+    max_bytes: Optional[int] = None,
+    deps_lock_file: Optional[Path] = None,
+):
+    try:
+        filtered_model = build_dir / zip_name
+        ignored_patterns = get_files_to_ignore(
+            directory=build_dir / TEMP_LOCAL_MODEL_DIR, patterns=ignored_patterns
+        )
+
+        for included_dir in dirs_to_include:
+            dir_to_copy = build_dir / TEMP_LOCAL_MODEL_DIR / included_dir
+            if dir_to_copy.is_dir():
+                copytree(
+                    src=build_dir / TEMP_LOCAL_MODEL_DIR / included_dir,
+                    dst=filtered_model / included_dir,
+                    ignore=shutil.ignore_patterns(*ignored_patterns),
+                    dirs_exist_ok=True,
+                )
+
+        deps_file = build_dir / TEMP_LOCAL_MODEL_DIR / dependency_file
+        shutil.copy(deps_file, filtered_model / dependency_file)
+
+        if deps_lock_file:
+            deps_lock_file_full_path = build_dir / TEMP_LOCAL_MODEL_DIR / deps_lock_file
+            shutil.copy(deps_lock_file_full_path, filtered_model / deps_lock_file)
+
+        _replace_large_files_with_too_large_file_message(filtered_model, max_bytes)
+
+        zip_path = Path(
+            shutil.make_archive(
+                base_name=str(filtered_model),
+                format="zip",
+                root_dir=filtered_model,
+            )
+        )
+
+        shutil.rmtree(filtered_model)
+        return zip_path
+
+    except Exception as e:
+        raise FrogmlGeneralBuildException(
+            message="Unable to zip model before upload",
+            src_exception=e,
+        )
+
+
+class UploadInChunks(object):
+    def __init__(
+        self,
+        file: Path,
+        build_logger: BuildLogger,
+        all_files_size_to_upload: int,
+        read_so_far: int = 0,
+        chunk_size_bytes: int = 1 << 13,
+    ):
+        self._file = file
+        self._chunk_size = chunk_size_bytes
+        self._total_size = self._file.stat().st_size
+        self._read_so_far = read_so_far
+        self._build_logger = build_logger
+        self._all_files_size_to_upload = (
+            all_files_size_to_upload  # Used for calculating percentage for both files
+        )
+        self._last_percent_update = 0
+
+    def __iter__(self):
+        with self._file.open("rb") as file:
+            while True:
+                data = file.read(self._chunk_size)
+                if not data:
+                    break
+                self._read_so_far += len(data)
+                percent = self._read_so_far * 1e2 / self._all_files_size_to_upload
+                msg = "{percent:3.0f}%".format(percent=percent)
+                if int(percent / 10) > self._last_percent_update:
+                    self._build_logger.info(
+                        msg
+                    )  # Updating only after 10 percent change
+                    self._last_percent_update = int(percent / 10)
+                if hasattr(self._build_logger, "spinner_text"):
+                    self._build_logger.spinner_text(line=msg)
+                yield data
+
+    def __len__(self):
+        return self._total_size
+
+
+def cleaning_up_after_build(step: Step):
+    if os.getenv("FROGML_DEBUG") != "true":
+        step.build_logger.debug("Removing Frogml temp artifacts directory")
+        shutil.rmtree(step.context.host_temp_local_build_dir, ignore_errors=True)

frogml_core/inner/build_logic/tools/ignore_files.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from frogml_core.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+
+def load_patterns_from_ignore_file(build_logger: BuildLogger, ignore_file_path: Path):
+    if Path(ignore_file_path).is_file():
+        build_logger.info("Found a Qwak ignore file - will ignore listed patterns")
+
+        with open(ignore_file_path, "r") as igonre_file:
+            patterns_to_ignore = [
+                pattern.strip() for pattern in igonre_file.readlines()
+            ]
+            build_logger.debug(
+                f"Patterns from Qwak igonre file detected - {str(patterns_to_ignore)}"
+            )
+            return patterns_to_ignore
+
+    build_logger.debug("no Qwak ignore file was found, skipping")
+    return []

frogml_core/inner/build_logic/trigger_build_context.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from frogml_core.inner.build_logic.interface.context_interface import Context
+
+
+@dataclass
+class TriggerBuildContext(Context):
+    pass

frogml_core/inner/di_configuration/__init__.py

@@ -0,0 +1,75 @@
+import os
+
+from .account import UserAccountConfiguration
+from .containers import FrogmlContainer
+
+
+def wire_dependencies():
+    container = FrogmlContainer()
+
+    default_config_file = os.path.join(os.path.dirname(__file__), "config.yml")
+    container.config.from_yaml(default_config_file)
+
+    from frogml_core.clients import (
+        administration,
+        alert_management,
+        alerts_registry,
+        analytics,
+        audience,
+        automation_management,
+        autoscaling,
+        batch_job_management,
+        build_management,
+        build_orchestrator,
+        data_versioning,
+        deployment,
+        feature_store,
+        file_versioning,
+        instance_template,
+        integration_management,
+        kube_deployment_captain,
+        logging_client,
+        model_management,
+        model_version_manager,
+        project,
+        prompt_manager,
+        system_secret,
+        user_application_instance,
+        vector_store,
+        workspace_manager,
+        jfrog_gateway,
+    )
+
+    container.wire(
+        packages=[
+            administration,
+            alert_management,
+            audience,
+            automation_management,
+            autoscaling,
+            analytics,
+            batch_job_management,
+            build_management,
+            build_orchestrator,
+            data_versioning,
+            deployment,
+            file_versioning,
+            instance_template,
+            kube_deployment_captain,
+            logging_client,
+            model_management,
+            project,
+            feature_store,
+            user_application_instance,
+            alerts_registry,
+            workspace_manager,
+            vector_store,
+            integration_management,
+            system_secret,
+            prompt_manager,
+            model_version_manager,
+            jfrog_gateway,
+        ]
+    )
+
+    return container

frogml_core/inner/di_configuration/account.py

@@ -0,0 +1,106 @@
+import errno
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+from frogml_storage.authentication._authentication_utils import get_credentials
+from frogml_storage.cli._login_cli import login as frogml_login
+from frogml_storage.constants import CONFIG_FILE_PATH
+
+from frogml_core.exceptions import FrogmlLoginException
+from frogml_core.inner.tool.auth import FrogMLAuthClient
+
+
+@dataclass
+class UserAccount:
+    """
+    User Account Configuration
+    """
+
+    # Assigned Token
+    token: Optional[str] = None
+
+    # Assigned username
+    username: Optional[str] = None
+
+    # Assigned password
+    password: Optional[str] = None
+
+    # Assigned URL
+    url: Optional[str] = None
+
+    # Anonymous login
+    anonymous: bool = False
+
+    # Interactive login
+    is_interactive: bool = False
+
+
+class UserAccountConfiguration:
+    def __init__(
+        self,
+        config_file=CONFIG_FILE_PATH,
+    ):
+        self._config_file = config_file
+        self._auth_client = FrogMLAuthClient()
+
+    def configure_user(self, user_account: UserAccount):
+        """
+        Configure user authentication based on the authentication client type
+        """
+
+        # Use FrogML login flow
+        success = frogml_login(
+            url=user_account.url,
+            username=user_account.username,
+            password=user_account.password,
+            token=user_account.token,
+            anonymous=user_account.anonymous,
+            is_interactive=user_account.is_interactive,
+        )
+
+        if not success:
+            raise FrogmlLoginException("Failed to authenticate with JFrog")
+
+        # Validate token
+        token: Optional[str] = self._auth_client.get_token()
+
+        if not token or len(token) <= 64:
+            raise FrogmlLoginException(
+                "Authentication with JFrog failed: Only Access Tokens are supported. "
+                "Please ensure you are using a valid Access Token."
+            )
+
+    @staticmethod
+    def _mkdir_p(path):
+        try:
+            os.makedirs(path)
+        except OSError as exc:  # Python >2.5
+            if not (exc.errno == errno.EEXIST and os.path.isdir(path)):
+                raise
+
+    @staticmethod
+    def _safe_open(path):
+        UserAccountConfiguration._mkdir_p(os.path.dirname(path))
+        return open(path, "w")
+
+    @staticmethod
+    def get_user_config() -> UserAccount:
+        """
+        Get persisted user account from config file
+        :return:
+        """
+        return UserAccount()
+
+    @staticmethod
+    def get_user_token() -> str:
+        """
+        Get persisted user account from config file
+        :return:
+        """
+        _, auth = get_credentials()
+
+        try:
+            return getattr(auth, "token")
+        except AttributeError as e:
+            raise FrogmlLoginException("Token is not configured") from e

frogml_core/inner/di_configuration/containers.py

@@ -0,0 +1,40 @@
+import grpc
+from dependency_injector import containers, providers
+
+from frogml_core.inner.tool.grpc.grpc_tools import (
+    create_grpc_channel,
+    create_grpc_channel_or_none,
+)
+
+
+class FrogmlContainer(containers.DeclarativeContainer):
+    """
+    JFrog ML CLI dependencies
+    """
+
+    config = providers.Configuration(strict=True)
+
+    core_grpc_channel = providers.Singleton(
+        create_grpc_channel,
+        url=config.grpc.core.address,
+        enable_ssl=config.grpc.core.enable_ssl,
+        status_for_retry=(
+            grpc.StatusCode.UNAVAILABLE,
+            grpc.StatusCode.CANCELLED,
+        ),
+    )
+
+    unauthenticated_core_grpc_channel = providers.Singleton(
+        create_grpc_channel,
+        url=config.grpc.core.address,
+        enable_ssl=config.grpc.authentication.enable_ssl,
+        enable_auth=False,
+    )
+
+    internal_grpc_channel_for_builds_orchestrator_factory = providers.Singleton(
+        create_grpc_channel_or_none,
+        url=config.grpc.builds.internal_address,
+        enable_ssl=config.grpc.builds.enable_ssl,
+        enable_auth=False,
+        status_for_retry=(grpc.StatusCode.UNAVAILABLE,),
+    )