frogml-core 0.0.1__py3-none-any.whl

frogml/inner/build_logic/tools/files.py

@@ -0,0 +1,237 @@
+import fnmatch
+import os
+import re
+import shutil
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+from frogml.exceptions import FrogmlGeneralBuildException
+from frogml.inner.build_logic.constants.temp_dir import TEMP_LOCAL_MODEL_DIR
+from frogml.inner.build_logic.interface.build_logger_interface import BuildLogger
+from frogml.inner.build_logic.interface.step_inteface import Step
+from frogml.inner.build_logic.tools.dependencies_tools import DEPS_MANAGER_FILE_MAP
+
+# Hidden directories setting
+HIDDEN_FILES_PREFIX = "."
+HIDDEN_DIRS_TO_INCLUDE = [".dvc"]
+IGNORED_PATTERNS_FOR_UPLOAD = [r"\..*", r"__pycache__"]
+FROGML_IGNORE_FILE_NAME = ".frogmlignore"
+
+
+def copytree(
+    src,
+    dst,
+    symlinks=False,
+    ignore=None,
+    ignore_dangling_symlinks=False,
+    dirs_exist_ok=False,
+):
+    names = os.listdir(src)
+    if ignore is not None:
+        ignored_names = ignore(src, names)
+    else:
+        ignored_names = set()
+
+    os.makedirs(dst, exist_ok=dirs_exist_ok)
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if os.path.islink(srcname):
+                linkto = os.readlink(srcname)
+                if symlinks:
+                    # We can't just leave it to `copy_function` because legacy
+                    # code with a custom `copy_function` may rely on copytree
+                    # doing the right thing.
+                    os.symlink(linkto, dstname)
+                    shutil.copystat(srcname, dstname, follow_symlinks=not symlinks)
+                else:
+                    # ignore dangling symlink if the flag is on
+                    if not os.path.exists(linkto) and ignore_dangling_symlinks:
+                        continue
+                    # otherwise let the copy occurs. copy2 will raise an error
+                    if os.path.isdir(srcname):
+                        copytree(srcname, dstname, symlinks, ignore)
+                    else:
+                        shutil.copy2(srcname, dstname)
+            elif os.path.isdir(srcname):
+                copytree(srcname, dstname, symlinks, ignore)
+            else:
+                # Will raise a SpecialFileError for unsupported file types
+                shutil.copy2(srcname, dstname)
+        # catch the Error from the recursive copytree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        shutil.copystat(src, dst)
+    except OSError as why:
+        # Copying file access times may fail on Windows
+        if getattr(why, "winerror", None) is None:
+            errors.append((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+    return dst
+
+
+def get_possible_dependency_lock_paths(dependency_path: Path):
+    paths = []
+    for _, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        if dependency_file_object.lock_file_name:
+            lock_file_path = (
+                dependency_path.parent / dependency_file_object.lock_file_name
+            )
+            paths.append(lock_file_path)
+    return paths
+
+
+def get_files_to_ignore(directory: Path, patterns: Iterable[str] = ()):
+    def ignore_hidden(file: Path, exclusions: List[str]):
+        name = os.path.basename(os.path.abspath(file))
+        is_hidden = name.startswith(HIDDEN_FILES_PREFIX) and (
+            name != FROGML_IGNORE_FILE_NAME and name not in exclusions
+        )
+        return is_hidden
+
+    def is_ignore_by_pattern(file: Path):
+        return (
+            len(
+                [
+                    pattern
+                    for pattern in patterns
+                    if re.search(fnmatch.translate(pattern), str(file))
+                ]
+            )
+            != 0
+        )
+
+    return [
+        file.name
+        for file in Path(directory).rglob("*")
+        if is_ignore_by_pattern(file)
+        or ignore_hidden(file, exclusions=HIDDEN_DIRS_TO_INCLUDE)
+    ]
+
+
+def _replace_large_files_with_too_large_file_message(
+    filtered_model: Path, max_bytes: Optional[int]
+):
+    def does_exceed_size(file: Path):
+        file_size = file.lstat().st_size
+        return file_size > max_bytes, file.lstat().st_size
+
+    if max_bytes is None:
+        return
+
+    for root, dirs, files in os.walk(filtered_model):
+        for file in files:
+            file_path = Path(os.path.join(root, file))
+            replace_content, file_size = does_exceed_size(file_path)
+            if replace_content:
+                Path(file_path).write_text(
+                    f"File is too big to display. Size: {file_size} bytes"
+                )
+
+
+def zip_model(
+    build_dir: Path,
+    dependency_file: Path,
+    dirs_to_include: List[str],
+    zip_name: str,
+    ignored_patterns: Iterable[str],
+    max_bytes: Optional[int] = None,
+    deps_lock_file: Optional[Path] = None,
+):
+    try:
+        filtered_model = build_dir / zip_name
+        ignored_patterns = get_files_to_ignore(
+            directory=build_dir / TEMP_LOCAL_MODEL_DIR, patterns=ignored_patterns
+        )
+
+        for included_dir in dirs_to_include:
+            dir_to_copy = build_dir / TEMP_LOCAL_MODEL_DIR / included_dir
+            if dir_to_copy.is_dir():
+                copytree(
+                    src=build_dir / TEMP_LOCAL_MODEL_DIR / included_dir,
+                    dst=filtered_model / included_dir,
+                    ignore=shutil.ignore_patterns(*ignored_patterns),
+                    dirs_exist_ok=True,
+                )
+
+        deps_file = build_dir / TEMP_LOCAL_MODEL_DIR / dependency_file
+        shutil.copy(deps_file, filtered_model / dependency_file)
+
+        if deps_lock_file:
+            deps_lock_file_full_path = build_dir / TEMP_LOCAL_MODEL_DIR / deps_lock_file
+            shutil.copy(deps_lock_file_full_path, filtered_model / deps_lock_file)
+
+        _replace_large_files_with_too_large_file_message(filtered_model, max_bytes)
+
+        zip_path = Path(
+            shutil.make_archive(
+                base_name=str(filtered_model),
+                format="zip",
+                root_dir=filtered_model,
+            )
+        )
+
+        shutil.rmtree(filtered_model)
+        return zip_path
+
+    except Exception as e:
+        raise FrogmlGeneralBuildException(
+            message="Unable to zip model before upload",
+            src_exception=e,
+        )
+
+
+class UploadInChunks(object):
+    def __init__(
+        self,
+        file: Path,
+        build_logger: BuildLogger,
+        all_files_size_to_upload: int,
+        read_so_far: int = 0,
+        chunk_size_bytes: int = 1 << 13,
+    ):
+        self._file = file
+        self._chunk_size = chunk_size_bytes
+        self._total_size = self._file.stat().st_size
+        self._read_so_far = read_so_far
+        self._build_logger = build_logger
+        self._all_files_size_to_upload = (
+            all_files_size_to_upload  # Used for calculating percentage for both files
+        )
+        self._last_percent_update = 0
+
+    def __iter__(self):
+        with self._file.open("rb") as file:
+            while True:
+                data = file.read(self._chunk_size)
+                if not data:
+                    break
+                self._read_so_far += len(data)
+                percent = self._read_so_far * 1e2 / self._all_files_size_to_upload
+                msg = "{percent:3.0f}%".format(percent=percent)
+                if int(percent / 10) > self._last_percent_update:
+                    self._build_logger.info(
+                        msg
+                    )  # Updating only after 10 percent change
+                    self._last_percent_update = int(percent / 10)
+                if hasattr(self._build_logger, "spinner_text"):
+                    self._build_logger.spinner_text(line=msg)
+                yield data
+
+    def __len__(self):
+        return self._total_size
+
+
+def cleaning_up_after_build(step: Step):
+    if os.getenv("FROGML_DEBUG") != "true":
+        step.build_logger.debug("Removing Frogml temp artifacts directory")
+        shutil.rmtree(step.context.host_temp_local_build_dir, ignore_errors=True)

frogml/inner/build_logic/tools/ignore_files.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from frogml.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+
+def load_patterns_from_ignore_file(build_logger: BuildLogger, ignore_file_path: Path):
+    if Path(ignore_file_path).is_file():
+        build_logger.info("Found a Qwak ignore file - will ignore listed patterns")
+
+        with open(ignore_file_path, "r") as igonre_file:
+            patterns_to_ignore = [
+                pattern.strip() for pattern in igonre_file.readlines()
+            ]
+            build_logger.debug(
+                f"Patterns from Qwak igonre file detected - {str(patterns_to_ignore)}"
+            )
+            return patterns_to_ignore
+
+    build_logger.debug("no Qwak ignore file was found, skipping")
+    return []

frogml/inner/build_logic/tools/text.py

@@ -0,0 +1,9 @@
+from re import sub
+
+
+def snake_case(s):
+    return "_".join(
+        sub(
+            "([A-Z][a-z]+)", r" \1", sub("([A-Z]+)", r" \1", s.replace("-", " "))
+        ).split()
+    ).lower()

frogml/inner/build_logic/trigger_build_context.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from frogml.inner.build_logic.interface.context_interface import Context
+
+
+@dataclass
+class TriggerBuildContext(Context):
+    pass

frogml/inner/const.py

@@ -0,0 +1,6 @@
+class FrogMLConstants:
+    """
+    JFrog ML Configuration settings
+    """
+
+    JFROG_TENANT_HEADER_KEY = "X-JFrog-Tenant-Id"

frogml/inner/di_configuration/__init__.py

@@ -0,0 +1,73 @@
+import os
+
+from .account import UserAccountConfiguration
+from .containers import FrogmlContainer
+
+
+def wire_dependencies():
+    container = FrogmlContainer()
+
+    default_config_file = os.path.join(os.path.dirname(__file__), "config.yml")
+    container.config.from_yaml(default_config_file)
+
+    from frogml.clients import (
+        administration,
+        alert_management,
+        alerts_registry,
+        analytics,
+        audience,
+        automation_management,
+        autoscaling,
+        batch_job_management,
+        build_management,
+        build_orchestrator,
+        data_versioning,
+        deployment,
+        feature_store,
+        file_versioning,
+        instance_template,
+        integration_management,
+        kube_deployment_captain,
+        logging_client,
+        model_management,
+        model_version_manager,
+        project,
+        prompt_manager,
+        system_secret,
+        user_application_instance,
+        vector_store,
+        workspace_manager,
+    )
+
+    container.wire(
+        packages=[
+            administration,
+            alert_management,
+            audience,
+            automation_management,
+            autoscaling,
+            analytics,
+            batch_job_management,
+            build_management,
+            build_orchestrator,
+            data_versioning,
+            deployment,
+            file_versioning,
+            instance_template,
+            kube_deployment_captain,
+            logging_client,
+            model_management,
+            project,
+            feature_store,
+            user_application_instance,
+            alerts_registry,
+            workspace_manager,
+            vector_store,
+            integration_management,
+            system_secret,
+            prompt_manager,
+            model_version_manager,
+        ]
+    )
+
+    return container

frogml/inner/di_configuration/account.py

@@ -0,0 +1,106 @@
+import errno
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+from frogml_storage.authentication._authentication_utils import get_credentials
+from frogml_storage.cli._login_cli import login as frogml_login
+from frogml_storage.constants import CONFIG_FILE_PATH
+
+from frogml.exceptions import FrogmlLoginException
+from frogml.inner.tool.auth import FrogMLAuthClient
+
+
+@dataclass
+class UserAccount:
+    """
+    User Account Configuration
+    """
+
+    # Assigned Token
+    token: Optional[str] = None
+
+    # Assigned username
+    username: Optional[str] = None
+
+    # Assigned password
+    password: Optional[str] = None
+
+    # Assigned URL
+    url: Optional[str] = None
+
+    # Anonymous login
+    anonymous: bool = False
+
+    # Interactive login
+    is_interactive: bool = False
+
+
+class UserAccountConfiguration:
+    def __init__(
+        self,
+        config_file=CONFIG_FILE_PATH,
+    ):
+        self._config_file = config_file
+        self._auth_client = FrogMLAuthClient()
+
+    def configure_user(self, user_account: UserAccount):
+        """
+        Configure user authentication based on the authentication client type
+        """
+
+        # Use FrogML login flow
+        success = frogml_login(
+            url=user_account.url,
+            username=user_account.username,
+            password=user_account.password,
+            token=user_account.token,
+            anonymous=user_account.anonymous,
+            is_interactive=user_account.is_interactive,
+        )
+
+        if not success:
+            raise FrogmlLoginException("Failed to authenticate with JFrog")
+
+        # Validate token
+        token: Optional[str] = self._auth_client.get_token()
+
+        if not token or len(token) <= 64:
+            raise FrogmlLoginException(
+                "Authentication with JFrog failed: Only Access Tokens are supported. "
+                "Please ensure you are using a valid Access Token."
+            )
+
+    @staticmethod
+    def _mkdir_p(path):
+        try:
+            os.makedirs(path)
+        except OSError as exc:  # Python >2.5
+            if not (exc.errno == errno.EEXIST and os.path.isdir(path)):
+                raise
+
+    @staticmethod
+    def _safe_open(path):
+        UserAccountConfiguration._mkdir_p(os.path.dirname(path))
+        return open(path, "w")
+
+    @staticmethod
+    def get_user_config() -> UserAccount:
+        """
+        Get persisted user account from config file
+        :return:
+        """
+        return UserAccount()
+
+    @staticmethod
+    def get_user_token() -> str:
+        """
+        Get persisted user account from config file
+        :return:
+        """
+        _, auth = get_credentials()
+
+        try:
+            return getattr(auth, "token")
+        except AttributeError as e:
+            raise FrogmlLoginException("Token is not configured") from e

frogml/inner/di_configuration/config.yml

@@ -0,0 +1,9 @@
+grpc:
+    core:
+        address: 'grpc.qwak.ai'
+        enable_ssl: True
+    builds:
+        internal_address: 'builds-orchestrator-webapp.default.svc.cluster.local:6577'
+        enable_ssl: False
+    authentication:
+        enable_ssl: True

frogml/inner/di_configuration/containers.py

@@ -0,0 +1,40 @@
+import grpc
+from dependency_injector import containers, providers
+
+from frogml.inner.tool.grpc.grpc_tools import (
+    create_grpc_channel,
+    create_grpc_channel_or_none,
+)
+
+
+class FrogmlContainer(containers.DeclarativeContainer):
+    """
+    JFrog ML CLI dependencies
+    """
+
+    config = providers.Configuration(strict=True)
+
+    core_grpc_channel = providers.Singleton(
+        create_grpc_channel,
+        url=config.grpc.core.address,
+        enable_ssl=config.grpc.core.enable_ssl,
+        status_for_retry=(
+            grpc.StatusCode.UNAVAILABLE,
+            grpc.StatusCode.CANCELLED,
+        ),
+    )
+
+    unauthenticated_core_grpc_channel = providers.Singleton(
+        create_grpc_channel,
+        url=config.grpc.core.address,
+        enable_ssl=config.grpc.authentication.enable_ssl,
+        enable_auth=False,
+    )
+
+    internal_grpc_channel_for_builds_orchestrator_factory = providers.Singleton(
+        create_grpc_channel_or_none,
+        url=config.grpc.builds.internal_address,
+        enable_ssl=config.grpc.builds.enable_ssl,
+        enable_auth=False,
+        status_for_retry=(grpc.StatusCode.UNAVAILABLE,),
+    )

frogml/inner/instance_template/verify_template_id.py

@@ -0,0 +1,48 @@
+from typing import Optional
+
+from _frogml_proto.qwak.instance_template.instance_template_pb2 import InstanceType
+from frogml.clients.instance_template.client import InstanceTemplateManagementClient
+from frogml.exceptions import FrogmlException
+from frogml.inner.provider import Provider
+
+INVALID_TEMPLATE_ID_ERROR_FORMAT = (
+    "Invalid instance: {template_id}. Valid instances are: {cpu_templates}. "
+    "For GPU instances you can use {gpu_templates}."
+)
+
+
+def verify_template_id(
+    template_id: str,
+    instance_template_client: InstanceTemplateManagementClient,
+    provider: Optional[Provider] = None,
+) -> None:
+    all_templates = instance_template_client.list_instance_templates()
+    valid_templates = [template for template in all_templates if template.enabled]
+    if provider == Provider.AWS:
+        valid_templates = [
+            template for template in valid_templates if template.aws_supported
+        ]
+    elif provider == Provider.GCP:
+        valid_templates = [
+            template for template in valid_templates if template.gcp_supported
+        ]
+
+    existing_templates = {template.id: template for template in valid_templates}
+    if template_id not in existing_templates.keys():
+        cpu_template_ids = [
+            template.id
+            for template in sorted(valid_templates, key=lambda template: template.order)
+            if template.instance_type == InstanceType.INSTANCE_TYPE_CPU
+        ]
+        gpu_template_ids = [
+            template.id
+            for template in sorted(valid_templates, key=lambda template: template.order)
+            if template.instance_type == InstanceType.INSTANCE_TYPE_GPU
+        ]
+        raise FrogmlException(
+            INVALID_TEMPLATE_ID_ERROR_FORMAT.format(
+                template_id=template_id,
+                cpu_templates=", ".join(cpu_template_ids),
+                gpu_templates=", ".join(gpu_template_ids),
+            )
+        )

frogml/inner/model_loggers_utils.py

@@ -0,0 +1,110 @@
+import os
+import re
+from typing import Optional
+
+import requests
+
+from _frogml_proto.qwak.builds.builds_orchestrator_service_pb2 import (
+    AuthenticationDetail,
+)
+from frogml.clients.model_management.client import ModelsManagementClient
+from frogml.exceptions import FrogmlException
+
+TAG_REGEX = re.compile(r"^[a-zA-Z0-9_-]+$")
+
+
+def validate_tag(tag: str) -> bool:
+    """
+    Check if tag exists.
+
+    Args:
+        tag: tag to check
+
+    Returns: If tag is valid
+    """
+    return re.match(TAG_REGEX, tag) is not None
+
+
+def fetch_model_id() -> Optional[str]:
+    """
+    Get model id from environment.
+
+    Returns: model id if found.
+
+    Notes:
+        1. Checking if called inside a model - then model id saved as environment variable.
+    """
+    # Checking if called inside a model - then model id saved as environment variable
+    return os.getenv("QWAK_MODEL_ID", None)
+
+
+def validate_model(model_id: str) -> str:
+    """
+    Validate a model ID validity and existence
+    """
+    if not model_id:
+        model_id = fetch_model_id()
+        if not model_id:
+            raise FrogmlException("Failed to determined model ID.")
+
+    try:
+        ModelsManagementClient().get_model(model_id=model_id)
+    except Exception:
+        raise FrogmlException("Failed to find model.")
+
+    return model_id
+
+
+def fetch_build_id() -> Optional[str]:
+    """
+    Get Build id from environment
+
+    Returns: Build id if found
+
+    Notes:
+        1. Checking if called inside a model - then build id saved as environment variable.
+    """
+    # Checking if called inside a model - then model id saved as environment variable
+    return os.getenv("QWAK_BUILD_ID", None)
+
+
+def upload_data(
+    upload_url: str,
+    data: bytes,
+    authentication_details: Optional[AuthenticationDetail],
+    content_type: str = "text/plain",
+) -> None:
+    """
+    Upload data
+    Args:
+        upload_url: the url to upload to.
+        data: the data to upload
+        authentication_details: authentication details for upload data
+        content_type: Uploaded content-type
+    """
+    try:
+        auth = None
+        if (
+            authentication_details.WhichOneof("integration_type")
+            == "jfrog_authentication_detail"
+        ):
+            auth = (
+                authentication_details.jfrog_authentication_detail.username,
+                authentication_details.jfrog_authentication_detail.token,
+            )
+
+        http_response = requests.put(  # nosec B113
+            upload_url,
+            data=data,
+            headers={"content-type": content_type},
+            auth=auth,
+        )
+
+        if http_response.status_code not in [200, 201]:
+            raise FrogmlException(
+                f"Failed to upload data. "
+                f"Status: [{http_response.status_code}], "
+                f"reason: [{http_response.reason}]"
+            )
+    except Exception as e:
+        raise FrogmlException(f"Failed to upload data. Error is {e}")

frogml/inner/provider.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class Provider(Enum):
+    AWS = 1
+    GCP = 2