qwak-core 0.1.37__py3-none-any.whl → 0.5.19__py3-none-any.whl

qwak/inner/build_logic/run_handlers/programmatic_phase_run_handler.py

@@ -0,0 +1,99 @@
+from contextlib import contextmanager
+from logging import Logger
+
+from qwak.exceptions import QwakException
+from qwak.inner.build_logic.build_loggers.trigger_build_logger import TriggerBuildLogger
+from qwak.inner.build_logic.constants.messages import (
+    FAILED_CONTACT_QWAK_SUPPORT_PROGRAMMATIC,
+)
+from qwak.inner.build_logic.interface.build_phase import BuildPhase
+from qwak.inner.build_logic.interface.phase_run_handler import PhaseRunHandler
+from qwak.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+
+
+class ProgrammaticPhaseRunHandler(PhaseRunHandler):
+    BUILD_IN_PROGRESS_FORMAT = "Build phase in progress: {}"
+    BUILD_FINISHED_FORMAT = "Phase successfully finished: {} after {} seconds"
+    BUILD_FAILURE_FORMAT = "Build phase failed: {} after {} seconds"
+
+    def __init__(self, python_logger: Logger, verbose: int, json_logs: bool):
+        self.build_logger = None
+        self.python_logger = python_logger
+        self.json_logs = json_logs
+        self.verbose = verbose
+
+    @contextmanager
+    def handle_current_phase(self, phase: PhasesPipeline):
+        self.build_logger = TriggerBuildLogger(
+            self.python_logger,
+            prefix="" if self.json_logs else phase.build_phase.description,
+            build_phase=phase.build_phase,
+        )
+        yield
+
+    def handle_phase_in_progress(self, build_phase: BuildPhase):
+        logger = self.build_logger or self.python_logger
+        logger.debug(f"Build phase in progress: {build_phase.name}")
+
+    def handle_phase_finished_successfully(
+        self, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        logger = self.build_logger or self.python_logger
+        logger.debug(
+            f"Phase successfully finished: {build_phase.name} after {duration_in_seconds} seconds"
+        )
+
+    def _report_failure(self, build_phase: BuildPhase, duration_in_seconds: int):
+        logger = self.build_logger or self.python_logger
+        logger.debug(
+            self.BUILD_FAILURE_FORMAT.format(build_phase.name, duration_in_seconds)
+        )
+
+    def handle_contact_support_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        print(
+            FAILED_CONTACT_QWAK_SUPPORT_PROGRAMMATIC.format(
+                build_id=build_id,
+            )
+        )
+        self._report_failure(build_phase, duration_in_seconds)
+        raise QwakException(str(ex))
+
+    def handle_keyboard_interrupt(
+        self, build_id: str, build_phase: BuildPhase, duration_in_seconds: int
+    ):
+        self._report_failure(build_phase, duration_in_seconds)
+        raise QwakException("KeyboardInterrupt")
+
+    def handle_pipeline_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        self._report_failure(build_phase, duration_in_seconds)
+        raise QwakException(str(ex))
+
+    def handle_pipeline_quiet_exception(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        self._report_failure(build_phase, duration_in_seconds)
+
+    def handle_remote_build_error(
+        self,
+        build_id: str,
+        build_phase: BuildPhase,
+        ex: BaseException,
+        duration_in_seconds: int,
+    ):
+        self.handle_pipeline_exception(build_id, build_phase, ex, duration_in_seconds)

qwak/inner/build_logic/tools/dependencies_tools.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from qwak.inner.build_logic.dependency_manager_type import DependencyManagerType
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+
+@dataclass
+class DependencyFileObject:
+    dep_file_name: List[str]
+    lock_file_name: str = field(default="")
+
+
+DEPS_MANAGER_FILE_MAP = {
+    DependencyManagerType.PIP: DependencyFileObject(dep_file_name=["requirements.txt"]),
+    DependencyManagerType.POETRY: DependencyFileObject(
+        dep_file_name=["pyproject.toml"], lock_file_name="poetry.lock"
+    ),
+    DependencyManagerType.CONDA: DependencyFileObject(
+        dep_file_name=["conda.yml", "conda.yaml"]
+    ),
+}
+
+
+def find_dependency_files(model_uri, main_dir, build_logger: BuildLogger):
+    dependency_manager_type = None
+    model_relative_dependency_file = None
+    model_relative_dependency_lock_file = None
+    for dep_type, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        for filename in dependency_file_object.dep_file_name:
+            dep_file_path = find_file_location(model_uri, main_dir, filename)
+            if dep_file_path:
+                build_logger.info(
+                    f"Found dependency type: {dep_type.name} by file: {dep_file_path}"
+                )
+                dependency_manager_type = dep_type
+                model_relative_dependency_file = dep_file_path.relative_to(model_uri)
+                if dependency_file_object.lock_file_name:
+                    dep_lock_file_path = find_file_location(
+                        model_uri,
+                        main_dir,
+                        dependency_file_object.lock_file_name,
+                    )
+                    if dep_lock_file_path:
+                        build_logger.info(
+                            f"Found dependency lock file {dep_lock_file_path}"
+                        )
+                        model_relative_dependency_lock_file = (
+                            dep_lock_file_path.relative_to(model_uri)
+                        )
+                break
+
+    return (
+        dependency_manager_type,
+        model_relative_dependency_file,
+        model_relative_dependency_lock_file,
+    )
+
+
+def find_file_location(model_uri, main_dir, filename) -> Path:
+    file_locations: List[Path] = [
+        model_uri / filename,
+        model_uri / main_dir / filename,
+    ]
+    for file in file_locations:
+        if file.is_file():
+            return file

qwak/inner/build_logic/tools/files.py

@@ -0,0 +1,237 @@
+import fnmatch
+import os
+import re
+import shutil
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+from qwak.exceptions import QwakGeneralBuildException
+from qwak.inner.build_logic.constants.temp_dir import TEMP_LOCAL_MODEL_DIR
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+from qwak.inner.build_logic.interface.step_inteface import Step
+from qwak.inner.build_logic.tools.dependencies_tools import DEPS_MANAGER_FILE_MAP
+
+# Hidden directories setting
+HIDDEN_FILES_PREFIX = "."
+HIDDEN_DIRS_TO_INCLUDE = [".dvc"]
+IGNORED_PATTERNS_FOR_UPLOAD = [r"\..*", r"__pycache__"]
+QWAK_IGNORE_FILE_NAME = ".qwakignore"
+
+
+def copytree(
+    src,
+    dst,
+    symlinks=False,
+    ignore=None,
+    ignore_dangling_symlinks=False,
+    dirs_exist_ok=False,
+):
+    names = os.listdir(src)
+    if ignore is not None:
+        ignored_names = ignore(src, names)
+    else:
+        ignored_names = set()
+
+    os.makedirs(dst, exist_ok=dirs_exist_ok)
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if os.path.islink(srcname):
+                linkto = os.readlink(srcname)
+                if symlinks:
+                    # We can't just leave it to `copy_function` because legacy
+                    # code with a custom `copy_function` may rely on copytree
+                    # doing the right thing.
+                    os.symlink(linkto, dstname)
+                    shutil.copystat(srcname, dstname, follow_symlinks=not symlinks)
+                else:
+                    # ignore dangling symlink if the flag is on
+                    if not os.path.exists(linkto) and ignore_dangling_symlinks:
+                        continue
+                    # otherwise let the copy occurs. copy2 will raise an error
+                    if os.path.isdir(srcname):
+                        copytree(srcname, dstname, symlinks, ignore)
+                    else:
+                        shutil.copy2(srcname, dstname)
+            elif os.path.isdir(srcname):
+                copytree(srcname, dstname, symlinks, ignore)
+            else:
+                # Will raise a SpecialFileError for unsupported file types
+                shutil.copy2(srcname, dstname)
+        # catch the Error from the recursive copytree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        shutil.copystat(src, dst)
+    except OSError as why:
+        # Copying file access times may fail on Windows
+        if getattr(why, "winerror", None) is None:
+            errors.append((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+    return dst
+
+
+def get_possible_dependency_lock_paths(dependency_path: Path):
+    paths = []
+    for _, dependency_file_object in DEPS_MANAGER_FILE_MAP.items():
+        if dependency_file_object.lock_file_name:
+            lock_file_path = (
+                dependency_path.parent / dependency_file_object.lock_file_name
+            )
+            paths.append(lock_file_path)
+    return paths
+
+
+def get_files_to_ignore(directory: Path, patterns: Iterable[str] = ()):
+    def ignore_hidden(file: Path, exclusions: List[str]):
+        name = os.path.basename(os.path.abspath(file))
+        is_hidden = name.startswith(HIDDEN_FILES_PREFIX) and (
+            name != QWAK_IGNORE_FILE_NAME and name not in exclusions
+        )
+        return is_hidden
+
+    def is_ignore_by_pattern(file: Path):
+        return (
+            len(
+                [
+                    pattern
+                    for pattern in patterns
+                    if re.search(fnmatch.translate(pattern), str(file))
+                ]
+            )
+            != 0
+        )
+
+    return [
+        file.name
+        for file in Path(directory).rglob("*")
+        if is_ignore_by_pattern(file)
+        or ignore_hidden(file, exclusions=HIDDEN_DIRS_TO_INCLUDE)
+    ]
+
+
+def _replace_large_files_with_too_large_file_message(
+    filtered_model: Path, max_bytes: Optional[int]
+):
+    def does_exceed_size(file: Path):
+        file_size = file.lstat().st_size
+        return file_size > max_bytes, file.lstat().st_size
+
+    if max_bytes is None:
+        return
+
+    for root, dirs, files in os.walk(filtered_model):
+        for file in files:
+            file_path = Path(os.path.join(root, file))
+            replace_content, file_size = does_exceed_size(file_path)
+            if replace_content:
+                Path(file_path).write_text(
+                    f"File is too big to display. Size: {file_size} bytes"
+                )
+
+
+def zip_model(
+    build_dir: Path,
+    dependency_file: Path,
+    dirs_to_include: List[str],
+    zip_name: str,
+    ignored_patterns: Iterable[str],
+    max_bytes: Optional[int] = None,
+    deps_lock_file: Optional[Path] = None,
+):
+    try:
+        filtered_model = build_dir / zip_name
+        ignored_patterns = get_files_to_ignore(
+            directory=build_dir / TEMP_LOCAL_MODEL_DIR, patterns=ignored_patterns
+        )
+
+        for included_dir in dirs_to_include:
+            dir_to_copy = build_dir / TEMP_LOCAL_MODEL_DIR / included_dir
+            if dir_to_copy.is_dir():
+                copytree(
+                    src=build_dir / TEMP_LOCAL_MODEL_DIR / included_dir,
+                    dst=filtered_model / included_dir,
+                    ignore=shutil.ignore_patterns(*ignored_patterns),
+                    dirs_exist_ok=True,
+                )
+
+        deps_file = build_dir / TEMP_LOCAL_MODEL_DIR / dependency_file
+        shutil.copy(deps_file, filtered_model / dependency_file)
+
+        if deps_lock_file:
+            deps_lock_file_full_path = build_dir / TEMP_LOCAL_MODEL_DIR / deps_lock_file
+            shutil.copy(deps_lock_file_full_path, filtered_model / deps_lock_file)
+
+        _replace_large_files_with_too_large_file_message(filtered_model, max_bytes)
+
+        zip_path = Path(
+            shutil.make_archive(
+                base_name=str(filtered_model),
+                format="zip",
+                root_dir=filtered_model,
+            )
+        )
+
+        shutil.rmtree(filtered_model)
+        return zip_path
+
+    except Exception as e:
+        raise QwakGeneralBuildException(
+            message="Unable to zip model before upload",
+            src_exception=e,
+        )
+
+
+class UploadInChunks(object):
+    def __init__(
+        self,
+        file: Path,
+        build_logger: BuildLogger,
+        all_files_size_to_upload: int,
+        read_so_far: int = 0,
+        chunk_size_bytes: int = 1 << 13,
+    ):
+        self._file = file
+        self._chunk_size = chunk_size_bytes
+        self._total_size = self._file.stat().st_size
+        self._read_so_far = read_so_far
+        self._build_logger = build_logger
+        self._all_files_size_to_upload = (
+            all_files_size_to_upload  # Used for calculating percentage for both files
+        )
+        self._last_percent_update = 0
+
+    def __iter__(self):
+        with self._file.open("rb") as file:
+            while True:
+                data = file.read(self._chunk_size)
+                if not data:
+                    break
+                self._read_so_far += len(data)
+                percent = self._read_so_far * 1e2 / self._all_files_size_to_upload
+                msg = "{percent:3.0f}%".format(percent=percent)
+                if int(percent / 10) > self._last_percent_update:
+                    self._build_logger.info(
+                        msg
+                    )  # Updating only after 10 percent change
+                    self._last_percent_update = int(percent / 10)
+                if hasattr(self._build_logger, "spinner_text"):
+                    self._build_logger.spinner_text(line=msg)
+                yield data
+
+    def __len__(self):
+        return self._total_size
+
+
+def cleaning_up_after_build(step: Step):
+    if os.getenv("QWAK_DEBUG") != "true":
+        step.build_logger.debug("Removing Qwak temp artifacts directory")
+        shutil.rmtree(step.context.host_temp_local_build_dir, ignore_errors=True)

qwak/inner/build_logic/tools/ignore_files.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+
+
+def load_patterns_from_ignore_file(build_logger: BuildLogger, ignore_file_path: Path):
+    if Path(ignore_file_path).is_file():
+        build_logger.info("Found a Qwak ignore file - will ignore listed patterns")
+
+        with open(ignore_file_path, "r") as igonre_file:
+            patterns_to_ignore = [
+                pattern.strip() for pattern in igonre_file.readlines()
+            ]
+            build_logger.debug(
+                f"Patterns from Qwak igonre file detected - {str(patterns_to_ignore)}"
+            )
+            return patterns_to_ignore
+
+    build_logger.debug("no Qwak ignore file was found, skipping")
+    return []

qwak/inner/build_logic/tools/text.py

@@ -0,0 +1,9 @@
+from re import sub
+
+
+def snake_case(s):
+    return "_".join(
+        sub(
+            "([A-Z][a-z]+)", r" \1", sub("([A-Z]+)", r" \1", s.replace("-", " "))
+        ).split()
+    ).lower()

qwak/inner/build_logic/trigger_build_context.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from qwak.inner.build_logic.interface.context_interface import Context
+
+
+@dataclass
+class TriggerBuildContext(Context):
+    pass

qwak/inner/const.py

@@ -33,8 +33,6 @@ class QwakConstants:
 
     TOKEN_AUDIENCE: str = "https://auth-token.qwak.ai/"  # nosec B105
 
-    QWAK_AUTHENTICATION_URL = "https://grpc.qwak.ai/api/v1/authentication/qwak-api-key"
+    QWAK_APP_URL: str = "https://app.qwak.ai"
 
-    QWAK_AUTHENTICATED_USER_ENDPOINT: str = (
-        "https://grpc.qwak.ai/api/v0/runtime/get-authenticated-user-context"
-    )
+    CONTROL_PLANE_GRPC_ADDRESS_ENVAR_NAME: str = "CONTROL_PLANE_GRPC_ADDRESS"

qwak/inner/di_configuration/__init__.py

@@ -1,65 +1,3 @@
-import os
-
 from .account import UserAccountConfiguration
 from .containers import QwakContainer
-
-
-def wire_dependencies():
-    container = QwakContainer()
-
-    default_config_file = os.path.join(os.path.dirname(__file__), "config.yml")
-    container.config.from_yaml(default_config_file)
-
-    from qwak.clients import (
-        administration,
-        alert_management,
-        alerts_registry,
-        analytics,
-        audience,
-        automation_management,
-        autoscaling,
-        batch_job_management,
-        build_management,
-        build_orchestrator,
-        data_versioning,
-        deployment,
-        feature_store,
-        file_versioning,
-        instance_template,
-        kube_deployment_captain,
-        logging_client,
-        model_management,
-        project,
-        user_application_instance,
-        vector_store,
-        workspace_manager,
-    )
-
-    container.wire(
-        packages=[
-            administration,
-            alert_management,
-            audience,
-            automation_management,
-            autoscaling,
-            analytics,
-            batch_job_management,
-            build_management,
-            build_orchestrator,
-            data_versioning,
-            deployment,
-            file_versioning,
-            instance_template,
-            kube_deployment_captain,
-            logging_client,
-            model_management,
-            project,
-            feature_store,
-            user_application_instance,
-            alerts_registry,
-            workspace_manager,
-            vector_store,
-        ]
-    )
-
-    return container
+from .dependency_wiring import wire_dependencies

qwak/inner/di_configuration/account.py

@@ -2,7 +2,7 @@ import configparser
 import errno
 import os
 from dataclasses import dataclass
-from typing import Optional
+from typing import Optional, Type
 
 from qwak.exceptions import QwakLoginException
 from qwak.inner.const import QwakConstants
@@ -17,7 +17,7 @@ class UserAccount:
     """
 
     # Assigned API Key
-    api_key: str
+    api_key: Optional[str] = None
 
     # Assigned username
     username: Optional[str] = None
@@ -30,23 +30,24 @@ class UserAccountConfiguration:
 
     def __init__(
         self,
-        config_file=QwakConstants.QWAK_CONFIG_FILE,
-        auth_file=QwakConstants.QWAK_AUTHORIZATION_FILE,
-        auth_client=Auth0ClientBase,
+        config_file: str = QwakConstants.QWAK_CONFIG_FILE,
+        auth_file: str = QwakConstants.QWAK_AUTHORIZATION_FILE,
+        auth_client: Type[Auth0ClientBase] = Auth0ClientBase,
     ):
         self._config_file = config_file
         self._auth_file = auth_file
         self._config = configparser.ConfigParser()
         self._auth = configparser.ConfigParser()
-        self._auth_client = auth_client
         self._environment = Session().get_environment()
+        self._auth_client = auth_client
 
     def configure_user(self, user_account: UserAccount):
         """
-        Write user account to the given config file in an ini format. Configuration will be written under the 'default'
-        section
-        :param user_account: user account properties to be written
+        Configure user authentication based on the authentication client type
         """
+        self.__qwak_login(user_account)
+
+    def __qwak_login(self, user_account: UserAccount):
         self._auth.read(self._auth_file)
         self._auth.remove_section(self._environment)
         with self._safe_open(self._auth_file) as authfile:
@@ -57,8 +58,8 @@ class UserAccountConfiguration:
             auth_file=self._auth_file,
         ).login()
 
+        # Store configuration only for Qwak auth
         self._config.read(self._config_file)
-
         with self._safe_open(self._config_file) as configfile:
             self._config[self._environment] = {}
             if user_account.username:
@@ -87,6 +88,10 @@ class UserAccountConfiguration:
         try:
             username = os.environ.get("QWAK_USERNAME")
             api_key = os.environ.get("QWAK_API_KEY")
+            if not api_key and (
+                len(self._environment) >= 60 and "@" in self._environment
+            ):
+                api_key = self._environment
             if api_key:
                 Session().set_environment(api_key)
                 return UserAccount(username=username, api_key=api_key)
@@ -104,13 +109,13 @@ class UserAccountConfiguration:
         except FileNotFoundError:
             raise QwakLoginException(
                 f"Could not read user configuration from {self._config_file}. "
-                f"Please make sure one has been set using `qwak configure` command"
+                f"Please ensure it is configured using `qwak configure` command"
             )
 
         except configparser.NoSectionError:
             raise QwakLoginException(
-                f"Could not read user configuration from {self._config_file}. "
-                f"Please make sure one has been set using `qwak configure` command"
+                f"Environment {self._environment} has not be configured."
+                f"Please ensure it is configured using the `qwak configure` command."
             )
 
     def get_user_apikey(self) -> str:
@@ -137,6 +142,18 @@ class UserAccountConfiguration:
 
         except configparser.NoSectionError:
             raise QwakLoginException(
-                f"Could not read user configuration from {self._config_file}. "
-                f"Please make sure one has been set using `qwak configure` command"
+                f"Environment {self._environment} has not be configured."
+                f"Please ensure it is configured using the `qwak configure` command."
             )
+
+    def retrieve_platform_url(self) -> str:
+        """
+        Retrieve the platform URL based on the configured authentication client.
+
+        Returns:
+            str: The platform URL.
+        """
+        auth_client_instance = self._auth_client()
+        base_url = auth_client_instance.get_base_url()
+
+        return base_url

qwak/inner/di_configuration/config.yml

@@ -5,3 +5,5 @@ grpc:
     builds:
         internal_address: 'builds-orchestrator-webapp.default.svc.cluster.local:6577'
         enable_ssl: False
+    authentication:
+        enable_ssl: True

qwak/inner/di_configuration/containers.py

@@ -19,12 +19,17 @@ class QwakContainer(containers.DeclarativeContainer):
         create_grpc_channel,
         url=config.grpc.core.address,
         enable_ssl=config.grpc.core.enable_ssl,
+        status_for_retry=(
+            grpc.StatusCode.UNAVAILABLE,
+            grpc.StatusCode.CANCELLED,
+            grpc.StatusCode.DEADLINE_EXCEEDED,
+        ),
     )
 
     unauthenticated_core_grpc_channel = providers.Singleton(
         create_grpc_channel,
         url=config.grpc.core.address,
-        enable_ssl=True,
+        enable_ssl=config.grpc.authentication.enable_ssl,
         enable_auth=False,
     )
 
@@ -33,5 +38,8 @@ class QwakContainer(containers.DeclarativeContainer):
         url=config.grpc.builds.internal_address,
         enable_ssl=config.grpc.builds.enable_ssl,
         enable_auth=False,
-        status_for_retry=(grpc.StatusCode.UNAVAILABLE,),
+        status_for_retry=(
+            grpc.StatusCode.UNAVAILABLE,
+            grpc.StatusCode.DEADLINE_EXCEEDED,
+        ),
     )