qwak-core 0.1.37__py3-none-any.whl → 0.5.19__py3-none-any.whl

qwak/feature_store/validations/validation_response.py

@@ -0,0 +1,119 @@
+import ast
+from abc import ABC
+from dataclasses import dataclass
+from io import StringIO
+from typing import TYPE_CHECKING, List
+
+from _qwak_proto.qwak.feature_store.features.feature_set_pb2 import (
+    Feature as ProtoFeature,
+)
+from _qwak_proto.qwak.features_operator.v3.features_operator_async_service_pb2 import (
+    GetValidationResultResponse as ProtoGetValidationResultResponse,
+)
+from _qwak_proto.qwak.features_operator.v3.features_operator_pb2 import (
+    SparkColumnDescription as ProtoSparkColumnDescription,
+    ValidationFailureResponse as ProtoValidationFailureResponse,
+    ValidationSuccessResponse as ProtoValidationSuccessResponse,
+)
+from qwak.exceptions import QwakException
+
+if TYPE_CHECKING:
+    try:
+        import pandas as pd
+    except ImportError:
+        pass
+
+_VALIDATION_RESPONSE_TYPE_SUCCESS = "success_response"
+_VALIDATION_RESPONSE_TYPE_FAILURE = "failure_response"
+_VALIDATION_RESPONSE_TYPE_TIMEOUT = "not_ready_response"
+
+
+class ValidationResponse(ABC):
+    pass
+
+
+@dataclass
+class TimeoutValidationResponse(ValidationResponse):
+    pass
+
+
+@dataclass
+class FailureValidationResponse(ValidationResponse):
+    phase: str
+    message: str
+    stdout: str
+    stderr: str
+
+
+@dataclass
+class SuccessValidationResponse(ValidationResponse):
+    sample: "pd.DataFrame"
+    features: List[ProtoFeature]
+    stdout: str
+    stderr: str
+
+
+class ValidationResponseFactory:
+    @staticmethod
+    def _get_features(
+        spark_columns_description: List[ProtoSparkColumnDescription],
+    ) -> List[ProtoFeature]:
+        spark_features = []
+        for spark_column in spark_columns_description:
+            feature_col_name_list = str(spark_column.column_name).split(".")
+            feature_name = (
+                feature_col_name_list[1]
+                if feature_col_name_list.__len__() == 2
+                else spark_column.column_name
+            )
+            spark_features.append(
+                ProtoFeature(
+                    feature_name=feature_name,
+                    feature_type=spark_column.spark_type,
+                )
+            )
+        return spark_features
+
+    @staticmethod
+    def from_proto(
+        validation_response: ProtoGetValidationResultResponse,
+    ) -> ValidationResponse:
+        try:
+            import pandas as pd
+        except ImportError:
+            raise QwakException("Missing required Pandas dependency")
+
+        validation_type: str = validation_response.WhichOneof("type")
+        if validation_type == _VALIDATION_RESPONSE_TYPE_SUCCESS:
+            success: ProtoValidationSuccessResponse = (
+                validation_response.success_response
+            )
+
+            pd_sample: pd.DataFrame = pd.read_json(
+                path_or_buf=StringIO(ast.literal_eval(success.sample)),
+                dtype=success.spark_column_description,
+            )
+
+            features = ValidationResponseFactory._get_features(
+                success.spark_column_description
+            )
+            return SuccessValidationResponse(
+                sample=pd_sample,
+                features=features,
+                stdout=success.outputs.stdout,
+                stderr=success.outputs.stderr,
+            )
+        elif validation_type == _VALIDATION_RESPONSE_TYPE_FAILURE:
+            failure: ProtoValidationFailureResponse = (
+                validation_response.failure_response
+            )
+            return FailureValidationResponse(
+                phase=failure.phase,
+                message=failure.error_message,
+                stdout=failure.outputs.stdout,
+                stderr=failure.outputs.stderr,
+            )
+        elif validation_type == _VALIDATION_RESPONSE_TYPE_TIMEOUT:
+            return TimeoutValidationResponse()
+
+        raise QwakException(f"Got unsupported response type: {validation_type}")

qwak/feature_store/validations/validator.py

@@ -0,0 +1,114 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple
+
+from _qwak_proto.qwak.features_operator.v3.features_operator_async_service_pb2 import (
+    GetValidationResultResponse as ProtoGetValidationResultResponse,
+)
+from qwak.clients.feature_store import FeatureRegistryClient
+from qwak.clients.feature_store.operator_client import FeaturesOperatorClient
+from qwak.exceptions import QwakException
+from qwak.feature_store.data_sources.base import BaseSource
+from qwak.feature_store.feature_sets.base_feature_set import BaseFeatureSet
+from qwak.feature_store.validations.validation_decorators import (
+    silence_backend_specific_validation_exceptions,
+)
+from qwak.feature_store.validations.validation_options import (
+    DataSourceValidationOptions,
+    FeatureSetValidationOptions,
+)
+from qwak.feature_store.validations.validation_response import (
+    ValidationResponse,
+    ValidationResponseFactory,
+)
+
+
+class Validator(ABC):
+    @abstractmethod
+    def validate_featureset(
+        self,
+        featureset: BaseFeatureSet,
+        sample_size: int = 10,
+        validation_options: Optional[FeatureSetValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        pass
+
+    @abstractmethod
+    def validate_data_source(
+        self,
+        data_source: BaseSource,
+        sample_size: int = 10,
+        validation_options: Optional[DataSourceValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        pass
+
+
+class FeaturesOperatorValidator(Validator):
+    _operator_client: FeaturesOperatorClient
+    _registry_client: FeatureRegistryClient
+
+    def __init__(self):
+        try:
+            import pandas  # noqa: F401
+        except ImportError:
+            raise QwakException("Missing required Pandas dependency")
+
+        self._operator_client = FeaturesOperatorClient()
+        self._registry_client = FeatureRegistryClient()
+
+    @silence_backend_specific_validation_exceptions(
+        "Validating DataSource is not supported for self-hosted environments"
+    )
+    def validate_data_source(
+        self,
+        data_source: BaseSource,
+        sample_size: int = 10,
+        validation_options: Optional[DataSourceValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        if sample_size <= 0 or 1_000 < sample_size:
+            raise ValueError(
+                f"sample_size must be under 1000 and positive, got: {sample_size}"
+            )
+        artifact_url: Optional[str] = None
+        data_source_spec, artifact_url = data_source._prepare_and_get()
+        proto_response: ProtoGetValidationResultResponse = (
+            self._operator_client.validate_data_source_blocking(
+                data_source_spec=data_source_spec,
+                num_samples=sample_size,
+                validation_options=(
+                    validation_options.to_proto() if validation_options else None
+                ),
+            )
+        )
+
+        return ValidationResponseFactory.from_proto(proto_response), artifact_url
+
+    @silence_backend_specific_validation_exceptions(
+        "Validating FeatureSet is not supported for self-hosted environments"
+    )
+    def validate_featureset(
+        self,
+        featureset: BaseFeatureSet,
+        sample_size: int = 10,
+        validation_options: Optional[FeatureSetValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        if sample_size <= 0 or 1_000 < sample_size:
+            raise ValueError(
+                f"sample_size must be under 1000 and positive, got: {sample_size}"
+            )
+        artifact_url: Optional[str] = None
+        featureset_spec, artifact_url = featureset._to_proto(
+            feature_registry=self._registry_client, features=None, git_commit=None
+        )
+
+        proto_response: ProtoGetValidationResultResponse = (
+            self._operator_client.validate_featureset_blocking(
+                featureset_spec=featureset_spec,
+                resource_path=artifact_url,
+                num_samples=sample_size,
+                validation_options=(
+                    validation_options.to_proto() if validation_options else None
+                ),
+            )
+        )
+
+        return ValidationResponseFactory.from_proto(proto_response), artifact_url

qwak/inner/build_config/build_config_v1.py

@@ -0,0 +1,311 @@
+from dataclasses import dataclass, field
+from os.path import expandvars
+from typing import Dict, List, Optional, Tuple, Union
+
+import grpc
+from _qwak_proto.qwak.builds.build_pb2 import BaseDockerImageType
+from _qwak_proto.qwak.builds.builds_pb2 import (
+    BuildConfiguration,
+    BuildEnvironment,
+    BuildModelUri,
+    BuildPropertiesProto,
+    CondaBuild,
+    DockerBuild,
+    PoetryBuild,
+    PythonBuild,
+    RemoteBuild,
+    RemoteBuildResources as RemoteBuildResourcesProto,
+    Step,
+    VirtualEnvironmentBuild,
+)
+from _qwak_proto.qwak.instance_template.instance_template_pb2 import InstanceType
+from qwak.clients.build_orchestrator import BuildOrchestratorClient
+from qwak.clients.instance_template.client import InstanceTemplateManagementClient
+from qwak.exceptions import QwakException
+from qwak.inner.tool.protobuf_factory import protobuf_factory
+from qwak.inner.tool.run_config import (
+    ConfigCliMap,
+    QwakConfigBase,
+    YamlConfigMixin,
+    validate_bool,
+    validate_float,
+    validate_int,
+    validate_list_of_strings,
+    validate_string,
+)
+from qwak.inner.tool.run_config.utils import (
+    validate_positive_int,
+    validate_purchase_option,
+)
+from qwak.model.base import QwakModel
+
+CONFIG_MAPPING: List[ConfigCliMap] = [
+    # ConfigV1
+    ConfigCliMap(
+        "validate_build_artifact", "step.validate_build_artifact", validate_bool, True
+    ),
+    ConfigCliMap(
+        "validate_build_artifact_timeout",
+        "step.validate_build_artifact_timeout",
+        validate_positive_int,
+        True,
+    ),
+    ConfigCliMap("tests", "step.tests", validate_bool, True),
+    # BuildProperties
+    ConfigCliMap("model_id", "build_properties.model_id", validate_string, True),
+    ConfigCliMap("build_id", "build_properties.build_id", validate_string, False),
+    ConfigCliMap(
+        "gpu_compatible", "build_properties.gpu_compatible", validate_bool, True
+    ),
+    ConfigCliMap("branch", "build_properties.branch", validate_string),
+    ConfigCliMap("tags", "build_properties.tags", validate_list_of_strings),
+    ConfigCliMap("build_name", "build_properties.build_name", validate_string),
+    # ModelUri
+    ConfigCliMap("uri", "build_properties.model_uri.uri", validate_string, False),
+    ConfigCliMap(
+        "git_branch", "build_properties.model_uri.git_branch", validate_string
+    ),
+    ConfigCliMap("main_dir", "build_properties.model_uri.main_dir", validate_string),
+    ConfigCliMap(
+        "git_credentials", "build_properties.model_uri.git_credentials", validate_string
+    ),
+    ConfigCliMap(
+        "git_secret_ssh", "build_properties.model_uri.git_secret_ssh", validate_string
+    ),
+    ConfigCliMap(
+        "git_credentials_secret",
+        "build_properties.model_uri.git_credentials_secret",
+        validate_string,
+    ),
+    ConfigCliMap(
+        "dependency_required_folders",
+        "build_properties.model_uri.dependency_required_folders",
+        validate_list_of_strings,
+    ),
+    # BuildEnv
+    # PythonEnv
+    ConfigCliMap(
+        "dependency_file_path",
+        "build_env.python_env.dependency_file_path",
+        validate_string,
+    ),
+    # DockerConf
+    ConfigCliMap("base_image", "build_env.docker.base_image", validate_string),
+    ConfigCliMap("param_list", "build_env.docker.params", validate_list_of_strings),
+    ConfigCliMap("env_vars", "build_env.docker.env_vars", validate_list_of_strings),
+    ConfigCliMap("cache", "build_env.docker.cache", validate_bool),
+    ConfigCliMap("push", "build_env.docker.push", validate_bool),
+    ConfigCliMap(
+        "iam_role_arn", "build_env.docker.assumed_iam_role_arn", validate_string
+    ),
+    ConfigCliMap(
+        "service_account_key_secret_name",
+        "build_env.docker.service_account_key_secret_name",
+        validate_string,
+    ),
+    # RemoteConf
+    ConfigCliMap("cpus", "build_env.remote.resources.cpus", validate_float),
+    ConfigCliMap("memory", "build_env.remote.resources.memory", validate_string),
+    ConfigCliMap("gpu_type", "build_env.remote.resources.gpu_type", validate_string),
+    ConfigCliMap(
+        "gpu_amount", "build_env.remote.resources.gpu_amount", validate_positive_int
+    ),
+    ConfigCliMap("instance", "build_env.remote.resources.instance", validate_string),
+    # Verbosity level
+    ConfigCliMap("verbose", "verbose", validate_int),
+    ConfigCliMap("deploy", "deploy", validate_bool),
+    ConfigCliMap("deployment_instance", "deployment_instance", validate_string),
+    ConfigCliMap("purchase_option", "purchase_option", validate_purchase_option),
+    ConfigCliMap(
+        "provision_instance_timeout",
+        "provision_instance_timeout",
+        validate_positive_int,
+    ),
+]
+
+
+@protobuf_factory(Step)
+@dataclass
+class Step:
+    validate_build_artifact: bool = field(default=True)
+    validate_build_artifact_timeout: int = field(default=120)
+    tests: bool = field(default=True)
+
+
+@protobuf_factory(
+    BuildModelUri,
+    exclude_fields=[
+        "git_credentials",
+        "git_credentials_secret",
+        "dependency_required_folders",
+    ],
+)
+@dataclass
+class ModelUri:
+    uri: Optional[str] = field(default=None)
+    git_branch: Optional[str] = field(default="master")
+    main_dir: Optional[str] = field(default="main")
+    dependency_required_folders: List[str] = field(default_factory=list)
+    git_credentials: Optional[str] = field(default=None)
+    git_credentials_secret: Optional[str] = field(default=None)
+    git_secret_ssh: Optional[str] = field(default=None)
+
+
+@protobuf_factory(BuildPropertiesProto)
+@dataclass
+class BuildProperties:
+    build_id: Optional[str] = field(default=None)
+    gpu_compatible: bool = field(default=False)
+    branch: Optional[str] = field(default="main")
+    model_id: str = field(default="")
+    tags: List[str] = field(default_factory=list)
+    model_uri: ModelUri = field(default_factory=ModelUri)
+    build_name: Optional[str] = field(default="")
+
+
+@protobuf_factory(DockerBuild)
+@dataclass
+class DockerConf:
+    base_image: Optional[str] = field(default=None)
+    env_vars: Optional[Union[List[str], Dict[str, str], Tuple]] = field(
+        default_factory=list
+    )
+    assumed_iam_role_arn: str = field(default=None)
+    service_account_key_secret_name: str = field(default=None)
+    params: Optional[List[str]] = field(default_factory=list)
+    no_cache: bool = field(default=False)
+    cache: bool = field(default=True)
+    push: bool = field(default=True)
+
+    def __post_init__(self):
+        if isinstance(self.env_vars, (list, tuple)):
+            self.env_vars = [expandvars(var) for var in self.env_vars]
+
+
+@protobuf_factory(VirtualEnvironmentBuild)
+@dataclass
+class VirtualenvType:
+    python_version: Optional[str] = field(default=None)
+    requirements_txt: Optional[str] = field(default=None)
+
+
+@protobuf_factory(PoetryBuild)
+@dataclass
+class PoetryType:
+    python_version: Optional[str] = field(default=None)
+    lock_file: Optional[str] = field(default=None)
+
+
+@protobuf_factory(CondaBuild)
+@dataclass
+class CondaType:
+    conda_file: Optional[str] = field(default=None)
+
+
+@protobuf_factory(
+    PythonBuild, exclude_fields=["git_credentials_secret", "git_credentials"]
+)
+@dataclass
+class PythonEnv:
+    git_credentials: Optional[str] = field(default=None)
+    git_credentials_secret: Optional[str] = field(default=None)
+    poetry: PoetryType = field(default=None)
+    conda: CondaType = field(default=None)
+    virtualenv: VirtualenvType = field(default=None)
+    dependency_file_path: Optional[str] = field(default=None)
+
+    def __post_init__(self):
+        if not (self.poetry or self.conda or self.virtualenv):
+            self.conda = CondaType("conda.yml")
+
+
+@protobuf_factory(RemoteBuildResourcesProto)
+@dataclass
+class RemoteBuildResources:
+    cpus: Optional[float] = field(default=None)
+    memory: Optional[str] = field(default=None)
+    gpu_type: Optional[str] = field(default=None)
+    gpu_amount: Optional[int] = field(default=None)
+    instance: Optional[str] = field(default=None)
+
+
+@protobuf_factory(RemoteBuild)
+@dataclass
+class RemoteConf:
+    is_remote: bool = field(default=True)
+    resources: RemoteBuildResources = field(default_factory=RemoteBuildResources)
+
+
+@protobuf_factory(BuildEnvironment)
+@dataclass
+class BuildEnv:
+    docker: DockerConf = field(default_factory=DockerConf)
+    python_env: PythonEnv = field(default_factory=PythonEnv)
+    remote: RemoteConf = field(default_factory=RemoteConf)
+
+
+@protobuf_factory(BuildConfiguration, exclude_fields=["deploy"])
+@dataclass
+class BuildConfigV1(YamlConfigMixin, QwakConfigBase):
+    @property
+    def _config_mapping(self) -> List[ConfigCliMap]:
+        return CONFIG_MAPPING
+
+    build_properties: BuildProperties = field(default_factory=BuildProperties)
+    build_env: Optional[BuildEnv] = field(default_factory=BuildEnv)
+    pre_build: Optional[BuildEnv] = field(default=None)
+    post_build: Optional[BuildEnv] = field(default=None)
+    step: Step = field(default_factory=Step)
+    deploy: bool = field(default=False)
+    deployment_instance: Optional[str] = field(default=None)
+    purchase_option: Optional[str] = field(default=None)
+    provision_instance_timeout: Optional[int] = field(default=None)
+    verbose: int = field(default=0)
+    pre_built_model: Optional[QwakModel] = field(default=None, init=False)
+
+    def _post_merge_cli(self):
+        """Implementing post merge actions."""
+        self.build_properties.model_id = self.build_properties.model_id.lower()
+        self.build_properties.tags = list(self.build_properties.tags)
+        self.build_properties.model_uri.dependency_required_folders = list(
+            self.build_properties.model_uri.dependency_required_folders
+        )
+        self.build_env.docker.env_vars = list(self.build_env.docker.env_vars)
+        self.build_env.docker.params = list(self.build_env.docker.params)
+
+    def fetch_base_docker_image_name(
+        self,
+        build_orchestrator: BuildOrchestratorClient,
+        instance_template: InstanceTemplateManagementClient,
+    ):
+        if not self.build_env.docker.base_image:
+            is_gpu_used = (
+                self.build_env.remote.resources.gpu_type
+                and self.build_env.remote.resources.gpu_amount
+                and self.build_env.remote.resources.gpu_amount > 0
+            )
+
+            if self.build_env.remote.resources.instance and not is_gpu_used:
+                is_gpu_used = self.deduce_gpu_from_template(instance_template)
+
+            if is_gpu_used or self.build_properties.gpu_compatible:
+                result = build_orchestrator.fetch_base_docker_image_name(
+                    BaseDockerImageType.GPU
+                )
+                self.build_env.docker.base_image = result.base_docker_image_name
+            else:
+                result = build_orchestrator.fetch_base_docker_image_name(
+                    BaseDockerImageType.CPU
+                )
+                self.build_env.docker.base_image = result.base_docker_image_name
+
+    def deduce_gpu_from_template(self, instance_template):
+        try:
+            instance_spec = instance_template.get_instance_template(
+                self.build_env.remote.resources.instance
+            )
+            return instance_spec.instance_type == InstanceType.INSTANCE_TYPE_GPU
+
+        except QwakException as e:
+            if not e.status_code or e.status_code != grpc.StatusCode.NOT_FOUND:
+                raise e

qwak/inner/build_logic/build_loggers/trigger_build_logger.py

@@ -0,0 +1,51 @@
+import logging
+from logging import Logger
+
+from qwak.inner.build_logic.interface.build_logger_interface import BuildLogger
+from qwak.inner.build_logic.interface.build_phase import BuildPhase
+
+
+PREFIX_FORMAT = "{prefix} - "
+EXCEPTION_FORMAT = """Message: {line}
+Exception: {e}
+"""
+
+
+class TriggerBuildLogger(BuildLogger):
+    def __init__(
+        self,
+        logger: Logger,
+        prefix: str,
+        build_phase: BuildPhase,
+        verbose: int = 0,
+        json_logs: bool = False,
+    ) -> None:
+        self.logger = logging.LoggerAdapter(
+            logger,
+            {
+                "phase": build_phase.description,
+                "phase_id": build_phase.phase_id,
+            },
+        )
+        self.prefix = PREFIX_FORMAT.format(prefix=prefix) if prefix else ""
+        self.spinner = None
+        self.verbose = verbose
+        self.json_logs = json_logs
+
+    def exception(self, line: str, e: BaseException) -> None:
+        self.logger.error(
+            EXCEPTION_FORMAT.format(line=line, e=e),
+            exc_info=False,
+        )
+
+    def error(self, line: str) -> None:
+        self.logger.error(f"{self.prefix}{line}")
+
+    def warning(self, line: str) -> None:
+        self.logger.warning(f"{self.prefix}{line}")
+
+    def info(self, line: str) -> None:
+        self.logger.info(f"{self.prefix}{line}")
+
+    def debug(self, line: str) -> None:
+        self.logger.debug(f"{self.prefix}{line}")

qwak/inner/build_logic/constants/dependencies.py

@@ -0,0 +1,5 @@
+PIP_DEPENDENCY_FILE = "requirements.txt"
+CONDA_YML_DEPENDENCY_FILE = "conda.yml"
+CONDA_YAML_DEPENDENCY_FILE = "conda.yaml"
+POETRY_DEPENDENCY_FILE = "pyproject.toml"
+POETRY_LOCK_DEPENDENCY_FILE = "poetry.lock"

qwak/inner/build_logic/constants/host_resource.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+HOST_QWAK_HIDDEN_FOLDER = Path.home() / ".qwak"
+HOST_TEMP_BUILD_DIR = HOST_QWAK_HIDDEN_FOLDER / "builds"

qwak/inner/build_logic/constants/messages.py

@@ -0,0 +1,3 @@
+FAILED_CONTACT_QWAK_SUPPORT_PROGRAMMATIC = """
+Build ID \033[4m{build_id}\033[0m failed!!
+"""

qwak/inner/build_logic/constants/temp_dir.py

@@ -0,0 +1 @@
+TEMP_LOCAL_MODEL_DIR = "model_code"

qwak/inner/build_logic/constants/upload_tag.py

@@ -0,0 +1,7 @@
+MODEL_CODE_TAG = "code.zip"
+SKINNY_MODEL_CODE_TAG = "skinny_code.zip"
+QWAK_SDK_VERSION_TAG = "qwak_sdk_version"
+QWAK_RUNTIME_WHEEL_TAG = "qwak_runtime_wheel"
+QWAK_CORE_WHEEL_TAG = "qwak_core_wheel"
+BUILD_CONFIG_TAG = "qwak_build_config"
+QWAK_BUILT_MODEL_TAG = "qwak_built_model"

qwak/inner/build_logic/dependency_manager_type.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class DependencyManagerType(Enum):
+    UNKNOWN = 0
+    PIP = 1
+    POETRY = 2
+    CONDA = 3