frogml-core 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

frogml_core/feature_store/validations/validation_response.py

@@ -0,0 +1,121 @@
+import ast
+from abc import ABC
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List
+
+from frogml_proto.qwak.feature_store.features.feature_set_pb2 import (
+    Feature as ProtoFeature,
+)
+from frogml_proto.qwak.features_operator.v3.features_operator_async_service_pb2 import (
+    GetValidationResultResponse as ProtoGetValidationResultResponse,
+)
+from frogml_proto.qwak.features_operator.v3.features_operator_pb2 import (
+    SparkColumnDescription as ProtoSparkColumnDescription,
+)
+from frogml_proto.qwak.features_operator.v3.features_operator_pb2 import (
+    ValidationFailureResponse as ProtoValidationFailureResponse,
+)
+from frogml_proto.qwak.features_operator.v3.features_operator_pb2 import (
+    ValidationSuccessResponse as ProtoValidationSuccessResponse,
+)
+from frogml_core.exceptions import FrogmlException
+
+if TYPE_CHECKING:
+    try:
+        import pandas as pd
+    except ImportError:
+        pass
+
+_VALIDATION_RESPONSE_TYPE_SUCCESS = "success_response"
+_VALIDATION_RESPONSE_TYPE_FAILURE = "failure_response"
+_VALIDATION_RESPONSE_TYPE_TIMEOUT = "not_ready_response"
+
+
+class ValidationResponse(ABC):
+    pass
+
+
+@dataclass
+class TimeoutValidationResponse(ValidationResponse):
+    pass
+
+
+@dataclass
+class FailureValidationResponse(ValidationResponse):
+    phase: str
+    message: str
+    stdout: str
+    stderr: str
+
+
+@dataclass
+class SuccessValidationResponse(ValidationResponse):
+    sample: "pd.DataFrame"
+    features: List[ProtoFeature]
+    stdout: str
+    stderr: str
+
+
+class ValidationResponseFactory:
+    @staticmethod
+    def _get_features(
+        spark_columns_description: List[ProtoSparkColumnDescription],
+    ) -> List[ProtoFeature]:
+        spark_features = []
+        for spark_column in spark_columns_description:
+            feature_col_name_list = str(spark_column.column_name).split(".")
+            feature_name = (
+                feature_col_name_list[1]
+                if feature_col_name_list.__len__() == 2
+                else spark_column.column_name
+            )
+            spark_features.append(
+                ProtoFeature(
+                    feature_name=feature_name,
+                    feature_type=spark_column.spark_type,
+                )
+            )
+        return spark_features
+
+    @staticmethod
+    def from_proto(
+        validation_response: ProtoGetValidationResultResponse,
+    ) -> ValidationResponse:
+        try:
+            import pandas as pd
+        except ImportError:
+            raise FrogmlException("Missing required Pandas dependency")
+
+        validation_type: str = validation_response.WhichOneof("type")
+        if validation_type == _VALIDATION_RESPONSE_TYPE_SUCCESS:
+            success: ProtoValidationSuccessResponse = (
+                validation_response.success_response
+            )
+            pd_sample: pd.DataFrame = pd.read_json(
+                path_or_buf=ast.literal_eval(success.sample),
+                dtype=success.spark_column_description,
+            )
+
+            features = ValidationResponseFactory._get_features(
+                success.spark_column_description
+            )
+            return SuccessValidationResponse(
+                sample=pd_sample,
+                features=features,
+                stdout=success.outputs.stdout,
+                stderr=success.outputs.stderr,
+            )
+        elif validation_type == _VALIDATION_RESPONSE_TYPE_FAILURE:
+            failure: ProtoValidationFailureResponse = (
+                validation_response.failure_response
+            )
+            return FailureValidationResponse(
+                phase=failure.phase,
+                message=failure.error_message,
+                stdout=failure.outputs.stdout,
+                stderr=failure.outputs.stderr,
+            )
+        elif validation_type == _VALIDATION_RESPONSE_TYPE_TIMEOUT:
+            return TimeoutValidationResponse()
+
+        raise FrogmlException(f"Got unsupported response type: {validation_type}")

frogml_core/feature_store/validations/validator.py

@@ -0,0 +1,106 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Tuple
+
+from frogml_proto.qwak.features_operator.v3.features_operator_async_service_pb2 import (
+    GetValidationResultResponse as ProtoGetValidationResultResponse,
+)
+from frogml_core.clients.feature_store import FeatureRegistryClient
+from frogml_core.clients.feature_store.operator_client import FeaturesOperatorClient
+from frogml_core.exceptions import FrogmlException
+from frogml_core.feature_store.data_sources.base import BaseSource
+from frogml_core.feature_store.feature_sets.base_feature_set import BaseFeatureSet
+from frogml_core.feature_store.validations.validation_options import (
+    DataSourceValidationOptions,
+    FeatureSetValidationOptions,
+)
+from frogml_core.feature_store.validations.validation_response import (
+    ValidationResponse,
+    ValidationResponseFactory,
+)
+
+
+class Validator(ABC):
+    @abstractmethod
+    def validate_featureset(
+        self,
+        featureset: BaseFeatureSet,
+        sample_size: int = 10,
+        validation_options: Optional[FeatureSetValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        pass
+
+    @abstractmethod
+    def validate_data_source(
+        self,
+        data_source: BaseSource,
+        sample_size: int = 10,
+        validation_options: Optional[DataSourceValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        pass
+
+
+class FeaturesOperatorValidator(Validator):
+    _operator_client: FeaturesOperatorClient
+    _registry_client: FeatureRegistryClient
+
+    def __init__(self):
+        try:
+            import pandas  # noqa: F401
+        except ImportError:
+            raise FrogmlException("Missing required Pandas dependency")
+
+        self._operator_client = FeaturesOperatorClient()
+        self._registry_client = FeatureRegistryClient()
+
+    def validate_data_source(
+        self,
+        data_source: BaseSource,
+        sample_size: int = 10,
+        validation_options: Optional[DataSourceValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        if 0 >= sample_size > 1000:
+            raise ValueError(
+                f"sample_size must be under 1000 and positive, got: {sample_size}"
+            )
+        artifact_url: Optional[str] = None
+        data_source_spec, artifact_url = data_source._prepare_and_get()
+        proto_response: ProtoGetValidationResultResponse = (
+            self._operator_client.validate_data_source_blocking(
+                data_source_spec=data_source_spec,
+                num_samples=sample_size,
+                validation_options=(
+                    validation_options.to_proto() if validation_options else None
+                ),
+            )
+        )
+
+        return ValidationResponseFactory.from_proto(proto_response), artifact_url
+
+    def validate_featureset(
+        self,
+        featureset: BaseFeatureSet,
+        sample_size: int = 10,
+        validation_options: Optional[FeatureSetValidationOptions] = None,
+    ) -> Tuple[ValidationResponse, Optional[str]]:
+        if 0 >= sample_size > 1000:
+            raise ValueError(
+                f"sample_size must be under 1000 and positive, got: {sample_size}"
+            )
+        artifact_url: Optional[str] = None
+
+        featureset_spec, artifact_url = featureset._to_proto(
+            feature_registry=self._registry_client, features=None, git_commit=None
+        )
+
+        proto_response: ProtoGetValidationResultResponse = (
+            self._operator_client.validate_featureset_blocking(
+                featureset_spec=featureset_spec,
+                resource_path=artifact_url,
+                num_samples=sample_size,
+                validation_options=(
+                    validation_options.to_proto() if validation_options else None
+                ),
+            )
+        )
+
+        return ValidationResponseFactory.from_proto(proto_response), artifact_url

frogml_core/frogml_client/batch_jobs/execution.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from frogml_proto.qwak.batch_job.v1.batch_job_service_pb2 import (
+    BatchJobDetails as BatchJobDetailsProto,
+)
+
+
+class ExecutionStatus(Enum):
+    UNDEFINED_BATCH_JOB_STATUS = 0
+    BATCH_JOB_COMMITTED_STATUS = 1
+    BATCH_JOB_PENDING_STATUS = 2
+    BATCH_JOB_RUNNING_STATUS = 3
+    BATCH_JOB_FINISHED_STATUS = 4
+    BATCH_JOB_FAILED_STATUS = 5
+    BATCH_JOB_CANCELLED_STATUS = 6
+    BATCH_JOB_TIMEOUT_STATUS = 7
+
+
+@dataclass
+class Execution:
+    environment_id: str = field(default=None)
+    build_id: str = field(default=None)
+    execution_id: str = field(default=None)
+    execution_status: ExecutionStatus = field(
+        default=ExecutionStatus.UNDEFINED_BATCH_JOB_STATUS
+    )
+    start_time: datetime = field(default=datetime.now())
+    end_time: datetime = field(default=datetime.now())
+    failure_message: str = field(default=None)
+
+    @staticmethod
+    def from_proto(batch_job_details_proto: BatchJobDetailsProto):
+        return Execution(
+            environment_id=batch_job_details_proto.environment_id,
+            build_id=batch_job_details_proto.build_id,
+            execution_id=batch_job_details_proto.job_id,
+            execution_status=ExecutionStatus(batch_job_details_proto.job_status),
+            start_time=datetime.fromtimestamp(
+                batch_job_details_proto.start_time.seconds
+                + batch_job_details_proto.start_time.nanos / 1e9
+            ),
+            end_time=datetime.fromtimestamp(
+                batch_job_details_proto.end_time.seconds
+                + batch_job_details_proto.end_time.nanos / 1e9
+            ),
+            failure_message=batch_job_details_proto.failure_message,
+        )

frogml_core/frogml_client/batch_jobs/task.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from frogml_proto.qwak.batch_job.v1.batch_job_service_pb2 import (
+    TaskExecutionDetails as TaskExecutionDetailsProto,
+)
+
+
+class TaskStatus(Enum):
+    UNDEFINED_BATCH_TASK_STATUS = 0
+    BATCH_TASK_COMMITTED_STATUS = 1
+    BATCH_TASK_PENDING_START_STATUS = 2
+    BATCH_TASK_RUNNING_STATUS = 3
+    BATCH_TASK_FINISHED_STATUS = 4
+    BATCH_TASK_FAILED_STATUS = 5
+    BATCH_TASK_CANCELLED_STATUS = 6
+    BATCH_TASK_TIMEOUT_STATUS = 7
+
+
+@dataclass
+class Task:
+    task_id: str = field(default=None)
+    task_status: TaskStatus = field(default=TaskStatus.UNDEFINED_BATCH_TASK_STATUS)
+    start_time: datetime = field(default=datetime.now())
+    end_time: datetime = field(default=datetime.now())
+    filename: str = field(default=None)
+
+    @staticmethod
+    def from_proto(task_execution_details_proto: TaskExecutionDetailsProto):
+        return Task(
+            task_id=task_execution_details_proto.task_id,
+            task_status=TaskStatus(task_execution_details_proto.status),
+            start_time=datetime.fromtimestamp(
+                task_execution_details_proto.start_time.seconds
+                + task_execution_details_proto.start_time.nanos / 1e9
+            ),
+            end_time=datetime.fromtimestamp(
+                task_execution_details_proto.end_time.seconds
+                + task_execution_details_proto.end_time.nanos / 1e9
+            ),
+            filename=task_execution_details_proto.filename,
+        )

frogml_core/frogml_client/build_api_helpers/build_api_steps.py

@@ -0,0 +1,53 @@
+from frogml_core.inner.build_config.build_config_v1 import BuildConfigV1
+from frogml_core.inner.build_logic.interface.build_phase import BuildPhase
+from frogml_core.inner.build_logic.phases.phase_010_fetch_model.fetch_model_step import (
+    FetchModelStep,
+)
+from frogml_core.inner.build_logic.phases.phase_010_fetch_model.post_fetch_validation_step import (
+    PostFetchValidationStep,
+)
+from frogml_core.inner.build_logic.phases.phase_010_fetch_model.pre_fetch_validation_step import (
+    PreFetchValidationStep,
+)
+from frogml_core.inner.build_logic.phases.phase_010_fetch_model.set_version_step import (
+    SetVersionStep,
+)
+from frogml_core.inner.build_logic.phases.phase_020_remote_register_frogml_build.cleanup_step import (
+    CleanupStep,
+)
+from frogml_core.inner.build_logic.phases.phase_020_remote_register_frogml_build.start_remote_build_step import (
+    StartRemoteBuildStep,
+)
+from frogml_core.inner.build_logic.phases.phase_020_remote_register_frogml_build.upload_step import (
+    UploadStep,
+)
+from frogml_core.inner.build_logic.phases.phases_pipeline import PhasesPipeline
+from frogml_core.inner.build_logic.trigger_build_context import TriggerBuildContext
+
+FETCHING_MODEL_CODE_PHASE = "FETCHING_MODEL_CODE"
+REGISTERING_QWAK_BUILD_PHASE = "REGISTERING_QWAK_BUILD"
+
+
+def get_trigger_build_api_steps(
+    config: BuildConfigV1, context: TriggerBuildContext
+) -> PhasesPipeline:
+    steps_root = PhasesPipeline(config=config, context=context)
+    steps_root.add_phase(
+        steps=[
+            SetVersionStep(),
+            PreFetchValidationStep(),
+            FetchModelStep(),
+            PostFetchValidationStep(),
+        ],
+        build_phase=BuildPhase(phase_id=FETCHING_MODEL_CODE_PHASE),
+    )
+    steps_root.add_phase(
+        steps=[
+            UploadStep(),
+            StartRemoteBuildStep(),
+            CleanupStep(),
+        ],
+        build_phase=BuildPhase(phase_id=REGISTERING_QWAK_BUILD_PHASE),
+    )
+
+    return steps_root

frogml_core/frogml_client/build_api_helpers/trigger_build_api.py

@@ -0,0 +1,67 @@
+import contextlib
+import os
+import shutil
+import uuid
+from pathlib import Path
+
+from frogml_core.frogml_client.build_api_helpers.build_api_steps import (
+    get_trigger_build_api_steps,
+)
+from frogml_core.frogml_client.build_api_helpers.messages import SUCCESS_MSG_API
+from frogml_core.inner.build_config.build_config_v1 import BuildConfigV1
+from frogml_core.inner.build_logic.constants.host_resource import (
+    HOST_FROGML_HIDDEN_FOLDER,
+)
+from frogml_core.inner.build_logic.execute_build_pipeline import execute_build_pipeline
+from frogml_core.inner.build_logic.run_handlers.programmatic_phase_run_handler import (
+    ProgrammaticPhaseRunHandler,
+)
+from frogml_core.inner.build_logic.trigger_build_context import TriggerBuildContext
+from frogml_core.tools.logger import get_frogml_logger, setup_frogml_logger
+
+BUILDS_LOGS = HOST_FROGML_HIDDEN_FOLDER / "logs" / "build"
+BUILD_LOG_NAME = "build.log"
+MAX_LOGS_NUMBER = 15
+DEBUG_LEVEL = 2
+
+
+@contextlib.contextmanager
+def get_build_logger(config: BuildConfigV1, build_id: str):
+    log_path = BUILDS_LOGS / config.build_properties.model_id / build_id
+    log_path.mkdir(parents=True, exist_ok=True)
+    try:
+        (log_path / "build_config.yml").write_text(config.to_yaml())
+        setup_frogml_logger(logs_folder=log_path, log_file_name=BUILD_LOG_NAME)
+        yield get_frogml_logger()
+    finally:
+        # Cleanup - Save only x last zips
+        logs_zip_sorted_by_data = sorted(
+            BUILDS_LOGS.rglob("**/*"), key=os.path.getmtime
+        )[:-MAX_LOGS_NUMBER]
+        path: Path
+        for path in logs_zip_sorted_by_data:
+            if path.is_file():
+                os.remove(path)
+            elif path.is_dir():
+                shutil.rmtree(path, ignore_errors=True)
+
+
+def trigger_build_api(config: BuildConfigV1):
+    build_id = str(uuid.uuid4())
+    with get_build_logger(config=config, build_id=build_id) as logger:
+        context = TriggerBuildContext()
+        context.build_id = build_id
+        pipeline = get_trigger_build_api_steps(config, context)
+        build_runner = ProgrammaticPhaseRunHandler(
+            logger, config.verbose, json_logs=False
+        )
+        execute_build_pipeline(pipeline, build_runner)
+        logger.info(
+            SUCCESS_MSG_API.format(
+                build_id=build_id,
+                model_id=context.model_id,
+                project_uuid=context.project_uuid,
+            )
+        )
+
+    return build_id

frogml_core/frogml_client/builds/build.py

@@ -0,0 +1,97 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Dict, List
+
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from frogml_proto.qwak.build.v1.build_pb2 import Audit as AuditProto
+from frogml_proto.qwak.build.v1.build_pb2 import Build as BuildProto
+from frogml_proto.qwak.builds.builds_pb2 import Build as BuildsManagementBuild
+
+
+class BuildStatus(Enum):
+    INVALID = 0
+    IN_PROGRESS = 1
+    SUCCESSFUL = 2
+    FAILED = 3
+    REMOTE_BUILD_INITIALIZING = 4
+    REMOTE_BUILD_CANCELLED = 5
+    REMOTE_BUILD_TIMED_OUT = 6
+    REMOTE_BUILD_UNKNOWN = 7
+    SYNCING_ENVIRONMENTS = 8
+    FINISHED_SYNCING = 9
+
+
+@dataclass
+class Build:
+    model_id: str = field(default=None)
+    build_id: str = field(default=None)
+    commit_id: str = field(default=None)
+    build_status: BuildStatus = field(default=BuildStatus.INVALID)
+    tags: List[str] = field(default_factory=list)
+    parameters: Dict[str, str] = field(default_factory=dict)
+    metrics: Dict[str, float] = field(default_factory=dict)
+    created_by: str = field(default="")
+    created_at: datetime = field(default=datetime.now())
+    last_modified_by: str = field(default="")
+    last_modified_at: datetime = field(default=datetime.now())
+
+    def to_proto(self):
+        timestamp = Timestamp()
+        return BuildProto(
+            buildId=self.build_id,
+            commitId=self.commit_id,
+            build_status=self.build_status.value,
+            tags=self.tags,
+            params=self.parameters,
+            metrics=self.metrics,
+            audit=AuditProto(
+                created_by=self.created_by,
+                created_at=timestamp.FromDatetime(self.created_at),
+                last_modified_by=self.last_modified_by,
+                last_modified_at=timestamp.FromDatetime(self.last_modified_at),
+            ),
+        )
+
+    @staticmethod
+    def from_proto(build_proto: BuildProto):
+        return Build(
+            build_id=build_proto.buildId,
+            commit_id=build_proto.commitId,
+            build_status=BuildStatus(build_proto.build_status),
+            tags=build_proto.tags,
+            parameters=build_proto.params,
+            metrics=build_proto.metrics,
+            created_by=build_proto.audit.created_by,
+            created_at=datetime.fromtimestamp(
+                build_proto.audit.created_at.seconds
+                + build_proto.audit.created_at.nanos / 1e9
+            ),
+            last_modified_by=build_proto.audit.last_modified_by,
+            last_modified_at=datetime.fromtimestamp(
+                build_proto.audit.last_modified_at.seconds
+                + build_proto.audit.last_modified_at.nanos / 1e9
+            ),
+        )
+
+    @staticmethod
+    def from_builds_management(build_proto: BuildsManagementBuild):
+        return Build(
+            model_id=build_proto.build_spec.model_id,
+            build_id=build_proto.build_spec.build_id,
+            commit_id=build_proto.build_spec.commit_id,
+            build_status=BuildStatus(build_proto.build_status),
+            tags=build_proto.build_spec.tags,
+            parameters=build_proto.build_spec.experiment_tracking_values.params,
+            metrics=build_proto.build_spec.experiment_tracking_values.metrics,
+            created_by=build_proto.created_by,
+            created_at=datetime.fromtimestamp(
+                build_proto.created_at.seconds + build_proto.created_at.nanos / 1e9
+            ),
+            last_modified_by=build_proto.last_modified_by,
+            last_modified_at=datetime.fromtimestamp(
+                build_proto.last_modified_at.seconds
+                + build_proto.last_modified_at.nanos / 1e9
+            ),
+        )

frogml_core/frogml_client/builds/filters/metric_filter.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass, field
+from enum import Enum
+
+from frogml_proto.qwak.build.v1.build_pb2 import MetricFilter as MetricFilterProto
+
+
+class MetricOperatorType(Enum):
+    METRIC_OPERATOR_TYPE_INVALID = 0
+    METRIC_OPERATOR_TYPE_EQUALS = 1
+    METRIC_OPERATOR_TYPE_NOT_EQUALS = 2
+    METRIC_OPERATOR_TYPE_LESS_THAN = 3
+    METRIC_OPERATOR_TYPE_LESS_THAN_EQUALS = 4
+    METRIC_OPERATOR_TYPE_GREATER_THAN = 5
+    METRIC_OPERATOR_TYPE_GREATER_THAN_EQUALS = 6
+
+
+@dataclass
+class MetricFilter:
+    metric_name: str = field(default=None)
+    metric_value: float = field(default=None)
+    operator: MetricOperatorType = field(
+        default=MetricOperatorType.METRIC_OPERATOR_TYPE_INVALID
+    )
+
+    def to_proto(self):
+        return MetricFilterProto(
+            metric_name=self.metric_name,
+            metric_value=self.metric_value,
+            operator=self.operator.value,
+        )
+
+    @staticmethod
+    def from_proto(metric_filter_proto: MetricFilterProto):
+        return MetricFilter(
+            metric_name=metric_filter_proto.metric_name,
+            metric_value=metric_filter_proto.metric_value,
+            operator=metric_filter_proto.operator,
+        )

frogml_core/frogml_client/builds/filters/parameter_filter.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass, field
+from enum import Enum
+
+from frogml_proto.qwak.build.v1.build_pb2 import (
+    ParameterFilter as ParameterFilterProto,
+)
+
+
+class ParameterOperatorType(Enum):
+    PARAMETER_OPERATOR_TYPE_INVALID = 0
+    PARAMETER_OPERATOR_TYPE_EQUALS = 1
+    PARAMETER_OPERATOR_TYPE_NOT_EQUALS = 2
+
+
+@dataclass
+class ParameterFilter:
+    parameter_name: str = field(default=None)
+    parameter_value: str = field(default=None)
+    operator: ParameterOperatorType = field(
+        default=ParameterOperatorType.PARAMETER_OPERATOR_TYPE_INVALID
+    )
+
+    def to_proto(self):
+        return ParameterFilterProto(
+            parameter_name=self.parameter_name,
+            parameter_value=self.parameter_value,
+            operator=self.operator.value,
+        )
+
+    @staticmethod
+    def from_proto(parameter_filter_proto: ParameterFilterProto):
+        return ParameterFilter(
+            parameter_name=parameter_filter_proto.parameter_name,
+            parameter_value=parameter_filter_proto.parameter_value,
+            operator=parameter_filter_proto.operator,
+        )