digitalhub 0.9.1__py3-none-any.whl → 0.10.0b0__py3-none-any.whl

digitalhub/entities/model/_base/entity.py

@@ -4,6 +4,8 @@ import typing
 
 from digitalhub.entities._base.material.entity import MaterialEntity
 from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities._commons.utils import validate_metric_value
+from digitalhub.entities._operations.processor import processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -32,3 +34,173 @@ class Model(MaterialEntity):
         super().__init__(project, name, uuid, kind, metadata, spec, status, user)
         self.spec: ModelSpec
         self.status: ModelStatus
+
+        # Initialize metrics
+        self._init_metrics()
+
+    def save(self, update: bool = False) -> Model:
+        """
+        Save entity into backend.
+
+        Parameters
+        ----------
+        update : bool
+            Flag to indicate update.
+
+        Returns
+        -------
+        Model
+            Entity saved.
+        """
+        obj: Model = super().save(update)
+        obj._get_metrics()
+        return obj
+
+    def log_metric(
+        self,
+        key: str,
+        value: list[float | int] | float | int,
+        overwrite: bool = False,
+        single_value: bool = False,
+    ) -> None:
+        """
+        Log metric into entity status.
+        A metric is named by a key and value (single number or list of numbers).
+        The metric by default is put in a list or appended to an existing list.
+        If single_value is True, the value will be a single number.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : list[float | int] | float | int
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+        single_value : bool
+            If True, value is a single value.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        Log a new value in a list
+        >>> entity.log_metric("loss", 0.002)
+
+        Append a new value in a list
+        >>> entity.log_metric("loss", 0.0019)
+
+        Log a list of values and append them to existing metric:
+        >>> entity.log_metric("loss", [0.0018, 0.0015])
+
+        Log a single value (not represented as list):
+        >>> entity.log_metric("accuracy", 0.9, single_value=True)
+
+        Log a list of values and overwrite existing metric:
+        >>> entity.log_metric("accuracy", [0.8, 0.9], overwrite=True)
+        """
+        value = validate_metric_value(value)
+
+        if isinstance(value, list):
+            self._handle_metric_list(key, value, overwrite)
+        elif single_value:
+            self._handle_metric_single(key, value, overwrite)
+        else:
+            self._handle_metric_list_append(key, value, overwrite)
+
+        processor.update_metric(self.project, self.ENTITY_TYPE, self.id, key, self.status.metrics[key])
+
+    ##############################
+    # Helper methods
+    ##############################
+
+    def _init_metrics(self) -> None:
+        """
+        Initialize metrics.
+
+        Returns
+        -------
+        None
+        """
+        if self.status.metrics is None:
+            self.status.metrics = {}
+
+    def _get_metrics(self) -> None:
+        """
+        Get model metrics from backend.
+
+        Returns
+        -------
+        None
+        """
+        self.status.metrics = processor.read_metrics(
+            project=self.project,
+            entity_type=self.ENTITY_TYPE,
+            entity_id=self.id,
+        )
+
+    def _handle_metric_single(self, key: str, value: float | int, overwrite: bool = False) -> None:
+        """
+        Handle metric single value.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : float
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+
+        Returns
+        -------
+        None
+        """
+        if key not in self.status.metrics or overwrite:
+            self.status.metrics[key] = value
+
+    def _handle_metric_list_append(self, key: str, value: float | int, overwrite: bool = False) -> None:
+        """
+        Handle metric list append.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : float
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+
+        Returns
+        -------
+        None
+        """
+        if key not in self.status.metrics or overwrite:
+            self.status.metrics[key] = [value]
+        else:
+            self.status.metrics[key].append(value)
+
+    def _handle_metric_list(self, key: str, value: list[int | float], overwrite: bool = False) -> None:
+        """
+        Handle metric list.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : list[int | float]
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+
+        Returns
+        -------
+        None
+        """
+        if key not in self.status.metrics or overwrite:
+            self.status.metrics[key] = value
+        else:
+            self.status.metrics[key].extend(value)

digitalhub/entities/model/_base/spec.py

@@ -13,16 +13,12 @@ class ModelSpec(MaterialSpec):
         path: str,
         framework: str | None = None,
         algorithm: str | None = None,
-        base_model: str | None = None,
         parameters: dict | None = None,
-        metrics: dict | None = None,
     ) -> None:
         self.path = path
         self.framework = framework
         self.algorithm = algorithm
-        self.base_model = base_model
         self.parameters = parameters
-        self.metrics = metrics
 
 
 class ModelValidator(MaterialValidator):
@@ -39,11 +35,5 @@ class ModelValidator(MaterialValidator):
     algorithm: str = None
     """Model algorithm (e.g. 'resnet')."""
 
-    base_model: str = None
-    """Base model."""
-
     parameters: dict = None
     """Model validator."""
-
-    metrics: dict = None
-    """Model metrics."""

digitalhub/entities/model/_base/status.py

@@ -7,3 +7,13 @@ class ModelStatus(MaterialStatus):
     """
     ModelStatus status.
     """
+
+    def __init__(
+        self,
+        state: str,
+        message: str | None = None,
+        files: list[dict] | None = None,
+        metrics: dict | None = None,
+    ):
+        super().__init__(state, message, files)
+        self.metrics = metrics

digitalhub/entities/model/crud.py

@@ -157,7 +157,7 @@ def get_model(
     >>>                 project="my-project",
     >>>                 entity_id="my-model-id")
     """
-    return processor.read_material_entity(
+    return processor.read_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -197,7 +197,7 @@ def get_model_versions(
     >>> objs = get_model_versions("my-model-name",
     >>>                           project="my-project")
     """
-    return processor.read_material_entity_versions(
+    return processor.read_context_entity_versions(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -225,7 +225,7 @@ def list_models(project: str, **kwargs) -> list[Model]:
     --------
     >>> objs = list_models(project="my-project")
     """
-    return processor.list_material_entities(
+    return processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,

digitalhub/entities/model/huggingface/spec.py

@@ -15,13 +15,13 @@ class ModelSpecHuggingface(ModelSpec):
         path: str,
         framework: str | None = None,
         algorithm: str | None = None,
-        base_model: str | None = None,
         parameters: dict | None = None,
-        metrics: dict | None = None,
+        base_model: str | None = None,
         model_id: str | None = None,
         model_revision: str = None,
     ) -> None:
-        super().__init__(path, framework, algorithm, base_model, parameters, metrics)
+        super().__init__(path, framework, algorithm, parameters)
+        self.base_model = base_model
         self.model_id = model_id
         self.model_revision = model_revision
 
@@ -31,6 +31,9 @@ class ModelValidatorHuggingface(ModelValidator):
     ModelValidatorHuggingface validator.
     """
 
+    base_model: str = None
+    """Base model."""
+
     placeholder_model_id: str = Field(default=None, alias="model_id")
     """Huggingface model id. If not specified, the model is loaded from the model path."""
 

digitalhub/entities/model/mlflow/models.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 
 class Signature(BaseModel):
@@ -26,7 +26,7 @@ class Dataset(BaseModel):
     name: Optional[str] = None
     digest: Optional[str] = None
     profile: Optional[str] = None
-    schema_: Optional[str] = Field(default=None, alias="schema")
+    dataset_schema: Optional[str] = None
     source: Optional[str] = None
     source_type: Optional[str] = None
 

digitalhub/entities/model/mlflow/spec.py

@@ -16,15 +16,13 @@ class ModelSpecMlflow(ModelSpec):
         path: str,
         framework: str | None = None,
         algorithm: str | None = None,
-        base_model: str | None = None,
         parameters: dict | None = None,
-        metrics: dict | None = None,
         flavor: str | None = None,
         model_config: dict | None = None,
         input_datasets: list[Dataset] | None = None,
         signature: Signature = None,
     ) -> None:
-        super().__init__(path, framework, algorithm, base_model, parameters, metrics)
+        super().__init__(path, framework, algorithm, parameters)
         self.flavor = flavor
         self.model_config = model_config
         self.input_datasets = input_datasets

digitalhub/entities/model/mlflow/utils.py

@@ -1,11 +1,32 @@
 from __future__ import annotations
 
+import typing
 from urllib.parse import urlparse
 
 import mlflow
 
 from digitalhub.entities.model.mlflow.models import Dataset, Signature
 
+if typing.TYPE_CHECKING:
+    from mlflow.entities import Run
+
+
+def get_mlflow_run(run_id: str) -> Run:
+    """
+    Get MLFlow run.
+
+    Parameters
+    ----------
+    run_id : str
+        The id of the mlflow run.
+
+    Returns
+    -------
+    Run
+        The extracted run.
+    """
+    return mlflow.MlflowClient().get_run(run_id)
+
 
 def from_mlflow_run(run_id: str) -> dict:
     """
@@ -23,13 +44,11 @@ def from_mlflow_run(run_id: str) -> dict:
     """
 
     # Get MLFlow run
-    client = mlflow.MlflowClient()
-    run = client.get_run(run_id)
+    run = get_mlflow_run(run_id)
 
     # Extract spec
     data = run.data
     parameters = data.params
-    metrics = data.metrics
     source_path = urlparse(run.info.artifact_uri).path
     model_uri = f"runs:/{run_id}/model"
     model = mlflow.pyfunc.load_model(model_uri=model_uri)
@@ -56,7 +75,7 @@ def from_mlflow_run(run_id: str) -> dict:
                 name=d.dataset.name,
                 digest=d.dataset.digest,
                 profile=d.dataset.profile,
-                schema=d.dataset.schema,
+                dataset_schema=d.dataset.schema,
                 source=d.dataset.source,
                 source_type=d.dataset.source_type,
             ).to_dict()
@@ -72,7 +91,6 @@ def from_mlflow_run(run_id: str) -> dict:
     # common properties
     model_params["framework"] = flavor
     model_params["parameters"] = parameters
-    model_params["metrics"] = metrics
 
     # specific to MLFlow
     model_params["flavor"] = flavor
@@ -81,3 +99,24 @@ def from_mlflow_run(run_id: str) -> dict:
     model_params["signature"] = signature
 
     return model_params
+
+
+def get_mlflow_model_metrics(run_id: str) -> dict:
+    """
+    Get MLFlow model metrics for a given run id.
+
+    Parameters
+    ----------
+    run_id : str
+        The id of the mlflow run.
+
+    Returns
+    -------
+    dict
+        The extracted metrics.
+    """
+    # Get MLFlow run
+    run = get_mlflow_run(run_id)
+
+    # Extract metrics
+    return run.data.metrics

digitalhub/entities/run/_base/entity.py

@@ -5,6 +5,7 @@ import typing
 
 from digitalhub.entities._base.unversioned.entity import UnversionedEntity
 from digitalhub.entities._commons.enums import EntityTypes, State
+from digitalhub.entities._commons.utils import validate_metric_value
 from digitalhub.entities._operations.processor import processor
 from digitalhub.factory.api import (
     build_runtime,
@@ -45,6 +46,9 @@ class Run(UnversionedEntity):
         self.spec: RunSpec
         self.status: RunStatus
 
+        # Initialize metrics
+        self._init_metrics()
+
     ##############################
     #  Run Methods
     ##############################
@@ -161,6 +165,62 @@ class Run(UnversionedEntity):
         if not self.spec.local_execution:
             return processor.resume_run(self.project, self.ENTITY_TYPE, self.id)
 
+    def log_metric(
+        self,
+        key: str,
+        value: list[float | int] | float | int,
+        overwrite: bool = False,
+        single_value: bool = False,
+    ) -> None:
+        """
+        Log metric into entity status.
+        A metric is named by a key and value (single number or list of numbers).
+        The metric by default is put in a list or appended to an existing list.
+        If single_value is True, the value will be a single number.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : list[float | int] | float | int
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+        single_value : bool
+            If True, value is a single value.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        Log a new value in a list
+        >>> entity.log_metric("loss", 0.002)
+
+        Append a new value in a list
+        >>> entity.log_metric("loss", 0.0019)
+
+        Log a list of values and append them to existing metric:
+        >>> entity.log_metric("loss", [0.0018, 0.0015])
+
+        Log a single value (not represented as list):
+        >>> entity.log_metric("accuracy", 0.9, single_value=True)
+
+        Log a list of values and overwrite existing metric:
+        >>> entity.log_metric("accuracy", [0.8, 0.9], overwrite=True)
+        """
+        value = validate_metric_value(value)
+
+        if isinstance(value, list):
+            self._handle_metric_list(key, value, overwrite)
+        elif single_value:
+            self._handle_metric_single(key, value, overwrite)
+        else:
+            self._handle_metric_list_append(key, value, overwrite)
+
+        processor.update_metric(self.project, self.ENTITY_TYPE, self.id, key, self.status.metrics[key])
+
     ##############################
     #  Helpers
     ##############################
@@ -303,3 +363,92 @@ class Run(UnversionedEntity):
             entity_type=EntityTypes.TASK.value,
             project=self.project,
         ).to_dict()
+
+    def _init_metrics(self) -> None:
+        """
+        Initialize metrics.
+
+        Returns
+        -------
+        None
+        """
+        if self.status.metrics is None:
+            self.status.metrics = {}
+
+    def _get_metrics(self) -> None:
+        """
+        Get run metrics from backend.
+
+        Returns
+        -------
+        None
+        """
+        self.status.metrics = processor.read_metrics(
+            project=self.project,
+            entity_type=self.ENTITY_TYPE,
+            entity_id=self.id,
+        )
+
+    def _handle_metric_single(self, key: str, value: float | int, overwrite: bool = False) -> None:
+        """
+        Handle metric single value.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : float
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+
+        Returns
+        -------
+        None
+        """
+        if key not in self.status.metrics or overwrite:
+            self.status.metrics[key] = value
+
+    def _handle_metric_list_append(self, key: str, value: float | int, overwrite: bool = False) -> None:
+        """
+        Handle metric list append.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : float
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+
+        Returns
+        -------
+        None
+        """
+        if key not in self.status.metrics or overwrite:
+            self.status.metrics[key] = [value]
+        else:
+            self.status.metrics[key].append(value)
+
+    def _handle_metric_list(self, key: str, value: list[int | float], overwrite: bool = False) -> None:
+        """
+        Handle metric list.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : list[int | float]
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+
+        Returns
+        -------
+        None
+        """
+        if key not in self.status.metrics or overwrite:
+            self.status.metrics[key] = value
+        else:
+            self.status.metrics[key].extend(value)

digitalhub/entities/run/_base/status.py

@@ -7,3 +7,15 @@ class RunStatus(Status):
     """
     RunStatus status.
     """
+
+    def __init__(
+        self,
+        state: str,
+        message: str | None = None,
+        transitions: list[dict] | None = None,
+        k8s: dict | None = None,
+        metrics: dict | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(state, message, transitions, k8s, **kwargs)
+        self.metrics = metrics

digitalhub/entities/task/_base/spec.py

@@ -66,6 +66,8 @@ class TaskSpecWorkflow(TaskSpec):
         self.envs = envs
         self.secrets = secrets
         self.profile = profile
+        self.runtime_class = runtime_class
+        self.priority_class = priority_class
 
 
 class TaskValidator(SpecValidator):

digitalhub/entities/task/crud.py

@@ -19,6 +19,7 @@ def new_task(
     uuid: str | None = None,
     labels: list[str] | None = None,
     function: str | None = None,
+    workflow: str | None = None,
     **kwargs,
 ) -> Task:
     """
@@ -36,6 +37,8 @@ def new_task(
         List of labels.
     function : str
         Name of the executable associated with the task.
+    workflow : str
+        Name of the workflow associated with the task.
     **kwargs : dict
         Spec keyword arguments.
 
@@ -56,6 +59,7 @@ def new_task(
         uuid=uuid,
         labels=labels,
         function=function,
+        workflow=workflow,
         **kwargs,
     )
 

digitalhub/readers/{_commons → pandas}/enums.py

@@ -10,4 +10,8 @@ class Extensions(Enum):
 
     CSV = "csv"
     PARQUET = "parquet"
+    JSON = "json"
+    EXCEL_OLD = "xls"
+    EXCEL = "xlsx"
+    TXT = "txt"
     FILE = "file"