digitalhub 0.9.1__py3-none-any.whl → 0.10.0__py3-none-any.whl

digitalhub/entities/dataitem/table/models.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -40,16 +41,16 @@ class TableSchemaFieldEntry(BaseModel):
     type_: FieldType = Field(alias="type")
     """Field type."""
 
-    title: str = None
+    title: Optional[str] = None
     """Field title."""
 
     format_: str = Field(default=None, alias="format")
     """Field format."""
 
-    example: str = None
+    example: Optional[str] = None
     """Field example."""
 
-    description: str = None
+    description: Optional[str] = None
     """Field description."""
 
 

digitalhub/{utils/data_utils.py → entities/dataitem/table/utils.py}

@@ -1,49 +1,23 @@
 from __future__ import annotations
 
-import json
+from digitalhub.utils.generic_utils import dump_json
 
 
-def build_data_preview(preview: list[dict] | None = None, rows_count: int | None = None) -> dict:
+def prepare_data(data: list[list], columnar: bool = False) -> list[list]:
     """
-    Build data preview.
+    Prepare data.
 
     Parameters
     ----------
-    preview : list[dict] | None
-        Preview.
-    rows_count : int | None
-        Row count.
-
-    Returns
-    -------
-    dict
-        Data preview.
-    """
-    dict_ = {}
-    if preview is not None:
-        dict_["cols"] = preview
-    if rows_count is not None:
-        dict_["rows"] = rows_count
-    return dict_
-
-
-def get_data_preview(columns: list, data: list[list], columnar: bool = False) -> list[dict]:
-    """
-    Prepare preview.
-
-    Parameters
-    ----------
-    columns : list
-        Columns names.
-    data : list[list]
-        Data to preview.
-    columnar : bool
+    data : list
+        Data.
+    columnar : bool | None
         If data are arranged in columns. If False, data are arranged in rows.
 
     Returns
     -------
-    list[dict]
-        Data preview.
+    list[list]
+        Prepared data.
     """
     # Reduce data to 10 rows
     if not columnar:
@@ -56,17 +30,10 @@ def get_data_preview(columns: list, data: list[list], columnar: bool = False) ->
     if not columnar:
         data = list(map(list, list(zip(*data))))
 
-    # Prepare the preview
-    data_dict = prepare_preview(columns, data)
-
-    # Filter memoryview values
-    filtered_memview = filter_memoryview(data_dict)
-
-    # Check the size of the preview data
-    return check_preview_size(filtered_memview)
+    return data
 
 
-def prepare_preview(column_names: list, data: list[list]) -> list[dict]:
+def prepare_preview(columns: list, data: list[list]) -> list[dict]:
     """
     Get preview.
 
@@ -80,9 +47,10 @@ def prepare_preview(column_names: list, data: list[list]) -> list[dict]:
     list[dict]
         Preview.
     """
-    if len(column_names) != len(data):
+    if len(columns) != len(data):
         raise ValueError("Column names and data must have the same length")
-    return [{"name": column, "value": values} for column, values in zip(column_names, data)]
+    preview = [{"name": column, "value": values} for column, values in zip(columns, data)]
+    return filter_memoryview(preview)
 
 
 def filter_memoryview(data: list[dict]) -> list[dict]:
@@ -91,13 +59,13 @@ def filter_memoryview(data: list[dict]) -> list[dict]:
 
     Parameters
     ----------
-    data : pd.DataFrame
+    data : list[dict]
         Data.
 
     Returns
     -------
-    list[str]
-        Column to filter out from preview.
+    list[dict]
+        Preview.
     """
     key_to_filter = []
     for i in data:
@@ -108,20 +76,44 @@ def filter_memoryview(data: list[dict]) -> list[dict]:
     return data
 
 
-def check_preview_size(preview: list[dict]) -> list:
+def check_preview_size(preview: dict) -> dict:
     """
-    Check preview size. If it's too big, return empty list.
+    Check preview size. If it's too big, return empty dict.
 
     Parameters
     ----------
-    preview : list[dict]
+    preview : dict
         Preview.
 
     Returns
     -------
-    list
+    dict
         Preview.
     """
-    if len(json.dumps(preview).encode("utf-8")) >= 64000:
-        return []
+    if len(dump_json(preview)) >= 64000:
+        return {}
     return preview
+
+
+def finalize_preview(preview: list[dict] | None = None, rows_count: int | None = None) -> dict:
+    """
+    Finalize preview.
+
+    Parameters
+    ----------
+    preview : list[dict]
+        Preview.
+    rows_count : int
+        Row count.
+
+    Returns
+    -------
+    dict
+        Data preview.
+    """
+    data: dict[str, list[dict] | int] = {}
+    if preview is not None:
+        data["cols"] = preview
+    if rows_count is not None:
+        data["rows"] = rows_count
+    return data

digitalhub/entities/dataitem/utils.py

@@ -8,16 +8,21 @@ from digitalhub.context.api import get_context
 from digitalhub.entities._base.entity._constructors.uuid import build_uuid
 from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
 from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
-from digitalhub.readers._commons.enums import Extensions
-from digitalhub.readers.api import get_reader_by_object
+from digitalhub.readers.data.api import get_reader_by_object
+from digitalhub.stores.api import get_store
+from digitalhub.utils.enums import FileExtensions
 from digitalhub.utils.generic_utils import slugify_string
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.dataitem._base.entity import Dataitem
 
 
+DEFAULT_EXTENSION = FileExtensions.PARQUET.value
+
+
 def eval_source(
-    source: str | list[str] | None = None,
+    source: SourcesOrListOfSources | None = None,
     data: Any | None = None,
     kind: str | None = None,
     name: str | None = None,
@@ -28,7 +33,7 @@ def eval_source(
 
     Parameters
     ----------
-    source : str | list[str]
+    source : SourcesOrListOfSources
         Source(s).
 
     Returns
@@ -39,11 +44,12 @@ def eval_source(
         raise ValueError("You must provide source or data.")
 
     if source is not None:
-        return eval_local_source(source)
+        eval_local_source(source)
+        return source
 
     if kind == EntityKinds.DATAITEM_TABLE.value:
         ctx = get_context(project)
-        pth = ctx.root / f"{slugify_string(name)}.{Extensions.PARQUET.value}"
+        pth = ctx.root / f"{slugify_string(name)}.{DEFAULT_EXTENSION}"
         reader = get_reader_by_object(data)
         reader.write_parquet(data, pth)
         return str(pth)
@@ -51,11 +57,49 @@ def eval_source(
     raise NotImplementedError
 
 
+def eval_data(
+    project: str,
+    kind: str,
+    source: SourcesOrListOfSources,
+    data: Any | None = None,
+    file_format: str | None = None,
+    engine: str | None = None,
+) -> Any:
+    """
+    Evaluate data is loaded.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    source : str
+        Source(s).
+    data : Any
+        Dataframe to log. Alternative to source.
+    file_format : str
+        Extension of the file.
+    engine : str
+        Engine to use.
+
+    Returns
+    -------
+    None
+    """
+    if kind == EntityKinds.DATAITEM_TABLE.value:
+        if data is None:
+            return get_store(project, source).read_df(
+                source,
+                file_format=file_format,
+                engine=engine,
+            )
+    return data
+
+
 def process_kwargs(
     project: str,
     name: str,
     kind: str,
-    source: str | list[str],
+    source: SourcesOrListOfSources,
     data: Any | None = None,
     path: str | None = None,
     **kwargs,
@@ -71,7 +115,7 @@ def process_kwargs(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Source(s).
     data : Any
         Dataframe to log. Alternative to source.
@@ -98,19 +142,23 @@ def process_kwargs(
     return kwargs
 
 
-def clean_tmp_path(pth: str) -> None:
+def clean_tmp_path(pth: SourcesOrListOfSources) -> None:
     """
     Clean temporary path.
 
     Parameters
     ----------
-    pth : str
+    pth : SourcesOrListOfSources
         Path to clean.
 
     Returns
     -------
     None
     """
+    if isinstance(pth, list):
+        for p in pth:
+            shutil.rmtree(p, ignore_errors=True)
+        return
     shutil.rmtree(pth, ignore_errors=True)
 
 

digitalhub/entities/function/crud.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.context import context_processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.function._base.entity import Function
@@ -56,7 +56,7 @@ def new_function(
     >>>                    code_src="function.py",
     >>>                    handler="function-handler")
     """
-    return processor.create_context_entity(
+    return context_processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -103,7 +103,7 @@ def get_function(
     >>>                    project="my-project",
     >>>                    entity_id="my-function-id")
     """
-    return processor.read_context_entity(
+    return context_processor.read_context_entity(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -143,7 +143,7 @@ def get_function_versions(
     >>> obj = get_function_versions("my-function-name"
     >>>                             project="my-project")
     """
-    return processor.read_context_entity_versions(
+    return context_processor.read_context_entity_versions(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -171,7 +171,7 @@ def list_functions(project: str, **kwargs) -> list[Function]:
     --------
     >>> objs = list_functions(project="my-project")
     """
-    return processor.list_context_entities(
+    return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -196,7 +196,7 @@ def import_function(file: str) -> Function:
     --------
     >>> obj = import_function("my-function.yaml")
     """
-    return processor.import_executable_entity(file)
+    return context_processor.import_executable_entity(file)
 
 
 def load_function(file: str) -> Function:
@@ -217,7 +217,7 @@ def load_function(file: str) -> Function:
     --------
     >>> obj = load_function("my-function.yaml")
     """
-    return processor.load_executable_entity(file)
+    return context_processor.load_executable_entity(file)
 
 
 def update_function(entity: Function) -> Function:
@@ -238,7 +238,7 @@ def update_function(entity: Function) -> Function:
     --------
     >>> obj = update_function(obj)
     """
-    return processor.update_context_entity(
+    return context_processor.update_context_entity(
         project=entity.project,
         entity_type=entity.ENTITY_TYPE,
         entity_id=entity.id,
@@ -287,7 +287,7 @@ def delete_function(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return processor.delete_context_entity(
+    return context_processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/model/_base/entity.py

@@ -4,6 +4,8 @@ import typing
 
 from digitalhub.entities._base.material.entity import MaterialEntity
 from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities._commons.metrics import MetricType, set_metrics, validate_metric_value
+from digitalhub.entities._processors.context import context_processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -32,3 +34,121 @@ class Model(MaterialEntity):
         super().__init__(project, name, uuid, kind, metadata, spec, status, user)
         self.spec: ModelSpec
         self.status: ModelStatus
+
+    def save(self, update: bool = False) -> Model:
+        """
+        Save entity into backend.
+
+        Parameters
+        ----------
+        update : bool
+            Flag to indicate update.
+
+        Returns
+        -------
+        Model
+            Entity saved.
+        """
+        obj: Model = super().save(update)
+        obj._get_metrics()
+        return obj
+
+    def log_metric(
+        self,
+        key: str,
+        value: MetricType,
+        overwrite: bool = False,
+        single_value: bool = False,
+    ) -> None:
+        """
+        Log metric into entity status.
+        A metric is named by a key and value (single number or list of numbers).
+        The metric by default is put in a list or appended to an existing list.
+        If single_value is True, the value will be a single number.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : MetricType
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+        single_value : bool
+            If True, value is a single value.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        Log a new value in a list
+        >>> entity.log_metric("loss", 0.002)
+
+        Append a new value in a list
+        >>> entity.log_metric("loss", 0.0019)
+
+        Log a list of values and append them to existing metric:
+        >>> entity.log_metric("loss", [0.0018, 0.0015])
+
+        Log a single value (not represented as list):
+        >>> entity.log_metric("accuracy", 0.9, single_value=True)
+
+        Log a list of values and overwrite existing metric:
+        >>> entity.log_metric("accuracy", [0.8, 0.9], overwrite=True)
+        """
+        self._set_metrics(key, value, overwrite, single_value)
+        context_processor.update_metric(self.project, self.ENTITY_TYPE, self.id, key, self.status.metrics[key])
+
+    ##############################
+    # Helper methods
+    ##############################
+
+    def _get_metrics(self) -> None:
+        """
+        Get model metrics from backend.
+
+        Returns
+        -------
+        None
+        """
+        self.status.metrics = context_processor.read_metrics(
+            project=self.project,
+            entity_type=self.ENTITY_TYPE,
+            entity_id=self.id,
+        )
+
+    def _set_metrics(
+        self,
+        key: str,
+        value: MetricType,
+        overwrite: bool,
+        single_value: bool,
+    ) -> None:
+        """
+        Set model metrics.
+
+        Parameters
+        ----------
+        key : str
+            Key of the metric.
+        value : MetricType
+            Value of the metric.
+        overwrite : bool
+            If True, overwrite existing metric.
+        single_value : bool
+            If True, value is a single value.
+
+        Returns
+        -------
+        None
+        """
+        value = validate_metric_value(value)
+        self.status.metrics = set_metrics(
+            self.status.metrics,
+            key,
+            value,
+            overwrite,
+            single_value,
+        )

digitalhub/entities/model/_base/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from digitalhub.entities._base.material.spec import MaterialSpec, MaterialValidator
 
 
@@ -13,16 +15,12 @@ class ModelSpec(MaterialSpec):
         path: str,
         framework: str | None = None,
         algorithm: str | None = None,
-        base_model: str | None = None,
         parameters: dict | None = None,
-        metrics: dict | None = None,
     ) -> None:
-        self.path = path
+        super().__init__(path)
         self.framework = framework
         self.algorithm = algorithm
-        self.base_model = base_model
         self.parameters = parameters
-        self.metrics = metrics
 
 
 class ModelValidator(MaterialValidator):
@@ -30,20 +28,11 @@ class ModelValidator(MaterialValidator):
     ModelValidator validator.
     """
 
-    path: str
-    """Path to the model."""
-
-    framework: str = None
+    framework: Optional[str] = None
     """Model framework (e.g. 'pytorch')."""
 
-    algorithm: str = None
+    algorithm: Optional[str] = None
     """Model algorithm (e.g. 'resnet')."""
 
-    base_model: str = None
-    """Base model."""
-
-    parameters: dict = None
+    parameters: Optional[dict] = None
     """Model validator."""
-
-    metrics: dict = None
-    """Model metrics."""

digitalhub/entities/model/_base/status.py

@@ -7,3 +7,13 @@ class ModelStatus(MaterialStatus):
     """
     ModelStatus status.
     """
+
+    def __init__(
+        self,
+        state: str,
+        message: str | None = None,
+        files: list[dict] | None = None,
+        metrics: dict | None = None,
+    ):
+        super().__init__(state, message, files)
+        self.metrics = metrics if metrics is not None else {}

digitalhub/entities/model/crud.py

@@ -3,8 +3,9 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.context import context_processor
 from digitalhub.entities.artifact.utils import eval_source, process_kwargs
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.model._base.entity import Model
@@ -60,7 +61,7 @@ def new_model(
     >>>                    kind="model",
     >>>                    path="s3://my-bucket/my-key")
     """
-    return processor.create_context_entity(
+    return context_processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -77,7 +78,7 @@ def log_model(
     project: str,
     name: str,
     kind: str,
-    source: list[str] | str,
+    source: SourcesOrListOfSources,
     path: str | None = None,
     **kwargs,
 ) -> Model:
@@ -92,7 +93,7 @@ def log_model(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Model location on local path.
     path : str
         Destination path of the model. If not provided, it's generated.
@@ -113,7 +114,7 @@ def log_model(
     """
     eval_source(source)
     kwargs = process_kwargs(project, name, source=source, path=path, **kwargs)
-    return processor.log_material_entity(
+    return context_processor.log_material_entity(
         source=source,
         project=project,
         name=name,
@@ -157,7 +158,7 @@ def get_model(
     >>>                 project="my-project",
     >>>                 entity_id="my-model-id")
     """
-    return processor.read_material_entity(
+    return context_processor.read_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -197,7 +198,7 @@ def get_model_versions(
     >>> objs = get_model_versions("my-model-name",
     >>>                           project="my-project")
     """
-    return processor.read_material_entity_versions(
+    return context_processor.read_context_entity_versions(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -225,7 +226,7 @@ def list_models(project: str, **kwargs) -> list[Model]:
     --------
     >>> objs = list_models(project="my-project")
     """
-    return processor.list_material_entities(
+    return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -250,7 +251,7 @@ def import_model(file: str) -> Model:
     --------
     >>> obj = import_model("my-model.yaml")
     """
-    return processor.import_context_entity(file)
+    return context_processor.import_context_entity(file)
 
 
 def load_model(file: str) -> Model:
@@ -271,7 +272,7 @@ def load_model(file: str) -> Model:
     --------
     >>> obj = load_model("my-model.yaml")
     """
-    return processor.load_context_entity(file)
+    return context_processor.load_context_entity(file)
 
 
 def update_model(entity: Model) -> Model:
@@ -292,7 +293,7 @@ def update_model(entity: Model) -> Model:
     --------
     >>> obj = get_model("store://my-model-key")
     """
-    return processor.update_context_entity(
+    return context_processor.update_context_entity(
         project=entity.project,
         entity_type=entity.ENTITY_TYPE,
         entity_id=entity.id,
@@ -338,7 +339,7 @@ def delete_model(
     >>>                    project="my-project",
     >>>                    delete_all_versions=True)
     """
-    return processor.delete_context_entity(
+    return context_processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,