digitalhub 0.10.0b2__py3-none-any.whl → 0.10.0b4__py3-none-any.whl

digitalhub/entities/_base/entity/builder.py

@@ -24,10 +24,10 @@ class EntityBuilder:
 
     # Class variables
     ENTITY_TYPE: str = None
-    ENTITY_CLASS: Entity = None
-    ENTITY_SPEC_CLASS: Spec = None
-    ENTITY_SPEC_VALIDATOR: SpecValidator = None
-    ENTITY_STATUS_CLASS: Status = None
+    ENTITY_CLASS: type[Entity] = None
+    ENTITY_SPEC_CLASS: type[Spec] = None
+    ENTITY_SPEC_VALIDATOR: type[SpecValidator] = None
+    ENTITY_STATUS_CLASS: type[Status] = None
     ENTITY_KIND: str = None
 
     def __init__(self) -> None:
@@ -60,7 +60,7 @@ class EntityBuilder:
         """
         return build_name(name)
 
-    def build_uuid(self, uuid: str) -> str:
+    def build_uuid(self, uuid: str | None = None) -> str:
         """
         Build entity uuid.
 

digitalhub/entities/_base/material/entity.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from digitalhub.entities._base.versioned.entity import VersionedEntity
 from digitalhub.entities._operations.processor import processor
 from digitalhub.stores.api import get_store
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -34,9 +35,6 @@ class MaterialEntity(VersionedEntity):
         self.spec: MaterialSpec
         self.status: MaterialStatus
 
-        # Init files info
-        self._init_files_info()
-
     def save(self, update: bool = False) -> MaterialEntity:
         """
         Save entity into backend.
@@ -139,7 +137,7 @@ class MaterialEntity(VersionedEntity):
 
         return store.download(self.spec.path, dst=dst, src=paths, overwrite=overwrite)
 
-    def upload(self, source: str | list[str]) -> None:
+    def upload(self, source: SourcesOrListOfSources) -> None:
         """
         Upload object from given local path to spec path destination.
         Source must be a local path. If the path is a folder, destination
@@ -178,17 +176,6 @@ class MaterialEntity(VersionedEntity):
     #  Public Helpers
     ##############################
 
-    def _init_files_info(self) -> None:
-        """
-        Initialize files info.
-
-        Returns
-        -------
-        None
-        """
-        if self.status.files is None:
-            self.status.files = []
-
     def add_files_info(self, files: list[dict]) -> None:
         """
         Add a file to the status.
@@ -207,13 +194,13 @@ class MaterialEntity(VersionedEntity):
             if f.get("path") not in path_list:
                 self.status.files.append(f)
 
-    def get_file_paths(self) -> list[str]:
+    def get_file_paths(self) -> list:
         """
         Get the paths of the files in the status.
 
         Returns
         -------
-        list[str]
+        list
             Paths of the files in the status.
         """
         return [f.get("path") for f in self.status.files]

digitalhub/entities/_base/material/status.py

@@ -15,4 +15,4 @@ class MaterialStatus(Status):
         files: list[dict] | None = None,
     ) -> None:
         super().__init__(state, message)
-        self.files = files
+        self.files = files if files is not None else []

digitalhub/entities/_commons/models.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
-from typing import Union
-
 from pydantic import BaseModel
 
+from digitalhub.utils.types import MetricType
+
 
 class Metric(BaseModel):
-    value: Union[float, int, list[Union[float, int]]]
+    """
+    Metric.
+    """
+
+    value: MetricType

digitalhub/entities/_commons/utils.py

@@ -6,9 +6,10 @@ from pydantic import ValidationError
 
 from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._commons.models import Metric
+from digitalhub.utils.types import MetricType
 
 
-def parse_entity_key(key: str) -> tuple[str]:
+def parse_entity_key(key: str) -> tuple[str, str, str, str | None, str]:
     """
     Parse the entity key. Returns project, entity type, kind, name and uuid.
 
@@ -19,7 +20,7 @@ def parse_entity_key(key: str) -> tuple[str]:
 
     Returns
     -------
-    tuple[str]
+    tuple[str, str, str, str | None, str]
         Project, entity type, kind, name and uuid.
     """
     try:
@@ -88,7 +89,7 @@ def get_project_from_key(key: str) -> str:
     return project
 
 
-def validate_metric_value(value: Any) -> float | int | list[float | int]:
+def validate_metric_value(value: Any) -> MetricType:
     """
     Validate metric value.
 
@@ -99,7 +100,7 @@ def validate_metric_value(value: Any) -> float | int | list[float | int]:
 
     Returns
     -------
-    float | int | list[float | int]
+    MetricType
         The validated value.
     """
     try:

digitalhub/entities/_operations/processor.py

@@ -10,6 +10,7 @@ from digitalhub.entities._commons.utils import get_project_from_key, parse_entit
 from digitalhub.factory.api import build_entity_from_dict, build_entity_from_params
 from digitalhub.utils.exceptions import ContextError, EntityAlreadyExistsError, EntityError, EntityNotExistsError
 from digitalhub.utils.io_utils import read_yaml
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.client._base.client import Client
@@ -605,7 +606,7 @@ class OperationsProcessor:
         MaterialEntity
             Object instance.
         """
-        source = kwargs.pop("source")
+        source: SourcesOrListOfSources = kwargs.pop("source")
         context = self._get_context(kwargs["project"])
         obj = build_entity_from_params(**kwargs)
         if context.is_running:
@@ -1382,7 +1383,7 @@ class OperationsProcessor:
             project=context.name,
             entity_type=entity_type,
         )
-        return context.client.update_object(api, data, **kwargs)
+        context.client.update_object(api, data, **kwargs)
 
     def read_run_logs(
         self,
@@ -1453,7 +1454,7 @@ class OperationsProcessor:
             entity_type=entity_type,
             entity_id=entity_id,
         )
-        return context.client.create_object(api, **kwargs)
+        context.client.create_object(api, **kwargs)
 
     def resume_run(
         self,
@@ -1488,7 +1489,7 @@ class OperationsProcessor:
             entity_type=entity_type,
             entity_id=entity_id,
         )
-        return context.client.create_object(api, **kwargs)
+        context.client.create_object(api, **kwargs)
 
     def read_files_info(
         self,

digitalhub/entities/artifact/artifact/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from digitalhub.entities.artifact._base.spec import ArtifactSpec, ArtifactValidator
 
 
@@ -23,5 +25,5 @@ class ArtifactValidatorArtifact(ArtifactValidator):
     ArtifactValidatorArtifact validator.
     """
 
-    src_path: str = None
+    src_path: Optional[str] = None
     """Source path of the artifact."""

digitalhub/entities/artifact/crud.py

@@ -6,6 +6,7 @@ from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._operations.processor import processor
 from digitalhub.entities.artifact._base.entity import Artifact
 from digitalhub.entities.artifact.utils import eval_source, process_kwargs
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.artifact._base.entity import Artifact
@@ -78,7 +79,7 @@ def log_artifact(
     project: str,
     name: str,
     kind: str,
-    source: list[str] | str,
+    source: SourcesOrListOfSources,
     path: str | None = None,
     **kwargs,
 ) -> Artifact:
@@ -93,7 +94,7 @@ def log_artifact(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Artifact location on local path.
     path : str
         Destination path of the artifact. If not provided, it's generated.

digitalhub/entities/builders.py

@@ -7,7 +7,7 @@ from digitalhub.entities.model.mlflow.builder import ModelModelBuilder
 from digitalhub.entities.project._base.builder import ProjectProjectBuilder
 from digitalhub.entities.secret._base.builder import SecretSecretBuilder
 
-entity_builders = (
+entity_builders: tuple = (
     (ProjectProjectBuilder.ENTITY_KIND, ProjectProjectBuilder),
     (SecretSecretBuilder.ENTITY_KIND, SecretSecretBuilder),
     (ArtifactArtifactBuilder.ENTITY_KIND, ArtifactArtifactBuilder),
@@ -17,17 +17,14 @@ entity_builders = (
 )
 
 ##############################
-#  Potential uninstalled entities
+#  Add custom entities here
 ##############################
 
 
 try:
     from digitalhub.entities.dataitem.iceberg.builder import DataitemIcebergBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (DataitemIcebergBuilder.ENTITY_KIND, DataitemIcebergBuilder),
-    )
+    entity_builders += ((DataitemIcebergBuilder.ENTITY_KIND, DataitemIcebergBuilder),)
 except ImportError:
     ...
 
@@ -35,29 +32,20 @@ except ImportError:
 try:
     from digitalhub.entities.model.model.builder import ModelMlflowBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (ModelMlflowBuilder.ENTITY_KIND, ModelMlflowBuilder),
-    )
+    entity_builders += ((ModelMlflowBuilder.ENTITY_KIND, ModelMlflowBuilder),)
 except ImportError:
     ...
 
 try:
     from digitalhub.entities.model.sklearn.builder import ModelSklearnBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (ModelSklearnBuilder.ENTITY_KIND, ModelSklearnBuilder),
-    )
+    entity_builders += ((ModelSklearnBuilder.ENTITY_KIND, ModelSklearnBuilder),)
 except ImportError:
     ...
 
 try:
     from digitalhub.entities.model.huggingface.builder import ModelHuggingfaceBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (ModelHuggingfaceBuilder.ENTITY_KIND, ModelHuggingfaceBuilder),
-    )
+    entity_builders += ((ModelHuggingfaceBuilder.ENTITY_KIND, ModelHuggingfaceBuilder),)
 except ImportError:
     ...

digitalhub/entities/dataitem/crud.py

@@ -5,7 +5,8 @@ from typing import Any
 
 from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._operations.processor import processor
-from digitalhub.entities.dataitem.utils import clean_tmp_path, eval_source, post_process, process_kwargs
+from digitalhub.entities.dataitem.utils import clean_tmp_path, eval_data, eval_source, post_process, process_kwargs
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.dataitem._base.entity import Dataitem
@@ -78,9 +79,11 @@ def log_dataitem(
     project: str,
     name: str,
     kind: str,
-    source: list[str] | str | None = None,
+    source: SourcesOrListOfSources | None = None,
     data: Any | None = None,
     path: str | None = None,
+    file_format: str | None = None,
+    engine: str | None = "pandas",
     **kwargs,
 ) -> Dataitem:
     """
@@ -94,12 +97,16 @@ def log_dataitem(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Dataitem location on local path.
     data : Any
         Dataframe to log. Alternative to source.
     path : str
         Destination path of the dataitem. If not provided, it's generated.
+    file_format : str
+        Extension of the file.
+    engine : str
+        Dataframe engine (pandas, polars, etc.).
     **kwargs : dict
         New dataitem spec parameters.
 
@@ -115,7 +122,12 @@ def log_dataitem(
     >>>                    kind="table",
     >>>                    data=df)
     """
+    cleanup = False
+    if data is not None:
+        cleanup = True
+
     source = eval_source(source, data, kind, name, project)
+    data = eval_data(project, kind, source, data, file_format, engine)
     kwargs = process_kwargs(project, name, kind, source=source, data=data, path=path, **kwargs)
     obj = processor.log_material_entity(
         source=source,
@@ -124,8 +136,8 @@ def log_dataitem(
         kind=kind,
         **kwargs,
     )
-    if data is not None:
-        obj = post_process(obj, data)
+    obj = post_process(obj, data)
+    if cleanup:
         clean_tmp_path(source)
     return obj
 

digitalhub/entities/dataitem/table/models.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -40,16 +41,16 @@ class TableSchemaFieldEntry(BaseModel):
     type_: FieldType = Field(alias="type")
     """Field type."""
 
-    title: str = None
+    title: Optional[str] = None
     """Field title."""
 
     format_: str = Field(default=None, alias="format")
     """Field format."""
 
-    example: str = None
+    example: Optional[str] = None
     """Field example."""
 
-    description: str = None
+    description: Optional[str] = None
     """Field description."""
 
 

digitalhub/entities/dataitem/table/utils.py

@@ -76,9 +76,9 @@ def filter_memoryview(data: list[dict]) -> list[dict]:
     return data
 
 
-def check_preview_size(preview: dict) -> list:
+def check_preview_size(preview: dict) -> dict:
     """
-    Check preview size. If it's too big, return empty list.
+    Check preview size. If it's too big, return empty dict.
 
     Parameters
     ----------
@@ -87,11 +87,11 @@ def check_preview_size(preview: dict) -> list:
 
     Returns
     -------
-    list
+    dict
         Preview.
     """
     if len(dump_json(preview)) >= 64000:
-        return []
+        return {}
     return preview
 
 
@@ -101,9 +101,9 @@ def finalize_preview(preview: list[dict] | None = None, rows_count: int | None =
 
     Parameters
     ----------
-    preview : list[dict] | None
+    preview : list[dict]
         Preview.
-    rows_count : int | None
+    rows_count : int
         Row count.
 
     Returns
@@ -111,7 +111,7 @@ def finalize_preview(preview: list[dict] | None = None, rows_count: int | None =
     dict
         Data preview.
     """
-    data = {}
+    data: dict[str, list[dict] | int] = {}
     if preview is not None:
         data["cols"] = preview
     if rows_count is not None:

digitalhub/entities/dataitem/utils.py

@@ -9,7 +9,9 @@ from digitalhub.entities._base.entity._constructors.uuid import build_uuid
 from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
 from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
 from digitalhub.readers.data.api import get_reader_by_object
+from digitalhub.stores.api import get_store
 from digitalhub.utils.generic_utils import slugify_string
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.dataitem._base.entity import Dataitem
@@ -19,7 +21,7 @@ DEFAULT_EXTENSION = "parquet"
 
 
 def eval_source(
-    source: str | list[str] | None = None,
+    source: SourcesOrListOfSources | None = None,
     data: Any | None = None,
     kind: str | None = None,
     name: str | None = None,
@@ -30,7 +32,7 @@ def eval_source(
 
     Parameters
     ----------
-    source : str | list[str]
+    source : SourcesOrListOfSources
         Source(s).
 
     Returns
@@ -54,11 +56,49 @@ def eval_source(
     raise NotImplementedError
 
 
+def eval_data(
+    project: str,
+    kind: str,
+    source: SourcesOrListOfSources,
+    data: Any | None = None,
+    file_format: str | None = None,
+    engine: str | None = None,
+) -> Any:
+    """
+    Evaluate data is loaded.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    source : str
+        Source(s).
+    data : Any
+        Dataframe to log. Alternative to source.
+    file_format : str
+        Extension of the file.
+    engine : str
+        Engine to use.
+
+    Returns
+    -------
+    None
+    """
+    if kind == EntityKinds.DATAITEM_TABLE.value:
+        if data is None:
+            return get_store(project, source).read_df(
+                source,
+                file_format=file_format,
+                engine=engine,
+            )
+    return data
+
+
 def process_kwargs(
     project: str,
     name: str,
     kind: str,
-    source: str | list[str],
+    source: SourcesOrListOfSources,
     data: Any | None = None,
     path: str | None = None,
     **kwargs,
@@ -74,7 +114,7 @@ def process_kwargs(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Source(s).
     data : Any
         Dataframe to log. Alternative to source.
@@ -101,19 +141,23 @@ def process_kwargs(
     return kwargs
 
 
-def clean_tmp_path(pth: str) -> None:
+def clean_tmp_path(pth: SourcesOrListOfSources) -> None:
     """
     Clean temporary path.
 
     Parameters
     ----------
-    pth : str
+    pth : SourcesOrListOfSources
         Path to clean.
 
     Returns
     -------
     None
     """
+    if isinstance(pth, list):
+        for p in pth:
+            shutil.rmtree(p, ignore_errors=True)
+        return
     shutil.rmtree(pth, ignore_errors=True)
 
 

digitalhub/entities/model/_base/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from digitalhub.entities._base.material.spec import MaterialSpec, MaterialValidator
 
 
@@ -15,7 +17,7 @@ class ModelSpec(MaterialSpec):
         algorithm: str | None = None,
         parameters: dict | None = None,
     ) -> None:
-        self.path = path
+        super().__init__(path)
         self.framework = framework
         self.algorithm = algorithm
         self.parameters = parameters
@@ -26,14 +28,11 @@ class ModelValidator(MaterialValidator):
     ModelValidator validator.
     """
 
-    path: str
-    """Path to the model."""
-
-    framework: str = None
+    framework: Optional[str] = None
     """Model framework (e.g. 'pytorch')."""
 
-    algorithm: str = None
+    algorithm: Optional[str] = None
     """Model algorithm (e.g. 'resnet')."""
 
-    parameters: dict = None
+    parameters: Optional[dict] = None
     """Model validator."""

digitalhub/entities/model/crud.py

@@ -5,6 +5,7 @@ import typing
 from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._operations.processor import processor
 from digitalhub.entities.artifact.utils import eval_source, process_kwargs
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.model._base.entity import Model
@@ -77,7 +78,7 @@ def log_model(
     project: str,
     name: str,
     kind: str,
-    source: list[str] | str,
+    source: SourcesOrListOfSources,
     path: str | None = None,
     **kwargs,
 ) -> Model:
@@ -92,7 +93,7 @@ def log_model(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Model location on local path.
     path : str
         Destination path of the model. If not provided, it's generated.

digitalhub/entities/model/huggingface/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from pydantic import Field
 
 from digitalhub.entities.model._base.spec import ModelSpec, ModelValidator
@@ -18,7 +20,7 @@ class ModelSpecHuggingface(ModelSpec):
         parameters: dict | None = None,
         base_model: str | None = None,
         model_id: str | None = None,
-        model_revision: str = None,
+        model_revision: str | None = None,
     ) -> None:
         super().__init__(path, framework, algorithm, parameters)
         self.base_model = base_model
@@ -31,7 +33,7 @@ class ModelValidatorHuggingface(ModelValidator):
     ModelValidatorHuggingface validator.
     """
 
-    base_model: str = None
+    base_model: Optional[str] = None
     """Base model."""
 
     placeholder_model_id: str = Field(default=None, alias="model_id")

digitalhub/entities/model/mlflow/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from pydantic import Field
 
 from digitalhub.entities.model._base.spec import ModelSpec, ModelValidator
@@ -20,7 +22,7 @@ class ModelSpecMlflow(ModelSpec):
         flavor: str | None = None,
         model_config: dict | None = None,
         input_datasets: list[Dataset] | None = None,
-        signature: Signature = None,
+        signature: Signature | None = None,
     ) -> None:
         super().__init__(path, framework, algorithm, parameters)
         self.flavor = flavor
@@ -34,14 +36,14 @@ class ModelValidatorMlflow(ModelValidator):
     ModelValidatorMlflow validator.
     """
 
-    flavor: str = None
+    flavor: Optional[str] = None
     """Mlflow model flavor."""
 
     placeholder_cfg_: dict = Field(default=None, alias="model_config")
     """Mlflow model config."""
 
-    input_datasets: list[Dataset] = None
+    input_datasets: Optional[list[Dataset]] = None
     """Mlflow input datasets."""
 
-    signature: Signature = None
+    signature: Optional[Signature] = None
     """Mlflow model signature."""