digitalhub 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

digitalhub/entities/dataitem/utils.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import shutil
+import typing
+from typing import Any
+
+from digitalhub.context.api import get_context
+from digitalhub.entities._base.entity._constructors.uuid import build_uuid
+from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
+from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
+from digitalhub.readers._commons.enums import Extensions
+from digitalhub.readers.api import get_reader_by_object
+from digitalhub.utils.generic_utils import slugify_string
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities.dataitem._base.entity import Dataitem
+
+
+def eval_source(
+    source: str | list[str] | None = None,
+    data: Any | None = None,
+    kind: str | None = None,
+    name: str | None = None,
+    project: str | None = None,
+) -> Any:
+    """
+    Evaluate if source is local.
+
+    Parameters
+    ----------
+    source : str | list[str]
+        Source(s).
+
+    Returns
+    -------
+    None
+    """
+    if (source is None) == (data is None):
+        raise ValueError("You must provide source or data.")
+
+    if source is not None:
+        return eval_local_source(source)
+
+    if kind == EntityKinds.DATAITEM_TABLE.value:
+        ctx = get_context(project)
+        pth = ctx.root / f"{slugify_string(name)}.{Extensions.PARQUET.value}"
+        reader = get_reader_by_object(data)
+        reader.write_parquet(data, pth)
+        return str(pth)
+
+    raise NotImplementedError
+
+
+def process_kwargs(
+    project: str,
+    name: str,
+    kind: str,
+    source: str | list[str],
+    data: Any | None = None,
+    path: str | None = None,
+    **kwargs,
+) -> dict:
+    """
+    Process spec kwargs.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    name : str
+        Object name.
+    kind : str
+        Kind the object.
+    source : str
+        Source(s).
+    data : Any
+        Dataframe to log. Alternative to source.
+    path : str
+        Destination path of the entity. If not provided, it's generated.
+    **kwargs : dict
+        Spec parameters.
+
+    Returns
+    -------
+    dict
+        Kwargs updated.
+    """
+    if data is not None:
+        if kind == EntityKinds.DATAITEM_TABLE.value:
+            reader = get_reader_by_object(data)
+            kwargs["schema"] = reader.get_schema(data)
+    if path is None:
+        uuid = build_uuid()
+        kwargs["uuid"] = uuid
+        kwargs["path"] = build_log_path_from_source(project, EntityTypes.DATAITEM.value, name, uuid, source)
+    else:
+        kwargs["path"] = path
+    return kwargs
+
+
+def clean_tmp_path(pth: str) -> None:
+    """
+    Clean temporary path.
+
+    Parameters
+    ----------
+    pth : str
+        Path to clean.
+
+    Returns
+    -------
+    None
+    """
+    shutil.rmtree(pth, ignore_errors=True)
+
+
+def post_process(obj: Dataitem, data: Any) -> Dataitem:
+    """
+    Post process object.
+
+    Parameters
+    ----------
+    obj : Dataitem
+        The object.
+    data : Any
+        The data.
+
+    Returns
+    -------
+    Dataitem
+        The object.
+    """
+    if obj.kind == EntityKinds.DATAITEM_TABLE.value:
+        reader = get_reader_by_object(data)
+        obj.status.preview = reader.get_preview(data)
+        obj.save(update=True)
+    return obj

digitalhub/entities/function/_base/builder.py

@@ -4,7 +4,7 @@ import typing
 
 from digitalhub.entities._base.runtime_entity.builder import RuntimeEntityBuilder
 from digitalhub.entities._base.versioned.builder import VersionedBuilder
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.function._base.entity import Function

digitalhub/entities/function/_base/entity.py

@@ -4,7 +4,7 @@ import typing
 from concurrent.futures import ThreadPoolExecutor
 
 from digitalhub.entities._base.executable.entity import ExecutableEntity
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes, Relationship
 from digitalhub.factory.api import get_run_kind, get_task_kind_from_action
 from digitalhub.utils.exceptions import BackendError
 
@@ -79,7 +79,11 @@ class Function(ExecutableEntity):
         task = self._get_or_create_task(task_kind)
 
         # Run function from task
-        run = task.run(run_kind, local_execution, **kwargs)
+        run = task.run(run_kind, save=False, local_execution=local_execution, **kwargs)
+
+        # Set as run's parent
+        run.add_relationship(Relationship.RUN_OF.value, self.key)
+        run.save()
 
         # If execution is done by DHCore backend, return the object
         if not local_execution:

digitalhub/entities/function/crud.py

@@ -2,15 +2,8 @@ from __future__ import annotations
 
 import typing
 
-from digitalhub.entities._base.crud import (
-    delete_entity,
-    get_context_entity_versions,
-    get_versioned_entity,
-    import_executable_entity,
-    list_context_entities,
-    new_context_entity,
-)
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities._operations.processor import processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.function._base.entity import Function
@@ -63,7 +56,7 @@ def new_function(
     >>>                    code_src="function.py",
     >>>                    handler="function-handler")
     """
-    return new_context_entity(
+    return processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -110,7 +103,7 @@ def get_function(
     >>>                    project="my-project",
     >>>                    entity_id="my-function-id")
     """
-    return get_versioned_entity(
+    return processor.read_context_entity(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -150,7 +143,7 @@ def get_function_versions(
     >>> obj = get_function_versions("my-function-name"
     >>>                             project="my-project")
     """
-    return get_context_entity_versions(
+    return processor.read_context_entity_versions(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -178,7 +171,7 @@ def list_functions(project: str, **kwargs) -> list[Function]:
     --------
     >>> objs = list_functions(project="my-project")
     """
-    return list_context_entities(
+    return processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -187,7 +180,7 @@ def list_functions(project: str, **kwargs) -> list[Function]:
 
 def import_function(file: str) -> Function:
     """
-    Get object from file.
+    Import object from a YAML file and create a new object into the backend.
 
     Parameters
     ----------
@@ -203,7 +196,28 @@ def import_function(file: str) -> Function:
     --------
     >>> obj = import_function("my-function.yaml")
     """
-    return import_executable_entity(file)
+    return processor.import_executable_entity(file)
+
+
+def load_function(file: str) -> Function:
+    """
+    Load object from a YAML file and update an existing object into the backend.
+
+    Parameters
+    ----------
+    file : str
+        Path to YAML file.
+
+    Returns
+    -------
+    Function
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = load_function("my-function.yaml")
+    """
+    return processor.load_executable_entity(file)
 
 
 def update_function(entity: Function) -> Function:
@@ -224,7 +238,12 @@ def update_function(entity: Function) -> Function:
     --------
     >>> obj = update_function(obj)
     """
-    return entity.save(update=True)
+    return processor.update_context_entity(
+        project=entity.project,
+        entity_type=entity.ENTITY_TYPE,
+        entity_id=entity.id,
+        entity_dict=entity.to_dict(),
+    )
 
 
 def delete_function(
@@ -268,7 +287,7 @@ def delete_function(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return delete_entity(
+    return processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/model/_base/builder.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._base.versioned.builder import VersionedBuilder
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.utils.exceptions import EntityError
 
 if typing.TYPE_CHECKING:

digitalhub/entities/model/_base/entity.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._base.material.entity import MaterialEntity
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata

digitalhub/entities/model/crud.py

@@ -2,17 +2,9 @@ from __future__ import annotations
 
 import typing
 
-from digitalhub.entities._base.crud import (
-    delete_entity,
-    get_material_entity,
-    get_material_entity_versions,
-    import_context_entity,
-    list_material_entities,
-    new_context_entity,
-)
-from digitalhub.entities._base.entity._constructors.uuid import build_uuid
-from digitalhub.entities.utils.entity_types import EntityTypes
-from digitalhub.entities.utils.utils import build_log_path_from_source, eval_local_source
+from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities._operations.processor import processor
+from digitalhub.entities.artifact.utils import eval_source, process_kwargs
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.model._base.entity import Model
@@ -68,7 +60,7 @@ def new_model(
     >>>                    kind="model",
     >>>                    path="s3://my-bucket/my-key")
     """
-    return new_context_entity(
+    return processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -119,16 +111,15 @@ def log_model(
     >>>                 kind="model",
     >>>                 source="./local-path")
     """
-    eval_local_source(source)
-
-    if path is None:
-        uuid = build_uuid()
-        kwargs["uuid"] = uuid
-        path = build_log_path_from_source(project, ENTITY_TYPE, name, uuid, source)
-
-    obj = new_model(project=project, name=name, kind=kind, path=path, **kwargs)
-    obj.upload(source)
-    return obj
+    eval_source(source)
+    kwargs = process_kwargs(project, name, source=source, path=path, **kwargs)
+    return processor.log_material_entity(
+        source=source,
+        project=project,
+        name=name,
+        kind=kind,
+        **kwargs,
+    )
 
 
 def get_model(
@@ -166,7 +157,7 @@ def get_model(
     >>>                 project="my-project",
     >>>                 entity_id="my-model-id")
     """
-    return get_material_entity(
+    return processor.read_material_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -206,7 +197,7 @@ def get_model_versions(
     >>> objs = get_model_versions("my-model-name",
     >>>                           project="my-project")
     """
-    return get_material_entity_versions(
+    return processor.read_material_entity_versions(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -234,7 +225,7 @@ def list_models(project: str, **kwargs) -> list[Model]:
     --------
     >>> objs = list_models(project="my-project")
     """
-    return list_material_entities(
+    return processor.list_material_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -243,7 +234,7 @@ def list_models(project: str, **kwargs) -> list[Model]:
 
 def import_model(file: str) -> Model:
     """
-    Import object from a YAML file.
+    Import object from a YAML file and create a new object into the backend.
 
     Parameters
     ----------
@@ -259,7 +250,28 @@ def import_model(file: str) -> Model:
     --------
     >>> obj = import_model("my-model.yaml")
     """
-    return import_context_entity(file)
+    return processor.import_context_entity(file)
+
+
+def load_model(file: str) -> Model:
+    """
+    Load object from a YAML file and update an existing object into the backend.
+
+    Parameters
+    ----------
+    file : str
+        Path to YAML file.
+
+    Returns
+    -------
+    Model
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = load_model("my-model.yaml")
+    """
+    return processor.load_context_entity(file)
 
 
 def update_model(entity: Model) -> Model:
@@ -280,7 +292,12 @@ def update_model(entity: Model) -> Model:
     --------
     >>> obj = get_model("store://my-model-key")
     """
-    return entity.save(update=True)
+    return processor.update_context_entity(
+        project=entity.project,
+        entity_type=entity.ENTITY_TYPE,
+        entity_id=entity.id,
+        entity_dict=entity.to_dict(),
+    )
 
 
 def delete_model(
@@ -321,7 +338,7 @@ def delete_model(
     >>>                    project="my-project",
     >>>                    delete_all_versions=True)
     """
-    return delete_entity(
+    return processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/model/huggingface/builder.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from digitalhub.entities._commons.enums import EntityKinds
 from digitalhub.entities.model._base.builder import ModelBuilder
 from digitalhub.entities.model.huggingface.entity import ModelHuggingface
 from digitalhub.entities.model.huggingface.spec import ModelSpecHuggingface, ModelValidatorHuggingface
@@ -15,4 +16,4 @@ class ModelHuggingfaceBuilder(ModelBuilder):
     ENTITY_SPEC_CLASS = ModelSpecHuggingface
     ENTITY_SPEC_VALIDATOR = ModelValidatorHuggingface
     ENTITY_STATUS_CLASS = ModelStatusHuggingface
-    ENTITY_KIND = "huggingface"
+    ENTITY_KIND = EntityKinds.MODEL_HUGGINGFACE.value

digitalhub/entities/model/huggingface/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pydantic import Field
+
 from digitalhub.entities.model._base.spec import ModelSpec, ModelValidator
 
 
@@ -29,8 +31,8 @@ class ModelValidatorHuggingface(ModelValidator):
     ModelValidatorHuggingface validator.
     """
 
-    model_id: str = None
+    placeholder_model_id: str = Field(default=None, alias="model_id")
     """Huggingface model id. If not specified, the model is loaded from the model path."""
 
-    model_revision: str = None
+    placeholder_model_revision: str = Field(default=None, alias="model_revision")
     """Huggingface model revision."""

digitalhub/entities/model/mlflow/builder.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from digitalhub.entities._commons.enums import EntityKinds
 from digitalhub.entities.model._base.builder import ModelBuilder
 from digitalhub.entities.model.mlflow.entity import ModelMlflow
 from digitalhub.entities.model.mlflow.spec import ModelSpecMlflow, ModelValidatorMlflow
@@ -15,4 +16,4 @@ class ModelModelBuilder(ModelBuilder):
     ENTITY_SPEC_CLASS = ModelSpecMlflow
     ENTITY_SPEC_VALIDATOR = ModelValidatorMlflow
     ENTITY_STATUS_CLASS = ModelStatusMlflow
-    ENTITY_KIND = "mlflow"
+    ENTITY_KIND = EntityKinds.MODEL_MLFLOW.value

digitalhub/entities/model/mlflow/models.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from pydantic import BaseModel, Field
 
 
@@ -8,9 +10,12 @@ class Signature(BaseModel):
     MLFlow model signature.
     """
 
-    inputs: str = None
-    outputs: str = None
-    params: str = None
+    inputs: Optional[str] = None
+    outputs: Optional[str] = None
+    params: Optional[str] = None
+
+    def to_dict(self):
+        return self.model_dump()
 
 
 class Dataset(BaseModel):
@@ -18,9 +23,12 @@ class Dataset(BaseModel):
     MLFlow model dataset.
     """
 
-    name: str = None
-    digest: str = None
-    profile: str = None
-    schema_: str = Field(default=None, alias="schema")
-    source: str = None
-    source_type: str = None
+    name: Optional[str] = None
+    digest: Optional[str] = None
+    profile: Optional[str] = None
+    schema_: Optional[str] = Field(default=None, alias="schema")
+    source: Optional[str] = None
+    source_type: Optional[str] = None
+
+    def to_dict(self):
+        return self.model_dump()

digitalhub/entities/model/mlflow/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pydantic import Field
+
 from digitalhub.entities.model._base.spec import ModelSpec, ModelValidator
 from digitalhub.entities.model.mlflow.models import Dataset, Signature
 
@@ -36,9 +38,12 @@ class ModelValidatorMlflow(ModelValidator):
 
     flavor: str = None
     """Mlflow model flavor."""
-    model_config: dict = None
+
+    placeholder_cfg_: dict = Field(default=None, alias="model_config")
     """Mlflow model config."""
+
     input_datasets: list[Dataset] = None
     """Mlflow input datasets."""
+
     signature: Signature = None
     """Mlflow model signature."""

digitalhub/entities/model/mlflow/utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from urllib.parse import urlparse
 
 import mlflow
@@ -44,7 +46,7 @@ def from_mlflow_run(run_id: str) -> dict:
         inputs=mlflow_signature.inputs.to_json() if mlflow_signature.inputs else None,
         outputs=mlflow_signature.outputs.to_json() if mlflow_signature.outputs else None,
         params=mlflow_signature.params.to_json() if mlflow_signature.params else None,
-    ).dict()
+    ).to_dict()
 
     # Extract datasets
     datasets = []
@@ -57,7 +59,7 @@ def from_mlflow_run(run_id: str) -> dict:
                 schema=d.dataset.schema,
                 source=d.dataset.source,
                 source_type=d.dataset.source_type,
-            ).dict()
+            ).to_dict()
             for d in run.inputs.dataset_inputs
         ]
 

digitalhub/entities/model/model/builder.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from digitalhub.entities._commons.enums import EntityKinds
 from digitalhub.entities.model._base.builder import ModelBuilder
 from digitalhub.entities.model.model.entity import ModelModel
 from digitalhub.entities.model.model.spec import ModelSpecModel, ModelValidatorModel
@@ -15,4 +16,4 @@ class ModelMlflowBuilder(ModelBuilder):
     ENTITY_SPEC_CLASS = ModelSpecModel
     ENTITY_SPEC_VALIDATOR = ModelValidatorModel
     ENTITY_STATUS_CLASS = ModelStatusModel
-    ENTITY_KIND = "model"
+    ENTITY_KIND = EntityKinds.MODEL_MODEL.value

digitalhub/entities/model/sklearn/builder.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from digitalhub.entities._commons.enums import EntityKinds
 from digitalhub.entities.model._base.builder import ModelBuilder
 from digitalhub.entities.model.sklearn.entity import ModelSklearn
 from digitalhub.entities.model.sklearn.spec import ModelSpecSklearn, ModelValidatorSklearn
@@ -15,4 +16,4 @@ class ModelSklearnBuilder(ModelBuilder):
     ENTITY_SPEC_CLASS = ModelSpecSklearn
     ENTITY_SPEC_VALIDATOR = ModelValidatorSklearn
     ENTITY_STATUS_CLASS = ModelStatusSklearn
-    ENTITY_KIND = "sklearn"
+    ENTITY_KIND = EntityKinds.MODEL_SKLEARN.value

digitalhub/entities/model/utils.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import Any
+
+from digitalhub.entities._base.entity._constructors.uuid import build_uuid
+from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
+from digitalhub.entities._commons.enums import EntityTypes
+
+
+def eval_source(
+    source: str | list[str] | None = None,
+) -> Any:
+    """
+    Evaluate if source is local.
+
+    Parameters
+    ----------
+    source : str | list[str]
+        Source(s).
+
+    Returns
+    -------
+    None
+    """
+    return eval_local_source(source)
+
+
+def process_kwargs(
+    project: str,
+    name: str,
+    source: str | list[str],
+    path: str | None = None,
+    **kwargs,
+) -> dict:
+    """
+    Process spec kwargs.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    name : str
+        Object name.
+    source : str
+        Source(s).
+    path : str
+        Destination path of the entity. If not provided, it's generated.
+    **kwargs : dict
+        Spec parameters.
+
+    Returns
+    -------
+    dict
+        Kwargs updated.
+    """
+    if path is None:
+        uuid = build_uuid()
+        kwargs["uuid"] = uuid
+        kwargs["path"] = build_log_path_from_source(project, EntityTypes.MODEL.value, name, uuid, source)
+    else:
+        kwargs["path"] = path
+    return kwargs