digitalhub 0.9.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

digitalhub/entities/model/huggingface/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from pydantic import Field
 
 from digitalhub.entities.model._base.spec import ModelSpec, ModelValidator
@@ -15,13 +17,13 @@ class ModelSpecHuggingface(ModelSpec):
         path: str,
         framework: str | None = None,
         algorithm: str | None = None,
-        base_model: str | None = None,
         parameters: dict | None = None,
-        metrics: dict | None = None,
+        base_model: str | None = None,
         model_id: str | None = None,
-        model_revision: str = None,
+        model_revision: str | None = None,
     ) -> None:
-        super().__init__(path, framework, algorithm, base_model, parameters, metrics)
+        super().__init__(path, framework, algorithm, parameters)
+        self.base_model = base_model
         self.model_id = model_id
         self.model_revision = model_revision
 
@@ -31,6 +33,9 @@ class ModelValidatorHuggingface(ModelValidator):
     ModelValidatorHuggingface validator.
     """
 
+    base_model: Optional[str] = None
+    """Base model."""
+
     placeholder_model_id: str = Field(default=None, alias="model_id")
     """Huggingface model id. If not specified, the model is loaded from the model path."""
 

digitalhub/entities/model/mlflow/models.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 
 class Signature(BaseModel):
@@ -26,7 +26,7 @@ class Dataset(BaseModel):
     name: Optional[str] = None
     digest: Optional[str] = None
     profile: Optional[str] = None
-    schema_: Optional[str] = Field(default=None, alias="schema")
+    dataset_schema: Optional[str] = None
     source: Optional[str] = None
     source_type: Optional[str] = None
 

digitalhub/entities/model/mlflow/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from pydantic import Field
 
 from digitalhub.entities.model._base.spec import ModelSpec, ModelValidator
@@ -16,15 +18,13 @@ class ModelSpecMlflow(ModelSpec):
         path: str,
         framework: str | None = None,
         algorithm: str | None = None,
-        base_model: str | None = None,
         parameters: dict | None = None,
-        metrics: dict | None = None,
         flavor: str | None = None,
         model_config: dict | None = None,
         input_datasets: list[Dataset] | None = None,
-        signature: Signature = None,
+        signature: Signature | None = None,
     ) -> None:
-        super().__init__(path, framework, algorithm, base_model, parameters, metrics)
+        super().__init__(path, framework, algorithm, parameters)
         self.flavor = flavor
         self.model_config = model_config
         self.input_datasets = input_datasets
@@ -36,14 +36,14 @@ class ModelValidatorMlflow(ModelValidator):
     ModelValidatorMlflow validator.
     """
 
-    flavor: str = None
+    flavor: Optional[str] = None
     """Mlflow model flavor."""
 
     placeholder_cfg_: dict = Field(default=None, alias="model_config")
     """Mlflow model config."""
 
-    input_datasets: list[Dataset] = None
+    input_datasets: Optional[list[Dataset]] = None
     """Mlflow input datasets."""
 
-    signature: Signature = None
+    signature: Optional[Signature] = None
     """Mlflow model signature."""

digitalhub/entities/model/mlflow/utils.py

@@ -1,11 +1,32 @@
 from __future__ import annotations
 
+import typing
 from urllib.parse import urlparse
 
 import mlflow
 
 from digitalhub.entities.model.mlflow.models import Dataset, Signature
 
+if typing.TYPE_CHECKING:
+    from mlflow.entities import Run
+
+
+def get_mlflow_run(run_id: str) -> Run:
+    """
+    Get MLFlow run.
+
+    Parameters
+    ----------
+    run_id : str
+        The id of the mlflow run.
+
+    Returns
+    -------
+    Run
+        The extracted run.
+    """
+    return mlflow.MlflowClient().get_run(run_id)
+
 
 def from_mlflow_run(run_id: str) -> dict:
     """
@@ -23,13 +44,11 @@ def from_mlflow_run(run_id: str) -> dict:
     """
 
     # Get MLFlow run
-    client = mlflow.MlflowClient()
-    run = client.get_run(run_id)
+    run = get_mlflow_run(run_id)
 
     # Extract spec
     data = run.data
     parameters = data.params
-    metrics = data.metrics
     source_path = urlparse(run.info.artifact_uri).path
     model_uri = f"runs:/{run_id}/model"
     model = mlflow.pyfunc.load_model(model_uri=model_uri)
@@ -56,7 +75,7 @@ def from_mlflow_run(run_id: str) -> dict:
                 name=d.dataset.name,
                 digest=d.dataset.digest,
                 profile=d.dataset.profile,
-                schema=d.dataset.schema,
+                dataset_schema=d.dataset.schema,
                 source=d.dataset.source,
                 source_type=d.dataset.source_type,
             ).to_dict()
@@ -72,7 +91,6 @@ def from_mlflow_run(run_id: str) -> dict:
     # common properties
     model_params["framework"] = flavor
     model_params["parameters"] = parameters
-    model_params["metrics"] = metrics
 
     # specific to MLFlow
     model_params["flavor"] = flavor
@@ -81,3 +99,24 @@ def from_mlflow_run(run_id: str) -> dict:
     model_params["signature"] = signature
 
     return model_params
+
+
+def get_mlflow_model_metrics(run_id: str) -> dict:
+    """
+    Get MLFlow model metrics for a given run id.
+
+    Parameters
+    ----------
+    run_id : str
+        The id of the mlflow run.
+
+    Returns
+    -------
+    dict
+        The extracted metrics.
+    """
+    # Get MLFlow run
+    run = get_mlflow_run(run_id)
+
+    # Extract metrics
+    return run.data.metrics

digitalhub/entities/project/_base/entity.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 import typing
+from pathlib import Path
 from typing import Any
 
-from digitalhub.entities._base.project.entity import ProjectEntity
+from digitalhub.client.api import get_client
+from digitalhub.context.api import build_context
+from digitalhub.entities._base.entity.entity import Entity
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.base import base_processor
+from digitalhub.entities._processors.context import context_processor
 from digitalhub.entities.artifact.crud import (
     delete_artifact,
     get_artifact,
@@ -64,9 +68,14 @@ from digitalhub.entities.workflow.crud import (
     new_workflow,
     update_workflow,
 )
-from digitalhub.utils.exceptions import EntityError
+from digitalhub.factory.api import build_entity_from_dict
+from digitalhub.utils.exceptions import BackendError, EntityAlreadyExistsError, EntityError
+from digitalhub.utils.generic_utils import get_timestamp
+from digitalhub.utils.io_utils import write_yaml
+from digitalhub.utils.uri_utils import has_local_scheme
 
 if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.context.entity import ContextEntity
     from digitalhub.entities._base.entity.metadata import Metadata
     from digitalhub.entities.artifact._base.entity import Artifact
     from digitalhub.entities.dataitem._base.entity import Dataitem
@@ -79,7 +88,7 @@ if typing.TYPE_CHECKING:
     from digitalhub.entities.workflow._base.entity import Workflow
 
 
-class Project(ProjectEntity):
+class Project(Entity):
     """
     A class representing a project.
     """
@@ -96,9 +105,308 @@ class Project(ProjectEntity):
         user: str | None = None,
         local: bool = False,
     ) -> None:
-        super().__init__(name, kind, metadata, spec, status, user, local)
-        self.spec: ProjectSpec
-        self.status: ProjectStatus
+        super().__init__(kind, metadata, spec, status, user)
+        self.id = name
+        self.name = name
+        self.key = base_processor.build_project_key(self.name, local=local)
+
+        self._obj_attr.extend(["id", "name"])
+
+        # Set client
+        self._client = get_client(local)
+
+        # Set context
+        build_context(self)
+
+    ##############################
+    #  Save / Refresh / Export
+    ##############################
+
+    def save(self, update: bool = False) -> Project:
+        """
+        Save entity into backend.
+
+        Parameters
+        ----------
+        update : bool
+            If True, the object will be updated.
+
+        Returns
+        -------
+        Project
+            Entity saved.
+        """
+        if update:
+            if self._client.is_local():
+                self.metadata.updated = get_timestamp()
+            new_obj = base_processor.update_project_entity(
+                entity_type=self.ENTITY_TYPE,
+                entity_name=self.name,
+                entity_dict=self.to_dict(),
+                local=self._client.is_local(),
+            )
+        else:
+            new_obj = base_processor.create_project_entity(_entity=self)
+        self._update_attributes(new_obj)
+        return self
+
+    def refresh(self) -> Project:
+        """
+        Refresh object from backend.
+
+        Returns
+        -------
+        Project
+            Project object.
+        """
+        new_obj = base_processor.read_project_entity(
+            entity_type=self.ENTITY_TYPE,
+            entity_name=self.name,
+            local=self._client.is_local(),
+        )
+        self._update_attributes(new_obj)
+        return self
+
+    def search_entity(
+        self,
+        query: str | None = None,
+        entity_types: list[str] | None = None,
+        name: str | None = None,
+        kind: str | None = None,
+        created: str | None = None,
+        updated: str | None = None,
+        description: str | None = None,
+        labels: list[str] | None = None,
+        **kwargs,
+    ) -> list[ContextEntity]:
+        """
+        Search objects from backend.
+
+        Parameters
+        ----------
+        query : str
+            Search query.
+        entity_types : list[str]
+            Entity types.
+        name : str
+            Entity name.
+        kind : str
+            Entity kind.
+        created : str
+            Entity creation date.
+        updated : str
+            Entity update date.
+        description : str
+            Entity description.
+        labels : list[str]
+            Entity labels.
+        **kwargs : dict
+            Parameters to pass to the API call.
+
+            Returns
+            -------
+            list[ContextEntity]
+                List of object instances.
+        """
+        objs = context_processor.search_entity(
+            self.name,
+            query=query,
+            entity_types=entity_types,
+            name=name,
+            kind=kind,
+            created=created,
+            updated=updated,
+            description=description,
+            labels=labels,
+            **kwargs,
+        )
+        self.refresh()
+        return objs
+
+    def export(self) -> str:
+        """
+        Export object as a YAML file in the context folder.
+        If the objects are not embedded, the objects are exported as a YAML file.
+
+        Returns
+        -------
+        str
+            Exported filepath.
+        """
+        obj = self._refresh_to_dict()
+        pth = Path(self.spec.context) / f"{self.ENTITY_TYPE}s-{self.name}.yaml"
+        obj = self._export_not_embedded(obj)
+        write_yaml(pth, obj)
+        return str(pth)
+
+    def _refresh_to_dict(self) -> dict:
+        """
+        Try to refresh object to collect entities related to project.
+
+        Returns
+        -------
+        dict
+            Entity object in dictionary format.
+        """
+        try:
+            return self.refresh().to_dict()
+        except BackendError:
+            return self.to_dict()
+
+    def _export_not_embedded(self, obj: dict) -> dict:
+        """
+        Export project objects if not embedded.
+
+        Parameters
+        ----------
+        obj : dict
+            Project object in dictionary format.
+
+        Returns
+        -------
+        dict
+            Updatated project object in dictionary format with referenced entities.
+        """
+        # Cycle over entity types
+        for entity_type in self._get_entity_types():
+            # Entity types are stored as a list of entities
+            for idx, entity in enumerate(obj.get("spec", {}).get(entity_type, [])):
+                # Export entity if not embedded is in metadata, else do nothing
+                if not self._is_embedded(entity):
+                    # Get entity object from backend
+                    ent = context_processor.read_context_entity(entity["key"])
+
+                    # Export and store ref in object metadata inside project
+                    pth = ent.export()
+                    obj["spec"][entity_type][idx]["metadata"]["ref"] = pth
+
+        # Return updated object
+        return obj
+
+    def _import_entities(self, obj: dict) -> None:
+        """
+        Import project entities.
+
+        Parameters
+        ----------
+        obj : dict
+            Project object in dictionary format.
+
+        Returns
+        -------
+        None
+        """
+        entity_types = self._get_entity_types()
+
+        # Cycle over entity types
+        for entity_type in entity_types:
+            # Entity types are stored as a list of entities
+            for entity in obj.get("spec", {}).get(entity_type, []):
+                embedded = self._is_embedded(entity)
+                ref = entity["metadata"].get("ref")
+
+                # Import entity if not embedded and there is a ref
+                if not embedded and ref is not None:
+                    # Import entity from local ref
+                    if has_local_scheme(ref):
+                        try:
+                            # Artifacts, Dataitems and Models
+                            if entity_type in entity_types[:3]:
+                                context_processor.import_context_entity(ref)
+
+                            # Functions and Workflows
+                            elif entity_type in entity_types[3:]:
+                                context_processor.import_executable_entity(ref)
+
+                        except FileNotFoundError:
+                            msg = f"File not found: {ref}."
+                            raise EntityError(msg)
+
+                # If entity is embedded, create it and try to save
+                elif embedded:
+                    # It's possible that embedded field in metadata is not shown
+                    if entity["metadata"].get("embedded") is None:
+                        entity["metadata"]["embedded"] = True
+
+                    try:
+                        build_entity_from_dict(entity).save()
+                    except EntityAlreadyExistsError:
+                        pass
+
+    def _load_entities(self, obj: dict) -> None:
+        """
+        Load project entities.
+
+        Parameters
+        ----------
+        obj : dict
+            Project object in dictionary format.
+
+        Returns
+        -------
+        None
+        """
+        entity_types = self._get_entity_types()
+
+        # Cycle over entity types
+        for entity_type in entity_types:
+            # Entity types are stored as a list of entities
+            for entity in obj.get("spec", {}).get(entity_type, []):
+                embedded = self._is_embedded(entity)
+                ref = entity["metadata"].get("ref")
+
+                # Load entity if not embedded and there is a ref
+                if not embedded and ref is not None:
+                    # Load entity from local ref
+                    if has_local_scheme(ref):
+                        try:
+                            # Artifacts, Dataitems and Models
+                            if entity_type in entity_types[:3]:
+                                context_processor.load_context_entity(ref)
+
+                            # Functions and Workflows
+                            elif entity_type in entity_types[3:]:
+                                context_processor.load_executable_entity(ref)
+
+                        except FileNotFoundError:
+                            msg = f"File not found: {ref}."
+                            raise EntityError(msg)
+
+    def _is_embedded(self, entity: dict) -> bool:
+        """
+        Check if entity is embedded.
+
+        Parameters
+        ----------
+        entity : dict
+            Entity in dictionary format.
+
+        Returns
+        -------
+        bool
+            True if entity is embedded.
+        """
+        metadata_embedded = entity["metadata"].get("embedded", False)
+        no_status = entity.get("status", None) is None
+        no_spec = entity.get("spec", None) is None
+        return metadata_embedded or not (no_status and no_spec)
+
+    def _get_entity_types(self) -> list[str]:
+        """
+        Get entity types.
+
+        Returns
+        -------
+        list
+            Entity types.
+        """
+        return [
+            f"{EntityTypes.ARTIFACT.value}s",
+            f"{EntityTypes.DATAITEM.value}s",
+            f"{EntityTypes.MODEL.value}s",
+            f"{EntityTypes.FUNCTION.value}s",
+            f"{EntityTypes.WORKFLOW.value}s",
+        ]
 
     ##############################
     #  Artifacts
@@ -1859,7 +2167,7 @@ class Project(ProjectEntity):
         -------
         None
         """
-        return processor.share_project_entity(
+        return base_processor.share_project_entity(
             entity_type=self.ENTITY_TYPE,
             entity_name=self.name,
             user=user,
@@ -1879,7 +2187,7 @@ class Project(ProjectEntity):
         -------
         None
         """
-        return processor.share_project_entity(
+        return base_processor.share_project_entity(
             entity_type=self.ENTITY_TYPE,
             entity_name=self.name,
             user=user,

digitalhub/entities/project/_base/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from digitalhub.entities._base.entity.spec import Spec, SpecValidator
 
 
@@ -31,20 +33,20 @@ class ProjectValidator(SpecValidator):
     ProjectValidator validator.
     """
 
-    context: str = None
+    context: Optional[str] = None
     """The project's context."""
 
-    functions: list = None
+    functions: Optional[list] = None
     """List of project's functions."""
 
-    artifacts: list = None
+    artifacts: Optional[list] = None
     """List of project's artifacts."""
 
-    workflows: list = None
+    workflows: Optional[list] = None
     """List of project's workflows."""
 
-    dataitems: list = None
+    dataitems: Optional[list] = None
     """List of project's dataitems."""
 
-    models: list = None
+    models: Optional[list] = None
     """List of project's models."""

digitalhub/entities/project/crud.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.base import base_processor
+from digitalhub.entities._processors.context import context_processor
 from digitalhub.entities.project.utils import setup_project
 from digitalhub.utils.exceptions import BackendError
 
@@ -58,7 +59,7 @@ def new_project(
     """
     if context is None:
         context = "./"
-    obj = processor.create_project_entity(
+    obj = base_processor.create_project_entity(
         name=name,
         kind="project",
         description=description,
@@ -103,7 +104,7 @@ def get_project(
     --------
     >>> obj = get_project("my-project")
     """
-    obj = processor.read_project_entity(
+    obj = base_processor.read_project_entity(
         entity_type=ENTITY_TYPE,
         entity_name=name,
         local=local,
@@ -142,7 +143,7 @@ def import_project(
     --------
     >>> obj = import_project("my-project.yaml")
     """
-    obj = processor.import_project_entity(file=file, local=local, config=config)
+    obj = base_processor.import_project_entity(file=file, local=local, config=config)
     return setup_project(obj, setup_kwargs)
 
 
@@ -175,7 +176,7 @@ def load_project(
     --------
     >>> obj = load_project("my-project.yaml")
     """
-    obj = processor.load_project_entity(file=file, local=local, config=config)
+    obj = base_processor.load_project_entity(file=file, local=local, config=config)
     return setup_project(obj, setup_kwargs)
 
 
@@ -195,7 +196,7 @@ def list_projects(local: bool = False, **kwargs) -> list[Project]:
     list
         List of objects.
     """
-    return processor.list_project_entities(local=local, **kwargs)
+    return base_processor.list_project_entities(local=local, **kwargs)
 
 
 def get_or_create_project(
@@ -268,11 +269,11 @@ def update_project(entity: Project, **kwargs) -> Project:
     --------
     >>> obj = update_project(obj)
     """
-    return processor.update_project_entity(
+    return base_processor.update_project_entity(
         entity_type=entity.ENTITY_TYPE,
         entity_name=entity.name,
         entity_dict=entity.to_dict(),
-        local=entity.local,
+        local=entity._client.is_local(),
         **kwargs,
     )
 
@@ -283,7 +284,7 @@ def delete_project(
     clean_context: bool = True,
     local: bool = False,
     **kwargs,
-) -> list[dict]:
+) -> dict:
     """
     Delete a project.
 
@@ -309,7 +310,7 @@ def delete_project(
     --------
     >>> delete_project("my-project")
     """
-    return processor.delete_project_entity(
+    return base_processor.delete_project_entity(
         entity_type=ENTITY_TYPE,
         entity_name=name,
         local=local,
@@ -362,7 +363,7 @@ def search_entity(
         list[ContextEntity]
             List of object instances.
     """
-    return processor.search_entity(
+    return context_processor.search_entity(
         project_name,
         query=query,
         entity_types=entity_types,