digitalhub 0.10.3__py3-none-any.whl → 0.11.0__py3-none-any.whl

digitalhub/entities/_processors/context.py

@@ -3,13 +3,13 @@ from __future__ import annotations
 import typing
 from typing import Any
 
-from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, Relationship
+from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, Relationship, State
 from digitalhub.entities._processors.utils import (
     get_context_from_identifier,
     get_context_from_project,
     parse_identifier,
 )
-from digitalhub.factory.api import build_entity_from_dict, build_entity_from_params
+from digitalhub.factory.factory import factory
 from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError, EntityNotExistsError
 from digitalhub.utils.io_utils import read_yaml
 from digitalhub.utils.types import SourcesOrListOfSources
@@ -91,9 +91,9 @@ class ContextEntityOperationsProcessor:
             obj = _entity
         else:
             context = get_context_from_project(kwargs["project"])
-            obj: ContextEntity = build_entity_from_params(**kwargs)
+            obj: ContextEntity = factory.build_entity_from_params(**kwargs)
         new_obj = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
-        return build_entity_from_dict(new_obj)
+        return factory.build_entity_from_dict(new_obj)
 
     def log_material_entity(
         self,
@@ -114,13 +114,36 @@ class ContextEntityOperationsProcessor:
         """
         source: SourcesOrListOfSources = kwargs.pop("source")
         context = get_context_from_project(kwargs["project"])
-        obj = build_entity_from_params(**kwargs)
+        obj = factory.build_entity_from_params(**kwargs)
         if context.is_running:
             obj.add_relationship(Relationship.PRODUCEDBY.value, context.get_run_ctx())
 
         new_obj: MaterialEntity = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
-        new_obj = build_entity_from_dict(new_obj)
-        new_obj.upload(source)
+        new_obj = factory.build_entity_from_dict(new_obj)
+
+        new_obj.status.state = State.UPLOADING.value
+        new_obj = self._update_material_entity(new_obj)
+
+        # Handle file upload
+        try:
+            new_obj.upload(source)
+            uploaded = True
+            msg = None
+        except Exception as e:
+            uploaded = False
+            msg = str(e)
+
+        new_obj.status.message = msg
+
+        # Update status after upload
+        if uploaded:
+            new_obj.status.state = State.READY.value
+            new_obj = self._update_material_entity(new_obj)
+        else:
+            new_obj.status.state = State.ERROR.value
+            new_obj = self._update_material_entity(new_obj)
+            raise EntityError(msg)
+
         return new_obj
 
     def _read_context_entity(
@@ -226,7 +249,7 @@ class ContextEntityOperationsProcessor:
             entity_id=entity_id,
             **kwargs,
         )
-        entity = build_entity_from_dict(obj)
+        entity = factory.build_entity_from_dict(obj)
         return self._post_process_get(entity)
 
     def read_unversioned_entity(
@@ -292,7 +315,7 @@ class ContextEntityOperationsProcessor:
         dict_obj: dict = read_yaml(file)
         dict_obj["status"] = {}
         context = get_context_from_project(dict_obj["project"])
-        obj = build_entity_from_dict(dict_obj)
+        obj = factory.build_entity_from_dict(dict_obj)
         try:
             self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
         except EntityAlreadyExistsError:
@@ -329,7 +352,7 @@ class ContextEntityOperationsProcessor:
             tsk_dicts = []
 
         context = get_context_from_project(exec_dict["project"])
-        obj: ExecutableEntity = build_entity_from_dict(exec_dict)
+        obj: ExecutableEntity = factory.build_entity_from_dict(exec_dict)
         try:
             self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
         except EntityAlreadyExistsError:
@@ -358,7 +381,7 @@ class ContextEntityOperationsProcessor:
         """
         dict_obj: dict = read_yaml(file)
         context = get_context_from_project(dict_obj["project"])
-        obj: ContextEntity = build_entity_from_dict(dict_obj)
+        obj: ContextEntity = factory.build_entity_from_dict(dict_obj)
         try:
             self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
         except EntityNotExistsError:
@@ -391,7 +414,7 @@ class ContextEntityOperationsProcessor:
             tsk_dicts = []
 
         context = get_context_from_project(exec_dict["project"])
-        obj: ExecutableEntity = build_entity_from_dict(exec_dict)
+        obj: ExecutableEntity = factory.build_entity_from_dict(exec_dict)
 
         try:
             self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
@@ -486,7 +509,7 @@ class ContextEntityOperationsProcessor:
         )
         objects = []
         for o in objs:
-            entity: ContextEntity = build_entity_from_dict(o)
+            entity: ContextEntity = factory.build_entity_from_dict(o)
             entity = self._post_process_get(entity)
             objects.append(entity)
         return objects
@@ -549,11 +572,35 @@ class ContextEntityOperationsProcessor:
         objs = self._list_context_entities(context, entity_type, **kwargs)
         objects = []
         for o in objs:
-            entity: ContextEntity = build_entity_from_dict(o)
+            entity: ContextEntity = factory.build_entity_from_dict(o)
             entity = self._post_process_get(entity)
             objects.append(entity)
         return objects
 
+    def _update_material_entity(
+        self,
+        new_obj: MaterialEntity,
+    ) -> dict:
+        """
+        Update material object shortcut.
+
+        Parameters
+        ----------
+        new_obj : MaterialEntity
+            Object instance.
+
+        Returns
+        -------
+        dict
+            Response from backend.
+        """
+        return self.update_context_entity(
+            new_obj.project,
+            new_obj.ENTITY_TYPE,
+            new_obj.id,
+            new_obj.to_dict(),
+        )
+
     def _update_context_entity(
         self,
         context: Context,
@@ -629,7 +676,7 @@ class ContextEntityOperationsProcessor:
             entity_dict,
             **kwargs,
         )
-        return build_entity_from_dict(obj)
+        return factory.build_entity_from_dict(obj)
 
     def _delete_context_entity(
         self,

digitalhub/entities/_processors/utils.py

@@ -5,7 +5,7 @@ import typing
 from digitalhub.context.api import get_context
 from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, EntityTypes
 from digitalhub.entities._commons.utils import get_project_from_key, parse_entity_key
-from digitalhub.factory.api import build_entity_from_dict
+from digitalhub.factory.factory import factory
 from digitalhub.stores.client.api import get_client
 from digitalhub.utils.exceptions import ContextError, EntityError, EntityNotExistsError
 
@@ -118,7 +118,7 @@ def get_context_from_remote(
     try:
         client = get_client()
         obj = _read_base_entity(client, EntityTypes.PROJECT.value, project)
-        build_entity_from_dict(obj)
+        factory.build_entity_from_dict(obj)
         return get_context(project)
     except EntityNotExistsError:
         raise ContextError(f"Project '{project}' not found.")

digitalhub/entities/builders.py

@@ -6,6 +6,7 @@ from digitalhub.entities.dataitem.table.builder import DataitemTableBuilder
 from digitalhub.entities.model.mlflow.builder import ModelModelBuilder
 from digitalhub.entities.project._base.builder import ProjectProjectBuilder
 from digitalhub.entities.secret._base.builder import SecretSecretBuilder
+from digitalhub.entities.trigger.scheduler.builder import TriggerSchedulerBuilder
 
 entity_builders: tuple = (
     (ProjectProjectBuilder.ENTITY_KIND, ProjectProjectBuilder),
@@ -14,6 +15,7 @@ entity_builders: tuple = (
     (DataitemDataitemBuilder.ENTITY_KIND, DataitemDataitemBuilder),
     (DataitemTableBuilder.ENTITY_KIND, DataitemTableBuilder),
     (ModelModelBuilder.ENTITY_KIND, ModelModelBuilder),
+    (TriggerSchedulerBuilder.ENTITY_KIND, TriggerSchedulerBuilder),
 )
 
 ##############################

digitalhub/entities/function/_base/entity.py

@@ -5,7 +5,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 from digitalhub.entities._base.executable.entity import ExecutableEntity
 from digitalhub.entities._commons.enums import EntityTypes, Relationship
-from digitalhub.factory.api import get_run_kind, get_task_kind_from_action
+from digitalhub.factory.factory import factory
 from digitalhub.utils.exceptions import BackendError
 
 if typing.TYPE_CHECKING:
@@ -72,8 +72,8 @@ class Function(ExecutableEntity):
             Run instance.
         """
         # Get task and run kind
-        task_kind = get_task_kind_from_action(self.kind, action)
-        run_kind = get_run_kind(self.kind)
+        task_kind = factory.get_task_kind_from_action(self.kind, action)
+        run_kind = factory.get_run_kind(self.kind)
 
         # Create or update new task
         task = self._get_or_create_task(task_kind)

digitalhub/entities/project/_base/builder.py

@@ -26,6 +26,7 @@ class ProjectProjectBuilder(EntityBuilder):
         description: str | None = None,
         labels: list[str] | None = None,
         local: bool = False,
+        config: dict | None = None,
         context: str | None = None,
         **kwargs,
     ) -> Project:
@@ -44,6 +45,8 @@ class ProjectProjectBuilder(EntityBuilder):
             List of labels.
         local : bool
             If True, use local backend, if False use DHCore backend. Default to False.
+        config : dict
+            DHCore environment configuration.
         context : str
             The context local folder of the project.
         **kwargs : dict
@@ -62,6 +65,7 @@ class ProjectProjectBuilder(EntityBuilder):
             labels=labels,
         )
         spec = self.build_spec(
+            config=config,
             context=context,
             **kwargs,
         )

digitalhub/entities/project/_base/entity.py

@@ -67,7 +67,7 @@ from digitalhub.entities.workflow.crud import (
     new_workflow,
     update_workflow,
 )
-from digitalhub.factory.api import build_entity_from_dict
+from digitalhub.factory.factory import factory
 from digitalhub.stores.client.api import get_client
 from digitalhub.utils.exceptions import BackendError, EntityAlreadyExistsError, EntityError
 from digitalhub.utils.generic_utils import get_timestamp
@@ -106,6 +106,9 @@ class Project(Entity):
         local: bool = False,
     ) -> None:
         super().__init__(kind, metadata, spec, status, user)
+        self.spec: ProjectSpec
+        self.status: ProjectStatus
+
         self.id = name
         self.name = name
         self.key = base_processor.build_project_key(self.name, local=local)
@@ -329,7 +332,7 @@ class Project(Entity):
                         entity["metadata"]["embedded"] = True
 
                     try:
-                        build_entity_from_dict(entity).save()
+                        factory.build_entity_from_dict(entity).save()
                     except EntityAlreadyExistsError:
                         pass
 

digitalhub/entities/project/_base/models.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel
+
+_DEFAULT_FILES_STORE = "s3://datalake"
+
+
+class ProfileConfig(BaseModel):
+    """
+    Configuration profiles.
+    """
+
+    default_files_store: Optional[str] = _DEFAULT_FILES_STORE
+
+    def to_dict(self) -> dict:
+        return self.model_dump(by_alias=True, exclude_none=True)

digitalhub/entities/project/_base/spec.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import Optional
 
 from digitalhub.entities._base.entity.spec import Spec, SpecValidator
+from digitalhub.entities.project._base.models import ProfileConfig
 
 
 class ProjectSpec(Spec):
@@ -18,6 +19,7 @@ class ProjectSpec(Spec):
         workflows: list | None = None,
         dataitems: list | None = None,
         models: list | None = None,
+        config: dict | None = None,
         **kwargs,
     ) -> None:
         self.context = context if context is not None else "./"
@@ -26,6 +28,7 @@ class ProjectSpec(Spec):
         self.workflows = workflows if workflows is not None else []
         self.dataitems = dataitems if dataitems is not None else []
         self.models = models if models is not None else []
+        self.config = config if config is not None else {}
 
 
 class ProjectValidator(SpecValidator):
@@ -50,3 +53,6 @@ class ProjectValidator(SpecValidator):
 
     models: Optional[list] = None
     """List of project's models."""
+
+    config: Optional[ProfileConfig] = None
+    """Project's config."""

digitalhub/entities/project/crud.py

@@ -75,7 +75,6 @@ def new_project(
 def get_project(
     name: str,
     local: bool = False,
-    config: dict | None = None,
     setup_kwargs: dict | None = None,
     **kwargs,
 ) -> Project:
@@ -88,8 +87,6 @@ def get_project(
         The Project name.
     local : bool
         Flag to determine if backend is local.
-    config : dict
-        DHCore environment configuration.
     setup_kwargs : dict
         Setup keyword arguments passed to setup_project() function.
     **kwargs : dict
@@ -108,7 +105,6 @@ def get_project(
         entity_type=ENTITY_TYPE,
         entity_name=name,
         local=local,
-        config=config,
         **kwargs,
     )
     return setup_project(obj, setup_kwargs)
@@ -117,7 +113,6 @@ def get_project(
 def import_project(
     file: str,
     local: bool = False,
-    config: dict | None = None,
     setup_kwargs: dict | None = None,
 ) -> Project:
     """
@@ -129,8 +124,6 @@ def import_project(
         Path to YAML file.
     local : bool
         Flag to determine if backend is local.
-    config : dict
-        DHCore environment configuration.
     setup_kwargs : dict
         Setup keyword arguments passed to setup_project() function.
 
@@ -143,14 +136,13 @@ def import_project(
     --------
     >>> obj = import_project("my-project.yaml")
     """
-    obj = base_processor.import_project_entity(file=file, local=local, config=config)
+    obj = base_processor.import_project_entity(file=file, local=local)
     return setup_project(obj, setup_kwargs)
 
 
 def load_project(
     file: str,
     local: bool = False,
-    config: dict | None = None,
     setup_kwargs: dict | None = None,
 ) -> Project:
     """
@@ -162,8 +154,6 @@ def load_project(
         Path to YAML file.
     local : bool
         Flag to determine if backend is local.
-    config : dict
-        DHCore environment configuration.
     setup_kwargs : dict
         Setup keyword arguments passed to setup_project() function.
 
@@ -176,7 +166,7 @@ def load_project(
     --------
     >>> obj = load_project("my-project.yaml")
     """
-    obj = base_processor.load_project_entity(file=file, local=local, config=config)
+    obj = base_processor.load_project_entity(file=file, local=local)
     return setup_project(obj, setup_kwargs)
 
 
@@ -234,7 +224,6 @@ def get_or_create_project(
         return get_project(
             name,
             local=local,
-            config=config,
             setup_kwargs=setup_kwargs,
             **kwargs,
         )

digitalhub/entities/run/_base/entity.py

@@ -7,13 +7,7 @@ from digitalhub.entities._base.unversioned.entity import UnversionedEntity
 from digitalhub.entities._commons.enums import EntityTypes, State
 from digitalhub.entities._commons.metrics import MetricType, set_metrics, validate_metric_value
 from digitalhub.entities._processors.context import context_processor
-from digitalhub.factory.api import (
-    build_runtime,
-    build_spec,
-    build_status,
-    get_entity_type_from_kind,
-    get_executable_kind,
-)
+from digitalhub.factory.factory import factory
 from digitalhub.utils.exceptions import EntityError
 from digitalhub.utils.logger import LOGGER
 
@@ -61,7 +55,7 @@ class Run(UnversionedEntity):
         executable = self._get_executable()
         task = self._get_task()
         new_spec = self._get_runtime().build(executable, task, self.to_dict())
-        self.spec = build_spec(self.kind, **new_spec)
+        self.spec = factory.build_spec(self.kind, **new_spec)
         self._set_state(State.BUILT.value)
         self.save(update=True)
 
@@ -302,7 +296,7 @@ class Run(UnversionedEntity):
         -------
         None
         """
-        self.status: RunStatus = build_status(self.kind, **status)
+        self.status: RunStatus = factory.build_status(self.kind, **status)
 
     def _set_state(self, state: str) -> None:
         """
@@ -343,7 +337,7 @@ class Run(UnversionedEntity):
         Runtime
             Runtime object.
         """
-        return build_runtime(self.kind, self.project)
+        return factory.build_runtime(self.kind, self.project)
 
     def _get_executable(self) -> dict:
         """
@@ -355,8 +349,8 @@ class Run(UnversionedEntity):
         dict
             Executable (function or workflow) from backend.
         """
-        exec_kind = get_executable_kind(self.kind)
-        exec_type = get_entity_type_from_kind(exec_kind)
+        exec_kind = factory.get_executable_kind(self.kind)
+        exec_type = factory.get_entity_type_from_kind(exec_kind)
         string_to_split = getattr(self.spec, exec_type)
         exec_name, exec_id = string_to_split.split("://")[-1].split("/")[-1].split(":")
         return context_processor.read_context_entity(

digitalhub/entities/task/_base/entity.py

@@ -5,7 +5,7 @@ import typing
 from digitalhub.entities._base.unversioned.entity import UnversionedEntity
 from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._processors.context import context_processor
-from digitalhub.factory.api import build_entity_from_params, get_entity_type_from_kind, get_executable_kind
+from digitalhub.factory.factory import factory
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -63,8 +63,8 @@ class Task(UnversionedEntity):
         Run
             Run object.
         """
-        exec_kind = get_executable_kind(self.kind)
-        exec_type = get_entity_type_from_kind(exec_kind)
+        exec_kind = factory.get_executable_kind(self.kind)
+        exec_type = factory.get_entity_type_from_kind(exec_kind)
         kwargs[exec_type] = getattr(self.spec, exec_type)
         return self.new_run(
             save=save,
@@ -108,7 +108,7 @@ class Task(UnversionedEntity):
         """
         if save:
             return context_processor.create_context_entity(**kwargs)
-        return build_entity_from_params(**kwargs)
+        return factory.build_entity_from_params(**kwargs)
 
     def get_run(self, entity_key: str) -> Run:
         """

digitalhub/entities/task/_base/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional
+from typing import Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -15,6 +15,24 @@ class VolumeType(Enum):
     EMPTY_DIR = "empty_dir"
 
 
+class SpecEmptyDir(BaseModel):
+    """
+    Spec empty dir model.
+    """
+
+    size_limit: str
+
+    medium: Optional[str] = None
+
+
+class SpecPVC(BaseModel):
+    """
+    Spec PVC model.
+    """
+
+    size: str
+
+
 class Volume(BaseModel):
     """
     Volume model.
@@ -31,7 +49,7 @@ class Volume(BaseModel):
     mount_path: str
     """Volume mount path inside the container."""
 
-    spec: Optional[dict[str, str]] = None
+    spec: Optional[Union[SpecEmptyDir, SpecPVC]] = None
     """Volume spec."""
 
 

digitalhub/entities/trigger/_base/builder.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from digitalhub.entities._base.versioned.builder import VersionedBuilder
+from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities.trigger._base.entity import Trigger
+
+
+class TriggerBuilder(VersionedBuilder):
+    """
+    TriggerTriggerBuilder builder.
+    """
+
+    ENTITY_TYPE = EntityTypes.TRIGGER.value
+
+    def build(
+        self,
+        kind: str,
+        project: str,
+        name: str,
+        uuid: str | None = None,
+        description: str | None = None,
+        labels: list[str] | None = None,
+        **kwargs,
+    ) -> Trigger:
+        """
+        Create a new object.
+
+        Parameters
+        ----------
+        project : str
+            Project name.
+        name : str
+            Object name.
+        kind : str
+            Kind the object.
+        uuid : str
+            ID of the object.
+        description : str
+            Description of the object (human readable).
+        labels : list[str]
+            List of labels.
+        **kwargs : dict
+            Spec keyword arguments.
+
+        Returns
+        -------
+        Trigger
+            Object instance.
+        """
+        name = self.build_name(name)
+        uuid = self.build_uuid(uuid)
+        metadata = self.build_metadata(
+            project=project,
+            name=name,
+            description=description,
+            labels=labels,
+        )
+        spec = self.build_spec(
+            **kwargs,
+        )
+        status = self.build_status()
+        return self.build_entity(
+            project=project,
+            name=name,
+            uuid=uuid,
+            kind=kind,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )

digitalhub/entities/trigger/_base/entity.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.entities._base.versioned.entity import VersionedEntity
+from digitalhub.entities._commons.enums import EntityTypes
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.entity.metadata import Metadata
+    from digitalhub.entities.trigger._base.spec import TriggerSpec
+    from digitalhub.entities.trigger._base.status import TriggerStatus
+
+
+class Trigger(VersionedEntity):
+    """
+    A class representing a trigger.
+    """
+
+    ENTITY_TYPE = EntityTypes.TRIGGER.value
+
+    def __init__(
+        self,
+        project: str,
+        name: str,
+        uuid: str,
+        kind: str,
+        metadata: Metadata,
+        spec: TriggerSpec,
+        status: TriggerStatus,
+        user: str | None = None,
+    ) -> None:
+        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
+        self.spec: TriggerSpec
+        self.status: TriggerStatus

digitalhub/entities/trigger/_base/spec.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from digitalhub.entities._base.entity.spec import Spec, SpecValidator
+
+
+class TriggerSpec(Spec):
+    """
+    TriggerSpec specifications.
+    """
+
+    def __init__(
+        self,
+        task: str,
+        template: dict,
+        function: str | None = None,
+        workflow: str | None = None,
+    ) -> None:
+        super().__init__()
+        self.task = task
+        self.template = template
+        self.function = function
+        self.workflow = workflow
+
+
+class TriggerValidator(SpecValidator):
+    """
+    TriggerValidator validator.
+    """
+
+    task: str
+    """Task string."""
+
+    template: dict
+    """Template map."""
+
+    function: str = None
+    """Function string."""
+
+    workflow: str = None
+    """Workflow string."""