digitalhub 0.7.0b2__py3-none-any.whl → 0.8.0__py3-none-any.whl

digitalhub/entities/_base/executable/entity.py

@@ -0,0 +1,405 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.entities._base.api_utils import list_entity_api_ctx
+from digitalhub.entities._base.versioned.entity import VersionedEntity
+from digitalhub.entities.run.crud import delete_run, get_run, list_runs
+from digitalhub.entities.task.crud import delete_task
+from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.factory.api import build_entity_from_dict, build_entity_from_params
+from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.entity.metadata import Metadata
+    from digitalhub.entities._base.entity.spec import Spec
+    from digitalhub.entities._base.entity.status import Status
+    from digitalhub.entities.run._base.entity import Run
+    from digitalhub.entities.task._base.entity import Task
+
+
+class ExecutableEntity(VersionedEntity):
+    """
+    A class representing an entity that can be executed.
+    """
+
+    def __init__(
+        self,
+        project: str,
+        name: str,
+        uuid: str,
+        kind: str,
+        metadata: Metadata,
+        spec: Spec,
+        status: Status,
+        user: str | None = None,
+    ) -> None:
+        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
+
+        # Initialize tasks
+        self._tasks: dict[str, Task] = {}
+
+    ##############################
+    #  Helpers
+    ##############################
+
+    def _get_executable_string(self) -> str:
+        """
+        Get executable string.
+
+        Returns
+        -------
+        str
+            Executable string.
+        """
+        return f"{self.kind}://{self.project}/{self.name}:{self.id}"
+
+    ##############################
+    #  Tasks
+    ##############################
+
+    def _get_or_create_task(self, kind: str) -> Task:
+        """
+        Get or create task.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+
+        Returns
+        -------
+        Task
+            Task.
+        """
+        if self._tasks.get(kind) is None:
+            try:
+                self._tasks[kind] = self.get_task(kind)
+            except EntityError:
+                self._tasks[kind] = self.new_task(kind)
+        return self._tasks[kind]
+
+    def import_tasks(self, tasks: list[dict]) -> None:
+        """
+        Import tasks from yaml.
+
+        Parameters
+        ----------
+        tasks : list[dict]
+            List of tasks to import.
+
+        Returns
+        -------
+        None
+        """
+        # Loop over tasks list, in the case where the function
+        # is imported from local file.
+        for task in tasks:
+            # If task is not a dictionary, skip it
+            if not isinstance(task, dict):
+                continue
+
+            # Create a new object from dictionary.
+            # the form in which tasks are stored in function
+            # status
+            task_obj = build_entity_from_dict(task)
+
+            # Try to save it in backend to been able to use
+            # it for launching runs. In fact, tasks must be
+            # persisted in backend to be able to launch runs.
+            # Ignore if task already exists
+            try:
+                task_obj.save()
+            except EntityAlreadyExistsError:
+                pass
+
+            # Set task if function is the same. Overwrite
+            # status task dict with the new task object
+            if task_obj.spec.function == self._get_executable_string():
+                self._tasks[task_obj.kind] = task_obj
+
+    def new_task(self, task_kind: str, **kwargs) -> Task:
+        """
+        Create new task. If the task already exists, update it.
+
+        Parameters
+        ----------
+        task_kind : str
+            Kind the object.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        Task
+            New task.
+        """
+        self._raise_if_exists(task_kind)
+
+        if kwargs is None:
+            kwargs = {}
+
+        # Override kwargs
+        kwargs["project"] = self.project
+        kwargs["function"] = self._get_executable_string()
+        kwargs["kind"] = task_kind
+
+        # Create object instance
+        task = build_entity_from_params(**kwargs)
+        task.save()
+
+        self._tasks[task_kind] = task
+        return task
+
+    def get_task(self, kind: str) -> Task:
+        """
+        Get task.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+
+        Returns
+        -------
+        Task
+            Task.
+
+        Raises
+        ------
+        EntityError
+            If task is not created.
+        """
+        try:
+            return self._tasks[kind]
+        except KeyError:
+            resp = self._get_task_from_backend(kind)
+            if not resp:
+                raise EntityError(f"Task {kind} is not created")
+            self._tasks[kind] = build_entity_from_dict(resp[0])
+            return self._tasks[kind]
+
+    def update_task(self, kind: str, **kwargs) -> Task:
+        """
+        Update task.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        Task
+            Task.
+        """
+        self._raise_if_not_exists(kind)
+
+        if kwargs is None:
+            kwargs = {}
+
+        # Update kwargs
+        kwargs["project"] = self.project
+        kwargs["kind"] = kind
+        kwargs["function"] = self._get_executable_string()
+        kwargs["uuid"] = self._tasks[kind].id
+
+        # Update task
+        task = build_entity_from_params(**kwargs)
+        task.save(update=True)
+        self._tasks[kind] = task
+        return task
+
+    def delete_task(self, kind: str, cascade: bool = True) -> dict:
+        """
+        Delete task.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+        cascade : bool
+            Flag to determine if cascade deletion must be performed.
+
+        Returns
+        -------
+        dict
+            Response from backend.
+        """
+        resp = delete_task(self._tasks[kind].key, cascade=cascade)
+        self._tasks.pop(kind, None)
+        return resp
+
+    def _get_task_from_backend(self, kind: str) -> list:
+        """
+        List tasks from backend filtered by function and kind.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+
+        Returns
+        -------
+        list
+            Response from backend.
+        """
+        params = {"function": self._get_executable_string(), "kind": kind}
+        return list_entity_api_ctx(self.project, EntityTypes.TASK.value, params=params)
+
+    def _check_task_in_backend(self, kind: str) -> bool:
+        """
+        Check if task exists in backend.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+
+        Returns
+        -------
+        bool
+            Flag to determine if task exists in backend.
+        """
+        resp = self._get_task_from_backend(kind)
+        if not resp:
+            return False
+        return True
+
+    def _raise_if_exists(self, kind: str) -> None:
+        """
+        Raise error if task is created.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        EntityError
+            If task already exists.
+        """
+        if self._check_task_in_backend(kind):
+            raise EntityError(f"Task '{kind}' already exists.")
+
+    def _raise_if_not_exists(self, kind: str) -> None:
+        """
+        Raise error if task is not created.
+
+        Parameters
+        ----------
+        kind : str
+            Kind the object.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        EntityError
+            If task does not exist.
+        """
+        if self._tasks.get(kind) is None:
+            raise EntityError(f"Task '{kind}' does not exist.")
+
+    ##############################
+    #  Runs
+    ##############################
+
+    def get_run(
+        self,
+        identifier: str,
+        **kwargs,
+    ) -> Run:
+        """
+        Get object from backend.
+
+        Parameters
+        ----------
+        identifier : str
+            Entity key (store://...) or entity ID.
+        **kwargs : dict
+            Parameters to pass to the API call.
+
+        Returns
+        -------
+        Run
+            Object instance.
+
+        Examples
+        --------
+        Using entity key:
+        >>> obj = executable.get_run("store://my-secret-key")
+
+        Using entity ID:
+        >>> obj = executable.get_run("123")
+        """
+        obj = get_run(
+            identifier=identifier,
+            project=self.project,
+            **kwargs,
+        )
+        self.refresh()
+        return obj
+
+    def list_runs(self, **kwargs) -> list[Run]:
+        """
+        List all runs from backend.
+
+        Parameters
+        ----------
+        **kwargs : dict
+            Parameters to pass to the API call.
+
+        Returns
+        -------
+        list[Run]
+            List of object instances.
+
+        Examples
+        --------
+        >>> objs = executable.list_runs()
+        """
+        if kwargs is None:
+            kwargs = {}
+        kwargs["params"] = {"function": self._get_executable_string()}
+        return list_runs(self.project, **kwargs)
+
+    def delete_run(
+        self,
+        identifier: str,
+        **kwargs,
+    ) -> None:
+        """
+        Delete run from backend.
+
+        Parameters
+        ----------
+        identifier : str
+            Entity key (store://...) or entity ID.
+        **kwargs : dict
+            Parameters to pass to the API call.
+
+        Returns
+        -------
+        dict
+            Response from backend.
+
+        Examples
+        --------
+        >>> executable.delete_run("store://my-run-key")
+
+        """
+        delete_run(
+            identifier=identifier,
+            project=self.project,
+            **kwargs,
+        )

digitalhub/entities/_base/material/entity.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import typing
+from pathlib import Path
+
+from digitalhub.entities._base.api_utils import files_info_get_api, files_info_put_api
+from digitalhub.entities._base.versioned.entity import VersionedEntity
+from digitalhub.stores.api import get_store
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.entity.metadata import Metadata
+    from digitalhub.entities._base.material.spec import MaterialSpec
+    from digitalhub.entities._base.material.status import MaterialStatus
+
+
+class MaterialEntity(VersionedEntity):
+    """
+    A class representing an entity that can be materialized
+    as file(s).
+    """
+
+    def __init__(
+        self,
+        project: str,
+        name: str,
+        uuid: str,
+        kind: str,
+        metadata: Metadata,
+        spec: MaterialSpec,
+        status: MaterialStatus,
+        user: str | None = None,
+    ) -> None:
+        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
+        self.spec: MaterialSpec
+        self.status: MaterialStatus
+
+    def save(self, update: bool = False) -> MaterialEntity:
+        """
+        Save entity into backend.
+
+        Parameters
+        ----------
+        update : bool
+            Flag to indicate update.
+
+        Returns
+        -------
+        MaterialEntity
+            Entity saved.
+        """
+        obj = self.to_dict()
+
+        files = None
+        if self.status.files is not None and len(self.status.files) > 5 and not self._context().local:
+            files = obj["status"].pop("files")
+
+        if not update:
+            new_obj: MaterialEntity = self._save(obj)
+        else:
+            new_obj: MaterialEntity = self._update(obj)
+
+        # Handle files info
+        if files is not None:
+            files_info_put_api(self.project, self.ENTITY_TYPE, self.id, files)
+            self.status.add_files_info(files)
+
+        return new_obj
+
+    ##############################
+    # I/O Methods
+    ##############################
+
+    def as_file(self) -> list[str]:
+        """
+        Get object as file(s). It downloads the object from storage in
+        a temporary folder and returns the list of downloaded files paths.
+
+        Returns
+        -------
+        list[str]
+            List of file paths.
+        """
+        store = get_store(self.spec.path)
+        paths = self.status.get_file_paths()
+        dst = store._build_temp()
+        return store.download(self.spec.path, dst=dst, src=paths)
+
+    def download(
+        self,
+        destination: str | None = None,
+        overwrite: bool = False,
+    ) -> str:
+        """
+        This function downloads one or more file from storage on local
+        machine.
+        It looks inside the object's status for the file(s) path under
+        files attribute. If it does not find it, it will try to download
+        what it can from spec.path.
+        The files are downloaded into a destination folder. If the destination
+        is not specified, it will set by default under the context path
+        as '<ctx-root>/<entity_type>', e.g. './dataitem'.
+        The overwrite flag allows to overwrite existing file(s) in the
+        destination folder.
+
+        Parameters
+        ----------
+        destination : str
+            Destination path as filename or directory.
+        overwrite : bool
+            Specify if overwrite existing file(s). If file(s) already
+            exist and overwrite is False, it will raise an error.
+
+        Returns
+        -------
+        str
+            Downloaded path.
+
+        Examples
+        --------
+        Download a single file:
+
+        >>> entity.status.files[0]
+        {
+            "path ": "data.csv",
+            "name ": "data.csv",
+            "content_type ": "text/csv;charset=utf-8 "
+        }
+        >>> path = entity.download()
+        >>> print(path)
+        dataitem/data.csv
+        """
+        store = get_store(self.spec.path)
+        paths = self.status.get_file_paths()
+
+        if destination is None:
+            dst = self._context().root / self.ENTITY_TYPE
+        else:
+            dst = Path(destination)
+
+        return store.download(self.spec.path, dst=dst, src=paths, overwrite=overwrite)
+
+    def upload(self, source: str | list[str]) -> None:
+        """
+        Upload object from given local path to spec path destination.
+        Source must be a local path. If the path is a folder, destination
+        path (object's spec path) must be a folder or a partition ending
+        with '/' (s3).
+
+        Parameters
+        ----------
+        source : str | list[str]
+            Local filepath, directory or list of filepaths.
+
+        Returns
+        -------
+        None
+
+        Examples
+        --------
+        Upload a single file:
+
+        >>> entity.spec.path = 's3://bucket/data.csv'
+        >>> entity.upload('./data.csv')
+
+        Upload a folder:
+        >>> entity.spec.path = 's3://bucket/data/'
+        >>> entity.upload('./data')
+        """
+        # Get store and upload object
+        store = get_store(self.spec.path)
+        paths = store.upload(source, self.spec.path)
+
+        # Update files info
+        files_info = store.get_file_info(paths)
+        self._update_files_info(files_info)
+
+    ##############################
+    #  Private Helpers
+    ##############################
+
+    def _update_files_info(self, files_info: list[dict] | None = None) -> None:
+        """
+        Update files info.
+
+        Parameters
+        ----------
+        files_info : list[dict] | None
+            Files info.
+
+        Returns
+        -------
+        None
+        """
+        if files_info is None:
+            return
+        self.refresh()
+        self.status.add_files_info(files_info)
+        self.save(update=True)
+
+    def _get_files_info(self) -> None:
+        """
+        Get files info from backend.
+
+        Returns
+        -------
+        None
+        """
+        if not self._context().local and not self.status.files:
+            files = files_info_get_api(
+                project=self.project,
+                entity_type=self.ENTITY_TYPE,
+                entity_id=self.id,
+            )
+            self.status.add_files_info(files)

digitalhub/entities/_base/material/spec.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from digitalhub.entities._base.entity.spec import Spec, SpecValidator
+
+
+class MaterialSpec(Spec):
+    """
+    Material specifications.class.
+    """
+
+    def __init__(self, path: str, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.path = path
+
+
+class MaterialValidator(SpecValidator):
+    """
+    Material parameters class.
+    """
+
+    path: str
+    """Target path to file(s)"""

digitalhub/entities/_base/material/status.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from digitalhub.entities._base.entity.status import Status
+
+
+class MaterialStatus(Status):
+    """
+    Material Status class.
+    """
+
+    def __init__(
+        self,
+        state: str,
+        message: str | None = None,
+        files: list[dict] | None = None,
+    ) -> None:
+        super().__init__(state, message)
+        if files is None:
+            files = []
+        self.files = files
+
+    def add_files_info(self, files: list[dict]) -> None:
+        """
+        Add a file to the status.
+
+        Parameters
+        ----------
+        files : list[dict]
+            Files to add.
+
+        Returns
+        -------
+        None
+        """
+        path_list = self.get_file_paths()
+        for f in files:
+            if f.get("path") not in path_list:
+                self.files.append(f)
+
+    def get_file_paths(self) -> list[str]:
+        """
+        Get the paths of the files in the status.
+
+        Returns
+        -------
+        list[str]
+            Paths of the files in the status.
+        """
+        return [f.get("path") for f in self.files]