digitalhub 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

digitalhub/entities/task/crud.py

@@ -2,14 +2,8 @@ from __future__ import annotations
 
 import typing
 
-from digitalhub.entities._base.crud import (
-    delete_entity,
-    get_unversioned_entity,
-    import_context_entity,
-    list_context_entities,
-    new_context_entity,
-)
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities._operations.processor import processor
 from digitalhub.utils.exceptions import EntityError
 
 if typing.TYPE_CHECKING:
@@ -56,7 +50,7 @@ def new_task(
     >>>                kind="python+job",
     >>>                function="function-string")
     """
-    return new_context_entity(
+    return processor.create_context_entity(
         project=project,
         kind=kind,
         uuid=uuid,
@@ -97,7 +91,7 @@ def get_task(
     >>> obj = get_task("my-task-id"
     >>>               project="my-project")
     """
-    return get_unversioned_entity(
+    return processor.read_unversioned_entity(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -125,7 +119,7 @@ def list_tasks(project: str, **kwargs) -> list[Task]:
     --------
     >>> objs = list_tasks(project="my-project")
     """
-    return list_context_entities(
+    return processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -134,7 +128,7 @@ def list_tasks(project: str, **kwargs) -> list[Task]:
 
 def import_task(file: str) -> Task:
     """
-    Get object from file.
+    Import object from a YAML file and create a new object into the backend.
 
     Parameters
     ----------
@@ -150,7 +144,28 @@ def import_task(file: str) -> Task:
     -------
     >>> obj = import_task("my-task.yaml")
     """
-    return import_context_entity(file)
+    return processor.import_context_entity(file)
+
+
+def load_task(file: str) -> Task:
+    """
+    Load object from a YAML file and update an existing object into the backend.
+
+    Parameters
+    ----------
+    file : str
+        Path to YAML file.
+
+    Returns
+    -------
+    Task
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = load_task("my-task.yaml")
+    """
+    return processor.load_context_entity(file)
 
 
 def update_task(entity: Task) -> Task:
@@ -171,7 +186,12 @@ def update_task(entity: Task) -> Task:
     --------
     >>> obj = update_task(obj)
     """
-    return entity.save(update=True)
+    return processor.update_context_entity(
+        project=entity.project,
+        entity_type=entity.ENTITY_TYPE,
+        entity_id=entity.id,
+        entity_dict=entity.to_dict(),
+    )
 
 
 def delete_task(
@@ -217,7 +237,7 @@ def delete_task(
     """
     if not identifier.startswith("store://"):
         raise EntityError("Task has no name. Use key instead.")
-    return delete_entity(
+    return processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/workflow/_base/builder.py

@@ -4,7 +4,7 @@ import typing
 
 from digitalhub.entities._base.runtime_entity.builder import RuntimeEntityBuilder
 from digitalhub.entities._base.versioned.builder import VersionedBuilder
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.workflow._base.entity import Workflow

digitalhub/entities/workflow/_base/entity.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._base.executable.entity import ExecutableEntity
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes, Relationship
 from digitalhub.factory.api import get_run_kind, get_task_kind_from_action
 from digitalhub.utils.exceptions import BackendError
 
@@ -41,7 +41,13 @@ class Workflow(ExecutableEntity):
     #  Workflow Methods
     ##############################
 
-    def run(self, action: str | None = None, **kwargs) -> Run:
+    def run(
+        self,
+        action: str,
+        wait: bool = False,
+        log_info: bool = True,
+        **kwargs,
+    ) -> Run:
         """
         Run workflow.
 
@@ -49,6 +55,10 @@ class Workflow(ExecutableEntity):
         ----------
         action : str
             Action to execute.
+        wait : bool
+            Flag to wait for execution to finish.
+        log_info : bool
+            Flag to log information while waiting.
         **kwargs : dict
             Keyword arguments passed to Run builder.
 
@@ -57,9 +67,6 @@ class Workflow(ExecutableEntity):
         Run
             Run instance.
         """
-        if action is None:
-            action = "pipeline"
-
         # Get task and run kind
         task_kind = get_task_kind_from_action(self.kind, action)
         run_kind = get_run_kind(self.kind)
@@ -71,4 +78,13 @@ class Workflow(ExecutableEntity):
         if self._context().local:
             raise BackendError("Cannot run workflow with local backend.")
 
-        return task.run(run_kind, local_execution=False, **kwargs)
+        # Run task
+        run = task.run(run_kind, save=False, local_execution=False, **kwargs)
+
+        # Set as run's parent
+        run.add_relationship(Relationship.RUN_OF.value, self.key)
+        run.save()
+
+        if wait:
+            return run.wait(log_info=log_info)
+        return run

digitalhub/entities/workflow/crud.py

@@ -2,15 +2,8 @@ from __future__ import annotations
 
 import typing
 
-from digitalhub.entities._base.crud import (
-    delete_entity,
-    get_context_entity_versions,
-    get_versioned_entity,
-    import_executable_entity,
-    list_context_entities,
-    new_context_entity,
-)
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._commons.enums import EntityTypes
+from digitalhub.entities._operations.processor import processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.workflow._base.entity import Workflow
@@ -62,7 +55,7 @@ def new_workflow(
     >>>                    code_src="pipeline.py",
     >>>                    handler="pipeline-handler")
     """
-    return new_context_entity(
+    return processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -109,7 +102,7 @@ def get_workflow(
     >>>                    project="my-project",
     >>>                    entity_id="my-workflow-id")
     """
-    return get_versioned_entity(
+    return processor.read_context_entity(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -149,7 +142,7 @@ def get_workflow_versions(
     >>> obj = get_workflow_versions("my-workflow-name"
     >>>                             project="my-project")
     """
-    return get_context_entity_versions(
+    return processor.read_context_entity_versions(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -177,7 +170,7 @@ def list_workflows(project: str, **kwargs) -> list[Workflow]:
     --------
     >>> objs = list_workflows(project="my-project")
     """
-    return list_context_entities(
+    return processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -186,7 +179,7 @@ def list_workflows(project: str, **kwargs) -> list[Workflow]:
 
 def import_workflow(file: str) -> Workflow:
     """
-    Import object from a YAML file.
+    Import object from a YAML file and create a new object into the backend.
 
     Parameters
     ----------
@@ -202,7 +195,28 @@ def import_workflow(file: str) -> Workflow:
     --------
     >>> obj = import_workflow("my-workflow.yaml")
     """
-    return import_executable_entity(file)
+    return processor.import_executable_entity(file)
+
+
+def load_workflow(file: str) -> Workflow:
+    """
+    Load object from a YAML file and update an existing object into the backend.
+
+    Parameters
+    ----------
+    file : str
+        Path to YAML file.
+
+    Returns
+    -------
+    Workflow
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = load_workflow("my-workflow.yaml")
+    """
+    return processor.load_executable_entity(file)
 
 
 def update_workflow(entity: Workflow) -> Workflow:
@@ -223,7 +237,12 @@ def update_workflow(entity: Workflow) -> Workflow:
     --------
     >>> obj = update_workflow(obj)
     """
-    return entity.save(update=True)
+    return processor.update_context_entity(
+        project=entity.project,
+        entity_type=entity.ENTITY_TYPE,
+        entity_id=entity.id,
+        entity_dict=entity.to_dict(),
+    )
 
 
 def delete_workflow(
@@ -267,7 +286,7 @@ def delete_workflow(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return delete_entity(
+    return processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/factory/utils.py

@@ -87,4 +87,4 @@ def register_entities() -> None:
             kind, builder = entity_builder_tuple
             factory.add_entity_builder(kind, builder)
     except Exception:
-        pass
+        raise

digitalhub/readers/_base/reader.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
-from abc import ABCMeta, abstractmethod
+from abc import abstractmethod
 from typing import Any
 
 
-class DataframeReader(metaclass=ABCMeta):
+class DataframeReader:
     """
     Dataframe reader abstract class.
     """

digitalhub/readers/_commons/enums.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class Extensions(Enum):
+    """
+    Supported file extensions.
+    """
+
+    CSV = "csv"
+    PARQUET = "parquet"
+    FILE = "file"

digitalhub/readers/api.py

@@ -4,6 +4,7 @@ import typing
 from typing import Any
 
 from digitalhub.readers.factory import factory
+from digitalhub.utils.exceptions import ReaderError
 
 if typing.TYPE_CHECKING:
     from digitalhub.readers._base.reader import DataframeReader
@@ -30,7 +31,7 @@ def get_reader_by_engine(engine: str | None = None) -> DataframeReader:
     except KeyError:
         engines = factory.list_supported_engines()
         msg = f"Unsupported dataframe engine: '{engine}'. Supported engines: {engines}"
-        raise ValueError(msg)
+        raise ReaderError(msg)
 
 
 def get_reader_by_object(obj: Any) -> DataframeReader:
@@ -53,7 +54,7 @@ def get_reader_by_object(obj: Any) -> DataframeReader:
     except KeyError:
         types = factory.list_supported_dataframes()
         msg = f"Unsupported dataframe type: '{obj}'. Supported types: {types}"
-        raise ValueError(msg)
+        raise ReaderError(msg)
 
 
 def get_supported_engines() -> list[str]:

digitalhub/readers/factory.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import typing
 
+from digitalhub.utils.exceptions import BuilderError
+
 if typing.TYPE_CHECKING:
     from digitalhub.readers._base.builder import ReaderBuilder
     from digitalhub.readers._base.reader import DataframeReader
@@ -35,13 +37,13 @@ class ReaderFactory:
         if self._engine_builders is None:
             self._engine_builders = {}
         if engine in self._engine_builders:
-            raise ValueError(f"Engine {engine} already exists.")
+            raise BuilderError(f"Builder for engine '{engine}' already exists.")
         self._engine_builders[engine] = builder
 
         if self._dataframe_builders is None:
             self._dataframe_builders = {}
         if dataframe in self._dataframe_builders:
-            raise ValueError(f"Dataframe {dataframe} already exists.")
+            raise BuilderError(f"Builder for dataframe '{dataframe}' already exists.")
         self._dataframe_builders[dataframe] = builder
 
     def build(self, engine: str | None = None, dataframe: str | None = None, **kwargs) -> DataframeReader:
@@ -63,7 +65,7 @@ class ReaderFactory:
             Reader object.
         """
         if (engine is None) == (dataframe is None):
-            raise ValueError("Either engine or dataframe must be provided.")
+            raise BuilderError("Either engine or dataframe must be provided.")
         if engine is not None:
             return self._engine_builders[engine].build(**kwargs)
         return self._dataframe_builders[dataframe].build(**kwargs)
@@ -104,7 +106,7 @@ class ReaderFactory:
         None
         """
         if engine not in self._engine_builders:
-            raise ValueError(f"Engine {engine} not found.")
+            raise BuilderError(f"Engine {engine} not found.")
         self._default = engine
 
     def get_default(self) -> str:
@@ -117,7 +119,7 @@ class ReaderFactory:
             Default engine.
         """
         if self._default is None:
-            raise ValueError("No default engine set.")
+            raise BuilderError("No default engine set.")
         return self._default
 
 
@@ -126,7 +128,11 @@ factory = ReaderFactory()
 try:
     from digitalhub.readers.pandas.builder import ReaderBuilderPandas
 
-    factory.add_builder(ReaderBuilderPandas.ENGINE, ReaderBuilderPandas.DATAFRAME_CLASS, ReaderBuilderPandas())
+    factory.add_builder(
+        ReaderBuilderPandas.ENGINE,
+        ReaderBuilderPandas.DATAFRAME_CLASS,
+        ReaderBuilderPandas(),
+    )
     factory.set_default(ReaderBuilderPandas.ENGINE)
 
 except ImportError:

digitalhub/readers/pandas/reader.py

@@ -8,7 +8,9 @@ import pandas as pd
 from pandas.errors import ParserError
 
 from digitalhub.readers._base.reader import DataframeReader
+from digitalhub.readers._commons.enums import Extensions
 from digitalhub.utils.data_utils import build_data_preview, get_data_preview
+from digitalhub.utils.exceptions import ReaderError
 
 
 class DataframeReaderPandas(DataframeReader):
@@ -38,15 +40,17 @@ class DataframeReaderPandas(DataframeReader):
         pd.DataFrame
             Pandas DataFrame.
         """
-        if extension == "csv":
+        if extension == Extensions.CSV.value:
             method = pd.read_csv
-        elif extension == "parquet":
+        elif extension == Extensions.PARQUET.value:
             method = pd.read_parquet
-        elif extension == "file":
+        elif extension == Extensions.FILE.value:
             try:
-                return self.read_df(path, "csv", **kwargs)
+                return self.read_df(path, Extensions.CSV.value, **kwargs)
             except ParserError:
-                raise ValueError(f"Unable to read from {path}.")
+                raise ReaderError(f"Unable to read from {path}.")
+        else:
+            raise ReaderError(f"Unsupported extension '{extension}' for reading.")
 
         if isinstance(path, list):
             dfs = [method(p, **kwargs) for p in path]
@@ -57,7 +61,13 @@ class DataframeReaderPandas(DataframeReader):
     # Write methods
     ##############################
 
-    def write_df(self, df: pd.DataFrame, dst: str | BytesIO, extension: str | None = None, **kwargs) -> None:
+    def write_df(
+        self,
+        df: pd.DataFrame,
+        dst: str | BytesIO,
+        extension: str | None = None,
+        **kwargs,
+    ) -> None:
         """
         Write DataFrame as parquet.
 
@@ -74,9 +84,11 @@ class DataframeReaderPandas(DataframeReader):
         -------
         None
         """
-        if extension == "csv":
+        if extension == Extensions.CSV.value:
             return self.write_csv(df, dst, **kwargs)
-        self.write_parquet(df, dst, **kwargs)
+        elif extension == Extensions.PARQUET.value:
+            return self.write_parquet(df, dst, **kwargs)
+        raise ReaderError(f"Unsupported extension '{extension}' for writing.")
 
     @staticmethod
     def write_csv(df: pd.DataFrame, dst: str | BytesIO, **kwargs) -> None:

digitalhub/runtimes/_base.py

@@ -34,13 +34,6 @@ class Runtime:
         Execute run task.
         """
 
-    @staticmethod
-    @abstractmethod
-    def _get_executable(action: str) -> Callable:
-        """
-        Get executable from action.
-        """
-
     ##############################
     # Private methods
     ##############################

digitalhub/runtimes/enums.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class RuntimeEnvVar(Enum):
+    """
+    Environment variables.
+    """
+
+    PROJECT = "PROJECT_NAME"
+    RUN_ID = "RUN_ID"

digitalhub/stores/_base/store.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
 
-from abc import ABCMeta, abstractmethod
+from abc import abstractmethod
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import Literal
+from typing import Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
+from digitalhub.readers.api import get_reader_by_engine
 from digitalhub.utils.exceptions import StoreError
-from digitalhub.utils.uri_utils import map_uri_scheme
+from digitalhub.utils.uri_utils import SchemeCategory, has_local_scheme
 
 
-class Store(metaclass=ABCMeta):
+class Store:
     """
     Store abstract class.
     """
@@ -35,7 +36,7 @@ class Store(metaclass=ABCMeta):
         self.type = store_type
 
     ##############################
-    # IO methods
+    # I/O methods
     ##############################
 
     @abstractmethod
@@ -51,17 +52,60 @@ class Store(metaclass=ABCMeta):
         """
 
     @abstractmethod
-    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+    def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
         """
         Method to upload artifact to storage.
         """
 
     @abstractmethod
-    def get_file_info(self, paths: list[str]) -> list[dict]:
+    def get_file_info(
+        self,
+        root: str,
+        paths: list[tuple[str, str]],
+    ) -> list[dict]:
         """
         Method to get file metadata.
         """
 
+    ##############################
+    # Datastore methods
+    ##############################
+
+    @abstractmethod
+    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
+        """
+        Write DataFrame as parquet or csv.
+        """
+
+    def read_df(
+        self,
+        path: str | list[str],
+        extension: str,
+        engine: str | None = None,
+        **kwargs,
+    ) -> Any:
+        """
+        Read DataFrame from path.
+
+        Parameters
+        ----------
+        path : str | list[str]
+            Path(s) to read DataFrame from.
+        extension : str
+            Extension of the file.
+        engine : str
+            Dataframe engine (pandas, polars, etc.).
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        Any
+            DataFrame.
+        """
+        reader = get_reader_by_engine(engine)
+        return reader.read_df(path, extension, **kwargs)
+
     ##############################
     # Helpers methods
     ##############################
@@ -84,7 +128,7 @@ class Store(metaclass=ABCMeta):
         StoreError
             If the source is not a local path.
         """
-        if map_uri_scheme(src) != "local":
+        if not has_local_scheme(src):
             raise StoreError(f"Source '{src}' is not a local path.")
 
     def _check_local_dst(self, dst: str) -> None:
@@ -105,7 +149,7 @@ class Store(metaclass=ABCMeta):
         StoreError
             If the destination is not a local path.
         """
-        if map_uri_scheme(dst) != "local":
+        if not has_local_scheme(dst):
             raise StoreError(f"Destination '{dst}' is not a local path.")
 
     def _check_overwrite(self, dst: Path, overwrite: bool) -> None:
@@ -170,16 +214,20 @@ class StoreConfig(BaseModel):
     Store configuration base class.
     """
 
+    model_config = ConfigDict(use_enum_values=True)
+
 
 class StoreParameters(BaseModel):
     """
     Store configuration class.
     """
 
+    model_config = ConfigDict(use_enum_values=True)
+
     name: str
     """Store id."""
 
-    type: Literal["local", "s3", "remote", "sql"]
+    type: SchemeCategory
     """Store type to instantiate."""
 
     config: StoreConfig = None

digitalhub/stores/builder.py

@@ -11,7 +11,7 @@ from digitalhub.stores.remote.store import RemoteStore, RemoteStoreConfig
 from digitalhub.stores.s3.store import S3Store, S3StoreConfig
 from digitalhub.stores.sql.store import SqlStore, SQLStoreConfig
 from digitalhub.utils.exceptions import StoreError
-from digitalhub.utils.uri_utils import map_uri_scheme
+from digitalhub.utils.uri_utils import SchemeCategory, map_uri_scheme
 
 if typing.TYPE_CHECKING:
     from digitalhub.stores._base.store import Store
@@ -167,7 +167,7 @@ def get_env_store_config(scheme: str) -> StoreParameters:
     ValueError
         If the scheme is not supported.
     """
-    if scheme == "s3":
+    if scheme == SchemeCategory.S3.value:
         return StoreParameters(
             name="s3",
             type="s3",
@@ -178,7 +178,7 @@ def get_env_store_config(scheme: str) -> StoreParameters:
                 bucket_name=os.getenv("S3_BUCKET_NAME"),  # type: ignore
             ),
         )
-    if scheme == "sql":
+    if scheme == SchemeCategory.SQL.value:
         return StoreParameters(
             name="sql",
             type="sql",
@@ -191,13 +191,13 @@ def get_env_store_config(scheme: str) -> StoreParameters:
                 pg_schema=os.getenv("POSTGRES_SCHEMA"),  # type: ignore
             ),
         )
-    if scheme == "remote":
+    if scheme == SchemeCategory.REMOTE.value:
         return StoreParameters(
             name="remote",
             type="remote",
             config=RemoteStoreConfig(),
         )
-    if scheme == "local":
+    if scheme == SchemeCategory.LOCAL.value:
         return StoreParameters(
             name="local",
             type="local",