digitalhub 0.9.1__py3-none-any.whl → 0.10.0__py3-none-any.whl

digitalhub/entities/task/crud.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.context import context_processor
 from digitalhub.utils.exceptions import EntityError
 
 if typing.TYPE_CHECKING:
@@ -19,6 +19,7 @@ def new_task(
     uuid: str | None = None,
     labels: list[str] | None = None,
     function: str | None = None,
+    workflow: str | None = None,
     **kwargs,
 ) -> Task:
     """
@@ -36,6 +37,8 @@ def new_task(
         List of labels.
     function : str
         Name of the executable associated with the task.
+    workflow : str
+        Name of the workflow associated with the task.
     **kwargs : dict
         Spec keyword arguments.
 
@@ -50,12 +53,13 @@ def new_task(
     >>>                kind="python+job",
     >>>                function="function-string")
     """
-    return processor.create_context_entity(
+    return context_processor.create_context_entity(
         project=project,
         kind=kind,
         uuid=uuid,
         labels=labels,
         function=function,
+        workflow=workflow,
         **kwargs,
     )
 
@@ -91,7 +95,7 @@ def get_task(
     >>> obj = get_task("my-task-id"
     >>>               project="my-project")
     """
-    return processor.read_unversioned_entity(
+    return context_processor.read_unversioned_entity(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -119,7 +123,7 @@ def list_tasks(project: str, **kwargs) -> list[Task]:
     --------
     >>> objs = list_tasks(project="my-project")
     """
-    return processor.list_context_entities(
+    return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -144,7 +148,7 @@ def import_task(file: str) -> Task:
     -------
     >>> obj = import_task("my-task.yaml")
     """
-    return processor.import_context_entity(file)
+    return context_processor.import_context_entity(file)
 
 
 def load_task(file: str) -> Task:
@@ -165,7 +169,7 @@ def load_task(file: str) -> Task:
     --------
     >>> obj = load_task("my-task.yaml")
     """
-    return processor.load_context_entity(file)
+    return context_processor.load_context_entity(file)
 
 
 def update_task(entity: Task) -> Task:
@@ -186,7 +190,7 @@ def update_task(entity: Task) -> Task:
     --------
     >>> obj = update_task(obj)
     """
-    return processor.update_context_entity(
+    return context_processor.update_context_entity(
         project=entity.project,
         entity_type=entity.ENTITY_TYPE,
         entity_id=entity.id,
@@ -237,7 +241,7 @@ def delete_task(
     """
     if not identifier.startswith("store://"):
         raise EntityError("Task has no name. Use key instead.")
-    return processor.delete_context_entity(
+    return context_processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/workflow/crud.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.context import context_processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.workflow._base.entity import Workflow
@@ -55,7 +55,7 @@ def new_workflow(
     >>>                    code_src="pipeline.py",
     >>>                    handler="pipeline-handler")
     """
-    return processor.create_context_entity(
+    return context_processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -102,7 +102,7 @@ def get_workflow(
     >>>                    project="my-project",
     >>>                    entity_id="my-workflow-id")
     """
-    return processor.read_context_entity(
+    return context_processor.read_context_entity(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -142,7 +142,7 @@ def get_workflow_versions(
     >>> obj = get_workflow_versions("my-workflow-name"
     >>>                             project="my-project")
     """
-    return processor.read_context_entity_versions(
+    return context_processor.read_context_entity_versions(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -170,7 +170,7 @@ def list_workflows(project: str, **kwargs) -> list[Workflow]:
     --------
     >>> objs = list_workflows(project="my-project")
     """
-    return processor.list_context_entities(
+    return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -195,7 +195,7 @@ def import_workflow(file: str) -> Workflow:
     --------
     >>> obj = import_workflow("my-workflow.yaml")
     """
-    return processor.import_executable_entity(file)
+    return context_processor.import_executable_entity(file)
 
 
 def load_workflow(file: str) -> Workflow:
@@ -216,7 +216,7 @@ def load_workflow(file: str) -> Workflow:
     --------
     >>> obj = load_workflow("my-workflow.yaml")
     """
-    return processor.load_executable_entity(file)
+    return context_processor.load_executable_entity(file)
 
 
 def update_workflow(entity: Workflow) -> Workflow:
@@ -237,7 +237,7 @@ def update_workflow(entity: Workflow) -> Workflow:
     --------
     >>> obj = update_workflow(obj)
     """
-    return processor.update_context_entity(
+    return context_processor.update_context_entity(
         project=entity.project,
         entity_type=entity.ENTITY_TYPE,
         entity_id=entity.id,
@@ -286,7 +286,7 @@ def delete_workflow(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return processor.delete_context_entity(
+    return context_processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/factory/utils.py

@@ -25,10 +25,10 @@ def import_module(package: str) -> ModuleType:
     """
     try:
         return importlib.import_module(package)
-    except ModuleNotFoundError:
-        raise ModuleNotFoundError(f"Package {package} not found.")
+    except ModuleNotFoundError as e:
+        raise ModuleNotFoundError(f"Package {package} not found.") from e
     except Exception as e:
-        raise e
+        raise RuntimeError(f"An error occurred while importing {package}.") from e
 
 
 def list_runtimes() -> list[str]:
@@ -37,18 +37,18 @@ def list_runtimes() -> list[str]:
 
     Returns
     -------
-    list
+    list[str]
         List of installed runtimes names.
     """
     pattern = r"digitalhub_runtime_.*"
-    runtimes = []
+    runtimes: list[str] = []
     try:
         for _, name, _ in pkgutil.iter_modules():
             if re.match(pattern, name):
                 runtimes.append(name)
         return runtimes
-    except Exception:
-        raise RuntimeError("Error listing installed runtimes.")
+    except Exception as e:
+        raise RuntimeError("Error listing installed runtimes.") from e
 
 
 def register_runtimes_entities() -> None:
@@ -86,5 +86,5 @@ def register_entities() -> None:
         for entity_builder_tuple in entities_builders_list:
             kind, builder = entity_builder_tuple
             factory.add_entity_builder(kind, builder)
-    except Exception:
-        raise
+    except Exception as e:
+        raise RuntimeError("Error registering entities.") from e

digitalhub/readers/{_base → data/_base}/builder.py

@@ -6,7 +6,7 @@ from abc import abstractmethod
 from digitalhub.utils.exceptions import BuilderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers._base.reader import DataframeReader
+    from digitalhub.readers.data._base.reader import DataframeReader
 
 
 class ReaderBuilder:

digitalhub/readers/{_base → data/_base}/reader.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
-from typing import Any
+from typing import IO, Any
 
 
 class DataframeReader:
@@ -13,11 +13,16 @@ class DataframeReader:
     # Read methods
     ##############################
 
-    @staticmethod
     @abstractmethod
-    def read_df(path: str | list[str], extension: str, **kwargs) -> Any:
+    def read_df(self, path_or_buffer: str | IO, extension: str, **kwargs) -> Any:
+        """
+        Read DataFrame from path or buffer.
+        """
+
+    @abstractmethod
+    def read_table(self, *args, **kwargs) -> Any:
         """
-        Read DataFrame from path.
+        Read table from db.
         """
 
     ##############################
@@ -68,3 +73,10 @@ class DataframeReader:
         """
         Get preview.
         """
+
+    @staticmethod
+    @abstractmethod
+    def concat_dfs(dfs: list[Any]) -> Any:
+        """
+        Concatenate a list of DataFrames.
+        """

digitalhub/readers/{api.py → data/api.py}

@@ -3,11 +3,11 @@ from __future__ import annotations
 import typing
 from typing import Any
 
-from digitalhub.readers.factory import factory
+from digitalhub.readers.data.factory import factory
 from digitalhub.utils.exceptions import ReaderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers._base.reader import DataframeReader
+    from digitalhub.readers.data._base.reader import DataframeReader
 
 
 def get_reader_by_engine(engine: str | None = None) -> DataframeReader:

digitalhub/readers/{factory.py → data/factory.py}

@@ -5,8 +5,8 @@ import typing
 from digitalhub.utils.exceptions import BuilderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers._base.builder import ReaderBuilder
-    from digitalhub.readers._base.reader import DataframeReader
+    from digitalhub.readers.data._base.builder import ReaderBuilder
+    from digitalhub.readers.data._base.reader import DataframeReader
 
 
 class ReaderFactory:
@@ -126,7 +126,7 @@ class ReaderFactory:
 factory = ReaderFactory()
 
 try:
-    from digitalhub.readers.pandas.builder import ReaderBuilderPandas
+    from digitalhub.readers.data.pandas.builder import ReaderBuilderPandas
 
     factory.add_builder(
         ReaderBuilderPandas.ENGINE,

digitalhub/readers/{pandas → data/pandas}/builder.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from digitalhub.readers._base.builder import ReaderBuilder
-from digitalhub.readers.pandas.reader import DataframeReaderPandas
+from digitalhub.readers.data._base.builder import ReaderBuilder
+from digitalhub.readers.data.pandas.reader import DataframeReaderPandas
 
 
 class ReaderBuilderPandas(ReaderBuilder):

digitalhub/readers/{pandas → data/pandas}/reader.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
+import json
 from io import BytesIO
-from typing import Any
+from typing import IO, Any
 
 import numpy as np
 import pandas as pd
 from pandas.errors import ParserError
 
-from digitalhub.readers._base.reader import DataframeReader
-from digitalhub.readers._commons.enums import Extensions
-from digitalhub.utils.data_utils import build_data_preview, get_data_preview
+from digitalhub.entities.dataitem.table.utils import check_preview_size, finalize_preview, prepare_data, prepare_preview
+from digitalhub.readers.data._base.reader import DataframeReader
+from digitalhub.utils.enums import FileExtensions
 from digitalhub.utils.exceptions import ReaderError
+from digitalhub.utils.generic_utils import CustomJsonEncoder
 
 
 class DataframeReaderPandas(DataframeReader):
@@ -22,14 +24,14 @@ class DataframeReaderPandas(DataframeReader):
     # Read methods
     ##############################
 
-    def read_df(self, path: str | list[str], extension: str, **kwargs) -> pd.DataFrame:
+    def read_df(self, path_or_buffer: str | IO, extension: str, **kwargs) -> pd.DataFrame:
         """
-        Read DataFrame from path.
+        Read DataFrame from path or buffer.
 
         Parameters
         ----------
-        path : str | list[str]
-            Path(s) to read DataFrame from.
+        path_or_buffer : str | IO
+            Path or buffer to read DataFrame from.
         extension : str
             Extension of the file.
         **kwargs : dict
@@ -40,22 +42,41 @@ class DataframeReaderPandas(DataframeReader):
         pd.DataFrame
             Pandas DataFrame.
         """
-        if extension == Extensions.CSV.value:
-            method = pd.read_csv
-        elif extension == Extensions.PARQUET.value:
-            method = pd.read_parquet
-        elif extension == Extensions.FILE.value:
+        if extension == FileExtensions.CSV.value:
+            return pd.read_csv(path_or_buffer, **kwargs)
+        if extension == FileExtensions.PARQUET.value:
+            return pd.read_parquet(path_or_buffer, **kwargs)
+        if extension == FileExtensions.JSON.value:
+            return pd.read_json(path_or_buffer, **kwargs)
+        if extension in (FileExtensions.EXCEL.value, FileExtensions.EXCEL_OLD.value):
+            return pd.read_excel(path_or_buffer, **kwargs)
+        if extension in (FileExtensions.TXT.value, FileExtensions.FILE.value):
             try:
-                return self.read_df(path, Extensions.CSV.value, **kwargs)
+                return self.read_df(path_or_buffer, FileExtensions.CSV.value, **kwargs)
             except ParserError:
-                raise ReaderError(f"Unable to read from {path}.")
+                raise ReaderError(f"Unable to read from {path_or_buffer}.")
         else:
             raise ReaderError(f"Unsupported extension '{extension}' for reading.")
 
-        if isinstance(path, list):
-            dfs = [method(p, **kwargs) for p in path]
-            return pd.concat(dfs)
-        return method(path, **kwargs)
+    def read_table(self, sql: str, engine: Any, **kwargs) -> pd.DataFrame:
+        """
+        Read table from db.
+
+        Parameters
+        ----------
+        sql : str
+            SQL query.
+        engine : Any
+            SQL Engine.
+        **kwargs
+            Keyword arguments.
+
+        Returns
+        -------
+        pd.DataFrame
+            Pandas DataFrame.
+        """
+        return pd.read_sql(sql=sql, con=engine, **kwargs)
 
     ##############################
     # Write methods
@@ -84,9 +105,9 @@ class DataframeReaderPandas(DataframeReader):
         -------
         None
         """
-        if extension == Extensions.CSV.value:
+        if extension == FileExtensions.CSV.value:
             return self.write_csv(df, dst, **kwargs)
-        elif extension == Extensions.PARQUET.value:
+        if extension == FileExtensions.PARQUET.value:
             return self.write_parquet(df, dst, **kwargs)
         raise ReaderError(f"Unsupported extension '{extension}' for writing.")
 
@@ -131,7 +152,7 @@ class DataframeReaderPandas(DataframeReader):
         df.to_parquet(dst, index=False, **kwargs)
 
     @staticmethod
-    def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str, **kwargs) -> None:
+    def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str | None = None, **kwargs) -> None:
         """
         Write DataFrame as table.
 
@@ -142,7 +163,7 @@ class DataframeReaderPandas(DataframeReader):
         table : str
             The destination table.
         engine : Any
-            The SQLAlchemy engine.
+            SQL Engine.
         schema : str
             The destination schema.
         **kwargs : dict
@@ -158,6 +179,23 @@ class DataframeReaderPandas(DataframeReader):
     # Utils
     ##############################
 
+    @staticmethod
+    def concat_dfs(dfs: list[pd.DataFrame]) -> pd.DataFrame:
+        """
+        Concatenate a list of DataFrames.
+
+        Parameters
+        ----------
+        dfs : list[pd.DataFrame]
+            The DataFrames to concatenate.
+
+        Returns
+        -------
+        pd.DataFrame
+            The concatenated DataFrame.
+        """
+        return pd.concat(dfs, ignore_index=True)
+
     @staticmethod
     def get_schema(df: pd.DataFrame) -> Any:
         """
@@ -196,7 +234,7 @@ class DataframeReaderPandas(DataframeReader):
         return schema
 
     @staticmethod
-    def get_preview(df: pd.DataFrame) -> Any:
+    def get_preview(df: pd.DataFrame) -> dict:
         """
         Get preview.
 
@@ -211,9 +249,51 @@ class DataframeReaderPandas(DataframeReader):
             The preview.
         """
         columns = [str(col) for col, _ in df.dtypes.items()]
-        head = df.head(10)
-        head = head.replace({np.nan: None})
-        head = head.values.tolist()
-        preview = get_data_preview(columns, head)
-        len_df = len(df)
-        return build_data_preview(preview, len_df)
+        head = df.head(10).replace({np.nan: None})
+        data = head.values.tolist()
+        prepared_data = prepare_data(data)
+        preview = prepare_preview(columns, prepared_data)
+        finalizes = finalize_preview(preview, df.shape[0])
+        serialized = _serialize_deserialize_preview(finalizes)
+        return check_preview_size(serialized)
+
+
+class PandasJsonEncoder(CustomJsonEncoder):
+    """
+    JSON pd.Timestamp to ISO format serializer.
+    """
+
+    def default(self, obj: Any) -> Any:
+        """
+        Pandas datetime to ISO format serializer.
+
+        Parameters
+        ----------
+        obj : Any
+            The object to serialize.
+
+        Returns
+        -------
+        Any
+            The serialized object.
+        """
+        if isinstance(obj, pd.Timestamp):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+def _serialize_deserialize_preview(preview: dict) -> dict:
+    """
+    Serialize and deserialize preview.
+
+    Parameters
+    ----------
+    preview : dict
+        The preview.
+
+    Returns
+    -------
+    dict
+        The serialized preview.
+    """
+    return json.loads(json.dumps(preview, cls=PandasJsonEncoder))