digitalhub 0.7.0b2__py3-none-any.whl → 0.8.0__py3-none-any.whl

digitalhub/readers/pandas/reader.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+from io import BytesIO
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from pandas.errors import ParserError
+
+from digitalhub.readers._base.reader import DataframeReader
+from digitalhub.utils.data_utils import build_data_preview, get_data_preview
+
+
+class DataframeReaderPandas(DataframeReader):
+    """
+    Pandas reader class.
+    """
+
+    ##############################
+    # Read methods
+    ##############################
+
+    def read_df(self, path: str | list[str], extension: str, **kwargs) -> pd.DataFrame:
+        """
+        Read DataFrame from path.
+
+        Parameters
+        ----------
+        path : str | list[str]
+            Path(s) to read DataFrame from.
+        extension : str
+            Extension of the file.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        pd.DataFrame
+            Pandas DataFrame.
+        """
+        if extension == "csv":
+            method = pd.read_csv
+        elif extension == "parquet":
+            method = pd.read_parquet
+        elif extension == "file":
+            try:
+                return self.read_df(path, "csv", **kwargs)
+            except ParserError:
+                raise ValueError(f"Unable to read from {path}.")
+
+        if isinstance(path, list):
+            dfs = [method(p, **kwargs) for p in path]
+            return pd.concat(dfs)
+        return method(path, **kwargs)
+
+    ##############################
+    # Write methods
+    ##############################
+
+    def write_df(self, df: pd.DataFrame, dst: str | BytesIO, extension: str | None = None, **kwargs) -> None:
+        """
+        Write DataFrame as parquet.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        dst : str | BytesIO
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        if extension == "csv":
+            return self.write_csv(df, dst, **kwargs)
+        self.write_parquet(df, dst, **kwargs)
+
+    @staticmethod
+    def write_csv(df: pd.DataFrame, dst: str | BytesIO, **kwargs) -> None:
+        """
+        Write DataFrame as csv.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        dst : str | BytesIO
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        df.to_csv(dst, index=False, **kwargs)
+
+    @staticmethod
+    def write_parquet(df: pd.DataFrame, dst: str | BytesIO, **kwargs) -> None:
+        """
+        Write DataFrame as parquet.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        dst : str | BytesIO
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        df.to_parquet(dst, index=False, **kwargs)
+
+    @staticmethod
+    def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str, **kwargs) -> None:
+        """
+        Write DataFrame as table.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        table : str
+            The destination table.
+        engine : Any
+            The SQLAlchemy engine.
+        schema : str
+            The destination schema.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        df.to_sql(table, engine, schema=schema, index=False, **kwargs)
+
+    ##############################
+    # Utils
+    ##############################
+
+    @staticmethod
+    def get_schema(df: pd.DataFrame) -> Any:
+        """
+        Get schema.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe.
+
+        Returns
+        -------
+        Any
+            The schema.
+        """
+        schema = {"fields": []}
+
+        for column_name, dtype in df.dtypes.items():
+            field = {"name": str(column_name), "type": ""}
+
+            if pd.api.types.is_integer_dtype(dtype):
+                field["type"] = "integer"
+            elif pd.api.types.is_float_dtype(dtype):
+                field["type"] = "number"
+            elif pd.api.types.is_bool_dtype(dtype):
+                field["type"] = "boolean"
+            elif pd.api.types.is_string_dtype(dtype):
+                field["type"] = "string"
+            elif pd.api.types.is_datetime64_any_dtype(dtype):
+                field["type"] = "datetime"
+            else:
+                field["type"] = "any"
+
+            schema["fields"].append(field)
+
+        return schema
+
+    @staticmethod
+    def get_preview(df: pd.DataFrame) -> Any:
+        """
+        Get preview.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe.
+
+        Returns
+        -------
+        Any
+            The preview.
+        """
+        columns = [str(col) for col, _ in df.dtypes.items()]
+        head = df.head(10)
+        head = head.replace({np.nan: None})
+        head = head.values.tolist()
+        preview = get_data_preview(columns, head)
+        len_df = len(df)
+        return build_data_preview(preview, len_df)

digitalhub/runtimes/_base.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Any, Callable
+
+from digitalhub.factory.api import get_action_from_task_kind
+from digitalhub.utils.exceptions import EntityError
+from digitalhub.utils.logger import LOGGER
+
+
+class Runtime:
+    """
+    Base Runtime class.
+
+    Runtimes are the entities responsible for the actual execution
+    of a given run. They are highly specialized components which
+    can translate the representation of a given execution as expressed
+    in the run into an actual execution operation performed via
+    libraries, code, external tools etc.
+    """
+
+    def __init__(self, project: str) -> None:
+        self.project = project
+
+    @abstractmethod
+    def build(self, executable: dict, task: dict, run: dict) -> dict:
+        """
+        Build run spec.
+        """
+
+    @abstractmethod
+    def run(self, run: dict) -> dict:
+        """
+        Execute run task.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def _get_executable(action: str) -> Callable:
+        """
+        Get executable from action.
+        """
+
+    ##############################
+    # Private methods
+    ##############################
+
+    def _validate_task(self, run: dict) -> str:
+        """
+        Check if task is allowed. This presumes that the
+        runtime holds a list of allowed actions in the self.allowed_actions
+        attribute.
+
+        Parameters
+        ----------
+        run : dict
+            Run object dictionary.
+
+        Returns
+        -------
+        str
+            Action to execute.
+        """
+        try:
+            task_kind = run["spec"]["task"].split(":")[0]
+        except (KeyError, IndexError):
+            msg = "Malformed run spec."
+            LOGGER.exception(msg)
+            raise RuntimeError(msg)
+
+        try:
+            return get_action_from_task_kind(task_kind, task_kind)
+        except EntityError:
+            msg = f"Task {task_kind} not allowed."
+            LOGGER.exception(msg)
+            raise RuntimeError(msg)
+
+    @staticmethod
+    def _execute(func: Callable, *args, **kwargs) -> Any:
+        """
+        Execute function.
+
+        Parameters
+        ----------
+        func : Callable
+            Function to execute.
+        *args
+            Function arguments.
+        **kwargs : dict
+            Function keyword arguments.
+
+        Returns
+        -------
+        Any
+            Function result.
+        """
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            msg = "Something got wrong during function execution."
+            LOGGER.exception(msg)
+            raise RuntimeError(msg)

digitalhub/runtimes/builder.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.utils.exceptions import BuilderError
+
+if typing.TYPE_CHECKING:
+    from digitalhub.runtimes._base import Runtime
+
+
+class RuntimeBuilder:
+    """
+    Builder class for building runtimes.
+    """
+
+    # Class variables
+    RUNTIME_CLASS: Runtime = None
+
+    def __init__(self) -> None:
+        if self.RUNTIME_CLASS is None:
+            raise BuilderError("RUNTIME_CLASS must be set")
+
+    def build(self, project: str, *args, **kwargs) -> Runtime:
+        """
+        Build runtime object.
+
+        Returns
+        -------
+        Runtime
+            Runtime object.
+        """
+        return self.RUNTIME_CLASS(project, *args, **kwargs)

digitalhub/stores/_base/store.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from pathlib import Path
+from tempfile import mkdtemp
+from typing import Literal
+
+from pydantic import BaseModel
+
+from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.uri_utils import map_uri_scheme
+
+
+class Store(metaclass=ABCMeta):
+    """
+    Store abstract class.
+    """
+
+    def __init__(self, name: str, store_type: str) -> None:
+        """
+        Constructor.
+
+        Parameters
+        ----------
+        name : str
+            Store name.
+        store_type : str
+            Store type. Used to choose the right store implementation.
+
+        Returns
+        -------
+        None
+        """
+        self.name = name
+        self.type = store_type
+
+    ##############################
+    # IO methods
+    ##############################
+
+    @abstractmethod
+    def download(
+        self,
+        root: str,
+        dst: Path,
+        src: list[str],
+        overwrite: bool = False,
+    ) -> str:
+        """
+        Method to download artifact from storage.
+        """
+
+    @abstractmethod
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
+        """
+        Method to upload artifact to storage.
+        """
+
+    @abstractmethod
+    def get_file_info(self, paths: list[str]) -> list[dict]:
+        """
+        Method to get file metadata.
+        """
+
+    ##############################
+    # Helpers methods
+    ##############################
+
+    def _check_local_src(self, src: str) -> None:
+        """
+        Check if the source path is local.
+
+        Parameters
+        ----------
+        src : str
+            The source path.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        StoreError
+            If the source is not a local path.
+        """
+        if map_uri_scheme(src) != "local":
+            raise StoreError(f"Source '{src}' is not a local path.")
+
+    def _check_local_dst(self, dst: str) -> None:
+        """
+        Check if the destination path is local.
+
+        Parameters
+        ----------
+        dst : str
+            The destination path.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        StoreError
+            If the destination is not a local path.
+        """
+        if map_uri_scheme(dst) != "local":
+            raise StoreError(f"Destination '{dst}' is not a local path.")
+
+    def _check_overwrite(self, dst: Path, overwrite: bool) -> None:
+        """
+        Check if destination path exists for overwrite.
+
+        Parameters
+        ----------
+        dst : Path
+            Destination path as filename.
+        overwrite : bool
+            Specify if overwrite an existing file.
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        StoreError
+            If destination path exists and overwrite is False.
+        """
+        if dst.exists() and not overwrite:
+            raise StoreError(f"Destination {str(dst)} already exists.")
+
+    @staticmethod
+    def _build_path(path: str | Path) -> None:
+        """
+        Get path from store path and path.
+
+        Parameters
+        ----------
+        path : str
+            The path to build.
+
+        Returns
+        -------
+        None
+        """
+        if not isinstance(path, Path):
+            path = Path(path)
+        if path.suffix != "":
+            path = path.parent
+        path.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def _build_temp() -> Path:
+        """
+        Build a temporary path.
+
+        Returns
+        -------
+        Path
+            Temporary path.
+        """
+        tmpdir = mkdtemp()
+        return Path(tmpdir)
+
+
+class StoreConfig(BaseModel):
+    """
+    Store configuration base class.
+    """
+
+
+class StoreParameters(BaseModel):
+    """
+    Store configuration class.
+    """
+
+    name: str
+    """Store id."""
+
+    type: Literal["local", "s3", "remote", "sql"]
+    """Store type to instantiate."""
+
+    config: StoreConfig = None
+    """Configuration for the store."""
+
+    is_default: bool = False
+    """Flag to determine if the store is the default one."""

digitalhub/stores/api.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.stores.builder import store_builder
+
+if typing.TYPE_CHECKING:
+    from digitalhub.stores._base.store import Store, StoreParameters
+
+
+def set_store(store_cfg: StoreParameters) -> None:
+    """
+    Set a new store instance with the given configuration.
+
+    Parameters
+    ----------
+    store_cfg : StoreParameters
+        Store configuration.
+
+    Returns
+    -------
+    None
+    """
+    store_builder.build(store_cfg)
+
+
+def get_store(uri: str) -> Store:
+    """
+    Get store instance by uri.
+
+    Parameters
+    ---------
+    uri : str
+        URI to parse.
+
+    Returns
+    -------
+    Store
+        Store instance.
+    """
+    return store_builder.get(uri)
+
+
+def get_default_store() -> Store:
+    """
+    Get the default store instance. The default store is the one that
+    can persist artifacts and dataitems.
+
+    Returns
+    -------
+    Store
+        Default store instance.
+    """
+    return store_builder.default()