digitalhub 0.8.0b0__py3-none-any.whl → 0.8.0b1__py3-none-any.whl

digitalhub/readers/builder.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import typing
+from typing import Any
+
+from digitalhub.readers.registry import REGISTRY_DATAFRAME, REGISTRY_FRAMEWORK
+
+if typing.TYPE_CHECKING:
+    from digitalhub.readers.objects.base import DataframeReader
+
+
+def get_reader_by_engine(engine: str) -> DataframeReader:
+    """
+    Get Dataframe reader.
+
+    Parameters
+    ----------
+    engine : str
+        Dataframe engine (pandas, polars, etc.).
+
+    Returns
+    -------
+    DataframeReader
+        Reader object.
+    """
+    try:
+        return REGISTRY_FRAMEWORK[engine]()
+    except KeyError:
+        engines = list(REGISTRY_FRAMEWORK.keys())
+        msg = f"Unsupported dataframe engine: {engine}. Supported engines: {engines}"
+        raise ValueError(msg)
+
+
+def get_reader_by_object(obj: Any) -> DataframeReader:
+    """
+    Get Dataframe reader by object.
+
+    Parameters
+    ----------
+    obj : Any
+        Object to get reader from.
+
+    Returns
+    -------
+    DataframeReader
+        Reader object.
+    """
+    try:
+        obj_name = f"{obj.__class__.__module__}.{obj.__class__.__name__}"
+        return REGISTRY_DATAFRAME[obj_name]()
+    except KeyError:
+        types = list(REGISTRY_DATAFRAME.keys())
+        msg = f"Unsupported dataframe type: {obj}. Supported types: {types}"
+        raise ValueError(msg)

digitalhub/readers/objects/base.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from abc import ABCMeta, abstractmethod
+from typing import Any
+
+
+class DataframeReader(metaclass=ABCMeta):
+    """
+    Dataframe reader abstract class.
+    """
+
+    ##############################
+    # Read methods
+    ##############################
+
+    @staticmethod
+    @abstractmethod
+    def read_df(path: str | list[str], extension: str, **kwargs) -> Any:
+        """
+        Read DataFrame from path.
+        """
+
+    ##############################
+    # Write methods
+    ##############################
+
+    @abstractmethod
+    def write_df(self, df: Any, dst: Any, extension: str | None = None, **kwargs) -> None:
+        """
+        Method to write a dataframe to a file.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def write_csv(df: Any, *args, **kwargs) -> str:
+        """
+        Write DataFrame as csv.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def write_parquet(df: Any, *args, **kwargs) -> str:
+        """
+        Write DataFrame as parquet.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def write_table(df: Any, *args, **kwargs) -> str:
+        """
+        Write DataFrame as table.
+        """
+
+    ##############################
+    # Utils
+    ##############################
+
+    @staticmethod
+    @abstractmethod
+    def get_schema(df: Any) -> Any:
+        """
+        Get schema.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def get_preview(df: Any) -> Any:
+        """
+        Get preview.
+        """

digitalhub/readers/objects/pandas.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+from io import BytesIO
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from pandas.errors import ParserError
+
+from digitalhub.readers.objects.base import DataframeReader
+from digitalhub.utils.data_utils import build_data_preview, get_data_preview
+
+
+class DataframeReaderPandas(DataframeReader):
+    """
+    Pandas reader class.
+    """
+
+    ##############################
+    # Read methods
+    ##############################
+
+    def read_df(self, path: str | list[str], extension: str, **kwargs) -> pd.DataFrame:
+        """
+        Read DataFrame from path.
+
+        Parameters
+        ----------
+        path : str | list[str]
+            Path(s) to read DataFrame from.
+        extension : str
+            Extension of the file.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        pd.DataFrame
+            Pandas DataFrame.
+        """
+        if extension == "csv":
+            method = pd.read_csv
+        elif extension == "parquet":
+            method = pd.read_parquet
+        elif extension == "file":
+            try:
+                return self.read_df(path, "csv", **kwargs)
+            except ParserError:
+                raise ValueError(f"Unable to read from {path}.")
+
+        if isinstance(path, list):
+            dfs = [method(p, **kwargs) for p in path]
+            return pd.concat(dfs)
+        return method(path, **kwargs)
+
+    ##############################
+    # Write methods
+    ##############################
+
+    def write_df(self, df: pd.DataFrame, dst: str | BytesIO, extension: str | None = None, **kwargs) -> None:
+        """
+        Write DataFrame as parquet.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        dst : str | BytesIO
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        if extension == "csv":
+            return self.write_csv(df, dst, **kwargs)
+        self.write_parquet(df, dst, **kwargs)
+
+    @staticmethod
+    def write_csv(df: pd.DataFrame, dst: str | BytesIO, **kwargs) -> None:
+        """
+        Write DataFrame as csv.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        dst : str | BytesIO
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        df.to_csv(dst, index=False, **kwargs)
+
+    @staticmethod
+    def write_parquet(df: pd.DataFrame, dst: str | BytesIO, **kwargs) -> None:
+        """
+        Write DataFrame as parquet.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        dst : str | BytesIO
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        df.to_parquet(dst, index=False, **kwargs)
+
+    @staticmethod
+    def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str, **kwargs) -> None:
+        """
+        Write DataFrame as table.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe to write.
+        table : str
+            The destination table.
+        engine : Any
+            The SQLAlchemy engine.
+        schema : str
+            The destination schema.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        None
+        """
+        df.to_sql(table, engine, schema=schema, index=False, **kwargs)
+
+    ##############################
+    # Utils
+    ##############################
+
+    @staticmethod
+    def get_schema(df: pd.DataFrame) -> Any:
+        """
+        Get schema.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe.
+
+        Returns
+        -------
+        Any
+            The schema.
+        """
+        schema = {"fields": []}
+
+        for column_name, dtype in df.dtypes.items():
+            field = {"name": str(column_name), "type": ""}
+
+            if pd.api.types.is_integer_dtype(dtype):
+                field["type"] = "integer"
+            elif pd.api.types.is_float_dtype(dtype):
+                field["type"] = "number"
+            elif pd.api.types.is_bool_dtype(dtype):
+                field["type"] = "boolean"
+            elif pd.api.types.is_string_dtype(dtype):
+                field["type"] = "string"
+            elif pd.api.types.is_datetime64_any_dtype(dtype):
+                field["type"] = "datetime"
+            else:
+                field["type"] = "any"
+
+            schema["fields"].append(field)
+
+        return schema
+
+    @staticmethod
+    def get_preview(df: pd.DataFrame) -> Any:
+        """
+        Get preview.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The dataframe.
+
+        Returns
+        -------
+        Any
+            The preview.
+        """
+        columns = [str(col) for col, _ in df.dtypes.items()]
+        head = df.head(10)
+        head = head.replace({np.nan: None})
+        head = head.values.tolist()
+        preview = get_data_preview(columns, head)
+        len_df = len(df)
+        return build_data_preview(preview, len_df)

digitalhub/readers/registry.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+REGISTRY_FRAMEWORK = {}
+REGISTRY_DATAFRAME = {}
+DATAFRAME_TYPES = []
+
+try:
+    from digitalhub.readers.objects.pandas import DataframeReaderPandas
+
+    REGISTRY_FRAMEWORK["pandas"] = DataframeReaderPandas
+    REGISTRY_DATAFRAME["pandas.core.frame.DataFrame"] = DataframeReaderPandas
+    DATAFRAME_TYPES.append("pandas.core.frame.DataFrame")
+
+except ImportError:
+    pass

digitalhub/registry/models.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class RuntimeEntry(BaseModel):
+    """
+    Model for Runtime entry.
+    It specifies the module path and the runtime class name.
+    """
+
+    module: str
+    """Module path."""
+
+    class_name: str
+    """Class name."""
+
+    kind_registry_module: str
+    """Kind registry module path."""
+
+    kind_registry_class_name: str
+    """Kind registry class name."""
+
+
+class SpecEntry(BaseModel):
+    """
+    Model for Spec entry.
+    It specifies the module path, the spec class name
+    and the spec pydantic class validator.
+    """
+
+    module: str
+    """Module path."""
+
+    class_name: str
+    """Class name."""
+
+    parameters_validator: str
+    """Class name of the parameter validator."""
+
+
+class StatusEntry(BaseModel):
+    """
+    Model for Status entry.
+    It specifies the module path and the status class name.
+    """
+
+    module: str
+    """Module path."""
+
+    class_name: str
+    """Class name."""
+
+
+class MetadataEntry(BaseModel):
+    """
+    Model for Metadata entry.
+    It specifies the module path and the metadata class name.
+    """
+
+    module: str
+    """Module path."""
+
+    class_name: str
+    """Class name."""
+
+
+class RegistryEntry(BaseModel):
+    """
+    Basic entry model for every entity.
+    Functions, tasks and runs specify also the runtime field.
+    """
+
+    entity_type: str
+    """Entity type."""
+
+    spec: SpecEntry
+    """Spec infos."""
+
+    status: StatusEntry
+    """Status infos."""
+
+    metadata: MetadataEntry
+    """Metadata infos."""
+
+    runtime: RuntimeEntry = None
+    """Runtime infos."""

digitalhub/registry/registry.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from digitalhub.registry.models import RegistryEntry
+
+
+class GlobalEntityRegistry:
+    """
+    Global entity registry where all entities classpaths are stored.
+    It keeps track of all entities that are registered in the system.
+    """
+
+    def register(self, entity_kind: str, entry: dict) -> None:
+        """
+        Register an entity kind.
+
+        Parameters
+        ----------
+        entity_kind : str
+            Entity kind to be registered.
+        entry : dict
+            Entry to be registered.
+
+        Returns
+        -------
+        None
+
+        """
+        if hasattr(self, entity_kind):
+            raise ValueError(f"Entity kind {entity_kind} already registered")
+        entry = RegistryEntry(**entry)
+        setattr(self, entity_kind, entry)
+
+    def update(self, entity_kind: str, entry: dict) -> None:
+        """
+        Update an entity kind.
+
+        Parameters
+        ----------
+        entity_kind : str
+            Entity kind to be updated.
+        entry : dict
+            Entry to be updated.
+
+        Returns
+        -------
+        None
+        """
+        if hasattr(self, entity_kind):
+            entry = RegistryEntry(**entry)
+            setattr(self, entity_kind, entry)
+        else:
+            raise ValueError(f"Entity kind {entity_kind} not registered")
+
+    def get_entity_type(self, entity_kind: str) -> str:
+        """
+        Get entity type from kind.
+
+        Parameters
+        ----------
+        entity_kind : str
+            Entity kind.
+
+        Returns
+        -------
+        str
+            Entity type.
+        """
+        if not hasattr(self, entity_kind):
+            raise ValueError(f"Entity kind {entity_kind} not registered")
+        entry: RegistryEntry = getattr(self, entity_kind)
+        return entry.entity_type
+
+
+registry = GlobalEntityRegistry()

digitalhub/registry/utils.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import importlib
+import importlib.metadata
+import pkgutil
+import re
+from types import ModuleType
+
+
+def import_module(package: str) -> ModuleType:
+    """
+    Import modules from package name.
+
+    Parameters
+    ----------
+    package : str
+        Package name.
+
+    Returns
+    -------
+    ModuleType
+        Module.
+    """
+    try:
+        return importlib.import_module(package)
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(f"Package {package} not found.")
+    except Exception as e:
+        raise e
+
+
+def import_class(module_to_import: str, class_name: str) -> type:
+    """
+    Import class from implemented module.
+
+    Parameters
+    ----------
+    module_to_import : str
+        Module name.
+    class_name : str
+        Class name.
+
+    Returns
+    -------
+    type
+        Class.
+    """
+    module = import_module(module_to_import)
+    try:
+        return getattr(module, class_name)
+    except AttributeError:
+        raise ModuleNotFoundError(f"Module {module_to_import} has no '{class_name}' class.")
+
+
+def list_runtimes() -> list[str]:
+    """
+    List installed runtimes for digitalhub.
+
+    Returns
+    -------
+    list
+        List of installed runtimes names.
+    """
+    pattern = r"digitalhub_runtime_.*"
+    runtimes = []
+    try:
+        for _, name, _ in pkgutil.iter_modules():
+            if re.match(pattern, name):
+                runtimes.append(name)
+        return runtimes
+    except Exception:
+        raise RuntimeError("Error listing installed runtimes.")
+
+
+def register_runtimes_entities() -> None:
+    """
+    Register runtimes and related entities into registry.
+
+    Returns
+    -------
+    None
+    """
+    for package in list_runtimes():
+        import_module(package)
+
+
+def register_entities() -> None:
+    """
+    Register layer and related entities into registry.
+
+    Returns
+    -------
+    None
+    """
+    # Try ot import registry from entities.registries module
+    try:
+        import_module(f"digitalhub.entities.registries")
+    except Exception:
+        pass
+
+
+def create_info(
+    root: str,
+    entity_type: str,
+    prefix: str,
+    suffix: str | None = "",
+    runtime_info: dict | None = None,
+) -> dict:
+    """
+    Create entity info.
+
+    Parameters
+    ----------
+    root : str
+        Root module.
+    entity_type : str
+        Entity type.
+    prefix : str
+        Entity prefix.
+    suffix : str
+        Entity suffix.
+    runtime_info : dict
+        Runtime info.
+
+    Returns
+    -------
+    dict
+        Entity info.
+    """
+    dict_ = {
+        "entity_type": entity_type,
+        "spec": {
+            "module": f"{root}.{entity_type}.spec",
+            "class_name": f"{prefix}Spec{suffix}",
+            "parameters_validator": f"{prefix}Params{suffix}",
+        },
+        "status": {
+            "module": f"{root}.{entity_type}.status",
+            "class_name": f"{prefix}Status{suffix}",
+        },
+        "metadata": {
+            "module": "digitalhub.entities._base.metadata",
+            "class_name": "Metadata",
+        },
+    }
+    # Add runtime only if provided
+    # (in functions, tasks, runs and workflows)
+    if runtime_info is not None:
+        dict_["runtime"] = runtime_info
+    return dict_