digitalhub 0.9.2__py3-none-any.whl → 0.10.0b1__py3-none-any.whl

digitalhub/readers/pandas/reader.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from io import BytesIO
 from typing import Any
 
@@ -7,10 +8,11 @@ import numpy as np
 import pandas as pd
 from pandas.errors import ParserError
 
+from digitalhub.entities.dataitem.table.utils import check_preview_size, finalize_preview, prepare_data, prepare_preview
 from digitalhub.readers._base.reader import DataframeReader
-from digitalhub.readers._commons.enums import Extensions
-from digitalhub.utils.data_utils import build_data_preview, get_data_preview
+from digitalhub.readers.pandas.enums import Extensions
 from digitalhub.utils.exceptions import ReaderError
+from digitalhub.utils.generic_utils import CustomJsonEncoder
 
 
 class DataframeReaderPandas(DataframeReader):
@@ -44,7 +46,11 @@ class DataframeReaderPandas(DataframeReader):
             method = pd.read_csv
         elif extension == Extensions.PARQUET.value:
             method = pd.read_parquet
-        elif extension == Extensions.FILE.value:
+        elif extension == Extensions.JSON.value:
+            method = pd.read_json
+        elif extension in (Extensions.EXCEL.value, Extensions.EXCEL_OLD.value):
+            method = pd.read_excel
+        elif extension in (Extensions.TXT.value, Extensions.FILE.value):
             try:
                 return self.read_df(path, Extensions.CSV.value, **kwargs)
             except ParserError:
@@ -196,7 +202,7 @@ class DataframeReaderPandas(DataframeReader):
         return schema
 
     @staticmethod
-    def get_preview(df: pd.DataFrame) -> Any:
+    def get_preview(df: pd.DataFrame) -> dict:
         """
         Get preview.
 
@@ -211,9 +217,51 @@ class DataframeReaderPandas(DataframeReader):
             The preview.
         """
         columns = [str(col) for col, _ in df.dtypes.items()]
-        head = df.head(10)
-        head = head.replace({np.nan: None})
-        head = head.values.tolist()
-        preview = get_data_preview(columns, head)
-        len_df = len(df)
-        return build_data_preview(preview, len_df)
+        head = df.head(10).replace({np.nan: None})
+        data = head.values.tolist()
+        prepared_data = prepare_data(data)
+        preview = prepare_preview(columns, prepared_data)
+        finalizes = finalize_preview(preview, df.shape[0])
+        serialized = _serialize_deserialize_preview(finalizes)
+        return check_preview_size(serialized)
+
+
+class PandasJsonEncoder(CustomJsonEncoder):
+    """
+    JSON pd.Timestamp to ISO format serializer.
+    """
+
+    def default(self, obj: Any) -> Any:
+        """
+        Pandas datetime to ISO format serializer.
+
+        Parameters
+        ----------
+        obj : Any
+            The object to serialize.
+
+        Returns
+        -------
+        Any
+            The serialized object.
+        """
+        if isinstance(obj, pd.Timestamp):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+def _serialize_deserialize_preview(preview: dict) -> dict:
+    """
+    Serialize and deserialize preview.
+
+    Parameters
+    ----------
+    preview : dict
+        The preview.
+
+    Returns
+    -------
+    dict
+        The serialized preview.
+    """
+    return json.loads(json.dumps(preview, cls=PandasJsonEncoder))

digitalhub/stores/_base/store.py

@@ -5,11 +5,9 @@ from pathlib import Path
 from tempfile import mkdtemp
 from typing import Any
 
-from pydantic import BaseModel, ConfigDict
-
 from digitalhub.readers.api import get_reader_by_engine
 from digitalhub.utils.exceptions import StoreError
-from digitalhub.utils.uri_utils import SchemeCategory, has_local_scheme
+from digitalhub.utils.uri_utils import has_local_scheme
 
 
 class Store:
@@ -17,24 +15,6 @@ class Store:
     Store abstract class.
     """
 
-    def __init__(self, name: str, store_type: str) -> None:
-        """
-        Constructor.
-
-        Parameters
-        ----------
-        name : str
-            Store name.
-        store_type : str
-            Store type. Used to choose the right store implementation.
-
-        Returns
-        -------
-        None
-        """
-        self.name = name
-        self.type = store_type
-
     ##############################
     # I/O methods
     ##############################
@@ -207,31 +187,3 @@ class Store:
         """
         tmpdir = mkdtemp()
         return Path(tmpdir)
-
-
-class StoreConfig(BaseModel):
-    """
-    Store configuration base class.
-    """
-
-    model_config = ConfigDict(use_enum_values=True)
-
-
-class StoreParameters(BaseModel):
-    """
-    Store configuration class.
-    """
-
-    model_config = ConfigDict(use_enum_values=True)
-
-    name: str
-    """Store id."""
-
-    type: SchemeCategory
-    """Store type to instantiate."""
-
-    config: StoreConfig = None
-    """Configuration for the store."""
-
-    is_default: bool = False
-    """Flag to determine if the store is the default one."""

digitalhub/stores/api.py

@@ -5,50 +5,25 @@ import typing
 from digitalhub.stores.builder import store_builder
 
 if typing.TYPE_CHECKING:
-    from digitalhub.stores._base.store import Store, StoreParameters
+    from digitalhub.stores._base.store import Store
 
 
-def set_store(store_cfg: StoreParameters) -> None:
+def get_store(project: str, uri: str, config: dict | None = None) -> Store:
     """
-    Set a new store instance with the given configuration.
-
-    Parameters
-    ----------
-    store_cfg : StoreParameters
-        Store configuration.
-
-    Returns
-    -------
-    None
-    """
-    store_builder.build(store_cfg)
-
-
-def get_store(uri: str) -> Store:
-    """
-    Get store instance by uri.
+    Get store instance by URI.
 
     Parameters
     ---------
+    project : str
+        Project name.
     uri : str
         URI to parse.
+    config : dict
+        Store configuration.
 
     Returns
     -------
     Store
         Store instance.
     """
-    return store_builder.get(uri)
-
-
-def get_default_store() -> Store:
-    """
-    Get the default store instance. The default store is the one that
-    can persist artifacts and dataitems.
-
-    Returns
-    -------
-    Store
-        Default store instance.
-    """
-    return store_builder.default()
+    return store_builder.get(uri, config)

digitalhub/stores/builder.py

@@ -1,28 +1,41 @@
 from __future__ import annotations
 
-import os
 import typing
 
-from pydantic import ValidationError
-
-from digitalhub.stores._base.store import StoreParameters
-from digitalhub.stores.local.store import LocalStore, LocalStoreConfig
-from digitalhub.stores.remote.store import RemoteStore, RemoteStoreConfig
-from digitalhub.stores.s3.store import S3Store, S3StoreConfig
-from digitalhub.stores.sql.store import SqlStore, SQLStoreConfig
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.configurator.api import get_current_env
+from digitalhub.stores.local.store import LocalStore
+from digitalhub.stores.remote.store import RemoteStore
+from digitalhub.stores.s3.store import S3Store
+from digitalhub.stores.sql.store import SqlStore
 from digitalhub.utils.uri_utils import SchemeCategory, map_uri_scheme
 
 if typing.TYPE_CHECKING:
     from digitalhub.stores._base.store import Store
 
 
-REGISTRY_STORES = {
-    "local": LocalStore,
-    "s3": S3Store,
-    "remote": RemoteStore,
-    "sql": SqlStore,
-}
+def _get_class_from_type(type: str) -> Store:
+    """
+    Get a store class from its type.
+
+    Parameters
+    ----------
+    type : str
+        Store type.
+
+    Returns
+    -------
+    Store
+        The store class.
+    """
+    if type == SchemeCategory.LOCAL.value:
+        return LocalStore
+    if type == SchemeCategory.S3.value:
+        return S3Store
+    if type == SchemeCategory.REMOTE.value:
+        return RemoteStore
+    if type == SchemeCategory.SQL.value:
+        return SqlStore
+    raise ValueError(f"Unknown store type: {type}")
 
 
 class StoreBuilder:
@@ -32,27 +45,28 @@ class StoreBuilder:
 
     def __init__(self) -> None:
         self._instances: dict[str, Store] = {}
-        self._default: Store | None = None
-        self._def_scheme = "s3"
 
-    def build(self, store_cfg: StoreParameters) -> None:
+    def build(self, store_type: str, config: dict | None = None) -> None:
         """
         Build a store instance and register it.
-        It overrides any existing instance.
 
         Parameters
         ----------
-        store_cfg : StoreParameters
+        store_type : str
+            Store type.
+        config : dict
             Store configuration.
 
         Returns
         -------
         None
         """
-        scheme = map_uri_scheme(store_cfg.type)
-        self._instances[scheme] = self.build_store(store_cfg)
+        env = get_current_env()
+        if env not in self._instances:
+            self._instances[env] = {}
+        self._instances[env][store_type] = _get_class_from_type(store_type)(config)
 
-    def get(self, uri: str) -> Store:
+    def get(self, uri: str, config: dict | None = None) -> Store:
         """
         Get a store instance by URI.
 
@@ -60,152 +74,21 @@ class StoreBuilder:
         ----------
         uri : str
             URI to parse.
-
-        Returns
-        -------
-        Store
-            The store instance.
-        """
-        scheme = map_uri_scheme(uri)
-        if scheme not in self._instances:
-            store_cfg = get_env_store_config(scheme)
-            self._instances[scheme] = self.build_store(store_cfg)
-        return self._instances[scheme]
-
-    def default(self) -> Store:
-        """
-        Get the default store instance.
-
-        Returns
-        -------
-        Store
-            The default store instance.
-
-        Raises
-        ------
-        StoreError
-            If no default store is set.
-        """
-        if self._default is None:
-            store_cfg = get_env_store_config(self._def_scheme)
-            self._default = self.build_store(store_cfg)
-        return self._default
-
-    def build_store(self, cfg: StoreParameters) -> Store:
-        """
-        Build a store instance.
-
-        Parameters
-        ----------
-        cfg : StoreParameters
+        config : dict
             Store configuration.
 
         Returns
         -------
         Store
             The store instance.
-
-        Raises
-        ------
-        NotImplementedError
-            If the store type is not implemented.
         """
+        env = get_current_env()
+        store_type = map_uri_scheme(uri)
         try:
-            obj = REGISTRY_STORES[cfg.type](cfg.name, cfg.type, cfg.config)
-            if cfg.is_default and self._default is not None:
-                raise StoreError("Only one default store!")
-            return obj
-        except KeyError as e:
-            raise NotImplementedError from e
-
-    @staticmethod
-    def _check_config(config: StoreParameters | dict) -> StoreParameters:
-        """
-        Check the store configuration validity.
-
-        Parameters
-        ----------
-        config : StoreParameters | dict
-            The store configuration.
-
-        Returns
-        -------
-        StoreParameters
-            The store configuration.
-
-        Raises
-        ------
-        TypeError
-            If the config parameter is not a StoreParameters instance or a well-formed dictionary.
-        """
-        if not isinstance(config, StoreParameters):
-            try:
-                return StoreParameters(**config)
-            except TypeError as e:
-                raise StoreError("Invalid store configuration type.") from e
-            except ValidationError as e:
-                raise StoreError("Malformed store configuration parameters.") from e
-        return config
-
-
-def get_env_store_config(scheme: str) -> StoreParameters:
-    """
-    Get a store configuration from the environment.
-
-    Parameters
-    ----------
-    scheme : str
-        URI scheme.
-
-    Returns
-    -------
-    StoreParameters
-        The store configuration based on the scheme.
-
-    Raises
-    ------
-    ValueError
-        If the scheme is not supported.
-    """
-    if scheme == SchemeCategory.S3.value:
-        return StoreParameters(
-            name="s3",
-            type="s3",
-            config=S3StoreConfig(
-                endpoint_url=os.getenv("S3_ENDPOINT_URL"),  # type: ignore
-                aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),  # type: ignore
-                aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),  # type: ignore
-                bucket_name=os.getenv("S3_BUCKET_NAME"),  # type: ignore
-            ),
-        )
-    if scheme == SchemeCategory.SQL.value:
-        return StoreParameters(
-            name="sql",
-            type="sql",
-            config=SQLStoreConfig(
-                host=os.getenv("POSTGRES_HOST"),  # type: ignore
-                port=os.getenv("POSTGRES_PORT"),  # type: ignore
-                user=os.getenv("POSTGRES_USER"),  # type: ignore
-                password=os.getenv("POSTGRES_PASSWORD"),  # type: ignore
-                database=os.getenv("POSTGRES_DATABASE"),  # type: ignore
-                pg_schema=os.getenv("POSTGRES_SCHEMA"),  # type: ignore
-            ),
-        )
-    if scheme == SchemeCategory.REMOTE.value:
-        return StoreParameters(
-            name="remote",
-            type="remote",
-            config=RemoteStoreConfig(),
-        )
-    if scheme == SchemeCategory.LOCAL.value:
-        return StoreParameters(
-            name="local",
-            type="local",
-            config=LocalStoreConfig(
-                path="tempsdk",
-            ),
-        )
-    raise ValueError(f"Unsupported scheme {scheme}")
+            return self._instances[env][store_type]
+        except KeyError:
+            self.build(store_type, config)
+            return self._instances[env][store_type]
 
 
 store_builder = StoreBuilder()

digitalhub/stores/local/store.py

@@ -5,29 +5,19 @@ from pathlib import Path
 from typing import Any
 
 from digitalhub.readers.api import get_reader_by_object
-from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.stores._base.store import Store
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_local
 
 
-class LocalStoreConfig(StoreConfig):
-    """
-    Local store configuration class.
-    """
-
-    path: str
-    """Local path."""
-
-
 class LocalStore(Store):
     """
     Local store class. It implements the Store interface and provides methods to fetch and persist
     artifacts on local filesystem based storage.
     """
 
-    def __init__(self, name: str, store_type: str, config: LocalStoreConfig) -> None:
-        super().__init__(name, store_type)
-        self.config = config
+    def __init__(self, config: dict | None = None) -> None:
+        super().__init__()
 
     ##############################
     # I/O methods
@@ -218,13 +208,9 @@ class LocalStore(Store):
         str
             Path of written dataframe.
         """
-        self.store._check_local_dst(dst)
-        self._validate_extension(Path(dst).suffix.removeprefix("."))
-
-        # Write dataframe
+        self._check_local_dst(dst)
         reader = get_reader_by_object(df)
         reader.write_df(df, dst, extension=extension, **kwargs)
-
         return dst
 
     ##############################

digitalhub/stores/remote/store.py

@@ -5,25 +5,18 @@ from typing import Any
 
 import requests
 
-from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.stores._base.store import Store
 from digitalhub.utils.exceptions import StoreError
 
 
-class RemoteStoreConfig(StoreConfig):
-    """
-    Remote store configuration class.
-    """
-
-
 class RemoteStore(Store):
     """
     HTTP store class. It implements the Store interface and provides methods to fetch
     artifacts from remote HTTP based storage.
     """
 
-    def __init__(self, name: str, store_type: str, config: RemoteStoreConfig) -> None:
-        super().__init__(name, store_type)
-        self.config = config
+    def __init__(self, config: dict | None = None) -> None:
+        super().__init__()
 
     ##############################
     # I/O methods

digitalhub/stores/s3/configurator.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from botocore.config import Config
+
+from digitalhub.configurator.configurator import configurator
+from digitalhub.stores.s3.enums import S3StoreEnv
+from digitalhub.stores.s3.models import S3StoreConfig
+from digitalhub.utils.exceptions import StoreError
+
+
+class S3StoreConfigurator:
+    """
+    Configure the store by getting the credentials from user
+    provided config or from environment.
+    """
+
+    def __init__(self, config: dict | None = None) -> None:
+        self.configure(config)
+
+    ##############################
+    # Configuration methods
+    ##############################
+
+    def configure(self, config: dict | None = None) -> None:
+        """
+        Configure the store by getting the credentials from user
+        provided config or from environment.
+
+        Parameters
+        ----------
+        config : dict
+            Configuration dictionary.
+
+        Returns
+        -------
+        None
+        """
+        # Validate config
+        if config is None:
+            self._get_config()
+        else:
+            config = S3StoreConfig(**config)
+            for pair in [
+                (S3StoreEnv.ENDPOINT_URL.value, config.endpoint),
+                (S3StoreEnv.ACCESS_KEY_ID.value, config.access_key),
+                (S3StoreEnv.SECRET_ACCESS_KEY.value, config.secret_key),
+                (S3StoreEnv.SESSION_TOKEN.value, config.session_token),
+                (S3StoreEnv.BUCKET_NAME.value, config.bucket_name),
+                (S3StoreEnv.REGION.value, config.region),
+                (S3StoreEnv.SIGNATURE_VERSION.value, config.signature_version),
+            ]:
+                configurator.set_credential(*pair)
+
+    def get_s3_creds(self) -> dict:
+        """
+        Get endpoint, access key and secret key.
+
+        Returns
+        -------
+        dict
+            The credentials.
+        """
+        creds = configurator.get_all_cred()
+        try:
+            return {
+                "endpoint_url": creds[S3StoreEnv.ENDPOINT_URL.value],
+                "aws_access_key_id": creds[S3StoreEnv.ACCESS_KEY_ID.value],
+                "aws_secret_access_key": creds[S3StoreEnv.SECRET_ACCESS_KEY.value],
+                "aws_session_token": creds[S3StoreEnv.SESSION_TOKEN.value],
+                "config": Config(
+                    region_name=creds[S3StoreEnv.REGION.value],
+                    signature_version=creds[S3StoreEnv.SIGNATURE_VERSION.value],
+                ),
+            }
+        except KeyError as e:
+            raise StoreError(f"Missing credentials for S3 store. {str(e)}")
+
+    def _get_config(self) -> None:
+        """
+        Get the store configuration.
+
+        Returns
+        -------
+        None
+        """
+        required_vars = [
+            S3StoreEnv.ENDPOINT_URL,
+            S3StoreEnv.ACCESS_KEY_ID,
+            S3StoreEnv.SECRET_ACCESS_KEY,
+            S3StoreEnv.BUCKET_NAME,
+        ]
+        optional_vars = [S3StoreEnv.REGION, S3StoreEnv.SIGNATURE_VERSION, S3StoreEnv.SESSION_TOKEN]
+
+        # Load required environment variables
+        credentials = {var.value: configurator.load_var(var.value) for var in required_vars}
+
+        # Check for missing required credentials
+        missing_vars = [key for key, value in credentials.items() if value is None]
+        if missing_vars:
+            raise StoreError(f"Missing credentials for S3 store: {', '.join(missing_vars)}")
+
+        # Load optional environment variables
+        credentials.update({var.value: configurator.load_var(var.value) for var in optional_vars})
+
+        # Set credentials
+        for key, value in credentials.items():
+            configurator.set_credential(key, value)

digitalhub/stores/s3/enums.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class S3StoreEnv(Enum):
+    """
+    S3Store environment
+    """
+
+    ENDPOINT_URL = "S3_ENDPOINT_URL"
+    ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
+    SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
+    SESSION_TOKEN = "AWS_SESSION_TOKEN"
+    BUCKET_NAME = "S3_BUCKET"
+    REGION = "S3_REGION"
+    SIGNATURE_VERSION = "S3_SIGNATURE_VERSION"