digitalhub 0.11.0b7__py3-none-any.whl → 0.13.0__py3-none-any.whl

digitalhub/stores/data/_base/store.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import typing
@@ -12,6 +16,7 @@ from digitalhub.utils.types import SourcesOrListOfSources
 from digitalhub.utils.uri_utils import has_local_scheme
 
 if typing.TYPE_CHECKING:
+    from digitalhub.stores.credentials.configurator import Configurator
     from digitalhub.stores.readers.data._base.reader import DataframeReader
 
 
@@ -20,6 +25,9 @@ class Store:
     Store abstract class.
     """
 
+    def __init__(self, configurator: Configurator | None = None) -> None:
+        self._configurator = configurator
+
     ##############################
     # I/O methods
     ##############################
@@ -27,19 +35,22 @@ class Store:
     @abstractmethod
     def download(
         self,
-        root: str,
+        src: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
-        Method to download artifact from storage.
+        Method to download material entity from storage.
         """
 
     @abstractmethod
-    def upload(self, src: SourcesOrListOfSources, dst: str) -> list[tuple[str, str]]:
+    def upload(
+        self,
+        src: SourcesOrListOfSources,
+        dst: str,
+    ) -> list[tuple[str, str]]:
         """
-        Method to upload artifact to storage.
+        Method to upload material entity to storage.
         """
 
     @abstractmethod
@@ -79,7 +90,13 @@ class Store:
         """
 
     @abstractmethod
-    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
+    def write_df(
+        self,
+        df: Any,
+        dst: str,
+        extension: str | None = None,
+        **kwargs,
+    ) -> str:
         """
         Write DataFrame as parquet or csv.
         """
@@ -169,9 +186,14 @@ class Store:
         """
         if not isinstance(path, Path):
             path = Path(path)
-        if path.suffix != "":
-            path = path.parent
-        path.mkdir(parents=True, exist_ok=True)
+        # If the path does not exist, we need to infer if it's a file or directory
+        if path.suffix and not path.name.startswith("."):
+            # Looks like a file, use parent
+            dir_path = path.parent
+        else:
+            # Looks like a directory (even if it contains dots)
+            dir_path = path
+        dir_path.mkdir(parents=True, exist_ok=True)
 
     @staticmethod
     def _build_temp() -> Path:

digitalhub/stores/data/api.py

@@ -1,29 +1,73 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import typing
 
+from digitalhub.context.api import get_context
+from digitalhub.stores.credentials.handler import creds_handler
 from digitalhub.stores.data.builder import store_builder
+from digitalhub.stores.data.enums import StoreEnv
 
 if typing.TYPE_CHECKING:
     from digitalhub.stores.data._base.store import Store
 
 
-def get_store(project: str, uri: str) -> Store:
+def get_default_store(project: str) -> str:
     """
-    Get store instance by URI.
+    Returns the default store URI for a given project.
 
     Parameters
-    ---------
+    ----------
     project : str
-        Project name.
+        The name of the project.
+
+    Returns
+    -------
+    str
+        The default store URI.
+
+    Raises
+    ------
+    ValueError
+        If no default store is found.
+    """
+    var = StoreEnv.DEFAULT_FILES_STORE.value
+
+    context = get_context(project)
+    store = context.config.get(var.lower().replace("dhcore_", ""))
+    if store is not None:
+        return store
+
+    store = creds_handler.load_from_env([var]).get(var)
+    if store is None:
+        store = creds_handler.load_from_file([var]).get(var)
+
+    if store is None or store == "":
+        raise ValueError(
+            "No default store found. "
+            "Please set a default store "
+            f"in your environment (e.g. export {var}=) "
+            " in the .dhcore.ini file "
+            "or set it in project config."
+        )
+    return store
+
+
+def get_store(uri: str) -> Store:
+    """
+    Returns a store instance for the given URI.
+
+    Parameters
+    ----------
     uri : str
-        URI to parse.
-    config : dict
-        Store configuration.
+        The URI to parse.
 
     Returns
     -------
     Store
-        Store instance.
+        The store instance corresponding to the URI.
     """
-    return store_builder.get(project, uri)
+    return store_builder.get(uri)

digitalhub/stores/data/builder.py

@@ -1,93 +1,146 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import typing
 
-from digitalhub.stores.configurator.api import get_current_env
 from digitalhub.stores.data.local.store import LocalStore
 from digitalhub.stores.data.remote.store import RemoteStore
+from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.stores.data.s3.store import S3Store
+from digitalhub.stores.data.sql.configurator import SqlStoreConfigurator
 from digitalhub.stores.data.sql.store import SqlStore
 from digitalhub.utils.uri_utils import SchemeCategory, map_uri_scheme
 
 if typing.TYPE_CHECKING:
+    from digitalhub.stores.credentials.configurator import Configurator
     from digitalhub.stores.data._base.store import Store
+    from digitalhub.utils.exceptions import StoreError
 
 
-def _get_class_from_type(type: str) -> Store:
+class StoreInfo:
     """
-    Get a store class from its type.
+    Container for store class and configurator information.
 
-    Parameters
-    ----------
-    type : str
-        Store type.
+    Holds store class references and their associated configurators
+    for registration and instantiation in the store builder system.
 
-    Returns
-    -------
-    Store
-        The store class.
+    Attributes
+    ----------
+    _store : Store
+        The store class to be instantiated.
+    _configurator : Configurator or None
+        The configurator class for store configuration, if required.
     """
-    if type == SchemeCategory.LOCAL.value:
-        return LocalStore
-    if type == SchemeCategory.S3.value:
-        return S3Store
-    if type == SchemeCategory.REMOTE.value:
-        return RemoteStore
-    if type == SchemeCategory.SQL.value:
-        return SqlStore
-    raise ValueError(f"Unknown store type: {type}")
+
+    def __init__(self, store: Store, configurator: Configurator | None = None) -> None:
+        self._store = store
+        self._configurator = configurator
 
 
 class StoreBuilder:
     """
-    Store builder class.
+    Store factory and registry for managing data store instances.
+
+    Provides registration, instantiation, and caching of data store
+    instances based on URI schemes. Supports various store types
+    including S3, SQL, local, and remote stores with their respective
+    configurators.
+
+    Attributes
+    ----------
+    _builders : dict[str, StoreInfo]
+        Registry of store types mapped to their StoreInfo instances.
+    _instances : dict[str, Store]
+        Cache of instantiated store instances by store type.
     """
 
     def __init__(self) -> None:
+        self._builders: dict[str, StoreInfo] = {}
         self._instances: dict[str, dict[str, Store]] = {}
 
-    def build(self, project: str, store_type: str) -> None:
+    def register(
+        self,
+        store_type: str,
+        store: Store,
+        configurator: Configurator | None = None,
+    ) -> None:
         """
-        Build a store instance and register it.
+        Register a store type with its class and optional configurator.
+
+        Adds a new store type to the builder registry, associating it
+        with a store class and optional configurator for later instantiation.
 
         Parameters
         ----------
         store_type : str
-            Store type.
-        config : dict
+            The unique identifier for the store type (e.g., 's3', 'sql').
+        store : Store
+            The store class to register for this type.
+        configurator : Configurator, optional
+            The configurator class for store configuration.
+            If None, the store will be instantiated without configuration.
 
         Returns
         -------
         None
+
+        Raises
+        ------
+        StoreError
+            If the store type is already registered in the builder.
         """
-        env = get_current_env()
-        if env not in self._instances:
-            self._instances[env] = {}
-        self._instances[env][store_type] = _get_class_from_type(store_type)()
+        if store_type not in self._builders:
+            self._builders[store_type] = StoreInfo(store, configurator)
+        else:
+            raise StoreError(f"Store type {store_type} already registered")
 
-    def get(self, project: str, uri: str) -> Store:
+    def get(self, uri: str) -> Store:
         """
-        Get a store instance by URI.
+        Get or create a store instance based on URI scheme.
+
+        Determines the appropriate store type from the URI scheme,
+        instantiates the store if not already cached, and returns
+        the store instance. Store instances are cached for reuse.
 
         Parameters
         ----------
         uri : str
-            URI to parse.
-        config : dict
-            Store configuration.
+            The URI to parse for determining the store type.
+            The scheme (e.g., 's3://', 'sql://') determines which
+            store type to instantiate.
 
         Returns
         -------
         Store
-            The store instance.
+            The store instance appropriate for handling the given URI.
+
+        Raises
+        ------
+        KeyError
+            If no store is registered for the URI scheme.
         """
-        env = get_current_env()
         store_type = map_uri_scheme(uri)
-        try:
-            return self._instances[env][store_type]
-        except KeyError:
-            self.build(project, store_type)
-            return self._instances[env][store_type]
+
+        # Build the store instance if not already present
+        if store_type not in self._instances:
+            store_info = self._builders[store_type]
+            store_cls = store_info._store
+            cfgrt_cls = store_info._configurator
+
+            if cfgrt_cls is None:
+                store = store_cls()
+            else:
+                store = store_cls(cfgrt_cls())
+            self._instances[store_type] = store
+
+        return self._instances[store_type]
 
 
 store_builder = StoreBuilder()
+store_builder.register(SchemeCategory.S3.value, S3Store, S3StoreConfigurator)
+store_builder.register(SchemeCategory.SQL.value, SqlStore, SqlStoreConfigurator)
+store_builder.register(SchemeCategory.LOCAL.value, LocalStore)
+store_builder.register(SchemeCategory.REMOTE.value, RemoteStore)

digitalhub/stores/data/enums.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from enum import Enum

digitalhub/stores/data/local/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0

digitalhub/stores/data/local/store.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import shutil
@@ -25,7 +29,6 @@ class LocalStore(Store):
         self,
         root: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -33,19 +36,17 @@ class LocalStore(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
         raise StoreError("Local store does not support download.")
 

digitalhub/stores/data/remote/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0

digitalhub/stores/data/remote/store.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from pathlib import Path
@@ -24,7 +28,6 @@ class RemoteStore(Store):
         self,
         root: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -32,19 +35,17 @@ class RemoteStore(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
         # Handle destination
         if dst is None:

digitalhub/stores/data/s3/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0

digitalhub/stores/data/s3/configurator.py

@@ -1,126 +1,115 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
+from datetime import datetime, timezone
+
 from botocore.config import Config
 
-from digitalhub.stores.configurator.configurator import configurator
-from digitalhub.stores.configurator.enums import CredsOrigin
-from digitalhub.stores.data.s3.enums import S3StoreEnv
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.stores.client.dhcore.utils import refresh_token
+from digitalhub.stores.credentials.configurator import Configurator
+from digitalhub.stores.credentials.enums import CredsEnvVar
 
 
-class S3StoreConfigurator:
+class S3StoreConfigurator(Configurator):
     """
     Configure the store by getting the credentials from user
     provided config or from environment.
     """
 
-    required_vars = [
-        S3StoreEnv.ENDPOINT_URL,
-        S3StoreEnv.ACCESS_KEY_ID,
-        S3StoreEnv.SECRET_ACCESS_KEY,
+    keys = [
+        CredsEnvVar.S3_ENDPOINT_URL.value,
+        CredsEnvVar.S3_ACCESS_KEY_ID.value,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY.value,
+        CredsEnvVar.S3_REGION.value,
+        CredsEnvVar.S3_SIGNATURE_VERSION.value,
+        CredsEnvVar.S3_SESSION_TOKEN.value,
+        CredsEnvVar.S3_PATH_STYLE.value,
+        CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value,
     ]
-    optional_vars = [
-        S3StoreEnv.REGION,
-        S3StoreEnv.SIGNATURE_VERSION,
-        S3StoreEnv.SESSION_TOKEN,
+    required_keys = [
+        CredsEnvVar.S3_ENDPOINT_URL.value,
+        CredsEnvVar.S3_ACCESS_KEY_ID.value,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY.value,
     ]
 
+    def __init__(self):
+        super().__init__()
+        self.load_configs()
+
     ##############################
     # Configuration methods
     ##############################
 
-    def get_boto3_client_config(self, origin: str) -> dict:
+    def load_env_vars(self) -> None:
         """
-        Get S3 credentials (access key, secret key,
-        session token and other config).
-
-        Parameters
-        ----------
-        origin : str
-            The origin of the credentials.
+        Loads the credentials from the environment variables.
 
         Returns
         -------
-        dict
-            The credentials.
+        None
         """
-        if origin == CredsOrigin.ENV.value:
-            creds = self._get_env_config()
-        elif origin == CredsOrigin.FILE.value:
-            creds = self._get_file_config()
-        else:
-            raise StoreError(f"Unknown origin: {origin}")
-        return {
-            "endpoint_url": creds[S3StoreEnv.ENDPOINT_URL.value],
-            "aws_access_key_id": creds[S3StoreEnv.ACCESS_KEY_ID.value],
-            "aws_secret_access_key": creds[S3StoreEnv.SECRET_ACCESS_KEY.value],
-            "aws_session_token": creds[S3StoreEnv.SESSION_TOKEN.value],
-            "config": Config(
-                region_name=creds[S3StoreEnv.REGION.value],
-                signature_version=creds[S3StoreEnv.SIGNATURE_VERSION.value],
-            ),
-        }
+        env_creds = self._creds_handler.load_from_env(self.keys)
+        self._creds_handler.set_credentials(self._env, env_creds)
 
-    def _get_env_config(self) -> dict:
+    def load_file_vars(self) -> None:
         """
-        Get the store configuration from environment variables.
+        Loads the credentials from a file.
 
         Returns
         -------
-        dict
-            The credentials.
+        None
         """
-        credentials = {
-            var.value: configurator.load_from_env(var.value) for var in self.required_vars + self.optional_vars
-        }
-        self._set_credentials(credentials)
-        return credentials
+        file_creds = self._creds_handler.load_from_file(self.keys)
+        self._creds_handler.set_credentials(self._file, file_creds)
 
-    def _get_file_config(self) -> dict:
+    def get_client_config(self) -> dict:
         """
-        Get the store configuration from file.
+        Gets S3 credentials (access key, secret key, session token, and other config).
 
         Returns
         -------
         dict
-            The credentials.
+            Dictionary containing S3 credentials and configuration.
         """
-        credentials = {
-            var.value: configurator.load_from_file(var.value) for var in self.required_vars + self.optional_vars
+        creds = self.get_credentials(self._origin)
+        expired = creds[CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value]
+        if self._origin == self._file and self._is_expired(expired):
+            refresh_token()
+            self.load_file_vars()
+            creds = self.get_credentials(self._origin)
+        return {
+            "endpoint_url": creds[CredsEnvVar.S3_ENDPOINT_URL.value],
+            "aws_access_key_id": creds[CredsEnvVar.S3_ACCESS_KEY_ID.value],
+            "aws_secret_access_key": creds[CredsEnvVar.S3_SECRET_ACCESS_KEY.value],
+            "aws_session_token": creds[CredsEnvVar.S3_SESSION_TOKEN.value],
+            "config": Config(
+                region_name=creds[CredsEnvVar.S3_REGION.value],
+                signature_version=creds[CredsEnvVar.S3_SIGNATURE_VERSION.value],
+            ),
         }
-        self._set_credentials(credentials)
-        return credentials
-
-    def _check_credentials(self, credentials: dict) -> None:
-        """
-        Check for missing credentials.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
 
-        Returns
-        -------
-        None
+    @staticmethod
+    def _is_expired(timestamp: str | None) -> bool:
         """
-        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
-        if missing_vars:
-            raise StoreError(f"Missing credentials for S3 store: {', '.join(missing_vars)}")
-
-    def _set_credentials(self, credentials: dict) -> None:
-        """
-        Set the store credentials into the configurator.
+        Determines whether a given timestamp is after the current UTC time.
 
         Parameters
         ----------
-        credentials : dict
-            The credentials.
+        timestamp : str or None
+            Timestamp string in the format 'YYYY-MM-DDTHH:MM:SSZ'.
 
         Returns
         -------
-        None
+        bool
+            True if the given timestamp is later than the current UTC time,
+            otherwise False.
         """
-        # Set credentials
-        for key, value in credentials.items():
-            configurator.set_credential(key, value)
+        if timestamp is None:
+            return False
+        dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ")
+        dt = dt.replace(tzinfo=timezone.utc)
+        now = datetime.now(timezone.utc) + datetime.timedelta(seconds=120)
+        return dt < now