digitalhub 0.11.0b7__py3-none-any.whl → 0.13.0b0__py3-none-any.whl

digitalhub/stores/data/builder.py

@@ -1,41 +1,30 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import typing
 
-from digitalhub.stores.configurator.api import get_current_env
+from digitalhub.stores.credentials.api import get_current_env
 from digitalhub.stores.data.local.store import LocalStore
 from digitalhub.stores.data.remote.store import RemoteStore
+from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.stores.data.s3.store import S3Store
+from digitalhub.stores.data.sql.configurator import SqlStoreConfigurator
 from digitalhub.stores.data.sql.store import SqlStore
 from digitalhub.utils.uri_utils import SchemeCategory, map_uri_scheme
 
 if typing.TYPE_CHECKING:
+    from digitalhub.stores.credentials.configurator import Configurator
     from digitalhub.stores.data._base.store import Store
+    from digitalhub.utils.exceptions import StoreError
 
 
-def _get_class_from_type(type: str) -> Store:
-    """
-    Get a store class from its type.
-
-    Parameters
-    ----------
-    type : str
-        Store type.
-
-    Returns
-    -------
-    Store
-        The store class.
-    """
-    if type == SchemeCategory.LOCAL.value:
-        return LocalStore
-    if type == SchemeCategory.S3.value:
-        return S3Store
-    if type == SchemeCategory.REMOTE.value:
-        return RemoteStore
-    if type == SchemeCategory.SQL.value:
-        return SqlStore
-    raise ValueError(f"Unknown store type: {type}")
+class StoreInfo:
+    def __init__(self, store: Store, configurator: Configurator | None = None) -> None:
+        self._store = store
+        self._configurator = configurator
 
 
 class StoreBuilder:
@@ -44,50 +33,58 @@ class StoreBuilder:
     """
 
     def __init__(self) -> None:
+        self._builders: dict[str, StoreInfo] = {}
         self._instances: dict[str, dict[str, Store]] = {}
 
-    def build(self, project: str, store_type: str) -> None:
-        """
-        Build a store instance and register it.
-
-        Parameters
-        ----------
-        store_type : str
-            Store type.
-        config : dict
-
-        Returns
-        -------
-        None
-        """
-        env = get_current_env()
-        if env not in self._instances:
-            self._instances[env] = {}
-        self._instances[env][store_type] = _get_class_from_type(store_type)()
-
-    def get(self, project: str, uri: str) -> Store:
+    def register(
+        self,
+        store_type: str,
+        store: Store,
+        configurator: Configurator | None = None,
+    ) -> None:
+        if store_type not in self._builders:
+            self._builders[store_type] = StoreInfo(store, configurator)
+        else:
+            raise StoreError(f"Store type {store_type} already registered")
+
+    def get(self, uri: str) -> Store:
         """
-        Get a store instance by URI.
+        Get a store instance by URI, building it if necessary.
 
         Parameters
         ----------
         uri : str
             URI to parse.
-        config : dict
-            Store configuration.
 
         Returns
         -------
         Store
             The store instance.
         """
-        env = get_current_env()
+        env_profile = get_current_env()
         store_type = map_uri_scheme(uri)
-        try:
-            return self._instances[env][store_type]
-        except KeyError:
-            self.build(project, store_type)
-            return self._instances[env][store_type]
+
+        # Ensure env exists in _instances
+        if env_profile not in self._instances:
+            self._instances[env_profile] = {}
+
+        # Build the store instance if not already present
+        if store_type not in self._instances[env_profile]:
+            store_info = self._builders[store_type]
+            store_cls = store_info._store
+            cfgrt_cls = store_info._configurator
+
+            if cfgrt_cls is None:
+                store = store_cls()
+            else:
+                store = store_cls(cfgrt_cls())
+            self._instances[env_profile][store_type] = store
+
+        return self._instances[env_profile][store_type]
 
 
 store_builder = StoreBuilder()
+store_builder.register(SchemeCategory.S3.value, S3Store, S3StoreConfigurator)
+store_builder.register(SchemeCategory.SQL.value, SqlStore, SqlStoreConfigurator)
+store_builder.register(SchemeCategory.LOCAL.value, LocalStore)
+store_builder.register(SchemeCategory.REMOTE.value, RemoteStore)

digitalhub/stores/data/enums.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from enum import Enum

digitalhub/stores/data/local/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0

digitalhub/stores/data/local/store.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 import shutil
@@ -25,7 +29,6 @@ class LocalStore(Store):
         self,
         root: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -33,19 +36,17 @@ class LocalStore(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
         raise StoreError("Local store does not support download.")
 

digitalhub/stores/data/remote/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0

digitalhub/stores/data/remote/store.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from pathlib import Path
@@ -24,7 +28,6 @@ class RemoteStore(Store):
         self,
         root: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -32,19 +35,17 @@ class RemoteStore(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
         # Handle destination
         if dst is None:

digitalhub/stores/data/s3/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0

digitalhub/stores/data/s3/configurator.py

@@ -1,38 +1,55 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from botocore.config import Config
 
-from digitalhub.stores.configurator.configurator import configurator
-from digitalhub.stores.configurator.enums import CredsOrigin
-from digitalhub.stores.data.s3.enums import S3StoreEnv
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.stores.credentials.configurator import Configurator
+from digitalhub.stores.credentials.enums import CredsEnvVar
 
 
-class S3StoreConfigurator:
+class S3StoreConfigurator(Configurator):
     """
     Configure the store by getting the credentials from user
     provided config or from environment.
     """
 
-    required_vars = [
-        S3StoreEnv.ENDPOINT_URL,
-        S3StoreEnv.ACCESS_KEY_ID,
-        S3StoreEnv.SECRET_ACCESS_KEY,
+    keys = [
+        CredsEnvVar.S3_ENDPOINT_URL,
+        CredsEnvVar.S3_ACCESS_KEY_ID,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY,
+        CredsEnvVar.S3_REGION,
+        CredsEnvVar.S3_SIGNATURE_VERSION,
+        CredsEnvVar.S3_SESSION_TOKEN,
     ]
-    optional_vars = [
-        S3StoreEnv.REGION,
-        S3StoreEnv.SIGNATURE_VERSION,
-        S3StoreEnv.SESSION_TOKEN,
+    required_keys = [
+        CredsEnvVar.S3_ENDPOINT_URL,
+        CredsEnvVar.S3_ACCESS_KEY_ID,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY,
     ]
 
+    def __init__(self):
+        super().__init__()
+        self.load_configs()
+
     ##############################
     # Configuration methods
     ##############################
 
-    def get_boto3_client_config(self, origin: str) -> dict:
+    def load_configs(self) -> None:
+        # Load from env
+        env_creds = {var.value: self._creds_handler.load_from_env(var.value) for var in self.keys}
+        self._creds_handler.set_credentials(self._env, env_creds)
+
+        # Load from file
+        file_creds = {var.value: self._creds_handler.load_from_file(var.value) for var in self.keys}
+        self._creds_handler.set_credentials(self._file, file_creds)
+
+    def get_client_config(self, origin: str) -> dict:
         """
-        Get S3 credentials (access key, secret key,
-        session token and other config).
+        Get S3 credentials (access key, secret key, session token and other config).
 
         Parameters
         ----------
@@ -44,83 +61,14 @@ class S3StoreConfigurator:
         dict
             The credentials.
         """
-        if origin == CredsOrigin.ENV.value:
-            creds = self._get_env_config()
-        elif origin == CredsOrigin.FILE.value:
-            creds = self._get_file_config()
-        else:
-            raise StoreError(f"Unknown origin: {origin}")
+        creds = self.get_credentials(origin)
         return {
-            "endpoint_url": creds[S3StoreEnv.ENDPOINT_URL.value],
-            "aws_access_key_id": creds[S3StoreEnv.ACCESS_KEY_ID.value],
-            "aws_secret_access_key": creds[S3StoreEnv.SECRET_ACCESS_KEY.value],
-            "aws_session_token": creds[S3StoreEnv.SESSION_TOKEN.value],
+            "endpoint_url": creds[CredsEnvVar.S3_ENDPOINT_URL.value],
+            "aws_access_key_id": creds[CredsEnvVar.S3_ACCESS_KEY_ID.value],
+            "aws_secret_access_key": creds[CredsEnvVar.S3_SECRET_ACCESS_KEY.value],
+            "aws_session_token": creds[CredsEnvVar.S3_SESSION_TOKEN.value],
             "config": Config(
-                region_name=creds[S3StoreEnv.REGION.value],
-                signature_version=creds[S3StoreEnv.SIGNATURE_VERSION.value],
+                region_name=creds[CredsEnvVar.S3_REGION.value],
+                signature_version=creds[CredsEnvVar.S3_SIGNATURE_VERSION.value],
             ),
         }
-
-    def _get_env_config(self) -> dict:
-        """
-        Get the store configuration from environment variables.
-
-        Returns
-        -------
-        dict
-            The credentials.
-        """
-        credentials = {
-            var.value: configurator.load_from_env(var.value) for var in self.required_vars + self.optional_vars
-        }
-        self._set_credentials(credentials)
-        return credentials
-
-    def _get_file_config(self) -> dict:
-        """
-        Get the store configuration from file.
-
-        Returns
-        -------
-        dict
-            The credentials.
-        """
-        credentials = {
-            var.value: configurator.load_from_file(var.value) for var in self.required_vars + self.optional_vars
-        }
-        self._set_credentials(credentials)
-        return credentials
-
-    def _check_credentials(self, credentials: dict) -> None:
-        """
-        Check for missing credentials.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
-
-        Returns
-        -------
-        None
-        """
-        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
-        if missing_vars:
-            raise StoreError(f"Missing credentials for S3 store: {', '.join(missing_vars)}")
-
-    def _set_credentials(self, credentials: dict) -> None:
-        """
-        Set the store credentials into the configurator.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
-
-        Returns
-        -------
-        None
-        """
-        # Set credentials
-        for key, value in credentials.items():
-            configurator.set_credential(key, value)

digitalhub/stores/data/s3/store.py

@@ -1,5 +1,10 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
+import typing
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Type
@@ -7,17 +12,22 @@ from urllib.parse import urlparse
 
 import boto3
 import botocore.client  # pylint: disable=unused-import
+from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
-from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.stores.data.s3.utils import get_bucket_name
 from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
+if typing.TYPE_CHECKING:
+    from digitalhub.stores.credentials.configurator import Configurator
+    from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
+
+
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
@@ -28,8 +38,9 @@ class S3Store(Store):
     artifacts on S3 based storage.
     """
 
-    def __init__(self) -> None:
-        self._configurator = S3StoreConfigurator()
+    def __init__(self, configurator: Configurator | None = None) -> None:
+        super().__init__(configurator)
+        self._configurator: S3StoreConfigurator
 
     ##############################
     # I/O methods
@@ -37,9 +48,8 @@ class S3Store(Store):
 
     def download(
         self,
-        root: str,
+        src: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -47,21 +57,19 @@ class S3Store(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
-        client, bucket = self._check_factory(root)
+        client, bucket = self._check_factory(src)
 
         # Build destination directory
         if dst.suffix == "":
@@ -70,20 +78,13 @@ class S3Store(Store):
             dst.parent.mkdir(parents=True, exist_ok=True)
 
         # Handle src and tree destination
-        if self.is_partition(root):
-            if not src:
-                keys = self._list_objects(client, bucket, root)
-                strip_root = self._get_key(root)
-                trees = [k.removeprefix(strip_root) for k in keys]
-            else:
-                keys = self._build_key_from_root(root, src)
-                trees = [s for s in src]
+        if self.is_partition(src):
+            keys = self._list_objects(client, bucket, src)
+            strip_root = self._get_key(src)
+            trees = [k.removeprefix(strip_root) for k in keys]
         else:
-            keys = [self._get_key(root)]
-            if not src:
-                trees = [Path(self._get_key(root)).name]
-            else:
-                trees = [s for s in src]
+            keys = [self._get_key(src)]
+            trees = [Path(self._get_key(src)).name]
 
         if len(keys) != len(trees):
             raise StoreError("Keys and trees must have the same length.")
@@ -124,7 +125,7 @@ class S3Store(Store):
         src : SourcesOrListOfSources
             Source(s).
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
 
         Returns
         -------
@@ -410,7 +411,7 @@ class S3Store(Store):
         src : str
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -453,7 +454,7 @@ class S3Store(Store):
         src : list
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -493,7 +494,7 @@ class S3Store(Store):
         src : str
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -542,7 +543,13 @@ class S3Store(Store):
         mime_type = get_file_mime_type(src)
         if mime_type is not None:
             extra_args["ContentType"] = mime_type
-        client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=extra_args)
+        client.upload_file(
+            Filename=src,
+            Bucket=bucket,
+            Key=key,
+            ExtraArgs=extra_args,
+            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+        )
 
     @staticmethod
     def _upload_fileobject(
@@ -569,7 +576,12 @@ class S3Store(Store):
         -------
         None
         """
-        client.put_object(Bucket=bucket, Key=key, Body=fileobj.getvalue())
+        client.upload_fileobj(
+            Fileobj=fileobj,
+            Bucket=bucket,
+            Key=key,
+            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+        )
 
     ##############################
     # Helper methods
@@ -615,13 +627,13 @@ class S3Store(Store):
 
         # Try to get client from environment variables
         try:
-            cfg = self._configurator.get_boto3_client_config(CredsOrigin.ENV.value)
+            cfg = self._configurator.get_client_config(CredsOrigin.ENV.value)
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
 
         # Fallback to file
         except StoreError:
-            cfg = self._configurator.get_boto3_client_config(CredsOrigin.FILE.value)
+            cfg = self._configurator.get_client_config(CredsOrigin.FILE.value)
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
 
@@ -672,29 +684,6 @@ class S3Store(Store):
             key = key[1:]
         return key
 
-    def _build_key_from_root(self, root: str, paths: list[str]) -> list[str]:
-        """
-        Method to build object path.
-
-        Parameters
-        ----------
-        root : str
-            The root of the object path.
-        paths : list[str]
-            The path to build.
-
-        Returns
-        -------
-        list[str]
-            List of keys.
-        """
-        keys = []
-        for path in paths:
-            clean_path = self._get_key(path)
-            key = self._get_key(f"{root}{clean_path}")
-            keys.append(key)
-        return keys
-
     def _list_objects(self, client: S3Client, bucket: str, partition: str) -> list[str]:
         """
         List objects in a S3 partition.

digitalhub/stores/data/s3/utils.py

@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import annotations
 
 from pathlib import Path
@@ -5,7 +9,7 @@ from urllib.parse import urlparse
 
 from boto3 import client as boto3_client
 
-from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.utils.exceptions import StoreError
 

digitalhub/stores/data/sql/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2025 DSLab - Fondazione Bruno Kessler
+#
+# SPDX-License-Identifier: Apache-2.0