digitalhub 0.12.0__py3-none-any.whl → 0.13.0b1__py3-none-any.whl

digitalhub/stores/data/s3/configurator.py

@@ -6,37 +6,50 @@ from __future__ import annotations
 
 from botocore.config import Config
 
-from digitalhub.stores.configurator.configurator import configurator
-from digitalhub.stores.configurator.enums import CredsOrigin
-from digitalhub.stores.data.s3.enums import S3StoreEnv
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.stores.credentials.configurator import Configurator
+from digitalhub.stores.credentials.enums import CredsEnvVar
 
 
-class S3StoreConfigurator:
+class S3StoreConfigurator(Configurator):
     """
     Configure the store by getting the credentials from user
     provided config or from environment.
     """
 
-    required_vars = [
-        S3StoreEnv.ENDPOINT_URL,
-        S3StoreEnv.ACCESS_KEY_ID,
-        S3StoreEnv.SECRET_ACCESS_KEY,
+    keys = [
+        CredsEnvVar.S3_ENDPOINT_URL,
+        CredsEnvVar.S3_ACCESS_KEY_ID,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY,
+        CredsEnvVar.S3_REGION,
+        CredsEnvVar.S3_SIGNATURE_VERSION,
+        CredsEnvVar.S3_SESSION_TOKEN,
     ]
-    optional_vars = [
-        S3StoreEnv.REGION,
-        S3StoreEnv.SIGNATURE_VERSION,
-        S3StoreEnv.SESSION_TOKEN,
+    required_keys = [
+        CredsEnvVar.S3_ENDPOINT_URL,
+        CredsEnvVar.S3_ACCESS_KEY_ID,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY,
     ]
 
+    def __init__(self):
+        super().__init__()
+        self.load_configs()
+
     ##############################
     # Configuration methods
     ##############################
 
-    def get_boto3_client_config(self, origin: str) -> dict:
+    def load_configs(self) -> None:
+        # Load from env
+        env_creds = {var.value: self._creds_handler.load_from_env(var.value) for var in self.keys}
+        self._creds_handler.set_credentials(self._env, env_creds)
+
+        # Load from file
+        file_creds = {var.value: self._creds_handler.load_from_file(var.value) for var in self.keys}
+        self._creds_handler.set_credentials(self._file, file_creds)
+
+    def get_client_config(self, origin: str) -> dict:
         """
-        Get S3 credentials (access key, secret key,
-        session token and other config).
+        Get S3 credentials (access key, secret key, session token and other config).
 
         Parameters
         ----------
@@ -48,83 +61,14 @@ class S3StoreConfigurator:
         dict
             The credentials.
         """
-        if origin == CredsOrigin.ENV.value:
-            creds = self._get_env_config()
-        elif origin == CredsOrigin.FILE.value:
-            creds = self._get_file_config()
-        else:
-            raise StoreError(f"Unknown origin: {origin}")
+        creds = self.get_credentials(origin)
         return {
-            "endpoint_url": creds[S3StoreEnv.ENDPOINT_URL.value],
-            "aws_access_key_id": creds[S3StoreEnv.ACCESS_KEY_ID.value],
-            "aws_secret_access_key": creds[S3StoreEnv.SECRET_ACCESS_KEY.value],
-            "aws_session_token": creds[S3StoreEnv.SESSION_TOKEN.value],
+            "endpoint_url": creds[CredsEnvVar.S3_ENDPOINT_URL.value],
+            "aws_access_key_id": creds[CredsEnvVar.S3_ACCESS_KEY_ID.value],
+            "aws_secret_access_key": creds[CredsEnvVar.S3_SECRET_ACCESS_KEY.value],
+            "aws_session_token": creds[CredsEnvVar.S3_SESSION_TOKEN.value],
             "config": Config(
-                region_name=creds[S3StoreEnv.REGION.value],
-                signature_version=creds[S3StoreEnv.SIGNATURE_VERSION.value],
+                region_name=creds[CredsEnvVar.S3_REGION.value],
+                signature_version=creds[CredsEnvVar.S3_SIGNATURE_VERSION.value],
             ),
         }
-
-    def _get_env_config(self) -> dict:
-        """
-        Get the store configuration from environment variables.
-
-        Returns
-        -------
-        dict
-            The credentials.
-        """
-        credentials = {
-            var.value: configurator.load_from_env(var.value) for var in self.required_vars + self.optional_vars
-        }
-        self._set_credentials(credentials)
-        return credentials
-
-    def _get_file_config(self) -> dict:
-        """
-        Get the store configuration from file.
-
-        Returns
-        -------
-        dict
-            The credentials.
-        """
-        credentials = {
-            var.value: configurator.load_from_file(var.value) for var in self.required_vars + self.optional_vars
-        }
-        self._set_credentials(credentials)
-        return credentials
-
-    def _check_credentials(self, credentials: dict) -> None:
-        """
-        Check for missing credentials.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
-
-        Returns
-        -------
-        None
-        """
-        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
-        if missing_vars:
-            raise StoreError(f"Missing credentials for S3 store: {', '.join(missing_vars)}")
-
-    def _set_credentials(self, credentials: dict) -> None:
-        """
-        Set the store credentials into the configurator.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
-
-        Returns
-        -------
-        None
-        """
-        # Set credentials
-        for key, value in credentials.items():
-            configurator.set_credential(key, value)

digitalhub/stores/data/s3/store.py

@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import typing
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Type
@@ -11,17 +12,22 @@ from urllib.parse import urlparse
 
 import boto3
 import botocore.client  # pylint: disable=unused-import
+from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
-from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.stores.data.s3.utils import get_bucket_name
 from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
+if typing.TYPE_CHECKING:
+    from digitalhub.stores.credentials.configurator import Configurator
+    from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
+
+
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
@@ -32,8 +38,9 @@ class S3Store(Store):
     artifacts on S3 based storage.
     """
 
-    def __init__(self) -> None:
-        self._configurator = S3StoreConfigurator()
+    def __init__(self, configurator: Configurator | None = None) -> None:
+        super().__init__(configurator)
+        self._configurator: S3StoreConfigurator
 
     ##############################
     # I/O methods
@@ -41,9 +48,8 @@ class S3Store(Store):
 
     def download(
         self,
-        root: str,
+        src: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -51,21 +57,19 @@ class S3Store(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
-        client, bucket = self._check_factory(root)
+        client, bucket = self._check_factory(src)
 
         # Build destination directory
         if dst.suffix == "":
@@ -74,20 +78,13 @@ class S3Store(Store):
             dst.parent.mkdir(parents=True, exist_ok=True)
 
         # Handle src and tree destination
-        if self.is_partition(root):
-            if not src:
-                keys = self._list_objects(client, bucket, root)
-                strip_root = self._get_key(root)
-                trees = [k.removeprefix(strip_root) for k in keys]
-            else:
-                keys = self._build_key_from_root(root, src)
-                trees = [s for s in src]
+        if self.is_partition(src):
+            keys = self._list_objects(client, bucket, src)
+            strip_root = self._get_key(src)
+            trees = [k.removeprefix(strip_root) for k in keys]
         else:
-            keys = [self._get_key(root)]
-            if not src:
-                trees = [Path(self._get_key(root)).name]
-            else:
-                trees = [s for s in src]
+            keys = [self._get_key(src)]
+            trees = [Path(self._get_key(src)).name]
 
         if len(keys) != len(trees):
             raise StoreError("Keys and trees must have the same length.")
@@ -128,7 +125,7 @@ class S3Store(Store):
         src : SourcesOrListOfSources
             Source(s).
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
 
         Returns
         -------
@@ -414,7 +411,7 @@ class S3Store(Store):
         src : str
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -457,7 +454,7 @@ class S3Store(Store):
         src : list
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -497,7 +494,7 @@ class S3Store(Store):
         src : str
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -546,7 +543,13 @@ class S3Store(Store):
         mime_type = get_file_mime_type(src)
         if mime_type is not None:
             extra_args["ContentType"] = mime_type
-        client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=extra_args)
+        client.upload_file(
+            Filename=src,
+            Bucket=bucket,
+            Key=key,
+            ExtraArgs=extra_args,
+            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+        )
 
     @staticmethod
     def _upload_fileobject(
@@ -573,7 +576,12 @@ class S3Store(Store):
         -------
         None
         """
-        client.put_object(Bucket=bucket, Key=key, Body=fileobj.getvalue())
+        client.upload_fileobj(
+            Fileobj=fileobj,
+            Bucket=bucket,
+            Key=key,
+            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+        )
 
     ##############################
     # Helper methods
@@ -619,13 +627,13 @@ class S3Store(Store):
 
         # Try to get client from environment variables
         try:
-            cfg = self._configurator.get_boto3_client_config(CredsOrigin.ENV.value)
+            cfg = self._configurator.get_client_config(CredsOrigin.ENV.value)
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
 
         # Fallback to file
         except StoreError:
-            cfg = self._configurator.get_boto3_client_config(CredsOrigin.FILE.value)
+            cfg = self._configurator.get_client_config(CredsOrigin.FILE.value)
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
 
@@ -676,29 +684,6 @@ class S3Store(Store):
             key = key[1:]
         return key
 
-    def _build_key_from_root(self, root: str, paths: list[str]) -> list[str]:
-        """
-        Method to build object path.
-
-        Parameters
-        ----------
-        root : str
-            The root of the object path.
-        paths : list[str]
-            The path to build.
-
-        Returns
-        -------
-        list[str]
-            List of keys.
-        """
-        keys = []
-        for path in paths:
-            clean_path = self._get_key(path)
-            key = self._get_key(f"{root}{clean_path}")
-            keys.append(key)
-        return keys
-
     def _list_objects(self, client: S3Client, bucket: str, partition: str) -> list[str]:
         """
         List objects in a S3 partition.

digitalhub/stores/data/s3/utils.py

@@ -9,7 +9,7 @@ from urllib.parse import urlparse
 
 from boto3 import client as boto3_client
 
-from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.utils.exceptions import StoreError
 

digitalhub/stores/data/sql/configurator.py

@@ -4,30 +4,48 @@
 
 from __future__ import annotations
 
-from digitalhub.stores.configurator.configurator import configurator
-from digitalhub.stores.configurator.enums import CredsOrigin
-from digitalhub.stores.data.sql.enums import SqlStoreEnv
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.stores.credentials.configurator import Configurator
+from digitalhub.stores.credentials.enums import CredsEnvVar
 
 
-class SqlStoreConfigurator:
+class SqlStoreConfigurator(Configurator):
     """
     Configure the store by getting the credentials from user
     provided config or from environment.
     """
 
-    required_vars = [
-        SqlStoreEnv.USERNAME,
-        SqlStoreEnv.PASSWORD,
-        SqlStoreEnv.HOST,
-        SqlStoreEnv.PORT,
-        SqlStoreEnv.DATABASE,
+    keys = [
+        CredsEnvVar.DB_USERNAME,
+        CredsEnvVar.DB_PASSWORD,
+        CredsEnvVar.DB_HOST,
+        CredsEnvVar.DB_PORT,
+        CredsEnvVar.DB_DATABASE,
     ]
+    required_keys = [
+        CredsEnvVar.DB_USERNAME,
+        CredsEnvVar.DB_PASSWORD,
+        CredsEnvVar.DB_HOST,
+        CredsEnvVar.DB_PORT,
+        CredsEnvVar.DB_DATABASE,
+    ]
+
+    def __init__(self):
+        super().__init__()
+        self.load_configs()
 
     ##############################
     # Configuration methods
     ##############################
 
+    def load_configs(self) -> None:
+        # Load from env
+        env_creds = {var.value: self._creds_handler.load_from_env(var.value) for var in self.keys}
+        self._creds_handler.set_credentials(self._env, env_creds)
+
+        # Load from file
+        file_creds = {var.value: self._creds_handler.load_from_file(var.value) for var in self.keys}
+        self._creds_handler.set_credentials(self._file, file_creds)
+
     def get_sql_conn_string(self, origin: str) -> str:
         """
         Get the connection string from environment variables.
@@ -42,76 +60,10 @@ class SqlStoreConfigurator:
         str
             The connection string.
         """
-        if origin == CredsOrigin.ENV.value:
-            creds = self._get_env_config()
-        elif origin == CredsOrigin.FILE.value:
-            creds = self._get_file_config()
-        else:
-            raise StoreError(f"Unknown origin: {origin}")
-
-        user = creds[SqlStoreEnv.USERNAME.value]
-        password = creds[SqlStoreEnv.PASSWORD.value]
-        host = creds[SqlStoreEnv.HOST.value]
-        port = creds[SqlStoreEnv.PORT.value]
-        database = creds[SqlStoreEnv.DATABASE.value]
+        creds = self.get_credentials(origin)
+        user = creds[CredsEnvVar.DB_USERNAME.value]
+        password = creds[CredsEnvVar.DB_PASSWORD.value]
+        host = creds[CredsEnvVar.DB_HOST.value]
+        port = creds[CredsEnvVar.DB_PORT.value]
+        database = creds[CredsEnvVar.DB_DATABASE.value]
         return f"postgresql://{user}:{password}@{host}:{port}/{database}"
-
-    def _get_env_config(self) -> dict:
-        """
-        Get the store configuration from environment variables.
-
-        Returns
-        -------
-        dict
-            The credentials.
-        """
-        credentials = {var.value: configurator.load_from_env(var.value) for var in self.required_vars}
-        self._set_credentials(credentials)
-        return credentials
-
-    def _get_file_config(self) -> dict:
-        """
-        Get the store configuration from file.
-
-        Returns
-        -------
-        dict
-            The credentials.
-        """
-        credentials = {var.value: configurator.load_from_file(var.value) for var in self.required_vars}
-        self._set_credentials(credentials)
-        return credentials
-
-    def _check_credentials(self, credentials: dict) -> None:
-        """
-        Check for missing credentials.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
-
-        Returns
-        -------
-        None
-        """
-        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
-        if missing_vars:
-            raise StoreError(f"Missing credentials for SQL store: {', '.join(missing_vars)}")
-
-    def _set_credentials(self, credentials: dict) -> None:
-        """
-        Set the store credentials into the configurator.
-
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
-
-        Returns
-        -------
-        None
-        """
-        # Set credentials
-        for key, value in credentials.items():
-            configurator.set_credential(key, value)

digitalhub/stores/data/sql/store.py

@@ -14,9 +14,8 @@ from sqlalchemy import MetaData, Table, create_engine, select
 from sqlalchemy.engine import Engine
 from sqlalchemy.exc import SQLAlchemyError
 
-from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
-from digitalhub.stores.data.sql.configurator import SqlStoreConfigurator
 from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.types import SourcesOrListOfSources
@@ -24,6 +23,9 @@ from digitalhub.utils.types import SourcesOrListOfSources
 if typing.TYPE_CHECKING:
     from sqlalchemy.engine.row import Row
 
+    from digitalhub.stores.credentials.configurator import Configurator
+    from digitalhub.stores.data.sql.configurator import SqlStoreConfigurator
+
 
 class SqlStore(Store):
     """
@@ -31,8 +33,9 @@ class SqlStore(Store):
     artifacts on SQL based storage.
     """
 
-    def __init__(self) -> None:
-        self._configurator = SqlStoreConfigurator()
+    def __init__(self, configurator: Configurator | None = None) -> None:
+        super().__init__(configurator)
+        self._configurator: SqlStoreConfigurator
 
     ##############################
     # I/O methods
@@ -40,9 +43,8 @@ class SqlStore(Store):
 
     def download(
         self,
-        root: str,
+        src: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -50,21 +52,19 @@ class SqlStore(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
-        table_name = self._get_table_name(root) + ".parquet"
+        table_name = self._get_table_name(src) + ".parquet"
         # Case where dst is not provided
         if dst is None:
             dst = Path(self._build_temp("sql")) / table_name
@@ -83,8 +83,8 @@ class SqlStore(Store):
             self._check_overwrite(dst, overwrite)
             self._build_path(dst)
 
-        schema = self._get_schema(root)
-        table = self._get_table_name(root)
+        schema = self._get_schema(src)
+        table = self._get_table_name(src)
         return self._download_table(schema, table, str(dst))
 
     def upload(

{digitalhub-0.12.0.dist-info → digitalhub-0.13.0b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalhub
-Version: 0.12.0
+Version: 0.13.0b1
 Summary: Python SDK for Digitalhub
 Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
 Author-email: Fondazione Bruno Kessler <digitalhub@fbk.eu>, Matteo Martini <mmartini@fbk.eu>