digitalhub 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

digitalhub/stores/data/s3/store.py

@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import typing
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Type
@@ -11,17 +12,21 @@ from urllib.parse import urlparse
 
 import boto3
 import botocore.client  # pylint: disable=unused-import
+from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from digitalhub.stores.configurator.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
-from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.stores.data.s3.utils import get_bucket_name
 from digitalhub.stores.readers.data.api import get_reader_by_object
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.exceptions import ConfigError, StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
+if typing.TYPE_CHECKING:
+    from digitalhub.stores.credentials.configurator import Configurator
+    from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
+
+
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
@@ -32,8 +37,9 @@ class S3Store(Store):
     artifacts on S3 based storage.
     """
 
-    def __init__(self) -> None:
-        self._configurator = S3StoreConfigurator()
+    def __init__(self, configurator: Configurator | None = None) -> None:
+        super().__init__(configurator)
+        self._configurator: S3StoreConfigurator
 
     ##############################
     # I/O methods
@@ -41,9 +47,8 @@ class S3Store(Store):
 
     def download(
         self,
-        root: str,
+        src: str,
         dst: Path,
-        src: list[str],
         overwrite: bool = False,
     ) -> str:
         """
@@ -51,21 +56,19 @@ class S3Store(Store):
 
         Parameters
         ----------
-        root : str
-            The root path of the artifact.
+        src : str
+            Path of the material entity.
         dst : str
-            The destination of the artifact on local filesystem.
-        src : list[str]
-            List of sources.
+            The destination of the material entity on local filesystem.
         overwrite : bool
             Specify if overwrite existing file(s).
 
         Returns
         -------
         str
-            Destination path of the downloaded artifact.
+            Destination path of the downloaded files.
         """
-        client, bucket = self._check_factory(root)
+        client, bucket = self._check_factory(src)
 
         # Build destination directory
         if dst.suffix == "":
@@ -74,20 +77,13 @@ class S3Store(Store):
             dst.parent.mkdir(parents=True, exist_ok=True)
 
         # Handle src and tree destination
-        if self.is_partition(root):
-            if not src:
-                keys = self._list_objects(client, bucket, root)
-                strip_root = self._get_key(root)
-                trees = [k.removeprefix(strip_root) for k in keys]
-            else:
-                keys = self._build_key_from_root(root, src)
-                trees = [s for s in src]
+        if self.is_partition(src):
+            keys = self._list_objects(client, bucket, src)
+            strip_root = self._get_key(src)
+            trees = [k.removeprefix(strip_root) for k in keys]
         else:
-            keys = [self._get_key(root)]
-            if not src:
-                trees = [Path(self._get_key(root)).name]
-            else:
-                trees = [s for s in src]
+            keys = [self._get_key(src)]
+            trees = [Path(self._get_key(src)).name]
 
         if len(keys) != len(trees):
             raise StoreError("Keys and trees must have the same length.")
@@ -128,7 +124,7 @@ class S3Store(Store):
         src : SourcesOrListOfSources
             Source(s).
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
 
         Returns
         -------
@@ -414,7 +410,7 @@ class S3Store(Store):
         src : str
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -457,7 +453,7 @@ class S3Store(Store):
         src : list
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -497,7 +493,7 @@ class S3Store(Store):
         src : str
             List of sources.
         dst : str
-            The destination of the artifact on storage.
+            The destination of the material entity on storage.
         client : S3Client
             The S3 client object.
         bucket : str
@@ -546,7 +542,13 @@ class S3Store(Store):
         mime_type = get_file_mime_type(src)
         if mime_type is not None:
             extra_args["ContentType"] = mime_type
-        client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=extra_args)
+        client.upload_file(
+            Filename=src,
+            Bucket=bucket,
+            Key=key,
+            ExtraArgs=extra_args,
+            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+        )
 
     @staticmethod
     def _upload_fileobject(
@@ -573,7 +575,12 @@ class S3Store(Store):
         -------
         None
         """
-        client.put_object(Bucket=bucket, Key=key, Body=fileobj.getvalue())
+        client.upload_fileobj(
+            Fileobj=fileobj,
+            Bucket=bucket,
+            Key=key,
+            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+        )
 
     ##############################
     # Helper methods
@@ -606,55 +613,59 @@ class S3Store(Store):
         """
         return boto3.client("s3", **cfg)
 
-    def _check_factory(self, root: str) -> tuple[S3Client, str]:
+    def _check_factory(self, s3_path: str, retry: bool = True) -> tuple[S3Client, str]:
         """
-        Check if the S3 bucket is accessible by sending a head_bucket request.
+        Checks if the S3 bucket collected from the URI is accessible.
+
+        Parameters
+        ----------
+        s3_path : str
+            Path to the S3 bucket (e.g., 's3://bucket/path').
+        retry : bool, optional
+            Whether to retry the operation if a ConfigError is raised. Default is True.
 
         Returns
         -------
-        tuple[S3Client, str]
-            A tuple containing the S3 client object and the name of the S3 bucket.
-        """
-        bucket = self._get_bucket(root)
+        tuple of S3Client and str
+            Tuple containing the S3 client object and the name of the S3 bucket.
 
-        # Try to get client from environment variables
+        Raises
+        ------
+        ConfigError
+            If access to the specified bucket is not available and retry is False.
+        """
+        bucket = self._get_bucket(s3_path)
         try:
-            cfg = self._configurator.get_boto3_client_config(CredsOrigin.ENV.value)
-            client = self._get_client(cfg)
-            self._check_access_to_storage(client, bucket)
-
-        # Fallback to file
-        except StoreError:
-            cfg = self._configurator.get_boto3_client_config(CredsOrigin.FILE.value)
+            cfg = self._configurator.get_client_config()
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
-
-        return client, bucket
+            return client, bucket
+        except ConfigError as e:
+            if retry:
+                self._configurator.eval_change_origin()
+                return self._check_factory(s3_path, False)
+            raise e
 
     def _check_access_to_storage(self, client: S3Client, bucket: str) -> None:
         """
-        Check if the S3 bucket is accessible by sending a head_bucket request.
+        Checks if the S3 bucket is accessible by sending a head_bucket request.
 
         Parameters
         ----------
         client : S3Client
-            The S3 client object.
+            S3 client object.
         bucket : str
-            The name of the S3 bucket.
-
-        Returns
-        -------
-        None
+            Name of the S3 bucket.
 
         Raises
         ------
-        ClientError:
+        ConfigError
             If access to the specified bucket is not available.
         """
         try:
             client.head_bucket(Bucket=bucket)
         except (ClientError, NoCredentialsError) as err:
-            raise StoreError(f"No access to s3 bucket! Error: {err}")
+            raise ConfigError(f"No access to s3 bucket! Error: {err}")
 
     @staticmethod
     def _get_key(path: str) -> str:
@@ -676,29 +687,6 @@ class S3Store(Store):
             key = key[1:]
         return key
 
-    def _build_key_from_root(self, root: str, paths: list[str]) -> list[str]:
-        """
-        Method to build object path.
-
-        Parameters
-        ----------
-        root : str
-            The root of the object path.
-        paths : list[str]
-            The path to build.
-
-        Returns
-        -------
-        list[str]
-            List of keys.
-        """
-        keys = []
-        for path in paths:
-            clean_path = self._get_key(path)
-            key = self._get_key(f"{root}{clean_path}")
-            keys.append(key)
-        return keys
-
     def _list_objects(self, client: S3Client, bucket: str, partition: str) -> list[str]:
         """
         List objects in a S3 partition.

digitalhub/stores/data/s3/utils.py

@@ -9,41 +9,41 @@ from urllib.parse import urlparse
 
 from boto3 import client as boto3_client
 
-from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.utils.exceptions import StoreError
 
 
 def get_bucket_name(path: str) -> str:
     """
-    Get bucket name from path.
+    Extract the bucket name from an S3 path.
 
     Parameters
     ----------
     path : str
-        The source path to get the key from.
+        S3 URI (e.g., 's3://bucket/key').
 
     Returns
     -------
     str
-        The bucket name.
+        The bucket name extracted from the URI.
     """
     return urlparse(path).netloc
 
 
 def get_bucket_and_key(path: str) -> tuple[str, str]:
     """
-    Get bucket and key from path.
+    Extract the bucket name and key from an S3 path.
 
     Parameters
     ----------
     path : str
-        The source path to get the key from.
+        S3 URI (e.g., 's3://bucket/key').
 
     Returns
     -------
-    tuple[str, str]
-        The bucket and key.
+    tuple of str
+        Tuple containing (bucket, key) extracted from the URI.
     """
     parsed = urlparse(path)
     return parsed.netloc, parsed.path
@@ -51,7 +51,7 @@ def get_bucket_and_key(path: str) -> tuple[str, str]:
 
 def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     """
-    Get S3 source.
+    Download an object from S3 and save it to a local file.
 
     Parameters
     ----------
@@ -60,7 +60,7 @@ def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     key : str
         S3 object key.
     filename : Path
-        Path where to save the function source.
+        Local path where the downloaded object will be saved.
 
     Returns
     -------

digitalhub/stores/data/sql/configurator.py

@@ -4,114 +4,117 @@
 
 from __future__ import annotations
 
-from digitalhub.stores.configurator.configurator import configurator
-from digitalhub.stores.configurator.enums import CredsOrigin
-from digitalhub.stores.data.sql.enums import SqlStoreEnv
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.stores.credentials.configurator import Configurator
+from digitalhub.stores.credentials.enums import CredsEnvVar
 
 
-class SqlStoreConfigurator:
+class SqlStoreConfigurator(Configurator):
     """
-    Configure the store by getting the credentials from user
-    provided config or from environment.
+    SQL store configuration manager for database connections.
+
+    Handles credential management and configuration for SQL database
+    connections. Loads credentials from environment variables or
+    configuration files and provides connection string generation
+    for database access.
+
+    Attributes
+    ----------
+    keys : list[str]
+        List of all supported credential keys for SQL connections.
+    required_keys : list[str]
+        List of mandatory credential keys that must be provided.
     """
 
-    required_vars = [
-        SqlStoreEnv.USERNAME,
-        SqlStoreEnv.PASSWORD,
-        SqlStoreEnv.HOST,
-        SqlStoreEnv.PORT,
-        SqlStoreEnv.DATABASE,
+    keys = [
+        CredsEnvVar.DB_USERNAME.value,
+        CredsEnvVar.DB_PASSWORD.value,
+        CredsEnvVar.DB_HOST.value,
+        CredsEnvVar.DB_PORT.value,
+        CredsEnvVar.DB_DATABASE.value,
+        CredsEnvVar.DB_PLATFORM.value,
     ]
+    required_keys = [
+        CredsEnvVar.DB_USERNAME.value,
+        CredsEnvVar.DB_PASSWORD.value,
+        CredsEnvVar.DB_HOST.value,
+        CredsEnvVar.DB_PORT.value,
+        CredsEnvVar.DB_DATABASE.value,
+    ]
+
+    def __init__(self):
+        super().__init__()
+        self.load_configs()
 
     ##############################
     # Configuration methods
     ##############################
 
-    def get_sql_conn_string(self, origin: str) -> str:
+    def load_env_vars(self) -> None:
         """
-        Get the connection string from environment variables.
+        Load database credentials from environment variables.
 
-        Parameters
-        ----------
-        origin : str
-            The origin of the credentials.
+        Retrieves SQL database connection credentials from the system
+        environment variables and stores them in the configurator's
+        credential handler for use in database connections.
 
         Returns
         -------
-        str
-            The connection string.
-        """
-        if origin == CredsOrigin.ENV.value:
-            creds = self._get_env_config()
-        elif origin == CredsOrigin.FILE.value:
-            creds = self._get_file_config()
-        else:
-            raise StoreError(f"Unknown origin: {origin}")
-
-        user = creds[SqlStoreEnv.USERNAME.value]
-        password = creds[SqlStoreEnv.PASSWORD.value]
-        host = creds[SqlStoreEnv.HOST.value]
-        port = creds[SqlStoreEnv.PORT.value]
-        database = creds[SqlStoreEnv.DATABASE.value]
-        return f"postgresql://{user}:{password}@{host}:{port}/{database}"
-
-    def _get_env_config(self) -> dict:
+        None
         """
-        Get the store configuration from environment variables.
+        env_creds = self._creds_handler.load_from_env(self.keys)
+        self._creds_handler.set_credentials(self._env, env_creds)
 
-        Returns
-        -------
-        dict
-            The credentials.
+    def load_file_vars(self) -> None:
         """
-        credentials = {var.value: configurator.load_from_env(var.value) for var in self.required_vars}
-        self._set_credentials(credentials)
-        return credentials
+        Load database credentials from configuration file.
 
-    def _get_file_config(self) -> dict:
-        """
-        Get the store configuration from file.
+        Retrieves SQL database connection credentials from a
+        configuration file and stores them in the configurator's
+        credential handler for use in database connections.
 
         Returns
         -------
-        dict
-            The credentials.
+        None
         """
-        credentials = {var.value: configurator.load_from_file(var.value) for var in self.required_vars}
-        self._set_credentials(credentials)
-        return credentials
+        file_creds = self._creds_handler.load_from_file(self.keys)
+        self._creds_handler.set_credentials(self._file, file_creds)
 
-    def _check_credentials(self, credentials: dict) -> None:
+    def get_sql_conn_string(self) -> str:
         """
-        Check for missing credentials.
+        Generate PostgreSQL connection string from stored credentials.
 
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
+        Constructs a PostgreSQL connection string using the configured
+        database credentials including username, password, host, port,
+        and database name.
 
         Returns
         -------
-        None
+        str
+            A PostgreSQL connection string in the format:
+            'postgresql://username:password@host:port/database'
         """
-        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
-        if missing_vars:
-            raise StoreError(f"Missing credentials for SQL store: {', '.join(missing_vars)}")
+        creds = self.get_sql_credentials()
+        user = creds[CredsEnvVar.DB_USERNAME.value]
+        password = creds[CredsEnvVar.DB_PASSWORD.value]
+        host = creds[CredsEnvVar.DB_HOST.value]
+        port = creds[CredsEnvVar.DB_PORT.value]
+        database = creds[CredsEnvVar.DB_DATABASE.value]
+        return f"postgresql://{user}:{password}@{host}:{port}/{database}"
 
-    def _set_credentials(self, credentials: dict) -> None:
+    def get_sql_credentials(self) -> dict:
         """
-        Set the store credentials into the configurator.
+        Get all configured database credentials as a dictionary.
 
-        Parameters
-        ----------
-        credentials : dict
-            The credentials.
+        Retrieves all available database credentials from the configured
+        source and returns them as a dictionary with all credential keys
+        from self.keys mapped to their values.
 
         Returns
         -------
-        None
+        dict
+            Dictionary containing all credential key-value pairs from self.keys.
+            Keys correspond to database connection parameters such as
+            username, password, host, port, database, and platform.
         """
-        # Set credentials
-        for key, value in credentials.items():
-            configurator.set_credential(key, value)
+        creds = self.get_credentials(self._origin)
+        return {key: creds.get(key) for key in self.keys}