digitalhub 0.13.0b3__py3-none-any.whl → 0.14.9__py3-none-any.whl

digitalhub/stores/data/s3/configurator.py

@@ -4,112 +4,74 @@
 
 from __future__ import annotations
 
-from datetime import datetime, timezone
-
 from botocore.config import Config
 
-from digitalhub.stores.client.dhcore.utils import refresh_token
-from digitalhub.stores.credentials.configurator import Configurator
-from digitalhub.stores.credentials.enums import CredsEnvVar
+from digitalhub.stores.configurator.configurator import configurator
+from digitalhub.stores.configurator.enums import ConfigurationVars, CredentialsVars
 
 
-class S3StoreConfigurator(Configurator):
+class S3StoreConfigurator:
     """
-    Configure the store by getting the credentials from user
-    provided config or from environment.
+    Configurator class for S3 store configuration and credentials management.
     """
 
-    keys = [
-        CredsEnvVar.S3_ENDPOINT_URL.value,
-        CredsEnvVar.S3_ACCESS_KEY_ID.value,
-        CredsEnvVar.S3_SECRET_ACCESS_KEY.value,
-        CredsEnvVar.S3_REGION.value,
-        CredsEnvVar.S3_SIGNATURE_VERSION.value,
-        CredsEnvVar.S3_SESSION_TOKEN.value,
-        CredsEnvVar.S3_PATH_STYLE.value,
-        CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value,
-    ]
-    required_keys = [
-        CredsEnvVar.S3_ENDPOINT_URL.value,
-        CredsEnvVar.S3_ACCESS_KEY_ID.value,
-        CredsEnvVar.S3_SECRET_ACCESS_KEY.value,
-    ]
-
     def __init__(self):
-        super().__init__()
-        self.load_configs()
+        self._validate()
 
     ##############################
     # Configuration methods
     ##############################
 
-    def load_env_vars(self) -> None:
-        """
-        Loads the credentials from the environment variables.
-
-        Returns
-        -------
-        None
-        """
-        env_creds = self._creds_handler.load_from_env(self.keys)
-        self._creds_handler.set_credentials(self._env, env_creds)
-
-    def load_file_vars(self) -> None:
-        """
-        Loads the credentials from a file.
-
-        Returns
-        -------
-        None
-        """
-        file_creds = self._creds_handler.load_from_file(self.keys)
-        self._creds_handler.set_credentials(self._file, file_creds)
-
     def get_client_config(self) -> dict:
         """
         Gets S3 credentials (access key, secret key, session token, and other config).
 
+        Parameters
+        ----------
+        creds : dict
+            The credentials dictionary.
+
         Returns
         -------
         dict
-            Dictionary containing S3 credentials and configuration.
+            A dictionary containing the S3 credentials.
         """
-        creds = self.get_credentials(self._origin)
-        expired = creds[CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value]
-        if self._origin == self._file and self._is_expired(expired):
-            refresh_token()
-            self.load_file_vars()
-            creds = self.get_credentials(self._origin)
+        creds = configurator.get_config_creds()
         return {
-            "endpoint_url": creds[CredsEnvVar.S3_ENDPOINT_URL.value],
-            "aws_access_key_id": creds[CredsEnvVar.S3_ACCESS_KEY_ID.value],
-            "aws_secret_access_key": creds[CredsEnvVar.S3_SECRET_ACCESS_KEY.value],
-            "aws_session_token": creds[CredsEnvVar.S3_SESSION_TOKEN.value],
+            "endpoint_url": creds[ConfigurationVars.S3_ENDPOINT_URL.value],
+            "aws_access_key_id": creds[CredentialsVars.S3_ACCESS_KEY_ID.value],
+            "aws_secret_access_key": creds[CredentialsVars.S3_SECRET_ACCESS_KEY.value],
+            "aws_session_token": creds[CredentialsVars.S3_SESSION_TOKEN.value],
             "config": Config(
-                region_name=creds[CredsEnvVar.S3_REGION.value],
-                signature_version=creds[CredsEnvVar.S3_SIGNATURE_VERSION.value],
+                region_name=creds[ConfigurationVars.S3_REGION.value],
+                signature_version=creds[ConfigurationVars.S3_SIGNATURE_VERSION.value],
             ),
         }
 
-    @staticmethod
-    def _is_expired(timestamp: str | None) -> bool:
+    def _validate(self) -> None:
         """
-        Determines whether a given timestamp is after the current UTC time.
-
-        Parameters
-        ----------
-        timestamp : str or None
-            Timestamp string in the format 'YYYY-MM-DDTHH:MM:SSZ'.
+        Validate if all required keys are present in the configuration.
+        """
+        required_keys = [
+            ConfigurationVars.S3_ENDPOINT_URL.value,
+            CredentialsVars.S3_ACCESS_KEY_ID.value,
+            CredentialsVars.S3_SECRET_ACCESS_KEY.value,
+        ]
+        current_keys = configurator.get_config_creds()
+        missing_keys = []
+        for key in required_keys:
+            if key not in current_keys or current_keys[key] is None:
+                missing_keys.append(key)
+        if missing_keys:
+            raise ValueError(f"Missing required variables for S3 store: {', '.join(missing_keys)}")
+
+    def eval_retry(self) -> bool:
+        """
+        Evaluate the status of retry lifecycle.
 
         Returns
         -------
         bool
-            True if the given timestamp is later than the current UTC time,
-            otherwise False.
+            True if a retry action was performed, otherwise False.
         """
-        if timestamp is None:
-            return False
-        dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ")
-        dt = dt.replace(tzinfo=timezone.utc)
-        now = datetime.now(timezone.utc) + datetime.timedelta(seconds=120)
-        return dt < now
+        return configurator.eval_retry()

digitalhub/stores/data/s3/store.py

@@ -4,7 +4,6 @@
 
 from __future__ import annotations
 
-import typing
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Type
@@ -16,20 +15,17 @@ from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
 from digitalhub.stores.data._base.store import Store
-from digitalhub.stores.data.s3.utils import get_bucket_name
+from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
 from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import ConfigError, StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
-if typing.TYPE_CHECKING:
-    from digitalhub.stores.credentials.configurator import Configurator
-    from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
-
-
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
+MULTIPART_THRESHOLD = 100 * 1024 * 1024
+
 
 class S3Store(Store):
     """
@@ -37,9 +33,9 @@ class S3Store(Store):
     artifacts on S3 based storage.
     """
 
-    def __init__(self, configurator: Configurator | None = None) -> None:
-        super().__init__(configurator)
-        self._configurator: S3StoreConfigurator
+    def __init__(self) -> None:
+        super().__init__()
+        self._configurator: S3StoreConfigurator = S3StoreConfigurator()
 
     ##############################
     # I/O methods
@@ -332,10 +328,50 @@ class S3Store(Store):
         str
             The S3 path where the dataframe was saved.
         """
-        fileobj = BytesIO()
         reader = get_reader_by_object(df)
-        reader.write_df(df, fileobj, extension=extension, **kwargs)
-        return self.upload_fileobject(fileobj, dst)
+        with BytesIO() as fileobj:
+            reader.write_df(df, fileobj, extension=extension, **kwargs)
+            fileobj.seek(0)
+            return self.upload_fileobject(fileobj, dst)
+
+    ##############################
+    # Wrapper methods
+    ##############################
+
+    def get_s3_source(self, src: str, filename: Path) -> None:
+        """
+        Download a file from S3 and save it to a local file.
+
+        Parameters
+        ----------
+        src : str
+            S3 path of the object to be downloaded (e.g., 's3://bucket
+        filename : Path
+            Local path where the downloaded object will be saved.
+        """
+        client, bucket = self._check_factory(src)
+        key = self._get_key(src)
+        self._download_file(key, filename, client, bucket)
+
+    def get_s3_client(self, file: bool = True) -> S3Client:
+        """
+        Get an S3 client object.
+
+        Parameters
+        ----------
+        file : bool
+            Whether to use file-based credentials. Default is True.
+
+        Returns
+        -------
+        S3Client
+            Returns a client object that interacts with the S3 storage service.
+        """
+        if file:
+            cfg = self._configurator.get_file_config()
+        else:
+            cfg = self._configurator.get_env_config()
+        return self._get_client(cfg)
 
     ##############################
     # Private I/O methods
@@ -533,10 +569,6 @@ class S3Store(Store):
             The S3 client object.
         bucket : str
             The name of the S3 bucket.
-
-        Returns
-        -------
-        None
         """
         extra_args = {}
         mime_type = get_file_mime_type(src)
@@ -547,7 +579,7 @@ class S3Store(Store):
             Bucket=bucket,
             Key=key,
             ExtraArgs=extra_args,
-            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+            Config=TransferConfig(multipart_threshold=MULTIPART_THRESHOLD),
         )
 
     @staticmethod
@@ -570,16 +602,12 @@ class S3Store(Store):
             The S3 client object.
         bucket : str
             The name of the S3 bucket.
-
-        Returns
-        -------
-        None
         """
         client.upload_fileobj(
             Fileobj=fileobj,
             Bucket=bucket,
             Key=key,
-            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+            Config=TransferConfig(multipart_threshold=MULTIPART_THRESHOLD),
         )
 
     ##############################
@@ -595,7 +623,7 @@ class S3Store(Store):
         str
             The name of the S3 bucket.
         """
-        return get_bucket_name(root)
+        return urlparse(root).netloc
 
     def _get_client(self, cfg: dict) -> S3Client:
         """
@@ -613,7 +641,7 @@ class S3Store(Store):
         """
         return boto3.client("s3", **cfg)
 
-    def _check_factory(self, s3_path: str, retry: bool = True) -> tuple[S3Client, str]:
+    def _check_factory(self, s3_path: str) -> tuple[S3Client, str]:
         """
         Checks if the S3 bucket collected from the URI is accessible.
 
@@ -621,29 +649,21 @@ class S3Store(Store):
         ----------
         s3_path : str
             Path to the S3 bucket (e.g., 's3://bucket/path').
-        retry : bool, optional
-            Whether to retry the operation if a ConfigError is raised. Default is True.
 
         Returns
         -------
         tuple of S3Client and str
             Tuple containing the S3 client object and the name of the S3 bucket.
-
-        Raises
-        ------
-        ConfigError
-            If access to the specified bucket is not available and retry is False.
         """
         bucket = self._get_bucket(s3_path)
+        cfg = self._configurator.get_client_config()
+        client = self._get_client(cfg)
         try:
-            cfg = self._configurator.get_client_config()
-            client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
             return client, bucket
         except ConfigError as e:
-            if retry:
-                self._configurator.eval_change_origin()
-                return self._check_factory(s3_path, False)
+            if self._configurator.eval_retry():
+                return self._check_factory(s3_path)
             raise e
 
     def _check_access_to_storage(self, client: S3Client, bucket: str) -> None:

digitalhub/stores/data/sql/configurator.py

@@ -4,67 +4,87 @@
 
 from __future__ import annotations
 
-from digitalhub.stores.credentials.configurator import Configurator
-from digitalhub.stores.credentials.enums import CredsEnvVar
+from digitalhub.stores.configurator.configurator import configurator
+from digitalhub.stores.configurator.enums import ConfigurationVars, CredentialsVars
 
 
-class SqlStoreConfigurator(Configurator):
+class SqlStoreConfigurator:
     """
-    Configure the store by getting the credentials from user
-    provided config or from environment.
+    Configurator class for SQL store configuration and credentials management.
     """
 
-    keys = [
-        CredsEnvVar.DB_USERNAME.value,
-        CredsEnvVar.DB_PASSWORD.value,
-        CredsEnvVar.DB_HOST.value,
-        CredsEnvVar.DB_PORT.value,
-        CredsEnvVar.DB_DATABASE.value,
-        CredsEnvVar.DB_PLATFORM.value,
-    ]
-    required_keys = [
-        CredsEnvVar.DB_USERNAME.value,
-        CredsEnvVar.DB_PASSWORD.value,
-        CredsEnvVar.DB_HOST.value,
-        CredsEnvVar.DB_PORT.value,
-        CredsEnvVar.DB_DATABASE.value,
-    ]
-
     def __init__(self):
-        super().__init__()
-        self.load_configs()
-
-    ##############################
-    # Configuration methods
-    ##############################
+        self._validate()
 
-    def load_env_vars(self) -> None:
+    def get_sql_conn_string(self) -> str:
         """
-        Load the credentials from the environment.
+        Generate PostgreSQL connection string from stored credentials.
+
+        Returns
+        -------
+        str
+            A PostgreSQL connection string in the format:
+            'postgresql://username:password@host:port/database'
         """
-        env_creds = self._creds_handler.load_from_env(self.keys)
-        self._creds_handler.set_credentials(self._env, env_creds)
+        creds = self.get_sql_credentials()
+        user = creds[CredentialsVars.DB_USERNAME.value]
+        password = creds[CredentialsVars.DB_PASSWORD.value]
+        host = creds[ConfigurationVars.DB_HOST.value]
+        port = creds[ConfigurationVars.DB_PORT.value]
+        database = creds[ConfigurationVars.DB_DATABASE.value]
+        return f"postgresql://{user}:{password}@{host}:{port}/{database}"
 
-    def load_file_vars(self) -> None:
+    def get_sql_credentials(self) -> dict:
         """
-        Load the credentials from the file.
+        Get all configured database credentials as a dictionary.
+
+        Retrieves all available database credentials from the configured
+        source and returns them as a dictionary with all credential keys
+        from self.keys mapped to their values.
+
+        Returns
+        -------
+        dict
+            Dictionary containing all credential key-value pairs from self.keys.
+            Keys correspond to database connection parameters such as
+            username, password, host, port, database, and platform.
         """
-        file_creds = self._creds_handler.load_from_file(self.keys)
-        self._creds_handler.set_credentials(self._file, file_creds)
+        keys = [
+            CredentialsVars.DB_USERNAME.value,
+            CredentialsVars.DB_PASSWORD.value,
+            ConfigurationVars.DB_HOST.value,
+            ConfigurationVars.DB_PORT.value,
+            ConfigurationVars.DB_DATABASE.value,
+        ]
+        creds = configurator.get_config_creds()
+        return {key: creds.get(key) for key in keys}
 
-    def get_sql_conn_string(self) -> str:
+    def eval_retry(self) -> bool:
         """
-        Get the connection string from environment variables.
+        Evaluate the status of retry lifecycle.
 
         Returns
         -------
-        str
-            The connection string.
+        bool
+            True if a retry should be attempted, False otherwise.
         """
-        creds = self.get_credentials(self._origin)
-        user = creds[CredsEnvVar.DB_USERNAME.value]
-        password = creds[CredsEnvVar.DB_PASSWORD.value]
-        host = creds[CredsEnvVar.DB_HOST.value]
-        port = creds[CredsEnvVar.DB_PORT.value]
-        database = creds[CredsEnvVar.DB_DATABASE.value]
-        return f"postgresql://{user}:{password}@{host}:{port}/{database}"
+        return configurator.eval_retry()
+
+    def _validate(self) -> None:
+        """
+        Validate if all required keys are present in the configuration.
+        """
+        required_keys = [
+            ConfigurationVars.DB_HOST.value,
+            ConfigurationVars.DB_PORT.value,
+            ConfigurationVars.DB_DATABASE.value,
+            CredentialsVars.DB_USERNAME.value,
+            CredentialsVars.DB_PASSWORD.value,
+        ]
+        current_keys = configurator.get_config_creds()
+        missing_keys = []
+        for key in required_keys:
+            if key not in current_keys or current_keys[key] is None:
+                missing_keys.append(key)
+        if missing_keys:
+            raise ValueError(f"Missing required variables for SQL store: {', '.join(missing_keys)}")