digitalhub 0.13.0b2__py3-none-any.whl → 0.13.0b4__py3-none-any.whl

digitalhub/stores/data/s3/configurator.py

@@ -4,8 +4,11 @@
 
 from __future__ import annotations
 
+from datetime import datetime, timezone
+
 from botocore.config import Config
 
+from digitalhub.stores.client.dhcore.utils import refresh_token
 from digitalhub.stores.credentials.configurator import Configurator
 from digitalhub.stores.credentials.enums import CredsEnvVar
 
@@ -17,17 +20,19 @@ class S3StoreConfigurator(Configurator):
     """
 
     keys = [
-        CredsEnvVar.S3_ENDPOINT_URL,
-        CredsEnvVar.S3_ACCESS_KEY_ID,
-        CredsEnvVar.S3_SECRET_ACCESS_KEY,
-        CredsEnvVar.S3_REGION,
-        CredsEnvVar.S3_SIGNATURE_VERSION,
-        CredsEnvVar.S3_SESSION_TOKEN,
+        CredsEnvVar.S3_ENDPOINT_URL.value,
+        CredsEnvVar.S3_ACCESS_KEY_ID.value,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY.value,
+        CredsEnvVar.S3_REGION.value,
+        CredsEnvVar.S3_SIGNATURE_VERSION.value,
+        CredsEnvVar.S3_SESSION_TOKEN.value,
+        CredsEnvVar.S3_PATH_STYLE.value,
+        CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value,
     ]
     required_keys = [
-        CredsEnvVar.S3_ENDPOINT_URL,
-        CredsEnvVar.S3_ACCESS_KEY_ID,
-        CredsEnvVar.S3_SECRET_ACCESS_KEY,
+        CredsEnvVar.S3_ENDPOINT_URL.value,
+        CredsEnvVar.S3_ACCESS_KEY_ID.value,
+        CredsEnvVar.S3_SECRET_ACCESS_KEY.value,
     ]
 
     def __init__(self):
@@ -38,30 +43,43 @@ class S3StoreConfigurator(Configurator):
     # Configuration methods
     ##############################
 
-    def load_configs(self) -> None:
-        # Load from env
-        env_creds = {var.value: self._creds_handler.load_from_env(var.value) for var in self.keys}
+    def load_env_vars(self) -> None:
+        """
+        Loads the credentials from the environment variables.
+
+        Returns
+        -------
+        None
+        """
+        env_creds = self._creds_handler.load_from_env(self.keys)
         self._creds_handler.set_credentials(self._env, env_creds)
 
-        # Load from file
-        file_creds = {var.value: self._creds_handler.load_from_file(var.value) for var in self.keys}
-        self._creds_handler.set_credentials(self._file, file_creds)
+    def load_file_vars(self) -> None:
+        """
+        Loads the credentials from a file.
 
-    def get_client_config(self, origin: str) -> dict:
+        Returns
+        -------
+        None
         """
-        Get S3 credentials (access key, secret key, session token and other config).
+        file_creds = self._creds_handler.load_from_file(self.keys)
+        self._creds_handler.set_credentials(self._file, file_creds)
 
-        Parameters
-        ----------
-        origin : str
-            The origin of the credentials.
+    def get_client_config(self) -> dict:
+        """
+        Gets S3 credentials (access key, secret key, session token, and other config).
 
         Returns
         -------
         dict
-            The credentials.
+            Dictionary containing S3 credentials and configuration.
         """
-        creds = self.get_credentials(origin)
+        creds = self.get_credentials(self._origin)
+        expired = creds[CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value]
+        if self._origin == self._file and self._is_expired(expired):
+            refresh_token()
+            self.load_file_vars()
+            creds = self.get_credentials(self._origin)
         return {
             "endpoint_url": creds[CredsEnvVar.S3_ENDPOINT_URL.value],
             "aws_access_key_id": creds[CredsEnvVar.S3_ACCESS_KEY_ID.value],
@@ -72,3 +90,26 @@ class S3StoreConfigurator(Configurator):
                 signature_version=creds[CredsEnvVar.S3_SIGNATURE_VERSION.value],
             ),
         }
+
+    @staticmethod
+    def _is_expired(timestamp: str | None) -> bool:
+        """
+        Determines whether a given timestamp is after the current UTC time.
+
+        Parameters
+        ----------
+        timestamp : str or None
+            Timestamp string in the format 'YYYY-MM-DDTHH:MM:SSZ'.
+
+        Returns
+        -------
+        bool
+            True if the given timestamp is later than the current UTC time,
+            otherwise False.
+        """
+        if timestamp is None:
+            return False
+        dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ")
+        dt = dt.replace(tzinfo=timezone.utc)
+        now = datetime.now(timezone.utc) + datetime.timedelta(seconds=120)
+        return dt < now

digitalhub/stores/data/s3/store.py

@@ -15,11 +15,10 @@ import botocore.client  # pylint: disable=unused-import
 from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
 from digitalhub.stores.data.s3.utils import get_bucket_name
 from digitalhub.stores.readers.data.api import get_reader_by_object
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.exceptions import ConfigError, StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
@@ -614,55 +613,59 @@ class S3Store(Store):
         """
         return boto3.client("s3", **cfg)
 
-    def _check_factory(self, root: str) -> tuple[S3Client, str]:
+    def _check_factory(self, s3_path: str, retry: bool = True) -> tuple[S3Client, str]:
         """
-        Check if the S3 bucket is accessible by sending a head_bucket request.
+        Checks if the S3 bucket collected from the URI is accessible.
+
+        Parameters
+        ----------
+        s3_path : str
+            Path to the S3 bucket (e.g., 's3://bucket/path').
+        retry : bool, optional
+            Whether to retry the operation if a ConfigError is raised. Default is True.
 
         Returns
         -------
-        tuple[S3Client, str]
-            A tuple containing the S3 client object and the name of the S3 bucket.
-        """
-        bucket = self._get_bucket(root)
+        tuple of S3Client and str
+            Tuple containing the S3 client object and the name of the S3 bucket.
 
-        # Try to get client from environment variables
+        Raises
+        ------
+        ConfigError
+            If access to the specified bucket is not available and retry is False.
+        """
+        bucket = self._get_bucket(s3_path)
         try:
-            cfg = self._configurator.get_client_config(CredsOrigin.ENV.value)
+            cfg = self._configurator.get_client_config()
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
-
-        # Fallback to file
-        except StoreError:
-            cfg = self._configurator.get_client_config(CredsOrigin.FILE.value)
-            client = self._get_client(cfg)
-            self._check_access_to_storage(client, bucket)
-
-        return client, bucket
+            return client, bucket
+        except ConfigError as e:
+            if retry:
+                self._configurator.eval_change_origin()
+                return self._check_factory(s3_path, False)
+            raise e
 
     def _check_access_to_storage(self, client: S3Client, bucket: str) -> None:
         """
-        Check if the S3 bucket is accessible by sending a head_bucket request.
+        Checks if the S3 bucket is accessible by sending a head_bucket request.
 
         Parameters
         ----------
         client : S3Client
-            The S3 client object.
+            S3 client object.
         bucket : str
-            The name of the S3 bucket.
-
-        Returns
-        -------
-        None
+            Name of the S3 bucket.
 
         Raises
         ------
-        ClientError:
+        ConfigError
             If access to the specified bucket is not available.
         """
         try:
             client.head_bucket(Bucket=bucket)
         except (ClientError, NoCredentialsError) as err:
-            raise StoreError(f"No access to s3 bucket! Error: {err}")
+            raise ConfigError(f"No access to s3 bucket! Error: {err}")
 
     @staticmethod
     def _get_key(path: str) -> str:

digitalhub/stores/data/s3/utils.py

@@ -16,34 +16,34 @@ from digitalhub.utils.exceptions import StoreError
 
 def get_bucket_name(path: str) -> str:
     """
-    Get bucket name from path.
+    Extract the bucket name from an S3 path.
 
     Parameters
     ----------
     path : str
-        The source path to get the key from.
+        S3 URI (e.g., 's3://bucket/key').
 
     Returns
     -------
     str
-        The bucket name.
+        The bucket name extracted from the URI.
     """
     return urlparse(path).netloc
 
 
 def get_bucket_and_key(path: str) -> tuple[str, str]:
     """
-    Get bucket and key from path.
+    Extract the bucket name and key from an S3 path.
 
     Parameters
     ----------
     path : str
-        The source path to get the key from.
+        S3 URI (e.g., 's3://bucket/key').
 
     Returns
     -------
-    tuple[str, str]
-        The bucket and key.
+    tuple of str
+        Tuple containing (bucket, key) extracted from the URI.
     """
     parsed = urlparse(path)
     return parsed.netloc, parsed.path
@@ -51,7 +51,7 @@ def get_bucket_and_key(path: str) -> tuple[str, str]:
 
 def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     """
-    Get S3 source.
+    Download an object from S3 and save it to a local file.
 
     Parameters
     ----------
@@ -60,7 +60,7 @@ def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     key : str
         S3 object key.
     filename : Path
-        Path where to save the function source.
+        Local path where the downloaded object will be saved.
 
     Returns
     -------

digitalhub/stores/data/sql/configurator.py

@@ -10,23 +10,35 @@ from digitalhub.stores.credentials.enums import CredsEnvVar
 
 class SqlStoreConfigurator(Configurator):
     """
-    Configure the store by getting the credentials from user
-    provided config or from environment.
+    SQL store configuration manager for database connections.
+
+    Handles credential management and configuration for SQL database
+    connections. Loads credentials from environment variables or
+    configuration files and provides connection string generation
+    for database access.
+
+    Attributes
+    ----------
+    keys : list[str]
+        List of all supported credential keys for SQL connections.
+    required_keys : list[str]
+        List of mandatory credential keys that must be provided.
     """
 
     keys = [
-        CredsEnvVar.DB_USERNAME,
-        CredsEnvVar.DB_PASSWORD,
-        CredsEnvVar.DB_HOST,
-        CredsEnvVar.DB_PORT,
-        CredsEnvVar.DB_DATABASE,
+        CredsEnvVar.DB_USERNAME.value,
+        CredsEnvVar.DB_PASSWORD.value,
+        CredsEnvVar.DB_HOST.value,
+        CredsEnvVar.DB_PORT.value,
+        CredsEnvVar.DB_DATABASE.value,
+        CredsEnvVar.DB_PLATFORM.value,
     ]
     required_keys = [
-        CredsEnvVar.DB_USERNAME,
-        CredsEnvVar.DB_PASSWORD,
-        CredsEnvVar.DB_HOST,
-        CredsEnvVar.DB_PORT,
-        CredsEnvVar.DB_DATABASE,
+        CredsEnvVar.DB_USERNAME.value,
+        CredsEnvVar.DB_PASSWORD.value,
+        CredsEnvVar.DB_HOST.value,
+        CredsEnvVar.DB_PORT.value,
+        CredsEnvVar.DB_DATABASE.value,
     ]
 
     def __init__(self):
@@ -37,33 +49,72 @@ class SqlStoreConfigurator(Configurator):
     # Configuration methods
     ##############################
 
-    def load_configs(self) -> None:
-        # Load from env
-        env_creds = {var.value: self._creds_handler.load_from_env(var.value) for var in self.keys}
+    def load_env_vars(self) -> None:
+        """
+        Load database credentials from environment variables.
+
+        Retrieves SQL database connection credentials from the system
+        environment variables and stores them in the configurator's
+        credential handler for use in database connections.
+
+        Returns
+        -------
+        None
+        """
+        env_creds = self._creds_handler.load_from_env(self.keys)
         self._creds_handler.set_credentials(self._env, env_creds)
 
-        # Load from file
-        file_creds = {var.value: self._creds_handler.load_from_file(var.value) for var in self.keys}
+    def load_file_vars(self) -> None:
+        """
+        Load database credentials from configuration file.
+
+        Retrieves SQL database connection credentials from a
+        configuration file and stores them in the configurator's
+        credential handler for use in database connections.
+
+        Returns
+        -------
+        None
+        """
+        file_creds = self._creds_handler.load_from_file(self.keys)
         self._creds_handler.set_credentials(self._file, file_creds)
 
-    def get_sql_conn_string(self, origin: str) -> str:
+    def get_sql_conn_string(self) -> str:
         """
-        Get the connection string from environment variables.
+        Generate PostgreSQL connection string from stored credentials.
 
-        Parameters
-        ----------
-        origin : str
-            The origin of the credentials.
+        Constructs a PostgreSQL connection string using the configured
+        database credentials including username, password, host, port,
+        and database name.
 
         Returns
         -------
         str
-            The connection string.
+            A PostgreSQL connection string in the format:
+            'postgresql://username:password@host:port/database'
         """
-        creds = self.get_credentials(origin)
+        creds = self.get_sql_credentials()
         user = creds[CredsEnvVar.DB_USERNAME.value]
         password = creds[CredsEnvVar.DB_PASSWORD.value]
         host = creds[CredsEnvVar.DB_HOST.value]
         port = creds[CredsEnvVar.DB_PORT.value]
         database = creds[CredsEnvVar.DB_DATABASE.value]
         return f"postgresql://{user}:{password}@{host}:{port}/{database}"
+
+    def get_sql_credentials(self) -> dict:
+        """
+        Get all configured database credentials as a dictionary.
+
+        Retrieves all available database credentials from the configured
+        source and returns them as a dictionary with all credential keys
+        from self.keys mapped to their values.
+
+        Returns
+        -------
+        dict
+            Dictionary containing all credential key-value pairs from self.keys.
+            Keys correspond to database connection parameters such as
+            username, password, host, port, database, and platform.
+        """
+        creds = self.get_credentials(self._origin)
+        return {key: creds.get(key) for key in self.keys}