digitalhub 0.13.0b1__py3-none-any.whl → 0.13.0b3__py3-none-any.whl

digitalhub/stores/data/s3/store.py

@@ -15,11 +15,10 @@ import botocore.client  # pylint: disable=unused-import
 from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
 from digitalhub.stores.data.s3.utils import get_bucket_name
 from digitalhub.stores.readers.data.api import get_reader_by_object
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.exceptions import ConfigError, StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
@@ -614,55 +613,59 @@ class S3Store(Store):
         """
         return boto3.client("s3", **cfg)
 
-    def _check_factory(self, root: str) -> tuple[S3Client, str]:
+    def _check_factory(self, s3_path: str, retry: bool = True) -> tuple[S3Client, str]:
         """
-        Check if the S3 bucket is accessible by sending a head_bucket request.
+        Checks if the S3 bucket collected from the URI is accessible.
+
+        Parameters
+        ----------
+        s3_path : str
+            Path to the S3 bucket (e.g., 's3://bucket/path').
+        retry : bool, optional
+            Whether to retry the operation if a ConfigError is raised. Default is True.
 
         Returns
         -------
-        tuple[S3Client, str]
-            A tuple containing the S3 client object and the name of the S3 bucket.
-        """
-        bucket = self._get_bucket(root)
+        tuple of S3Client and str
+            Tuple containing the S3 client object and the name of the S3 bucket.
 
-        # Try to get client from environment variables
+        Raises
+        ------
+        ConfigError
+            If access to the specified bucket is not available and retry is False.
+        """
+        bucket = self._get_bucket(s3_path)
         try:
-            cfg = self._configurator.get_client_config(CredsOrigin.ENV.value)
+            cfg = self._configurator.get_client_config()
             client = self._get_client(cfg)
             self._check_access_to_storage(client, bucket)
-
-        # Fallback to file
-        except StoreError:
-            cfg = self._configurator.get_client_config(CredsOrigin.FILE.value)
-            client = self._get_client(cfg)
-            self._check_access_to_storage(client, bucket)
-
-        return client, bucket
+            return client, bucket
+        except ConfigError as e:
+            if retry:
+                self._configurator.eval_change_origin()
+                return self._check_factory(s3_path, False)
+            raise e
 
     def _check_access_to_storage(self, client: S3Client, bucket: str) -> None:
         """
-        Check if the S3 bucket is accessible by sending a head_bucket request.
+        Checks if the S3 bucket is accessible by sending a head_bucket request.
 
         Parameters
         ----------
         client : S3Client
-            The S3 client object.
+            S3 client object.
         bucket : str
-            The name of the S3 bucket.
-
-        Returns
-        -------
-        None
+            Name of the S3 bucket.
 
         Raises
         ------
-        ClientError:
+        ConfigError
             If access to the specified bucket is not available.
         """
         try:
             client.head_bucket(Bucket=bucket)
         except (ClientError, NoCredentialsError) as err:
-            raise StoreError(f"No access to s3 bucket! Error: {err}")
+            raise ConfigError(f"No access to s3 bucket! Error: {err}")
 
     @staticmethod
     def _get_key(path: str) -> str:

digitalhub/stores/data/s3/utils.py

@@ -16,34 +16,34 @@ from digitalhub.utils.exceptions import StoreError
 
 def get_bucket_name(path: str) -> str:
     """
-    Get bucket name from path.
+    Extract the bucket name from an S3 path.
 
     Parameters
     ----------
     path : str
-        The source path to get the key from.
+        S3 URI (e.g., 's3://bucket/key').
 
     Returns
     -------
     str
-        The bucket name.
+        The bucket name extracted from the URI.
     """
     return urlparse(path).netloc
 
 
 def get_bucket_and_key(path: str) -> tuple[str, str]:
     """
-    Get bucket and key from path.
+    Extract the bucket name and key from an S3 path.
 
     Parameters
     ----------
     path : str
-        The source path to get the key from.
+        S3 URI (e.g., 's3://bucket/key').
 
     Returns
     -------
-    tuple[str, str]
-        The bucket and key.
+    tuple of str
+        Tuple containing (bucket, key) extracted from the URI.
     """
     parsed = urlparse(path)
     return parsed.netloc, parsed.path
@@ -51,7 +51,7 @@ def get_bucket_and_key(path: str) -> tuple[str, str]:
 
 def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     """
-    Get S3 source.
+    Download an object from S3 and save it to a local file.
 
     Parameters
     ----------
@@ -60,7 +60,7 @@ def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     key : str
         S3 object key.
     filename : Path
-        Path where to save the function source.
+        Local path where the downloaded object will be saved.
 
     Returns
     -------

digitalhub/stores/data/sql/configurator.py

@@ -15,18 +15,19 @@ class SqlStoreConfigurator(Configurator):
     """
 
     keys = [
-        CredsEnvVar.DB_USERNAME,
-        CredsEnvVar.DB_PASSWORD,
-        CredsEnvVar.DB_HOST,
-        CredsEnvVar.DB_PORT,
-        CredsEnvVar.DB_DATABASE,
+        CredsEnvVar.DB_USERNAME.value,
+        CredsEnvVar.DB_PASSWORD.value,
+        CredsEnvVar.DB_HOST.value,
+        CredsEnvVar.DB_PORT.value,
+        CredsEnvVar.DB_DATABASE.value,
+        CredsEnvVar.DB_PLATFORM.value,
     ]
     required_keys = [
-        CredsEnvVar.DB_USERNAME,
-        CredsEnvVar.DB_PASSWORD,
-        CredsEnvVar.DB_HOST,
-        CredsEnvVar.DB_PORT,
-        CredsEnvVar.DB_DATABASE,
+        CredsEnvVar.DB_USERNAME.value,
+        CredsEnvVar.DB_PASSWORD.value,
+        CredsEnvVar.DB_HOST.value,
+        CredsEnvVar.DB_PORT.value,
+        CredsEnvVar.DB_DATABASE.value,
     ]
 
     def __init__(self):
@@ -37,30 +38,30 @@ class SqlStoreConfigurator(Configurator):
     # Configuration methods
     ##############################
 
-    def load_configs(self) -> None:
-        # Load from env
-        env_creds = {var.value: self._creds_handler.load_from_env(var.value) for var in self.keys}
+    def load_env_vars(self) -> None:
+        """
+        Load the credentials from the environment.
+        """
+        env_creds = self._creds_handler.load_from_env(self.keys)
         self._creds_handler.set_credentials(self._env, env_creds)
 
-        # Load from file
-        file_creds = {var.value: self._creds_handler.load_from_file(var.value) for var in self.keys}
+    def load_file_vars(self) -> None:
+        """
+        Load the credentials from the file.
+        """
+        file_creds = self._creds_handler.load_from_file(self.keys)
         self._creds_handler.set_credentials(self._file, file_creds)
 
-    def get_sql_conn_string(self, origin: str) -> str:
+    def get_sql_conn_string(self) -> str:
         """
         Get the connection string from environment variables.
 
-        Parameters
-        ----------
-        origin : str
-            The origin of the credentials.
-
         Returns
         -------
         str
             The connection string.
         """
-        creds = self.get_credentials(origin)
+        creds = self.get_credentials(self._origin)
         user = creds[CredsEnvVar.DB_USERNAME.value]
         password = creds[CredsEnvVar.DB_PASSWORD.value]
         host = creds[CredsEnvVar.DB_HOST.value]

digitalhub/stores/data/sql/store.py

@@ -14,10 +14,9 @@ from sqlalchemy import MetaData, Table, create_engine, select
 from sqlalchemy.engine import Engine
 from sqlalchemy.exc import SQLAlchemyError
 
-from digitalhub.stores.credentials.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
 from digitalhub.stores.readers.data.api import get_reader_by_object
-from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.exceptions import ConfigError, StoreError
 from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
@@ -287,21 +286,16 @@ class SqlStore(Store):
     # Helper methods
     ##############################
 
-    def _get_connection_string(self, origin: str) -> str:
+    def _get_connection_string(self) -> str:
         """
         Get the connection string.
 
-        Parameters
-        ----------
-        origin : str
-            The origin of the credentials.
-
         Returns
         -------
         str
             The connection string.
         """
-        return self._configurator.get_sql_conn_string(origin)
+        return self._configurator.get_sql_conn_string()
 
     def _get_engine(self, origin: str, schema: str | None = None) -> Engine:
         """
@@ -330,12 +324,14 @@ class SqlStore(Store):
         except Exception as ex:
             raise StoreError(f"Something wrong with connection string. Arguments: {str(ex.args)}")
 
-    def _check_factory(self, schema: str | None = None) -> Engine:
+    def _check_factory(self, retry: bool = True, schema: str | None = None) -> Engine:
         """
         Check if the database is accessible and return the engine.
 
         Parameters
         ----------
+        retry : bool
+            Whether to retry if the database is not accessible.
         schema : str
             The schema.
 
@@ -345,12 +341,14 @@ class SqlStore(Store):
             The database engine.
         """
         try:
-            engine = self._get_engine(CredsOrigin.ENV.value, schema)
-            self._check_access_to_storage(engine)
-        except StoreError:
-            engine = self._get_engine(CredsOrigin.FILE.value, schema)
+            engine = self._get_engine(schema)
             self._check_access_to_storage(engine)
-        return engine
+            return engine
+        except ConfigError as e:
+            if retry:
+                self._configurator.eval_change_origin()
+                return self._check_factory(retry=False, schema=schema)
+            raise e
 
     @staticmethod
     def _parse_path(path: str) -> dict:
@@ -435,4 +433,4 @@ class SqlStore(Store):
             engine.connect()
         except SQLAlchemyError:
             engine.dispose()
-            raise StoreError("No access to db!")
+            raise ConfigError("No access to db!")

digitalhub/utils/exceptions.py

@@ -81,3 +81,9 @@ class ClientError(Exception):
     """
     Raised when incontered errors on clients.
     """
+
+
+class ConfigError(Exception):
+    """
+    Raised when incontered errors on configs.
+    """

digitalhub/utils/file_utils.py

@@ -16,6 +16,21 @@ from pydantic import BaseModel
 class FileInfo(BaseModel):
     """
     File info class.
+
+    Attributes
+    ----------
+    path : str or None
+        Path to the file.
+    name : str or None
+        Name of the file.
+    content_type : str or None
+        MIME type of the file.
+    size : int or None
+        Size of the file in bytes.
+    hash : str or None
+        Hash of the file contents.
+    last_modified : str or None
+        Last modified date/time in ISO format.
     """
 
     path: Optional[str] = None
@@ -25,13 +40,21 @@ class FileInfo(BaseModel):
     hash: Optional[str] = None
     last_modified: Optional[str] = None
 
-    def to_dict(self):
+    def to_dict(self) -> dict:
+        """
+        Convert FileInfo to dictionary.
+
+        Returns
+        -------
+        dict
+            Dictionary representation of the FileInfo object.
+        """
         return self.model_dump()
 
 
 def calculate_blob_hash(data_path: str) -> str:
     """
-    Calculate the hash of a file.
+    Calculate the SHA-256 hash of a file.
 
     Parameters
     ----------
@@ -41,7 +64,7 @@ def calculate_blob_hash(data_path: str) -> str:
     Returns
     -------
     str
-        The hash of the file.
+        The SHA-256 hash of the file, prefixed with 'sha256:'.
     """
     with open(data_path, "rb") as f:
         data = f.read()
@@ -50,7 +73,7 @@ def calculate_blob_hash(data_path: str) -> str:
 
 def get_file_size(data_path: str) -> int:
     """
-    Get the size of a file.
+    Get the size of a file in bytes.
 
     Parameters
     ----------
@@ -60,14 +83,14 @@ def get_file_size(data_path: str) -> int:
     Returns
     -------
     int
-        The size of the file.
+        Size of the file in bytes.
     """
     return Path(data_path).stat().st_size
 
 
 def get_file_mime_type(data_path: str) -> str | None:
     """
-    Get the mime type of a file.
+    Get the MIME type of a file.
 
     Parameters
     ----------
@@ -76,15 +99,15 @@ def get_file_mime_type(data_path: str) -> str | None:
 
     Returns
     -------
-    str
-        The mime type of the file.
+    str or None
+        The MIME type of the file, or None if unknown.
     """
     return guess_type(data_path)[0]
 
 
 def get_path_name(data_path: str) -> str:
     """
-    Get the name of a file.
+    Get the name of a file from its path.
 
     Parameters
     ----------
@@ -101,7 +124,7 @@ def get_path_name(data_path: str) -> str:
 
 def get_last_modified(data_path: str) -> str:
     """
-    Get the last modified date of a file.
+    Get the last modified date/time of a file in ISO format.
 
     Parameters
     ----------
@@ -111,7 +134,7 @@ def get_last_modified(data_path: str) -> str:
     Returns
     -------
     str
-        The last modified date of the file.
+        The last modified date/time in ISO format.
     """
     path = Path(data_path)
     timestamp = path.stat().st_mtime
@@ -120,7 +143,7 @@ def get_last_modified(data_path: str) -> str:
 
 def get_s3_path(src_path: str) -> str:
     """
-    Get the S3 path of a file.
+    Get the S3 URI of a file path.
 
     Parameters
     ----------
@@ -130,26 +153,26 @@ def get_s3_path(src_path: str) -> str:
     Returns
     -------
     str
-        The S3 path of the file.
+        The S3 URI of the file.
     """
     return Path(src_path).as_uri()
 
 
 def get_file_info_from_local(path: str, src_path: str) -> None | dict:
     """
-    Get file info from path.
+    Get file info from a local path.
 
     Parameters
     ----------
     path : str
         Target path of the object.
     src_path : str
-        Local path of some source.
+        Local path of the source file.
 
     Returns
     -------
-    dict
-        File info.
+    dict or None
+        File info dictionary, or None if an error occurs.
     """
     try:
         name = get_path_name(path)
@@ -172,19 +195,19 @@ def get_file_info_from_local(path: str, src_path: str) -> None | dict:
 
 def get_file_info_from_s3(path: str, metadata: dict) -> None | dict:
     """
-    Get file info from path.
+    Get file info from S3 metadata.
 
     Parameters
     ----------
     path : str
         Object source path.
     metadata : dict
-        Metadata of the object from S3.
+        Metadata dictionary of the object from S3.
 
     Returns
     -------
-    dict
-        File info.
+    dict or None
+        File info dictionary, or None if an error occurs.
     """
     try:
         size = metadata["ContentLength"]
@@ -214,17 +237,17 @@ def get_file_info_from_s3(path: str, metadata: dict) -> None | dict:
 
 def eval_zip_type(source: str) -> bool:
     """
-    Evaluate zip type.
+    Evaluate whether the source is a zip file.
 
     Parameters
     ----------
     source : str
-        Source.
+        Source file path.
 
     Returns
     -------
     bool
-        True if path is zip.
+        True if the path is a zip file, False otherwise.
     """
     extension = source.endswith(".zip")
     mime_zip = get_file_mime_type(source) == "application/zip"
@@ -233,34 +256,34 @@ def eval_zip_type(source: str) -> bool:
 
 def eval_text_type(source: str) -> bool:
     """
-    Evaluate text type.
+    Evaluate whether the source is a plain text file.
 
     Parameters
     ----------
     source : str
-        Source.
+        Source file path.
 
     Returns
     -------
     bool
-        True if path is text.
+        True if the path is a plain text file, False otherwise.
     """
     return get_file_mime_type(source) == "text/plain"
 
 
 def eval_py_type(source: str) -> bool:
     """
-    Evaluate python type.
+    Evaluate whether the source is a Python file.
 
     Parameters
     ----------
     source : str
-        Source.
+        Source file path.
 
     Returns
     -------
     bool
-        True if path is python.
+        True if the path is a Python file, False otherwise.
     """
     extension = source.endswith(".py")
     mime_py = get_file_mime_type(source) == "text/x-python"