digitalhub 0.13.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

digitalhub/stores/data/_base/store.py

@@ -114,10 +114,6 @@ class Store:
         src : str
             The source path.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         StoreError
@@ -135,10 +131,6 @@ class Store:
         dst : str
             The destination path.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         StoreError
@@ -158,10 +150,6 @@ class Store:
         overwrite : bool
             Specify if overwrite an existing file.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         StoreError
@@ -179,10 +167,6 @@ class Store:
         ----------
         path : str | Path
             The path to build.
-
-        Returns
-        -------
-        None
         """
         if not isinstance(path, Path):
             path = Path(path)

digitalhub/stores/data/builder.py

@@ -79,14 +79,10 @@ class StoreBuilder:
             The unique identifier for the store type (e.g., 's3', 'sql').
         store : Store
             The store class to register for this type.
-        configurator : Configurator, optional
+        configurator : Configurator
             The configurator class for store configuration.
             If None, the store will be instantiated without configuration.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         StoreError

digitalhub/stores/data/local/store.py

@@ -4,7 +4,6 @@
 
 from __future__ import annotations
 
-import shutil
 from pathlib import Path
 from typing import Any
 
@@ -180,105 +179,3 @@ class LocalStore(Store):
         reader = get_reader_by_object(df)
         reader.write_df(df, dst, extension=extension, **kwargs)
         return dst
-
-    ##############################
-    # Private I/O methods
-    ##############################
-
-    def _get_src_dst_files(self, src: Path, dst: Path) -> list[str]:
-        """
-        Copy files from source to destination.
-
-        Parameters
-        ----------
-        src : Path
-            The source path.
-        dst : Path
-            The destination path.
-
-        Returns
-        -------
-        list[str]
-            Returns the list of destination and source paths of the
-            copied files.
-        """
-        return [self._get_src_dst_file(i, dst) for i in src.rglob("*") if i.is_file()]
-
-    def _get_src_dst_file(self, src: Path, dst: Path) -> str:
-        """
-        Copy file from source to destination.
-
-        Parameters
-        ----------
-        src : Path
-            The source path.
-        dst : Path
-            The destination path.
-
-        Returns
-        -------
-        str
-        """
-        dst_pth = self._copy_file(src, dst, True)
-        return str(dst_pth), str(src)
-
-    def _copy_dir(self, src: Path, dst: Path, overwrite: bool) -> list[str]:
-        """
-        Download file from source to destination.
-
-        Parameters
-        ----------
-        src : Path
-            The source path.
-        dst : Path
-            The destination path.
-
-        Returns
-        -------
-        list[str]
-        """
-        dst = self._rebuild_path(dst, src)
-        shutil.copytree(src, dst, dirs_exist_ok=overwrite)
-        return [str(i) for i in dst.rglob("*") if i.is_file()]
-
-    def _copy_file(self, src: Path, dst: Path, overwrite: bool) -> str:
-        """
-        Copy file from source to destination.
-
-        Parameters
-        ----------
-        src : Path
-            The source path.
-        dst : Path
-            The destination path.
-
-        Returns
-        -------
-        str
-        """
-        dst = self._rebuild_path(dst, src)
-        self._check_overwrite(dst, overwrite)
-        return str(shutil.copy2(src, dst))
-
-    def _rebuild_path(self, dst: Path, src: Path) -> Path:
-        """
-        Rebuild path.
-
-        Parameters
-        ----------
-        dst : Path
-            The destination path.
-        src : Path
-            The source path.
-
-        Returns
-        -------
-        Path
-            The rebuilt path.
-        """
-        if dst.is_dir():
-            if src.is_absolute():
-                raise StoreError("Source must be a relative path if the destination is a directory.")
-            dst = dst / src
-        self._build_path(dst)
-        return dst

digitalhub/stores/data/remote/store.py

@@ -174,10 +174,6 @@ class RemoteStore(Store):
         src : str
             The source location.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         HTTPError

digitalhub/stores/data/s3/configurator.py

@@ -46,10 +46,6 @@ class S3StoreConfigurator(Configurator):
     def load_env_vars(self) -> None:
         """
         Loads the credentials from the environment variables.
-
-        Returns
-        -------
-        None
         """
         env_creds = self._creds_handler.load_from_env(self.keys)
         self._creds_handler.set_credentials(self._env, env_creds)
@@ -57,10 +53,6 @@ class S3StoreConfigurator(Configurator):
     def load_file_vars(self) -> None:
         """
         Loads the credentials from a file.
-
-        Returns
-        -------
-        None
         """
         file_creds = self._creds_handler.load_from_file(self.keys)
         self._creds_handler.set_credentials(self._file, file_creds)
@@ -74,12 +66,23 @@ class S3StoreConfigurator(Configurator):
         dict
             Dictionary containing S3 credentials and configuration.
         """
-        creds = self.get_credentials(self._origin)
-        expired = creds[CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value]
-        if self._origin == self._file and self._is_expired(expired):
-            refresh_token()
-            self.load_file_vars()
-            creds = self.get_credentials(self._origin)
+        creds = self.evaluate_credentials()
+        return self.get_creds_dict(creds)
+
+    def get_creds_dict(self, creds: dict) -> dict:
+        """
+        Returns a dictionary containing the S3 credentials.
+
+        Parameters
+        ----------
+        creds : dict
+            The credentials dictionary.
+
+        Returns
+        -------
+        dict
+            A dictionary containing the S3 credentials.
+        """
         return {
             "endpoint_url": creds[CredsEnvVar.S3_ENDPOINT_URL.value],
             "aws_access_key_id": creds[CredsEnvVar.S3_ACCESS_KEY_ID.value],
@@ -91,6 +94,49 @@ class S3StoreConfigurator(Configurator):
             ),
         }
 
+    def evaluate_credentials(self) -> dict:
+        """
+        Evaluates and returns the current valid credentials.
+        If the credentials are expired and were loaded from file,
+        it refreshes them.
+
+        Returns
+        -------
+        dict
+            The current valid credentials.
+        """
+        creds = self.get_credentials(self._origin)
+        expired = creds[CredsEnvVar.S3_CREDENTIALS_EXPIRATION.value]
+        if self._origin == self._file and self._is_expired(expired):
+            refresh_token()
+            self.load_file_vars()
+            creds = self.get_credentials(self._origin)
+        return creds
+
+    def get_file_config(self) -> dict:
+        """
+        Returns the credentials loaded from file.
+
+        Returns
+        -------
+        dict
+            The credentials loaded from file.
+        """
+        creds = self.get_credentials(self._file)
+        return self.get_creds_dict(creds)
+
+    def get_env_config(self) -> dict:
+        """
+        Returns the credentials loaded from environment variables.
+
+        Returns
+        -------
+        dict
+            The credentials loaded from environment variables.
+        """
+        creds = self.get_credentials(self._env)
+        return self.get_creds_dict(creds)
+
     @staticmethod
     def _is_expired(timestamp: str | None) -> bool:
         """

digitalhub/stores/data/s3/store.py

@@ -16,7 +16,6 @@ from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError, NoCredentialsError
 
 from digitalhub.stores.data._base.store import Store
-from digitalhub.stores.data.s3.utils import get_bucket_name
 from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import ConfigError, StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
@@ -30,6 +29,8 @@ if typing.TYPE_CHECKING:
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
+MULTIPART_THRESHOLD = 100 * 1024 * 1024
+
 
 class S3Store(Store):
     """
@@ -332,10 +333,50 @@ class S3Store(Store):
         str
             The S3 path where the dataframe was saved.
         """
-        fileobj = BytesIO()
         reader = get_reader_by_object(df)
-        reader.write_df(df, fileobj, extension=extension, **kwargs)
-        return self.upload_fileobject(fileobj, dst)
+        with BytesIO() as fileobj:
+            reader.write_df(df, fileobj, extension=extension, **kwargs)
+            fileobj.seek(0)
+            return self.upload_fileobject(fileobj, dst)
+
+    ##############################
+    # Wrapper methods
+    ##############################
+
+    def get_s3_source(self, src: str, filename: Path) -> None:
+        """
+        Download a file from S3 and save it to a local file.
+
+        Parameters
+        ----------
+        src : str
+            S3 path of the object to be downloaded (e.g., 's3://bucket
+        filename : Path
+            Local path where the downloaded object will be saved.
+        """
+        client, bucket = self._check_factory(src)
+        key = self._get_key(src)
+        self._download_file(key, filename, client, bucket)
+
+    def get_s3_client(self, file: bool = True) -> S3Client:
+        """
+        Get an S3 client object.
+
+        Parameters
+        ----------
+        file : bool
+            Whether to use file-based credentials. Default is True.
+
+        Returns
+        -------
+        S3Client
+            Returns a client object that interacts with the S3 storage service.
+        """
+        if file:
+            cfg = self._configurator.get_file_config()
+        else:
+            cfg = self._configurator.get_env_config()
+        return self._get_client(cfg)
 
     ##############################
     # Private I/O methods
@@ -533,10 +574,6 @@ class S3Store(Store):
             The S3 client object.
         bucket : str
             The name of the S3 bucket.
-
-        Returns
-        -------
-        None
         """
         extra_args = {}
         mime_type = get_file_mime_type(src)
@@ -547,7 +584,7 @@ class S3Store(Store):
             Bucket=bucket,
             Key=key,
             ExtraArgs=extra_args,
-            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+            Config=TransferConfig(multipart_threshold=MULTIPART_THRESHOLD),
         )
 
     @staticmethod
@@ -570,16 +607,12 @@ class S3Store(Store):
             The S3 client object.
         bucket : str
             The name of the S3 bucket.
-
-        Returns
-        -------
-        None
         """
         client.upload_fileobj(
             Fileobj=fileobj,
             Bucket=bucket,
             Key=key,
-            Config=TransferConfig(multipart_threshold=100 * 1024 * 1024),
+            Config=TransferConfig(multipart_threshold=MULTIPART_THRESHOLD),
         )
 
     ##############################
@@ -595,7 +628,7 @@ class S3Store(Store):
         str
             The name of the S3 bucket.
         """
-        return get_bucket_name(root)
+        return urlparse(root).netloc
 
     def _get_client(self, cfg: dict) -> S3Client:
         """
@@ -621,7 +654,7 @@ class S3Store(Store):
         ----------
         s3_path : str
             Path to the S3 bucket (e.g., 's3://bucket/path').
-        retry : bool, optional
+        retry : bool
             Whether to retry the operation if a ConfigError is raised. Default is True.
 
         Returns

digitalhub/stores/data/sql/configurator.py

@@ -56,10 +56,6 @@ class SqlStoreConfigurator(Configurator):
         Retrieves SQL database connection credentials from the system
         environment variables and stores them in the configurator's
         credential handler for use in database connections.
-
-        Returns
-        -------
-        None
         """
         env_creds = self._creds_handler.load_from_env(self.keys)
         self._creds_handler.set_credentials(self._env, env_creds)
@@ -71,10 +67,6 @@ class SqlStoreConfigurator(Configurator):
         Retrieves SQL database connection credentials from a
         configuration file and stores them in the configurator's
         credential handler for use in database connections.
-
-        Returns
-        -------
-        None
         """
         file_creds = self._creds_handler.load_from_file(self.keys)
         self._creds_handler.set_credentials(self._file, file_creds)

digitalhub/stores/data/sql/store.py

@@ -159,9 +159,9 @@ class SqlStore(Store):
         path : SourcesOrListOfSources
             The SQL URI path to read from in the format
             'sql://database/schema/table'. Only single paths are supported.
-        file_format : str, optional
+        file_format : str
             File format specification (not used for SQL operations).
-        engine : str, optional
+        engine : str
             DataFrame engine to use (e.g., 'pandas', 'polars').
             If None, uses the default engine.
         **kwargs : dict
@@ -209,7 +209,7 @@ class SqlStore(Store):
         path : str
             The SQL URI path specifying the database connection
             in the format 'sql://database/schema/table'.
-        engine : str, optional
+        engine : str
             DataFrame engine to use for result processing
             (e.g., 'pandas', 'polars'). If None, uses the default.
 
@@ -238,7 +238,7 @@ class SqlStore(Store):
         dst : str
             The destination SQL URI in the format
             'sql://database/schema/table' or 'sql://database/table'.
-        extension : str, optional
+        extension : str
             File extension parameter (not used for SQL operations).
         **kwargs : dict
             Additional keyword arguments passed to the DataFrame's
@@ -253,6 +253,21 @@ class SqlStore(Store):
         table = self._get_table_name(dst)
         return self._upload_table(df, schema, table, **kwargs)
 
+    ##############################
+    # Wrapper methods
+    ##############################
+
+    def get_engine(self, schema: str | None = None) -> Engine:
+        """
+        Get a SQLAlchemy engine connected to the database.
+
+        Returns
+        -------
+        Engine
+            A SQLAlchemy engine instance connected to the database.
+        """
+        return self._check_factory(schema=schema)
+
     ##############################
     # Private I/O methods
     ##############################
@@ -359,7 +374,7 @@ class SqlStore(Store):
 
         Parameters
         ----------
-        schema : str, optional
+        schema : str
             The database schema to set in the search path.
             If provided, sets the PostgreSQL search_path option.
 
@@ -397,7 +412,7 @@ class SqlStore(Store):
         retry : bool, default True
             Whether to attempt a retry with different configuration
             if the initial connection fails.
-        schema : str, optional
+        schema : str
             The database schema to configure in the engine.
 
         Returns
@@ -511,10 +526,6 @@ class SqlStore(Store):
         engine : Engine
             The SQLAlchemy engine to test for connectivity.
 
-        Returns
-        -------
-        None
-
         Raises
         ------
         ConfigError

digitalhub/stores/readers/data/factory.py

@@ -33,10 +33,6 @@ class ReaderFactory:
             Reader name.
         builder : DataframeReader
             Builder object.
-
-        Returns
-        -------
-        None
         """
         if self._engine_builders is None:
             self._engine_builders = {}
@@ -104,10 +100,6 @@ class ReaderFactory:
         ----------
         engine : str
             Engine name.
-
-        Returns
-        -------
-        None
         """
         if engine not in self._engine_builders:
             raise BuilderError(f"Engine {engine} not found.")

digitalhub/stores/readers/data/pandas/reader.py

@@ -104,10 +104,6 @@ class DataframeReaderPandas(DataframeReader):
             The destination of the dataframe.
         **kwargs : dict
             Keyword arguments.
-
-        Returns
-        -------
-        None
         """
         if extension == FileExtensions.CSV.value:
             return self.write_csv(df, dst, **kwargs)
@@ -128,10 +124,6 @@ class DataframeReaderPandas(DataframeReader):
             The destination of the dataframe.
         **kwargs : dict
             Keyword arguments.
-
-        Returns
-        -------
-        None
         """
         if "index" not in kwargs:
             kwargs["index"] = False
@@ -150,10 +142,6 @@ class DataframeReaderPandas(DataframeReader):
             The destination of the dataframe.
         **kwargs : dict
             Keyword arguments.
-
-        Returns
-        -------
-        None
         """
         if "index" not in kwargs:
             kwargs["index"] = False
@@ -176,10 +164,6 @@ class DataframeReaderPandas(DataframeReader):
             The destination schema.
         **kwargs : dict
             Keyword arguments.
-
-        Returns
-        -------
-        None
         """
         if "index" not in kwargs:
             kwargs["index"] = False

digitalhub/utils/file_utils.py

@@ -141,23 +141,6 @@ def get_last_modified(data_path: str) -> str:
     return datetime.fromtimestamp(timestamp).astimezone().isoformat()
 
 
-def get_s3_path(src_path: str) -> str:
-    """
-    Get the S3 URI of a file path.
-
-    Parameters
-    ----------
-    src_path : str
-        Path to the file.
-
-    Returns
-    -------
-    str
-        The S3 URI of the file.
-    """
-    return Path(src_path).as_uri()
-
-
 def get_file_info_from_local(path: str, src_path: str) -> None | dict:
     """
     Get file info from a local path.

digitalhub/utils/generic_utils.py

@@ -95,10 +95,6 @@ def requests_chunk_download(source: str, filename: Path) -> None:
         URL to download the file from.
     filename : Path
         Path where to save the downloaded file.
-
-    Returns
-    -------
-    None
     """
     with requests.get(source, stream=True) as r:
         r.raise_for_status()
@@ -117,10 +113,6 @@ def extract_archive(path: Path, filename: Path) -> None:
         Directory where to extract the archive.
     filename : Path
         Path to the zip archive file.
-
-    Returns
-    -------
-    None
     """
     with ZipFile(filename, "r") as zip_file:
         zip_file.extractall(path)
@@ -256,10 +248,6 @@ def carriage_return_warn(string: str) -> None:
     ----------
     string : str
         The string to check.
-
-    Returns
-    -------
-    None
     """
     if "\r\n" in string:
         warn("String contains a carriage return. It may not be parsed correctly from remote runtimes.")

digitalhub/utils/git_utils.py

@@ -47,10 +47,6 @@ def clone_repository(path: Path, url: str) -> None:
         Path where to save the repository.
     url : str
         URL of the repository.
-
-    Returns
-    -------
-    None
     """
     clean_path(path)
     checkout_object = get_checkout_object(url)
@@ -85,10 +81,6 @@ def clean_path(path: Path) -> None:
     ----------
     path : Path
         Path to clean.
-
-    Returns
-    -------
-    None
     """
 
     shutil.rmtree(path, ignore_errors=True)

digitalhub/utils/io_utils.py

@@ -23,10 +23,6 @@ def write_yaml(filepath: str | Path, obj: dict | list[dict]) -> None:
         The YAML file path to write.
     obj : dict or list of dict
         The dict or list of dicts to write.
-
-    Returns
-    -------
-    None
     """
     if isinstance(obj, list):
         with open(filepath, "w", encoding="utf-8") as out_file:
@@ -46,10 +42,6 @@ def write_text(filepath: Path, text: str) -> None:
         The file path to write.
     text : str
         The text to write.
-
-    Returns
-    -------
-    None
     """
     filepath.write_text(text, encoding="utf-8")
 
@@ -80,10 +72,6 @@ class NoDatesSafeLoader(yaml.SafeLoader):
         ----------
         tag_to_remove : str
             The tag to remove.
-
-        Returns
-        -------
-        None
         """
         if "yaml_implicit_resolvers" not in cls.__dict__:
             cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()