pypomes-s3 0.3.1__tar.gz

pypomes_s3-0.3.1/.gitignore

@@ -0,0 +1,16 @@
+.cache
+.env
+.venv
+.idea
+.vscode
+env
+venv
+temp
+tmp
+**/__pycache__
+*.py[cod]
+*.sh
+*.ps1
+/dist
+/env
+deploy.txt

pypomes_s3-0.3.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 GT Nunes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pypomes_s3-0.3.1/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.3
+Name: pypomes_s3
+Version: 0.3.1
+Summary: A collection of Python pomes, pennyeach (S3 storage modules)
+Project-URL: Homepage, https://github.com/TheWiseCoder/PyPomes-S3
+Project-URL: Bug Tracker, https://github.com/TheWiseCoder/PyPomes-S3/issues
+Author-email: GT Nunes <wisecoder01@gmail.com>
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: boto3>=1.34.108
+Requires-Dist: minio>=7.2.5
+Requires-Dist: pip>=24.0
+Requires-Dist: pypomes-core>=1.0.4
+Requires-Dist: setuptools>=68.0.0
+Requires-Dist: unidecode>=1.3.6
+Requires-Dist: wheel>=0.42.0

pypomes_s3-0.3.1/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = [
+    "hatchling>=1.22.2"
+]
+build-backend = "hatchling.build"
+
+[project]
+name = "pypomes_s3"
+version = "0.3.1"
+authors = [
+  { name="GT Nunes", email="wisecoder01@gmail.com" }
+]
+description = "A collection of Python pomes, pennyeach (S3 storage modules)"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent"
+]
+dependencies = [
+    "boto3>=1.34.108",
+    "minio>=7.2.5",
+    "pip>=24.0",
+    "pypomes_core>=1.0.4",
+    "setuptools>=68.0.0",
+    "Unidecode>=1.3.6",
+    "wheel>=0.42.0"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/TheWiseCoder/PyPomes-S3"
+"Bug Tracker" = "https://github.com/TheWiseCoder/PyPomes-S3/issues"

pypomes_s3-0.3.1/src/pypomes_s3/__init__.py

@@ -0,0 +1,24 @@
+from .minio_client import (
+    MinioCM
+)
+from .minio_pomes import (
+    MINIO_BUCKET_NAME, MINIO_ENDPOINT_URL, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE_ACCESS, MINIO_TEMP_PATH,
+    minio_access, minio_file_store, minio_object_store, minio_object_stat,
+    minio_object_delete, minio_objects_list, minio_object_retrieve, minio_object_exists,
+    minio_object_tags_retrieve, minio_file_retrieve, minio_setup,
+)
+
+__all__ = [
+    # minio_client
+    "MinioCM",
+    # minio_pomes
+    "MINIO_BUCKET_NAME", "MINIO_ENDPOINT_URL", "MINIO_ACCESS_KEY",
+    "MINIO_SECRET_KEY", "MINIO_SECURE_ACCESS", "MINIO_TEMP_PATH",
+    "minio_access", "minio_file_store", "minio_object_store", "minio_object_stat",
+    "minio_object_delete", "minio_objects_list", "minio_object_retrieve", "minio_object_exists",
+    "minio_object_tags_retrieve", "minio_file_retrieve", "minio_setup",
+]
+
+from importlib.metadata import version
+__version__ = version("pypomes_s3")
+__version_info__ = tuple(int(i) for i in __version__.split(".") if i.isdigit())

pypomes_s3-0.3.1/src/pypomes_s3/aws_pomes.py

@@ -0,0 +1,42 @@
+# import pickle
+# import uuid
+from boto3.session import Session
+from botocore.client import BaseClient
+from logging import Logger
+from pathlib import Path
+from pypomes_core import (
+    APP_PREFIX, TEMP_DIR,
+    env_get_str, env_get_path
+)
+from typing import Final
+# from unidecode import unidecode
+
+S3_BUCKET_NAME: Final[str] = env_get_str(f"{APP_PREFIX}_S3_BUCKET_NAME")
+S3_ACCESS_KEY: Final[str] = env_get_str(f"{APP_PREFIX}_S3_ACCESS_KEY")
+S3_SECRET_KEY: Final[str] = env_get_str(f"{APP_PREFIX}_S3_SECRET_KEY")
+S3_ENDPOINT_URL: Final[str] = env_get_str(f"{APP_PREFIX}_S3_ENDPOINT_URL")
+S3_TEMP_PATH: Final[Path] = env_get_path(f"{APP_PREFIX}_S3_TEMP_PATH", TEMP_DIR)
+
+
+def s3_access(errors: list[str],
+              logger: Logger = None) -> BaseClient:
+    """
+    Obtain and return a *S3* client object.
+
+    :param errors: incidental error messages
+    :param logger: optional logger
+    :return: the S3 client object
+    """
+    # initialize the return variable
+    result: BaseClient | None = None
+
+    try:
+        result = Session().client(service_name="s3",
+                                  aws_access_key_id=S3_ACCESS_KEY,
+                                  aws_secret_access_key=S3_SECRET_KEY,
+                                  endpoint_url=S3_ENDPOINT_URL)
+    except Exception as e:
+        errors.append(str(e))
+        logger.debug(e)
+
+    return result

pypomes_s3-0.3.1/src/pypomes_s3/minio_client.py

@@ -0,0 +1,67 @@
+from minio import Minio
+from minio.credentials.providers import Provider
+from types import TracebackType
+from urllib3 import PoolManager
+
+
+class MinioCM:
+    """
+    Simple Storage Service (*S3*) client, extended to provide context-managed operations.
+
+    Please, refer to the *MinIO* package documentation for details on its implementation.
+    """
+    def __init__(self,
+                 endpoint: str,
+                 access_key: str | None = None,
+                 secret_key: str | None = None,
+                 session_token: str | None = None,
+                 secure: bool = True,
+                 region: str | None = None,
+                 http_client: PoolManager | None = None,
+                 credentials: Provider | None = None,
+                 cert_check: bool = True) -> None:
+        """
+        Extend *MinIO* package's implementation of a Simple Storage Service (*S3*) client, .
+
+        The aim is to provide context-managed *MinIO* clients.
+        Please, refer to the *MinIO* package documentation for details on its implementation.
+
+        :param endpoint: Hostname of a S3 service
+        :param access_key: Access key (aka user ID) of your account in S3 service
+        :param secret_key: Secret Key (aka password) of your account in S3 service
+        :param session_token: Session token of your account in S3 service
+        :param secure: Flag to indicate to use secure (TLS) connection to S3 service or not
+        :param region: Region name of buckets in S3 service
+        :param http_client: Customized HTTP client
+        :param credentials: Credentials provider of your account in S3 service
+        :param cert_check: Flag to indicate to verify SSL certificate or not
+        """
+        self.minio_client: Minio = Minio(endpoint=endpoint,
+                                         access_key=access_key,
+                                         secret_key=secret_key,
+                                         session_token=session_token,
+                                         secure=secure,
+                                         region=region,
+                                         http_client=http_client,
+                                         credentials=credentials,
+                                         cert_check=cert_check)
+
+    def __enter__(self) -> Minio:
+        """
+        First step in providing a context-managed *MinIO* client.
+
+        :return: a Minio client object
+        """
+        return self.minio_client
+
+    def __exit__(self,
+                 exception_type: type[BaseException],
+                 exception_value: BaseException,
+                 traceback: TracebackType) -> bool:
+        """
+        Last step in providing a context-managed *MinIO* client.
+        """
+        self.minio_client = None
+
+        # make sure an eventual exception is propagated
+        return exception_type is None

pypomes_s3-0.3.1/src/pypomes_s3/minio_pomes.py

@@ -0,0 +1,578 @@
+import pickle
+import uuid
+from collections.abc import Iterator
+from logging import Logger
+from minio import Minio
+from minio.datatypes import Object as MinioObject
+from minio.commonconfig import Tags
+from pathlib import Path
+from pypomes_core import (
+    APP_PREFIX, TEMP_DIR,
+    env_get_bool, env_get_str, env_get_path
+)
+from typing import Final
+from unidecode import unidecode
+
+MINIO_BUCKET_NAME: Final[str] = env_get_str(f"{APP_PREFIX}_MINIO_BUCKET_URL")
+MINIO_ENDPOINT_URL: Final[str] = env_get_str(f"{APP_PREFIX}_MINIO_ENDPOINT_URL")
+MINIO_ACCESS_KEY: Final[str] = env_get_str(f"{APP_PREFIX}_MINIO_ACCESS_KEY")
+MINIO_SECRET_KEY: Final[str] = env_get_str(f"{APP_PREFIX}_MINIO_SECRET_KEY")
+MINIO_SECURE_ACCESS: Final[bool] = env_get_bool(f"{APP_PREFIX}_MINIO_SECURE_ACCESS")
+MINIO_TEMP_PATH: Final[Path] = env_get_path(f"{APP_PREFIX}_MINIO_TEMP_PATH", TEMP_DIR)
+
+
+def minio_access(errors: list[str],
+                 logger: Logger = None) -> Minio:
+    """
+    Obtain and return a *MinIO* client object.
+
+    :param errors: incidental error messages
+    :param logger: optional logger
+    :return: the MinIO client object
+    """
+    # initialize the return variable
+    result: Minio | None = None
+
+    # obtain the MinIO client
+    try:
+        result = Minio(endpoint=MINIO_ENDPOINT_URL,
+                       access_key=MINIO_ACCESS_KEY,
+                       secret_key=MINIO_SECRET_KEY,
+                       secure=MINIO_SECURE_ACCESS)
+        if logger:
+            logger.debug("Minio client created")
+
+    except Exception as e:
+        __minio_except_msg(errors=errors,
+                           exception=e,
+                           logger=logger)
+
+    return result
+
+
+def minio_setup(errors: list[str],
+                bucket: str = MINIO_BUCKET_NAME,
+                client: Minio = None,
+                logger: Logger = None) -> bool:
+    """
+    Prepare the *MinIO* client for operations.
+
+    This function should be called just once, at startup,
+    to make sure the interaction with the MinIo service is fully functional.
+
+    :param errors: incidental error messages
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: True if service is fully functional
+    """
+    # initialize the return variable
+    result: bool = False
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # was the MinIO client obtained ?
+    if curr_client:
+        # yes, proceed
+        try:
+            if not curr_client.bucket_exists(bucket_name=bucket):
+                curr_client.make_bucket(bucket_name=bucket)
+            result = True
+            if logger:
+                logger.debug(f"Setup MinIO, endpoint={MINIO_ENDPOINT_URL}, bucket={bucket}, "
+                             f"access key={MINIO_ACCESS_KEY}, secure={MINIO_SECURE_ACCESS}")
+        except Exception as e:
+            __minio_except_msg(errors=errors,
+                               exception=e,
+                               logger=logger)
+
+    return result
+
+
+def minio_file_store(errors: list[str],
+                     basepath: Path | str,
+                     identifier: str,
+                     filepath: Path | str,
+                     mimetype: str,
+                     tags: dict = None,
+                     bucket: str = MINIO_BUCKET_NAME,
+                     client: Minio = None,
+                     logger: Logger = None) -> None:
+    """
+    Store a file at the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to store the file at
+    :param identifier: the file identifier, tipically a file name
+    :param filepath: the path specifying where the file is
+    :param mimetype: the file mimetype
+    :param tags: optional metadata describing the file
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    """
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # was the MinIO client obtained ?
+    if curr_client:
+        # yes, proceed
+        remotepath: Path = Path(basepath) / identifier
+        # have tags been defined ?
+        if tags is None or len(tags) == 0:
+            # no
+            doc_tags = None
+        else:
+            # sim, store them
+            doc_tags = Tags(for_object=True)
+            for key, value in tags.items():
+                # normalize text, by removing all diacritics
+                doc_tags[key] = unidecode(value)
+        # store the file
+        try:
+            curr_client.fput_object(bucket_name=bucket,
+                                    object_name=f"{remotepath}",
+                                    file_path=filepath,
+                                    content_type=mimetype,
+                                    tags=doc_tags)
+            if logger:
+                logger.debug(f"Stored {remotepath}, "
+                             f"content type {mimetype}, tags {tags}, bucket {bucket}")
+        except Exception as e:
+            __minio_except_msg(errors=errors,
+                               exception=e,
+                               logger=logger)
+
+
+def minio_file_retrieve(errors: list[str],
+                        basepath: Path | str,
+                        identifier: str,
+                        filepath: Path | str,
+                        bucket: str = MINIO_BUCKET_NAME,
+                        client: Minio = None,
+                        logger: Logger = None) -> any:
+    """
+    Retrieve a file from the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to retrieve the file from
+    :param identifier: the file identifier, tipically a file name
+    :param filepath: the path to save the retrieved file at
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: information about the file retrieved
+    """
+    # initialize the return variable
+    result: any = None
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # was the MinIO client obtained ?
+    if curr_client:
+        # yes, proceed
+        remotepath: Path = Path(basepath) / identifier
+        try:
+            result = curr_client.fget_object(bucket_name=bucket,
+                                             object_name=f"{remotepath}",
+                                             file_path=filepath)
+            if logger:
+                logger.debug(f"Retrieved {remotepath}, bucket {bucket}")
+        except Exception as e:
+            if not hasattr(e, "code") or e.code != "NoSuchKey":
+                __minio_except_msg(errors=errors,
+                                   exception=e,
+                                   logger=logger)
+
+    return result
+
+
+def minio_object_exists(errors: list[str],
+                        basepath: Path | str,
+                        identifier: str = None,
+                        bucket: str = MINIO_BUCKET_NAME,
+                        client: Minio = None,
+                        logger: Logger = None) -> bool:
+    """
+    Determine if a given object exists in the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to locate the object at
+    :param identifier: the object identifier
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: True if the object was found
+    """
+    # initialize the return variable
+    result: bool = False
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # proceed, if the MinIO client eas obtained
+    if curr_client:
+        # was the identifier provided ?
+        if identifier is None:
+            # no, object is a folder
+            objs: Iterator = minio_objects_list(errors=errors,
+                                                basepath=basepath,
+                                                recursive=False,
+                                                bucket=bucket,
+                                                client=curr_client,
+                                                logger=logger)
+            for _ in objs:
+                result = True
+                break
+        # verify the status of the object
+        elif minio_object_stat(errors=errors,
+                               basepath=basepath,
+                               identifier=identifier,
+                               bucket=bucket,
+                               client=curr_client,
+                               logger=logger):
+            result = True
+        if logger:
+            remotepath: Path = Path(basepath) / identifier
+            existence: str = "exists" if result else "do not exist"
+            logger.debug(f"Object {remotepath}, bucket {bucket}, {existence}")
+
+    return result
+
+
+def minio_object_stat(errors: list[str],
+                      basepath: Path | str,
+                      identifier: str,
+                      bucket: str = MINIO_BUCKET_NAME,
+                      client: Minio = None,
+                      logger: Logger = None) -> MinioObject:
+    """
+    Retrieve and return the information about an object in the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying where to locate the object
+    :param identifier: the object identifier
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: metadata and information about the object
+    """
+    # initialize the return variable
+    result: MinioObject | None = None
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # was the MinIO client obtained ?
+    if curr_client:
+        # yes, proceed
+        remotepath: Path = Path(basepath) / identifier
+        try:
+            result = curr_client.stat_object(bucket_name=bucket,
+                                             object_name=f"{remotepath}")
+            if logger:
+                logger.debug(f"Stat'ed {remotepath}, bucket {bucket}")
+        except Exception as e:
+            if not hasattr(e, "code") or e.code != "NoSuchKey":
+                __minio_except_msg(errors=errors,
+                                   exception=e,
+                                   logger=logger)
+
+    return result
+
+
+def minio_object_store(errors: list[str],
+                       basepath: Path | str,
+                       identifier: str,
+                       obj: any,
+                       tags: dict = None,
+                       bucket: str = MINIO_BUCKET_NAME,
+                       client: Minio = None,
+                       logger: Logger = None) -> None:
+    """
+    Store an object at the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to store the object at
+    :param identifier: the object identifier
+    :param obj: object to be stored
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :param tags: optional metadata describing the object
+    """
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # proceed, if the MinIO client was obtained
+    if curr_client:
+        # serialize the object into a file
+        filepath: Path = Path(MINIO_TEMP_PATH) / f"{uuid.uuid4()}.pickle"
+        with filepath.open("wb") as f:
+            pickle.dump(obj, f)
+
+        # store the file
+        op_errors: list[str] = []
+        minio_file_store(errors=op_errors,
+                         basepath=basepath,
+                         identifier=identifier,
+                         filepath=filepath,
+                         mimetype="application/octet-stream",
+                         tags=tags,
+                         bucket=bucket,
+                         client=curr_client,
+                         logger=logger)
+
+        # errors ?
+        if op_errors:
+            # yes, report them
+            errors.extend(op_errors)
+            storage: str = "Unable to store"
+        else:
+            # no, remove the file from the file system
+            filepath.unlink()
+            storage: str = "Stored "
+
+        if logger:
+            remotepath: Path = Path(basepath) / identifier
+            logger.debug(f"{storage} {remotepath}, bucket {bucket}")
+
+
+def minio_object_retrieve(errors: list[str],
+                          basepath: Path,
+                          identifier: str,
+                          bucket: str = MINIO_BUCKET_NAME,
+                          client: Minio = None,
+                          logger: Logger = None) -> any:
+    """
+    Retrieve an object from the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to retrieve the object from
+    :param identifier: the object identifier
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: the object retrieved
+    """
+    # initialize the return variable
+    result: any = None
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # proceed, if the MinIO client was obtained
+    if curr_client:
+        # retrieve the file containg the serialized object
+        filepath: Path = Path(MINIO_TEMP_PATH) / f"{uuid.uuid4()}.pickle"
+        stat: any = minio_file_retrieve(errors=errors,
+                                        basepath=basepath,
+                                        identifier=identifier,
+                                        filepath=filepath,
+                                        bucket=bucket,
+                                        client=curr_client,
+                                        logger=logger)
+
+        # was the file retrieved ?
+        if stat:
+            # yes, umarshall the corresponding object
+            with filepath.open("rb") as f:
+                result = pickle.load(f)
+            filepath.unlink()
+
+        if logger:
+            retrieval: str = "Retrieved" if result else "Unable to retrieve"
+            remotepath: Path = Path(basepath) / identifier
+            logger.debug(f"{retrieval} {remotepath}, bucket {bucket}")
+
+    return result
+
+
+def minio_object_delete(errors: list[str],
+                        basepath: str,
+                        identifier: str = None,
+                        bucket: str = MINIO_BUCKET_NAME,
+                        client: Minio = None,
+                        logger: Logger = None) -> None:
+    """
+    Remove an object from the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to retrieve the object from
+    :param identifier: the object identifier
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    """
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # proceed, if the MinIO client was obtained
+    if curr_client:
+        # was the identifier provided ?
+        if identifier is None:
+            # no, remove the folder
+            __minio_folder_delete(errors=errors,
+                                  client=curr_client,
+                                  basepath=basepath,
+                                  bucket=bucket,
+                                  logger=logger)
+        else:
+            # yes, remove the object
+            remotepath: Path = Path(basepath) / identifier
+            try:
+                curr_client.remove_object(bucket_name=bucket,
+                                          object_name=f"{remotepath}")
+                if logger:
+                    logger.debug(f"Deleted {remotepath}, bucket {bucket}")
+            except Exception as e:
+                if not hasattr(e, "code") or e.code != "NoSuchKey":
+                    __minio_except_msg(errors=errors,
+                                       exception=e,
+                                       logger=logger)
+
+
+# recupera as tags do objeto
+def minio_object_tags_retrieve(errors: list[str],
+                               basepath: str,
+                               identifier: str,
+                               bucket: str = MINIO_BUCKET_NAME,
+                               client: Minio = None,
+                               logger: Logger = None) -> dict:
+    """
+    Retrieve and return the metadata information for an object in the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to retrieve the object from
+    :param identifier: the object identifier
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: the metadata about the object
+    """
+    # initialize the return variable
+    result: dict | None = None
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # was the MinIO client obtained ?
+    if curr_client:
+        # yes, proceed
+        remotepath: Path = Path(basepath) / identifier
+        try:
+            tags: Tags = curr_client.get_object_tags(bucket_name=bucket,
+                                                     object_name=f"{remotepath}")
+            if tags is not None and len(tags) > 0:
+                result = {}
+                for key, value in tags.items():
+                    result[key] = value
+            if logger:
+                logger.debug(f"Retrieved {remotepath}, bucket {bucket}, tags {result}")
+        except Exception as e:
+            if not hasattr(e, "code") or e.code != "NoSuchKey":
+                __minio_except_msg(errors=errors,
+                                   exception=e,
+                                   logger=logger)
+
+    return result
+
+
+def minio_objects_list(errors: list[str],
+                       basepath: str,
+                       recursive: bool = False,
+                       bucket: str = MINIO_BUCKET_NAME,
+                       client: Minio = None,
+                       logger: Logger = None) -> Iterator:
+    """
+    Retrieve and return an iterator into the list of objects at *basepath*, in the *MinIO* store.
+
+    :param errors: incidental error messages
+    :param basepath: the path specifying the location to iterate from
+    :param recursive: whether the location is iterated recursively
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param client: optional MinIO client (obtains a new one, if not provided)
+    :param logger: optional logger
+    :return: the iterator into the list of objects, 'None' if the folder does not exist
+    """
+    # initialize the return variable
+    result: Iterator | None = None
+
+    # make sure to have a MinIO client
+    curr_client: Minio = client or minio_access(errors=errors,
+                                                logger=logger)
+    # was the MinIO client obtained ?
+    if curr_client:
+        # yes, proceed
+        try:
+            result = curr_client.list_objects(bucket_name=bucket,
+                                              prefix=basepath,
+                                              recursive=recursive)
+            if logger:
+                logger.debug(f"Listed {basepath}, bucket {bucket}")
+        except Exception as e:
+            __minio_except_msg(errors=errors,
+                               exception=e,
+                               logger=logger)
+
+    return result
+
+
+def __minio_folder_delete(errors: list[str],
+                          client: Minio,
+                          basepath: str,
+                          bucket: str = MINIO_BUCKET_NAME,
+                          logger: Logger = None) -> None:
+    """
+    Traverse the folders recursively, removing its objects.
+
+    :param errors: incidental error messages
+    :param client: the MinIO client object
+    :param basepath: the path specifying the location to delete the objects at
+    :param bucket: the bucket to use (defaults to MINIO_BUCKET)
+    :param logger: optional logger
+    """
+    # obtain the list of entries in the given folder
+    objs: Iterator = minio_objects_list(errors=errors,
+                                        basepath=basepath,
+                                        recursive=True,
+                                        bucket=bucket,
+                                        logger=logger)
+    # was the list obtained ?
+    if objs:
+        # yes, proceed
+        for obj in objs:
+            try:
+                client.remove_object(bucket_name=bucket,
+                                     object_name=obj.object_name)
+            except Exception as e:
+                # SANITY CHECK: in case of concurrent exclusion
+                if not hasattr(e, "code") or e.code != "NoSuchKey":
+                    __minio_except_msg(errors=errors,
+                                       exception=e,
+                                       logger=logger)
+        if logger:
+            logger.debug(msg=f"Removed folder {basepath}, bucket {bucket}")
+
+
+def __minio_except_msg(errors: list[str],
+                       exception: Exception,
+                       logger: Logger) -> None:
+    """
+    Format and return an error message from *exception*.
+
+    :param errors: incidental error messages
+    :param exception: the reference exception
+    :param logger: optional logger
+    :return: the error message
+    """
+    # interaction with MinIO raised the exception "<class 'exception_class'>"
+    cls: str = str(exception.__class__)
+    exc_msg: str = f"{cls[7:-1]} - {exception}"
+    err_msg: str = f"Error accessing the object storer: {exc_msg}"
+    if errors:
+        errors.append(err_msg)
+    if logger:
+        logger.error(msg=err_msg)

pypomes_s3-0.3.1/src/pypomes_s3/s3_common.py

@@ -0,0 +1,145 @@
+from logging import DEBUG, Logger
+from pypomes_core import (
+    APP_PREFIX,
+    env_get_str, str_sanitize, str_get_positional
+)
+
+# the preferred way to specify S3 storage parameters is dynamically with 's3_setup_params'
+# specifying S3 storage parameters with environment variables can be done in two ways:
+# 1. specify the set
+#   {APP_PREFIX}_S3_ENGINE (one of 'aws', 'minio')
+#   {APP_PREFIX}_S3_ACCESS_KEY
+#   {APP_PREFIX}_S3_SECRET_KEY
+#   {APP_PREFIX}_S3_BUCKET_NAME
+#   {APP_PREFIX}_S3_TEMP_PATH
+#   {APP_PREFIX}_S3_REGION_NAME (for aws)
+#   {APP_PREFIX}_S3_ENDPOINT_URL (for minio)
+#   {APP_PREFIX}_S3_SECURE_ACCESS (for minio)
+# 2. alternatively, specify a comma-separated list of servers in
+#   {APP_PREFIX}_S3_ENGINES
+#   and for each engine, specify the set above, replacing 'S3' with
+#   'AWS' and 'MINIO', respectively for the engines listed above
+
+_S3_ACCESS_DATA: dict = {}
+_S3_ENGINES: list[str] = []
+if env_get_str(f"{APP_PREFIX}_S3_ENGINE",  None):
+    _default_setup: bool = True
+    _S3_ENGINES.append(env_get_str(f"{APP_PREFIX}_S3_ENGINE"))
+else:
+    _default_setup: bool = False
+    _engines: str = env_get_str(f"{APP_PREFIX}_S3_ENGINES", None)
+    if _engines:
+        _S3_ENGINES.extend(_engines.split(sep=","))
+for engine in _S3_ENGINES:
+    if _default_setup:
+        _tag = "S3"
+        _default_setup = False
+    else:
+        _tag: str = str_get_positional(source=engine,
+                                       list_origin=["aws", "minio"],
+                                       list_dest=["AWS", "MINIO"])
+    _s3_data = {
+        "access-key":  env_get_str(f"{APP_PREFIX}_{_tag}_ACESS_KEY"),
+        "secret-key": env_get_str(f"{APP_PREFIX}_{_tag}_SECRET_KEY"),
+        "bucket-name": env_get_str(f"{APP_PREFIX}_{_tag}_BUCKET_NAME"),
+        "temp-path": env_get_str(f"{APP_PREFIX}_{_tag}_TEMP_PATH")
+    }
+    if engine == "aws":
+        _s3_data["client"] = env_get_str(f"{APP_PREFIX}_{_tag}_REGION_NAME")
+    elif engine == "minio":
+        _s3_data["endpoit-url"] = env_get_str(f"{APP_PREFIX}_{_tag}_ENDPOINT_URL")
+        _s3_data["secure-access"] = env_get_str(f"{APP_PREFIX}_{_tag}_SECURE_ACCESS")
+    _S3_ACCESS_DATA[engine] = _s3_data
+
+
+def _assert_engine(errors: list[str],
+                   engine: str) -> str:
+    """
+    Verify if *engine* is in the list of supported engines.
+
+    If *engine* is a supported engine, it is returned. If its value is 'None',
+    the first engine in the list of supported engines (the default engine) is returned.
+
+    :param errors: incidental errors
+    :param engine: the reference database engine
+    :return: the validated or default engine
+    """
+    # initialize the return valiable
+    result: str | None = None
+
+    if not engine and _S3_ENGINES:
+        result = _S3_ENGINES[0]
+    elif engine in _S3_ENGINES:
+        result = engine
+    else:
+        err_msg = f"S3 engine '{engine}' unknown or not configured"
+        errors.append(err_msg)
+
+    return result
+
+
+def _s3_get_params(engine: str) -> tuple:
+    """
+    Return the current parameters being used for *engine*.
+
+    The parameters are returned as a *tuple*, with the elements
+    *access-key*, *secret-key*, *bucket-name*, *temp-path*.
+    For *aws* engines, the extra element *region-name* is returned.
+    for *minio* engines, the elements *endpoint-url* and *secure-access are returned.
+    The meaning of some parameters may vary between different S3 engines.
+
+    :param engine: the reference database engine
+    :return: the current parameters for the engine
+    """
+    access_key: str = _S3_ACCESS_DATA[engine]["access-key"]
+    secret_key: str = _S3_ACCESS_DATA[engine]["secret-key"]
+    bucket_name: str = _S3_ACCESS_DATA[engine]["bucket-name"]
+    temp_path = _S3_ACCESS_DATA[engine]["temp-path"]
+
+    result: tuple | None = None
+    if engine == "aws":
+        region_name: str = _S3_ACCESS_DATA[engine]["region-name"]
+        result = (access_key, secret_key, bucket_name, temp_path, region_name)
+    elif engine == "minio":
+        endpoint_url: str = _S3_ACCESS_DATA[engine]["endpoint-url"]
+        secure_access: bool = _S3_ACCESS_DATA[engine]["secure-access"]
+        result = (access_key, secret_key, bucket_name, temp_path, endpoint_url, secure_access)
+
+    return result
+
+
+def _db_except_msg(exception: Exception,
+                   engine: str) -> str:
+    """
+    Format and return the error message corresponding to the exception raised while accessing the database.
+
+    :param exception: the exception raised
+    :param engine: the reference database engine
+    :return: the formatted error message
+    """
+    endpoint: str = _S3_ACCESS_DATA[engine].get("region-name") if engine == "aws" else \
+                    _S3_ACCESS_DATA[engine].get("endpoint-url")
+    return f"Error accessing '{engine}' at '{endpoint}': {str_sanitize(f'{exception}')}"
+
+
+def _s3_log(logger: Logger,
+            err_msg: str = None,
+            level: int = DEBUG,
+            errors: list[str] = None,
+            stmt: str = None) -> None:
+    """
+    Log *err_msg* and add it to *errors*, or else log *stmt*, whichever is applicable.
+
+    :param logger: the logger object
+    :param err_msg: the error message to log
+    :param level: log level (defaults to DEBUG)
+    :param errors: optional incidental errors
+    :param stmt: optional statement
+    """
+    if err_msg:
+        if logger:
+            logger.log(level, err_msg)
+        if isinstance(errors, list):
+            errors.append(err_msg)
+    if logger and stmt:
+        logger.log(level, stmt)

pypomes_s3-0.3.1/src/pypomes_s3/s3_pomes.py

@@ -0,0 +1,89 @@
+from logging import Logger
+from pathlib import Path
+from typing import Any
+
+from .s3_common import (
+    _S3_ENGINES, _S3_ACCESS_DATA, _assert_engine
+)
+
+
+def db_setup(engine: str,
+             access_key: str,
+             access_secret: str,
+             bucket_name: str,
+             temp_path: str | Path,
+             region_name: str = None,
+             endpoint_url: str = None,
+             secure_access: bool = None) -> bool:
+    """
+    Establish the provided parameters for access to *engine*.
+
+    The meaning of some parameters may vary between different database engines.
+    All parameters, with the exception of *db_client* and *db_driver*, are required.
+    *db_client* may be provided for *oracle*, but not for the other engines.
+    *db_driver* is required for *sqlserver*, but is not accepted for the other engines.
+
+    :param engine: the S3 engine (one of [aws, minio])
+    :param access_key: the access key for the service
+    :param access_secret: the access secret code
+    :param bucket_name: the name of the default bucket
+    :param temp_path: path for temporary files
+    :param region_name: the name of the region where the engine is located (AWS only)
+    :param endpoint_url: the access URL for the service (MinIO only)
+    :param secure_access: whether or not to use Transport Security Layer (MinIO only)
+    :return: True if the data was accepted, False otherwise
+    """
+    # initialize the return variable
+    result: bool = False
+
+    # are the parameters compliant ?
+    if (engine in ["aws", "minio"] and
+        access_key and access_secret and bucket_name and temp_path and
+        not (engine != "aws" and region_name) and
+        not (engine == "aws" and not region_name) and
+        not (engine != "minio" and endpoint_url) and
+        not (engine == "minio" and not endpoint_url) and
+        not (engine != "minio" and secure_access is not None) and
+        not (engine == "minio" and secure_access is None)):
+        _S3_ACCESS_DATA[engine] = {
+            "access-key": access_key,
+            "access-secret": access_secret,
+            "bucket-name": bucket_name,
+            "temp-path": temp_path
+        }
+        if engine == "aws":
+            _S3_ACCESS_DATA[engine]["region-name"] = region_name
+        elif engine == "minio":
+            _S3_ACCESS_DATA[engine]["endpoint-url"] = endpoint_url
+            _S3_ACCESS_DATA[engine]["secure-access"] = secure_access
+        if engine not in _S3_ENGINES:
+            _S3_ENGINES.append(engine)
+        result = True
+
+    return result
+
+
+def s3_get_engines() -> list[str]:
+    """
+    Retrieve and return the list of configured engines.
+
+    This list may include any of the supported engines:
+     *aws*, *minio*.
+
+    :return: the list of configured engines
+    """
+    return _S3_ENGINES
+
+
+def db_get_params(engine: str = None) -> dict:
+    """
+    Return the connection parameters a *dict*.
+
+    The returned *dict* contains the keys *name*, *user*, *pwd*, *host*, *port*.
+    The meaning of these parameters may vary between different database engines.
+
+    :param engine: the database engine
+    :return: the current connection parameters for the engine
+    """
+    curr_engine: str = _S3_ENGINES[0] if not engine and _S3_ENGINES else engine
+    return _S3_ACCESS_DATA.get(engine or curr_engine)