PyS3Uploader 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

{pys3uploader-0.2.2.dist-info → pys3uploader-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.2.2
+Version: 0.2.4
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -40,6 +40,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3==1.40.*
+Requires-Dist: python-dotenv==1.1.*
 Requires-Dist: tqdm==4.67.*
 Provides-Extra: dev
 Requires-Dist: sphinx==5.1.1; extra == "dev"
@@ -90,7 +91,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
+        exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run_in_parallel()
 ```
@@ -103,7 +104,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
+        exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run()
 ```
@@ -114,10 +115,15 @@ if __name__ == '__main__':
 
 #### Optional kwargs
 - **s3_prefix** - S3 object prefix for each file. Defaults to ``None``
-- **exclude_path** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
+- **exclude_prefix** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
 - **skip_dot_files** - Boolean flag to skip dot files. Defaults to ``True``
 - **overwrite** - Boolean flag to overwrite files present in S3. Defaults to ``False``
+- **file_exclusion** - Sequence of files to exclude during upload. Defaults to ``None``
+- **folder_exclusion** - Sequence of directories to exclude during upload. Defaults to ``None``
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
+- **log_handler** - Choose between `stdout` vs `file` logging. Defaults to `s3.LogHandler.stdout`
+- **log_level** - Choose the logging level. Defaults to `s3.LogLevel.debug`
+- **env_file** – Path to a `.env` file for loading environment variables. Defaults to scanning the current directory.
 <br><br>
 - **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION`
 - **profile_name** - AWS profile name. Defaults to the env var `PROFILE_NAME`

pys3uploader-0.2.4.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3/__init__.py,sha256=YsBU1Xy4sLbm_8jU5kKP8QP3ayKBmnJDaF3NCNaDOsk,66
+s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+s3/logger.py,sha256=igwMubdTQ_GrMkwie5DAIvmxIcgj6a9UA_EGFrwFYiQ,2571
+s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+s3/uploader.py,sha256=S480dteogtnA3xWVkHb2aRr5gtL1WnRIukw3_aAX1z0,14228
+s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
+pys3uploader-0.2.4.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.2.4.dist-info/METADATA,sha256=Dd-gyQ387bU7NcVC7Y0aqAeyXUzxlW1nsMWbMvfRDLg,7969
+pys3uploader-0.2.4.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.2.4.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
+pys3uploader-0.2.4.dist-info/RECORD,,

s3/__init__.py

@@ -1,3 +1,3 @@
 from s3.uploader import Uploader  # noqa: F401
 
-version = "0.2.2"
+version = "0.2.4"

s3/logger.py

@@ -5,9 +5,46 @@
 """
 
 import logging
+import os
+from datetime import datetime
+from enum import IntEnum, StrEnum
 
 
-def default_handler() -> logging.StreamHandler:
+class LogHandler(StrEnum):
+    """Logging handlers to choose from when default logger is used.
+
+    >>> LogHandler
+
+    """
+
+    file = "file"
+    stdout = "stdout"
+
+
+class LogLevel(IntEnum):
+    """Logging levels to choose from when default logger is used.
+
+    >>> LogLevel
+
+    """
+
+    debug = logging.DEBUG
+    info = logging.INFO
+    warning = logging.WARNING
+    error = logging.ERROR
+
+    @classmethod
+    def _missing_(cls, value):
+        """Allow constructing from string names."""
+        if isinstance(value, str):
+            value = value.lower()
+            for member in cls:
+                if member.name == value:
+                    return member
+        return None
+
+
+def stream_handler() -> logging.StreamHandler:
     """Creates a ``StreamHandler`` and assigns a default format to it.
 
     Returns:
@@ -19,6 +56,20 @@ def default_handler() -> logging.StreamHandler:
     return handler
 
 
+def file_handler() -> logging.FileHandler:
+    """Creates a ``StreamHandler`` and assigns a default format to it.
+
+    Returns:
+        logging.StreamHandler:
+        Returns an instance of the ``StreamHandler`` object.
+    """
+    os.makedirs("logs", exist_ok=True)
+    filename = os.path.join("logs", datetime.now().strftime("PyS3Uploader_%d-%m-%Y_%H:%M.log"))
+    handler = logging.FileHandler(filename, mode="a")
+    handler.setFormatter(fmt=default_format())
+    return handler
+
+
 def default_format() -> logging.Formatter:
     """Creates a logging ``Formatter`` with a custom message and datetime format.
 
@@ -32,7 +83,7 @@ def default_format() -> logging.Formatter:
     )
 
 
-def default_logger() -> logging.Logger:
+def setup_logger(handler: LogHandler, level: LogLevel):
     """Creates a default logger with debug mode enabled.
 
     Returns:
@@ -40,6 +91,10 @@ def default_logger() -> logging.Logger:
         Returns an instance of the ``Logger`` object.
     """
     logger = logging.getLogger(__name__)
-    logger.addHandler(hdlr=default_handler())
-    logger.setLevel(level=logging.DEBUG)
+    if handler == LogHandler.file:
+        logger.addHandler(hdlr=file_handler())
+    elif handler == LogHandler.stdout:
+        logger.addHandler(hdlr=stream_handler())
+
+    logger.setLevel(level)
     return logger

s3/uploader.py

@@ -2,15 +2,16 @@ import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Dict
+from typing import Dict, Iterable
 
 import boto3.resources.factory
+import dotenv
 from botocore.config import Config
 from botocore.exceptions import ClientError
 from tqdm import tqdm
 
 from s3.exceptions import BucketNotFound
-from s3.logger import default_logger
+from s3.logger import LogHandler, LogLevel, setup_logger
 from s3.utils import (
     RETRY_CONFIG,
     UploadResults,
@@ -33,15 +34,20 @@ class Uploader:
         bucket_name: str,
         upload_dir: str,
         s3_prefix: str = None,
-        exclude_path: str = None,
+        exclude_prefix: str = None,
         skip_dot_files: bool = True,
         overwrite: bool = False,
+        file_exclusion: Iterable[str] = None,
+        folder_exclusion: Iterable[str] = None,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
         aws_secret_access_key: str = None,
         retry_config: Config = RETRY_CONFIG,
         logger: logging.Logger = None,
+        log_handler: LogHandler = LogHandler.stdout,
+        log_level: LogLevel = LogLevel.debug,
+        env_file: str = None,
     ):
         """Initiates all the necessary args and creates a boto3 session with retry logic.
 
@@ -49,14 +55,19 @@ class Uploader:
             bucket_name: Name of the bucket.
             upload_dir: Full path of the directory to be uploaded.
             s3_prefix: Particular bucket prefix within which the upload should happen.
-            exclude_path: Full directory path to exclude from S3 object prefix.
+            exclude_prefix: Full directory path to exclude from S3 object prefix.
             skip_dot_files: Boolean flag to skip dot files.
             overwrite: Boolean flag to overwrite files in S3.
+            file_exclusion: Sequence of files to exclude during upload.
+            folder_exclusion: Sequence of directories to exclude during upload.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
             aws_secret_access_key: AWS secret access key.
             logger: Bring your own logger.
+            log_handler: Default log handler, can be ``file`` or ``stdout``.
+            log_level: Default log level, can be ``debug``, ``info``, ``warning`` or ``error``.
+            env_file: Dotenv file (.env) filepath to load environment variables.
 
         See Also:
             s3_prefix:
@@ -65,29 +76,56 @@ class Uploader:
                 If ``s3_prefix`` is set to: ``2025``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub/photo.jpg`` will be uploaded as ``2025/S3Upload/sub/photo.jpg``
 
-            exclude_path:
+            exclude_prefix:
                 When upload directory is "/home/ubuntu/Desktop/S3Upload", each file will naturally have the full prefix.
-                However, this behavior can be avoided by specifying the ``exclude_path`` parameter.
+                However, this behavior can be avoided by specifying the ``exclude_prefix`` parameter.
 
-                If exclude_path is set to: ``/home/ubuntu/Desktop``, then the file path
+                If exclude_prefix is set to: ``/home/ubuntu/Desktop``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub-dir/photo.jpg`` will be uploaded as ``S3Upload/sub-dir/photo.jpg``
+
+            env_file:
+                Environment variables can be loaded from a .env file.
+                The filepath can be set as ``env_file`` during object instantiation or as an environment variable.
+                If a filepath is provided, PyS3Uploader loads it directly or searches the root directory for the file.
+                If no filepath is provided, PyS3Uploader searches the current directory for a .env file.
         """
+        self.logger = logger or setup_logger(handler=LogHandler(log_handler), level=LogLevel(log_level))
+        self.env_file = env_file or getenv("ENV_FILE", default=".env")
+
+        # Check for env_file in current working directory
+        if os.path.isfile(self.env_file):
+            self.logger.debug("Loading env file: %s", self.env_file)
+            dotenv.load_dotenv(dotenv_path=self.env_file, override=True)
+        # Find the env_file from root
+        elif env_file := dotenv.find_dotenv(self.env_file, raise_error_if_not_found=False):
+            self.logger.debug("Loading env file: %s", env_file)
+            dotenv.load_dotenv(dotenv_path=env_file, override=True)
+        else:
+            # Scan current working directory for any .env files
+            for file in os.listdir():
+                if file.endswith(".env"):
+                    self.logger.debug("Loading env file: %s", file)
+                    dotenv.load_dotenv(dotenv_path=file, override=True)
+                    break
+            else:
+                self.logger.debug("No .env files found to load")
+
         self.session = boto3.Session(
-            profile_name=profile_name or getenv("PROFILE_NAME"),
+            profile_name=profile_name or getenv("PROFILE_NAME", "AWS_PROFILE_NAME"),
             region_name=region_name or getenv("AWS_DEFAULT_REGION"),
             aws_access_key_id=aws_access_key_id or getenv("AWS_ACCESS_KEY_ID"),
             aws_secret_access_key=aws_secret_access_key or getenv("AWS_SECRET_ACCESS_KEY"),
         )
         self.s3 = self.session.resource(service_name="s3", config=retry_config)
 
-        self.logger = logger or default_logger()
-
         self.bucket_name = bucket_name
-        self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "UPLOAD_SOURCE")
+        self.upload_dir = upload_dir
         self.s3_prefix = s3_prefix
-        self.exclude_path = exclude_path
+        self.exclude_prefix = exclude_prefix
         self.skip_dot_files = skip_dot_files
         self.overwrite = overwrite
+        self.file_exclusion = file_exclusion or []
+        self.folder_exclusion = folder_exclusion or []
 
         self.results = UploadResults()
         self.start = time.time()
@@ -106,9 +144,9 @@ class Uploader:
             BucketNotFound: If bucket name was not found.
         """
         self.start = time.time()
-        if self.exclude_path and self.exclude_path not in self.upload_dir:
+        if self.exclude_prefix and self.exclude_prefix not in self.upload_dir:
             raise ValueError(
-                f"\n\n\tStart folder {self.exclude_path!r} is not a part of upload directory {self.upload_dir!r}"
+                f"\n\n\tStart folder {self.exclude_prefix!r} is not a part of upload directory {self.upload_dir!r}"
             )
         if not self.upload_dir:
             raise ValueError("\n\n\tCannot proceed without an upload directory.")
@@ -150,19 +188,21 @@ class Uploader:
         """
         if self.overwrite:
             return True
+        try:
+            file_size = os.path.getsize(filepath)
+        except (OSError, PermissionError) as error:
+            self.logger.error(error)
+            file_size = 0
         # Indicates that the object path already exists in S3
         if object_size := self.object_size_map.get(objectpath):
-            try:
-                file_size = os.path.getsize(filepath)
-            except (OSError, PermissionError) as error:
-                self.logger.error(error)
-                return True
             if object_size == file_size:
                 self.logger.info("S3 object %s exists, and size [%d] matches, skipping..", objectpath, object_size)
                 return False
             self.logger.info(
                 "S3 object %s exists, but size mismatch. Local: [%d], S3: [%d]", objectpath, file_size, object_size
             )
+        else:
+            self.logger.debug("S3 object '%s' of size [%d bytes] doesn't exist, uploading..", objectpath, file_size)
         return True
 
     def _uploader(self, filepath: str, objectpath: str) -> None:
@@ -184,13 +224,20 @@ class Uploader:
         """
         files_to_upload = {}
         for __path, __directory, __files in os.walk(self.upload_dir):
+            scan_dir = os.path.split(__path)[-1]
+            if scan_dir in self.folder_exclusion:
+                self.logger.info("Skipping '%s' honoring folder exclusion", scan_dir)
+                continue
             for file_ in __files:
+                if file_ in self.file_exclusion:
+                    self.logger.info("Skipping '%s' honoring file exclusion", file_)
+                    continue
                 if self.skip_dot_files and file_.startswith("."):
                     self.logger.info("Skipping dot file: %s", file_)
                     continue
                 file_path = os.path.join(__path, file_)
-                if self.exclude_path:
-                    relative_path = file_path.replace(self.exclude_path, "")
+                if self.exclude_prefix:
+                    relative_path = file_path.replace(self.exclude_prefix, "")
                 else:
                     relative_path = file_path
                 # Lists in python are ordered, so s3 prefix will get loaded first when provided
@@ -213,7 +260,7 @@ class Uploader:
         self.logger.debug(keys)
         self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
         self.logger.info("Initiating upload process.")
-        for objectpath, filepath in tqdm(
+        for filepath, objectpath in tqdm(
             keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
         ):
             try:

pys3uploader-0.2.2.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=L12aFBb0plj8WISe0_He1vvQ55aOKd8VCyMlj_2LqrQ,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=I2An6Ix0rFMlvDLtLaDQ6F-YrN70IDCNFgh9E32cXHA,11489
-s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
-pys3uploader-0.2.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.2.2.dist-info/METADATA,sha256=fIlyxO6dFHYH3uhFg4yZa7RtGGBKqaOynAHlrbuffoY,7449
-pys3uploader-0.2.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.2.2.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.2.2.dist-info/RECORD,,