PyS3Uploader 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

{pys3uploader-0.2.1.dist-info → pys3uploader-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.2.1
+Version: 0.2.3
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -40,6 +40,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3==1.40.*
+Requires-Dist: python-dotenv==1.1.*
 Requires-Dist: tqdm==4.67.*
 Provides-Extra: dev
 Requires-Dist: sphinx==5.1.1; extra == "dev"
@@ -90,7 +91,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
+        exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run_in_parallel()
 ```
@@ -103,7 +104,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
+        exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run()
 ```
@@ -114,10 +115,13 @@ if __name__ == '__main__':
 
 #### Optional kwargs
 - **s3_prefix** - S3 object prefix for each file. Defaults to ``None``
-- **exclude_path** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
+- **exclude_prefix** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
 - **skip_dot_files** - Boolean flag to skip dot files. Defaults to ``True``
 - **overwrite** - Boolean flag to overwrite files present in S3. Defaults to ``False``
+- **file_exclusion** - Sequence of files to exclude during upload. Defaults to ``None``
+- **folder_exclusion** - Sequence of directories to exclude during upload. Defaults to ``None``
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
+- **env_file** – Path to a `.env` file for loading environment variables. Defaults to scanning the current directory.
 <br><br>
 - **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION`
 - **profile_name** - AWS profile name. Defaults to the env var `PROFILE_NAME`

pys3uploader-0.2.3.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3/__init__.py,sha256=aZ2woJ8TD2tgqXi0ElG-wWwJWoQLIdqTdm50FLaxL8w,66
+s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
+s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+s3/uploader.py,sha256=KxrWbIInXxXQszP_uJLf_dBI5rUNjNnhco3gr9Vdrto,13767
+s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
+pys3uploader-0.2.3.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.2.3.dist-info/METADATA,sha256=ae2lA8b7dsGWZSMfB8w4joDiAlaE6Wk1f3p1Fxywkc4,7795
+pys3uploader-0.2.3.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.2.3.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
+pys3uploader-0.2.3.dist-info/RECORD,,

s3/__init__.py

@@ -1,3 +1,3 @@
 from s3.uploader import Uploader  # noqa: F401
 
-version = "0.2.1"
+version = "0.2.3"

s3/uploader.py

@@ -2,9 +2,10 @@ import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Dict
+from typing import Dict, Iterable
 
 import boto3.resources.factory
+import dotenv
 from botocore.config import Config
 from botocore.exceptions import ClientError
 from tqdm import tqdm
@@ -14,6 +15,7 @@ from s3.logger import default_logger
 from s3.utils import (
     RETRY_CONFIG,
     UploadResults,
+    convert_seconds,
     convert_to_folder_structure,
     getenv,
     urljoin,
@@ -32,15 +34,18 @@ class Uploader:
         bucket_name: str,
         upload_dir: str,
         s3_prefix: str = None,
-        exclude_path: str = None,
+        exclude_prefix: str = None,
         skip_dot_files: bool = True,
         overwrite: bool = False,
+        file_exclusion: Iterable[str] = None,
+        folder_exclusion: Iterable[str] = None,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
         aws_secret_access_key: str = None,
         retry_config: Config = RETRY_CONFIG,
         logger: logging.Logger = None,
+        env_file: str = None,
     ):
         """Initiates all the necessary args and creates a boto3 session with retry logic.
 
@@ -48,14 +53,17 @@ class Uploader:
             bucket_name: Name of the bucket.
             upload_dir: Full path of the directory to be uploaded.
             s3_prefix: Particular bucket prefix within which the upload should happen.
-            exclude_path: Full directory path to exclude from S3 object prefix.
+            exclude_prefix: Full directory path to exclude from S3 object prefix.
             skip_dot_files: Boolean flag to skip dot files.
             overwrite: Boolean flag to overwrite files in S3.
+            file_exclusion: Sequence of files to exclude during upload.
+            folder_exclusion: Sequence of directories to exclude during upload.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
             aws_secret_access_key: AWS secret access key.
             logger: Bring your own logger.
+            env_file: Dotenv file (.env) filepath to load environment variables.
 
         See Also:
             s3_prefix:
@@ -64,29 +72,56 @@ class Uploader:
                 If ``s3_prefix`` is set to: ``2025``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub/photo.jpg`` will be uploaded as ``2025/S3Upload/sub/photo.jpg``
 
-            exclude_path:
+            exclude_prefix:
                 When upload directory is "/home/ubuntu/Desktop/S3Upload", each file will naturally have the full prefix.
-                However, this behavior can be avoided by specifying the ``exclude_path`` parameter.
+                However, this behavior can be avoided by specifying the ``exclude_prefix`` parameter.
 
-                If exclude_path is set to: ``/home/ubuntu/Desktop``, then the file path
+                If exclude_prefix is set to: ``/home/ubuntu/Desktop``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub-dir/photo.jpg`` will be uploaded as ``S3Upload/sub-dir/photo.jpg``
+
+            env_file:
+                Environment variables can be loaded from a .env file.
+                The filepath can be set as ``env_file`` during object instantiation or as an environment variable.
+                If a filepath is provided, PyS3Uploader loads it directly or searches the root directory for the file.
+                If no filepath is provided, PyS3Uploader searches the current directory for a .env file.
         """
+        self.logger = logger or default_logger()
+        self.env_file = env_file or getenv("ENV_FILE", default=".env")
+
+        # Check for env_file in current working directory
+        if os.path.isfile(self.env_file):
+            self.logger.debug("Loading env file: %s", self.env_file)
+            dotenv.load_dotenv(dotenv_path=self.env_file, override=True)
+        # Find the env_file from root
+        elif env_file := dotenv.find_dotenv(self.env_file, raise_error_if_not_found=False):
+            self.logger.debug("Loading env file: %s", env_file)
+            dotenv.load_dotenv(dotenv_path=env_file, override=True)
+        else:
+            # Scan current working directory for any .env files
+            for file in os.listdir():
+                if file.endswith(".env"):
+                    self.logger.debug("Loading env file: %s", file)
+                    dotenv.load_dotenv(dotenv_path=file, override=True)
+                    break
+            else:
+                self.logger.debug("No .env files found to load")
+
         self.session = boto3.Session(
-            profile_name=profile_name or getenv("PROFILE_NAME"),
+            profile_name=profile_name or getenv("PROFILE_NAME", "AWS_PROFILE_NAME"),
             region_name=region_name or getenv("AWS_DEFAULT_REGION"),
             aws_access_key_id=aws_access_key_id or getenv("AWS_ACCESS_KEY_ID"),
             aws_secret_access_key=aws_secret_access_key or getenv("AWS_SECRET_ACCESS_KEY"),
         )
         self.s3 = self.session.resource(service_name="s3", config=retry_config)
 
-        self.logger = logger or default_logger()
-
         self.bucket_name = bucket_name
-        self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "UPLOAD_SOURCE")
+        self.upload_dir = upload_dir
         self.s3_prefix = s3_prefix
-        self.exclude_path = exclude_path
+        self.exclude_prefix = exclude_prefix
         self.skip_dot_files = skip_dot_files
         self.overwrite = overwrite
+        self.file_exclusion = file_exclusion or []
+        self.folder_exclusion = folder_exclusion or []
 
         self.results = UploadResults()
         self.start = time.time()
@@ -105,9 +140,9 @@ class Uploader:
             BucketNotFound: If bucket name was not found.
         """
         self.start = time.time()
-        if self.exclude_path and self.exclude_path not in self.upload_dir:
+        if self.exclude_prefix and self.exclude_prefix not in self.upload_dir:
             raise ValueError(
-                f"\n\n\tStart folder {self.exclude_path!r} is not a part of upload directory {self.upload_dir!r}"
+                f"\n\n\tStart folder {self.exclude_prefix!r} is not a part of upload directory {self.upload_dir!r}"
             )
         if not self.upload_dir:
             raise ValueError("\n\n\tCannot proceed without an upload directory.")
@@ -134,7 +169,7 @@ class Uploader:
         self.logger.info(
             "Total number of uploads: %d, success: %d, failed: %d", total, self.results.success, self.results.failed
         )
-        self.logger.info("Run Time: %.2fs", time.time() - self.start)
+        self.logger.info("Run time: %s", convert_seconds(time.time() - self.start))
 
     def _proceed_to_upload(self, filepath: str, objectpath: str) -> bool:
         """Compares file size if the object already exists in S3.
@@ -183,13 +218,20 @@ class Uploader:
         """
         files_to_upload = {}
         for __path, __directory, __files in os.walk(self.upload_dir):
+            scan_dir = os.path.split(__path)[-1]
+            if scan_dir in self.folder_exclusion:
+                self.logger.info("Skipping '%s' honoring folder exclusion", scan_dir)
+                continue
             for file_ in __files:
+                if file_ in self.file_exclusion:
+                    self.logger.info("Skipping '%s' honoring file exclusion", file_)
+                    continue
                 if self.skip_dot_files and file_.startswith("."):
                     self.logger.info("Skipping dot file: %s", file_)
                     continue
                 file_path = os.path.join(__path, file_)
-                if self.exclude_path:
-                    relative_path = file_path.replace(self.exclude_path, "")
+                if self.exclude_prefix:
+                    relative_path = file_path.replace(self.exclude_prefix, "")
                 else:
                     relative_path = file_path
                 # Lists in python are ordered, so s3 prefix will get loaded first when provided
@@ -212,7 +254,7 @@ class Uploader:
         self.logger.debug(keys)
         self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
         self.logger.info("Initiating upload process.")
-        for objectpath, filepath in tqdm(
+        for filepath, objectpath in tqdm(
             keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
         ):
             try:

s3/utils.py

@@ -3,18 +3,6 @@ from typing import Dict, Set
 
 from botocore.config import Config
 
-
-class UploadResults(dict):
-    """Object to store results of S3 upload.
-
-    >>> UploadResults
-
-    """
-
-    success: int = 0
-    failed: int = 0
-
-
 RETRY_CONFIG: Config = Config(
     retries={
         "max_attempts": 10,
@@ -27,6 +15,17 @@ RETRY_CONFIG: Config = Config(
 )
 
 
+class UploadResults(dict):
+    """Object to store results of S3 upload.
+
+    >>> UploadResults
+
+    """
+
+    success: int = 0
+    failed: int = 0
+
+
 def getenv(*args, default: str = None) -> str:
     """Returns the key-ed environment variable or the default value."""
     for key in args:
@@ -82,3 +81,67 @@ def convert_to_folder_structure(sequence: Set[str]) -> str:
         return result
 
     return generate_folder_structure(folder_structure)
+
+
+def convert_seconds(seconds: int | float, n_elem: int = 2) -> str:
+    """Calculate years, months, days, hours, minutes, seconds, and milliseconds from given input.
+
+    Args:
+        seconds: Number of seconds to convert (supports float values).
+        n_elem: Number of elements required from the converted list.
+
+    Returns:
+        str:
+        Returns a humanized string notion of the number of seconds.
+    """
+    if not seconds:
+        return "0s"
+    elif seconds < 1:
+        return f"{seconds * 1000:.0f}ms"
+
+    seconds_in_year = 365 * 24 * 3600
+    seconds_in_month = 30 * 24 * 3600
+
+    years = seconds // seconds_in_year
+    seconds %= seconds_in_year
+
+    months = seconds // seconds_in_month
+    seconds %= seconds_in_month
+
+    days = seconds // (24 * 3600)
+    seconds %= 24 * 3600
+
+    hours = seconds // 3600
+    seconds %= 3600
+
+    minutes = seconds // 60
+    seconds %= 60
+
+    milliseconds = round((seconds % 1) * 1000)
+    seconds = int(seconds)  # Convert remaining seconds to int for display
+
+    time_parts = []
+
+    if years > 0:
+        time_parts.append(f"{int(years)} year{'s' if years > 1 else ''}")
+    if months > 0:
+        time_parts.append(f"{int(months)} month{'s' if months > 1 else ''}")
+    if days > 0:
+        time_parts.append(f"{int(days)} day{'s' if days > 1 else ''}")
+    if hours > 0:
+        time_parts.append(f"{int(hours)} hour{'s' if hours > 1 else ''}")
+    if minutes > 0:
+        time_parts.append(f"{int(minutes)} minute{'s' if minutes > 1 else ''}")
+    if seconds > 0 or milliseconds > 0:
+        if seconds > 0 and milliseconds > 0:
+            time_parts.append(f"{seconds + milliseconds / 1000:.1f}s")
+        elif seconds > 0:
+            time_parts.append(f"{seconds}s")
+        else:
+            time_parts.append(f"{milliseconds}ms")
+
+    if len(time_parts) == 1:
+        return time_parts[0]
+
+    list_ = time_parts[:n_elem]
+    return ", and ".join([", ".join(list_[:-1]), list_[-1]] if len(list_) > 2 else list_)

pys3uploader-0.2.1.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=IqcPR9iWMw0GDBEmKvLzW7P-AhInTkwRklkvYgiT1Xc,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=v6TGdm8EcAldoIw3GOAwkuoUzi9b9IRj8b94hl1Pkyw,11454
-s3/utils.py,sha256=dd1OeLbswLzFVyjYiXixkJlFsoGWRtRCOHha6wLG5zQ,2485
-pys3uploader-0.2.1.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.2.1.dist-info/METADATA,sha256=NA7x6YqpWEKvn3XEYzVug7XTl1vPcwiphUzWtMvlzHE,7449
-pys3uploader-0.2.1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.2.1.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.2.1.dist-info/RECORD,,