PyS3Uploader 0.2.2__py3-none-any.whl → 0.2.4a0__py3-none-any.whl

{pys3uploader-0.2.2.dist-info → pys3uploader-0.2.4a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.2.2
+Version: 0.2.4a0
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -40,6 +40,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3==1.40.*
+Requires-Dist: python-dotenv==1.1.*
 Requires-Dist: tqdm==4.67.*
 Provides-Extra: dev
 Requires-Dist: sphinx==5.1.1; extra == "dev"
@@ -90,7 +91,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
+        exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run_in_parallel()
 ```
@@ -103,7 +104,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
+        exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run()
 ```
@@ -114,10 +115,13 @@ if __name__ == '__main__':
 
 #### Optional kwargs
 - **s3_prefix** - S3 object prefix for each file. Defaults to ``None``
-- **exclude_path** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
+- **exclude_prefix** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
 - **skip_dot_files** - Boolean flag to skip dot files. Defaults to ``True``
 - **overwrite** - Boolean flag to overwrite files present in S3. Defaults to ``False``
+- **file_exclusion** - Sequence of files to exclude during upload. Defaults to ``None``
+- **folder_exclusion** - Sequence of directories to exclude during upload. Defaults to ``None``
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
+- **env_file** – Path to a `.env` file for loading environment variables. Defaults to scanning the current directory.
 <br><br>
 - **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION`
 - **profile_name** - AWS profile name. Defaults to the env var `PROFILE_NAME`

pys3uploader-0.2.4a0.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3/__init__.py,sha256=qPNHkWhXCMoofZe0kidyKkQ8YX3Ke0U35ViBo-oEWp0,68
+s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
+s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+s3/uploader.py,sha256=VZOajOlWLWU-E34qS40YH8VUH_RCoiVvIOUP3kSzyMA,13868
+s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
+pys3uploader-0.2.4a0.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.2.4a0.dist-info/METADATA,sha256=LCV-hgBLudyY3q9dq5E2sg63OnapaoEYv2ggUqDsc7I,7797
+pys3uploader-0.2.4a0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.2.4a0.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
+pys3uploader-0.2.4a0.dist-info/RECORD,,

s3/__init__.py

@@ -1,3 +1,3 @@
 from s3.uploader import Uploader  # noqa: F401
 
-version = "0.2.2"
+version = "0.2.4a0"

s3/uploader.py

@@ -2,9 +2,10 @@ import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Dict
+from typing import Dict, Iterable
 
 import boto3.resources.factory
+import dotenv
 from botocore.config import Config
 from botocore.exceptions import ClientError
 from tqdm import tqdm
@@ -33,15 +34,18 @@ class Uploader:
         bucket_name: str,
         upload_dir: str,
         s3_prefix: str = None,
-        exclude_path: str = None,
+        exclude_prefix: str = None,
         skip_dot_files: bool = True,
         overwrite: bool = False,
+        file_exclusion: Iterable[str] = None,
+        folder_exclusion: Iterable[str] = None,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
         aws_secret_access_key: str = None,
         retry_config: Config = RETRY_CONFIG,
         logger: logging.Logger = None,
+        env_file: str = None,
     ):
         """Initiates all the necessary args and creates a boto3 session with retry logic.
 
@@ -49,14 +53,17 @@ class Uploader:
             bucket_name: Name of the bucket.
             upload_dir: Full path of the directory to be uploaded.
             s3_prefix: Particular bucket prefix within which the upload should happen.
-            exclude_path: Full directory path to exclude from S3 object prefix.
+            exclude_prefix: Full directory path to exclude from S3 object prefix.
             skip_dot_files: Boolean flag to skip dot files.
             overwrite: Boolean flag to overwrite files in S3.
+            file_exclusion: Sequence of files to exclude during upload.
+            folder_exclusion: Sequence of directories to exclude during upload.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
             aws_secret_access_key: AWS secret access key.
             logger: Bring your own logger.
+            env_file: Dotenv file (.env) filepath to load environment variables.
 
         See Also:
             s3_prefix:
@@ -65,29 +72,56 @@ class Uploader:
                 If ``s3_prefix`` is set to: ``2025``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub/photo.jpg`` will be uploaded as ``2025/S3Upload/sub/photo.jpg``
 
-            exclude_path:
+            exclude_prefix:
                 When upload directory is "/home/ubuntu/Desktop/S3Upload", each file will naturally have the full prefix.
-                However, this behavior can be avoided by specifying the ``exclude_path`` parameter.
+                However, this behavior can be avoided by specifying the ``exclude_prefix`` parameter.
 
-                If exclude_path is set to: ``/home/ubuntu/Desktop``, then the file path
+                If exclude_prefix is set to: ``/home/ubuntu/Desktop``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub-dir/photo.jpg`` will be uploaded as ``S3Upload/sub-dir/photo.jpg``
+
+            env_file:
+                Environment variables can be loaded from a .env file.
+                The filepath can be set as ``env_file`` during object instantiation or as an environment variable.
+                If a filepath is provided, PyS3Uploader loads it directly or searches the root directory for the file.
+                If no filepath is provided, PyS3Uploader searches the current directory for a .env file.
         """
+        self.logger = logger or default_logger()
+        self.env_file = env_file or getenv("ENV_FILE", default=".env")
+
+        # Check for env_file in current working directory
+        if os.path.isfile(self.env_file):
+            self.logger.debug("Loading env file: %s", self.env_file)
+            dotenv.load_dotenv(dotenv_path=self.env_file, override=True)
+        # Find the env_file from root
+        elif env_file := dotenv.find_dotenv(self.env_file, raise_error_if_not_found=False):
+            self.logger.debug("Loading env file: %s", env_file)
+            dotenv.load_dotenv(dotenv_path=env_file, override=True)
+        else:
+            # Scan current working directory for any .env files
+            for file in os.listdir():
+                if file.endswith(".env"):
+                    self.logger.debug("Loading env file: %s", file)
+                    dotenv.load_dotenv(dotenv_path=file, override=True)
+                    break
+            else:
+                self.logger.debug("No .env files found to load")
+
         self.session = boto3.Session(
-            profile_name=profile_name or getenv("PROFILE_NAME"),
+            profile_name=profile_name or getenv("PROFILE_NAME", "AWS_PROFILE_NAME"),
             region_name=region_name or getenv("AWS_DEFAULT_REGION"),
             aws_access_key_id=aws_access_key_id or getenv("AWS_ACCESS_KEY_ID"),
             aws_secret_access_key=aws_secret_access_key or getenv("AWS_SECRET_ACCESS_KEY"),
         )
         self.s3 = self.session.resource(service_name="s3", config=retry_config)
 
-        self.logger = logger or default_logger()
-
         self.bucket_name = bucket_name
-        self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "UPLOAD_SOURCE")
+        self.upload_dir = upload_dir
         self.s3_prefix = s3_prefix
-        self.exclude_path = exclude_path
+        self.exclude_prefix = exclude_prefix
         self.skip_dot_files = skip_dot_files
         self.overwrite = overwrite
+        self.file_exclusion = file_exclusion or []
+        self.folder_exclusion = folder_exclusion or []
 
         self.results = UploadResults()
         self.start = time.time()
@@ -106,9 +140,9 @@ class Uploader:
             BucketNotFound: If bucket name was not found.
         """
         self.start = time.time()
-        if self.exclude_path and self.exclude_path not in self.upload_dir:
+        if self.exclude_prefix and self.exclude_prefix not in self.upload_dir:
             raise ValueError(
-                f"\n\n\tStart folder {self.exclude_path!r} is not a part of upload directory {self.upload_dir!r}"
+                f"\n\n\tStart folder {self.exclude_prefix!r} is not a part of upload directory {self.upload_dir!r}"
             )
         if not self.upload_dir:
             raise ValueError("\n\n\tCannot proceed without an upload directory.")
@@ -163,6 +197,8 @@ class Uploader:
             self.logger.info(
                 "S3 object %s exists, but size mismatch. Local: [%d], S3: [%d]", objectpath, file_size, object_size
             )
+        else:
+            self.logger.debug("S3 object '%s' doesn't exist, uploading..", objectpath)
         return True
 
     def _uploader(self, filepath: str, objectpath: str) -> None:
@@ -184,13 +220,20 @@ class Uploader:
         """
         files_to_upload = {}
         for __path, __directory, __files in os.walk(self.upload_dir):
+            scan_dir = os.path.split(__path)[-1]
+            if scan_dir in self.folder_exclusion:
+                self.logger.info("Skipping '%s' honoring folder exclusion", scan_dir)
+                continue
             for file_ in __files:
+                if file_ in self.file_exclusion:
+                    self.logger.info("Skipping '%s' honoring file exclusion", file_)
+                    continue
                 if self.skip_dot_files and file_.startswith("."):
                     self.logger.info("Skipping dot file: %s", file_)
                     continue
                 file_path = os.path.join(__path, file_)
-                if self.exclude_path:
-                    relative_path = file_path.replace(self.exclude_path, "")
+                if self.exclude_prefix:
+                    relative_path = file_path.replace(self.exclude_prefix, "")
                 else:
                     relative_path = file_path
                 # Lists in python are ordered, so s3 prefix will get loaded first when provided
@@ -213,7 +256,7 @@ class Uploader:
         self.logger.debug(keys)
         self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
         self.logger.info("Initiating upload process.")
-        for objectpath, filepath in tqdm(
+        for filepath, objectpath in tqdm(
             keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
         ):
             try:

pys3uploader-0.2.2.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=L12aFBb0plj8WISe0_He1vvQ55aOKd8VCyMlj_2LqrQ,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=I2An6Ix0rFMlvDLtLaDQ6F-YrN70IDCNFgh9E32cXHA,11489
-s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
-pys3uploader-0.2.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.2.2.dist-info/METADATA,sha256=fIlyxO6dFHYH3uhFg4yZa7RtGGBKqaOynAHlrbuffoY,7449
-pys3uploader-0.2.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.2.2.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.2.2.dist-info/RECORD,,