PyS3Uploader 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

{pys3uploader-0.2.0.dist-info → pys3uploader-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.2.0
+Version: 0.2.1
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -115,6 +115,8 @@ if __name__ == '__main__':
 #### Optional kwargs
 - **s3_prefix** - S3 object prefix for each file. Defaults to ``None``
 - **exclude_path** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
+- **skip_dot_files** - Boolean flag to skip dot files. Defaults to ``True``
+- **overwrite** - Boolean flag to overwrite files present in S3. Defaults to ``False``
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
 <br><br>
 - **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION`

pys3uploader-0.2.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3/__init__.py,sha256=IqcPR9iWMw0GDBEmKvLzW7P-AhInTkwRklkvYgiT1Xc,66
+s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
+s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+s3/uploader.py,sha256=v6TGdm8EcAldoIw3GOAwkuoUzi9b9IRj8b94hl1Pkyw,11454
+s3/utils.py,sha256=dd1OeLbswLzFVyjYiXixkJlFsoGWRtRCOHha6wLG5zQ,2485
+pys3uploader-0.2.1.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.2.1.dist-info/METADATA,sha256=NA7x6YqpWEKvn3XEYzVug7XTl1vPcwiphUzWtMvlzHE,7449
+pys3uploader-0.2.1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.2.1.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
+pys3uploader-0.2.1.dist-info/RECORD,,

s3/__init__.py

@@ -1,3 +1,3 @@
 from s3.uploader import Uploader  # noqa: F401
 
-version = "0.2.0"
+version = "0.2.1"

s3/uploader.py

@@ -11,7 +11,13 @@ from tqdm import tqdm
 
 from s3.exceptions import BucketNotFound
 from s3.logger import default_logger
-from s3.utils import UploadResults, convert_to_folder_structure, getenv, urljoin
+from s3.utils import (
+    RETRY_CONFIG,
+    UploadResults,
+    convert_to_folder_structure,
+    getenv,
+    urljoin,
+)
 
 
 class Uploader:
@@ -21,19 +27,19 @@ class Uploader:
 
     """
 
-    RETRY_CONFIG: Config = Config(retries={"max_attempts": 10, "mode": "standard"})
-
     def __init__(
         self,
         bucket_name: str,
         upload_dir: str,
         s3_prefix: str = None,
         exclude_path: str = None,
+        skip_dot_files: bool = True,
         overwrite: bool = False,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
         aws_secret_access_key: str = None,
+        retry_config: Config = RETRY_CONFIG,
         logger: logging.Logger = None,
     ):
         """Initiates all the necessary args and creates a boto3 session with retry logic.
@@ -43,6 +49,7 @@ class Uploader:
             upload_dir: Full path of the directory to be uploaded.
             s3_prefix: Particular bucket prefix within which the upload should happen.
             exclude_path: Full directory path to exclude from S3 object prefix.
+            skip_dot_files: Boolean flag to skip dot files.
             overwrite: Boolean flag to overwrite files in S3.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
@@ -51,18 +58,18 @@ class Uploader:
             logger: Bring your own logger.
 
         See Also:
+            s3_prefix:
+                If provided, ``s3_prefix`` will always be attached to each object.
+
+                If ``s3_prefix`` is set to: ``2025``, then the file path
+                ``/home/ubuntu/Desktop/S3Upload/sub/photo.jpg`` will be uploaded as ``2025/S3Upload/sub/photo.jpg``
+
             exclude_path:
                 When upload directory is "/home/ubuntu/Desktop/S3Upload", each file will naturally have the full prefix.
                 However, this behavior can be avoided by specifying the ``exclude_path`` parameter.
 
                 If exclude_path is set to: ``/home/ubuntu/Desktop``, then the file path
                 ``/home/ubuntu/Desktop/S3Upload/sub-dir/photo.jpg`` will be uploaded as ``S3Upload/sub-dir/photo.jpg``
-
-            s3_prefix:
-                If provided, ``s3_prefix`` will always be attached to each object.
-
-                If ``s3_prefix`` is set to: ``2025``, then the file path
-                ``/home/ubuntu/Desktop/S3Upload/sub/photo.jpg`` will be uploaded as ``2025/S3Upload/sub/photo.jpg``
         """
         self.session = boto3.Session(
             profile_name=profile_name or getenv("PROFILE_NAME"),
@@ -70,7 +77,7 @@ class Uploader:
             aws_access_key_id=aws_access_key_id or getenv("AWS_ACCESS_KEY_ID"),
             aws_secret_access_key=aws_secret_access_key or getenv("AWS_SECRET_ACCESS_KEY"),
         )
-        self.s3 = self.session.resource(service_name="s3", config=self.RETRY_CONFIG)
+        self.s3 = self.session.resource(service_name="s3", config=retry_config)
 
         self.logger = logger or default_logger()
 
@@ -78,6 +85,7 @@ class Uploader:
         self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "UPLOAD_SOURCE")
         self.s3_prefix = s3_prefix
         self.exclude_path = exclude_path
+        self.skip_dot_files = skip_dot_files
         self.overwrite = overwrite
 
         self.results = UploadResults()
@@ -176,6 +184,9 @@ class Uploader:
         files_to_upload = {}
         for __path, __directory, __files in os.walk(self.upload_dir):
             for file_ in __files:
+                if self.skip_dot_files and file_.startswith("."):
+                    self.logger.info("Skipping dot file: %s", file_)
+                    continue
                 file_path = os.path.join(__path, file_)
                 if self.exclude_path:
                     relative_path = file_path.replace(self.exclude_path, "")

s3/utils.py

@@ -1,6 +1,8 @@
 import os
 from typing import Dict, Set
 
+from botocore.config import Config
+
 
 class UploadResults(dict):
     """Object to store results of S3 upload.
@@ -13,6 +15,18 @@ class UploadResults(dict):
     failed: int = 0
 
 
+RETRY_CONFIG: Config = Config(
+    retries={
+        "max_attempts": 10,
+        "mode": "adaptive",  # Adaptive retry mode with jitter
+        "total_max_attempts": 20,  # Max retries across all requests
+    },
+    # Adding custom timeouts here:
+    connect_timeout=5,  # 5 seconds for establishing a connection
+    read_timeout=30,  # 30 seconds to wait for a response from the server
+)
+
+
 def getenv(*args, default: str = None) -> str:
     """Returns the key-ed environment variable or the default value."""
     for key in args:

pys3uploader-0.2.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=yLvvl4-uTLZwhdhCMQpWq5juX_zFuYAfKSf4aB0WjZw,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=IAlFrEjfBuexrfmBPGN9OZAfHjQuwcGRzWi2es0r_fU,11154
-s3/utils.py,sha256=0kcG0aE2olHhC8thaUEwx2J8tOI2-2TGCk6E6U-PiKw,2058
-pys3uploader-0.2.0.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.2.0.dist-info/METADATA,sha256=IXSmHXJJndlnd_6MHlpZrcVILPni8VUbVNJYQEjMIR8,7286
-pys3uploader-0.2.0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.2.0.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.2.0.dist-info/RECORD,,