PyS3Uploader 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

pys3uploader/progress.py

@@ -0,0 +1,39 @@
+import threading
+
+from alive_progress import alive_bar
+
+
+class ProgressPercentage:
+    """Tracks progress of a file upload to S3 and updates the alive_bar.
+
+    >>> ProgressPercentage
+
+    """
+
+    def __init__(self, filename: str, size: int, bar: alive_bar):
+        """Initializes the progress tracker.
+
+        Args:
+            filename: Name of the file being uploaded.
+            size: Total size of the file in bytes.
+            bar: alive_bar instance to update progress.
+        """
+        self._filename = filename
+        self._size = size
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+        self._bar = bar
+
+    def __call__(self, bytes_amount: int) -> None:
+        """Callback method to update progress.
+
+        Args:
+            bytes_amount: Number of bytes transferred in the last chunk.
+        """
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percent = (self._seen_so_far / self._size) * 100
+            bar_len = 20
+            filled = int(bar_len * percent / 100)
+            bar_str = "█" * filled + "." * (bar_len - filled)
+            self._bar.text(f" || {self._filename} [{bar_str}] {percent:.0f}%")

pys3uploader/uploader.py

@@ -6,12 +6,13 @@ from typing import Dict, Iterable
 
 import boto3.resources.factory
 import dotenv
+from alive_progress import alive_bar
 from botocore.config import Config
 from botocore.exceptions import ClientError
-from tqdm import tqdm
 
 from pys3uploader.exceptions import BucketNotFound
 from pys3uploader.logger import LogHandler, LogLevel, setup_logger
+from pys3uploader.progress import ProgressPercentage
 from pys3uploader.utils import (
     RETRY_CONFIG,
     UploadResults,
@@ -176,6 +177,22 @@ class Uploader:
         )
         self.logger.info("Run time: %s", convert_seconds(time.time() - self.start))
 
+    def filesize(self, filepath: str) -> int:
+        """Gets the file size of a given filepath.
+
+        Args:
+            filepath: Full path of the file.
+
+        Returns:
+            int:
+            Returns the file size in bytes.
+        """
+        try:
+            return os.path.getsize(filepath)
+        except (OSError, PermissionError) as error:
+            self.logger.error(error)
+            return 0
+
     def _proceed_to_upload(self, filepath: str, objectpath: str) -> bool:
         """Compares file size if the object already exists in S3.
 
@@ -189,11 +206,7 @@ class Uploader:
         """
         if self.overwrite:
             return True
-        try:
-            file_size = os.path.getsize(filepath)
-        except (OSError, PermissionError) as error:
-            self.logger.error(error)
-            file_size = 0
+        file_size = self.filesize(filepath)
         # Indicates that the object path already exists in S3
         if object_size := self.object_size_map.get(objectpath):
             if object_size == file_size:
@@ -220,7 +233,7 @@ class Uploader:
             )
         return True
 
-    def _uploader(self, filepath: str, objectpath: str) -> None:
+    def _uploader(self, filepath: str, objectpath: str, callback: ProgressPercentage) -> None:
         """Uploads the filepath to the specified S3 bucket.
 
         Args:
@@ -228,7 +241,7 @@ class Uploader:
             objectpath: Object path ref in S3.
         """
         if self._proceed_to_upload(filepath, objectpath):
-            self.bucket.upload_file(filepath, objectpath)
+            self.bucket.upload_file(filepath, objectpath, Callback=callback)
 
     def _get_files(self) -> Dict[str, str]:
         """Get a mapping for all the file path and object paths in upload directory.
@@ -272,19 +285,23 @@ class Uploader:
         """Initiates object upload in a traditional loop."""
         self.init()
         keys = self._get_files()
-        self.logger.debug(keys)
-        self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
-        self.logger.info("Initiating upload process.")
-        for filepath, objectpath in tqdm(
-            keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
-        ):
-            try:
-                self._uploader(filepath=filepath, objectpath=objectpath)
-                self.results.success += 1
-            except ClientError as error:
-                self.logger.error(error)
-                self.results.failed += 1
-        self.exit()
+        total_files = len(keys)
+
+        with alive_bar(total_files, title="Progress", bar="smooth", spinner="dots") as overall_bar:
+            for filepath, objectpath in keys.items():
+                progress_callback = ProgressPercentage(
+                    filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
+                )
+                try:
+                    self._uploader(filepath, objectpath, progress_callback)
+                    self.results.success += 1
+                except ClientError as error:
+                    self.logger.error(error)
+                    self.results.failed += 1
+                except KeyboardInterrupt:
+                    self.logger.warning("Upload interrupted by user")
+                    break
+                overall_bar()  # increment overall progress bar
 
     def run_in_parallel(self, max_workers: int = 5) -> None:
         """Initiates upload in multi-threading.
@@ -294,32 +311,32 @@ class Uploader:
         """
         self.init()
         keys = self._get_files()
-        self.logger.debug(keys)
+        total_files = len(keys)
+
         self.logger.info(
             "%d files from '%s' will be uploaded to '%s' with maximum concurrency of: %d",
-            len(keys),
+            total_files,
             self.upload_dir,
             self.bucket_name,
             max_workers,
         )
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(self._uploader, **dict(filepath=filepath, objectpath=objectpath))
-                for filepath, objectpath in keys.items()
-            ]
-            for future in tqdm(
-                iterable=as_completed(futures),
-                total=len(futures),
-                desc=f"Uploading files to {self.bucket_name}",
-                unit="files",
-                leave=True,
-            ):
-                try:
-                    future.result()
-                    self.results.success += 1
-                except ClientError as error:
-                    self.logger.error(f"Upload failed: {error}")
-                    self.results.failed += 1
+        with alive_bar(total_files, title="Progress", bar="smooth", spinner="dots") as overall_bar:
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = []
+                for filepath, objectpath in keys.items():
+                    progress_callback = ProgressPercentage(
+                        filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
+                    )
+                    futures.append(executor.submit(self._uploader, filepath, objectpath, callback=progress_callback))
+
+                for future in as_completed(futures):
+                    try:
+                        future.result()
+                        self.results.success += 1
+                    except ClientError as error:
+                        self.logger.error(f"Upload failed: {error}")
+                        self.results.failed += 1
+                    overall_bar()  # Increment overall bar after each upload finishes
         self.exit()
 
     def get_bucket_structure(self) -> str:

pys3uploader/version.py

@@ -1 +1 @@
-version = "0.3.1"
+version = "0.3.2"

{pys3uploader-0.3.1.dist-info → pys3uploader-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.3.1
+Version: 0.3.2
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -39,9 +39,9 @@ Classifier: Topic :: Internet :: File Transfer Protocol (FTP)
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: alive-progress==3.3.*
 Requires-Dist: boto3==1.40.*
 Requires-Dist: python-dotenv==1.1.*
-Requires-Dist: tqdm==4.67.*
 Provides-Extra: dev
 Requires-Dist: sphinx==5.1.1; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"

pys3uploader-0.3.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+pys3uploader/__init__.py,sha256=EqMScWbJNV4UWeMg4fMko2KB18xL2CO3a3o_od0H0Lc,124
+pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+pys3uploader/logger.py,sha256=igwMubdTQ_GrMkwie5DAIvmxIcgj6a9UA_EGFrwFYiQ,2571
+pys3uploader/progress.py,sha256=IladNMXLBhkPpxOntpANTam_hC9OWosmNDmdbweDNYM,1195
+pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+pys3uploader/uploader.py,sha256=1X0MCWwzm2O2tY9R_tVZl-CC8qnoGt32XlyQ0LIROYw,15460
+pys3uploader/utils.py,sha256=T-TIUtzQ6sYEfZqwxH_t-Cfhw5wuzr_REQ-OBkdcg00,5357
+pys3uploader/version.py,sha256=zjuPzQ2OlxNtrLXb5A3OlPsaRb_b_Ln51AWN6r9VRho,18
+pys3uploader-0.3.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.3.2.dist-info/METADATA,sha256=RmwnvqWSdcLYLbWz2WXfzilw6FOpSG8ydfOtii4t16Q,8048
+pys3uploader-0.3.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.3.2.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
+pys3uploader-0.3.2.dist-info/RECORD,,

pys3uploader-0.3.1.dist-info/RECORD

@@ -1,12 +0,0 @@
-pys3uploader/__init__.py,sha256=EqMScWbJNV4UWeMg4fMko2KB18xL2CO3a3o_od0H0Lc,124
-pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-pys3uploader/logger.py,sha256=igwMubdTQ_GrMkwie5DAIvmxIcgj6a9UA_EGFrwFYiQ,2571
-pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-pys3uploader/uploader.py,sha256=FW2Mg1yoMV6Qaq9F2n-EXnP1vqi3AGtWUuIWVMev65o,14644
-pys3uploader/utils.py,sha256=T-TIUtzQ6sYEfZqwxH_t-Cfhw5wuzr_REQ-OBkdcg00,5357
-pys3uploader/version.py,sha256=sEAhGxRzEBE5t0VjAcJ-336II62pGIQ0eLrs42I-sGU,18
-pys3uploader-0.3.1.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.3.1.dist-info/METADATA,sha256=-moIOf7765V6qYgXcOpJtawcnlriAlUOQ2a-abgGt5o,8039
-pys3uploader-0.3.1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.3.1.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
-pys3uploader-0.3.1.dist-info/RECORD,,