PyS3Uploader 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

pys3uploader/__init__.py

@@ -1,4 +1,2 @@
 from pys3uploader.logger import LogHandler, LogLevel  # noqa: F401
 from pys3uploader.uploader import Uploader  # noqa: F401
-
-version = "0.3.0"

pys3uploader/progress.py

@@ -0,0 +1,39 @@
+import threading
+
+from alive_progress import alive_bar
+
+
+class ProgressPercentage:
+    """Tracks progress of a file upload to S3 and updates the alive_bar.
+
+    >>> ProgressPercentage
+
+    """
+
+    def __init__(self, filename: str, size: int, bar: alive_bar):
+        """Initializes the progress tracker.
+
+        Args:
+            filename: Name of the file being uploaded.
+            size: Total size of the file in bytes.
+            bar: alive_bar instance to update progress.
+        """
+        self._filename = filename
+        self._size = size
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+        self._bar = bar
+
+    def __call__(self, bytes_amount: int) -> None:
+        """Callback method to update progress.
+
+        Args:
+            bytes_amount: Number of bytes transferred in the last chunk.
+        """
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percent = (self._seen_so_far / self._size) * 100
+            bar_len = 20
+            filled = int(bar_len * percent / 100)
+            bar_str = "█" * filled + "." * (bar_len - filled)
+            self._bar.text(f" || {self._filename} [{bar_str}] {percent:.0f}%")

pys3uploader/uploader.py

@@ -6,18 +6,20 @@ from typing import Dict, Iterable
 
 import boto3.resources.factory
 import dotenv
+from alive_progress import alive_bar
 from botocore.config import Config
 from botocore.exceptions import ClientError
-from tqdm import tqdm
 
 from pys3uploader.exceptions import BucketNotFound
 from pys3uploader.logger import LogHandler, LogLevel, setup_logger
+from pys3uploader.progress import ProgressPercentage
 from pys3uploader.utils import (
     RETRY_CONFIG,
     UploadResults,
     convert_seconds,
     convert_to_folder_structure,
     getenv,
+    size_converter,
     urljoin,
 )
 
@@ -175,6 +177,22 @@ class Uploader:
         )
         self.logger.info("Run time: %s", convert_seconds(time.time() - self.start))
 
+    def filesize(self, filepath: str) -> int:
+        """Gets the file size of a given filepath.
+
+        Args:
+            filepath: Full path of the file.
+
+        Returns:
+            int:
+            Returns the file size in bytes.
+        """
+        try:
+            return os.path.getsize(filepath)
+        except (OSError, PermissionError) as error:
+            self.logger.error(error)
+            return 0
+
     def _proceed_to_upload(self, filepath: str, objectpath: str) -> bool:
         """Compares file size if the object already exists in S3.
 
@@ -188,24 +206,34 @@ class Uploader:
         """
         if self.overwrite:
             return True
-        try:
-            file_size = os.path.getsize(filepath)
-        except (OSError, PermissionError) as error:
-            self.logger.error(error)
-            file_size = 0
+        file_size = self.filesize(filepath)
         # Indicates that the object path already exists in S3
         if object_size := self.object_size_map.get(objectpath):
             if object_size == file_size:
-                self.logger.info("S3 object %s exists, and size [%d] matches, skipping..", objectpath, object_size)
+                self.logger.info(
+                    "S3 object %s exists, and size [%d bytes / %s] matches, skipping..",
+                    objectpath,
+                    object_size,
+                    size_converter(object_size),
+                )
                 return False
             self.logger.info(
-                "S3 object %s exists, but size mismatch. Local: [%d], S3: [%d]", objectpath, file_size, object_size
+                "S3 object %s exists, but size mismatch. Local: [%d bytes / %s], S3: [%d bytes / %s]",
+                objectpath,
+                file_size,
+                object_size,
+                size_converter(object_size),
             )
         else:
-            self.logger.debug("S3 object '%s' of size [%d bytes] doesn't exist, uploading..", objectpath, file_size)
+            self.logger.debug(
+                "S3 object '%s' of size [%d bytes / %s] doesn't exist, uploading..",
+                objectpath,
+                file_size,
+                size_converter(file_size),
+            )
         return True
 
-    def _uploader(self, filepath: str, objectpath: str) -> None:
+    def _uploader(self, filepath: str, objectpath: str, callback: ProgressPercentage) -> None:
         """Uploads the filepath to the specified S3 bucket.
 
         Args:
@@ -213,7 +241,7 @@ class Uploader:
             objectpath: Object path ref in S3.
         """
         if self._proceed_to_upload(filepath, objectpath):
-            self.bucket.upload_file(filepath, objectpath)
+            self.bucket.upload_file(filepath, objectpath, Callback=callback)
 
     def _get_files(self) -> Dict[str, str]:
         """Get a mapping for all the file path and object paths in upload directory.
@@ -257,19 +285,23 @@ class Uploader:
         """Initiates object upload in a traditional loop."""
         self.init()
         keys = self._get_files()
-        self.logger.debug(keys)
-        self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
-        self.logger.info("Initiating upload process.")
-        for filepath, objectpath in tqdm(
-            keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
-        ):
-            try:
-                self._uploader(filepath=filepath, objectpath=objectpath)
-                self.results.success += 1
-            except ClientError as error:
-                self.logger.error(error)
-                self.results.failed += 1
-        self.exit()
+        total_files = len(keys)
+
+        with alive_bar(total_files, title="Progress", bar="smooth", spinner="dots") as overall_bar:
+            for filepath, objectpath in keys.items():
+                progress_callback = ProgressPercentage(
+                    filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
+                )
+                try:
+                    self._uploader(filepath, objectpath, progress_callback)
+                    self.results.success += 1
+                except ClientError as error:
+                    self.logger.error(error)
+                    self.results.failed += 1
+                except KeyboardInterrupt:
+                    self.logger.warning("Upload interrupted by user")
+                    break
+                overall_bar()  # increment overall progress bar
 
     def run_in_parallel(self, max_workers: int = 5) -> None:
         """Initiates upload in multi-threading.
@@ -279,32 +311,32 @@ class Uploader:
         """
         self.init()
         keys = self._get_files()
-        self.logger.debug(keys)
+        total_files = len(keys)
+
         self.logger.info(
             "%d files from '%s' will be uploaded to '%s' with maximum concurrency of: %d",
-            len(keys),
+            total_files,
             self.upload_dir,
             self.bucket_name,
             max_workers,
         )
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(self._uploader, **dict(filepath=filepath, objectpath=objectpath))
-                for filepath, objectpath in keys.items()
-            ]
-            for future in tqdm(
-                iterable=as_completed(futures),
-                total=len(futures),
-                desc=f"Uploading files to {self.bucket_name}",
-                unit="files",
-                leave=True,
-            ):
-                try:
-                    future.result()
-                    self.results.success += 1
-                except ClientError as error:
-                    self.logger.error(f"Upload failed: {error}")
-                    self.results.failed += 1
+        with alive_bar(total_files, title="Progress", bar="smooth", spinner="dots") as overall_bar:
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = []
+                for filepath, objectpath in keys.items():
+                    progress_callback = ProgressPercentage(
+                        filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
+                    )
+                    futures.append(executor.submit(self._uploader, filepath, objectpath, callback=progress_callback))
+
+                for future in as_completed(futures):
+                    try:
+                        future.result()
+                        self.results.success += 1
+                    except ClientError as error:
+                        self.logger.error(f"Upload failed: {error}")
+                        self.results.failed += 1
+                    overall_bar()  # Increment overall bar after each upload finishes
         self.exit()
 
     def get_bucket_structure(self) -> str:

pys3uploader/utils.py

@@ -1,3 +1,4 @@
+import math
 import os
 from typing import Dict, Set
 
@@ -145,3 +146,33 @@ def convert_seconds(seconds: int | float, n_elem: int = 2) -> str:
 
     list_ = time_parts[:n_elem]
     return ", and ".join([", ".join(list_[:-1]), list_[-1]] if len(list_) > 2 else list_)
+
+
+def format_nos(input_: float) -> int | float:
+    """Removes ``.0`` float values.
+
+    Args:
+        input_: Strings or integers with ``.0`` at the end.
+
+    Returns:
+        int | float:
+        Int if found, else returns the received float value.
+    """
+    return int(input_) if isinstance(input_, float) and input_.is_integer() else input_
+
+
+def size_converter(byte_size: int | float) -> str:
+    """Gets the current memory consumed and converts it to human friendly format.
+
+    Args:
+        byte_size: Receives byte size as argument.
+
+    Returns:
+        str:
+        Converted understandable size.
+    """
+    if not byte_size:
+        return "0 B"
+    size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    index = int(math.floor(math.log(byte_size, 1024)))
+    return f"{format_nos(round(byte_size / pow(1024, index), 2))} {size_name[index]}"

pys3uploader/version.py

@@ -0,0 +1 @@
+version = "0.3.2"

{pys3uploader-0.3.0.dist-info → pys3uploader-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.3.0
+Version: 0.3.2
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -39,9 +39,9 @@ Classifier: Topic :: Internet :: File Transfer Protocol (FTP)
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: alive-progress==3.3.*
 Requires-Dist: boto3==1.40.*
 Requires-Dist: python-dotenv==1.1.*
-Requires-Dist: tqdm==4.67.*
 Provides-Extra: dev
 Requires-Dist: sphinx==5.1.1; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"

pys3uploader-0.3.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+pys3uploader/__init__.py,sha256=EqMScWbJNV4UWeMg4fMko2KB18xL2CO3a3o_od0H0Lc,124
+pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+pys3uploader/logger.py,sha256=igwMubdTQ_GrMkwie5DAIvmxIcgj6a9UA_EGFrwFYiQ,2571
+pys3uploader/progress.py,sha256=IladNMXLBhkPpxOntpANTam_hC9OWosmNDmdbweDNYM,1195
+pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+pys3uploader/uploader.py,sha256=1X0MCWwzm2O2tY9R_tVZl-CC8qnoGt32XlyQ0LIROYw,15460
+pys3uploader/utils.py,sha256=T-TIUtzQ6sYEfZqwxH_t-Cfhw5wuzr_REQ-OBkdcg00,5357
+pys3uploader/version.py,sha256=zjuPzQ2OlxNtrLXb5A3OlPsaRb_b_Ln51AWN6r9VRho,18
+pys3uploader-0.3.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.3.2.dist-info/METADATA,sha256=RmwnvqWSdcLYLbWz2WXfzilw6FOpSG8ydfOtii4t16Q,8048
+pys3uploader-0.3.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.3.2.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
+pys3uploader-0.3.2.dist-info/RECORD,,

pys3uploader-0.3.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-pys3uploader/__init__.py,sha256=QszR24KVfcJXbrJI0oxZCIaMs5GOtwnIbtfrY2Zj7Nk,143
-pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-pys3uploader/logger.py,sha256=igwMubdTQ_GrMkwie5DAIvmxIcgj6a9UA_EGFrwFYiQ,2571
-pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-pys3uploader/uploader.py,sha256=-PRwu2RvzExjLTKbw9oQuFcyNI0Uxi1IsoJu6e6wayY,14258
-pys3uploader/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
-pys3uploader-0.3.0.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.3.0.dist-info/METADATA,sha256=0eS5eoAsatnmMi_wanq_kzHTZGVTvwo2c4pR_CIa3T4,8039
-pys3uploader-0.3.0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.3.0.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
-pys3uploader-0.3.0.dist-info/RECORD,,