PyS3Uploader 0.4.0a1__py3-none-any.whl → 0.4.1a0__py3-none-any.whl

pys3uploader/metadata.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 
+
 @dataclass
 class Metadata(dict):
     """Dataclass for metadata information."""

pys3uploader/timer.py

@@ -1,4 +1,5 @@
-from threading import Timer
+import logging
+import threading
 from typing import Any, Callable, Dict, Tuple
 
 
@@ -15,6 +16,7 @@ class RepeatedTimer:
         function: Callable,
         args: Tuple = None,
         kwargs: Dict[str, Any] = None,
+        logger: logging.Logger = None,
     ):
         """Repeats the ``Timer`` object from threading.
 
@@ -23,32 +25,33 @@ class RepeatedTimer:
             function: Function to trigger with intervals.
             args: Arguments for the function.
             kwargs: Keyword arguments for the function.
+            logger: Logger instance.
         """
-        self._timer = None
         self.interval = interval
         self.function = function
         self.args = args or ()
         self.kwargs = kwargs or {}
-        self.is_running = False
+        self.logger = logger or logging.getLogger(__name__)
+        self.thread = None
+        self._stop_event = threading.Event()
 
     def _run(self):
         """Triggers the target function."""
-        self.is_running = False
-        self.start()
-        self.function(*self.args, **self.kwargs)
+        while not self._stop_event.wait(self.interval):
+            try:
+                self.function(*self.args, **self.kwargs)
+            except Exception as error:
+                self.logger.error("Error in RepeatedTimer function [%s]: %s", self.function.__name__, error)
 
     def start(self):
         """Trigger target function if timer isn't running already."""
-        if not self.is_running:
-            self._timer = Timer(self.interval, self._run)
-            self._timer.start()
-            self.is_running = True
+        if self.thread and self.thread.is_alive():
+            return
+        self.thread = threading.Thread(target=self._run, daemon=True)
+        self.thread.start()
 
-    def stop(self):
+    def stop(self, timeout: int = 3):
         """Stop the timer and cancel all futures."""
-        self._timer.cancel()
-        self.is_running = False
-
-    def cancel(self):
-        """Initiate cancellation."""
-        self.stop()
+        self._stop_event.set()
+        if self.thread:
+            self.thread.join(timeout=timeout)

pys3uploader/uploader.py

@@ -2,8 +2,8 @@ import json
 import logging
 import os
 import time
-from datetime import datetime, UTC
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import UTC, datetime
 from typing import Dict, Iterable, NoReturn
 
 import boto3.resources.factory
@@ -13,10 +13,10 @@ from botocore.config import Config
 from botocore.exceptions import ClientError
 
 from pys3uploader.exceptions import BucketNotFound
-from pys3uploader.metadata import Metadata
-from pys3uploader.timer import RepeatedTimer
 from pys3uploader.logger import LogHandler, LogLevel, setup_logger
+from pys3uploader.metadata import Metadata
 from pys3uploader.progress import ProgressPercentage
+from pys3uploader.timer import RepeatedTimer
 from pys3uploader.utils import (
     RETRY_CONFIG,
     UploadResults,
@@ -45,6 +45,8 @@ class Uploader:
         overwrite: bool = False,
         file_exclusion: Iterable[str] = None,
         folder_exclusion: Iterable[str] = None,
+        metadata_upload_interval: int = None,
+        metadata_filename: str = None,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
@@ -66,10 +68,13 @@ class Uploader:
             overwrite: Boolean flag to overwrite files in S3.
             file_exclusion: Sequence of files to exclude during upload.
             folder_exclusion: Sequence of directories to exclude during upload.
+            metadata_upload_interval: Interval in seconds to upload metadata file.
+            metadata_filename: Metadata filename to upload periodically.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
             aws_secret_access_key: AWS secret access key.
+            retry_config: Boto3 retry configuration.
             logger: Bring your own logger.
             log_handler: Default log handler, can be ``file`` or ``stdout``.
             log_level: Default log level, can be ``debug``, ``info``, ``warning`` or ``error``.
@@ -145,10 +150,13 @@ class Uploader:
         self.upload_files: Dict[str, str] = {}
         self.file_size_map: Dict[str, int] = {}
 
+        self.metadata_filename = metadata_filename or getenv("METADATA_FILENAME", default="METADATA.json")
         self.timer = RepeatedTimer(
             function=self.metadata_uploader,
-            interval=int(getenv("METADATA_UPLOAD_INTERVAL", 300))
+            interval=metadata_upload_interval or int(getenv("METADATA_UPLOAD_INTERVAL", default="300")),
+            logger=self.logger,
         )
+        self.alive_bar_kwargs = dict(title="Progress", bar="smooth", spinner=None, enrich_print=False)
 
     def init(self) -> None | NoReturn:
         """Instantiates the bucket instance.
@@ -175,7 +183,7 @@ class Uploader:
         self.upload_dir = os.path.abspath(self.upload_dir)
         self.load_bucket_state()
 
-    def load_bucket_state(self):
+    def load_bucket_state(self) -> None:
         """Loads the bucket's current state."""
         # noinspection PyUnresolvedReferences
         self.bucket: boto3.resources.factory.s3.Bucket = self.s3.Bucket(self.bucket_name)
@@ -191,10 +199,11 @@ class Uploader:
     def exit(self) -> None:
         """Exits after printing results, and run time."""
         success = len(self.results.success)
+        skipped = len(self.results.skipped)
         failed = len(self.results.failed)
         total = success + failed
         self.logger.info(
-            "Total number of uploads: %d, success: %d, failed: %d", total, success, failed
+            "Total number of uploads: %d, skipped: %d, success: %d, failed: %d", total, skipped, success, failed
         )
         # Stop the timer and upload the final state as metadata file
         self.timer.stop()
@@ -253,11 +262,13 @@ class Uploader:
                     object_size,
                     size_converter(object_size),
                 )
+                self.results.skipped.append(filepath)
                 return False
             self.logger.info(
                 "S3 object %s exists, but size mismatch. Local: [%d bytes / %s], S3: [%d bytes / %s]",
                 objectpath,
                 file_size,
+                size_converter(file_size),
                 object_size,
                 size_converter(object_size),
             )
@@ -319,13 +330,29 @@ class Uploader:
                 files_to_upload[file_path] = object_path
         return files_to_upload
 
-    def run(self) -> None:
-        """Initiates object upload in a traditional loop."""
+    def _preflight(self) -> int:
+        """Preflight checks and tasks before upload.
+
+        Returns:
+            int:
+            Returns the total number of files to be uploaded.
+        """
+        # Verify and initiate bucket state
         self.init()
+        # Verify and initiate local state
         self.load_local_state()
+        # Make sure there are files to upload
+        assert self.upload_files, "\n\n\tNo files found to upload.\n"
+        # Log size details
         self.size_it()
+        # Start metadata upload timer
         self.timer.start()
-        total_files = len(self.upload_files)
+        # Return total files to upload
+        return len(self.upload_files)
+
+    def run(self) -> None:
+        """Initiates object upload in a traditional loop."""
+        total_files = self._preflight()
 
         self.logger.info(
             "%d files from '%s' will be uploaded to '%s' sequentially",
@@ -333,7 +360,7 @@ class Uploader:
             self.upload_dir,
             self.bucket_name,
         )
-        with alive_bar(total_files, title="Progress", bar="smooth", spinner="dots") as overall_bar:
+        with alive_bar(total_files, **self.alive_bar_kwargs) as overall_bar:
             for filepath, objectpath in self.upload_files.items():
                 progress_callback = ProgressPercentage(
                     filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
@@ -356,13 +383,7 @@ class Uploader:
         Args:
             max_workers: Number of maximum threads to use.
         """
-        # Verify and initiate bucket state
-        self.init()
-        # Verify and initiate local state
-        self.load_local_state()
-        self.size_it()
-        self.timer.start()
-        total_files = len(self.upload_files)
+        total_files = self._preflight()
 
         self.logger.info(
             "%d files from '%s' will be uploaded to '%s' with maximum concurrency of: %d",
@@ -371,16 +392,18 @@ class Uploader:
             self.bucket_name,
             max_workers,
         )
-        with alive_bar(total_files, title="Progress", bar="smooth", spinner="dots") as overall_bar:
+        with alive_bar(total_files, **self.alive_bar_kwargs) as overall_bar:
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                futures = []
+                futures = {}
                 for filepath, objectpath in self.upload_files.items():
                     progress_callback = ProgressPercentage(
                         filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
                     )
-                    futures.append(executor.submit(self._uploader, filepath, objectpath, callback=progress_callback))
+                    future = executor.submit(self._uploader, filepath, objectpath, progress_callback)
+                    futures[future] = filepath
 
                 for future in as_completed(futures):
+                    filepath = futures[future]
                     try:
                         future.result()
                         self.results.success.append(filepath)
@@ -392,29 +415,29 @@ class Uploader:
 
     def metadata_uploader(self) -> None:
         """Metadata uploader."""
-        filename = objectpath = getenv("METADATA_FILENAME", "METADATA.json")
         self.load_bucket_state()
-        objects_uploaded = len(self.results.success)
-        size_uploaded = sum([self.filesize(file) for file in self.results.success])
+        success = list(set(self.results.success + self.results.skipped))
+        objects_uploaded = len(success)
+        size_uploaded = sum(self.filesize(file) for file in success)
 
-        pending_files = self.upload_files.keys() - self.results.success
+        pending_files = set(self.upload_files.keys()) - set(success)
         objects_pending = len(pending_files)
-        size_pending = sum([self.filesize(file) for file in pending_files])
+        size_pending = sum(self.filesize(file) for file in pending_files)
 
         metadata = Metadata(
             timestamp=datetime.now(tz=UTC).strftime("%A %B %d, %Y %H:%M:%S"),
             objects_uploaded=objects_uploaded,
             objects_pending=objects_pending,
             size_uploaded=size_converter(size_uploaded),
-            size_pending=size_converter(size_pending)
+            size_pending=size_converter(size_pending),
         )
-        self.logger.debug("\n" + json.dumps(metadata, indent=2) + "\n")
+        self.logger.debug("\n" + json.dumps(metadata.__dict__, indent=2) + "\n")
         self.logger.debug("Uploading metadata to S3")
-        filepath = os.path.join(os.getcwd(), filename)
+        filepath = os.path.join(os.getcwd(), self.metadata_filename)
         with open(filepath, "w") as file:
             json.dump(metadata.__dict__, file, indent=2)
             file.flush()
-        self.bucket.upload_file(filepath, objectpath)
+        self.bucket.upload_file(filepath, self.metadata_filename)
 
     def get_bucket_structure(self) -> str:
         """Gets all the objects in an S3 bucket and forms it into a hierarchical folder like representation.

pys3uploader/utils.py

@@ -1,6 +1,6 @@
 import math
 import os
-from typing import Dict, Set, List
+from typing import Dict, List, Set
 
 from botocore.config import Config
 
@@ -25,6 +25,7 @@ class UploadResults(dict):
 
     success: List[str] = []
     failed: List[str] = []
+    skipped: List[str] = []
 
 
 def getenv(*args, default: str = None) -> str:

pys3uploader/version.py

@@ -1 +1 @@
-version = "0.4.0a1"
+version = "0.4.1a0"

{pys3uploader-0.4.0a1.dist-info → pys3uploader-0.4.1a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.4.0a1
+Version: 0.4.1a0
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -157,6 +157,8 @@ if __name__ == '__main__':
 - **overwrite** - Boolean flag to overwrite files present in S3. Defaults to ``False``
 - **file_exclusion** - Sequence of files to exclude during upload. Defaults to ``None``
 - **folder_exclusion** - Sequence of directories to exclude during upload. Defaults to ``None``
+- **metadata_upload_interval** - Interval in seconds to upload metadata file. Defaults to ``300``
+- **metadata_filename** - Metadata filename to upload periodically. Defaults to ``METADATA.json``
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
 - **log_handler** - Choose between `stdout` vs `file` logging. Defaults to `pys3uploader.LogHandler.stdout`
 - **log_level** - Choose the logging level. Defaults to `pys3uploader.LogLevel.debug`

pys3uploader-0.4.1a0.dist-info/RECORD

@@ -0,0 +1,15 @@
+pys3uploader/__init__.py,sha256=EqMScWbJNV4UWeMg4fMko2KB18xL2CO3a3o_od0H0Lc,124
+pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+pys3uploader/logger.py,sha256=z9JEnyf4nHIakey0bAaCgEN7oXOYJYOpskZyM_4s-D4,2678
+pys3uploader/metadata.py,sha256=4pn8Y9dVZLFXxq5Cocm20t1mfDkV5xJXY6YEekQ8ssQ,231
+pys3uploader/progress.py,sha256=IladNMXLBhkPpxOntpANTam_hC9OWosmNDmdbweDNYM,1195
+pys3uploader/timer.py,sha256=8tvlQaO1Z7TfFVBhl5qrDq9wsMiOR6P0J2bkVwfM96c,1816
+pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+pys3uploader/uploader.py,sha256=sKUcC5eFL_NPYoYi3FWUWRCwkjoW2OXlcozNP9OIrzM,19707
+pys3uploader/utils.py,sha256=8eIM5ZhS7-bYT5Dq4FNbmojQ16vvoAPF1ihCCTZezGE,5783
+pys3uploader/version.py,sha256=t9lIxi0skPjuG6f_lIZ9Hx9gXs5mbdbnIuC5KLIMc-Y,20
+pys3uploader-0.4.1a0.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.4.1a0.dist-info/METADATA,sha256=m4P0H49F97neV8LFFUOfASBHEmHGip3Ck-Yj4kvdbkU,9155
+pys3uploader-0.4.1a0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.4.1a0.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
+pys3uploader-0.4.1a0.dist-info/RECORD,,

pys3uploader-0.4.0a1.dist-info/RECORD

@@ -1,15 +0,0 @@
-pys3uploader/__init__.py,sha256=EqMScWbJNV4UWeMg4fMko2KB18xL2CO3a3o_od0H0Lc,124
-pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-pys3uploader/logger.py,sha256=z9JEnyf4nHIakey0bAaCgEN7oXOYJYOpskZyM_4s-D4,2678
-pys3uploader/metadata.py,sha256=tOOoLh2vISfH-GfH3yBcA_xtEjRwomaw7sCLEaDRK-8,230
-pys3uploader/progress.py,sha256=IladNMXLBhkPpxOntpANTam_hC9OWosmNDmdbweDNYM,1195
-pys3uploader/timer.py,sha256=qN2XNrGEyP3stsK3McvhE3VvIiUFh7mv4rbp5WDeyVU,1498
-pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-pys3uploader/uploader.py,sha256=h5DYQA2yv0fQ2SSyAnAl8SsgJUajmN_o1PdMSqMbACM,18588
-pys3uploader/utils.py,sha256=_2RYKUTyrQzwkxo7fSiLb5ASrpjcNpb3kZHqy_wByRk,5755
-pys3uploader/version.py,sha256=VAwBBgd_skAqJS9UL1T_xDXryTqN5m58fbTTEXcKxgM,20
-pys3uploader-0.4.0a1.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.4.0a1.dist-info/METADATA,sha256=FdJdNSesnP1xHfb4il5HBw1pxsPn7ToAYkQ_T3PrIb0,8959
-pys3uploader-0.4.0a1.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.4.0a1.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
-pys3uploader-0.4.0a1.dist-info/RECORD,,