PyS3Uploader 0.2.3__py3-none-any.whl → 0.4.0b2__py3-none-any.whl

pys3uploader/__init__.py

@@ -0,0 +1,2 @@
+from pys3uploader.logger import LogHandler, LogLevel  # noqa: F401
+from pys3uploader.uploader import Uploader  # noqa: F401

pys3uploader/logger.py

@@ -0,0 +1,104 @@
+"""Loads a default logger with StreamHandler set to DEBUG mode.
+
+>>> logging.Logger
+
+"""
+
+import logging
+import os
+from datetime import datetime
+from enum import IntEnum, StrEnum
+
+
+class LogHandler(StrEnum):
+    """Logging handlers to choose from when default logger is used.
+
+    >>> LogHandler
+
+    """
+
+    file = "file"
+    stdout = "stdout"
+
+
+class LogLevel(IntEnum):
+    """Logging levels to choose from when default logger is used.
+
+    >>> LogLevel
+
+    """
+
+    debug = logging.DEBUG
+    info = logging.INFO
+    warning = logging.WARNING
+    error = logging.ERROR
+
+    @classmethod
+    def _missing_(cls, value):
+        """Allow constructing from string names."""
+        if isinstance(value, str):
+            value = value.lower()
+            for member in cls:
+                if member.name == value:
+                    return member
+        return None
+
+
+def stream_handler() -> logging.StreamHandler:
+    """Creates a ``StreamHandler`` and assigns a default format to it.
+
+    Returns:
+        logging.StreamHandler:
+        Returns an instance of the ``StreamHandler`` object.
+    """
+    handler = logging.StreamHandler()
+    handler.setFormatter(fmt=default_format())
+    return handler
+
+
+def file_handler() -> logging.FileHandler:
+    """Creates a ``StreamHandler`` and assigns a default format to it.
+
+    Returns:
+        logging.StreamHandler:
+        Returns an instance of the ``StreamHandler`` object.
+    """
+    os.makedirs("logs", exist_ok=True)
+    filename = os.path.join("logs", datetime.now().strftime("PyS3Uploader_%d-%m-%Y_%H:%M.log"))
+    handler = logging.FileHandler(filename, mode="a")
+    handler.setFormatter(fmt=default_format())
+    return handler
+
+
+def default_format() -> logging.Formatter:
+    """Creates a logging ``Formatter`` with a custom message and datetime format.
+
+    Returns:
+        logging.Formatter:
+        Returns an instance of the ``Formatter`` object.
+    """
+    return logging.Formatter(
+        fmt="%(asctime)s - %(levelname)s - [%(module)s:%(lineno)d] - %(funcName)s - %(message)s",
+        datefmt="%b-%d-%Y %I:%M:%S %p",
+    )
+
+
+def setup_logger(handler: LogHandler, level: LogLevel) -> logging.Logger:
+    """Creates a default logger with debug mode enabled.
+
+    Args:
+        handler: Logging handler to use.
+        level: Logging level to use.
+
+    Returns:
+        logging.Logger:
+        Returns an instance of the ``Logger`` object.
+    """
+    logger = logging.getLogger(__name__)
+    if handler == LogHandler.file:
+        logger.addHandler(hdlr=file_handler())
+    elif handler == LogHandler.stdout:
+        logger.addHandler(hdlr=stream_handler())
+
+    logger.setLevel(level)
+    return logger

pys3uploader/metadata.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Metadata(dict):
+    """Dataclass for metadata information."""
+
+    timestamp: str
+    objects_uploaded: int
+    objects_pending: int
+    size_uploaded: str
+    size_pending: str

pys3uploader/progress.py

@@ -0,0 +1,39 @@
+import threading
+
+from alive_progress import alive_bar
+
+
+class ProgressPercentage:
+    """Tracks progress of a file upload to S3 and updates the alive_bar.
+
+    >>> ProgressPercentage
+
+    """
+
+    def __init__(self, filename: str, size: int, bar: alive_bar):
+        """Initializes the progress tracker.
+
+        Args:
+            filename: Name of the file being uploaded.
+            size: Total size of the file in bytes.
+            bar: alive_bar instance to update progress.
+        """
+        self._filename = filename
+        self._size = size
+        self._seen_so_far = 0
+        self._lock = threading.Lock()
+        self._bar = bar
+
+    def __call__(self, bytes_amount: int) -> None:
+        """Callback method to update progress.
+
+        Args:
+            bytes_amount: Number of bytes transferred in the last chunk.
+        """
+        with self._lock:
+            self._seen_so_far += bytes_amount
+            percent = (self._seen_so_far / self._size) * 100
+            bar_len = 20
+            filled = int(bar_len * percent / 100)
+            bar_str = "█" * filled + "." * (bar_len - filled)
+            self._bar.text(f" || {self._filename} [{bar_str}] {percent:.0f}%")

pys3uploader/timer.py

@@ -0,0 +1,57 @@
+import logging
+import threading
+from typing import Any, Callable, Dict, Tuple
+
+
+class RepeatedTimer:
+    """Instantiates RepeatedTimer object to kick off the threading.Timer object with custom intervals.
+
+    >>> RepeatedTimer
+
+    """
+
+    def __init__(
+        self,
+        interval: int,
+        function: Callable,
+        args: Tuple = None,
+        kwargs: Dict[str, Any] = None,
+        logger: logging.Logger = None,
+    ):
+        """Repeats the ``Timer`` object from threading.
+
+        Args:
+            interval: Interval in seconds.
+            function: Function to trigger with intervals.
+            args: Arguments for the function.
+            kwargs: Keyword arguments for the function.
+            logger: Logger instance.
+        """
+        self.interval = interval
+        self.function = function
+        self.args = args or ()
+        self.kwargs = kwargs or {}
+        self.logger = logger or logging.getLogger(__name__)
+        self.thread = None
+        self._stop_event = threading.Event()
+
+    def _run(self):
+        """Triggers the target function."""
+        while not self._stop_event.wait(self.interval):
+            try:
+                self.function(*self.args, **self.kwargs)
+            except Exception as error:
+                self.logger.error("Error in RepeatedTimer function [%s]: %s", self.function.__name__, error)
+
+    def start(self):
+        """Trigger target function if timer isn't running already."""
+        if self.thread and self.thread.is_alive():
+            return
+        self.thread = threading.Thread(target=self._run, daemon=True)
+        self.thread.start()
+
+    def stop(self, timeout: int = 3):
+        """Stop the timer and cancel all futures."""
+        self._stop_event.set()
+        if self.thread:
+            self.thread.join(timeout=timeout)

{s3 → pys3uploader}/uploader.py

@@ -1,23 +1,29 @@
+import json
 import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Dict, Iterable
+from datetime import UTC, datetime
+from typing import Dict, Iterable, NoReturn
 
 import boto3.resources.factory
 import dotenv
+from alive_progress import alive_bar
 from botocore.config import Config
 from botocore.exceptions import ClientError
-from tqdm import tqdm
 
-from s3.exceptions import BucketNotFound
-from s3.logger import default_logger
-from s3.utils import (
+from pys3uploader.exceptions import BucketNotFound
+from pys3uploader.logger import LogHandler, LogLevel, setup_logger
+from pys3uploader.metadata import Metadata
+from pys3uploader.progress import ProgressPercentage
+from pys3uploader.timer import RepeatedTimer
+from pys3uploader.utils import (
     RETRY_CONFIG,
     UploadResults,
     convert_seconds,
     convert_to_folder_structure,
     getenv,
+    size_converter,
     urljoin,
 )
 
@@ -39,12 +45,16 @@ class Uploader:
         overwrite: bool = False,
         file_exclusion: Iterable[str] = None,
         folder_exclusion: Iterable[str] = None,
+        metadata_upload_interval: int = None,
+        metadata_filename: str = None,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
         aws_secret_access_key: str = None,
         retry_config: Config = RETRY_CONFIG,
         logger: logging.Logger = None,
+        log_handler: LogHandler = LogHandler.stdout,
+        log_level: LogLevel = LogLevel.debug,
         env_file: str = None,
     ):
         """Initiates all the necessary args and creates a boto3 session with retry logic.
@@ -58,11 +68,15 @@ class Uploader:
             overwrite: Boolean flag to overwrite files in S3.
             file_exclusion: Sequence of files to exclude during upload.
             folder_exclusion: Sequence of directories to exclude during upload.
+            metadata_upload_interval: Interval in seconds to upload metadata file.
+            metadata_filename: Metadata filename to upload periodically.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
             aws_secret_access_key: AWS secret access key.
             logger: Bring your own logger.
+            log_handler: Default log handler, can be ``file`` or ``stdout``.
+            log_level: Default log level, can be ``debug``, ``info``, ``warning`` or ``error``.
             env_file: Dotenv file (.env) filepath to load environment variables.
 
         See Also:
@@ -85,7 +99,7 @@ class Uploader:
                 If a filepath is provided, PyS3Uploader loads it directly or searches the root directory for the file.
                 If no filepath is provided, PyS3Uploader searches the current directory for a .env file.
         """
-        self.logger = logger or default_logger()
+        self.logger = logger or setup_logger(handler=LogHandler(log_handler), level=LogLevel(log_level))
         self.env_file = env_file or getenv("ENV_FILE", default=".env")
 
         # Check for env_file in current working directory
@@ -132,7 +146,18 @@ class Uploader:
         self.bucket_objects: boto3.resources.factory.s3.ObjectSummary = []
         self.object_size_map: Dict[str, int] = {}
 
-    def init(self) -> None:
+        self.upload_files: Dict[str, str] = {}
+        self.file_size_map: Dict[str, int] = {}
+
+        self.metadata_filename = metadata_filename or getenv("METADATA_FILENAME", default="METADATA.json")
+        self.timer = RepeatedTimer(
+            function=self.metadata_uploader,
+            interval=metadata_upload_interval or int(getenv("METADATA_UPLOAD_INTERVAL", default="300")),
+            logger=self.logger,
+        )
+        self.alive_bar_kwargs = dict(title="Progress", bar="smooth", spinner=None, enrich_print=False)
+
+    def init(self) -> None | NoReturn:
         """Instantiates the bucket instance.
 
         Raises:
@@ -150,27 +175,69 @@ class Uploader:
             assert os.path.exists(self.upload_dir)
         except AssertionError:
             raise ValueError(f"\n\n\tPath not found: {self.upload_dir}")
-        buckets = [bucket.name for bucket in self.s3.buckets.all()]
         if not self.bucket_name:
-            raise ValueError(f"\n\n\tCannot proceed without a bucket name.\n\tAvailable: {buckets}")
-        _account_id, _alias = self.session.resource(service_name="iam").CurrentUser().arn.split("/")
-        if self.bucket_name not in buckets:
-            raise BucketNotFound(f"\n\n\t{self.bucket_name} was not found in {_alias} account.\n\tAvailable: {buckets}")
+            raise ValueError("\n\n\tCannot proceed without a bucket name.")
+        if (buckets := [bucket.name for bucket in self.s3.buckets.all()]) and self.bucket_name not in buckets:
+            raise BucketNotFound(f"\n\n\t{self.bucket_name} was not found.\n\tAvailable: {buckets}")
         self.upload_dir = os.path.abspath(self.upload_dir)
+        self.load_bucket_state()
+
+    def load_bucket_state(self) -> None:
+        """Loads the bucket's current state."""
         # noinspection PyUnresolvedReferences
         self.bucket: boto3.resources.factory.s3.Bucket = self.s3.Bucket(self.bucket_name)
         # noinspection PyUnresolvedReferences
         self.bucket_objects: boto3.resources.factory.s3.ObjectSummary = [obj for obj in self.bucket.objects.all()]
         self.object_size_map = {obj.key: obj.size for obj in self.bucket_objects}
 
+    def load_local_state(self):
+        """Loads the local file queue."""
+        self.upload_files = self._get_files()
+        self.file_size_map = {file: self.filesize(file) for file in self.upload_files}
+
     def exit(self) -> None:
         """Exits after printing results, and run time."""
-        total = self.results.success + self.results.failed
+        success = len(self.results.success)
+        skipped = len(self.results.skipped)
+        failed = len(self.results.failed)
+        total = success + failed
         self.logger.info(
-            "Total number of uploads: %d, success: %d, failed: %d", total, self.results.success, self.results.failed
+            "Total number of uploads: %d, skipped: %d, success: %d, failed: %d", total, skipped, success, failed
         )
+        # Stop the timer and upload the final state as metadata file
+        self.timer.stop()
+        self.metadata_uploader()
         self.logger.info("Run time: %s", convert_seconds(time.time() - self.start))
 
+    def filesize(self, filepath: str) -> int:
+        """Gets the file size of a given filepath.
+
+        Args:
+            filepath: Full path of the file.
+
+        Returns:
+            int:
+            Returns the file size in bytes.
+        """
+        try:
+            return os.path.getsize(filepath)
+        except (OSError, PermissionError) as error:
+            self.logger.error(error)
+            return 0
+
+    def size_it(self) -> None:
+        """Calculates and logs the total size of files in S3 and local."""
+        files_in_s3 = len(self.object_size_map)
+        files_local = len(self.upload_files)
+
+        total_size_s3 = sum(self.object_size_map.values())
+        total_size_local = sum(self.file_size_map.values())
+
+        self.logger.info("Files in S3: [#%d]: %s (%d bytes)", files_in_s3, size_converter(total_size_s3), total_size_s3)
+        self.logger.info(
+            "Files local: [#%d]: %s (%d bytes)", files_local, size_converter(total_size_local), total_size_local
+        )
+
     def _proceed_to_upload(self, filepath: str, objectpath: str) -> bool:
         """Compares file size if the object already exists in S3.
 
@@ -184,30 +251,44 @@ class Uploader:
         """
         if self.overwrite:
             return True
+        file_size = self.filesize(filepath)
         # Indicates that the object path already exists in S3
         if object_size := self.object_size_map.get(objectpath):
-            try:
-                file_size = os.path.getsize(filepath)
-            except (OSError, PermissionError) as error:
-                self.logger.error(error)
-                return True
             if object_size == file_size:
-                self.logger.info("S3 object %s exists, and size [%d] matches, skipping..", objectpath, object_size)
+                self.logger.info(
+                    "S3 object %s exists, and size [%d bytes / %s] matches, skipping..",
+                    objectpath,
+                    object_size,
+                    size_converter(object_size),
+                )
+                self.results.skipped.append(filepath)
                 return False
             self.logger.info(
-                "S3 object %s exists, but size mismatch. Local: [%d], S3: [%d]", objectpath, file_size, object_size
+                "S3 object %s exists, but size mismatch. Local: [%d bytes / %s], S3: [%d bytes / %s]",
+                objectpath,
+                file_size,
+                object_size,
+                size_converter(object_size),
+            )
+        else:
+            self.logger.debug(
+                "S3 object '%s' of size [%d bytes / %s] doesn't exist, uploading..",
+                objectpath,
+                file_size,
+                size_converter(file_size),
             )
         return True
 
-    def _uploader(self, filepath: str, objectpath: str) -> None:
+    def _uploader(self, filepath: str, objectpath: str, callback: ProgressPercentage) -> None:
         """Uploads the filepath to the specified S3 bucket.
 
         Args:
             filepath: Filepath to upload.
             objectpath: Object path ref in S3.
+            callback: ProgressPercentage callback to track upload progress.
         """
         if self._proceed_to_upload(filepath, objectpath):
-            self.bucket.upload_file(filepath, objectpath)
+            self.bucket.upload_file(filepath, objectpath, Callback=callback)
 
     def _get_files(self) -> Dict[str, str]:
         """Get a mapping for all the file path and object paths in upload directory.
@@ -247,22 +328,51 @@ class Uploader:
                 files_to_upload[file_path] = object_path
         return files_to_upload
 
+    def _preflight(self) -> int:
+        """Preflight checks and tasks before upload.
+
+        Returns:
+            int:
+            Returns the total number of files to be uploaded.
+        """
+        # Verify and initiate bucket state
+        self.init()
+        # Verify and initiate local state
+        self.load_local_state()
+        # Make sure there are files to upload
+        assert self.upload_files, "\n\n\tNo files found to upload.\n"
+        # Log size details
+        self.size_it()
+        # Start metadata upload timer
+        self.timer.start()
+        # Return total files to upload
+        return len(self.upload_files)
+
     def run(self) -> None:
         """Initiates object upload in a traditional loop."""
-        self.init()
-        keys = self._get_files()
-        self.logger.debug(keys)
-        self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
-        self.logger.info("Initiating upload process.")
-        for filepath, objectpath in tqdm(
-            keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
-        ):
-            try:
-                self._uploader(filepath=filepath, objectpath=objectpath)
-                self.results.success += 1
-            except ClientError as error:
-                self.logger.error(error)
-                self.results.failed += 1
+        total_files = self._preflight()
+
+        self.logger.info(
+            "%d files from '%s' will be uploaded to '%s' sequentially",
+            total_files,
+            self.upload_dir,
+            self.bucket_name,
+        )
+        with alive_bar(total_files, **self.alive_bar_kwargs) as overall_bar:
+            for filepath, objectpath in self.upload_files.items():
+                progress_callback = ProgressPercentage(
+                    filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
+                )
+                try:
+                    self._uploader(filepath, objectpath, progress_callback)
+                    self.results.success.append(filepath)
+                except ClientError as error:
+                    self.logger.error("Upload failed: %s", error)
+                    self.results.failed.append(filepath)
+                except KeyboardInterrupt:
+                    self.logger.warning("Upload interrupted by user")
+                    break
+                overall_bar()  # increment overall progress bar
         self.exit()
 
     def run_in_parallel(self, max_workers: int = 5) -> None:
@@ -271,36 +381,62 @@ class Uploader:
         Args:
             max_workers: Number of maximum threads to use.
         """
-        self.init()
-        keys = self._get_files()
-        self.logger.debug(keys)
+        total_files = self._preflight()
+
         self.logger.info(
             "%d files from '%s' will be uploaded to '%s' with maximum concurrency of: %d",
-            len(keys),
+            total_files,
             self.upload_dir,
             self.bucket_name,
             max_workers,
         )
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [
-                executor.submit(self._uploader, **dict(filepath=filepath, objectpath=objectpath))
-                for filepath, objectpath in keys.items()
-            ]
-            for future in tqdm(
-                iterable=as_completed(futures),
-                total=len(futures),
-                desc=f"Uploading files to {self.bucket_name}",
-                unit="files",
-                leave=True,
-            ):
-                try:
-                    future.result()
-                    self.results.success += 1
-                except ClientError as error:
-                    self.logger.error(f"Upload failed: {error}")
-                    self.results.failed += 1
+        with alive_bar(total_files, **self.alive_bar_kwargs) as overall_bar:
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = {}
+                for filepath, objectpath in self.upload_files.items():
+                    progress_callback = ProgressPercentage(
+                        filename=os.path.basename(filepath), size=self.filesize(filepath), bar=overall_bar
+                    )
+                    future = executor.submit(self._uploader, filepath, objectpath, progress_callback)
+                    futures[future] = filepath
+
+                for future in as_completed(futures):
+                    filepath = futures[future]
+                    try:
+                        future.result()
+                        self.results.success.append(filepath)
+                    except ClientError as error:
+                        self.logger.error("Upload failed: %s", error)
+                        self.results.failed.append(filepath)
+                    overall_bar()  # Increment overall bar after each upload finishes
         self.exit()
 
+    def metadata_uploader(self) -> None:
+        """Metadata uploader."""
+        self.load_bucket_state()
+        success = list(set(self.results.success + self.results.skipped))
+        objects_uploaded = len(success)
+        size_uploaded = sum(self.filesize(file) for file in success)
+
+        pending_files = set(self.upload_files.keys()) - set(success)
+        objects_pending = len(pending_files)
+        size_pending = sum(self.filesize(file) for file in pending_files)
+
+        metadata = Metadata(
+            timestamp=datetime.now(tz=UTC).strftime("%A %B %d, %Y %H:%M:%S"),
+            objects_uploaded=objects_uploaded,
+            objects_pending=objects_pending,
+            size_uploaded=size_converter(size_uploaded),
+            size_pending=size_converter(size_pending),
+        )
+        self.logger.debug("\n" + json.dumps(metadata.__dict__, indent=2) + "\n")
+        self.logger.debug("Uploading metadata to S3")
+        filepath = os.path.join(os.getcwd(), self.metadata_filename)
+        with open(filepath, "w") as file:
+            json.dump(metadata.__dict__, file, indent=2)
+            file.flush()
+        self.bucket.upload_file(filepath, self.metadata_filename)
+
     def get_bucket_structure(self) -> str:
         """Gets all the objects in an S3 bucket and forms it into a hierarchical folder like representation.
 

{s3 → pys3uploader}/utils.py

@@ -1,5 +1,6 @@
+import math
 import os
-from typing import Dict, Set
+from typing import Dict, List, Set
 
 from botocore.config import Config
 
@@ -22,12 +23,22 @@ class UploadResults(dict):
 
     """
 
-    success: int = 0
-    failed: int = 0
+    success: List[str] = []
+    failed: List[str] = []
+    skipped: List[str] = []
 
 
 def getenv(*args, default: str = None) -> str:
-    """Returns the key-ed environment variable or the default value."""
+    """Returns the key-ed environment variable or the default value.
+
+    Args:
+        args: Environment variable keys to search for.
+        default: Default value to return if no environment variable is found.
+
+    Returns:
+        str:
+        Environment variable value or the default value.
+    """
     for key in args:
         if value := os.environ.get(key.upper()) or os.environ.get(key.lower()):
             return value
@@ -37,6 +48,9 @@ def getenv(*args, default: str = None) -> str:
 def urljoin(*args) -> str:
     """Joins given arguments into a url. Trailing but not leading slashes are stripped for each argument.
 
+    Args:
+        args: Parts of the url to join.
+
     Returns:
         str:
         Joined url.
@@ -67,6 +81,10 @@ def convert_to_folder_structure(sequence: Set[str]) -> str:
         Args:
             structure: Structure of folder objects as key-value pairs.
             indent: Required indentation for the ASCII.
+
+        Returns:
+            str:
+            String representation of the folder structure.
         """
         result = ""
         for i, (key, value) in enumerate(structure.items()):
@@ -145,3 +163,33 @@ def convert_seconds(seconds: int | float, n_elem: int = 2) -> str:
 
     list_ = time_parts[:n_elem]
     return ", and ".join([", ".join(list_[:-1]), list_[-1]] if len(list_) > 2 else list_)
+
+
+def format_nos(input_: float) -> int | float:
+    """Removes ``.0`` float values.
+
+    Args:
+        input_: Strings or integers with ``.0`` at the end.
+
+    Returns:
+        int | float:
+        Int if found, else returns the received float value.
+    """
+    return int(input_) if isinstance(input_, float) and input_.is_integer() else input_
+
+
+def size_converter(byte_size: int | float) -> str:
+    """Gets the current memory consumed and converts it to human friendly format.
+
+    Args:
+        byte_size: Receives byte size as argument.
+
+    Returns:
+        str:
+        Converted understandable size.
+    """
+    if not byte_size:
+        return "0 B"
+    size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    index = int(math.floor(math.log(byte_size, 1024)))
+    return f"{format_nos(round(byte_size / pow(1024, index), 2))} {size_name[index]}"

pys3uploader/version.py

@@ -0,0 +1 @@
+version = "0.4.0b2"

{pys3uploader-0.2.3.dist-info → pys3uploader-0.4.0b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.2.3
+Version: 0.4.0b2
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -29,7 +29,7 @@ Project-URL: Homepage, https://github.com/thevickypedia/PyS3Uploader
 Project-URL: Docs, https://thevickypedia.github.io/PyS3Uploader/
 Project-URL: Source, https://github.com/thevickypedia/PyS3Uploader
 Project-URL: Bug Tracker, https://github.com/thevickypedia/PyS3Uploader/issues
-Keywords: s3
+Keywords: pys3uploader
 Classifier: Development Status :: 1 - Planning
 Classifier: Intended Audience :: Information Technology
 Classifier: Operating System :: OS Independent
@@ -39,9 +39,9 @@ Classifier: Topic :: Internet :: File Transfer Protocol (FTP)
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: alive-progress==3.3.*
 Requires-Dist: boto3==1.40.*
 Requires-Dist: python-dotenv==1.1.*
-Requires-Dist: tqdm==4.67.*
 Provides-Extra: dev
 Requires-Dist: sphinx==5.1.1; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
@@ -76,6 +76,43 @@ Requires-Dist: recommonmark; extra == "dev"
 # PyS3Uploader
 Python module to upload an entire directory to an S3 bucket.
 
+<details>
+<summary><strong>Bucket Policy Required</strong></summary>
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "ListBucketsForExistenceCheck",
+            "Effect": "Allow",
+            "Action": "s3:ListAllMyBuckets",
+            "Resource": "*"
+        },
+        {
+            "Sid": "ListAndUploadToSpecificBucket",
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:ListBucketMultipartUploads"
+            ],
+            "Resource": "arn:aws:s3:::bucketname"
+        },
+        {
+            "Sid": "UploadObjectsToBucket",
+            "Effect": "Allow",
+            "Action": [
+                "s3:PutObject",
+                "s3:AbortMultipartUpload",
+                "s3:ListMultipartUploadParts"
+            ],
+            "Resource": "arn:aws:s3:::bucketname/*"
+        }
+    ]
+}
+```
+</details>
+
 ### Installation
 ```shell
 pip install PyS3Uploader
@@ -85,10 +122,10 @@ pip install PyS3Uploader
 
 ##### Upload objects in parallel
 ```python
-import s3
+import pys3uploader
 
 if __name__ == '__main__':
-    wrapper = s3.Uploader(
+    wrapper = pys3uploader.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
         exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
@@ -98,10 +135,10 @@ if __name__ == '__main__':
 
 ##### Upload objects in sequence
 ```python
-import s3
+import pys3uploader
 
 if __name__ == '__main__':
-    wrapper = s3.Uploader(
+    wrapper = pys3uploader.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
         exclude_prefix="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
@@ -120,7 +157,11 @@ if __name__ == '__main__':
 - **overwrite** - Boolean flag to overwrite files present in S3. Defaults to ``False``
 - **file_exclusion** - Sequence of files to exclude during upload. Defaults to ``None``
 - **folder_exclusion** - Sequence of directories to exclude during upload. Defaults to ``None``
+- **metadata_upload_interval** - Interval in seconds to upload metadata file. Defaults to ``300``
+- **metadata_filename** - Metadata filename to upload periodically. Defaults to ``METADATA.json``
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
+- **log_handler** - Choose between `stdout` vs `file` logging. Defaults to `pys3uploader.LogHandler.stdout`
+- **log_level** - Choose the logging level. Defaults to `pys3uploader.LogLevel.debug`
 - **env_file** – Path to a `.env` file for loading environment variables. Defaults to scanning the current directory.
 <br><br>
 - **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION`

pys3uploader-0.4.0b2.dist-info/RECORD

@@ -0,0 +1,15 @@
+pys3uploader/__init__.py,sha256=EqMScWbJNV4UWeMg4fMko2KB18xL2CO3a3o_od0H0Lc,124
+pys3uploader/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+pys3uploader/logger.py,sha256=z9JEnyf4nHIakey0bAaCgEN7oXOYJYOpskZyM_4s-D4,2678
+pys3uploader/metadata.py,sha256=4pn8Y9dVZLFXxq5Cocm20t1mfDkV5xJXY6YEekQ8ssQ,231
+pys3uploader/progress.py,sha256=IladNMXLBhkPpxOntpANTam_hC9OWosmNDmdbweDNYM,1195
+pys3uploader/timer.py,sha256=8tvlQaO1Z7TfFVBhl5qrDq9wsMiOR6P0J2bkVwfM96c,1816
+pys3uploader/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+pys3uploader/uploader.py,sha256=iaLsYaqyI4SeRmuQhQWEQpEzaA_U3szloOegj_axp-0,19611
+pys3uploader/utils.py,sha256=8eIM5ZhS7-bYT5Dq4FNbmojQ16vvoAPF1ihCCTZezGE,5783
+pys3uploader/version.py,sha256=FqI7luRfbr03_FdtC3SRipnHn79CZY6AjZYAM2RmRi0,20
+pys3uploader-0.4.0b2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.4.0b2.dist-info/METADATA,sha256=yVVlf8rbYL48rOXj6J5mt290xlz0xjhA3f7jO_tY1-Q,9155
+pys3uploader-0.4.0b2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.4.0b2.dist-info/top_level.txt,sha256=lVIFMMoUx7dj_myetBmOUQTJiOzz5VyDqchnQElmrWw,13
+pys3uploader-0.4.0b2.dist-info/RECORD,,

pys3uploader-0.4.0b2.dist-info/top_level.txt

@@ -0,0 +1 @@
+pys3uploader

pys3uploader-0.2.3.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=aZ2woJ8TD2tgqXi0ElG-wWwJWoQLIdqTdm50FLaxL8w,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=KxrWbIInXxXQszP_uJLf_dBI5rUNjNnhco3gr9Vdrto,13767
-s3/utils.py,sha256=NbF28CYviK_St5qd1EOumMVyus9BvQON7clUFeR_SEQ,4473
-pys3uploader-0.2.3.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.2.3.dist-info/METADATA,sha256=ae2lA8b7dsGWZSMfB8w4joDiAlaE6Wk1f3p1Fxywkc4,7795
-pys3uploader-0.2.3.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.2.3.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.2.3.dist-info/RECORD,,

pys3uploader-0.2.3.dist-info/top_level.txt

@@ -1 +0,0 @@
-s3

s3/__init__.py

@@ -1,3 +0,0 @@
-from s3.uploader import Uploader  # noqa: F401
-
-version = "0.2.3"

s3/logger.py

@@ -1,45 +0,0 @@
-"""Loads a default logger with StreamHandler set to DEBUG mode.
-
->>> logging.Logger
-
-"""
-
-import logging
-
-
-def default_handler() -> logging.StreamHandler:
-    """Creates a ``StreamHandler`` and assigns a default format to it.
-
-    Returns:
-        logging.StreamHandler:
-        Returns an instance of the ``StreamHandler`` object.
-    """
-    handler = logging.StreamHandler()
-    handler.setFormatter(fmt=default_format())
-    return handler
-
-
-def default_format() -> logging.Formatter:
-    """Creates a logging ``Formatter`` with a custom message and datetime format.
-
-    Returns:
-        logging.Formatter:
-        Returns an instance of the ``Formatter`` object.
-    """
-    return logging.Formatter(
-        fmt="%(asctime)s - %(levelname)s - [%(module)s:%(lineno)d] - %(funcName)s - %(message)s",
-        datefmt="%b-%d-%Y %I:%M:%S %p",
-    )
-
-
-def default_logger() -> logging.Logger:
-    """Creates a default logger with debug mode enabled.
-
-    Returns:
-        logging.Logger:
-        Returns an instance of the ``Logger`` object.
-    """
-    logger = logging.getLogger(__name__)
-    logger.addHandler(hdlr=default_handler())
-    logger.setLevel(level=logging.DEBUG)
-    return logger