PyS3Uploader 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

{pys3uploader-0.1.0.dist-info → pys3uploader-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.1.0
+Version: 0.1.2
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License
@@ -63,7 +63,7 @@ Requires-Dist: recommonmark; extra == "dev"
 
 **Activity**
 
-[![GitHub Repo created](https://img.shields.io/date/1618966420)][repo]
+[![GitHub Repo created](https://img.shields.io/date/1760313686)][repo]
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/y/thevickypedia/PyS3Uploader)][repo]
 [![GitHub last commit](https://img.shields.io/github/last-commit/thevickypedia/PyS3Uploader)][repo]
 
@@ -90,7 +90,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        prefix_dir="START_DIRECTORY_IN_S3"
+        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run_in_parallel()
 ```
@@ -103,7 +103,7 @@ if __name__ == '__main__':
     wrapper = s3.Uploader(
         bucket_name="BUCKET_NAME",
         upload_dir="FULL_PATH_TO_UPLOAD",
-        prefix_dir="START_DIRECTORY_IN_S3"
+        exclude_path="PART_OF_UPLOAD_DIR_TO_EXCLUDE"
     )
     wrapper.run()
 ```
@@ -113,7 +113,8 @@ if __name__ == '__main__':
 - **upload_dir** - Directory to upload.
 
 #### Optional kwargs
-- **prefix_dir** - Start directory from ``upload_dir`` to use as root in S3. Defaults to `None`
+- **s3_prefix** - S3 object prefix for each file. Defaults to ``None``
+- **exclude_path** - Path in ``upload_dir`` that has to be excluded in object keys. Defaults to `None`
 - **logger** - Bring your own custom pre-configured logger. Defaults to on-screen logging.
 <br><br>
 - **region_name** - AWS region name. Defaults to the env var `AWS_DEFAULT_REGION`

pys3uploader-0.1.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3/__init__.py,sha256=qSltnC7r3AjwiYWzsD9JUs8SzeBEV16nrHldiWlrxtY,66
+s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
+s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+s3/uploader.py,sha256=kkv7d2EaMH3OsoIJgTx7yRUd00s0n9PbRbjj6Rm7qdA,9355
+s3/utils.py,sha256=0kcG0aE2olHhC8thaUEwx2J8tOI2-2TGCk6E6U-PiKw,2058
+pys3uploader-0.1.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.1.2.dist-info/METADATA,sha256=GtQq-ZDiZEMpl2CEs4VJw4AQ8tf5rzcfgjDu68oHX6c,7286
+pys3uploader-0.1.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.1.2.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
+pys3uploader-0.1.2.dist-info/RECORD,,

s3/__init__.py

@@ -1,3 +1,3 @@
 from s3.uploader import Uploader  # noqa: F401
 
-version = "0.1.0"
+version = "0.1.2"

s3/uploader.py

@@ -11,7 +11,7 @@ from tqdm import tqdm
 
 from s3.exceptions import BucketNotFound
 from s3.logger import default_logger
-from s3.utils import UploadResults, get_object_path, getenv
+from s3.utils import UploadResults, convert_to_folder_structure, getenv, urljoin
 
 
 class Uploader:
@@ -27,7 +27,8 @@ class Uploader:
         self,
         bucket_name: str,
         upload_dir: str,
-        prefix_dir: str = None,
+        s3_prefix: str = None,
+        exclude_path: str = None,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
@@ -38,13 +39,28 @@ class Uploader:
 
         Args:
             bucket_name: Name of the bucket.
-            upload_dir: Name of the directory to be uploaded.
-            prefix_dir: Start folder name from upload_dir.
+            upload_dir: Full path of the directory to be uploaded.
+            s3_prefix: Particular bucket prefix within which the upload should happen.
+            exclude_path: Full directory path to exclude from S3 object prefix.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
             aws_secret_access_key: AWS secret access key.
             logger: Bring your own logger.
+
+        See Also:
+            exclude_path:
+                When upload directory is "/home/ubuntu/Desktop/S3Upload", each file will naturally have the full prefix.
+                However, this behavior can be avoided by specifying the ``exclude_path`` parameter.
+
+                If exclude_path is set to: ``/home/ubuntu/Desktop``, then the file path
+                ``/home/ubuntu/Desktop/S3Upload/sub-dir/photo.jpg`` will be uploaded as ``S3Upload/sub-dir/photo.jpg``
+
+            s3_prefix:
+                If provided, ``s3_prefix`` will always be attached to each object.
+
+                If ``s3_prefix`` is set to: ``2025``, then the file path
+                ``/home/ubuntu/Desktop/S3Upload/sub/photo.jpg`` will be uploaded as ``2025/S3Upload/sub/photo.jpg``
         """
         self.session = boto3.Session(
             profile_name=profile_name or getenv("PROFILE_NAME"),
@@ -54,8 +70,9 @@ class Uploader:
         )
         self.s3 = self.session.resource(service_name="s3", config=self.RETRY_CONFIG)
         self.logger = logger or default_logger()
-        self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "SOURCE")
-        self.prefix_dir = prefix_dir
+        self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "UPLOAD_SOURCE")
+        self.s3_prefix = s3_prefix
+        self.exclude_path = exclude_path
         self.bucket_name = bucket_name
         # noinspection PyUnresolvedReferences
         self.bucket: boto3.resources.factory.s3.Bucket = None
@@ -70,9 +87,9 @@ class Uploader:
             BucketNotFound: If bucket name was not found.
         """
         self.start = time.time()
-        if self.prefix_dir and self.prefix_dir not in self.upload_dir.split(os.sep):
+        if self.exclude_path and self.exclude_path not in self.upload_dir:
             raise ValueError(
-                f"\n\n\tPrefix folder name {self.prefix_dir!r} is not a part of upload directory {self.upload_dir!r}"
+                f"\n\n\tStart folder {self.exclude_path!r} is not a part of upload directory {self.upload_dir!r}"
             )
         if not self.upload_dir:
             raise ValueError("\n\n\tCannot proceed without an upload directory.")
@@ -87,7 +104,6 @@ class Uploader:
         if self.bucket_name not in buckets:
             raise BucketNotFound(f"\n\n\t{self.bucket_name} was not found in {_alias} account.\n\tAvailable: {buckets}")
         self.upload_dir = os.path.abspath(self.upload_dir)
-        self.logger.info("Bucket objects from '%s' will be uploaded to '%s'", self.upload_dir, self.bucket_name)
         # noinspection PyUnresolvedReferences
         self.bucket: boto3.resources.factory.s3.Bucket = self.s3.Bucket(self.bucket_name)
 
@@ -119,14 +135,20 @@ class Uploader:
         for __path, __directory, __files in os.walk(self.upload_dir):
             for file_ in __files:
                 file_path = os.path.join(__path, file_)
-                if self.prefix_dir:
-                    try:
-                        object_path = get_object_path(file_path, self.prefix_dir)
-                    except ValueError as error:
-                        self.logger.error(error)
-                        continue
+                if self.exclude_path:
+                    relative_path = file_path.replace(self.exclude_path, "")
                 else:
-                    object_path = self.prefix_dir
+                    relative_path = file_path
+                # Lists in python are ordered, so s3 prefix will get loaded first when provided
+                url_parts = []
+                if self.s3_prefix:
+                    url_parts.extend(
+                        self.s3_prefix.split(os.sep) if os.sep in self.s3_prefix else self.s3_prefix.split("/")
+                    )
+                # Add rest of the file path to parts before normalizing as an S3 object URL
+                url_parts.extend(relative_path.split(os.sep))
+                # Remove falsy values using filter - "None", "bool", "len" or "lambda item: item"
+                object_path = urljoin(*filter(None, url_parts))
                 files_to_upload[object_path] = file_path
         return files_to_upload
 
@@ -135,6 +157,7 @@ class Uploader:
         self.init()
         keys = self._get_files()
         self.logger.debug(keys)
+        self.logger.info("%d files from '%s' will be uploaded to '%s'", len(keys), self.upload_dir, self.bucket_name)
         self.logger.info("Initiating upload process.")
         for objectpath, filepath in tqdm(
             keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
@@ -154,9 +177,15 @@ class Uploader:
             max_workers: Number of maximum threads to use.
         """
         self.init()
-        self.logger.info(f"Number of threads: {max_workers}")
         keys = self._get_files()
-        self.logger.info("Initiating upload process.")
+        self.logger.debug(keys)
+        self.logger.info(
+            "%d files from '%s' will be uploaded to '%s' with maximum concurrency of: %d",
+            len(keys),
+            self.upload_dir,
+            self.bucket_name,
+            max_workers,
+        )
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
             futures = [executor.submit(self._uploader, *kv) for kv in keys.items()]
             for future in tqdm(
@@ -173,3 +202,18 @@ class Uploader:
                     self.logger.error(f"Upload failed: {error}")
                     self.results.failed += 1
         self.exit()
+
+    def get_bucket_structure(self) -> str:
+        """Gets all the objects in an S3 bucket and forms it into a hierarchical folder like representation.
+
+        Returns:
+            str:
+            Returns a hierarchical folder like representation of the chosen bucket.
+        """
+        self.init()
+        # Using list and set will yield the same results but using set we can isolate directories from files
+        return convert_to_folder_structure(set([obj.key for obj in self.bucket.objects.all()]))
+
+    def print_bucket_structure(self) -> None:
+        """Prints all the objects in an S3 bucket with a folder like representation."""
+        print(self.get_bucket_structure())

s3/utils.py

@@ -1,4 +1,5 @@
 import os
+from typing import Dict, Set
 
 
 class UploadResults(dict):
@@ -20,26 +21,50 @@ def getenv(*args, default: str = None) -> str:
     return default
 
 
-def get_object_path(filepath: str, start_folder_name: str):
-    """Construct object path without absolute path's pretext.
+def urljoin(*args) -> str:
+    """Joins given arguments into a url. Trailing but not leading slashes are stripped for each argument.
+
+    Returns:
+        str:
+        Joined url.
+    """
+    return "/".join(map(lambda x: str(x).rstrip("/").lstrip("/"), args))
+
+
+def convert_to_folder_structure(sequence: Set[str]) -> str:
+    """Convert objects in a s3 buckets into a folder like representation.
 
     Args:
-        filepath: Absolute file path to upload.
-        start_folder_name: Folder name to begin object path.
+        sequence: Takes either a mutable or immutable sequence as an argument.
 
     Returns:
         str:
-        Returns the object name.
+        String representation of the architecture.
     """
-    # Split file_path into parts
-    parts = filepath.split(os.sep)
-    try:
-        # Find index of the folder to start from
-        start_index = parts.index(start_folder_name)
-    except ValueError:
-        # Folder not found in path, fallback to full path or raise error
-        raise ValueError(f"Folder '{start_folder_name}' not found in path '{filepath}'")
-    # Reconstruct path from start_folder_name onwards
-    relative_parts = parts[start_index:]
-    # Join with os.sep for system-appropriate separators
-    return os.sep.join(relative_parts)
+    folder_structure = {}
+    for item in sequence:
+        parts = item.split("/")
+        current_level = folder_structure
+        for part in parts:
+            current_level = current_level.setdefault(part, {})
+
+    def generate_folder_structure(structure: Dict[str, dict], indent: str = "") -> str:
+        """Generates the folder like structure.
+
+        Args:
+            structure: Structure of folder objects as key-value pairs.
+            indent: Required indentation for the ASCII.
+        """
+        result = ""
+        for i, (key, value) in enumerate(structure.items()):
+            if i == len(structure) - 1:
+                result += indent + "└── " + key + "\n"
+                sub_indent = indent + "    "
+            else:
+                result += indent + "├── " + key + "\n"
+                sub_indent = indent + "│   "
+            if value:
+                result += generate_folder_structure(value, sub_indent)
+        return result
+
+    return generate_folder_structure(folder_structure)

pys3uploader-0.1.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=zSLbLrsnVD-tRtiiTcT3JDWzmpnSC9mP6uHMXt2cyEc,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=Z2EvtUlR5jlL1xbeQWj4XLBfhTn4yWPm9E8WhPcz6Qk,7056
-s3/utils.py,sha256=swkdwkfn43e8I3dGL9HAGZ-dba3fIeorihVAjTE07wc,1291
-pys3uploader-0.1.0.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.1.0.dist-info/METADATA,sha256=gkAfOF-hEXYfW9p0PZiJhcrpLjhNorK7LFZNtU_ybrE,7188
-pys3uploader-0.1.0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.1.0.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.1.0.dist-info/RECORD,,