PyS3Uploader 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

{pys3uploader-0.1.2.dist-info → pys3uploader-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PyS3Uploader
-Version: 0.1.2
+Version: 0.2.0
 Summary: Python module to upload objects to an S3 bucket.
 Author-email: Vignesh Rao <svignesh1793@gmail.com>
 License: MIT License

pys3uploader-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+s3/__init__.py,sha256=yLvvl4-uTLZwhdhCMQpWq5juX_zFuYAfKSf4aB0WjZw,66
+s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
+s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
+s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
+s3/uploader.py,sha256=IAlFrEjfBuexrfmBPGN9OZAfHjQuwcGRzWi2es0r_fU,11154
+s3/utils.py,sha256=0kcG0aE2olHhC8thaUEwx2J8tOI2-2TGCk6E6U-PiKw,2058
+pys3uploader-0.2.0.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
+pys3uploader-0.2.0.dist-info/METADATA,sha256=IXSmHXJJndlnd_6MHlpZrcVILPni8VUbVNJYQEjMIR8,7286
+pys3uploader-0.2.0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+pys3uploader-0.2.0.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
+pys3uploader-0.2.0.dist-info/RECORD,,

s3/__init__.py

@@ -1,3 +1,3 @@
 from s3.uploader import Uploader  # noqa: F401
 
-version = "0.1.2"
+version = "0.2.0"

s3/uploader.py

@@ -29,6 +29,7 @@ class Uploader:
         upload_dir: str,
         s3_prefix: str = None,
         exclude_path: str = None,
+        overwrite: bool = False,
         region_name: str = None,
         profile_name: str = None,
         aws_access_key_id: str = None,
@@ -42,6 +43,7 @@ class Uploader:
             upload_dir: Full path of the directory to be uploaded.
             s3_prefix: Particular bucket prefix within which the upload should happen.
             exclude_path: Full directory path to exclude from S3 object prefix.
+            overwrite: Boolean flag to overwrite files in S3.
             region_name: Name of the AWS region.
             profile_name: AWS profile name.
             aws_access_key_id: AWS access key ID.
@@ -69,16 +71,24 @@ class Uploader:
             aws_secret_access_key=aws_secret_access_key or getenv("AWS_SECRET_ACCESS_KEY"),
         )
         self.s3 = self.session.resource(service_name="s3", config=self.RETRY_CONFIG)
+
         self.logger = logger or default_logger()
+
+        self.bucket_name = bucket_name
         self.upload_dir = upload_dir or getenv("UPLOAD_DIR", "UPLOAD_SOURCE")
         self.s3_prefix = s3_prefix
         self.exclude_path = exclude_path
-        self.bucket_name = bucket_name
-        # noinspection PyUnresolvedReferences
-        self.bucket: boto3.resources.factory.s3.Bucket = None
+        self.overwrite = overwrite
+
         self.results = UploadResults()
         self.start = time.time()
 
+        # noinspection PyUnresolvedReferences
+        self.bucket: boto3.resources.factory.s3.Bucket = None
+        # noinspection PyUnresolvedReferences
+        self.bucket_objects: boto3.resources.factory.s3.ObjectSummary = []
+        self.object_size_map: Dict[str, int] = {}
+
     def init(self) -> None:
         """Instantiates the bucket instance.
 
@@ -106,6 +116,9 @@ class Uploader:
         self.upload_dir = os.path.abspath(self.upload_dir)
         # noinspection PyUnresolvedReferences
         self.bucket: boto3.resources.factory.s3.Bucket = self.s3.Bucket(self.bucket_name)
+        # noinspection PyUnresolvedReferences
+        self.bucket_objects: boto3.resources.factory.s3.ObjectSummary = [obj for obj in self.bucket.objects.all()]
+        self.object_size_map = {obj.key: obj.size for obj in self.bucket_objects}
 
     def exit(self) -> None:
         """Exits after printing results, and run time."""
@@ -115,21 +128,50 @@ class Uploader:
         )
         self.logger.info("Run Time: %.2fs", time.time() - self.start)
 
-    def _uploader(self, objectpath: str, filepath: str) -> None:
+    def _proceed_to_upload(self, filepath: str, objectpath: str) -> bool:
+        """Compares file size if the object already exists in S3.
+
+        Args:
+            filepath: Source filepath.
+            objectpath: S3 object path.
+
+        Returns:
+            bool:
+            Returns a boolean flag to indicate upload flag.
+        """
+        if self.overwrite:
+            return True
+        # Indicates that the object path already exists in S3
+        if object_size := self.object_size_map.get(objectpath):
+            try:
+                file_size = os.path.getsize(filepath)
+            except (OSError, PermissionError) as error:
+                self.logger.error(error)
+                return True
+            if object_size == file_size:
+                self.logger.info("S3 object %s exists, and size [%d] matches, skipping..", objectpath, object_size)
+                return False
+            self.logger.info(
+                "S3 object %s exists, but size mismatch. Local: [%d], S3: [%d]", objectpath, file_size, object_size
+            )
+        return True
+
+    def _uploader(self, filepath: str, objectpath: str) -> None:
         """Uploads the filepath to the specified S3 bucket.
 
         Args:
-            objectpath: Object path ref in S3.
             filepath: Filepath to upload.
+            objectpath: Object path ref in S3.
         """
-        self.bucket.upload_file(filepath, objectpath)
+        if self._proceed_to_upload(filepath, objectpath):
+            self.bucket.upload_file(filepath, objectpath)
 
     def _get_files(self) -> Dict[str, str]:
         """Get a mapping for all the file path and object paths in upload directory.
 
         Returns:
             Dict[str, str]:
-            Returns a dictionary object path and filepath.
+            Returns a key-value pair of filepath and objectpath.
         """
         files_to_upload = {}
         for __path, __directory, __files in os.walk(self.upload_dir):
@@ -149,7 +191,7 @@ class Uploader:
                 url_parts.extend(relative_path.split(os.sep))
                 # Remove falsy values using filter - "None", "bool", "len" or "lambda item: item"
                 object_path = urljoin(*filter(None, url_parts))
-                files_to_upload[object_path] = file_path
+                files_to_upload[file_path] = object_path
         return files_to_upload
 
     def run(self) -> None:
@@ -163,7 +205,7 @@ class Uploader:
             keys.items(), total=len(keys), unit="file", leave=True, desc=f"Uploading files from {self.upload_dir}"
         ):
             try:
-                self._uploader(objectpath=objectpath, filepath=filepath)
+                self._uploader(filepath=filepath, objectpath=objectpath)
                 self.results.success += 1
             except ClientError as error:
                 self.logger.error(error)
@@ -187,7 +229,10 @@ class Uploader:
             max_workers,
         )
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [executor.submit(self._uploader, *kv) for kv in keys.items()]
+            futures = [
+                executor.submit(self._uploader, **dict(filepath=filepath, objectpath=objectpath))
+                for filepath, objectpath in keys.items()
+            ]
             for future in tqdm(
                 iterable=as_completed(futures),
                 total=len(futures),
@@ -212,7 +257,7 @@ class Uploader:
         """
         self.init()
         # Using list and set will yield the same results but using set we can isolate directories from files
-        return convert_to_folder_structure(set([obj.key for obj in self.bucket.objects.all()]))
+        return convert_to_folder_structure(set(obj.key for obj in self.bucket_objects))
 
     def print_bucket_structure(self) -> None:
         """Prints all the objects in an S3 bucket with a folder like representation."""

pys3uploader-0.1.2.dist-info/RECORD

@@ -1,11 +0,0 @@
-s3/__init__.py,sha256=qSltnC7r3AjwiYWzsD9JUs8SzeBEV16nrHldiWlrxtY,66
-s3/exceptions.py,sha256=hH3jlMOe8yjBatQK9EdndWZz4QESU74KSY_iDhQ37SY,2585
-s3/logger.py,sha256=oH540oq8jY723jA4lDWlgfFPLbNgGXTkDwFpB7TLO_o,1196
-s3/tree.py,sha256=DiQ2ekMMaj2m_P3-iKkEqSuJCJZ_UZxcAwHtAoPVa5c,1824
-s3/uploader.py,sha256=kkv7d2EaMH3OsoIJgTx7yRUd00s0n9PbRbjj6Rm7qdA,9355
-s3/utils.py,sha256=0kcG0aE2olHhC8thaUEwx2J8tOI2-2TGCk6E6U-PiKw,2058
-pys3uploader-0.1.2.dist-info/LICENSE,sha256=8k-hEraOzyum0GvmmK65YxNRTFXK7eIFHJ0OshJXeTk,1068
-pys3uploader-0.1.2.dist-info/METADATA,sha256=GtQq-ZDiZEMpl2CEs4VJw4AQ8tf5rzcfgjDu68oHX6c,7286
-pys3uploader-0.1.2.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-pys3uploader-0.1.2.dist-info/top_level.txt,sha256=iQp4y1P58Q633gj8M08kHE4mqqT0hixuDWcniDk_RJ4,3
-pys3uploader-0.1.2.dist-info/RECORD,,