sharedkernel 2.2.6__tar.gz → 2.3.0__tar.gz

{sharedkernel-2.2.6 → sharedkernel-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sharedkernel
-Version: 2.2.6
+Version: 2.3.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -29,6 +29,10 @@ Dynamic: summary
 this is a shared kernel package
 
 # Change Log
+### Version 2.3.0
+- Implement multipart uploader for S3
+### Version 2.2.7
+- add presigned url in s3 uploader
 ### Version 2.2.6
 - implement api key
 ### Version 2.2.5

{sharedkernel-2.2.6 → sharedkernel-2.3.0}/README.md

@@ -2,6 +2,10 @@
 this is a shared kernel package
 
 # Change Log
+### Version 2.3.0
+- Implement multipart uploader for S3
+### Version 2.2.7
+- add presigned url in s3 uploader
 ### Version 2.2.6
 - implement api key
 ### Version 2.2.5

{sharedkernel-2.2.6 → sharedkernel-2.3.0}/setup.py

@@ -37,7 +37,7 @@ setup(
         "deepdiff",
     ],
     # *strongly* suggested for sharing
-    version="2.2.6",
+    version="2.3.0",
     description="sharekernel is a shared package between all python projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

sharedkernel-2.3.0/sharedkernel/multipart_upload.py

@@ -0,0 +1,69 @@
+class MultipartUploadSession:
+    MIN_PART_SIZE = 5 * 1024 * 1024  # 5MB
+
+    def __init__(self, s3_client, bucket, object_key, acl="private"):
+        self.s3 = s3_client
+        self.bucket = bucket
+        self.key = object_key
+        self.acl = acl
+
+        self.upload_id = None
+        self.parts = []
+        self.part_number = 1
+        self.buffer = bytearray()
+
+        self._start()
+
+    def _start(self):
+        resp = self.s3.create_multipart_upload(
+            Bucket=self.bucket,
+            Key=self.key,
+            ACL=self.acl,
+        )
+        self.upload_id = resp["UploadId"]
+
+    def upload_chunk(self, chunk: bytes):
+        """
+        Receives small chunks (KB) and buffers them
+        """
+        self.buffer.extend(chunk)
+
+        if len(self.buffer) >= self.MIN_PART_SIZE:
+            self._upload_part(bytes(self.buffer))
+            self.buffer.clear()
+
+    def _upload_part(self, data: bytes):
+        resp = self.s3.upload_part(
+            Bucket=self.bucket,
+            Key=self.key,
+            UploadId=self.upload_id,
+            PartNumber=self.part_number,
+            Body=data,
+        )
+
+        self.parts.append({
+            "ETag": resp["ETag"],
+            "PartNumber": self.part_number
+        })
+        self.part_number += 1
+
+    def complete(self):
+        # upload remaining buffer (can be < 5MB)
+        if self.buffer:
+            self._upload_part(bytes(self.buffer))
+            self.buffer.clear()
+
+        self.s3.complete_multipart_upload(
+            Bucket=self.bucket,
+            Key=self.key,
+            UploadId=self.upload_id,
+            MultipartUpload={"Parts": self.parts},
+        )
+
+    def abort(self):
+        if self.upload_id:
+            self.s3.abort_multipart_upload(
+                Bucket=self.bucket,
+                Key=self.key,
+                UploadId=self.upload_id,
+            )

{sharedkernel-2.2.6 → sharedkernel-2.3.0}/sharedkernel/s3_uploader.py

@@ -1,62 +1,64 @@
+
 import requests
 import boto3
 from io import BytesIO
 import uuid
 import os
+from sharedkernel.multipart_upload import MultipartUploadSession
 
 class S3Uploader:
     def __init__(self, endpoint_url, bucket, access_key, secret_key):
-        """
-        Initializes the S3Uploader with credentials and optional endpoint/region.
-
-        :param endpoint_url: Custom endpoint URL (for S3-compatible services like MinIO)
-        :param bucket: Name of S3 bucket
-        :param access_key: Access key
-        :param secret_key: Secret key
-        """
         self.endpoint_url = endpoint_url
         self.bucket = bucket
         self.access_key = access_key
         self.secret_key = secret_key
 
-        # Initialize the S3 client
         self.s3 = boto3.client(
             's3',
             endpoint_url=self.endpoint_url,
             aws_access_key_id=self.access_key,
             aws_secret_access_key=self.secret_key,
         )
-
-    def upload_file_object(self, file_obj, object_name=None, file_extension=None, folder_name=None):
-        """
-        Uploads a file object to an S3 bucket and returns its URL.
-
-        :param file_obj: File object to upload
-        :param object_name: S3 file name to save as (optional)
-        :param file_extension: File extension to append to object name (optional)
-        :param folder_name: Optional folder name to save the file in
-        :return: URL of the uploaded file if successful, else False
-        """
-        # Use uuid4 to generate a unique object_name if not provided
+    
+    @staticmethod
+    def __generate_object_name(object_name=None, file_extension=None, folder_name=None):
         if object_name is None:
             object_name = str(uuid.uuid4())
 
-        # Append the file extension to the object_name
         if file_extension:
             object_name += file_extension
 
-        # Prefix the object name with the folder if provided
         if folder_name:
             object_name = f"{folder_name}/{object_name}"
+        
+        return object_name
+
 
-        # Upload the file object
-        self.s3.upload_fileobj(file_obj, self.bucket, object_name, ExtraArgs={'ACL':'public-read'})
+    def upload_file_object(self, file_obj, object_name=None, file_extension=None, folder_name=None, public_read=True):
+        object_name = self.__generate_object_name(object_name, file_extension, folder_name)
+
+        acl = "public-read" if public_read else "private"
+
+        self.s3.upload_fileobj(file_obj, self.bucket, object_name, ExtraArgs={'ACL': acl})
 
-        # Construct the URL of the uploaded file
         file_url = f"{self.endpoint_url}/{self.bucket}/{object_name}"
 
         return file_url
 
+    def generate_presigned_url(self, object_name=None, file_extension=None, folder_name=None, expire_time: int = 3600):
+        object_name = self.__generate_object_name(object_name, file_extension, folder_name)
+
+        temp_url = self.s3.generate_presigned_url(
+           'get_object',
+           Params={
+               'Bucket': self.bucket,
+               'Key': object_name
+           },
+           ExpiresIn=expire_time
+       )
+        
+        return temp_url
+        
     def upload_file_from_url(self, file_url, object_name=None, folder_name=None):
         """
         Downloads a file from a URL and uploads it to an S3 bucket, returning its URL.
@@ -80,3 +82,25 @@ class S3Uploader:
         # Step 2: Upload the file to S3
         file_obj = BytesIO(response.content)
         return self.upload_file_object(file_obj=file_obj, object_name=object_name, file_extension=file_extension, folder_name=folder_name)
+
+    def create_multipart_session(
+        self,
+        object_name=None,
+        file_extension=None,
+        folder_name=None,
+        public_read=True,
+    ):
+        object_key = self.__generate_object_name(
+            object_name, file_extension, folder_name
+        )
+
+        acl = "public-read" if public_read else "private"
+
+        return MultipartUploadSession(
+            s3_client=self.s3,
+            bucket=self.bucket,
+            object_key=object_key,
+            acl=acl,
+        )
+
+

{sharedkernel-2.2.6 → sharedkernel-2.3.0}/sharedkernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sharedkernel
-Version: 2.2.6
+Version: 2.3.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -29,6 +29,10 @@ Dynamic: summary
 this is a shared kernel package
 
 # Change Log
+### Version 2.3.0
+- Implement multipart uploader for S3
+### Version 2.2.7
+- add presigned url in s3 uploader
 ### Version 2.2.6
 - implement api key
 ### Version 2.2.5

{sharedkernel-2.2.6 → sharedkernel-2.3.0}/sharedkernel.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ sharedkernel/data_format_converter.py
 sharedkernel/date_converter.py
 sharedkernel/diff_utils.py
 sharedkernel/jwt_service.py
+sharedkernel/multipart_upload.py
 sharedkernel/regex_masking.py
 sharedkernel/s3_uploader.py
 sharedkernel/string_extentions.py