rclone-api 1.3.27__py2.py3-none-any.whl → 1.3.28__py2.py3-none-any.whl

rclone_api/s3/s3_multipart_uploader.py

@@ -0,0 +1,28 @@
+import _thread
+import os
+import traceback
+import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from queue import Queue
+from threading import Event, Thread
+from typing import Any, Callable
+
+from botocore.client import BaseClient
+
+from rclone_api.mount_read_chunker import FilePart
+from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
+from rclone_api.s3.chunk_types import (
+    FinishedPiece,
+    UploadInfo,
+    UploadState,
+)
+from rclone_api.s3.types import MultiUploadResult
+from rclone_api.types import EndOfStream
+from rclone_api.util import locked_print
+
+
+class S3MultiPartUploader:
+
+    def __init__(s3_client: BaseClient, bucket_name: str, object_name: str, chunk_size: int, file_size: int):
+        pass

rclone_api/s3/upload_file_multipart.py

@@ -149,6 +149,10 @@ def upload_runner(
     queue_upload: Queue[FilePart | EndOfStream],
     cancel_chunker_event: Event,
 ) -> None:
+    # import semaphre
+    import threading
+
+    semaphore = threading.Semaphore(upload_threads)
     with ThreadPoolExecutor(max_workers=upload_threads) as executor:
         try:
             while True:
@@ -159,9 +163,12 @@ def upload_runner(
                 def task(upload_info=upload_info, file_chunk=file_chunk):
                     return handle_upload(upload_info, file_chunk)
 
+                semaphore.acquire()
+
                 fut = executor.submit(task)
 
                 def done_cb(fut=fut):
+                    semaphore.release()
                     result = fut.result()
                     if isinstance(result, Exception):
                         warnings.warn(f"Error uploading part: {result}, skipping")
@@ -233,7 +240,7 @@ def upload_file_multipart(
         )
         return upload_state
 
-    work_que_max = upload_threads // 2 + 2
+    work_que_max = 1
 
     new_state = make_new_state()
     loaded_state = get_upload_state()

{rclone_api-1.3.27.dist-info → rclone_api-1.3.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.27
+Version: 1.3.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.27.dist-info → rclone_api-1.3.28.dist-info}/RECORD

@@ -40,11 +40,12 @@ rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,21
 rclone_api/s3/chunk_task.py,sha256=AanVCygDoUjmMOUdEIYl-hEpPEGSJTIU_MSFGQ0tI0Q,7421
 rclone_api/s3/chunk_types.py,sha256=oSWv8No9V3BeM7IcGnowyR2a7YrszdAXzEJlxaeZcp0,8852
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
+rclone_api/s3/s3_multipart_uploader.py,sha256=JxUjf-Dpeq_JnMr8RBGuNCFcljGr_GM11529W53n3AM,766
 rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
-rclone_api/s3/upload_file_multipart.py,sha256=bxAB_SFkyydBBzC_cAv7gShe93nBkFVS-TXwM2Xttfk,12425
-rclone_api-1.3.27.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.3.27.dist-info/METADATA,sha256=r9lu8qOOxcB0zfmU6Pnw9eQBo0PvivUPKWtlt2b_XEo,4628
-rclone_api-1.3.27.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.3.27.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.3.27.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.3.27.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=55TXp9q1It5nKkwBo49nCZNgvdMsqvIGByDMBFzo_ik,12576
+rclone_api-1.3.28.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.3.28.dist-info/METADATA,sha256=JkcK7CZWmK6d5PEF3vYvlWIOGnb7ZIXNraPEH3HQUHM,4628
+rclone_api-1.3.28.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.3.28.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.3.28.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.3.28.dist-info/RECORD,,