rclone-api 1.3.26__tar.gz → 1.3.28__tar.gz

{rclone_api-1.3.26 → rclone_api-1.3.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.26
+Version: 1.3.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.26 → rclone_api-1.3.28}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.3.26"
+version = "1.3.28"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.3.28/src/rclone_api/s3/s3_multipart_uploader.py

@@ -0,0 +1,28 @@
+import _thread
+import os
+import traceback
+import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from queue import Queue
+from threading import Event, Thread
+from typing import Any, Callable
+
+from botocore.client import BaseClient
+
+from rclone_api.mount_read_chunker import FilePart
+from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
+from rclone_api.s3.chunk_types import (
+    FinishedPiece,
+    UploadInfo,
+    UploadState,
+)
+from rclone_api.s3.types import MultiUploadResult
+from rclone_api.types import EndOfStream
+from rclone_api.util import locked_print
+
+
+class S3MultiPartUploader:
+
+    def __init__(s3_client: BaseClient, bucket_name: str, object_name: str, chunk_size: int, file_size: int):
+        pass

{rclone_api-1.3.26 → rclone_api-1.3.28}/src/rclone_api/s3/upload_file_multipart.py

@@ -149,6 +149,10 @@ def upload_runner(
     queue_upload: Queue[FilePart | EndOfStream],
     cancel_chunker_event: Event,
 ) -> None:
+    # import semaphre
+    import threading
+
+    semaphore = threading.Semaphore(upload_threads)
     with ThreadPoolExecutor(max_workers=upload_threads) as executor:
         try:
             while True:
@@ -159,9 +163,12 @@ def upload_runner(
                 def task(upload_info=upload_info, file_chunk=file_chunk):
                     return handle_upload(upload_info, file_chunk)
 
+                semaphore.acquire()
+
                 fut = executor.submit(task)
 
                 def done_cb(fut=fut):
+                    semaphore.release()
                     result = fut.result()
                     if isinstance(result, Exception):
                         warnings.warn(f"Error uploading part: {result}, skipping")
@@ -233,7 +240,7 @@ def upload_file_multipart(
         )
         return upload_state
 
-    work_que_max = upload_threads // 2 + 2
+    work_que_max = 1
 
     new_state = make_new_state()
     loaded_state = get_upload_state()

{rclone_api-1.3.26 → rclone_api-1.3.28}/src/rclone_api/types.py

@@ -1,12 +1,13 @@
 import atexit
 import os
 import re
+import threading
 import time
 import warnings
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
-from threading import Lock, Thread
+from threading import Lock
 from typing import Any
 
 
@@ -291,35 +292,32 @@ atexit.register(_on_exit_cleanup)
 _FILEPARTS: list["FilePart"] = []
 
 _FILEPARTS_LOCK = Lock()
+
+
 def _add_filepart(part: "FilePart") -> None:
     with _FILEPARTS_LOCK:
         if part not in _FILEPARTS:
             _FILEPARTS.append(part)
 
+
 def _remove_filepart(part: "FilePart") -> None:
     with _FILEPARTS_LOCK:
         if part in _FILEPARTS:
             _FILEPARTS.remove(part)
 
 
-
-class ListFileParts(Thread):
-    def __init__(self):
-        super().__init__(daemon=True)
-        self.start()
-
-    def run(self):
-        while True:
-            print("File parts:")
-            for part in _FILEPARTS:
-                print(part)
-                print(part.stacktrace)
-                print("\n")
-            print("\n\n")
-            time.sleep(5)
+def run_debug_parts():
+    while True:
+        print("\nAlive file parts:")
+        for part in list(_FILEPARTS):
+            print(part)
+            # print(part.stacktrace)
+        print("\n\n")
+        time.sleep(60)
 
 
-dbg_thread = ListFileParts()
+dbg_thread = threading.Thread(target=run_debug_parts)
+dbg_thread.start()
 
 
 class FilePart:

{rclone_api-1.3.26 → rclone_api-1.3.28}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.26
+Version: 1.3.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.26 → rclone_api-1.3.28}/src/rclone_api.egg-info/SOURCES.txt

@@ -68,6 +68,7 @@ src/rclone_api/s3/basic_ops.py
 src/rclone_api/s3/chunk_task.py
 src/rclone_api/s3/chunk_types.py
 src/rclone_api/s3/create.py
+src/rclone_api/s3/s3_multipart_uploader.py
 src/rclone_api/s3/types.py
 src/rclone_api/s3/upload_file_multipart.py
 tests/test_cmd_list_files.py