rclone-api 1.3.22__py2.py3-none-any.whl → 1.3.24__py2.py3-none-any.whl

rclone_api/rclone.py

@@ -831,9 +831,9 @@ class Rclone:
         vfs_read_chunk_size_limit = chunk_size
         vfs_read_chunk_streams = read_threads
         vfs_disk_space_total_size = chunk_size
-        assert (
-            chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
-        ), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
+        # assert (
+        #     chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
+        # ), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
         other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
         other_args += [
             "--vfs-read-chunk-size-limit",

rclone_api/s3/chunk_task.py

@@ -189,6 +189,8 @@ def file_chunker(
             )
             fut.add_done_callback(callback.on_complete)
             # wait until the queue_upload queue can accept the next chunk
+            qsize = queue_upload.qsize()
+            print(f"queue_upload_size: {qsize}")
             while queue_upload.full():
                 time.sleep(0.1)
     except Exception as e:

rclone_api/s3/upload_file_multipart.py

@@ -25,7 +25,10 @@ _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
 
 
 def upload_task(
-    info: UploadInfo, chunk: FilePart, part_number: int, retries: int
+    info: UploadInfo,
+    chunk: FilePart,
+    part_number: int,
+    retries: int,
 ) -> FinishedPiece:
     file_or_err: Path | Exception = chunk.get_file()
     if isinstance(file_or_err, Exception):
@@ -139,6 +142,40 @@ def _abort_previous_upload(upload_state: UploadState) -> None:
             locked_print(f"Error aborting previous upload: {e}")
 
 
+def upload_runner(
+    upload_state: UploadState,
+    upload_info: UploadInfo,
+    upload_threads: int,
+    queue_upload: Queue[FilePart | EndOfStream],
+    cancel_chunker_event: Event,
+) -> None:
+    with ThreadPoolExecutor(max_workers=upload_threads) as executor:
+        try:
+            while True:
+                file_chunk: FilePart | EndOfStream = queue_upload.get()
+                if isinstance(file_chunk, EndOfStream):
+                    break
+
+                def task(upload_info=upload_info, file_chunk=file_chunk):
+                    return handle_upload(upload_info, file_chunk)
+
+                fut = executor.submit(task)
+
+                def done_cb(fut=fut):
+                    result = fut.result()
+                    if isinstance(result, Exception):
+                        warnings.warn(f"Error uploading part: {result}, skipping")
+                        return
+                    # upload_state.finished_parts.put(result)
+                    upload_state.add_finished(result)
+
+                fut.add_done_callback(done_cb)
+        except Exception:
+            cancel_chunker_event.set()
+            executor.shutdown(wait=False, cancel_futures=True)
+            raise
+
+
 def upload_file_multipart(
     s3_client: BaseClient,
     chunk_fetcher: Callable[[int, int, Any], Future[FilePart]],
@@ -265,32 +302,13 @@ def upload_file_multipart(
     try:
         thread_chunker = Thread(target=chunker_task, daemon=True)
         thread_chunker.start()
-
-        with ThreadPoolExecutor(max_workers=upload_threads) as executor:
-            try:
-                while True:
-                    file_chunk: FilePart | EndOfStream = queue_upload.get()
-                    if isinstance(file_chunk, EndOfStream):
-                        break
-
-                    def task(upload_info=upload_info, file_chunk=file_chunk):
-                        return handle_upload(upload_info, file_chunk)
-
-                    fut = executor.submit(task)
-
-                    def done_cb(fut=fut):
-                        result = fut.result()
-                        if isinstance(result, Exception):
-                            warnings.warn(f"Error uploading part: {result}, skipping")
-                            return
-                        # upload_state.finished_parts.put(result)
-                        upload_state.add_finished(result)
-
-                    fut.add_done_callback(done_cb)
-            except Exception:
-                cancel_chunker_event.set()
-                executor.shutdown(wait=False, cancel_futures=True)
-                raise
+        upload_runner(
+            upload_state=upload_state,
+            upload_info=upload_info,
+            upload_threads=upload_threads,
+            queue_upload=queue_upload,
+            cancel_chunker_event=cancel_chunker_event,
+        )
         # upload_state.finished_parts.put(None)  # Signal the end of the queue
         upload_state.add_finished(EndOfStream())
         thread_chunker.join()

rclone_api/types.py

@@ -299,6 +299,7 @@ class FilePart:
             self.payload = payload
             return
         if isinstance(payload, bytes):
+            print(f"Creating file part with payload: {len(payload)}")
             self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
             with _TMP_DIR_ACCESS_LOCK:
                 if not self.payload.parent.exists():
@@ -306,7 +307,9 @@ class FilePart:
                 self.payload.write_bytes(payload)
             _add_for_cleanup(self.payload)
         if isinstance(payload, Path):
+            print("Adopting payload: ", payload)
             self.payload = payload
+            _add_for_cleanup(self.payload)
 
     def get_file(self) -> Path | Exception:
         return self.payload
@@ -344,18 +347,26 @@ class FilePart:
         return isinstance(self.payload, Exception)
 
     def dispose(self) -> None:
+        print("Disposing file part")
         with self._lock:
             if isinstance(self.payload, Exception):
                 warnings.warn(
                     f"Cannot close file part because the payload represents an error: {self.payload}"
                 )
+                print("Cannot close file part because the payload represents an error")
                 return
             if self.payload.exists():
+                print(f"File part {self.payload} exists")
                 try:
+                    print(f"Unlinking file part {self.payload}")
                     self.payload.unlink()
                     print(f"File part {self.payload} deleted")
                 except Exception as e:
                     warnings.warn(f"Cannot close file part because of error: {e}")
+            else:
+                warnings.warn(
+                    f"Cannot close file part because it does not exist: {self.payload}"
+                )
 
     def __del__(self):
         self.dispose()

{rclone_api-1.3.22.dist-info → rclone_api-1.3.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.22
+Version: 1.3.24
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.22.dist-info → rclone_api-1.3.24.dist-info}/RECORD

@@ -17,11 +17,11 @@ rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
 rclone_api/mount_read_chunker.py,sha256=7jaF1Rsjr-kXIZW--Ol1QuG7WArBgdIcpQ0AJMYn7bI,4764
 rclone_api/process.py,sha256=BGXJTZVT__jeaDyjN8_kRycliOhkBErMPdHO1hKRvJE,5271
-rclone_api/rclone.py,sha256=JYSilA0cFn6UQ0_72kqWgNhGyIAOqhbTilrbZkRnfXs,54111
+rclone_api/rclone.py,sha256=yRhQoCBJI-tfhxySR17a-vSEhWw5cMbk8_WbYs5WqRc,54117
 rclone_api/remote.py,sha256=jq3dPbAGvYZFW5cTviqxT2w6_jG2LLfS1RIcYSmMsQQ,503
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=aj3usJrIDKC4MKuObBo9WjlR1isyJColdOQWXo3Repo,10608
+rclone_api/types.py,sha256=CGwxlnhVgJumwSLukx68ZEM9m2gxLPDcAeVLbzS2JBA,11173
 rclone_api/util.py,sha256=F9Q3zbWRsgPF4NG6OWB63cZ7GVq82lsraP47gmmDohU,5416
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
@@ -37,14 +37,14 @@ rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ
 rclone_api/profile/mount_copy_bytes.py,sha256=M1vZn-Mrga14Ik7MHGZHbnwYli41Ep6Tyll7hQc7Wmo,9071
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_task.py,sha256=kA6_5fLNdtT3QdTFrfBY6y8sH9Og8nM2mrjgAz_g1Rc,7196
+rclone_api/s3/chunk_task.py,sha256=ZAsVOdiKae0aJ6_YjDuFYG8ZxCfF5jWWMToPM_Li9bo,7286
 rclone_api/s3/chunk_types.py,sha256=oSWv8No9V3BeM7IcGnowyR2a7YrszdAXzEJlxaeZcp0,8852
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
-rclone_api/s3/upload_file_multipart.py,sha256=UlrUl8fB0oK8_r0w8ZwH79jlOCHQrMOKWZeNCmHrT7M,12052
-rclone_api-1.3.22.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.3.22.dist-info/METADATA,sha256=jf9TZCxwZIaOD_4bqJQ_SrVJxygQS2fqlBuEHj0tEGc,4628
-rclone_api-1.3.22.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.3.22.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.3.22.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.3.22.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=bxAB_SFkyydBBzC_cAv7gShe93nBkFVS-TXwM2Xttfk,12425
+rclone_api-1.3.24.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.3.24.dist-info/METADATA,sha256=uP9r1gJ4ASyiISJg48vY86hiUBlvib2ZRRhJ_tBaSyg,4628
+rclone_api-1.3.24.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.3.24.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.3.24.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.3.24.dist-info/RECORD,,