rclone-api 1.1.42__tar.gz → 1.1.44__tar.gz

{rclone_api-1.1.42 → rclone_api-1.1.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.42
+Version: 1.1.44
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.42 → rclone_api-1.1.44}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.42"
+version = "1.1.44"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.42 → rclone_api-1.1.44}/src/rclone_api/rclone.py

@@ -832,15 +832,22 @@ class Rclone:
         tmp_mnt = Path("tmp_mnt") / random_str(12)
         src_parent_path = Path(src).parent.as_posix()
         src_file = Path(src).name
-        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
+        other_args: list[str] = [
+            "--no-modtime",
+            # "--vfs-read-wait", "1s"
+        ]
         vfs_read_chunk_size = SizeSuffix(length // transfers)
         vfs_read_chunk_size_limit = SizeSuffix(length)
         vfs_read_chunk_streams = transfers
         vfs_disk_space_total_size = SizeSuffix(length)
+        # --max-read-ahead SizeSuffix
+        max_read_ahead = SizeSuffix(vfs_read_chunk_size.as_int())
+
         other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
         other_args += ["--vfs-read-chunk-size-limit", str(vfs_read_chunk_size_limit)]
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
         other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
+        other_args += ["--max-read-ahead", str(max_read_ahead)]
         other_args += ["--read-only"]
         if direct_io:
             other_args += ["--direct-io"]
@@ -920,7 +927,7 @@ class Rclone:
 
         allow_writes = allow_writes or False
         use_links = use_links or True
-        verbose = get_verbose(verbose)
+        verbose = get_verbose(verbose) or (log is not None)
         vfs_cache_mode = vfs_cache_mode or "full"
         clean_mount(outdir, verbose=verbose)
         prepare_mount(outdir, verbose=verbose)

{rclone_api-1.1.42 → rclone_api-1.1.44}/src/rclone_api/s3/chunk_file.py

@@ -2,6 +2,7 @@ import time
 import warnings
 from pathlib import Path
 from queue import Queue
+from threading import Event
 
 from rclone_api.s3.chunk_types import FileChunk, UploadState
 from rclone_api.util import locked_print
@@ -25,12 +26,16 @@ def _get_file_size(file_path: Path, timeout: int = 60) -> int:
 
 
 def file_chunker(
-    upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
+    upload_state: UploadState,
+    max_chunks: int | None,
+    cancel_signal: Event,
+    output: Queue[FileChunk | None],
 ) -> None:
     count = 0
 
     def should_stop() -> bool:
         nonlocal count
+
         if max_chunks is None:
             return False
         if count >= max_chunks:
@@ -68,6 +73,12 @@ def file_chunker(
                 return None
             return part_number
 
+        if cancel_signal.is_set():
+            print(
+                f"Cancel signal is set for file chunker while processing {file_path}, returning"
+            )
+            return
+
         while not should_stop():
             curr_parth_num = next_part_number()
             if curr_parth_num is None:

{rclone_api-1.1.42 → rclone_api-1.1.44}/src/rclone_api/s3/upload_file_multipart.py

@@ -5,7 +5,7 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from queue import Queue
-from threading import Thread
+from threading import Event, Thread
 
 from botocore.client import BaseClient
 
@@ -207,16 +207,21 @@ def upload_file_multipart(
     upload_info = upload_state.upload_info
 
     chunker_errors: Queue[Exception] = Queue()
+    cancel_chunker_event = Event()
 
     def chunker_task(
         upload_state=upload_state,
         output=filechunks,
         max_chunks=max_chunks_before_suspension,
+        cancel_signal=cancel_chunker_event,
         queue_errors=chunker_errors,
     ) -> None:
         try:
             file_chunker(
-                upload_state=upload_state, output=output, max_chunks=max_chunks
+                upload_state=upload_state,
+                output=output,
+                max_chunks=max_chunks,
+                cancel_signal=cancel_signal,
             )
         except Exception as e:
             queue_errors.put(e)
@@ -228,25 +233,30 @@ def upload_file_multipart(
         thread_chunker.start()
 
         with ThreadPoolExecutor(max_workers=upload_threads) as executor:
-            while True:
-                file_chunk: FileChunk | None = filechunks.get()
-                if file_chunk is None:
-                    break
+            try:
+                while True:
+                    file_chunk: FileChunk | None = filechunks.get()
+                    if file_chunk is None:
+                        break
 
-                def task(upload_info=upload_info, file_chunk=file_chunk):
-                    return handle_upload(upload_info, file_chunk)
+                    def task(upload_info=upload_info, file_chunk=file_chunk):
+                        return handle_upload(upload_info, file_chunk)
 
-                fut = executor.submit(task)
+                    fut = executor.submit(task)
 
-                def done_cb(fut=fut):
-                    result = fut.result()
-                    if isinstance(result, Exception):
-                        warnings.warn(f"Error uploading part: {result}, skipping")
-                        return
-                    # upload_state.finished_parts.put(result)
-                    upload_state.add_finished(result)
+                    def done_cb(fut=fut):
+                        result = fut.result()
+                        if isinstance(result, Exception):
+                            warnings.warn(f"Error uploading part: {result}, skipping")
+                            return
+                        # upload_state.finished_parts.put(result)
+                        upload_state.add_finished(result)
 
-                fut.add_done_callback(done_cb)
+                    fut.add_done_callback(done_cb)
+            except Exception:
+                cancel_chunker_event.set()
+                executor.shutdown(wait=False, cancel_futures=True)
+                raise
         # upload_state.finished_parts.put(None)  # Signal the end of the queue
         upload_state.add_finished(None)
         thread_chunker.join()

{rclone_api-1.1.42 → rclone_api-1.1.44}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.42
+Version: 1.1.44
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License