rclone-api 1.1.43__py2.py3-none-any.whl → 1.1.45__py2.py3-none-any.whl

rclone_api/mount.py

@@ -31,7 +31,13 @@ class Mount:
         if self._closed:
             return
         self._closed = True
-        clean_mount(self, verbose=False)
+        clean_mount(self, verbose=False, wait=wait)
+
+    def __enter__(self) -> "Mount":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close(wait=True)
 
     def __del__(self):
         self.close(wait=False)

rclone_api/profile/mount_copy_bytes.py

@@ -3,6 +3,7 @@ Unit test file.
 """
 
 import os
+import shutil
 import time
 from dataclasses import dataclass
 from pathlib import Path
@@ -103,10 +104,11 @@ def _init() -> None:
     os.environ["RCLONE_API_VERBOSE"] = "1"
 
 
-def _run_profile(rclone: Rclone, src_file: str, transfers: int, size: int) -> None:
-    mount_log = (
-        Path("logs") / "mount" / f"mount_{SizeSuffix(size)}_threads_{transfers}.log"
-    )
+def _run_profile(
+    rclone: Rclone, src_file: str, transfers: int, size: int, log_dir: Path
+) -> None:
+
+    mount_log = log_dir / f"mount_{SizeSuffix(size)}_threads_{transfers}.log"
     print("\n\n")
     print("#" * 80)
     print(f"# Started test download of {SizeSuffix(size)} with {transfers} transfers")
@@ -177,10 +179,18 @@ def test_profile_copy_bytes() -> None:
     # sftp mount
     src_file = "src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
 
+    mount_root_path = Path("logs") / "mount"
+    if mount_root_path.exists():
+        shutil.rmtree(mount_root_path)
+
     for size in sizes:
         for transfers in transfer_list:
             _run_profile(
-                rclone=rclone, src_file=src_file, transfers=transfers, size=size
+                rclone=rclone,
+                src_file=src_file,
+                transfers=transfers,
+                size=size,
+                log_dir=mount_root_path,
             )
     print("done")
 

rclone_api/rclone.py

@@ -832,7 +832,10 @@ class Rclone:
         tmp_mnt = Path("tmp_mnt") / random_str(12)
         src_parent_path = Path(src).parent.as_posix()
         src_file = Path(src).name
-        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
+        other_args: list[str] = [
+            "--no-modtime",
+            # "--vfs-read-wait", "1s"
+        ]
         vfs_read_chunk_size = SizeSuffix(length // transfers)
         vfs_read_chunk_size_limit = SizeSuffix(length)
         vfs_read_chunk_streams = transfers
@@ -924,10 +927,11 @@ class Rclone:
 
         allow_writes = allow_writes or False
         use_links = use_links or True
-        verbose = get_verbose(verbose)
+        verbose = get_verbose(verbose) or (log is not None)
         vfs_cache_mode = vfs_cache_mode or "full"
         clean_mount(outdir, verbose=verbose)
         prepare_mount(outdir, verbose=verbose)
+        debug_fuse = log is not None
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:
@@ -937,12 +941,14 @@ class Rclone:
         if vfs_cache_mode:
             cmd_list.append("--vfs-cache-mode")
             cmd_list.append(vfs_cache_mode)
+        if debug_fuse:
+            cmd_list.append("--debug-fuse")
         if verbose:
             cmd_list.append("-vvvv")
         if other_args:
             cmd_list += other_args
         proc = self._launch_process(cmd_list, log=log)
-        mount_read_only = "--read-only" in cmd_list  # hint to allow fast teardown
+        mount_read_only = not allow_writes
         mount: Mount = Mount(mount_path=outdir, process=proc, read_only=mount_read_only)
         return mount
 
@@ -960,7 +966,6 @@ class Rclone:
     ) -> Generator[Mount, None, None]:
         """Like mount, but can be used in a context manager."""
         error_happened = False
-        verbose = get_verbose(verbose)
         mount: Mount = self.mount(
             src,
             outdir,
@@ -979,7 +984,7 @@ class Rclone:
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
         finally:
-            if not error_happened:
+            if not error_happened or (not allow_writes):
                 mount.close()
 
     # Settings optimized for s3.

rclone_api/s3/chunk_file.py

@@ -2,6 +2,7 @@ import time
 import warnings
 from pathlib import Path
 from queue import Queue
+from threading import Event
 
 from rclone_api.s3.chunk_types import FileChunk, UploadState
 from rclone_api.util import locked_print
@@ -25,12 +26,16 @@ def _get_file_size(file_path: Path, timeout: int = 60) -> int:
 
 
 def file_chunker(
-    upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
+    upload_state: UploadState,
+    max_chunks: int | None,
+    cancel_signal: Event,
+    output: Queue[FileChunk | None],
 ) -> None:
     count = 0
 
     def should_stop() -> bool:
         nonlocal count
+
         if max_chunks is None:
             return False
         if count >= max_chunks:
@@ -68,6 +73,12 @@ def file_chunker(
                 return None
             return part_number
 
+        if cancel_signal.is_set():
+            print(
+                f"Cancel signal is set for file chunker while processing {file_path}, returning"
+            )
+            return
+
         while not should_stop():
             curr_parth_num = next_part_number()
             if curr_parth_num is None:

rclone_api/s3/upload_file_multipart.py

@@ -5,7 +5,7 @@ import warnings
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from queue import Queue
-from threading import Thread
+from threading import Event, Thread
 
 from botocore.client import BaseClient
 
@@ -207,16 +207,21 @@ def upload_file_multipart(
     upload_info = upload_state.upload_info
 
     chunker_errors: Queue[Exception] = Queue()
+    cancel_chunker_event = Event()
 
     def chunker_task(
         upload_state=upload_state,
         output=filechunks,
         max_chunks=max_chunks_before_suspension,
+        cancel_signal=cancel_chunker_event,
         queue_errors=chunker_errors,
     ) -> None:
         try:
             file_chunker(
-                upload_state=upload_state, output=output, max_chunks=max_chunks
+                upload_state=upload_state,
+                output=output,
+                max_chunks=max_chunks,
+                cancel_signal=cancel_signal,
             )
         except Exception as e:
             queue_errors.put(e)
@@ -228,25 +233,30 @@ def upload_file_multipart(
         thread_chunker.start()
 
         with ThreadPoolExecutor(max_workers=upload_threads) as executor:
-            while True:
-                file_chunk: FileChunk | None = filechunks.get()
-                if file_chunk is None:
-                    break
+            try:
+                while True:
+                    file_chunk: FileChunk | None = filechunks.get()
+                    if file_chunk is None:
+                        break
 
-                def task(upload_info=upload_info, file_chunk=file_chunk):
-                    return handle_upload(upload_info, file_chunk)
+                    def task(upload_info=upload_info, file_chunk=file_chunk):
+                        return handle_upload(upload_info, file_chunk)
 
-                fut = executor.submit(task)
+                    fut = executor.submit(task)
 
-                def done_cb(fut=fut):
-                    result = fut.result()
-                    if isinstance(result, Exception):
-                        warnings.warn(f"Error uploading part: {result}, skipping")
-                        return
-                    # upload_state.finished_parts.put(result)
-                    upload_state.add_finished(result)
+                    def done_cb(fut=fut):
+                        result = fut.result()
+                        if isinstance(result, Exception):
+                            warnings.warn(f"Error uploading part: {result}, skipping")
+                            return
+                        # upload_state.finished_parts.put(result)
+                        upload_state.add_finished(result)
 
-                fut.add_done_callback(done_cb)
+                    fut.add_done_callback(done_cb)
+            except Exception:
+                cancel_chunker_event.set()
+                executor.shutdown(wait=False, cancel_futures=True)
+                raise
         # upload_state.finished_parts.put(None)  # Signal the end of the queue
         upload_state.add_finished(None)
         thread_chunker.join()

{rclone_api-1.1.43.dist-info → rclone_api-1.1.45.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.43
+Version: 1.1.45
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.43.dist-info → rclone_api-1.1.45.dist-info}/RECORD

@@ -11,9 +11,9 @@ rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/mount.py,sha256=ryAjkX4_kFeFZWLiBVpcGy2VilpvVhFbWeWfEX4jMKs,6104
+rclone_api/mount.py,sha256=RgzGfJakHxgiTsLu0bFX4ZYIzr4GAdIH8CMlz-M-hsY,6267
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
-rclone_api/rclone.py,sha256=J_8UNI1aVARaWA3_EMoCU0z_oabt05nstcWD2hEHsCk,42829
+rclone_api/rclone.py,sha256=-cMgLNykn4JFP83-sFeVWArFl28173LXgdWvpPr0_rE,42933
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -25,17 +25,17 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=Jsvb49CNTvkdSAkIAKDxBJ5TzAk7-dOuZdDrigiTjsY,5684
+rclone_api/profile/mount_copy_bytes.py,sha256=6hJJ7jo8brFzq-t96JCjQeCIYnzYT1Km9vRL1jbUyLo,5897
 rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_file.py,sha256=YELR-EzR7RHpzCDGpYdzlwu21NZW5wttIDvLoONI4aU,3477
+rclone_api/s3/chunk_file.py,sha256=D6wM9Nuu73LNGC8JCCfevqjF3qdZ21mQxYQClFLZLMU,3726
 rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,10316
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
-rclone_api/s3/upload_file_multipart.py,sha256=1jQAdk35Fa9Tcq36bS65262cs7AcNG2DAFQ-NdYlWSw,9961
-rclone_api-1.1.43.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.43.dist-info/METADATA,sha256=cL7VgGdbjfGJSJ7f3UO8ON4ucWQ_stuhj9vyKIUkAwE,4537
-rclone_api-1.1.43.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.43.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.1.43.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.43.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=y9azNAU8QH5Ovwz33V2HZwNmJdlFjJg-jrXLZ1gtMds,10364
+rclone_api-1.1.45.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.45.dist-info/METADATA,sha256=ZLx1uXz3hoeN194vwCxzj98SdfbBtdfRAeDtMKqafE0,4537
+rclone_api-1.1.45.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.45.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.1.45.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.45.dist-info/RECORD,,