rclone-api 1.1.61__py2.py3-none-any.whl → 1.1.63__py2.py3-none-any.whl

rclone_api/mount.py

@@ -6,12 +6,14 @@ import subprocess
 import time
 import warnings
 import weakref
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
+from threading import Lock, Semaphore
 from typing import Any
 
 from rclone_api.process import Process
+from rclone_api.types import SizeSuffix
 
 _SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
 
@@ -283,3 +285,83 @@ def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
             shutil.rmtree(cache_dir)
         except Exception as e:
             warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
+
+
+class MultiMountFileChunker:
+    def __init__(
+        self,
+        filename: str,
+        chunk_size: SizeSuffix,
+        mounts: list[Mount],
+        executor: ThreadPoolExecutor,
+    ) -> None:
+        self.filename = filename
+        self.chunk_size = chunk_size
+        self.executor = executor
+        self.mounts_processing: list[Mount] = []
+        self.mounts_availabe: list[Mount] = mounts
+        self.semaphore = Semaphore(len(mounts))
+        self.lock = Lock()
+
+    def close(self) -> None:
+        self.executor.shutdown(wait=False, cancel_futures=True)
+        with ThreadPoolExecutor() as executor:
+            for mount in self.mounts_processing:
+                executor.submit(lambda: mount.close())
+
+    def fetch(self, offset: int, size: int) -> bytes | Exception:
+        try:
+            assert offset % self.chunk_size.as_int() == 0
+            assert size <= self.chunk_size.as_int()
+            assert size > 0
+            assert offset >= 0
+
+            chunks: list[tuple[int, int]] = []
+            start = offset
+            end = offset + size
+            while start < end:
+                chunk_size = min(self.chunk_size.as_int(), end - start)
+                chunks.append((start, chunk_size))
+                start += chunk_size
+
+            futures: list[Future[bytes | Exception]] = []
+            for start, chunk_size in chunks:
+                self.semaphore.acquire()
+                with self.lock:
+                    mount = self.mounts_availabe.pop()
+                    self.mounts_processing.append(mount)
+
+                path = mount.mount_path / self.filename
+
+                def task() -> bytes | Exception:
+                    try:
+                        with path.open("rb") as f:
+                            f.seek(offset)
+                            return f.read(size)
+                    except KeyboardInterrupt as e:
+                        import _thread
+
+                        _thread.interrupt_main()
+                        return Exception(e)
+                    except Exception as e:
+                        return e
+                    finally:
+                        with self.lock:
+                            self.mounts_processing.remove(mount)
+                            self.mounts_availabe.append(mount)
+                            self.semaphore.release()
+
+                fut = self.executor.submit(task)
+                futures.append(fut)
+
+            out: bytes = b""
+            for fut in futures:
+                chunk: bytes | Exception = fut.result()
+                if isinstance(chunk, Exception):
+                    return chunk
+                else:
+                    out += chunk
+                    del chunk
+            return out
+        except Exception as e:
+            return e

rclone_api/profile/mount_copy_bytes.py

@@ -133,6 +133,7 @@ def _run_profile(
     if isinstance(bytes_or_err, Exception):
         print(bytes_or_err)
         stack_trace = bytes_or_err.__traceback__
+        print(f"Error: {bytes_or_err}\nStack trace:\n{stack_trace}")
         assert False, f"Error: {bytes_or_err}\nStack trace:\n{stack_trace}"
     assert isinstance(bytes_or_err, bytes)
     # self.assertEqual(len(bytes_or_err), size)

rclone_api/rclone.py

@@ -25,7 +25,7 @@ from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
 from rclone_api.group_files import group_files
-from rclone_api.mount import Mount, clean_mount, prepare_mount
+from rclone_api.mount import Mount, MultiMountFileChunker, clean_mount, prepare_mount
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
@@ -829,7 +829,7 @@ class Rclone:
         """Copy bytes from a file to another file."""
         from rclone_api.util import random_str
 
-        tmp_mnt = Path("tmp_mnt") / random_str(12)
+        tmp_mnts = Path("tmp_mnts") / random_str(12)
         src_parent_path = Path(src).parent.as_posix()
         src_file = Path(src).name
         other_args: list[str] = [
@@ -860,7 +860,7 @@ class Rclone:
                 # use scoped mount to do the read, then write the bytes to the destination
                 with self.scoped_mount(
                     src_parent_path,
-                    tmp_mnt,
+                    tmp_mnts,
                     use_links=True,
                     verbose=mount_log is not None,
                     vfs_cache_mode="minimal",
@@ -869,7 +869,7 @@ class Rclone:
                     cache_dir=cache_dir,
                     cache_dir_delete_on_exit=True,
                 ):
-                    src_file_mnt = tmp_mnt / src_file
+                    src_file_mnt = tmp_mnts / src_file
                     with open(src_file_mnt, "rb") as f:
                         f.seek(offset)
                         data = f.read(length)
@@ -883,6 +883,85 @@ class Rclone:
             except Exception as e:
                 return e
 
+    def get_multi_mount_file_chunker(
+        self,
+        src: str,
+        chunk_size: SizeSuffix,
+        threads: int,
+        mount_log: Path | None,
+        direct_io: bool,
+    ) -> MultiMountFileChunker:
+        from rclone_api.util import random_str
+
+        mounts: list[Mount] = []
+        vfs_read_chunk_size = chunk_size
+        vfs_read_chunk_size_limit = chunk_size
+        vfs_read_chunk_streams = 0
+        vfs_disk_space_total_size = chunk_size
+        other_args: list[str] = []
+        other_args += ["--no-modtime"]
+        other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
+        other_args += [
+            "--vfs-read-chunk-size-limit",
+            vfs_read_chunk_size_limit.as_str(),
+        ]
+        other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
+        other_args += [
+            "--vfs-disk-space-total-size",
+            vfs_disk_space_total_size.as_str(),
+        ]
+        other_args += ["--read-only"]
+        if direct_io:
+            other_args += ["--direct-io"]
+
+        filename = Path(src).name
+        with ThreadPoolExecutor(max_workers=threads) as executor:
+            futures: list[Future] = []
+            try:
+                for i in range(threads):
+                    tmp_mnts = Path("tmp_mnts") / random_str(12)
+                    verbose = mount_log is not None
+                    clean_mount(tmp_mnts, verbose=verbose)
+                    prepare_mount(tmp_mnts, verbose=verbose)
+                    src_parent_path = Path(src).parent.as_posix()
+
+                    def task(src_parent_path=src_parent_path, tmp_mnts=tmp_mnts):
+                        return self.mount(
+                            src=src_parent_path,
+                            outdir=tmp_mnts,
+                            allow_writes=False,
+                            use_links=True,
+                            vfs_cache_mode="minimal",
+                            verbose=False,
+                            cache_dir=Path("cache") / random_str(12),
+                            cache_dir_delete_on_exit=True,
+                            log=mount_log,
+                            other_args=other_args,
+                        )
+
+                    futures.append(executor.submit(task))
+                mount_errors: list[Exception] = []
+                for fut in futures:
+                    try:
+                        mount = fut.result()
+                        mounts.append(mount)
+                    except Exception as er:
+                        mount_errors.append(er)
+                if mount_errors:
+                    raise Exception(mount_errors)
+            except Exception:
+                for mount in mounts:
+                    mount.close()
+                raise
+        executor = ThreadPoolExecutor(max_workers=threads)
+        filechunker: MultiMountFileChunker = MultiMountFileChunker(
+            filename=filename,
+            chunk_size=chunk_size,
+            mounts=mounts,
+            executor=executor,
+        )
+        return filechunker
+
     def copy_bytes(
         self,
         src: str,
@@ -897,49 +976,28 @@ class Rclone:
     ) -> bytes | Exception:
         """Copy bytes from a file to another file."""
         # determine number of threads from chunk size
-        threads = min(max_threads, length // chunk_size.as_int())
-
-        if threads == 1:
-            return self._copy_bytes(
-                src,
-                offset,
-                length,
-                transfers=1,
-                outfile=outfile,
-                mount_log=mount_log,
-                direct_io=direct_io,
-            )
-        else:
-            futures: list[Future] = []
-            with ThreadPoolExecutor(max_workers=threads) as executor:
-                for i in range(threads):
-                    start = i * chunk_size.as_int()
-                    end = start + chunk_size.as_int()
-                    if i == threads - 1:
-                        end = length
-                    future = executor.submit(
-                        self._copy_bytes,
-                        src,
-                        offset + start,
-                        end - start,
-                        transfers=1,
-                        outfile=None,
-                        mount_log=mount_log,
-                        direct_io=direct_io,
-                    )
-                    futures.append(future)
-                data: bytes = b""
-                for future in futures:
-                    chunk: bytes | Exception = future.result()
-                    if isinstance(chunk, Exception):
-                        return chunk
-                    data += chunk
-                if outfile is not None:
-                    with open(outfile, "wb") as out:
-                        out.write(data)
-                        del data
-                        return bytes(0)
+        threads = max(1, min(max_threads, length // chunk_size.as_int()))
+        filechunker = self.get_multi_mount_file_chunker(
+            src=src,
+            chunk_size=chunk_size,
+            threads=threads,
+            mount_log=mount_log,
+            direct_io=direct_io,
+        )
+        try:
+            data = filechunker.fetch(offset, length)
+            if isinstance(data, Exception):
+                raise data
+            if outfile is None:
                 return data
+            with open(outfile, "wb") as out:
+                out.write(data)
+                del data
+            return bytes(0)
+        except Exception as e:
+            return e
+        finally:
+            filechunker.close()
 
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None

{rclone_api-1.1.61.dist-info → rclone_api-1.1.63.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.61
+Version: 1.1.63
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.61.dist-info → rclone_api-1.1.63.dist-info}/RECORD

@@ -11,9 +11,9 @@ rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/mount.py,sha256=5XiFwP7tst4h8cGQstZMJ0u5VzqFQ4YehnDl2AFP11A,10057
+rclone_api/mount.py,sha256=hd6_zmJ-wH337F0eh9JJMMlVzj92m32tLm2t5qZFxNc,12931
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
-rclone_api/rclone.py,sha256=_bXkNBKeAVfjEOmX7bEwpxlXaJURJ3RguY1TJzwIBzM,46038
+rclone_api/rclone.py,sha256=Sd4NrCoCwIe8wfOEpJ-eL6BRozIvCHI6fhnr6BI2ET8,48226
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -25,7 +25,7 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=KJhn2ufpRwT9c9bH_q2RWvklw5x63nvI-UMtv9O8CPg,7717
+rclone_api/profile/mount_copy_bytes.py,sha256=lpyviVpOyYq7E4HxKMy_YrCwx4wfy00mr4cn86TkE6I,7786
 rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=D6wM9Nuu73LNGC8JCCfevqjF3qdZ21mQxYQClFLZLMU,3726
@@ -33,9 +33,9 @@ rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
 rclone_api/s3/upload_file_multipart.py,sha256=y9azNAU8QH5Ovwz33V2HZwNmJdlFjJg-jrXLZ1gtMds,10364
-rclone_api-1.1.61.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.61.dist-info/METADATA,sha256=K94aUUT1DvHqha4sUrmbKR18VbBosxwPVUW0tQLAKqU,4537
-rclone_api-1.1.61.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.61.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.1.61.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.61.dist-info/RECORD,,
+rclone_api-1.1.63.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.63.dist-info/METADATA,sha256=kJBxXPg5Yzl1wxdJqir5X4YGJijUJ4Br6c_tlhbv0Lc,4537
+rclone_api-1.1.63.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.63.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.1.63.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.63.dist-info/RECORD,,