rclone-api 1.1.60__py2.py3-none-any.whl → 1.1.62__py2.py3-none-any.whl

rclone_api/mount.py

@@ -6,12 +6,14 @@ import subprocess
 import time
 import warnings
 import weakref
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
+from threading import Lock, Semaphore
 from typing import Any
 
 from rclone_api.process import Process
+from rclone_api.types import SizeSuffix
 
 _SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
 
@@ -283,3 +285,83 @@ def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
             shutil.rmtree(cache_dir)
         except Exception as e:
             warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
+
+
+class MultiMountFileChunker:
+    def __init__(
+        self,
+        filename: str,
+        chunk_size: SizeSuffix,
+        mounts: list[Mount],
+        executor: ThreadPoolExecutor,
+    ) -> None:
+        self.filename = filename
+        self.chunk_size = chunk_size
+        self.executor = executor
+        self.mounts_processing: list[Mount] = []
+        self.mounts_availabe: list[Mount] = mounts
+        self.semaphore = Semaphore(len(mounts))
+        self.lock = Lock()
+
+    def close(self) -> None:
+        self.executor.shutdown(wait=False, cancel_futures=True)
+        with ThreadPoolExecutor() as executor:
+            for mount in self.mounts_processing:
+                executor.submit(lambda: mount.close())
+
+    def fetch(self, offset: int, size: int) -> bytes | Exception:
+        try:
+            assert offset % self.chunk_size.as_int() == 0
+            assert size <= self.chunk_size.as_int()
+            assert size > 0
+            assert offset >= 0
+
+            chunks: list[tuple[int, int]] = []
+            start = offset
+            end = offset + size
+            while start < end:
+                chunk_size = min(self.chunk_size.as_int(), end - start)
+                chunks.append((start, chunk_size))
+                start += chunk_size
+
+            futures: list[Future[bytes | Exception]] = []
+            for start, chunk_size in chunks:
+                self.semaphore.acquire()
+                with self.lock:
+                    mount = self.mounts_availabe.pop()
+                    self.mounts_processing.append(mount)
+
+                path = mount.mount_path / self.filename
+
+                def task() -> bytes | Exception:
+                    try:
+                        with path.open("rb") as f:
+                            f.seek(offset)
+                            return f.read(size)
+                    except KeyboardInterrupt as e:
+                        import _thread
+
+                        _thread.interrupt_main()
+                        return Exception(e)
+                    except Exception as e:
+                        return e
+                    finally:
+                        with self.lock:
+                            self.mounts_processing.remove(mount)
+                            self.mounts_availabe.append(mount)
+                            self.semaphore.release()
+
+                fut = self.executor.submit(task)
+                futures.append(fut)
+
+            out: bytes = b""
+            for fut in futures:
+                chunk: bytes | Exception = fut.result()
+                if isinstance(chunk, Exception):
+                    return chunk
+                else:
+                    out += chunk
+                    del chunk
+            return out
+        except Exception as e:
+            return e

rclone_api/profile/mount_copy_bytes.py

@@ -6,7 +6,6 @@ import argparse
 import os
 import shutil
 import time
-from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -119,12 +118,14 @@ def _run_profile(
     print("#" * 80)
     net_io_start = psutil.net_io_counters()
     start = time.time()
+    chunk_size = size // transfers
     bytes_or_err: bytes | Exception = rclone.copy_bytes(
         src=src_file,
         offset=offset.as_int(),
         length=size.as_int(),
+        chunk_size=chunk_size,
         direct_io=direct_io,
-        transfers=transfers,
+        max_threads=transfers,
         mount_log=mount_log,
     )
     diff = time.time() - start
@@ -138,6 +139,8 @@ def _run_profile(
     assert len(bytes_or_err) == size.as_int(), f"Length: {len(bytes_or_err)} != {size}"
 
     # print io stats
+    # disabling num for now
+    num = 1
     bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num
     bytes_recv = (net_io_end.bytes_recv - net_io_start.bytes_recv) // num
     packets_sent = (net_io_end.packets_sent - net_io_start.packets_sent) // num
@@ -230,8 +233,8 @@ def main() -> None:
         shutil.rmtree(mount_root_path)
 
     args = _parse_args()
-    transfer_list = [1]
-    parallel_workers = args.num
+    transfer_list = [args.num]
+    # parallel_workers = args.num
 
     def task(
         offset: SizeSuffix,
@@ -250,12 +253,14 @@ def main() -> None:
             num=args.num,
         )
 
-    with ThreadPoolExecutor(max_workers=parallel_workers) as _:
-        tasks: list[Future] = []
-        for i in range(parallel_workers):
-            offset = SizeSuffix(i * 1024 * 1024 * 256)
-            future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
-            tasks.append(future)
+    task(SizeSuffix(0))
+
+    # with ThreadPoolExecutor(max_workers=parallel_workers) as _:
+    #     tasks: list[Future] = []
+    #     for i in range(parallel_workers):
+    #         offset = SizeSuffix(i * 1024 * 1024 * 256)
+    #         future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
+    #         tasks.append(future)
 
 
 if __name__ == "__main__":

rclone_api/rclone.py

@@ -25,7 +25,7 @@ from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
 from rclone_api.group_files import group_files
-from rclone_api.mount import Mount, clean_mount, prepare_mount
+from rclone_api.mount import Mount, MultiMountFileChunker, clean_mount, prepare_mount
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
@@ -815,7 +815,7 @@ class Rclone:
             )
             return out
 
-    def copy_bytes(
+    def _copy_bytes(
         self,
         src: str,
         offset: int,
@@ -829,7 +829,7 @@ class Rclone:
         """Copy bytes from a file to another file."""
         from rclone_api.util import random_str
 
-        tmp_mnt = Path("tmp_mnt") / random_str(12)
+        tmp_mnts = Path("tmp_mnts") / random_str(12)
         src_parent_path = Path(src).parent.as_posix()
         src_file = Path(src).name
         other_args: list[str] = [
@@ -844,7 +844,7 @@ class Rclone:
         max_read_ahead = SizeSuffix(vfs_read_chunk_size.as_int())
 
         # other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
-        other_args += ["--vfs-read-chunk-size", str(0)]
+        other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
         other_args += ["--vfs-read-chunk-size-limit", str(vfs_read_chunk_size_limit)]
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
         other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
@@ -860,7 +860,7 @@ class Rclone:
                 # use scoped mount to do the read, then write the bytes to the destination
                 with self.scoped_mount(
                     src_parent_path,
-                    tmp_mnt,
+                    tmp_mnts,
                     use_links=True,
                     verbose=mount_log is not None,
                     vfs_cache_mode="minimal",
@@ -869,7 +869,7 @@ class Rclone:
                     cache_dir=cache_dir,
                     cache_dir_delete_on_exit=True,
                 ):
-                    src_file_mnt = tmp_mnt / src_file
+                    src_file_mnt = tmp_mnts / src_file
                     with open(src_file_mnt, "rb") as f:
                         f.seek(offset)
                         data = f.read(length)
@@ -883,6 +883,122 @@ class Rclone:
             except Exception as e:
                 return e
 
+    def get_multi_mount_file_chunker(
+        self,
+        src: str,
+        chunk_size: SizeSuffix,
+        threads: int,
+        mount_log: Path | None,
+        direct_io: bool,
+    ) -> MultiMountFileChunker:
+        from rclone_api.util import random_str
+
+        mounts: list[Mount] = []
+        vfs_read_chunk_size = chunk_size
+        vfs_read_chunk_size_limit = chunk_size
+        vfs_read_chunk_streams = 0
+        vfs_disk_space_total_size = chunk_size
+        other_args: list[str] = []
+        other_args += ["--no-modtime"]
+        other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
+        other_args += [
+            "--vfs-read-chunk-size-limit",
+            vfs_read_chunk_size_limit.as_str(),
+        ]
+        other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
+        other_args += [
+            "--vfs-disk-space-total-size",
+            vfs_disk_space_total_size.as_str(),
+        ]
+        other_args += ["--read-only"]
+        if direct_io:
+            other_args += ["--direct-io"]
+
+        filename = Path(src).name
+        with ThreadPoolExecutor(max_workers=threads) as executor:
+            futures: list[Future] = []
+            try:
+                for i in range(threads):
+                    tmp_mnts = Path("tmp_mnts") / random_str(12)
+                    verbose = mount_log is not None
+                    clean_mount(tmp_mnts, verbose=verbose)
+                    prepare_mount(tmp_mnts, verbose=verbose)
+                    src_parent_path = Path(src).parent.as_posix()
+
+                    def task(src_parent_path=src_parent_path, tmp_mnts=tmp_mnts):
+                        return self.mount(
+                            src=src_parent_path,
+                            outdir=tmp_mnts,
+                            allow_writes=False,
+                            use_links=True,
+                            vfs_cache_mode="minimal",
+                            verbose=False,
+                            cache_dir=Path("cache") / random_str(12),
+                            cache_dir_delete_on_exit=True,
+                            log=mount_log,
+                            other_args=other_args,
+                        )
+
+                    futures.append(executor.submit(task))
+                mount_errors: list[Exception] = []
+                for fut in futures:
+                    try:
+                        mount = fut.result()
+                        mounts.append(mount)
+                    except Exception as er:
+                        mount_errors.append(er)
+                if mount_errors:
+                    raise Exception(mount_errors)
+            except Exception:
+                for mount in mounts:
+                    mount.close()
+                raise
+        executor = ThreadPoolExecutor(max_workers=threads)
+        filechunker: MultiMountFileChunker = MultiMountFileChunker(
+            filename=filename,
+            chunk_size=chunk_size,
+            mounts=mounts,
+            executor=executor,
+        )
+        return filechunker
+
+    def copy_bytes(
+        self,
+        src: str,
+        offset: int,
+        length: int,
+        chunk_size: SizeSuffix,
+        max_threads: int = 1,
+        # If outfile is supplied then bytes are written to this file and success returns bytes(0)
+        outfile: Path | None = None,
+        mount_log: Path | None = None,
+        direct_io: bool = True,
+    ) -> bytes | Exception:
+        """Copy bytes from a file to another file."""
+        # determine number of threads from chunk size
+        threads = max(1, min(max_threads, length // chunk_size.as_int()))
+        filechunker = self.get_multi_mount_file_chunker(
+            src=src,
+            chunk_size=chunk_size,
+            threads=threads,
+            mount_log=mount_log,
+            direct_io=direct_io,
+        )
+        try:
+            data = filechunker.fetch(offset, length)
+            if isinstance(data, Exception):
+                raise data
+            if outfile is None:
+                return data
+            with open(outfile, "wb") as out:
+                out.write(data)
+                del data
+            return bytes(0)
+        except Exception as e:
+            return e
+        finally:
+            filechunker.close()
+
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
     ) -> CompletedProcess:

rclone_api/types.py

@@ -170,3 +170,11 @@ class SizeSuffix:
             raise ZeroDivisionError("Division by zero is undefined")
         # Use floor division to maintain integer arithmetic.
         return SizeSuffix(self._size // other_int._size)
+
+    # support / division
+    def __floordiv__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        if other_int._size == 0:
+            raise ZeroDivisionError("Division by zero is undefined")
+        # Use floor division to maintain integer arithmetic.
+        return SizeSuffix(self._size // other_int._size)

{rclone_api-1.1.60.dist-info → rclone_api-1.1.62.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.60
+Version: 1.1.62
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.60.dist-info → rclone_api-1.1.62.dist-info}/RECORD

@@ -11,13 +11,13 @@ rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/mount.py,sha256=5XiFwP7tst4h8cGQstZMJ0u5VzqFQ4YehnDl2AFP11A,10057
+rclone_api/mount.py,sha256=hd6_zmJ-wH337F0eh9JJMMlVzj92m32tLm2t5qZFxNc,12931
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
-rclone_api/rclone.py,sha256=eHIxAseumxEFHiJytdpRZ88gGy5qx3IKkU3yKYueWNA,43906
+rclone_api/rclone.py,sha256=Sd4NrCoCwIe8wfOEpJ-eL6BRozIvCHI6fhnr6BI2ET8,48226
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=AWVIoCaRkUBCXOQffxdkFCOeUhPwolQh9d2albyAGHc,4730
+rclone_api/types.py,sha256=2x9APG861HFrMwt5AIpxYI_ahDY09vUjGMtTE5xHlEs,5085
 rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
@@ -25,7 +25,7 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=aHbIsIm2_fEvSZIHhRcXaNB2caT7xaE4zJi4dNYr7II,7621
+rclone_api/profile/mount_copy_bytes.py,sha256=KJhn2ufpRwT9c9bH_q2RWvklw5x63nvI-UMtv9O8CPg,7717
 rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=D6wM9Nuu73LNGC8JCCfevqjF3qdZ21mQxYQClFLZLMU,3726
@@ -33,9 +33,9 @@ rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
 rclone_api/s3/upload_file_multipart.py,sha256=y9azNAU8QH5Ovwz33V2HZwNmJdlFjJg-jrXLZ1gtMds,10364
-rclone_api-1.1.60.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.60.dist-info/METADATA,sha256=fUIrdhlIVOhWdheud6rnjJ2ikFhd4Z9hBw5fLiEJR2s,4537
-rclone_api-1.1.60.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.60.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.1.60.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.60.dist-info/RECORD,,
+rclone_api-1.1.62.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.62.dist-info/METADATA,sha256=d0_DUkLYpOf_vedOxoFXs7Qco4e_6A3RBGaPogBUkec,4537
+rclone_api-1.1.62.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.62.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.1.62.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.62.dist-info/RECORD,,