rclone-api 1.1.61__tar.gz → 1.1.62__tar.gz

{rclone_api-1.1.61 → rclone_api-1.1.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.61
+Version: 1.1.62
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.61 → rclone_api-1.1.62}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.61"
+version = "1.1.62"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.61 → rclone_api-1.1.62}/src/rclone_api/mount.py

@@ -6,12 +6,14 @@ import subprocess
 import time
 import warnings
 import weakref
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
+from threading import Lock, Semaphore
 from typing import Any
 
 from rclone_api.process import Process
+from rclone_api.types import SizeSuffix
 
 _SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
 
@@ -283,3 +285,83 @@ def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
             shutil.rmtree(cache_dir)
         except Exception as e:
             warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
+
+
+class MultiMountFileChunker:
+    def __init__(
+        self,
+        filename: str,
+        chunk_size: SizeSuffix,
+        mounts: list[Mount],
+        executor: ThreadPoolExecutor,
+    ) -> None:
+        self.filename = filename
+        self.chunk_size = chunk_size
+        self.executor = executor
+        self.mounts_processing: list[Mount] = []
+        self.mounts_availabe: list[Mount] = mounts
+        self.semaphore = Semaphore(len(mounts))
+        self.lock = Lock()
+
+    def close(self) -> None:
+        self.executor.shutdown(wait=False, cancel_futures=True)
+        with ThreadPoolExecutor() as executor:
+            for mount in self.mounts_processing:
+                executor.submit(lambda: mount.close())
+
+    def fetch(self, offset: int, size: int) -> bytes | Exception:
+        try:
+            assert offset % self.chunk_size.as_int() == 0
+            assert size <= self.chunk_size.as_int()
+            assert size > 0
+            assert offset >= 0
+
+            chunks: list[tuple[int, int]] = []
+            start = offset
+            end = offset + size
+            while start < end:
+                chunk_size = min(self.chunk_size.as_int(), end - start)
+                chunks.append((start, chunk_size))
+                start += chunk_size
+
+            futures: list[Future[bytes | Exception]] = []
+            for start, chunk_size in chunks:
+                self.semaphore.acquire()
+                with self.lock:
+                    mount = self.mounts_availabe.pop()
+                    self.mounts_processing.append(mount)
+
+                path = mount.mount_path / self.filename
+
+                def task() -> bytes | Exception:
+                    try:
+                        with path.open("rb") as f:
+                            f.seek(offset)
+                            return f.read(size)
+                    except KeyboardInterrupt as e:
+                        import _thread
+
+                        _thread.interrupt_main()
+                        return Exception(e)
+                    except Exception as e:
+                        return e
+                    finally:
+                        with self.lock:
+                            self.mounts_processing.remove(mount)
+                            self.mounts_availabe.append(mount)
+                            self.semaphore.release()
+
+                fut = self.executor.submit(task)
+                futures.append(fut)
+
+            out: bytes = b""
+            for fut in futures:
+                chunk: bytes | Exception = fut.result()
+                if isinstance(chunk, Exception):
+                    return chunk
+                else:
+                    out += chunk
+                    del chunk
+            return out
+        except Exception as e:
+            return e

{rclone_api-1.1.61 → rclone_api-1.1.62}/src/rclone_api/rclone.py

@@ -25,7 +25,7 @@ from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
 from rclone_api.group_files import group_files
-from rclone_api.mount import Mount, clean_mount, prepare_mount
+from rclone_api.mount import Mount, MultiMountFileChunker, clean_mount, prepare_mount
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
@@ -829,7 +829,7 @@ class Rclone:
         """Copy bytes from a file to another file."""
         from rclone_api.util import random_str
 
-        tmp_mnt = Path("tmp_mnt") / random_str(12)
+        tmp_mnts = Path("tmp_mnts") / random_str(12)
         src_parent_path = Path(src).parent.as_posix()
         src_file = Path(src).name
         other_args: list[str] = [
@@ -860,7 +860,7 @@ class Rclone:
                 # use scoped mount to do the read, then write the bytes to the destination
                 with self.scoped_mount(
                     src_parent_path,
-                    tmp_mnt,
+                    tmp_mnts,
                     use_links=True,
                     verbose=mount_log is not None,
                     vfs_cache_mode="minimal",
@@ -869,7 +869,7 @@ class Rclone:
                     cache_dir=cache_dir,
                     cache_dir_delete_on_exit=True,
                 ):
-                    src_file_mnt = tmp_mnt / src_file
+                    src_file_mnt = tmp_mnts / src_file
                     with open(src_file_mnt, "rb") as f:
                         f.seek(offset)
                         data = f.read(length)
@@ -883,6 +883,85 @@ class Rclone:
             except Exception as e:
                 return e
 
+    def get_multi_mount_file_chunker(
+        self,
+        src: str,
+        chunk_size: SizeSuffix,
+        threads: int,
+        mount_log: Path | None,
+        direct_io: bool,
+    ) -> MultiMountFileChunker:
+        from rclone_api.util import random_str
+
+        mounts: list[Mount] = []
+        vfs_read_chunk_size = chunk_size
+        vfs_read_chunk_size_limit = chunk_size
+        vfs_read_chunk_streams = 0
+        vfs_disk_space_total_size = chunk_size
+        other_args: list[str] = []
+        other_args += ["--no-modtime"]
+        other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
+        other_args += [
+            "--vfs-read-chunk-size-limit",
+            vfs_read_chunk_size_limit.as_str(),
+        ]
+        other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
+        other_args += [
+            "--vfs-disk-space-total-size",
+            vfs_disk_space_total_size.as_str(),
+        ]
+        other_args += ["--read-only"]
+        if direct_io:
+            other_args += ["--direct-io"]
+
+        filename = Path(src).name
+        with ThreadPoolExecutor(max_workers=threads) as executor:
+            futures: list[Future] = []
+            try:
+                for i in range(threads):
+                    tmp_mnts = Path("tmp_mnts") / random_str(12)
+                    verbose = mount_log is not None
+                    clean_mount(tmp_mnts, verbose=verbose)
+                    prepare_mount(tmp_mnts, verbose=verbose)
+                    src_parent_path = Path(src).parent.as_posix()
+
+                    def task(src_parent_path=src_parent_path, tmp_mnts=tmp_mnts):
+                        return self.mount(
+                            src=src_parent_path,
+                            outdir=tmp_mnts,
+                            allow_writes=False,
+                            use_links=True,
+                            vfs_cache_mode="minimal",
+                            verbose=False,
+                            cache_dir=Path("cache") / random_str(12),
+                            cache_dir_delete_on_exit=True,
+                            log=mount_log,
+                            other_args=other_args,
+                        )
+
+                    futures.append(executor.submit(task))
+                mount_errors: list[Exception] = []
+                for fut in futures:
+                    try:
+                        mount = fut.result()
+                        mounts.append(mount)
+                    except Exception as er:
+                        mount_errors.append(er)
+                if mount_errors:
+                    raise Exception(mount_errors)
+            except Exception:
+                for mount in mounts:
+                    mount.close()
+                raise
+        executor = ThreadPoolExecutor(max_workers=threads)
+        filechunker: MultiMountFileChunker = MultiMountFileChunker(
+            filename=filename,
+            chunk_size=chunk_size,
+            mounts=mounts,
+            executor=executor,
+        )
+        return filechunker
+
     def copy_bytes(
         self,
         src: str,
@@ -897,49 +976,28 @@ class Rclone:
     ) -> bytes | Exception:
         """Copy bytes from a file to another file."""
         # determine number of threads from chunk size
-        threads = min(max_threads, length // chunk_size.as_int())
-
-        if threads == 1:
-            return self._copy_bytes(
-                src,
-                offset,
-                length,
-                transfers=1,
-                outfile=outfile,
-                mount_log=mount_log,
-                direct_io=direct_io,
-            )
-        else:
-            futures: list[Future] = []
-            with ThreadPoolExecutor(max_workers=threads) as executor:
-                for i in range(threads):
-                    start = i * chunk_size.as_int()
-                    end = start + chunk_size.as_int()
-                    if i == threads - 1:
-                        end = length
-                    future = executor.submit(
-                        self._copy_bytes,
-                        src,
-                        offset + start,
-                        end - start,
-                        transfers=1,
-                        outfile=None,
-                        mount_log=mount_log,
-                        direct_io=direct_io,
-                    )
-                    futures.append(future)
-                data: bytes = b""
-                for future in futures:
-                    chunk: bytes | Exception = future.result()
-                    if isinstance(chunk, Exception):
-                        return chunk
-                    data += chunk
-                if outfile is not None:
-                    with open(outfile, "wb") as out:
-                        out.write(data)
-                        del data
-                        return bytes(0)
+        threads = max(1, min(max_threads, length // chunk_size.as_int()))
+        filechunker = self.get_multi_mount_file_chunker(
+            src=src,
+            chunk_size=chunk_size,
+            threads=threads,
+            mount_log=mount_log,
+            direct_io=direct_io,
+        )
+        try:
+            data = filechunker.fetch(offset, length)
+            if isinstance(data, Exception):
+                raise data
+            if outfile is None:
                 return data
+            with open(outfile, "wb") as out:
+                out.write(data)
+                del data
+            return bytes(0)
+        except Exception as e:
+            return e
+        finally:
+            filechunker.close()
 
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None

{rclone_api-1.1.61 → rclone_api-1.1.62}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.61
+Version: 1.1.62
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License