rclone-api 1.1.60__tar.gz → 1.1.61__tar.gz

{rclone_api-1.1.60 → rclone_api-1.1.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.60
+Version: 1.1.61
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.60 → rclone_api-1.1.61}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.60"
+version = "1.1.61"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.60 → rclone_api-1.1.61}/src/rclone_api/profile/mount_copy_bytes.py

@@ -6,7 +6,6 @@ import argparse
 import os
 import shutil
 import time
-from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -119,12 +118,14 @@ def _run_profile(
     print("#" * 80)
     net_io_start = psutil.net_io_counters()
     start = time.time()
+    chunk_size = size // transfers
     bytes_or_err: bytes | Exception = rclone.copy_bytes(
         src=src_file,
         offset=offset.as_int(),
         length=size.as_int(),
+        chunk_size=chunk_size,
         direct_io=direct_io,
-        transfers=transfers,
+        max_threads=transfers,
         mount_log=mount_log,
     )
     diff = time.time() - start
@@ -138,6 +139,8 @@ def _run_profile(
     assert len(bytes_or_err) == size.as_int(), f"Length: {len(bytes_or_err)} != {size}"
 
     # print io stats
+    # disabling num for now
+    num = 1
     bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num
     bytes_recv = (net_io_end.bytes_recv - net_io_start.bytes_recv) // num
     packets_sent = (net_io_end.packets_sent - net_io_start.packets_sent) // num
@@ -230,8 +233,8 @@ def main() -> None:
         shutil.rmtree(mount_root_path)
 
     args = _parse_args()
-    transfer_list = [1]
-    parallel_workers = args.num
+    transfer_list = [args.num]
+    # parallel_workers = args.num
 
     def task(
         offset: SizeSuffix,
@@ -250,12 +253,14 @@ def main() -> None:
             num=args.num,
         )
 
-    with ThreadPoolExecutor(max_workers=parallel_workers) as _:
-        tasks: list[Future] = []
-        for i in range(parallel_workers):
-            offset = SizeSuffix(i * 1024 * 1024 * 256)
-            future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
-            tasks.append(future)
+    task(SizeSuffix(0))
+
+    # with ThreadPoolExecutor(max_workers=parallel_workers) as _:
+    #     tasks: list[Future] = []
+    #     for i in range(parallel_workers):
+    #         offset = SizeSuffix(i * 1024 * 1024 * 256)
+    #         future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
+    #         tasks.append(future)
 
 
 if __name__ == "__main__":

{rclone_api-1.1.60 → rclone_api-1.1.61}/src/rclone_api/rclone.py

@@ -815,7 +815,7 @@ class Rclone:
             )
             return out
 
-    def copy_bytes(
+    def _copy_bytes(
         self,
         src: str,
         offset: int,
@@ -844,7 +844,7 @@ class Rclone:
         max_read_ahead = SizeSuffix(vfs_read_chunk_size.as_int())
 
         # other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
-        other_args += ["--vfs-read-chunk-size", str(0)]
+        other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
         other_args += ["--vfs-read-chunk-size-limit", str(vfs_read_chunk_size_limit)]
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
         other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
@@ -883,6 +883,64 @@ class Rclone:
             except Exception as e:
                 return e
 
+    def copy_bytes(
+        self,
+        src: str,
+        offset: int,
+        length: int,
+        chunk_size: SizeSuffix,
+        max_threads: int = 1,
+        # If outfile is supplied then bytes are written to this file and success returns bytes(0)
+        outfile: Path | None = None,
+        mount_log: Path | None = None,
+        direct_io: bool = True,
+    ) -> bytes | Exception:
+        """Copy bytes from a file to another file."""
+        # determine number of threads from chunk size
+        threads = min(max_threads, length // chunk_size.as_int())
+
+        if threads == 1:
+            return self._copy_bytes(
+                src,
+                offset,
+                length,
+                transfers=1,
+                outfile=outfile,
+                mount_log=mount_log,
+                direct_io=direct_io,
+            )
+        else:
+            futures: list[Future] = []
+            with ThreadPoolExecutor(max_workers=threads) as executor:
+                for i in range(threads):
+                    start = i * chunk_size.as_int()
+                    end = start + chunk_size.as_int()
+                    if i == threads - 1:
+                        end = length
+                    future = executor.submit(
+                        self._copy_bytes,
+                        src,
+                        offset + start,
+                        end - start,
+                        transfers=1,
+                        outfile=None,
+                        mount_log=mount_log,
+                        direct_io=direct_io,
+                    )
+                    futures.append(future)
+                data: bytes = b""
+                for future in futures:
+                    chunk: bytes | Exception = future.result()
+                    if isinstance(chunk, Exception):
+                        return chunk
+                    data += chunk
+                if outfile is not None:
+                    with open(outfile, "wb") as out:
+                        out.write(data)
+                        del data
+                        return bytes(0)
+                return data
+
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
     ) -> CompletedProcess:

{rclone_api-1.1.60 → rclone_api-1.1.61}/src/rclone_api/types.py

@@ -170,3 +170,11 @@ class SizeSuffix:
             raise ZeroDivisionError("Division by zero is undefined")
         # Use floor division to maintain integer arithmetic.
         return SizeSuffix(self._size // other_int._size)
+
+    # support / division
+    def __floordiv__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        if other_int._size == 0:
+            raise ZeroDivisionError("Division by zero is undefined")
+        # Use floor division to maintain integer arithmetic.
+        return SizeSuffix(self._size // other_int._size)

{rclone_api-1.1.60 → rclone_api-1.1.61}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.60
+Version: 1.1.61
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.60 → rclone_api-1.1.61}/tests/test_copy_bytes.py

@@ -9,7 +9,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from rclone_api import Config, Rclone
+from rclone_api import Config, Rclone, SizeSuffix
 
 load_dotenv()
 
@@ -76,7 +76,8 @@ class RcloneCopyBytesTester(unittest.TestCase):
             src="dst:rclone-api-unit-test/zachs_video/breaking_ai_mind.mp4",
             offset=0,
             length=1024 * 1024,
-            transfers=1,
+            chunk_size=SizeSuffix(1024 * 1024),
+            max_threads=1,
         )
         if isinstance(bytes_or_err, Exception):
             print(bytes_or_err)
@@ -95,6 +96,7 @@ class RcloneCopyBytesTester(unittest.TestCase):
                 src="dst:rclone-api-unit-test/zachs_video/breaking_ai_mind.mp4",
                 offset=0,
                 length=1024 * 1024,
+                chunk_size=SizeSuffix(1024 * 1024),
                 outfile=tmp,
                 mount_log=log,
             )