rclone-api 1.1.59__py2.py3-none-any.whl → 1.1.61__py2.py3-none-any.whl

rclone_api/profile/mount_copy_bytes.py

@@ -6,7 +6,6 @@ import argparse
 import os
 import shutil
 import time
-from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -108,7 +107,8 @@ def _run_profile(
     offset: SizeSuffix,
     size: SizeSuffix,
     log_dir: Path,
-    direct_io: bool = True,
+    num: int,
+    direct_io: bool,
 ) -> None:
 
     mount_log = log_dir / f"mount_{SizeSuffix(size)}_threads_{transfers}.log"
@@ -118,12 +118,14 @@ def _run_profile(
     print("#" * 80)
     net_io_start = psutil.net_io_counters()
     start = time.time()
+    chunk_size = size // transfers
     bytes_or_err: bytes | Exception = rclone.copy_bytes(
         src=src_file,
         offset=offset.as_int(),
         length=size.as_int(),
+        chunk_size=chunk_size,
         direct_io=direct_io,
-        transfers=transfers,
+        max_threads=transfers,
         mount_log=mount_log,
     )
     diff = time.time() - start
@@ -137,9 +139,11 @@ def _run_profile(
     assert len(bytes_or_err) == size.as_int(), f"Length: {len(bytes_or_err)} != {size}"
 
     # print io stats
-    bytes_sent = net_io_end.bytes_sent - net_io_start.bytes_sent
-    bytes_recv = net_io_end.bytes_recv - net_io_start.bytes_recv
-    packets_sent = net_io_end.packets_sent - net_io_start.packets_sent
+    # disabling num for now
+    num = 1
+    bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num
+    bytes_recv = (net_io_end.bytes_recv - net_io_start.bytes_recv) // num
+    packets_sent = (net_io_end.packets_sent - net_io_start.packets_sent) // num
     efficiency = size.as_int() / (bytes_recv)
     efficiency_100 = efficiency * 100
     efficiency_str = f"{efficiency_100:.2f}"
@@ -164,6 +168,7 @@ def test_profile_copy_bytes(
     transfer_list: list[int] | None,
     mount_root_path: Path,
     size: SizeSuffix | None,
+    num: int,
 ) -> None:
 
     if size:
@@ -197,6 +202,7 @@ def test_profile_copy_bytes(
                 size=SizeSuffix(sz),
                 direct_io=args.direct_io,
                 log_dir=mount_root_path,
+                num=num,
             )
     print("done")
 
@@ -227,8 +233,8 @@ def main() -> None:
         shutil.rmtree(mount_root_path)
 
     args = _parse_args()
-    transfer_list = [1]
-    parallel_workers = args.num
+    transfer_list = [args.num]
+    # parallel_workers = args.num
 
     def task(
         offset: SizeSuffix,
@@ -244,14 +250,17 @@ def main() -> None:
             mount_root_path=mount_root_path,
             transfer_list=transfer_list,
             size=args.size,
+            num=args.num,
         )
 
-    with ThreadPoolExecutor(max_workers=parallel_workers) as _:
-        tasks: list[Future] = []
-        for i in range(parallel_workers):
-            offset = SizeSuffix(i * 1024 * 1024 * 256)
-            future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
-            tasks.append(future)
+    task(SizeSuffix(0))
+
+    # with ThreadPoolExecutor(max_workers=parallel_workers) as _:
+    #     tasks: list[Future] = []
+    #     for i in range(parallel_workers):
+    #         offset = SizeSuffix(i * 1024 * 1024 * 256)
+    #         future = ThreadPoolExecutor().submit(lambda: task(offset=offset))
+    #         tasks.append(future)
 
 
 if __name__ == "__main__":

rclone_api/rclone.py

@@ -815,7 +815,7 @@ class Rclone:
             )
             return out
 
-    def copy_bytes(
+    def _copy_bytes(
         self,
         src: str,
         offset: int,
@@ -844,7 +844,7 @@ class Rclone:
         max_read_ahead = SizeSuffix(vfs_read_chunk_size.as_int())
 
         # other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
-        other_args += ["--vfs-read-chunk-size", str(0)]
+        other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
         other_args += ["--vfs-read-chunk-size-limit", str(vfs_read_chunk_size_limit)]
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
         other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
@@ -883,6 +883,64 @@ class Rclone:
             except Exception as e:
                 return e
 
+    def copy_bytes(
+        self,
+        src: str,
+        offset: int,
+        length: int,
+        chunk_size: SizeSuffix,
+        max_threads: int = 1,
+        # If outfile is supplied then bytes are written to this file and success returns bytes(0)
+        outfile: Path | None = None,
+        mount_log: Path | None = None,
+        direct_io: bool = True,
+    ) -> bytes | Exception:
+        """Copy bytes from a file to another file."""
+        # determine number of threads from chunk size
+        threads = min(max_threads, length // chunk_size.as_int())
+
+        if threads == 1:
+            return self._copy_bytes(
+                src,
+                offset,
+                length,
+                transfers=1,
+                outfile=outfile,
+                mount_log=mount_log,
+                direct_io=direct_io,
+            )
+        else:
+            futures: list[Future] = []
+            with ThreadPoolExecutor(max_workers=threads) as executor:
+                for i in range(threads):
+                    start = i * chunk_size.as_int()
+                    end = start + chunk_size.as_int()
+                    if i == threads - 1:
+                        end = length
+                    future = executor.submit(
+                        self._copy_bytes,
+                        src,
+                        offset + start,
+                        end - start,
+                        transfers=1,
+                        outfile=None,
+                        mount_log=mount_log,
+                        direct_io=direct_io,
+                    )
+                    futures.append(future)
+                data: bytes = b""
+                for future in futures:
+                    chunk: bytes | Exception = future.result()
+                    if isinstance(chunk, Exception):
+                        return chunk
+                    data += chunk
+                if outfile is not None:
+                    with open(outfile, "wb") as out:
+                        out.write(data)
+                        del data
+                        return bytes(0)
+                return data
+
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
     ) -> CompletedProcess:

rclone_api/types.py

@@ -170,3 +170,11 @@ class SizeSuffix:
             raise ZeroDivisionError("Division by zero is undefined")
         # Use floor division to maintain integer arithmetic.
         return SizeSuffix(self._size // other_int._size)
+
+    # support / division
+    def __floordiv__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        if other_int._size == 0:
+            raise ZeroDivisionError("Division by zero is undefined")
+        # Use floor division to maintain integer arithmetic.
+        return SizeSuffix(self._size // other_int._size)

{rclone_api-1.1.59.dist-info → rclone_api-1.1.61.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.59
+Version: 1.1.61
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.59.dist-info → rclone_api-1.1.61.dist-info}/RECORD

@@ -13,11 +13,11 @@ rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/mount.py,sha256=5XiFwP7tst4h8cGQstZMJ0u5VzqFQ4YehnDl2AFP11A,10057
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
-rclone_api/rclone.py,sha256=eHIxAseumxEFHiJytdpRZ88gGy5qx3IKkU3yKYueWNA,43906
+rclone_api/rclone.py,sha256=_bXkNBKeAVfjEOmX7bEwpxlXaJURJ3RguY1TJzwIBzM,46038
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=AWVIoCaRkUBCXOQffxdkFCOeUhPwolQh9d2albyAGHc,4730
+rclone_api/types.py,sha256=2x9APG861HFrMwt5AIpxYI_ahDY09vUjGMtTE5xHlEs,5085
 rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
@@ -25,7 +25,7 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=ZfwKJ9hYQgFN8Xo2IWTQwvcTJf_DZgTxqVSHWUFrWUc,7522
+rclone_api/profile/mount_copy_bytes.py,sha256=KJhn2ufpRwT9c9bH_q2RWvklw5x63nvI-UMtv9O8CPg,7717
 rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=D6wM9Nuu73LNGC8JCCfevqjF3qdZ21mQxYQClFLZLMU,3726
@@ -33,9 +33,9 @@ rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
 rclone_api/s3/upload_file_multipart.py,sha256=y9azNAU8QH5Ovwz33V2HZwNmJdlFjJg-jrXLZ1gtMds,10364
-rclone_api-1.1.59.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.59.dist-info/METADATA,sha256=QqrdtjJ4QCKu2FgqOVTNYX1z9J65tfdF7EsKUqvrHcg,4537
-rclone_api-1.1.59.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.59.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.1.59.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.59.dist-info/RECORD,,
+rclone_api-1.1.61.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.61.dist-info/METADATA,sha256=K94aUUT1DvHqha4sUrmbKR18VbBosxwPVUW0tQLAKqU,4537
+rclone_api-1.1.61.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.61.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.1.61.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.61.dist-info/RECORD,,