rclone-api 1.1.58__py2.py3-none-any.whl → 1.1.60__py2.py3-none-any.whl

rclone_api/profile/mount_copy_bytes.py

@@ -6,7 +6,7 @@ import argparse
 import os
 import shutil
 import time
-from concurrent.futures import ThreadPoolExecutor, Future
+from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -21,7 +21,8 @@ os.environ["RCLONE_API_VERBOSE"] = "1"
 @dataclass
 class Args:
     direct_io: bool
-    num: int = 1
+    num: int
+    size: SizeSuffix | None
 
 
 @dataclass
@@ -107,7 +108,8 @@ def _run_profile(
     offset: SizeSuffix,
     size: SizeSuffix,
     log_dir: Path,
-    direct_io: bool = True,
+    num: int,
+    direct_io: bool,
 ) -> None:
 
     mount_log = log_dir / f"mount_{SizeSuffix(size)}_threads_{transfers}.log"
@@ -136,9 +138,9 @@ def _run_profile(
     assert len(bytes_or_err) == size.as_int(), f"Length: {len(bytes_or_err)} != {size}"
 
     # print io stats
-    bytes_sent = net_io_end.bytes_sent - net_io_start.bytes_sent
-    bytes_recv = net_io_end.bytes_recv - net_io_start.bytes_recv
-    packets_sent = net_io_end.packets_sent - net_io_start.packets_sent
+    bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num
+    bytes_recv = (net_io_end.bytes_recv - net_io_start.bytes_recv) // num
+    packets_sent = (net_io_end.packets_sent - net_io_start.packets_sent) // num
     efficiency = size.as_int() / (bytes_recv)
     efficiency_100 = efficiency * 100
     efficiency_str = f"{efficiency_100:.2f}"
@@ -156,19 +158,30 @@ def _run_profile(
     print(f"Time: {diff:.1f} seconds")
 
 
-def test_profile_copy_bytes(args: Args, rclone: Rclone, offset: SizeSuffix, transfer_list: list[int] | None, mount_root_path: Path) -> None:
+def test_profile_copy_bytes(
+    args: Args,
+    rclone: Rclone,
+    offset: SizeSuffix,
+    transfer_list: list[int] | None,
+    mount_root_path: Path,
+    size: SizeSuffix | None,
+    num: int,
+) -> None:
 
-    sizes = [
-        #1024 * 1024 * 1,
-        #1024 * 1024 * 2,
-        #1024 * 1024 * 4,
-        #1024 * 1024 * 8,
-        1024 * 1024 * 16,
-        # 1024 * 1024 * 32,
-        1024 * 1024 * 64,
-        1024 * 1024 * 128,
-        # 1024 * 1024 * 256,
-    ]
+    if size:
+        sizes = [size.as_int()]
+    else:
+        sizes = [
+            # 1024 * 1024 * 1,
+            # 1024 * 1024 * 2,
+            # 1024 * 1024 * 4,
+            # 1024 * 1024 * 8,
+            1024 * 1024 * 16,
+            # 1024 * 1024 * 32,
+            1024 * 1024 * 64,
+            1024 * 1024 * 128,
+            # 1024 * 1024 * 256,
+        ]
     # transfer_list = [1, 2, 4, 8, 16]
     transfer_list = transfer_list or [1, 2, 4]
 
@@ -176,18 +189,17 @@ def test_profile_copy_bytes(args: Args, rclone: Rclone, offset: SizeSuffix, tran
     # sftp mount
     src_file = "src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
 
-
-
-    for size in sizes:
+    for sz in sizes:
         for transfers in transfer_list:
             _run_profile(
                 rclone=rclone,
                 src_file=src_file,
                 transfers=transfers,
                 offset=offset,
-                size=SizeSuffix(size),
+                size=SizeSuffix(sz),
                 direct_io=args.direct_io,
                 log_dir=mount_root_path,
+                num=num,
             )
     print("done")
 
@@ -196,8 +208,11 @@ def _parse_args() -> Args:
     parser = argparse.ArgumentParser(description="Profile copy_bytes")
     parser.add_argument("--direct-io", help="Use direct IO", action="store_true")
     parser.add_argument("-n", "--num", help="Number of workers", type=int, default=1)
+    parser.add_argument(
+        "--size", help="Size of the file to download", type=SizeSuffix, default=None
+    )
     args = parser.parse_args()
-    return Args(direct_io=args.direct_io, num=args.num)
+    return Args(direct_io=args.direct_io, num=args.num, size=args.size)
 
 
 def main() -> None:
@@ -218,11 +233,24 @@ def main() -> None:
     transfer_list = [1]
     parallel_workers = args.num
 
-    def task(offset: SizeSuffix, args=args, rclone=rclone, transfer_list=transfer_list, mount_root_path=mount_root_path):
-        return test_profile_copy_bytes(args, rclone, offset, transfer_list, mount_root_path)
-    
-
-    with ThreadPoolExecutor(max_workers=parallel_workers) as executor:
+    def task(
+        offset: SizeSuffix,
+        args=args,
+        rclone=rclone,
+        transfer_list=transfer_list,
+        mount_root_path=mount_root_path,
+    ):
+        return test_profile_copy_bytes(
+            args=args,
+            rclone=rclone,
+            offset=offset,
+            mount_root_path=mount_root_path,
+            transfer_list=transfer_list,
+            size=args.size,
+            num=args.num,
+        )
+
+    with ThreadPoolExecutor(max_workers=parallel_workers) as _:
         tasks: list[Future] = []
         for i in range(parallel_workers):
             offset = SizeSuffix(i * 1024 * 1024 * 256)
@@ -231,4 +259,8 @@ def main() -> None:
 
 
 if __name__ == "__main__":
+    import sys
+
+    sys.argv.append("--size")
+    sys.argv.append("16MB")
     main()

{rclone_api-1.1.58.dist-info → rclone_api-1.1.60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.58
+Version: 1.1.60
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.58.dist-info → rclone_api-1.1.60.dist-info}/RECORD

@@ -25,7 +25,7 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=SybEsKZ1fw-O6RbXTiV-cmo3fAjqwhOuFZdawr_X0HU,6960
+rclone_api/profile/mount_copy_bytes.py,sha256=aHbIsIm2_fEvSZIHhRcXaNB2caT7xaE4zJi4dNYr7II,7621
 rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=D6wM9Nuu73LNGC8JCCfevqjF3qdZ21mQxYQClFLZLMU,3726
@@ -33,9 +33,9 @@ rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
 rclone_api/s3/upload_file_multipart.py,sha256=y9azNAU8QH5Ovwz33V2HZwNmJdlFjJg-jrXLZ1gtMds,10364
-rclone_api-1.1.58.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.58.dist-info/METADATA,sha256=y62azQy7mgVjksQozOOsWUtVw_wMeyef2-Y0NbnTEC0,4537
-rclone_api-1.1.58.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.58.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.1.58.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.58.dist-info/RECORD,,
+rclone_api-1.1.60.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.60.dist-info/METADATA,sha256=fUIrdhlIVOhWdheud6rnjJ2ikFhd4Z9hBw5fLiEJR2s,4537
+rclone_api-1.1.60.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.60.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.1.60.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.60.dist-info/RECORD,,