rclone-api 1.1.32__py2.py3-none-any.whl → 1.1.33__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/profiling/mount_copy_bytes.py +86 -84
- {rclone_api-1.1.32.dist-info → rclone_api-1.1.33.dist-info}/METADATA +1 -1
- {rclone_api-1.1.32.dist-info → rclone_api-1.1.33.dist-info}/RECORD +7 -7
- {rclone_api-1.1.32.dist-info → rclone_api-1.1.33.dist-info}/LICENSE +0 -0
- {rclone_api-1.1.32.dist-info → rclone_api-1.1.33.dist-info}/WHEEL +0 -0
- {rclone_api-1.1.32.dist-info → rclone_api-1.1.33.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.1.32.dist-info → rclone_api-1.1.33.dist-info}/top_level.txt +0 -0
|
@@ -4,7 +4,6 @@ Unit test file.
|
|
|
4
4
|
|
|
5
5
|
import os
|
|
6
6
|
import time
|
|
7
|
-
import unittest
|
|
8
7
|
|
|
9
8
|
import psutil
|
|
10
9
|
from dotenv import load_dotenv
|
|
@@ -57,90 +56,93 @@ pass = {SRC_SFTP_PASS}
|
|
|
57
56
|
return Config(config_text)
|
|
58
57
|
|
|
59
58
|
|
|
60
|
-
|
|
61
|
-
"""
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
""
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
59
|
+
def _init() -> None:
|
|
60
|
+
"""Check if all required environment variables are set before running tests."""
|
|
61
|
+
required_vars = [
|
|
62
|
+
"BUCKET_NAME",
|
|
63
|
+
"BUCKET_KEY_SECRET",
|
|
64
|
+
"BUCKET_KEY_PUBLIC",
|
|
65
|
+
"BUCKET_URL",
|
|
66
|
+
]
|
|
67
|
+
missing = [var for var in required_vars if not os.getenv(var)]
|
|
68
|
+
if missing:
|
|
69
|
+
print(f"Missing required environment variables: {', '.join(missing)}")
|
|
70
|
+
os.environ["RCLONE_API_VERBOSE"] = "1"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_profile_copy_bytes() -> None:
|
|
74
|
+
print("Running test_profile_copy_bytes")
|
|
75
|
+
rclone = Rclone(_generate_rclone_config())
|
|
76
|
+
sizes = [
|
|
77
|
+
1024 * 1024 * 1,
|
|
78
|
+
1024 * 1024 * 2,
|
|
79
|
+
1024 * 1024 * 4,
|
|
80
|
+
1024 * 1024 * 8,
|
|
81
|
+
1024 * 1024 * 16,
|
|
82
|
+
1024 * 1024 * 32,
|
|
83
|
+
1024 * 1024 * 64,
|
|
84
|
+
]
|
|
85
|
+
# transfer_list = [1, 2, 4, 8, 16]
|
|
86
|
+
transfer_list = [1, 2, 4]
|
|
87
|
+
|
|
88
|
+
# src_file = "dst:rclone-api-unit-test/zachs_video/internaly_ai_alignment.mp4"
|
|
89
|
+
# sftp mount
|
|
90
|
+
src_file = "src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
|
|
91
|
+
|
|
92
|
+
for size in sizes:
|
|
93
|
+
for transfers in transfer_list:
|
|
94
|
+
print("\n\n")
|
|
95
|
+
print("#" * 80)
|
|
96
|
+
print(
|
|
97
|
+
f"# Started test download of {SizeSuffix(size)} with {transfers} transfers"
|
|
75
98
|
)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
# print io stats
|
|
123
|
-
bytes_sent = net_io_end.bytes_sent - net_io_start.bytes_sent
|
|
124
|
-
bytes_recv = net_io_end.bytes_recv - net_io_start.bytes_recv
|
|
125
|
-
packets_sent = net_io_end.packets_sent - net_io_start.packets_sent
|
|
126
|
-
efficiency = size / (bytes_recv)
|
|
127
|
-
efficiency_100 = efficiency * 100
|
|
128
|
-
efficiency_str = f"{efficiency_100:.2f}"
|
|
129
|
-
|
|
130
|
-
bytes_send_suffix = SizeSuffix(bytes_sent)
|
|
131
|
-
bytes_recv_suffix = SizeSuffix(bytes_recv)
|
|
132
|
-
range_size = SizeSuffix(size)
|
|
133
|
-
|
|
134
|
-
print(f"\nFinished downloading {range_size} with {transfers} transfers")
|
|
135
|
-
print("Net IO stats:")
|
|
136
|
-
print(f"Bytes sent: {bytes_send_suffix}")
|
|
137
|
-
print(f"Bytes received: {bytes_recv_suffix}")
|
|
138
|
-
print(f"Packets sent: {packets_sent}")
|
|
139
|
-
print(f"Efficiency: {efficiency_str}%")
|
|
140
|
-
print(f"Time: {diff:.1f} seconds")
|
|
141
|
-
|
|
142
|
-
print("done")
|
|
99
|
+
print("#" * 80)
|
|
100
|
+
net_io_start = psutil.net_io_counters()
|
|
101
|
+
start = time.time()
|
|
102
|
+
bytes_or_err: bytes | Exception = rclone.copy_bytes(
|
|
103
|
+
src=src_file,
|
|
104
|
+
offset=0,
|
|
105
|
+
length=size,
|
|
106
|
+
direct_io=True,
|
|
107
|
+
transfers=transfers,
|
|
108
|
+
)
|
|
109
|
+
diff = time.time() - start
|
|
110
|
+
net_io_end = psutil.net_io_counters()
|
|
111
|
+
if isinstance(bytes_or_err, Exception):
|
|
112
|
+
print(bytes_or_err)
|
|
113
|
+
assert False, f"Error: {bytes_or_err}"
|
|
114
|
+
assert isinstance(bytes_or_err, bytes)
|
|
115
|
+
# self.assertEqual(len(bytes_or_err), size)
|
|
116
|
+
assert len(bytes_or_err) == size, f"Length: {len(bytes_or_err)} != {size}"
|
|
117
|
+
|
|
118
|
+
# print io stats
|
|
119
|
+
bytes_sent = net_io_end.bytes_sent - net_io_start.bytes_sent
|
|
120
|
+
bytes_recv = net_io_end.bytes_recv - net_io_start.bytes_recv
|
|
121
|
+
packets_sent = net_io_end.packets_sent - net_io_start.packets_sent
|
|
122
|
+
efficiency = size / (bytes_recv)
|
|
123
|
+
efficiency_100 = efficiency * 100
|
|
124
|
+
efficiency_str = f"{efficiency_100:.2f}"
|
|
125
|
+
|
|
126
|
+
bytes_send_suffix = SizeSuffix(bytes_sent)
|
|
127
|
+
bytes_recv_suffix = SizeSuffix(bytes_recv)
|
|
128
|
+
range_size = SizeSuffix(size)
|
|
129
|
+
|
|
130
|
+
print(f"\nFinished downloading {range_size} with {transfers} transfers")
|
|
131
|
+
print("Net IO stats:")
|
|
132
|
+
print(f"Bytes sent: {bytes_send_suffix}")
|
|
133
|
+
print(f"Bytes received: {bytes_recv_suffix}")
|
|
134
|
+
print(f"Packets sent: {packets_sent}")
|
|
135
|
+
print(f"Efficiency: {efficiency_str}%")
|
|
136
|
+
print(f"Time: {diff:.1f} seconds")
|
|
137
|
+
|
|
138
|
+
print("done")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def main() -> None:
|
|
142
|
+
"""Main entry point."""
|
|
143
|
+
_init()
|
|
144
|
+
test_profile_copy_bytes()
|
|
143
145
|
|
|
144
146
|
|
|
145
147
|
if __name__ == "__main__":
|
|
146
|
-
|
|
148
|
+
main()
|
|
@@ -25,7 +25,7 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
|
|
|
25
25
|
rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
|
|
26
26
|
rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
|
|
27
27
|
rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
|
|
28
|
-
rclone_api/profiling/mount_copy_bytes.py,sha256=
|
|
28
|
+
rclone_api/profiling/mount_copy_bytes.py,sha256=XpVmEh0x0S2XlivWf85OcDzUxpAv_lE2hVaiJtvt4fg,4537
|
|
29
29
|
rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
|
|
30
30
|
rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
|
|
31
31
|
rclone_api/s3/chunk_file.py,sha256=YELR-EzR7RHpzCDGpYdzlwu21NZW5wttIDvLoONI4aU,3477
|
|
@@ -33,9 +33,9 @@ rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,
|
|
|
33
33
|
rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
|
|
34
34
|
rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
|
|
35
35
|
rclone_api/s3/upload_file_multipart.py,sha256=1jQAdk35Fa9Tcq36bS65262cs7AcNG2DAFQ-NdYlWSw,9961
|
|
36
|
-
rclone_api-1.1.
|
|
37
|
-
rclone_api-1.1.
|
|
38
|
-
rclone_api-1.1.
|
|
39
|
-
rclone_api-1.1.
|
|
40
|
-
rclone_api-1.1.
|
|
41
|
-
rclone_api-1.1.
|
|
36
|
+
rclone_api-1.1.33.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
|
37
|
+
rclone_api-1.1.33.dist-info/METADATA,sha256=zkDoV5_SNjYQ9D0n-hzVlXoNoKLBTV16-thm4p2gUCg,4537
|
|
38
|
+
rclone_api-1.1.33.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
|
|
39
|
+
rclone_api-1.1.33.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
|
|
40
|
+
rclone_api-1.1.33.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
|
41
|
+
rclone_api-1.1.33.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|