rclone-api 1.1.26__py2.py3-none-any.whl → 1.1.38__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/exec.py +4 -1
- rclone_api/mount.py +5 -4
- rclone_api/process.py +5 -7
- rclone_api/profile/mount_copy_bytes.py +187 -0
- rclone_api/rclone.py +66 -6
- rclone_api/s3/api.py +1 -1
- rclone_api/types.py +52 -25
- {rclone_api-1.1.26.dist-info → rclone_api-1.1.38.dist-info}/METADATA +2 -1
- {rclone_api-1.1.26.dist-info → rclone_api-1.1.38.dist-info}/RECORD +14 -13
- {rclone_api-1.1.26.dist-info → rclone_api-1.1.38.dist-info}/entry_points.txt +1 -0
- /rclone_api/s3/{chunk_uploader.py → upload_file_multipart.py} +0 -0
- {rclone_api-1.1.26.dist-info → rclone_api-1.1.38.dist-info}/LICENSE +0 -0
- {rclone_api-1.1.26.dist-info → rclone_api-1.1.38.dist-info}/WHEEL +0 -0
- {rclone_api-1.1.26.dist-info → rclone_api-1.1.38.dist-info}/top_level.txt +0 -0
rclone_api/exec.py
CHANGED
|
@@ -23,7 +23,9 @@ class RcloneExec:
|
|
|
23
23
|
cmd, self.rclone_config, self.rclone_exe, check=check, capture=capture
|
|
24
24
|
)
|
|
25
25
|
|
|
26
|
-
def launch_process(
|
|
26
|
+
def launch_process(
|
|
27
|
+
self, cmd: list[str], capture: bool | None, log: Path | None
|
|
28
|
+
) -> Process:
|
|
27
29
|
"""Launch rclone process."""
|
|
28
30
|
|
|
29
31
|
args: ProcessArgs = ProcessArgs(
|
|
@@ -32,6 +34,7 @@ class RcloneExec:
|
|
|
32
34
|
rclone_exe=self.rclone_exe,
|
|
33
35
|
cmd_list=cmd,
|
|
34
36
|
capture_stdout=capture,
|
|
37
|
+
log=log,
|
|
35
38
|
)
|
|
36
39
|
process = Process(args)
|
|
37
40
|
return process
|
rclone_api/mount.py
CHANGED
|
@@ -105,8 +105,8 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
|
|
|
105
105
|
mount_path = mount.mount_path if isinstance(mount, Mount) else mount
|
|
106
106
|
try:
|
|
107
107
|
mount_exists = mount_path.exists()
|
|
108
|
-
except OSError
|
|
109
|
-
warnings.warn(f"Error checking {mount_path}: {e}")
|
|
108
|
+
except OSError:
|
|
109
|
+
# warnings.warn(f"Error checking {mount_path}: {e}")
|
|
110
110
|
mount_exists = True
|
|
111
111
|
|
|
112
112
|
# Give the system a moment (if unmount is in progress, etc.)
|
|
@@ -139,8 +139,9 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
|
|
|
139
139
|
mount_path.rmdir()
|
|
140
140
|
if verbose:
|
|
141
141
|
print(f"Successfully removed mount directory {mount_path}")
|
|
142
|
-
except Exception
|
|
143
|
-
warnings.warn(f"Failed to remove mount {mount_path}: {e}")
|
|
142
|
+
except Exception:
|
|
143
|
+
# warnings.warn(f"Failed to remove mount {mount_path}: {e}")
|
|
144
|
+
pass
|
|
144
145
|
else:
|
|
145
146
|
warnings.warn(f"Unsupported platform: {_SYSTEM}")
|
|
146
147
|
|
rclone_api/process.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import atexit
|
|
2
|
-
import os
|
|
3
2
|
import subprocess
|
|
4
3
|
import threading
|
|
5
4
|
import time
|
|
@@ -13,12 +12,6 @@ from rclone_api.config import Config
|
|
|
13
12
|
from rclone_api.util import get_verbose
|
|
14
13
|
|
|
15
14
|
|
|
16
|
-
def _get_verbose(verbose: bool | None) -> bool:
|
|
17
|
-
if verbose is not None:
|
|
18
|
-
return verbose
|
|
19
|
-
return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
|
|
20
|
-
|
|
21
|
-
|
|
22
15
|
@dataclass
|
|
23
16
|
class ProcessArgs:
|
|
24
17
|
cmd: list[str]
|
|
@@ -27,12 +20,14 @@ class ProcessArgs:
|
|
|
27
20
|
cmd_list: list[str]
|
|
28
21
|
verbose: bool | None = None
|
|
29
22
|
capture_stdout: bool | None = None
|
|
23
|
+
log: Path | None = None
|
|
30
24
|
|
|
31
25
|
|
|
32
26
|
class Process:
|
|
33
27
|
def __init__(self, args: ProcessArgs) -> None:
|
|
34
28
|
assert args.rclone_exe.exists()
|
|
35
29
|
self.args = args
|
|
30
|
+
self.log = args.log
|
|
36
31
|
self.tempdir: TemporaryDirectory | None = None
|
|
37
32
|
verbose = get_verbose(args.verbose)
|
|
38
33
|
if isinstance(args.rclone_conf, Config):
|
|
@@ -52,6 +47,9 @@ class Process:
|
|
|
52
47
|
+ ["--config", str(rclone_conf.resolve())]
|
|
53
48
|
+ args.cmd
|
|
54
49
|
)
|
|
50
|
+
if self.args.log:
|
|
51
|
+
self.args.log.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
self.cmd += ["--log-file", str(self.args.log)]
|
|
55
53
|
if verbose:
|
|
56
54
|
cmd_str = subprocess.list2cmdline(self.cmd)
|
|
57
55
|
print(f"Running: {cmd_str}")
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unit test file.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import psutil
|
|
11
|
+
from dotenv import load_dotenv
|
|
12
|
+
|
|
13
|
+
from rclone_api import Config, Rclone, SizeSuffix
|
|
14
|
+
|
|
15
|
+
load_dotenv()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Credentials:
|
|
20
|
+
BUCKET_KEY_SECRET: str
|
|
21
|
+
BUCKET_KEY_PUBLIC: str
|
|
22
|
+
BUCKET_NAME: str
|
|
23
|
+
SRC_SFTP_HOST: str
|
|
24
|
+
SRC_SFTP_USER: str
|
|
25
|
+
SRC_SFTP_PORT: str
|
|
26
|
+
SRC_SFTP_PASS: str
|
|
27
|
+
BUCKET_URL: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _generate_rclone_config() -> tuple[Config, Credentials]:
|
|
31
|
+
|
|
32
|
+
cwd = Path.cwd()
|
|
33
|
+
print(f"Current working directory: {cwd}")
|
|
34
|
+
|
|
35
|
+
# assert that .env exists for this test
|
|
36
|
+
assert os.path.exists(
|
|
37
|
+
".env"
|
|
38
|
+
), "this test requires that the secret .env file exists with the credentials"
|
|
39
|
+
|
|
40
|
+
# BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks") # Default if not in .env
|
|
41
|
+
|
|
42
|
+
# Load additional environment variables
|
|
43
|
+
BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
|
|
44
|
+
BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
|
|
45
|
+
BUCKET_NAME = os.getenv("BUCKET_NAME")
|
|
46
|
+
SRC_SFTP_HOST = os.getenv("SRC_SFTP_HOST")
|
|
47
|
+
SRC_SFTP_USER = os.getenv("SRC_SFTP_USER")
|
|
48
|
+
SRC_SFTP_PORT = os.getenv("SRC_SFTP_PORT")
|
|
49
|
+
SRC_SFTP_PASS = os.getenv("SRC_SFTP_PASS")
|
|
50
|
+
# BUCKET_URL = os.getenv("BUCKET_URL")
|
|
51
|
+
BUCKET_URL = "sfo3.digitaloceanspaces.com"
|
|
52
|
+
|
|
53
|
+
config_text = f"""
|
|
54
|
+
[dst]
|
|
55
|
+
type = s3
|
|
56
|
+
provider = DigitalOcean
|
|
57
|
+
access_key_id = {BUCKET_KEY_PUBLIC}
|
|
58
|
+
secret_access_key = {BUCKET_KEY_SECRET}
|
|
59
|
+
endpoint = {BUCKET_URL}
|
|
60
|
+
bucket = {BUCKET_NAME}
|
|
61
|
+
|
|
62
|
+
[src]
|
|
63
|
+
type = sftp
|
|
64
|
+
host = {SRC_SFTP_HOST}
|
|
65
|
+
user = {SRC_SFTP_USER}
|
|
66
|
+
port = {SRC_SFTP_PORT}
|
|
67
|
+
pass = {SRC_SFTP_PASS}
|
|
68
|
+
|
|
69
|
+
"""
|
|
70
|
+
print("Config text:")
|
|
71
|
+
print(config_text)
|
|
72
|
+
# _CONFIG_PATH.write_text(config_text, encoding="utf-8")
|
|
73
|
+
# print(f"Config file written to: {_CONFIG_PATH}")
|
|
74
|
+
|
|
75
|
+
creds = Credentials(
|
|
76
|
+
BUCKET_KEY_SECRET=str(BUCKET_KEY_SECRET),
|
|
77
|
+
BUCKET_KEY_PUBLIC=str(BUCKET_KEY_PUBLIC),
|
|
78
|
+
BUCKET_NAME=str(BUCKET_NAME),
|
|
79
|
+
SRC_SFTP_HOST=str(SRC_SFTP_HOST),
|
|
80
|
+
SRC_SFTP_USER=str(SRC_SFTP_USER),
|
|
81
|
+
SRC_SFTP_PORT=str(SRC_SFTP_PORT),
|
|
82
|
+
SRC_SFTP_PASS=str(SRC_SFTP_PASS),
|
|
83
|
+
BUCKET_URL=str(BUCKET_URL),
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
return Config(config_text), creds
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _init() -> None:
|
|
90
|
+
"""Check if all required environment variables are set before running tests."""
|
|
91
|
+
required_vars = [
|
|
92
|
+
"BUCKET_NAME",
|
|
93
|
+
"BUCKET_KEY_SECRET",
|
|
94
|
+
"BUCKET_KEY_PUBLIC",
|
|
95
|
+
"BUCKET_URL",
|
|
96
|
+
]
|
|
97
|
+
missing = [var for var in required_vars if not os.getenv(var)]
|
|
98
|
+
if missing:
|
|
99
|
+
print(f"Missing required environment variables: {', '.join(missing)}")
|
|
100
|
+
os.environ["RCLONE_API_VERBOSE"] = "1"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_profile_copy_bytes() -> None:
|
|
104
|
+
print("Running test_profile_copy_bytes")
|
|
105
|
+
config, creds = _generate_rclone_config()
|
|
106
|
+
print("Config:")
|
|
107
|
+
print(config)
|
|
108
|
+
print("Credentials:")
|
|
109
|
+
print(creds)
|
|
110
|
+
rclone = Rclone(config)
|
|
111
|
+
|
|
112
|
+
sizes = [
|
|
113
|
+
1024 * 1024 * 1,
|
|
114
|
+
1024 * 1024 * 2,
|
|
115
|
+
1024 * 1024 * 4,
|
|
116
|
+
1024 * 1024 * 8,
|
|
117
|
+
1024 * 1024 * 16,
|
|
118
|
+
1024 * 1024 * 32,
|
|
119
|
+
1024 * 1024 * 64,
|
|
120
|
+
]
|
|
121
|
+
# transfer_list = [1, 2, 4, 8, 16]
|
|
122
|
+
transfer_list = [1, 2, 4]
|
|
123
|
+
|
|
124
|
+
# src_file = "dst:rclone-api-unit-test/zachs_video/internaly_ai_alignment.mp4"
|
|
125
|
+
# sftp mount
|
|
126
|
+
src_file = "src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
|
|
127
|
+
|
|
128
|
+
for size in sizes:
|
|
129
|
+
for transfers in transfer_list:
|
|
130
|
+
mount_log = Path("logs") / "mount" / f"mount_{size}_{transfers}.log"
|
|
131
|
+
print("\n\n")
|
|
132
|
+
print("#" * 80)
|
|
133
|
+
print(
|
|
134
|
+
f"# Started test download of {SizeSuffix(size)} with {transfers} transfers"
|
|
135
|
+
)
|
|
136
|
+
print("#" * 80)
|
|
137
|
+
net_io_start = psutil.net_io_counters()
|
|
138
|
+
start = time.time()
|
|
139
|
+
bytes_or_err: bytes | Exception = rclone.copy_bytes(
|
|
140
|
+
src=src_file,
|
|
141
|
+
offset=0,
|
|
142
|
+
length=size,
|
|
143
|
+
direct_io=True,
|
|
144
|
+
transfers=transfers,
|
|
145
|
+
mount_log=mount_log,
|
|
146
|
+
)
|
|
147
|
+
diff = time.time() - start
|
|
148
|
+
net_io_end = psutil.net_io_counters()
|
|
149
|
+
if isinstance(bytes_or_err, Exception):
|
|
150
|
+
print(bytes_or_err)
|
|
151
|
+
stack_trace = bytes_or_err.__traceback__
|
|
152
|
+
assert False, f"Error: {bytes_or_err}\nStack trace:\n{stack_trace}"
|
|
153
|
+
assert isinstance(bytes_or_err, bytes)
|
|
154
|
+
# self.assertEqual(len(bytes_or_err), size)
|
|
155
|
+
assert len(bytes_or_err) == size, f"Length: {len(bytes_or_err)} != {size}"
|
|
156
|
+
|
|
157
|
+
# print io stats
|
|
158
|
+
bytes_sent = net_io_end.bytes_sent - net_io_start.bytes_sent
|
|
159
|
+
bytes_recv = net_io_end.bytes_recv - net_io_start.bytes_recv
|
|
160
|
+
packets_sent = net_io_end.packets_sent - net_io_start.packets_sent
|
|
161
|
+
efficiency = size / (bytes_recv)
|
|
162
|
+
efficiency_100 = efficiency * 100
|
|
163
|
+
efficiency_str = f"{efficiency_100:.2f}"
|
|
164
|
+
|
|
165
|
+
bytes_send_suffix = SizeSuffix(bytes_sent)
|
|
166
|
+
bytes_recv_suffix = SizeSuffix(bytes_recv)
|
|
167
|
+
range_size = SizeSuffix(size)
|
|
168
|
+
|
|
169
|
+
print(f"\nFinished downloading {range_size} with {transfers} transfers")
|
|
170
|
+
print("Net IO stats:")
|
|
171
|
+
print(f"Bytes sent: {bytes_send_suffix}")
|
|
172
|
+
print(f"Bytes received: {bytes_recv_suffix}")
|
|
173
|
+
print(f"Packets sent: {packets_sent}")
|
|
174
|
+
print(f"Efficiency: {efficiency_str}%")
|
|
175
|
+
print(f"Time: {diff:.1f} seconds")
|
|
176
|
+
|
|
177
|
+
print("done")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def main() -> None:
|
|
181
|
+
"""Main entry point."""
|
|
182
|
+
_init()
|
|
183
|
+
test_profile_copy_bytes()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
if __name__ == "__main__":
|
|
187
|
+
main()
|
rclone_api/rclone.py
CHANGED
|
@@ -80,8 +80,10 @@ class Rclone:
|
|
|
80
80
|
) -> subprocess.CompletedProcess:
|
|
81
81
|
return self._exec.execute(cmd, check=check, capture=capture)
|
|
82
82
|
|
|
83
|
-
def _launch_process(
|
|
84
|
-
|
|
83
|
+
def _launch_process(
|
|
84
|
+
self, cmd: list[str], capture: bool | None = None, log: Path | None = None
|
|
85
|
+
) -> Process:
|
|
86
|
+
return self._exec.launch_process(cmd, capture=capture, log=log)
|
|
85
87
|
|
|
86
88
|
def webgui(self, other_args: list[str] | None = None) -> Process:
|
|
87
89
|
"""Launch the Rclone web GUI."""
|
|
@@ -684,6 +686,7 @@ class Rclone:
|
|
|
684
686
|
verbose: bool | None = None,
|
|
685
687
|
max_chunks_before_suspension: int | None = None,
|
|
686
688
|
mount_path: Path | None = None,
|
|
689
|
+
mount_log: Path | None = None,
|
|
687
690
|
) -> MultiUploadResult:
|
|
688
691
|
"""For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
|
|
689
692
|
from rclone_api.s3.api import S3Client
|
|
@@ -691,7 +694,7 @@ class Rclone:
|
|
|
691
694
|
from rclone_api.util import S3PathInfo, random_str, split_s3_path
|
|
692
695
|
|
|
693
696
|
other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
|
|
694
|
-
chunk_size = chunk_size or SizeSuffix("
|
|
697
|
+
chunk_size = chunk_size or SizeSuffix("64M")
|
|
695
698
|
unit_chunk_size = chunk_size / read_threads
|
|
696
699
|
vfs_read_chunk_size = unit_chunk_size
|
|
697
700
|
vfs_read_chunk_size_limit = chunk_size
|
|
@@ -724,10 +727,9 @@ class Rclone:
|
|
|
724
727
|
use_links=True,
|
|
725
728
|
vfs_cache_mode="minimal",
|
|
726
729
|
verbose=False,
|
|
730
|
+
log=mount_log,
|
|
727
731
|
other_args=other_args,
|
|
728
732
|
):
|
|
729
|
-
# raise NotImplementedError("Not implemented yet")
|
|
730
|
-
|
|
731
733
|
path_info: S3PathInfo = split_s3_path(dst)
|
|
732
734
|
remote = path_info.remote
|
|
733
735
|
bucket_name = path_info.bucket
|
|
@@ -813,6 +815,61 @@ class Rclone:
|
|
|
813
815
|
)
|
|
814
816
|
return out
|
|
815
817
|
|
|
818
|
+
def copy_bytes(
|
|
819
|
+
self,
|
|
820
|
+
src: str,
|
|
821
|
+
offset: int,
|
|
822
|
+
length: int,
|
|
823
|
+
transfers: int = 1, # Note, increasing transfers can result in devestating drop in performance.
|
|
824
|
+
# If outfile is supplied then bytes are written to this file and success returns bytes(0)
|
|
825
|
+
outfile: Path | None = None,
|
|
826
|
+
mount_log: Path | None = None,
|
|
827
|
+
direct_io: bool = True,
|
|
828
|
+
) -> bytes | Exception:
|
|
829
|
+
"""Copy bytes from a file to another file."""
|
|
830
|
+
from rclone_api.util import random_str
|
|
831
|
+
|
|
832
|
+
tmp_mnt = Path("tmp_mnt") / random_str(12)
|
|
833
|
+
src_parent_path = Path(src).parent.as_posix()
|
|
834
|
+
src_file = Path(src).name
|
|
835
|
+
other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
|
|
836
|
+
vfs_read_chunk_size = length // transfers
|
|
837
|
+
vfs_read_chunk_size_limit = length
|
|
838
|
+
vfs_read_chunk_streams = transfers
|
|
839
|
+
vfs_disk_space_total_size = length
|
|
840
|
+
other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
|
|
841
|
+
other_args += ["--vfs-read-chunk-size-limit", str(vfs_read_chunk_size_limit)]
|
|
842
|
+
other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
|
|
843
|
+
other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
|
|
844
|
+
other_args += ["--read-only"]
|
|
845
|
+
if direct_io:
|
|
846
|
+
other_args += ["--direct-io"]
|
|
847
|
+
|
|
848
|
+
try:
|
|
849
|
+
# use scoped mount to do the read, then write the bytes to the destination
|
|
850
|
+
with self.scoped_mount(
|
|
851
|
+
src_parent_path,
|
|
852
|
+
tmp_mnt,
|
|
853
|
+
use_links=True,
|
|
854
|
+
verbose=mount_log is not None,
|
|
855
|
+
vfs_cache_mode="minimal",
|
|
856
|
+
other_args=other_args,
|
|
857
|
+
log=mount_log,
|
|
858
|
+
):
|
|
859
|
+
src_file_mnt = tmp_mnt / src_file
|
|
860
|
+
with open(src_file_mnt, "rb") as f:
|
|
861
|
+
f.seek(offset)
|
|
862
|
+
data = f.read(length)
|
|
863
|
+
if outfile is None:
|
|
864
|
+
return data
|
|
865
|
+
with open(outfile, "wb") as out:
|
|
866
|
+
out.write(data)
|
|
867
|
+
del data
|
|
868
|
+
return bytes(0)
|
|
869
|
+
|
|
870
|
+
except Exception as e:
|
|
871
|
+
return e
|
|
872
|
+
|
|
816
873
|
def copy_dir(
|
|
817
874
|
self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
|
|
818
875
|
) -> CompletedProcess:
|
|
@@ -845,6 +902,7 @@ class Rclone:
|
|
|
845
902
|
use_links: bool | None = None,
|
|
846
903
|
vfs_cache_mode: str | None = None,
|
|
847
904
|
verbose: bool | None = None,
|
|
905
|
+
log: Path | None = None,
|
|
848
906
|
other_args: list[str] | None = None,
|
|
849
907
|
) -> Mount:
|
|
850
908
|
"""Mount a remote or directory to a local path.
|
|
@@ -879,7 +937,7 @@ class Rclone:
|
|
|
879
937
|
cmd_list.append("-vvvv")
|
|
880
938
|
if other_args:
|
|
881
939
|
cmd_list += other_args
|
|
882
|
-
proc = self._launch_process(cmd_list)
|
|
940
|
+
proc = self._launch_process(cmd_list, log=log)
|
|
883
941
|
mount_read_only = "--read-only" in cmd_list # hint to allow fast teardown
|
|
884
942
|
mount: Mount = Mount(mount_path=outdir, process=proc, read_only=mount_read_only)
|
|
885
943
|
return mount
|
|
@@ -893,6 +951,7 @@ class Rclone:
|
|
|
893
951
|
use_links: bool | None = None,
|
|
894
952
|
vfs_cache_mode: str | None = None,
|
|
895
953
|
verbose: bool | None = None,
|
|
954
|
+
log: Path | None = None,
|
|
896
955
|
other_args: list[str] | None = None,
|
|
897
956
|
) -> Generator[Mount, None, None]:
|
|
898
957
|
"""Like mount, but can be used in a context manager."""
|
|
@@ -905,6 +964,7 @@ class Rclone:
|
|
|
905
964
|
use_links=use_links,
|
|
906
965
|
vfs_cache_mode=vfs_cache_mode,
|
|
907
966
|
verbose=verbose,
|
|
967
|
+
log=log,
|
|
908
968
|
other_args=other_args,
|
|
909
969
|
)
|
|
910
970
|
try:
|
rclone_api/s3/api.py
CHANGED
|
@@ -9,9 +9,9 @@ from rclone_api.s3.basic_ops import (
|
|
|
9
9
|
list_bucket_contents,
|
|
10
10
|
upload_file,
|
|
11
11
|
)
|
|
12
|
-
from rclone_api.s3.chunk_uploader import MultiUploadResult, upload_file_multipart
|
|
13
12
|
from rclone_api.s3.create import create_s3_client
|
|
14
13
|
from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
|
|
14
|
+
from rclone_api.s3.upload_file_multipart import MultiUploadResult, upload_file_multipart
|
|
15
15
|
|
|
16
16
|
_MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
|
|
17
17
|
|
rclone_api/types.py
CHANGED
|
@@ -37,41 +37,68 @@ class SizeResult:
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
def _to_size_suffix(size: int) -> str:
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
40
|
+
def _convert(size: int) -> tuple[float, str]:
|
|
41
|
+
val: float
|
|
42
|
+
unit: str
|
|
43
|
+
if size < 1024:
|
|
44
|
+
val = size
|
|
45
|
+
unit = "B"
|
|
46
|
+
elif size < 1024**2:
|
|
47
|
+
val = size / 1024
|
|
48
|
+
unit = "K"
|
|
49
|
+
elif size < 1024**3:
|
|
50
|
+
val = size / (1024**2)
|
|
51
|
+
unit = "M"
|
|
52
|
+
elif size < 1024**4:
|
|
53
|
+
val = size / (1024**3)
|
|
54
|
+
unit = "G"
|
|
55
|
+
elif size < 1024**5:
|
|
56
|
+
val = size / (1024**4)
|
|
57
|
+
unit = "T"
|
|
58
|
+
elif size < 1024**6:
|
|
59
|
+
val = size / (1024**5)
|
|
60
|
+
unit = "P"
|
|
61
|
+
else:
|
|
62
|
+
raise ValueError(f"Invalid size: {size}")
|
|
63
|
+
|
|
64
|
+
return val, unit
|
|
65
|
+
|
|
66
|
+
def _fmt(_val: float | int, _unit: str) -> str:
|
|
67
|
+
# If the float is an integer, drop the decimal, otherwise format with one decimal.
|
|
68
|
+
val_str: str = str(_val)
|
|
69
|
+
if not val_str.endswith(".0"):
|
|
70
|
+
first_str: str = f"{_val:.1f}"
|
|
71
|
+
else:
|
|
72
|
+
first_str = str(int(_val))
|
|
73
|
+
return first_str + _unit
|
|
74
|
+
|
|
75
|
+
val, unit = _convert(size)
|
|
76
|
+
out = _fmt(val, unit)
|
|
77
|
+
# Now round trip the value to fix floating point issues via rounding.
|
|
78
|
+
int_val = _from_size_suffix(out)
|
|
79
|
+
val, unit = _convert(int_val)
|
|
80
|
+
out = _fmt(val, unit)
|
|
81
|
+
return out
|
|
62
82
|
|
|
63
83
|
|
|
64
84
|
# Update regex to allow decimals (e.g., 16.5MB)
|
|
65
85
|
_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+(?:\.\d+)?)([A-Za-z]+)$")
|
|
66
86
|
|
|
67
87
|
|
|
88
|
+
def _parse_elements(value: str) -> tuple[str, str] | None:
|
|
89
|
+
match = _PATTERN_SIZE_SUFFIX.match(value)
|
|
90
|
+
if match is None:
|
|
91
|
+
return None
|
|
92
|
+
return match.group(1), match.group(2)
|
|
93
|
+
|
|
94
|
+
|
|
68
95
|
def _from_size_suffix(size: str) -> int:
|
|
69
96
|
if size == "0":
|
|
70
97
|
return 0
|
|
71
|
-
|
|
72
|
-
if
|
|
98
|
+
pair = _parse_elements(size)
|
|
99
|
+
if pair is None:
|
|
73
100
|
raise ValueError(f"Invalid size suffix: {size}")
|
|
74
|
-
num_str, suffix =
|
|
101
|
+
num_str, suffix = pair
|
|
75
102
|
n = float(num_str)
|
|
76
103
|
# Determine the unit from the first letter (e.g., "M" from "MB")
|
|
77
104
|
unit = suffix[0].upper()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: rclone_api
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.38
|
|
4
4
|
Summary: rclone api in python
|
|
5
5
|
Home-page: https://github.com/zackees/rclone-api
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -12,6 +12,7 @@ License-File: LICENSE
|
|
|
12
12
|
Requires-Dist: pyright>=1.1.393
|
|
13
13
|
Requires-Dist: python-dotenv>=1.0.0
|
|
14
14
|
Requires-Dist: certifi>=2025.1.31
|
|
15
|
+
Requires-Dist: psutil
|
|
15
16
|
Requires-Dist: boto3<=1.35.99,>=1.20.1
|
|
16
17
|
Dynamic: home-page
|
|
17
18
|
|
|
@@ -7,17 +7,17 @@ rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
|
|
|
7
7
|
rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
|
|
8
8
|
rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
|
|
9
9
|
rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
|
|
10
|
-
rclone_api/exec.py,sha256=
|
|
10
|
+
rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
|
|
11
11
|
rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
|
|
12
12
|
rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
|
|
13
13
|
rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
|
|
14
|
-
rclone_api/mount.py,sha256=
|
|
15
|
-
rclone_api/process.py,sha256=
|
|
16
|
-
rclone_api/rclone.py,sha256=
|
|
14
|
+
rclone_api/mount.py,sha256=ryAjkX4_kFeFZWLiBVpcGy2VilpvVhFbWeWfEX4jMKs,6104
|
|
15
|
+
rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
|
|
16
|
+
rclone_api/rclone.py,sha256=meZkXmIM1M_L0Xu2hf0fcm-tG1cF5t6AHSIM6TOXUQM,42624
|
|
17
17
|
rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
|
|
18
18
|
rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
|
|
19
19
|
rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
|
|
20
|
-
rclone_api/types.py,sha256
|
|
20
|
+
rclone_api/types.py,sha256=AWVIoCaRkUBCXOQffxdkFCOeUhPwolQh9d2albyAGHc,4730
|
|
21
21
|
rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
|
|
22
22
|
rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
|
|
23
23
|
rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
|
|
@@ -25,16 +25,17 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
|
|
|
25
25
|
rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
|
|
26
26
|
rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
|
|
27
27
|
rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
|
|
28
|
-
rclone_api/
|
|
28
|
+
rclone_api/profile/mount_copy_bytes.py,sha256=eVxF830AcsUTrDh5CwyyBLtgxh9AZa5ae51tk0yn9Xo,5626
|
|
29
|
+
rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
|
|
29
30
|
rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
|
|
30
31
|
rclone_api/s3/chunk_file.py,sha256=YELR-EzR7RHpzCDGpYdzlwu21NZW5wttIDvLoONI4aU,3477
|
|
31
32
|
rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,10316
|
|
32
|
-
rclone_api/s3/chunk_uploader.py,sha256=1jQAdk35Fa9Tcq36bS65262cs7AcNG2DAFQ-NdYlWSw,9961
|
|
33
33
|
rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
|
|
34
34
|
rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
|
|
35
|
-
rclone_api
|
|
36
|
-
rclone_api-1.1.
|
|
37
|
-
rclone_api-1.1.
|
|
38
|
-
rclone_api-1.1.
|
|
39
|
-
rclone_api-1.1.
|
|
40
|
-
rclone_api-1.1.
|
|
35
|
+
rclone_api/s3/upload_file_multipart.py,sha256=1jQAdk35Fa9Tcq36bS65262cs7AcNG2DAFQ-NdYlWSw,9961
|
|
36
|
+
rclone_api-1.1.38.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
|
37
|
+
rclone_api-1.1.38.dist-info/METADATA,sha256=xjIzdVaEv0B_zQJLXJCkONJBns3rvpc91QC3-BWAAvM,4537
|
|
38
|
+
rclone_api-1.1.38.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
|
|
39
|
+
rclone_api-1.1.38.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
|
|
40
|
+
rclone_api-1.1.38.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
|
41
|
+
rclone_api-1.1.38.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|