rclone-api 1.1.7__py2.py3-none-any.whl → 1.1.10__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/cmd/copy_large_s3.py +14 -10
- rclone_api/experimental/flags.py +8 -1
- rclone_api/mount.py +61 -3
- rclone_api/process.py +4 -0
- rclone_api/rclone.py +29 -102
- rclone_api/s3/chunk_types.py +17 -2
- rclone_api/types.py +30 -19
- rclone_api/util.py +0 -21
- {rclone_api-1.1.7.dist-info → rclone_api-1.1.10.dist-info}/METADATA +1 -1
- {rclone_api-1.1.7.dist-info → rclone_api-1.1.10.dist-info}/RECORD +14 -14
- {rclone_api-1.1.7.dist-info → rclone_api-1.1.10.dist-info}/LICENSE +0 -0
- {rclone_api-1.1.7.dist-info → rclone_api-1.1.10.dist-info}/WHEEL +0 -0
- {rclone_api-1.1.7.dist-info → rclone_api-1.1.10.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.1.7.dist-info → rclone_api-1.1.10.dist-info}/top_level.txt +0 -0
rclone_api/cmd/copy_large_s3.py
CHANGED
|
@@ -12,8 +12,8 @@ class Args:
|
|
|
12
12
|
config_path: Path
|
|
13
13
|
src: str
|
|
14
14
|
dst: str
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
chunk_size: SizeSuffix
|
|
16
|
+
threads: int
|
|
17
17
|
retries: int
|
|
18
18
|
save_state_json: Path
|
|
19
19
|
verbose: bool
|
|
@@ -38,13 +38,13 @@ def _parse_args() -> Args:
|
|
|
38
38
|
"--chunk-size",
|
|
39
39
|
help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
|
|
40
40
|
type=str,
|
|
41
|
-
default="
|
|
41
|
+
default="1G",
|
|
42
42
|
)
|
|
43
43
|
parser.add_argument(
|
|
44
|
-
"--
|
|
45
|
-
help="
|
|
44
|
+
"--threads",
|
|
45
|
+
help="Number of threads to use per chunk",
|
|
46
46
|
type=int,
|
|
47
|
-
default=
|
|
47
|
+
default=64,
|
|
48
48
|
)
|
|
49
49
|
parser.add_argument("--retries", help="Number of retries", type=int, default=3)
|
|
50
50
|
parser.add_argument(
|
|
@@ -59,8 +59,8 @@ def _parse_args() -> Args:
|
|
|
59
59
|
config_path=Path(args.config),
|
|
60
60
|
src=args.src,
|
|
61
61
|
dst=args.dst,
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
chunk_size=SizeSuffix(args.chunk_size),
|
|
63
|
+
threads=args.threads,
|
|
64
64
|
retries=args.retries,
|
|
65
65
|
save_state_json=args.resume_json,
|
|
66
66
|
verbose=args.verbose,
|
|
@@ -72,11 +72,15 @@ def main() -> int:
|
|
|
72
72
|
"""Main entry point."""
|
|
73
73
|
args = _parse_args()
|
|
74
74
|
rclone = Rclone(rclone_conf=args.config_path)
|
|
75
|
+
# unit_chunk = args.chunk_size / args.threads
|
|
75
76
|
rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
|
|
76
77
|
src=args.src,
|
|
77
78
|
dst=args.dst,
|
|
78
|
-
chunk_size=args.
|
|
79
|
-
|
|
79
|
+
chunk_size=args.chunk_size,
|
|
80
|
+
threads=args.threads,
|
|
81
|
+
# vfs_read_chunk_size=unit_chunk,
|
|
82
|
+
# vfs_read_chunk_size_limit=args.chunk_size,
|
|
83
|
+
# vfs_read_chunk_streams=args.threads,
|
|
80
84
|
retries=args.retries,
|
|
81
85
|
save_state_json=args.save_state_json,
|
|
82
86
|
verbose=args.verbose,
|
rclone_api/experimental/flags.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
|
|
3
3
|
from rclone_api.experimental.flags_base import BaseFlags, merge_flags
|
|
4
|
+
from rclone_api.types import SizeSuffix
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
@dataclass
|
|
@@ -20,7 +21,7 @@ class CopyFlags(BaseFlags):
|
|
|
20
21
|
links: bool | None = None
|
|
21
22
|
max_backlog: int | None = None
|
|
22
23
|
max_duration: str | None = None
|
|
23
|
-
max_transfer:
|
|
24
|
+
max_transfer: SizeSuffix | None = None
|
|
24
25
|
metadata: bool | None = None
|
|
25
26
|
modify_window: str | None = None
|
|
26
27
|
multi_thread_chunk_size: str | None = None
|
|
@@ -77,6 +78,12 @@ def unit_test() -> None:
|
|
|
77
78
|
merged_d_c = copy_flags_d.merge(copy_flags_c)
|
|
78
79
|
print("C:", merged_d_c)
|
|
79
80
|
|
|
81
|
+
# now do the one with the SizeSuffix type
|
|
82
|
+
copy_flags_e = CopyFlags(max_transfer=SizeSuffix("128M"))
|
|
83
|
+
copy_flags_f = CopyFlags(max_transfer=SizeSuffix("256M"))
|
|
84
|
+
merged_e_f = copy_flags_e.merge(copy_flags_f)
|
|
85
|
+
print("D:", merged_e_f)
|
|
86
|
+
|
|
80
87
|
|
|
81
88
|
if __name__ == "__main__":
|
|
82
89
|
unit_test()
|
rclone_api/mount.py
CHANGED
|
@@ -1,12 +1,41 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import platform
|
|
2
3
|
import subprocess
|
|
3
4
|
import time
|
|
4
5
|
import warnings
|
|
6
|
+
from dataclasses import dataclass
|
|
5
7
|
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from rclone_api.process import Process
|
|
6
11
|
|
|
7
12
|
_SYSTEM = platform.system() # "Linux", "Darwin", "Windows", etc.
|
|
8
13
|
|
|
9
14
|
|
|
15
|
+
@dataclass
|
|
16
|
+
class Mount:
|
|
17
|
+
"""Mount information."""
|
|
18
|
+
|
|
19
|
+
mount_path: Path
|
|
20
|
+
process: Process
|
|
21
|
+
_closed: bool = False
|
|
22
|
+
|
|
23
|
+
def __post_init__(self):
|
|
24
|
+
assert isinstance(self.mount_path, Path)
|
|
25
|
+
assert self.process is not None
|
|
26
|
+
wait_for_mount(self.mount_path, self.process)
|
|
27
|
+
|
|
28
|
+
def close(self, wait=True) -> None:
|
|
29
|
+
"""Clean up the mount."""
|
|
30
|
+
if self._closed:
|
|
31
|
+
return
|
|
32
|
+
self._closed = True
|
|
33
|
+
clean_mount(self, verbose=False)
|
|
34
|
+
|
|
35
|
+
def __del__(self):
|
|
36
|
+
self.close(wait=False)
|
|
37
|
+
|
|
38
|
+
|
|
10
39
|
def run_command(cmd: str, verbose: bool) -> int:
|
|
11
40
|
"""Run a shell command and print its output if verbose is True."""
|
|
12
41
|
if verbose:
|
|
@@ -36,7 +65,27 @@ def prepare_mount(outdir: Path, verbose: bool) -> None:
|
|
|
36
65
|
outdir.mkdir(parents=True, exist_ok=True)
|
|
37
66
|
|
|
38
67
|
|
|
39
|
-
def
|
|
68
|
+
def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
|
|
69
|
+
from rclone_api.process import Process
|
|
70
|
+
|
|
71
|
+
assert isinstance(mount_process, Process)
|
|
72
|
+
expire_time = time.time() + timeout
|
|
73
|
+
while time.time() < expire_time:
|
|
74
|
+
rtn = mount_process.poll()
|
|
75
|
+
if rtn is not None:
|
|
76
|
+
cmd_str = subprocess.list2cmdline(mount_process.cmd)
|
|
77
|
+
raise subprocess.CalledProcessError(rtn, cmd_str)
|
|
78
|
+
if path.exists():
|
|
79
|
+
# how many files?
|
|
80
|
+
dircontents = os.listdir(str(path))
|
|
81
|
+
if len(dircontents) > 0:
|
|
82
|
+
print(f"Mount point {path}, waiting 5 seconds for files to appear.")
|
|
83
|
+
time.sleep(5)
|
|
84
|
+
return
|
|
85
|
+
time.sleep(1)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
|
|
40
89
|
"""
|
|
41
90
|
Clean up a mount path across Linux, macOS, and Windows.
|
|
42
91
|
|
|
@@ -45,7 +94,14 @@ def clean_mount(mount_path: Path, verbose: bool = False) -> None:
|
|
|
45
94
|
and 'umount'. On macOS it uses 'umount' (and optionally 'diskutil unmount'),
|
|
46
95
|
while on Windows it attempts to remove the mount point via 'mountvol /D'.
|
|
47
96
|
"""
|
|
97
|
+
proc = mount.process if isinstance(mount, Mount) else None
|
|
98
|
+
if proc is not None and proc.poll() is None:
|
|
99
|
+
if verbose:
|
|
100
|
+
print(f"Terminating mount process {proc.pid}")
|
|
101
|
+
proc.kill()
|
|
102
|
+
|
|
48
103
|
# Check if the mount path exists; if an OSError occurs, assume it exists.
|
|
104
|
+
mount_path = mount.mount_path if isinstance(mount, Mount) else mount
|
|
49
105
|
try:
|
|
50
106
|
mount_exists = mount_path.exists()
|
|
51
107
|
except OSError as e:
|
|
@@ -53,7 +109,8 @@ def clean_mount(mount_path: Path, verbose: bool = False) -> None:
|
|
|
53
109
|
mount_exists = True
|
|
54
110
|
|
|
55
111
|
# Give the system a moment (if unmount is in progress, etc.)
|
|
56
|
-
|
|
112
|
+
if wait:
|
|
113
|
+
time.sleep(2)
|
|
57
114
|
|
|
58
115
|
if not mount_exists:
|
|
59
116
|
if verbose:
|
|
@@ -87,7 +144,8 @@ def clean_mount(mount_path: Path, verbose: bool = False) -> None:
|
|
|
87
144
|
warnings.warn(f"Unsupported platform: {_SYSTEM}")
|
|
88
145
|
|
|
89
146
|
# Allow some time for the unmount commands to take effect.
|
|
90
|
-
|
|
147
|
+
if wait:
|
|
148
|
+
time.sleep(2)
|
|
91
149
|
|
|
92
150
|
# Re-check if the mount path still exists.
|
|
93
151
|
try:
|
rclone_api/process.py
CHANGED
rclone_api/rclone.py
CHANGED
|
@@ -25,6 +25,7 @@ from rclone_api.dir_listing import DirListing
|
|
|
25
25
|
from rclone_api.exec import RcloneExec
|
|
26
26
|
from rclone_api.file import File
|
|
27
27
|
from rclone_api.group_files import group_files
|
|
28
|
+
from rclone_api.mount import Mount, clean_mount, prepare_mount
|
|
28
29
|
from rclone_api.process import Process
|
|
29
30
|
from rclone_api.remote import Remote
|
|
30
31
|
from rclone_api.rpath import RPath
|
|
@@ -46,12 +47,9 @@ from rclone_api.util import (
|
|
|
46
47
|
get_rclone_exe,
|
|
47
48
|
get_verbose,
|
|
48
49
|
to_path,
|
|
49
|
-
wait_for_mount,
|
|
50
50
|
)
|
|
51
51
|
from rclone_api.walk import walk
|
|
52
52
|
|
|
53
|
-
_IS_WINDOWS = os.name == "nt"
|
|
54
|
-
|
|
55
53
|
|
|
56
54
|
def rclone_verbose(verbose: bool | None) -> bool:
|
|
57
55
|
if verbose is not None:
|
|
@@ -680,12 +678,7 @@ class Rclone:
|
|
|
680
678
|
dst: str,
|
|
681
679
|
save_state_json: Path,
|
|
682
680
|
chunk_size: SizeSuffix | None = None,
|
|
683
|
-
|
|
684
|
-
# * 1024
|
|
685
|
-
# * 1024, # This setting will scale the performance of the upload
|
|
686
|
-
concurrent_chunks: (
|
|
687
|
-
int | None
|
|
688
|
-
) = None, # This setting will scale the performance of the upload
|
|
681
|
+
threads: int = 16,
|
|
689
682
|
retries: int = 3,
|
|
690
683
|
verbose: bool | None = None,
|
|
691
684
|
max_chunks_before_suspension: int | None = None,
|
|
@@ -696,30 +689,29 @@ class Rclone:
|
|
|
696
689
|
from rclone_api.s3.create import S3Credentials
|
|
697
690
|
from rclone_api.util import S3PathInfo, random_str, split_s3_path
|
|
698
691
|
|
|
699
|
-
|
|
700
|
-
chunk_size = SizeSuffix(
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
"--vfs-read-chunk-size",
|
|
712
|
-
chunk_size.as_str(),
|
|
692
|
+
other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
|
|
693
|
+
chunk_size = chunk_size or SizeSuffix("128M")
|
|
694
|
+
unit_chunk_size = chunk_size / threads
|
|
695
|
+
vfs_read_chunk_size = unit_chunk_size
|
|
696
|
+
vfs_read_chunk_size_limit = chunk_size
|
|
697
|
+
vfs_read_chunk_streams = threads
|
|
698
|
+
vfs_disk_space_total_size = chunk_size
|
|
699
|
+
assert (
|
|
700
|
+
chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
|
|
701
|
+
), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
|
|
702
|
+
other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
|
|
703
|
+
other_args += [
|
|
713
704
|
"--vfs-read-chunk-size-limit",
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
705
|
+
vfs_read_chunk_size_limit.as_str(),
|
|
706
|
+
]
|
|
707
|
+
other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
|
|
708
|
+
other_args += [
|
|
709
|
+
"--vfs-disk-space-total-size",
|
|
710
|
+
vfs_disk_space_total_size.as_str(),
|
|
718
711
|
]
|
|
719
712
|
mount_path = mount_path or Path("tmp_mnts") / random_str(12)
|
|
720
713
|
src_path = Path(src)
|
|
721
714
|
name = src_path.name
|
|
722
|
-
|
|
723
715
|
parent_path = str(src_path.parent.as_posix())
|
|
724
716
|
with self.scoped_mount(
|
|
725
717
|
parent_path,
|
|
@@ -847,7 +839,7 @@ class Rclone:
|
|
|
847
839
|
vfs_cache_mode: str | None = None,
|
|
848
840
|
verbose: bool | None = None,
|
|
849
841
|
other_args: list[str] | None = None,
|
|
850
|
-
) ->
|
|
842
|
+
) -> Mount:
|
|
851
843
|
"""Mount a remote or directory to a local path.
|
|
852
844
|
|
|
853
845
|
Args:
|
|
@@ -860,7 +852,6 @@ class Rclone:
|
|
|
860
852
|
Raises:
|
|
861
853
|
subprocess.CalledProcessError: If the mount operation fails
|
|
862
854
|
"""
|
|
863
|
-
from rclone_api.mount import clean_mount, prepare_mount
|
|
864
855
|
|
|
865
856
|
allow_writes = allow_writes or False
|
|
866
857
|
use_links = use_links or True
|
|
@@ -882,8 +873,9 @@ class Rclone:
|
|
|
882
873
|
if other_args:
|
|
883
874
|
cmd_list += other_args
|
|
884
875
|
proc = self._launch_process(cmd_list)
|
|
885
|
-
|
|
886
|
-
|
|
876
|
+
|
|
877
|
+
mount: Mount = Mount(mount_path=outdir, process=proc)
|
|
878
|
+
return mount
|
|
887
879
|
|
|
888
880
|
@contextmanager
|
|
889
881
|
def scoped_mount(
|
|
@@ -895,11 +887,11 @@ class Rclone:
|
|
|
895
887
|
vfs_cache_mode: str | None = None,
|
|
896
888
|
verbose: bool | None = None,
|
|
897
889
|
other_args: list[str] | None = None,
|
|
898
|
-
) -> Generator[
|
|
890
|
+
) -> Generator[Mount, None, None]:
|
|
899
891
|
"""Like mount, but can be used in a context manager."""
|
|
900
892
|
error_happened = False
|
|
901
893
|
verbose = get_verbose(verbose)
|
|
902
|
-
|
|
894
|
+
mount: Mount = self.mount(
|
|
903
895
|
src,
|
|
904
896
|
outdir,
|
|
905
897
|
allow_writes=allow_writes,
|
|
@@ -909,77 +901,15 @@ class Rclone:
|
|
|
909
901
|
other_args=other_args,
|
|
910
902
|
)
|
|
911
903
|
try:
|
|
912
|
-
yield
|
|
904
|
+
yield mount
|
|
913
905
|
except Exception as e:
|
|
914
906
|
error_happened = True
|
|
915
907
|
stack_trace = traceback.format_exc()
|
|
916
908
|
warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
|
|
917
909
|
raise
|
|
918
910
|
finally:
|
|
919
|
-
if proc.poll() is None:
|
|
920
|
-
proc.terminate()
|
|
921
|
-
proc.wait()
|
|
922
911
|
if not error_happened:
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
clean_mount(outdir, verbose=verbose)
|
|
926
|
-
|
|
927
|
-
@deprecated("mount")
|
|
928
|
-
def mount_webdav(
|
|
929
|
-
self,
|
|
930
|
-
url: str,
|
|
931
|
-
outdir: Path,
|
|
932
|
-
vfs_cache_mode: str | None = None,
|
|
933
|
-
vfs_disk_space_total_size: str | None = "10G",
|
|
934
|
-
other_args: list[str] | None = None,
|
|
935
|
-
) -> Process:
|
|
936
|
-
"""Mount a remote or directory to a local path.
|
|
937
|
-
|
|
938
|
-
Args:
|
|
939
|
-
src: Remote or directory to mount
|
|
940
|
-
outdir: Local path to mount to
|
|
941
|
-
|
|
942
|
-
Returns:
|
|
943
|
-
CompletedProcess from the mount command execution
|
|
944
|
-
|
|
945
|
-
Raises:
|
|
946
|
-
subprocess.CalledProcessError: If the mount operation fails
|
|
947
|
-
"""
|
|
948
|
-
other_args = other_args or []
|
|
949
|
-
if vfs_cache_mode is None:
|
|
950
|
-
if "--vfs-cache-mode" in other_args:
|
|
951
|
-
pass
|
|
952
|
-
else:
|
|
953
|
-
vfs_cache_mode = "full"
|
|
954
|
-
elif "--vfs-cache-mode" in other_args:
|
|
955
|
-
warnings.warn(
|
|
956
|
-
f"vfs_cache_mode is set to {vfs_cache_mode} but --vfs-cache-mode is already in other_args"
|
|
957
|
-
)
|
|
958
|
-
idx = other_args.index("--vfs-cache-mode")
|
|
959
|
-
other_args.pop(idx)
|
|
960
|
-
other_args.pop(idx) # also the next value which will be the cache mode.
|
|
961
|
-
|
|
962
|
-
if outdir.exists():
|
|
963
|
-
is_empty = not list(outdir.iterdir())
|
|
964
|
-
if not is_empty:
|
|
965
|
-
raise ValueError(
|
|
966
|
-
f"Mount directory already exists and is not empty: {outdir}"
|
|
967
|
-
)
|
|
968
|
-
outdir.rmdir()
|
|
969
|
-
|
|
970
|
-
src_str = url
|
|
971
|
-
cmd_list: list[str] = ["mount", src_str, str(outdir)]
|
|
972
|
-
if vfs_cache_mode:
|
|
973
|
-
cmd_list.append("--vfs-cache-mode")
|
|
974
|
-
cmd_list.append(vfs_cache_mode)
|
|
975
|
-
if other_args:
|
|
976
|
-
cmd_list += other_args
|
|
977
|
-
if vfs_disk_space_total_size is not None:
|
|
978
|
-
cmd_list.append("--vfs-cache-max-size")
|
|
979
|
-
cmd_list.append(vfs_disk_space_total_size)
|
|
980
|
-
proc = self._launch_process(cmd_list)
|
|
981
|
-
wait_for_mount(outdir, proc)
|
|
982
|
-
return proc
|
|
912
|
+
mount.close()
|
|
983
913
|
|
|
984
914
|
# Settings optimized for s3.
|
|
985
915
|
def mount_s3(
|
|
@@ -988,11 +918,8 @@ class Rclone:
|
|
|
988
918
|
outdir: Path,
|
|
989
919
|
allow_writes=False,
|
|
990
920
|
vfs_cache_mode="full",
|
|
991
|
-
# dir-cache-time
|
|
992
921
|
dir_cache_time: str | None = "1h",
|
|
993
922
|
attribute_timeout: str | None = "1h",
|
|
994
|
-
# --vfs-cache-max-size
|
|
995
|
-
# vfs-cache-max-size
|
|
996
923
|
vfs_disk_space_total_size: str | None = "100M",
|
|
997
924
|
transfers: int | None = 128,
|
|
998
925
|
modtime_strategy: (
|
|
@@ -1004,7 +931,7 @@ class Rclone:
|
|
|
1004
931
|
# vfs-refresh
|
|
1005
932
|
vfs_refresh: bool = True,
|
|
1006
933
|
other_args: list[str] | None = None,
|
|
1007
|
-
) ->
|
|
934
|
+
) -> Mount:
|
|
1008
935
|
"""Mount a remote or directory to a local path.
|
|
1009
936
|
|
|
1010
937
|
Args:
|
rclone_api/s3/chunk_types.py
CHANGED
|
@@ -7,6 +7,7 @@ from threading import Lock
|
|
|
7
7
|
|
|
8
8
|
from botocore.client import BaseClient
|
|
9
9
|
|
|
10
|
+
from rclone_api.types import SizeSuffix
|
|
10
11
|
from rclone_api.util import locked_print
|
|
11
12
|
|
|
12
13
|
_MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024 # 5MB
|
|
@@ -108,16 +109,20 @@ class UploadInfo:
|
|
|
108
109
|
value = getattr(self, f.name)
|
|
109
110
|
# Convert non-serializable objects (like s3_client) to a string representation.
|
|
110
111
|
if f.name == "s3_client":
|
|
111
|
-
|
|
112
|
+
continue
|
|
112
113
|
else:
|
|
113
114
|
if isinstance(value, Path):
|
|
114
115
|
value = str(value)
|
|
115
116
|
json_dict[f.name] = value
|
|
117
|
+
|
|
116
118
|
return json_dict
|
|
117
119
|
|
|
118
120
|
@staticmethod
|
|
119
121
|
def from_json(s3_client: BaseClient, json_dict: dict) -> "UploadInfo":
|
|
120
|
-
json_dict.pop("s3_client") # Remove the placeholder string
|
|
122
|
+
# json_dict.pop("s3_client") # Remove the placeholder string
|
|
123
|
+
if "s3_client" in json_dict:
|
|
124
|
+
json_dict.pop("s3_client")
|
|
125
|
+
|
|
121
126
|
return UploadInfo(s3_client=s3_client, **json_dict)
|
|
122
127
|
|
|
123
128
|
|
|
@@ -224,6 +229,12 @@ class UploadState:
|
|
|
224
229
|
|
|
225
230
|
# self.count()
|
|
226
231
|
finished_count, total = self.count()
|
|
232
|
+
total_finished: SizeSuffix = SizeSuffix(
|
|
233
|
+
finished_count * self.upload_info.chunk_size
|
|
234
|
+
)
|
|
235
|
+
total_remaining: SizeSuffix = SizeSuffix(
|
|
236
|
+
self.remaining() * self.upload_info.chunk_size
|
|
237
|
+
)
|
|
227
238
|
|
|
228
239
|
# parts.sort(key=lambda x: x.part_number) # Some backends need this.
|
|
229
240
|
out_json = {
|
|
@@ -232,6 +243,10 @@ class UploadState:
|
|
|
232
243
|
"is_done": is_done,
|
|
233
244
|
"finished_count": finished_count,
|
|
234
245
|
"total_parts": total,
|
|
246
|
+
"total_size": SizeSuffix(self.upload_info.file_size).as_str(),
|
|
247
|
+
"total_finished": total_finished.as_str(),
|
|
248
|
+
"total_remaining": total_remaining.as_str(),
|
|
249
|
+
"completed": f"{(finished_count / total) * 100:.2f}%",
|
|
235
250
|
}
|
|
236
251
|
|
|
237
252
|
# check that we can sererialize
|
rclone_api/types.py
CHANGED
|
@@ -52,31 +52,33 @@ def _to_size_suffix(size: int) -> str:
|
|
|
52
52
|
raise ValueError(f"Invalid size: {size}")
|
|
53
53
|
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
# Update regex to allow decimals (e.g., 16.5MB)
|
|
56
|
+
_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+(?:\.\d+)?)([A-Za-z]+)$")
|
|
56
57
|
|
|
57
58
|
|
|
58
59
|
def _from_size_suffix(size: str) -> int:
|
|
59
|
-
|
|
60
|
-
|
|
60
|
+
if size == "0":
|
|
61
|
+
return 0
|
|
61
62
|
match = _PATTERN_SIZE_SUFFIX.match(size)
|
|
62
63
|
if match is None:
|
|
63
64
|
raise ValueError(f"Invalid size suffix: {size}")
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
65
|
+
num_str, suffix = match.group(1), match.group(2)
|
|
66
|
+
n = float(num_str)
|
|
67
|
+
# Determine the unit from the first letter (e.g., "M" from "MB")
|
|
68
|
+
unit = suffix[0].upper()
|
|
69
|
+
if unit == "B":
|
|
70
|
+
return int(n)
|
|
71
|
+
if unit == "K":
|
|
72
|
+
return int(n * 1024)
|
|
73
|
+
if unit == "M":
|
|
74
|
+
return int(n * 1024 * 1024)
|
|
75
|
+
if unit == "G":
|
|
76
|
+
return int(n * 1024 * 1024 * 1024)
|
|
77
|
+
if unit == "T":
|
|
78
|
+
return int(n * 1024**4)
|
|
79
|
+
if unit == "P":
|
|
80
|
+
return int(n * 1024**5)
|
|
81
|
+
raise ValueError(f"Invalid size suffix: {suffix}")
|
|
80
82
|
|
|
81
83
|
|
|
82
84
|
class SizeSuffix:
|
|
@@ -88,6 +90,8 @@ class SizeSuffix:
|
|
|
88
90
|
self._size = size
|
|
89
91
|
elif isinstance(size, str):
|
|
90
92
|
self._size = _from_size_suffix(size)
|
|
93
|
+
elif isinstance(size, float):
|
|
94
|
+
self._size = int(size)
|
|
91
95
|
else:
|
|
92
96
|
raise ValueError(f"Invalid type for size: {type(size)}")
|
|
93
97
|
|
|
@@ -123,3 +127,10 @@ class SizeSuffix:
|
|
|
123
127
|
def __sub__(self, other: "int | SizeSuffix") -> "SizeSuffix":
|
|
124
128
|
other_int = SizeSuffix(other)
|
|
125
129
|
return SizeSuffix(self._size - other_int._size)
|
|
130
|
+
|
|
131
|
+
def __truediv__(self, other: "int | SizeSuffix") -> "SizeSuffix":
|
|
132
|
+
other_int = SizeSuffix(other)
|
|
133
|
+
if other_int._size == 0:
|
|
134
|
+
raise ZeroDivisionError("Division by zero is undefined")
|
|
135
|
+
# Use floor division to maintain integer arithmetic.
|
|
136
|
+
return SizeSuffix(self._size // other_int._size)
|
rclone_api/util.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import shutil
|
|
3
3
|
import subprocess
|
|
4
|
-
import time
|
|
5
4
|
import warnings
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from tempfile import TemporaryDirectory
|
|
@@ -135,26 +134,6 @@ def rclone_execute(
|
|
|
135
134
|
print(f"Error cleaning up tempdir: {e}")
|
|
136
135
|
|
|
137
136
|
|
|
138
|
-
def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
|
|
139
|
-
from rclone_api.process import Process
|
|
140
|
-
|
|
141
|
-
assert isinstance(mount_process, Process)
|
|
142
|
-
expire_time = time.time() + timeout
|
|
143
|
-
while time.time() < expire_time:
|
|
144
|
-
rtn = mount_process.poll()
|
|
145
|
-
if rtn is not None:
|
|
146
|
-
cmd_str = subprocess.list2cmdline(mount_process.cmd)
|
|
147
|
-
raise subprocess.CalledProcessError(rtn, cmd_str)
|
|
148
|
-
if path.exists():
|
|
149
|
-
# how many files?
|
|
150
|
-
dircontents = os.listdir(str(path))
|
|
151
|
-
if len(dircontents) > 0:
|
|
152
|
-
print(f"Mount point {path}, waiting 5 seconds for files to appear.")
|
|
153
|
-
time.sleep(5)
|
|
154
|
-
return
|
|
155
|
-
time.sleep(1)
|
|
156
|
-
|
|
157
|
-
|
|
158
137
|
def split_s3_path(path: str) -> S3PathInfo:
|
|
159
138
|
if ":" not in path:
|
|
160
139
|
raise ValueError(f"Invalid S3 path: {path}")
|
|
@@ -11,30 +11,30 @@ rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
|
|
|
11
11
|
rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
|
|
12
12
|
rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
|
|
13
13
|
rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
|
|
14
|
-
rclone_api/mount.py,sha256=
|
|
15
|
-
rclone_api/process.py,sha256=
|
|
16
|
-
rclone_api/rclone.py,sha256=
|
|
14
|
+
rclone_api/mount.py,sha256=Xj3BMVSDEwUbtMi8ycn5mUom-fAf-F9lJOjR_BzVllw,6073
|
|
15
|
+
rclone_api/process.py,sha256=xYUgU17txkZfZdr4vtRfvD8YjvSfdrbjM7PYW1npAMI,4264
|
|
16
|
+
rclone_api/rclone.py,sha256=BA3DJvCraLIrglDGSi1mF-wCH8rsqpBzc-5O-IWQpq8,39809
|
|
17
17
|
rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
|
|
18
18
|
rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
|
|
19
19
|
rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
|
|
20
|
-
rclone_api/types.py,sha256=
|
|
21
|
-
rclone_api/util.py,sha256=
|
|
20
|
+
rclone_api/types.py,sha256=zfTb0iM6mhfqgaYS6j6T0NIOA4e9GymNOXLPhVELe4A,3853
|
|
21
|
+
rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
|
|
22
22
|
rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
|
|
23
23
|
rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
|
|
24
|
-
rclone_api/cmd/copy_large_s3.py,sha256=
|
|
24
|
+
rclone_api/cmd/copy_large_s3.py,sha256=UR5qkW9vLtGLC951Om8UWFcQKApZUF-SzUhlIo3YIGU,3069
|
|
25
25
|
rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
|
|
26
|
-
rclone_api/experimental/flags.py,sha256=
|
|
26
|
+
rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
|
|
27
27
|
rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
|
|
28
28
|
rclone_api/s3/api.py,sha256=VstlaEnBjO2JDQuCRLdTfUGvQLbfshlXXhAzimFv4Vc,3763
|
|
29
29
|
rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
|
|
30
30
|
rclone_api/s3/chunk_file.py,sha256=XPoDl7DJMJIGBMRoPO2wqwqCMT7ZrIsEkDqlbMH8jzs,3506
|
|
31
|
-
rclone_api/s3/chunk_types.py,sha256=
|
|
31
|
+
rclone_api/s3/chunk_types.py,sha256=6_ythhT7EcchJ9MI8qBKVKh-SVOe1Rf15t_JepHPilA,8587
|
|
32
32
|
rclone_api/s3/chunk_uploader.py,sha256=KO8834Gery9HKWSqjQTNW0pbBbVoGrza9gj-1OaNLQQ,9130
|
|
33
33
|
rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
|
|
34
34
|
rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
|
|
35
|
-
rclone_api-1.1.
|
|
36
|
-
rclone_api-1.1.
|
|
37
|
-
rclone_api-1.1.
|
|
38
|
-
rclone_api-1.1.
|
|
39
|
-
rclone_api-1.1.
|
|
40
|
-
rclone_api-1.1.
|
|
35
|
+
rclone_api-1.1.10.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
|
36
|
+
rclone_api-1.1.10.dist-info/METADATA,sha256=3djgVtUHz682-1HmO5v6MNSCA7-2Iw2-qD6KXlVbUcc,4479
|
|
37
|
+
rclone_api-1.1.10.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
|
|
38
|
+
rclone_api-1.1.10.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
|
|
39
|
+
rclone_api-1.1.10.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
|
40
|
+
rclone_api-1.1.10.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|