rclone-api 1.1.6__tar.gz → 1.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rclone_api-1.1.6 → rclone_api-1.1.8}/PKG-INFO +1 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/pyproject.toml +1 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/cmd/copy_large_s3.py +15 -11
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/experimental/flags.py +8 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/mount.py +46 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/process.py +4 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/rclone.py +25 -100
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/chunk_types.py +17 -2
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/types.py +11 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/util.py +0 -21
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api.egg-info/PKG-INFO +1 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api.egg-info/SOURCES.txt +0 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_mount.py +3 -1
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_mount_s3.py +3 -1
- rclone_api-1.1.6/tests/test_mount_webdav.py +0 -117
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.aiderignore +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.github/workflows/lint.yml +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.github/workflows/push_macos.yml +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.github/workflows/push_ubuntu.yml +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.github/workflows/push_win.yml +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.gitignore +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.pylintrc +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.vscode/launch.json +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.vscode/settings.json +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/.vscode/tasks.json +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/LICENSE +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/MANIFEST.in +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/README.md +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/clean +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/install +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/lint +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/requirements.testing.txt +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/setup.cfg +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/setup.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/__init__.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/assets/example.txt +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/cli.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/cmd/list_files.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/completed_process.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/config.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/convert.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/deprecated.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/diff.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/dir.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/dir_listing.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/exec.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/experimental/flags_base.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/file.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/filelist.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/group_files.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/remote.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/rpath.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/api.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/basic_ops.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/chunk_file.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/chunk_uploader.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/create.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/s3/types.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/scan_missing_folders.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api/walk.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api.egg-info/dependency_links.txt +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api.egg-info/entry_points.txt +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api.egg-info/requires.txt +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/src/rclone_api.egg-info/top_level.txt +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/test +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/archive/test_paramiko.py.disabled +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_cmd_list_files.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_copy.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_copy_files.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_diff.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_group_files.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_is_synced.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_ls.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_mounted_ranged_download.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_obscure.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_rclone_config.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_remote_control.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_remotes.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_s3.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_scan_missing_folders.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_size_files.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_size_suffix.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tests/test_walk.py +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/tox.ini +0 -0
- {rclone_api-1.1.6 → rclone_api-1.1.8}/upload_package.sh +0 -0
|
@@ -12,8 +12,8 @@ class Args:
|
|
|
12
12
|
config_path: Path
|
|
13
13
|
src: str
|
|
14
14
|
dst: str
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
chunk_size: SizeSuffix
|
|
16
|
+
threads: int
|
|
17
17
|
retries: int
|
|
18
18
|
save_state_json: Path
|
|
19
19
|
verbose: bool
|
|
@@ -38,13 +38,13 @@ def _parse_args() -> Args:
|
|
|
38
38
|
"--chunk-size",
|
|
39
39
|
help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
|
|
40
40
|
type=str,
|
|
41
|
-
default="
|
|
41
|
+
default="512M",
|
|
42
42
|
)
|
|
43
43
|
parser.add_argument(
|
|
44
|
-
"--
|
|
45
|
-
help="
|
|
44
|
+
"--threads",
|
|
45
|
+
help="Number of threads to use per chunk",
|
|
46
46
|
type=int,
|
|
47
|
-
default=
|
|
47
|
+
default=16,
|
|
48
48
|
)
|
|
49
49
|
parser.add_argument("--retries", help="Number of retries", type=int, default=3)
|
|
50
50
|
parser.add_argument(
|
|
@@ -59,10 +59,10 @@ def _parse_args() -> Args:
|
|
|
59
59
|
config_path=Path(args.config),
|
|
60
60
|
src=args.src,
|
|
61
61
|
dst=args.dst,
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
chunk_size=SizeSuffix(args.chunk_size),
|
|
63
|
+
threads=args.threads,
|
|
64
64
|
retries=args.retries,
|
|
65
|
-
save_state_json=args.
|
|
65
|
+
save_state_json=args.resume_json,
|
|
66
66
|
verbose=args.verbose,
|
|
67
67
|
)
|
|
68
68
|
return out
|
|
@@ -72,11 +72,15 @@ def main() -> int:
|
|
|
72
72
|
"""Main entry point."""
|
|
73
73
|
args = _parse_args()
|
|
74
74
|
rclone = Rclone(rclone_conf=args.config_path)
|
|
75
|
+
# unit_chunk = args.chunk_size / args.threads
|
|
75
76
|
rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
|
|
76
77
|
src=args.src,
|
|
77
78
|
dst=args.dst,
|
|
78
|
-
chunk_size=args.
|
|
79
|
-
|
|
79
|
+
chunk_size=args.chunk_size,
|
|
80
|
+
threads=args.threads,
|
|
81
|
+
# vfs_read_chunk_size=unit_chunk,
|
|
82
|
+
# vfs_read_chunk_size_limit=args.chunk_size,
|
|
83
|
+
# vfs_read_chunk_streams=args.threads,
|
|
80
84
|
retries=args.retries,
|
|
81
85
|
save_state_json=args.save_state_json,
|
|
82
86
|
verbose=args.verbose,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
|
|
3
3
|
from rclone_api.experimental.flags_base import BaseFlags, merge_flags
|
|
4
|
+
from rclone_api.types import SizeSuffix
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
@dataclass
|
|
@@ -20,7 +21,7 @@ class CopyFlags(BaseFlags):
|
|
|
20
21
|
links: bool | None = None
|
|
21
22
|
max_backlog: int | None = None
|
|
22
23
|
max_duration: str | None = None
|
|
23
|
-
max_transfer:
|
|
24
|
+
max_transfer: SizeSuffix | None = None
|
|
24
25
|
metadata: bool | None = None
|
|
25
26
|
modify_window: str | None = None
|
|
26
27
|
multi_thread_chunk_size: str | None = None
|
|
@@ -77,6 +78,12 @@ def unit_test() -> None:
|
|
|
77
78
|
merged_d_c = copy_flags_d.merge(copy_flags_c)
|
|
78
79
|
print("C:", merged_d_c)
|
|
79
80
|
|
|
81
|
+
# now do the one with the SizeSuffix type
|
|
82
|
+
copy_flags_e = CopyFlags(max_transfer=SizeSuffix("128M"))
|
|
83
|
+
copy_flags_f = CopyFlags(max_transfer=SizeSuffix("256M"))
|
|
84
|
+
merged_e_f = copy_flags_e.merge(copy_flags_f)
|
|
85
|
+
print("D:", merged_e_f)
|
|
86
|
+
|
|
80
87
|
|
|
81
88
|
if __name__ == "__main__":
|
|
82
89
|
unit_test()
|
|
@@ -1,12 +1,30 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import platform
|
|
2
3
|
import subprocess
|
|
3
4
|
import time
|
|
4
5
|
import warnings
|
|
6
|
+
from dataclasses import dataclass
|
|
5
7
|
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from rclone_api.process import Process
|
|
6
11
|
|
|
7
12
|
_SYSTEM = platform.system() # "Linux", "Darwin", "Windows", etc.
|
|
8
13
|
|
|
9
14
|
|
|
15
|
+
@dataclass
|
|
16
|
+
class Mount:
|
|
17
|
+
"""Mount information."""
|
|
18
|
+
|
|
19
|
+
mount_path: Path
|
|
20
|
+
process: Process
|
|
21
|
+
|
|
22
|
+
def __post_init__(self):
|
|
23
|
+
assert isinstance(self.mount_path, Path)
|
|
24
|
+
assert self.process is not None
|
|
25
|
+
wait_for_mount(self.mount_path, self.process)
|
|
26
|
+
|
|
27
|
+
|
|
10
28
|
def run_command(cmd: str, verbose: bool) -> int:
|
|
11
29
|
"""Run a shell command and print its output if verbose is True."""
|
|
12
30
|
if verbose:
|
|
@@ -36,7 +54,27 @@ def prepare_mount(outdir: Path, verbose: bool) -> None:
|
|
|
36
54
|
outdir.mkdir(parents=True, exist_ok=True)
|
|
37
55
|
|
|
38
56
|
|
|
39
|
-
def
|
|
57
|
+
def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
|
|
58
|
+
from rclone_api.process import Process
|
|
59
|
+
|
|
60
|
+
assert isinstance(mount_process, Process)
|
|
61
|
+
expire_time = time.time() + timeout
|
|
62
|
+
while time.time() < expire_time:
|
|
63
|
+
rtn = mount_process.poll()
|
|
64
|
+
if rtn is not None:
|
|
65
|
+
cmd_str = subprocess.list2cmdline(mount_process.cmd)
|
|
66
|
+
raise subprocess.CalledProcessError(rtn, cmd_str)
|
|
67
|
+
if path.exists():
|
|
68
|
+
# how many files?
|
|
69
|
+
dircontents = os.listdir(str(path))
|
|
70
|
+
if len(dircontents) > 0:
|
|
71
|
+
print(f"Mount point {path}, waiting 5 seconds for files to appear.")
|
|
72
|
+
time.sleep(5)
|
|
73
|
+
return
|
|
74
|
+
time.sleep(1)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
|
|
40
78
|
"""
|
|
41
79
|
Clean up a mount path across Linux, macOS, and Windows.
|
|
42
80
|
|
|
@@ -45,7 +83,14 @@ def clean_mount(mount_path: Path, verbose: bool = False) -> None:
|
|
|
45
83
|
and 'umount'. On macOS it uses 'umount' (and optionally 'diskutil unmount'),
|
|
46
84
|
while on Windows it attempts to remove the mount point via 'mountvol /D'.
|
|
47
85
|
"""
|
|
86
|
+
proc = mount.process if isinstance(mount, Mount) else None
|
|
87
|
+
if proc is not None and proc.poll() is None:
|
|
88
|
+
if verbose:
|
|
89
|
+
print(f"Terminating mount process {proc.pid}")
|
|
90
|
+
proc.kill()
|
|
91
|
+
|
|
48
92
|
# Check if the mount path exists; if an OSError occurs, assume it exists.
|
|
93
|
+
mount_path = mount.mount_path if isinstance(mount, Mount) else mount
|
|
49
94
|
try:
|
|
50
95
|
mount_exists = mount_path.exists()
|
|
51
96
|
except OSError as e:
|
|
@@ -25,6 +25,7 @@ from rclone_api.dir_listing import DirListing
|
|
|
25
25
|
from rclone_api.exec import RcloneExec
|
|
26
26
|
from rclone_api.file import File
|
|
27
27
|
from rclone_api.group_files import group_files
|
|
28
|
+
from rclone_api.mount import Mount, clean_mount, prepare_mount
|
|
28
29
|
from rclone_api.process import Process
|
|
29
30
|
from rclone_api.remote import Remote
|
|
30
31
|
from rclone_api.rpath import RPath
|
|
@@ -46,12 +47,9 @@ from rclone_api.util import (
|
|
|
46
47
|
get_rclone_exe,
|
|
47
48
|
get_verbose,
|
|
48
49
|
to_path,
|
|
49
|
-
wait_for_mount,
|
|
50
50
|
)
|
|
51
51
|
from rclone_api.walk import walk
|
|
52
52
|
|
|
53
|
-
_IS_WINDOWS = os.name == "nt"
|
|
54
|
-
|
|
55
53
|
|
|
56
54
|
def rclone_verbose(verbose: bool | None) -> bool:
|
|
57
55
|
if verbose is not None:
|
|
@@ -680,12 +678,7 @@ class Rclone:
|
|
|
680
678
|
dst: str,
|
|
681
679
|
save_state_json: Path,
|
|
682
680
|
chunk_size: SizeSuffix | None = None,
|
|
683
|
-
|
|
684
|
-
# * 1024
|
|
685
|
-
# * 1024, # This setting will scale the performance of the upload
|
|
686
|
-
concurrent_chunks: (
|
|
687
|
-
int | None
|
|
688
|
-
) = None, # This setting will scale the performance of the upload
|
|
681
|
+
threads: int = 16,
|
|
689
682
|
retries: int = 3,
|
|
690
683
|
verbose: bool | None = None,
|
|
691
684
|
max_chunks_before_suspension: int | None = None,
|
|
@@ -696,30 +689,24 @@ class Rclone:
|
|
|
696
689
|
from rclone_api.s3.create import S3Credentials
|
|
697
690
|
from rclone_api.util import S3PathInfo, random_str, split_s3_path
|
|
698
691
|
|
|
699
|
-
|
|
700
|
-
chunk_size = SizeSuffix(
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
size_limit.as_str(), # purge quickly.
|
|
711
|
-
"--vfs-read-chunk-size",
|
|
712
|
-
chunk_size.as_str(),
|
|
692
|
+
other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
|
|
693
|
+
chunk_size = chunk_size or SizeSuffix("128M")
|
|
694
|
+
unit_chunk_size = chunk_size / threads
|
|
695
|
+
vfs_read_chunk_size = unit_chunk_size
|
|
696
|
+
vfs_read_chunk_size_limit = chunk_size
|
|
697
|
+
vfs_read_chunk_streams = threads
|
|
698
|
+
assert (
|
|
699
|
+
chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
|
|
700
|
+
), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
|
|
701
|
+
other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
|
|
702
|
+
other_args += [
|
|
713
703
|
"--vfs-read-chunk-size-limit",
|
|
714
|
-
|
|
715
|
-
"--vfs-read-chunk-streams",
|
|
716
|
-
str(concurrent_chunks),
|
|
717
|
-
"--vfs-fast-fingerprint",
|
|
704
|
+
vfs_read_chunk_size_limit.as_str(),
|
|
718
705
|
]
|
|
706
|
+
other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
|
|
719
707
|
mount_path = mount_path or Path("tmp_mnts") / random_str(12)
|
|
720
708
|
src_path = Path(src)
|
|
721
709
|
name = src_path.name
|
|
722
|
-
|
|
723
710
|
parent_path = str(src_path.parent.as_posix())
|
|
724
711
|
with self.scoped_mount(
|
|
725
712
|
parent_path,
|
|
@@ -847,7 +834,7 @@ class Rclone:
|
|
|
847
834
|
vfs_cache_mode: str | None = None,
|
|
848
835
|
verbose: bool | None = None,
|
|
849
836
|
other_args: list[str] | None = None,
|
|
850
|
-
) ->
|
|
837
|
+
) -> Mount:
|
|
851
838
|
"""Mount a remote or directory to a local path.
|
|
852
839
|
|
|
853
840
|
Args:
|
|
@@ -860,7 +847,6 @@ class Rclone:
|
|
|
860
847
|
Raises:
|
|
861
848
|
subprocess.CalledProcessError: If the mount operation fails
|
|
862
849
|
"""
|
|
863
|
-
from rclone_api.mount import clean_mount, prepare_mount
|
|
864
850
|
|
|
865
851
|
allow_writes = allow_writes or False
|
|
866
852
|
use_links = use_links or True
|
|
@@ -882,8 +868,9 @@ class Rclone:
|
|
|
882
868
|
if other_args:
|
|
883
869
|
cmd_list += other_args
|
|
884
870
|
proc = self._launch_process(cmd_list)
|
|
885
|
-
|
|
886
|
-
|
|
871
|
+
|
|
872
|
+
mount: Mount = Mount(mount_path=outdir, process=proc)
|
|
873
|
+
return mount
|
|
887
874
|
|
|
888
875
|
@contextmanager
|
|
889
876
|
def scoped_mount(
|
|
@@ -895,11 +882,11 @@ class Rclone:
|
|
|
895
882
|
vfs_cache_mode: str | None = None,
|
|
896
883
|
verbose: bool | None = None,
|
|
897
884
|
other_args: list[str] | None = None,
|
|
898
|
-
) -> Generator[
|
|
885
|
+
) -> Generator[Mount, None, None]:
|
|
899
886
|
"""Like mount, but can be used in a context manager."""
|
|
900
887
|
error_happened = False
|
|
901
888
|
verbose = get_verbose(verbose)
|
|
902
|
-
|
|
889
|
+
mount: Mount = self.mount(
|
|
903
890
|
src,
|
|
904
891
|
outdir,
|
|
905
892
|
allow_writes=allow_writes,
|
|
@@ -909,77 +896,18 @@ class Rclone:
|
|
|
909
896
|
other_args=other_args,
|
|
910
897
|
)
|
|
911
898
|
try:
|
|
912
|
-
yield
|
|
899
|
+
yield mount
|
|
913
900
|
except Exception as e:
|
|
914
901
|
error_happened = True
|
|
915
902
|
stack_trace = traceback.format_exc()
|
|
916
903
|
warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
|
|
917
904
|
raise
|
|
918
905
|
finally:
|
|
919
|
-
|
|
920
|
-
proc.terminate()
|
|
921
|
-
proc.wait()
|
|
906
|
+
|
|
922
907
|
if not error_happened:
|
|
923
908
|
from rclone_api.mount import clean_mount
|
|
924
909
|
|
|
925
|
-
clean_mount(
|
|
926
|
-
|
|
927
|
-
@deprecated("mount")
|
|
928
|
-
def mount_webdav(
|
|
929
|
-
self,
|
|
930
|
-
url: str,
|
|
931
|
-
outdir: Path,
|
|
932
|
-
vfs_cache_mode: str | None = None,
|
|
933
|
-
vfs_disk_space_total_size: str | None = "10G",
|
|
934
|
-
other_args: list[str] | None = None,
|
|
935
|
-
) -> Process:
|
|
936
|
-
"""Mount a remote or directory to a local path.
|
|
937
|
-
|
|
938
|
-
Args:
|
|
939
|
-
src: Remote or directory to mount
|
|
940
|
-
outdir: Local path to mount to
|
|
941
|
-
|
|
942
|
-
Returns:
|
|
943
|
-
CompletedProcess from the mount command execution
|
|
944
|
-
|
|
945
|
-
Raises:
|
|
946
|
-
subprocess.CalledProcessError: If the mount operation fails
|
|
947
|
-
"""
|
|
948
|
-
other_args = other_args or []
|
|
949
|
-
if vfs_cache_mode is None:
|
|
950
|
-
if "--vfs-cache-mode" in other_args:
|
|
951
|
-
pass
|
|
952
|
-
else:
|
|
953
|
-
vfs_cache_mode = "full"
|
|
954
|
-
elif "--vfs-cache-mode" in other_args:
|
|
955
|
-
warnings.warn(
|
|
956
|
-
f"vfs_cache_mode is set to {vfs_cache_mode} but --vfs-cache-mode is already in other_args"
|
|
957
|
-
)
|
|
958
|
-
idx = other_args.index("--vfs-cache-mode")
|
|
959
|
-
other_args.pop(idx)
|
|
960
|
-
other_args.pop(idx) # also the next value which will be the cache mode.
|
|
961
|
-
|
|
962
|
-
if outdir.exists():
|
|
963
|
-
is_empty = not list(outdir.iterdir())
|
|
964
|
-
if not is_empty:
|
|
965
|
-
raise ValueError(
|
|
966
|
-
f"Mount directory already exists and is not empty: {outdir}"
|
|
967
|
-
)
|
|
968
|
-
outdir.rmdir()
|
|
969
|
-
|
|
970
|
-
src_str = url
|
|
971
|
-
cmd_list: list[str] = ["mount", src_str, str(outdir)]
|
|
972
|
-
if vfs_cache_mode:
|
|
973
|
-
cmd_list.append("--vfs-cache-mode")
|
|
974
|
-
cmd_list.append(vfs_cache_mode)
|
|
975
|
-
if other_args:
|
|
976
|
-
cmd_list += other_args
|
|
977
|
-
if vfs_disk_space_total_size is not None:
|
|
978
|
-
cmd_list.append("--vfs-cache-max-size")
|
|
979
|
-
cmd_list.append(vfs_disk_space_total_size)
|
|
980
|
-
proc = self._launch_process(cmd_list)
|
|
981
|
-
wait_for_mount(outdir, proc)
|
|
982
|
-
return proc
|
|
910
|
+
clean_mount(mount, verbose=verbose)
|
|
983
911
|
|
|
984
912
|
# Settings optimized for s3.
|
|
985
913
|
def mount_s3(
|
|
@@ -988,11 +916,8 @@ class Rclone:
|
|
|
988
916
|
outdir: Path,
|
|
989
917
|
allow_writes=False,
|
|
990
918
|
vfs_cache_mode="full",
|
|
991
|
-
# dir-cache-time
|
|
992
919
|
dir_cache_time: str | None = "1h",
|
|
993
920
|
attribute_timeout: str | None = "1h",
|
|
994
|
-
# --vfs-cache-max-size
|
|
995
|
-
# vfs-cache-max-size
|
|
996
921
|
vfs_disk_space_total_size: str | None = "100M",
|
|
997
922
|
transfers: int | None = 128,
|
|
998
923
|
modtime_strategy: (
|
|
@@ -1004,7 +929,7 @@ class Rclone:
|
|
|
1004
929
|
# vfs-refresh
|
|
1005
930
|
vfs_refresh: bool = True,
|
|
1006
931
|
other_args: list[str] | None = None,
|
|
1007
|
-
) ->
|
|
932
|
+
) -> Mount:
|
|
1008
933
|
"""Mount a remote or directory to a local path.
|
|
1009
934
|
|
|
1010
935
|
Args:
|
|
@@ -7,6 +7,7 @@ from threading import Lock
|
|
|
7
7
|
|
|
8
8
|
from botocore.client import BaseClient
|
|
9
9
|
|
|
10
|
+
from rclone_api.types import SizeSuffix
|
|
10
11
|
from rclone_api.util import locked_print
|
|
11
12
|
|
|
12
13
|
_MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024 # 5MB
|
|
@@ -108,16 +109,20 @@ class UploadInfo:
|
|
|
108
109
|
value = getattr(self, f.name)
|
|
109
110
|
# Convert non-serializable objects (like s3_client) to a string representation.
|
|
110
111
|
if f.name == "s3_client":
|
|
111
|
-
|
|
112
|
+
continue
|
|
112
113
|
else:
|
|
113
114
|
if isinstance(value, Path):
|
|
114
115
|
value = str(value)
|
|
115
116
|
json_dict[f.name] = value
|
|
117
|
+
|
|
116
118
|
return json_dict
|
|
117
119
|
|
|
118
120
|
@staticmethod
|
|
119
121
|
def from_json(s3_client: BaseClient, json_dict: dict) -> "UploadInfo":
|
|
120
|
-
json_dict.pop("s3_client") # Remove the placeholder string
|
|
122
|
+
# json_dict.pop("s3_client") # Remove the placeholder string
|
|
123
|
+
if "s3_client" in json_dict:
|
|
124
|
+
json_dict.pop("s3_client")
|
|
125
|
+
|
|
121
126
|
return UploadInfo(s3_client=s3_client, **json_dict)
|
|
122
127
|
|
|
123
128
|
|
|
@@ -224,6 +229,12 @@ class UploadState:
|
|
|
224
229
|
|
|
225
230
|
# self.count()
|
|
226
231
|
finished_count, total = self.count()
|
|
232
|
+
total_finished: SizeSuffix = SizeSuffix(
|
|
233
|
+
finished_count * self.upload_info.chunk_size
|
|
234
|
+
)
|
|
235
|
+
total_remaining: SizeSuffix = SizeSuffix(
|
|
236
|
+
self.remaining() * self.upload_info.chunk_size
|
|
237
|
+
)
|
|
227
238
|
|
|
228
239
|
# parts.sort(key=lambda x: x.part_number) # Some backends need this.
|
|
229
240
|
out_json = {
|
|
@@ -232,6 +243,10 @@ class UploadState:
|
|
|
232
243
|
"is_done": is_done,
|
|
233
244
|
"finished_count": finished_count,
|
|
234
245
|
"total_parts": total,
|
|
246
|
+
"total_size": SizeSuffix(self.upload_info.file_size).as_str(),
|
|
247
|
+
"total_finished": total_finished.as_str(),
|
|
248
|
+
"total_remaining": total_remaining.as_str(),
|
|
249
|
+
"completed": f"{(finished_count / total) * 100:.2f}%",
|
|
235
250
|
}
|
|
236
251
|
|
|
237
252
|
# check that we can sererialize
|
|
@@ -58,6 +58,8 @@ _PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+)([A-Za-z]+)$")
|
|
|
58
58
|
def _from_size_suffix(size: str) -> int:
|
|
59
59
|
# 16MiB
|
|
60
60
|
# parse out number and suffix
|
|
61
|
+
if size == "0":
|
|
62
|
+
return 0
|
|
61
63
|
match = _PATTERN_SIZE_SUFFIX.match(size)
|
|
62
64
|
if match is None:
|
|
63
65
|
raise ValueError(f"Invalid size suffix: {size}")
|
|
@@ -88,6 +90,8 @@ class SizeSuffix:
|
|
|
88
90
|
self._size = size
|
|
89
91
|
elif isinstance(size, str):
|
|
90
92
|
self._size = _from_size_suffix(size)
|
|
93
|
+
elif isinstance(size, float):
|
|
94
|
+
self._size = int(size)
|
|
91
95
|
else:
|
|
92
96
|
raise ValueError(f"Invalid type for size: {type(size)}")
|
|
93
97
|
|
|
@@ -123,3 +127,10 @@ class SizeSuffix:
|
|
|
123
127
|
def __sub__(self, other: "int | SizeSuffix") -> "SizeSuffix":
|
|
124
128
|
other_int = SizeSuffix(other)
|
|
125
129
|
return SizeSuffix(self._size - other_int._size)
|
|
130
|
+
|
|
131
|
+
def __truediv__(self, other: "int | SizeSuffix") -> "SizeSuffix":
|
|
132
|
+
other_int = SizeSuffix(other)
|
|
133
|
+
if other_int._size == 0:
|
|
134
|
+
raise ZeroDivisionError("Division by zero is undefined")
|
|
135
|
+
# Use floor division to maintain integer arithmetic.
|
|
136
|
+
return SizeSuffix(self._size // other_int._size)
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import shutil
|
|
3
3
|
import subprocess
|
|
4
|
-
import time
|
|
5
4
|
import warnings
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from tempfile import TemporaryDirectory
|
|
@@ -135,26 +134,6 @@ def rclone_execute(
|
|
|
135
134
|
print(f"Error cleaning up tempdir: {e}")
|
|
136
135
|
|
|
137
136
|
|
|
138
|
-
def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
|
|
139
|
-
from rclone_api.process import Process
|
|
140
|
-
|
|
141
|
-
assert isinstance(mount_process, Process)
|
|
142
|
-
expire_time = time.time() + timeout
|
|
143
|
-
while time.time() < expire_time:
|
|
144
|
-
rtn = mount_process.poll()
|
|
145
|
-
if rtn is not None:
|
|
146
|
-
cmd_str = subprocess.list2cmdline(mount_process.cmd)
|
|
147
|
-
raise subprocess.CalledProcessError(rtn, cmd_str)
|
|
148
|
-
if path.exists():
|
|
149
|
-
# how many files?
|
|
150
|
-
dircontents = os.listdir(str(path))
|
|
151
|
-
if len(dircontents) > 0:
|
|
152
|
-
print(f"Mount point {path}, waiting 5 seconds for files to appear.")
|
|
153
|
-
time.sleep(5)
|
|
154
|
-
return
|
|
155
|
-
time.sleep(1)
|
|
156
|
-
|
|
157
|
-
|
|
158
137
|
def split_s3_path(path: str) -> S3PathInfo:
|
|
159
138
|
if ":" not in path:
|
|
160
139
|
raise ValueError(f"Invalid S3 path: {path}")
|
|
@@ -68,7 +68,9 @@ class RcloneMountTests(unittest.TestCase):
|
|
|
68
68
|
|
|
69
69
|
try:
|
|
70
70
|
# Start the mount process
|
|
71
|
-
|
|
71
|
+
mount = self.rclone.mount(remote_path, self.mount_point)
|
|
72
|
+
process = mount.process
|
|
73
|
+
assert process is not None
|
|
72
74
|
self.assertIsNone(
|
|
73
75
|
process.poll(), "Mount process should still be running after 2 seconds"
|
|
74
76
|
)
|
|
@@ -70,7 +70,9 @@ class RcloneMountS3Tests(unittest.TestCase):
|
|
|
70
70
|
|
|
71
71
|
try:
|
|
72
72
|
# Start the mount process
|
|
73
|
-
|
|
73
|
+
mount = self.rclone.mount_s3(remote_path, self.mount_point)
|
|
74
|
+
process = mount.process
|
|
75
|
+
assert process
|
|
74
76
|
self.assertIsNone(
|
|
75
77
|
process.poll(), "Mount process should still be running after 2 seconds"
|
|
76
78
|
)
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Unit test file.
|
|
3
|
-
"""
|
|
4
|
-
|
|
5
|
-
import os
|
|
6
|
-
import unittest
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
from dotenv import load_dotenv
|
|
10
|
-
|
|
11
|
-
from rclone_api import Config, Process, Rclone, Remote
|
|
12
|
-
|
|
13
|
-
_ENABLED = False
|
|
14
|
-
|
|
15
|
-
load_dotenv()
|
|
16
|
-
|
|
17
|
-
BUCKET_NAME = os.getenv("BUCKET_NAME") # Default if not in .env
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def _generate_rclone_config(port: int) -> Config:
|
|
21
|
-
|
|
22
|
-
# BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks") # Default if not in .env
|
|
23
|
-
|
|
24
|
-
# Load additional environment variables
|
|
25
|
-
BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
|
|
26
|
-
BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
|
|
27
|
-
# BUCKET_URL = os.getenv("BUCKET_URL")
|
|
28
|
-
BUCKET_URL = "sfo3.digitaloceanspaces.com"
|
|
29
|
-
|
|
30
|
-
config_text = f"""
|
|
31
|
-
[dst]
|
|
32
|
-
type = s3
|
|
33
|
-
provider = DigitalOcean
|
|
34
|
-
access_key_id = {BUCKET_KEY_PUBLIC}
|
|
35
|
-
secret_access_key = {BUCKET_KEY_SECRET}
|
|
36
|
-
endpoint = {BUCKET_URL}
|
|
37
|
-
bucket = {BUCKET_NAME}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
[webdav]
|
|
41
|
-
type = webdav
|
|
42
|
-
user = guest
|
|
43
|
-
# obscured password for "1234", use Rclone.obscure("1234") to generate
|
|
44
|
-
pass = d4IbQLV9W0JhI2tm5Zp88hpMtEg
|
|
45
|
-
url = http://localhost:{port}
|
|
46
|
-
vendor = rclone
|
|
47
|
-
"""
|
|
48
|
-
|
|
49
|
-
out = Config(config_text)
|
|
50
|
-
return out
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class RcloneMountWebdavTester(unittest.TestCase):
|
|
54
|
-
"""Test rclone functionality."""
|
|
55
|
-
|
|
56
|
-
def setUp(self) -> None:
|
|
57
|
-
"""Check if all required environment variables are set before running tests."""
|
|
58
|
-
required_vars = [
|
|
59
|
-
"BUCKET_NAME",
|
|
60
|
-
"BUCKET_KEY_SECRET",
|
|
61
|
-
"BUCKET_KEY_PUBLIC",
|
|
62
|
-
"BUCKET_URL",
|
|
63
|
-
]
|
|
64
|
-
missing = [var for var in required_vars if not os.getenv(var)]
|
|
65
|
-
if missing:
|
|
66
|
-
self.skipTest(
|
|
67
|
-
f"Missing required environment variables: {', '.join(missing)}"
|
|
68
|
-
)
|
|
69
|
-
os.environ["RCLONE_API_VERBOSE"] = "1"
|
|
70
|
-
|
|
71
|
-
@unittest.skipIf(not _ENABLED, "Test not enabled")
|
|
72
|
-
def test_serve_webdav_and_mount(self) -> None:
|
|
73
|
-
"""Test basic NFS serve functionality."""
|
|
74
|
-
port = 8090
|
|
75
|
-
config = _generate_rclone_config(port)
|
|
76
|
-
rclone = Rclone(config)
|
|
77
|
-
|
|
78
|
-
# Start NFS server for the remote
|
|
79
|
-
remote = Remote("dst", rclone=rclone)
|
|
80
|
-
# serve = Remote("webdav", rclone=rclone)
|
|
81
|
-
test_addr = f"localhost:{port}"
|
|
82
|
-
user = "guest"
|
|
83
|
-
password = "1234"
|
|
84
|
-
|
|
85
|
-
process = rclone.serve_webdav(
|
|
86
|
-
f"{remote.name}:{BUCKET_NAME}", addr=test_addr, user=user, password=password
|
|
87
|
-
)
|
|
88
|
-
mount_proc: Process | None = None
|
|
89
|
-
|
|
90
|
-
try:
|
|
91
|
-
# Verify process is running
|
|
92
|
-
self.assertIsNone(process.poll())
|
|
93
|
-
mount_point = Path("test_mount2")
|
|
94
|
-
mount_proc = rclone.mount_webdav("webdav:", mount_point)
|
|
95
|
-
# test that the mount point exists
|
|
96
|
-
self.assertTrue(mount_point.exists())
|
|
97
|
-
# test the folder is not empty
|
|
98
|
-
next_path = next(mount_point.iterdir())
|
|
99
|
-
self.assertIsNotNone(next_path)
|
|
100
|
-
|
|
101
|
-
finally:
|
|
102
|
-
# Clean up
|
|
103
|
-
if mount_proc:
|
|
104
|
-
mount_proc.terminate()
|
|
105
|
-
mount_proc.wait()
|
|
106
|
-
process.terminate()
|
|
107
|
-
process.wait()
|
|
108
|
-
if mount_proc:
|
|
109
|
-
mount_proc.terminate()
|
|
110
|
-
mount_proc.wait()
|
|
111
|
-
|
|
112
|
-
# Verify process terminated
|
|
113
|
-
self.assertIsNotNone(process.poll())
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if __name__ == "__main__":
|
|
117
|
-
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|