rclone-api 1.0.12__py2.py3-none-any.whl → 1.0.14__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- rclone_api/__init__.py +1 -1
- rclone_api/config.py +3 -14
- rclone_api/convert.py +31 -0
- rclone_api/dir.py +23 -1
- rclone_api/dir_listing.py +28 -2
- rclone_api/{types.py → exec.py} +1 -6
- rclone_api/file.py +14 -2
- rclone_api/rclone.py +135 -11
- rclone_api/rpath.py +27 -9
- rclone_api/util.py +68 -4
- rclone_api/walk.py +49 -66
- rclone_api-1.0.14.dist-info/METADATA +34 -0
- rclone_api-1.0.14.dist-info/RECORD +19 -0
- rclone_api-1.0.12.dist-info/METADATA +0 -36
- rclone_api-1.0.12.dist-info/RECORD +0 -18
- {rclone_api-1.0.12.dist-info → rclone_api-1.0.14.dist-info}/LICENSE +0 -0
- {rclone_api-1.0.12.dist-info → rclone_api-1.0.14.dist-info}/WHEEL +0 -0
- {rclone_api-1.0.12.dist-info → rclone_api-1.0.14.dist-info}/top_level.txt +0 -0
rclone_api/__init__.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
+
from .config import Config
|
1
2
|
from .dir import Dir
|
2
3
|
from .dir_listing import DirListing
|
3
4
|
from .file import File
|
4
5
|
from .rclone import Rclone
|
5
6
|
from .remote import Remote
|
6
7
|
from .rpath import RPath
|
7
|
-
from .types import Config
|
8
8
|
|
9
9
|
__all__ = ["Rclone", "File", "Config", "Remote", "Dir", "RPath", "DirListing"]
|
rclone_api/config.py
CHANGED
@@ -1,19 +1,8 @@
|
|
1
|
-
import subprocess
|
2
1
|
from dataclasses import dataclass
|
3
|
-
from pathlib import Path
|
4
|
-
|
5
|
-
from rclone_api.types import Config
|
6
2
|
|
7
3
|
|
8
4
|
@dataclass
|
9
|
-
class
|
10
|
-
"""Rclone
|
11
|
-
|
12
|
-
rclone_config: Path | Config
|
13
|
-
rclone_exe: Path
|
14
|
-
|
15
|
-
def execute(self, cmd: list[str]) -> subprocess.CompletedProcess:
|
16
|
-
"""Execute rclone command."""
|
17
|
-
from rclone_api.util import rclone_execute
|
5
|
+
class Config:
|
6
|
+
"""Rclone configuration dataclass."""
|
18
7
|
|
19
|
-
|
8
|
+
text: str
|
rclone_api/convert.py
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
from rclone_api.dir import Dir
|
2
|
+
from rclone_api.file import File
|
3
|
+
from rclone_api.remote import Remote
|
4
|
+
|
5
|
+
|
6
|
+
def convert_to_filestr_list(files: str | File | list[str] | list[File]) -> list[str]:
|
7
|
+
out: list[str] = []
|
8
|
+
if isinstance(files, str):
|
9
|
+
out.append(files)
|
10
|
+
elif isinstance(files, File):
|
11
|
+
out.append(str(files.path))
|
12
|
+
elif isinstance(files, list):
|
13
|
+
for f in files:
|
14
|
+
if isinstance(f, File):
|
15
|
+
f = str(f.path)
|
16
|
+
out.append(f)
|
17
|
+
else:
|
18
|
+
raise ValueError(f"Invalid type for file: {type(files)}")
|
19
|
+
return out
|
20
|
+
|
21
|
+
|
22
|
+
def convert_to_str(file_or_dir: str | File | Dir | Remote) -> str:
|
23
|
+
if isinstance(file_or_dir, str):
|
24
|
+
return file_or_dir
|
25
|
+
if isinstance(file_or_dir, File):
|
26
|
+
return str(file_or_dir.path)
|
27
|
+
if isinstance(file_or_dir, Dir):
|
28
|
+
return str(file_or_dir.path)
|
29
|
+
if isinstance(file_or_dir, Remote):
|
30
|
+
return str(file_or_dir)
|
31
|
+
raise ValueError(f"Invalid type for file_or_dir: {type(file_or_dir)}")
|
rclone_api/dir.py
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import json
|
1
2
|
from typing import Generator
|
2
3
|
|
3
4
|
from rclone_api.dir_listing import DirListing
|
@@ -8,6 +9,14 @@ from rclone_api.rpath import RPath
|
|
8
9
|
class Dir:
|
9
10
|
"""Remote file dataclass."""
|
10
11
|
|
12
|
+
@property
|
13
|
+
def remote(self) -> Remote:
|
14
|
+
return self.path.remote
|
15
|
+
|
16
|
+
@property
|
17
|
+
def name(self) -> str:
|
18
|
+
return self.path.name
|
19
|
+
|
11
20
|
def __init__(self, path: RPath | Remote) -> None:
|
12
21
|
"""Initialize Dir with either an RPath or Remote.
|
13
22
|
|
@@ -17,6 +26,7 @@ class Dir:
|
|
17
26
|
if isinstance(path, Remote):
|
18
27
|
# Need to create an RPath for the Remote's root
|
19
28
|
self.path = RPath(
|
29
|
+
remote=path,
|
20
30
|
path=str(path),
|
21
31
|
name=str(path),
|
22
32
|
size=0,
|
@@ -28,11 +38,14 @@ class Dir:
|
|
28
38
|
self.path.set_rclone(path.rclone)
|
29
39
|
else:
|
30
40
|
self.path = path
|
41
|
+
# self.path.set_rclone(self.path.remote.rclone)
|
42
|
+
assert self.path.rclone is not None
|
31
43
|
|
32
44
|
def ls(self, max_depth: int = 0) -> DirListing:
|
33
45
|
"""List files and directories in the given path."""
|
34
46
|
assert self.path.rclone is not None
|
35
|
-
|
47
|
+
dir = Dir(self.path)
|
48
|
+
return self.path.rclone.ls(dir, max_depth=max_depth)
|
36
49
|
|
37
50
|
def walk(self, max_depth: int = -1) -> Generator[DirListing, None, None]:
|
38
51
|
"""List files and directories in the given path."""
|
@@ -41,5 +54,14 @@ class Dir:
|
|
41
54
|
assert self.path.rclone is not None
|
42
55
|
return walk(self, max_depth=max_depth)
|
43
56
|
|
57
|
+
def to_json(self) -> dict:
|
58
|
+
"""Convert the Dir to a JSON serializable dictionary."""
|
59
|
+
return self.path.to_json()
|
60
|
+
|
44
61
|
def __str__(self) -> str:
|
45
62
|
return str(self.path)
|
63
|
+
|
64
|
+
def __repr__(self) -> str:
|
65
|
+
data = self.path.to_json()
|
66
|
+
data_str = json.dumps(data)
|
67
|
+
return data_str
|
rclone_api/dir_listing.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
import json
|
2
|
+
|
3
3
|
from rclone_api.rpath import RPath
|
4
4
|
|
5
5
|
|
@@ -12,3 +12,29 @@ class DirListing:
|
|
12
12
|
|
13
13
|
self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
|
14
14
|
self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
|
15
|
+
|
16
|
+
def __str__(self) -> str:
|
17
|
+
n_files = len(self.files)
|
18
|
+
n_dirs = len(self.dirs)
|
19
|
+
msg = f"Files: {n_files}\n"
|
20
|
+
if n_files > 0:
|
21
|
+
for f in self.files:
|
22
|
+
msg += f" {f}\n"
|
23
|
+
msg += f"Dirs: {n_dirs}\n"
|
24
|
+
if n_dirs > 0:
|
25
|
+
for d in self.dirs:
|
26
|
+
msg += f" {d}\n"
|
27
|
+
return msg
|
28
|
+
|
29
|
+
def __repr__(self) -> str:
|
30
|
+
dirs: list = []
|
31
|
+
files: list = []
|
32
|
+
for d in self.dirs:
|
33
|
+
dirs.append(d.path.to_json())
|
34
|
+
for f in self.files:
|
35
|
+
files.append(f.path.to_json())
|
36
|
+
json_obj = {
|
37
|
+
"dirs": dirs,
|
38
|
+
"files": files,
|
39
|
+
}
|
40
|
+
return json.dumps(json_obj, indent=2)
|
rclone_api/{types.py → exec.py}
RENAMED
rclone_api/file.py
CHANGED
@@ -12,6 +12,10 @@ class File:
|
|
12
12
|
) -> None:
|
13
13
|
self.path = path
|
14
14
|
|
15
|
+
@property
|
16
|
+
def name(self) -> str:
|
17
|
+
return self.path.name
|
18
|
+
|
15
19
|
def read_text(self) -> str:
|
16
20
|
"""Read the file contents as bytes.
|
17
21
|
|
@@ -30,6 +34,14 @@ class File:
|
|
30
34
|
result = self.path.rclone._run(["cat", self.path.path])
|
31
35
|
return result.stdout
|
32
36
|
|
37
|
+
def to_json(self) -> dict:
|
38
|
+
"""Convert the File to a JSON serializable dictionary."""
|
39
|
+
return self.path.to_json()
|
40
|
+
|
33
41
|
def __str__(self) -> str:
|
34
|
-
|
35
|
-
|
42
|
+
return str(self.path)
|
43
|
+
|
44
|
+
def __repr__(self) -> str:
|
45
|
+
data = self.path.to_json()
|
46
|
+
data_str = json.dumps(data)
|
47
|
+
return data_str
|
rclone_api/rclone.py
CHANGED
@@ -3,15 +3,20 @@ Unit test file.
|
|
3
3
|
"""
|
4
4
|
|
5
5
|
import subprocess
|
6
|
+
from concurrent.futures import ThreadPoolExecutor
|
7
|
+
from fnmatch import fnmatch
|
6
8
|
from pathlib import Path
|
7
9
|
from typing import Generator
|
8
10
|
|
9
11
|
from rclone_api import Dir
|
12
|
+
from rclone_api.config import Config
|
13
|
+
from rclone_api.convert import convert_to_filestr_list, convert_to_str
|
10
14
|
from rclone_api.dir_listing import DirListing
|
15
|
+
from rclone_api.exec import RcloneExec
|
16
|
+
from rclone_api.file import File
|
11
17
|
from rclone_api.remote import Remote
|
12
18
|
from rclone_api.rpath import RPath
|
13
|
-
from rclone_api.
|
14
|
-
from rclone_api.util import get_rclone_exe
|
19
|
+
from rclone_api.util import get_rclone_exe, to_path
|
15
20
|
from rclone_api.walk import walk
|
16
21
|
|
17
22
|
|
@@ -27,7 +32,12 @@ class Rclone:
|
|
27
32
|
def _run(self, cmd: list[str]) -> subprocess.CompletedProcess:
|
28
33
|
return self._exec.execute(cmd)
|
29
34
|
|
30
|
-
def ls(
|
35
|
+
def ls(
|
36
|
+
self,
|
37
|
+
path: Dir | Remote | str,
|
38
|
+
max_depth: int | None = None,
|
39
|
+
glob: str | None = None,
|
40
|
+
) -> DirListing:
|
31
41
|
"""List files in the given path.
|
32
42
|
|
33
43
|
Args:
|
@@ -37,16 +47,35 @@ class Rclone:
|
|
37
47
|
Returns:
|
38
48
|
List of File objects found at the path
|
39
49
|
"""
|
50
|
+
|
51
|
+
if isinstance(path, str):
|
52
|
+
path = Dir(
|
53
|
+
to_path(path, self)
|
54
|
+
) # assume it's a directory if ls is being called.
|
55
|
+
|
40
56
|
cmd = ["lsjson"]
|
41
|
-
if max_depth
|
42
|
-
cmd.
|
57
|
+
if max_depth is not None:
|
58
|
+
cmd.append("--recursive")
|
59
|
+
if max_depth > -1:
|
60
|
+
cmd.append("--max-depth")
|
61
|
+
cmd.append(str(max_depth))
|
43
62
|
cmd.append(str(path))
|
63
|
+
remote = path.remote if isinstance(path, Dir) else path
|
64
|
+
assert isinstance(remote, Remote)
|
44
65
|
|
45
66
|
cp = self._run(cmd)
|
46
67
|
text = cp.stdout
|
47
|
-
|
68
|
+
parent_path: str | None = None
|
69
|
+
if isinstance(path, Dir):
|
70
|
+
parent_path = path.path.path
|
71
|
+
paths: list[RPath] = RPath.from_json_str(text, remote, parent_path=parent_path)
|
72
|
+
# print(parent_path)
|
48
73
|
for o in paths:
|
49
74
|
o.set_rclone(self)
|
75
|
+
|
76
|
+
# do we have a glob pattern?
|
77
|
+
if glob is not None:
|
78
|
+
paths = [p for p in paths if fnmatch(p.path, glob)]
|
50
79
|
return DirListing(paths)
|
51
80
|
|
52
81
|
def listremotes(self) -> list[Remote]:
|
@@ -61,7 +90,7 @@ class Rclone:
|
|
61
90
|
return out
|
62
91
|
|
63
92
|
def walk(
|
64
|
-
self, path:
|
93
|
+
self, path: Dir | Remote | str, max_depth: int = -1
|
65
94
|
) -> Generator[DirListing, None, None]:
|
66
95
|
"""Walk through the given path recursively.
|
67
96
|
|
@@ -72,11 +101,13 @@ class Rclone:
|
|
72
101
|
Yields:
|
73
102
|
DirListing: Directory listing for each directory encountered
|
74
103
|
"""
|
75
|
-
if isinstance(path,
|
104
|
+
if isinstance(path, Dir):
|
76
105
|
# Create a Remote object for the path
|
106
|
+
remote = path.remote
|
77
107
|
rpath = RPath(
|
78
|
-
|
79
|
-
|
108
|
+
remote=remote,
|
109
|
+
path=path.path.path,
|
110
|
+
name=path.path.name,
|
80
111
|
size=0,
|
81
112
|
mime_type="inode/directory",
|
82
113
|
mod_time="",
|
@@ -84,7 +115,100 @@ class Rclone:
|
|
84
115
|
)
|
85
116
|
rpath.set_rclone(self)
|
86
117
|
dir_obj = Dir(rpath)
|
87
|
-
|
118
|
+
elif isinstance(path, str):
|
119
|
+
dir_obj = Dir(to_path(path, self))
|
120
|
+
elif isinstance(path, Remote):
|
88
121
|
dir_obj = Dir(path)
|
122
|
+
else:
|
123
|
+
assert f"Invalid type for path: {type(path)}"
|
89
124
|
|
90
125
|
yield from walk(dir_obj, max_depth=max_depth)
|
126
|
+
|
127
|
+
def copyfile(self, src: File | str, dst: File | str) -> None:
|
128
|
+
"""Copy a single file from source to destination.
|
129
|
+
|
130
|
+
Args:
|
131
|
+
src: Source file path (including remote if applicable)
|
132
|
+
dst: Destination file path (including remote if applicable)
|
133
|
+
|
134
|
+
Raises:
|
135
|
+
subprocess.CalledProcessError: If the copy operation fails
|
136
|
+
"""
|
137
|
+
src = src if isinstance(src, str) else str(src.path)
|
138
|
+
dst = dst if isinstance(dst, str) else str(dst.path)
|
139
|
+
cmd_list: list[str] = ["copyto", src, dst]
|
140
|
+
self._run(cmd_list)
|
141
|
+
|
142
|
+
def copyfiles(self, filelist: dict[File, File] | dict[str, str]) -> None:
|
143
|
+
"""Copy multiple files from source to destination.
|
144
|
+
|
145
|
+
Warning - slow.
|
146
|
+
|
147
|
+
Args:
|
148
|
+
payload: Dictionary of source and destination file paths
|
149
|
+
"""
|
150
|
+
str_dict: dict[str, str] = {}
|
151
|
+
for src, dst in filelist.items():
|
152
|
+
src = src if isinstance(src, str) else str(src.path)
|
153
|
+
dst = dst if isinstance(dst, str) else str(dst.path)
|
154
|
+
str_dict[src] = dst
|
155
|
+
|
156
|
+
with ThreadPoolExecutor(max_workers=64) as executor:
|
157
|
+
for src, dst in str_dict.items(): # warning - slow
|
158
|
+
cmd_list: list[str] = ["copyto", src, dst]
|
159
|
+
# self._run(cmd_list)
|
160
|
+
executor.submit(self._run, cmd_list)
|
161
|
+
|
162
|
+
def copy(self, src: Dir, dst: Dir) -> None:
|
163
|
+
"""Copy files from source to destination.
|
164
|
+
|
165
|
+
Args:
|
166
|
+
src: Source directory
|
167
|
+
dst: Destination directory
|
168
|
+
"""
|
169
|
+
src_dir = src.path.path
|
170
|
+
dst_dir = dst.path.path
|
171
|
+
cmd_list: list[str] = ["copy", src_dir, dst_dir]
|
172
|
+
self._run(cmd_list)
|
173
|
+
|
174
|
+
def purge(self, path: Dir | str) -> None:
|
175
|
+
"""Purge a directory"""
|
176
|
+
# path should always be a string
|
177
|
+
path = path if isinstance(path, str) else str(path.path)
|
178
|
+
cmd_list: list[str] = ["purge", str(path)]
|
179
|
+
self._run(cmd_list)
|
180
|
+
|
181
|
+
def deletefiles(self, files: str | File | list[str] | list[File]) -> None:
|
182
|
+
"""Delete a directory"""
|
183
|
+
payload: list[str] = convert_to_filestr_list(files)
|
184
|
+
cmd_list: list[str] = ["delete"] + payload
|
185
|
+
self._run(cmd_list)
|
186
|
+
|
187
|
+
def exists(self, path: Dir | Remote | str | File) -> bool:
|
188
|
+
"""Check if a file or directory exists."""
|
189
|
+
arg: str = convert_to_str(path)
|
190
|
+
assert isinstance(arg, str)
|
191
|
+
try:
|
192
|
+
self.ls(arg)
|
193
|
+
return True
|
194
|
+
except subprocess.CalledProcessError:
|
195
|
+
return False
|
196
|
+
|
197
|
+
def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
|
198
|
+
"""Check if two directories are in sync."""
|
199
|
+
src = convert_to_str(src)
|
200
|
+
dst = convert_to_str(dst)
|
201
|
+
cmd_list: list[str] = ["check", str(src), str(dst)]
|
202
|
+
try:
|
203
|
+
self._run(cmd_list)
|
204
|
+
return True
|
205
|
+
except subprocess.CalledProcessError:
|
206
|
+
return False
|
207
|
+
|
208
|
+
def copy_dir(self, src: str | Dir, dst: str | Dir) -> None:
|
209
|
+
"""Copy a directory from source to destination."""
|
210
|
+
# convert src to str, also dst
|
211
|
+
src = convert_to_str(src)
|
212
|
+
dst = convert_to_str(dst)
|
213
|
+
cmd_list: list[str] = ["copy", src, dst]
|
214
|
+
self._run(cmd_list)
|
rclone_api/rpath.py
CHANGED
@@ -1,12 +1,15 @@
|
|
1
1
|
import json
|
2
2
|
from typing import Any
|
3
3
|
|
4
|
+
from rclone_api.remote import Remote
|
5
|
+
|
4
6
|
|
5
7
|
class RPath:
|
6
8
|
"""Remote file dataclass."""
|
7
9
|
|
8
10
|
def __init__(
|
9
11
|
self,
|
12
|
+
remote: Remote,
|
10
13
|
path: str,
|
11
14
|
name: str,
|
12
15
|
size: int,
|
@@ -16,6 +19,7 @@ class RPath:
|
|
16
19
|
) -> None:
|
17
20
|
from rclone_api.rclone import Rclone
|
18
21
|
|
22
|
+
self.remote = remote
|
19
23
|
self.path = path
|
20
24
|
self.name = name
|
21
25
|
self.size = size
|
@@ -32,10 +36,16 @@ class RPath:
|
|
32
36
|
self.rclone = rclone
|
33
37
|
|
34
38
|
@staticmethod
|
35
|
-
def from_dict(
|
39
|
+
def from_dict(
|
40
|
+
data: dict, remote: Remote, parent_path: str | None = None
|
41
|
+
) -> "RPath":
|
36
42
|
"""Create a File from a dictionary."""
|
43
|
+
path = data["Path"]
|
44
|
+
if parent_path is not None:
|
45
|
+
path = f"{parent_path}/{path}"
|
37
46
|
return RPath(
|
38
|
-
|
47
|
+
remote,
|
48
|
+
path,
|
39
49
|
data["Name"],
|
40
50
|
data["Size"],
|
41
51
|
data["MimeType"],
|
@@ -45,21 +55,25 @@ class RPath:
|
|
45
55
|
)
|
46
56
|
|
47
57
|
@staticmethod
|
48
|
-
def from_array(
|
58
|
+
def from_array(
|
59
|
+
data: list[dict], remote: Remote, parent_path: str | None = None
|
60
|
+
) -> list["RPath"]:
|
49
61
|
"""Create a File from a dictionary."""
|
50
62
|
out: list[RPath] = []
|
51
63
|
for d in data:
|
52
|
-
file: RPath = RPath.from_dict(d)
|
64
|
+
file: RPath = RPath.from_dict(d, remote, parent_path)
|
53
65
|
out.append(file)
|
54
66
|
return out
|
55
67
|
|
56
68
|
@staticmethod
|
57
|
-
def from_json_str(
|
69
|
+
def from_json_str(
|
70
|
+
json_str: str, remote: Remote, parent_path: str | None = None
|
71
|
+
) -> list["RPath"]:
|
58
72
|
"""Create a File from a JSON string."""
|
59
73
|
json_obj = json.loads(json_str)
|
60
74
|
if isinstance(json_obj, dict):
|
61
|
-
return [RPath.from_dict(json_obj)]
|
62
|
-
return RPath.from_array(json_obj)
|
75
|
+
return [RPath.from_dict(json_obj, remote, parent_path)]
|
76
|
+
return RPath.from_array(json_obj, remote, parent_path)
|
63
77
|
|
64
78
|
def to_json(self) -> dict:
|
65
79
|
return {
|
@@ -73,5 +87,9 @@ class RPath:
|
|
73
87
|
}
|
74
88
|
|
75
89
|
def __str__(self) -> str:
|
76
|
-
|
77
|
-
|
90
|
+
return f"{self.remote.name}:{self.path}"
|
91
|
+
|
92
|
+
def __repr__(self):
|
93
|
+
data = self.to_json()
|
94
|
+
data["Remote"] = self.remote.name
|
95
|
+
return json.dumps(data)
|
rclone_api/util.py
CHANGED
@@ -1,9 +1,64 @@
|
|
1
|
+
import os
|
1
2
|
import shutil
|
2
3
|
import subprocess
|
3
4
|
from pathlib import Path
|
4
5
|
from tempfile import TemporaryDirectory
|
6
|
+
from typing import Any
|
5
7
|
|
6
|
-
from rclone_api.
|
8
|
+
from rclone_api.config import Config
|
9
|
+
from rclone_api.dir import Dir
|
10
|
+
from rclone_api.remote import Remote
|
11
|
+
from rclone_api.rpath import RPath
|
12
|
+
|
13
|
+
# from .rclone import Rclone
|
14
|
+
|
15
|
+
|
16
|
+
def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
|
17
|
+
from rclone_api.rclone import Rclone
|
18
|
+
|
19
|
+
assert isinstance(rclone, Rclone)
|
20
|
+
# if str then it will be remote:path
|
21
|
+
if isinstance(item, str):
|
22
|
+
# return RPath(item)
|
23
|
+
# remote_name: str = item.split(":")[0]
|
24
|
+
parts = item.split(":")
|
25
|
+
remote_name = parts[0]
|
26
|
+
path = ":".join(parts[1:])
|
27
|
+
remote = Remote(name=remote_name, rclone=rclone)
|
28
|
+
out = RPath(
|
29
|
+
remote=remote,
|
30
|
+
path=path,
|
31
|
+
name="",
|
32
|
+
size=0,
|
33
|
+
mime_type="",
|
34
|
+
mod_time="",
|
35
|
+
is_dir=True,
|
36
|
+
)
|
37
|
+
out.set_rclone(rclone)
|
38
|
+
return out
|
39
|
+
elif isinstance(item, Dir):
|
40
|
+
return item.path
|
41
|
+
elif isinstance(item, Remote):
|
42
|
+
out = RPath(
|
43
|
+
remote=item,
|
44
|
+
path=str(item),
|
45
|
+
name=str(item),
|
46
|
+
size=0,
|
47
|
+
mime_type="inode/directory",
|
48
|
+
mod_time="",
|
49
|
+
is_dir=True,
|
50
|
+
)
|
51
|
+
out.set_rclone(rclone)
|
52
|
+
return out
|
53
|
+
else:
|
54
|
+
raise ValueError(f"Invalid type for item: {type(item)}")
|
55
|
+
|
56
|
+
|
57
|
+
def _get_verbose(verbose: bool | None) -> bool:
|
58
|
+
if verbose is not None:
|
59
|
+
return verbose
|
60
|
+
# get it from the environment
|
61
|
+
return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
|
7
62
|
|
8
63
|
|
9
64
|
def get_rclone_exe(rclone_exe: Path | None) -> Path:
|
@@ -20,10 +75,11 @@ def rclone_execute(
|
|
20
75
|
cmd: list[str],
|
21
76
|
rclone_conf: Path | Config,
|
22
77
|
rclone_exe: Path,
|
23
|
-
verbose: bool =
|
78
|
+
verbose: bool | None = None,
|
24
79
|
) -> subprocess.CompletedProcess:
|
25
|
-
print(subprocess.list2cmdline(cmd))
|
26
80
|
tempdir: TemporaryDirectory | None = None
|
81
|
+
verbose = _get_verbose(verbose)
|
82
|
+
assert verbose is not None
|
27
83
|
|
28
84
|
try:
|
29
85
|
if isinstance(rclone_conf, Config):
|
@@ -38,8 +94,16 @@ def rclone_execute(
|
|
38
94
|
cmd_str = subprocess.list2cmdline(cmd)
|
39
95
|
print(f"Running: {cmd_str}")
|
40
96
|
cp = subprocess.run(
|
41
|
-
cmd, capture_output=True, encoding="utf-8", check=
|
97
|
+
cmd, capture_output=True, encoding="utf-8", check=False, shell=False
|
42
98
|
)
|
99
|
+
if cp.returncode != 0:
|
100
|
+
cmd_str = subprocess.list2cmdline(cmd)
|
101
|
+
print(
|
102
|
+
f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
|
103
|
+
)
|
104
|
+
raise subprocess.CalledProcessError(
|
105
|
+
cp.returncode, cmd, cp.stdout, cp.stderr
|
106
|
+
)
|
43
107
|
return cp
|
44
108
|
finally:
|
45
109
|
if tempdir:
|
rclone_api/walk.py
CHANGED
@@ -1,87 +1,70 @@
|
|
1
|
-
from
|
2
|
-
from
|
1
|
+
from queue import Queue
|
2
|
+
from threading import Thread
|
3
3
|
from typing import Generator
|
4
4
|
|
5
5
|
from rclone_api import Dir
|
6
6
|
from rclone_api.dir_listing import DirListing
|
7
7
|
from rclone_api.remote import Remote
|
8
8
|
|
9
|
+
_MAX_OUT_QUEUE_SIZE = 50
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
|
12
|
+
def _walk_runner(
|
13
|
+
queue: Queue[Dir], max_depth: int, out_queue: Queue[DirListing | None]
|
14
|
+
) -> None:
|
15
|
+
try:
|
16
|
+
while not queue.empty():
|
17
|
+
current_dir = queue.get()
|
18
|
+
dirlisting = current_dir.ls()
|
19
|
+
out_queue.put(dirlisting)
|
20
|
+
dirs = dirlisting.dirs
|
21
|
+
|
22
|
+
if max_depth != 0 and len(dirs) > 0:
|
23
|
+
for child in dirs:
|
24
|
+
queue.put(child)
|
25
|
+
if max_depth < 0:
|
26
|
+
continue
|
27
|
+
if max_depth > 0:
|
28
|
+
max_depth -= 1
|
29
|
+
out_queue.put(None)
|
30
|
+
except KeyboardInterrupt:
|
31
|
+
import _thread
|
32
|
+
|
33
|
+
out_queue.put(None)
|
34
|
+
|
35
|
+
_thread.interrupt_main()
|
36
|
+
|
37
|
+
|
38
|
+
def walk(dir: Dir | Remote, max_depth: int = -1) -> Generator[DirListing, None, None]:
|
13
39
|
"""Walk through the given directory recursively.
|
14
40
|
|
15
41
|
Args:
|
16
42
|
dir: Directory or Remote to walk through
|
17
43
|
max_depth: Maximum depth to traverse (-1 for unlimited)
|
18
|
-
max_workers: Maximum number of concurrent workers
|
19
44
|
|
20
45
|
Yields:
|
21
46
|
DirListing: Directory listing for each directory encountered
|
22
47
|
"""
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
results: Queue[DirListing | Exception] = Queue()
|
48
|
+
try:
|
49
|
+
# Convert Remote to Dir if needed
|
50
|
+
if isinstance(dir, Remote):
|
51
|
+
dir = Dir(dir)
|
28
52
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
# Add timeout to allow checking for sentinel value
|
33
|
-
try:
|
34
|
-
current_dir, depth = pending.get(timeout=0.1)
|
35
|
-
except Empty:
|
36
|
-
continue
|
53
|
+
in_queue: Queue[Dir] = Queue()
|
54
|
+
out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
|
55
|
+
in_queue.put(dir)
|
37
56
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
57
|
+
# Start worker thread
|
58
|
+
worker = Thread(
|
59
|
+
target=_walk_runner, args=(in_queue, max_depth, out_queue), daemon=True
|
60
|
+
)
|
61
|
+
worker.start()
|
42
62
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
if max_depth == -1 or depth < max_depth:
|
47
|
-
for d in listing.dirs:
|
48
|
-
pending.put((d, depth + 1))
|
49
|
-
|
50
|
-
pending.task_done()
|
51
|
-
except Exception as e:
|
52
|
-
results.put(e)
|
53
|
-
pending.task_done()
|
63
|
+
while dirlisting := out_queue.get():
|
64
|
+
if dirlisting is None:
|
54
65
|
break
|
55
|
-
|
56
|
-
|
57
|
-
# Start workers
|
58
|
-
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
59
|
-
workers = [executor.submit(worker) for _ in range(max_workers)]
|
60
|
-
|
61
|
-
# Start walking
|
62
|
-
pending.put((dir, 0))
|
63
|
-
|
64
|
-
# Process results while workers are running
|
65
|
-
completed = 0
|
66
|
-
while completed < max_workers:
|
67
|
-
try:
|
68
|
-
result = results.get(timeout=0.1)
|
69
|
-
if isinstance(result, Exception):
|
70
|
-
# Propagate exception
|
71
|
-
raise result
|
72
|
-
yield result
|
73
|
-
except Empty:
|
74
|
-
# Check if any workers have completed
|
75
|
-
completed = sum(1 for w in workers if w.done())
|
76
|
-
continue
|
77
|
-
|
78
|
-
# Signal workers to stop
|
79
|
-
for _ in range(max_workers):
|
80
|
-
pending.put((None, 0))
|
66
|
+
yield dirlisting
|
81
67
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
if isinstance(result, Exception):
|
86
|
-
raise result
|
87
|
-
yield result
|
68
|
+
worker.join()
|
69
|
+
except KeyboardInterrupt:
|
70
|
+
pass
|
@@ -0,0 +1,34 @@
|
|
1
|
+
Metadata-Version: 2.2
|
2
|
+
Name: rclone_api
|
3
|
+
Version: 1.0.14
|
4
|
+
Summary: rclone api in python
|
5
|
+
Home-page: https://github.com/zackees/rclone-api
|
6
|
+
Maintainer: Zachary Vorhies
|
7
|
+
License: BSD 3-Clause License
|
8
|
+
Keywords: template-python-cmd
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
10
|
+
Requires-Python: >=3.10
|
11
|
+
Description-Content-Type: text/markdown
|
12
|
+
License-File: LICENSE
|
13
|
+
Requires-Dist: python-dotenv>=1.0.0
|
14
|
+
Dynamic: home-page
|
15
|
+
Dynamic: maintainer
|
16
|
+
|
17
|
+
# rclone-api
|
18
|
+
|
19
|
+
[](https://github.com/zackees/rclone-api/actions/workflows/lint.yml)
|
20
|
+
[](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml)
|
21
|
+
[](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
|
22
|
+
[](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
|
23
|
+
|
24
|
+
Api version of rclone. It's well tested. It's just released so this readme is a little to be desired.
|
25
|
+
|
26
|
+
To develop software, run `. ./activate`
|
27
|
+
|
28
|
+
# Windows
|
29
|
+
|
30
|
+
This environment requires you to use `git-bash`.
|
31
|
+
|
32
|
+
# Linting
|
33
|
+
|
34
|
+
Run `./lint`
|
@@ -0,0 +1,19 @@
|
|
1
|
+
rclone_api/__init__.py,sha256=gca3IDhDdKBtyoS1ekGQ-xTa6zrShZ6Gb4XS8ChdlYo,266
|
2
|
+
rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
|
3
|
+
rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
|
4
|
+
rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
|
5
|
+
rclone_api/dir.py,sha256=xUV4i9_E7QHrqHld2IPBlon1CGaAp8qQYMKfJTyVvoQ,2088
|
6
|
+
rclone_api/dir_listing.py,sha256=8t5Jx9ZVOJPqGKTJbWaES6bjgogUT2bnpbPVWwK1Fcs,1124
|
7
|
+
rclone_api/exec.py,sha256=PgiNF5eQpsY8Pb62ZPwLli1WA9iMEJeMZwX43VEYKII,472
|
8
|
+
rclone_api/file.py,sha256=bb9FWQy54wKRydy0M12MQwNxycBwRSn_10ubYGAw0ag,1234
|
9
|
+
rclone_api/rclone.py,sha256=m2V3BQX0buK3mwImBiTSNTUh_JcdQJ27OeJWxqvgRZs,7455
|
10
|
+
rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
|
11
|
+
rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
|
12
|
+
rclone_api/util.py,sha256=uiTfOlKCFB5FdUkYR2-Hriz0fVyrWeuZtSgHgc9RrpA,3328
|
13
|
+
rclone_api/walk.py,sha256=J78-bY2AhNpt2ICsI5LqmXRE7oC6wVDoKoicIoU6XMg,1953
|
14
|
+
rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
|
15
|
+
rclone_api-1.0.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
16
|
+
rclone_api-1.0.14.dist-info/METADATA,sha256=pATg2JeY8MQ2I7kywlJj6x4KPdL0QKKcbxjxbvq1OCs,1375
|
17
|
+
rclone_api-1.0.14.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
|
18
|
+
rclone_api-1.0.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
19
|
+
rclone_api-1.0.14.dist-info/RECORD,,
|
@@ -1,36 +0,0 @@
|
|
1
|
-
Metadata-Version: 2.2
|
2
|
-
Name: rclone_api
|
3
|
-
Version: 1.0.12
|
4
|
-
Summary: rclone api in python
|
5
|
-
Home-page: https://github.com/zackees/rclone-api
|
6
|
-
Maintainer: Zachary Vorhies
|
7
|
-
License: BSD 3-Clause License
|
8
|
-
Keywords: template-python-cmd
|
9
|
-
Classifier: Programming Language :: Python :: 3
|
10
|
-
Requires-Python: >=3.7
|
11
|
-
Description-Content-Type: text/markdown
|
12
|
-
License-File: LICENSE
|
13
|
-
Dynamic: home-page
|
14
|
-
Dynamic: maintainer
|
15
|
-
|
16
|
-
# template-python-cmd
|
17
|
-
A template for quickly making a python lib that has a command line program attached
|
18
|
-
|
19
|
-
[](../../actions/workflows/lint.yml)
|
20
|
-
|
21
|
-
[](../../actions/workflows/push_macos.yml)
|
22
|
-
[](../../actions/workflows/push_ubuntu.yml)
|
23
|
-
[](../../actions/workflows/push_win.yml)
|
24
|
-
|
25
|
-
Replace `template-python-cmd` and `template_python_cmd` with your command. Run tox until it's
|
26
|
-
correct.
|
27
|
-
|
28
|
-
To develop software, run `. ./activate.sh`
|
29
|
-
|
30
|
-
# Windows
|
31
|
-
|
32
|
-
This environment requires you to use `git-bash`.
|
33
|
-
|
34
|
-
# Linting
|
35
|
-
|
36
|
-
Run `./lint.sh` to find linting errors using `pylint`, `flake8` and `mypy`.
|
@@ -1,18 +0,0 @@
|
|
1
|
-
rclone_api/__init__.py,sha256=YH7KQaPwUiJWJiRf0NRKD7XHhMXsxWdXDjt9WLSwdjA,265
|
2
|
-
rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
|
3
|
-
rclone_api/config.py,sha256=UujjDNcpwgipgdyFPMhC3zikvlsrurUvvZiwXm5NlPg,471
|
4
|
-
rclone_api/dir.py,sha256=rd40egerhoKGAdIKfeGh_P70fSDzMo-EiTl0euABGOU,1497
|
5
|
-
rclone_api/dir_listing.py,sha256=NWleKHCCRW7_eh9JfRwE6r3QbjmiHD5ZEGQcd2vm4uY,458
|
6
|
-
rclone_api/file.py,sha256=409neYPfCUKQX2w7Uw7_D0wR2PtH42srNe9u_nnOLxM,925
|
7
|
-
rclone_api/rclone.py,sha256=EUsLW5qcipI5RzhpHUd0PQqQow43m2uKJmNgVR8ZBuU,2859
|
8
|
-
rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
|
9
|
-
rclone_api/rpath.py,sha256=_k59z-O75hAhpPyUSH4glPEoiYtIS3RUQb7ltU9Ianw,2009
|
10
|
-
rclone_api/types.py,sha256=Yp15HrjwZonSQhE323R0WP7fA4NWqKjAfM7z3OwHpWI,518
|
11
|
-
rclone_api/util.py,sha256=EXnijLLdFHqwoZvHrZdqeM8r6T7ad9-pN7qBN1b0I_g,1465
|
12
|
-
rclone_api/walk.py,sha256=3GKnu9P5aPNiftfoi52U7wo1wixZxwxdqqS0_IlMGCE,2816
|
13
|
-
rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
|
14
|
-
rclone_api-1.0.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
15
|
-
rclone_api-1.0.12.dist-info/METADATA,sha256=fkbaFAbZssh_WIKBJs-M1VNNUf3o3YKcEo3wIYDGO8o,1245
|
16
|
-
rclone_api-1.0.12.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
|
17
|
-
rclone_api-1.0.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
18
|
-
rclone_api-1.0.12.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|