rclone-api 1.4.33__py2.py3-none-any.whl → 1.5.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/__init__.py +523 -523
- rclone_api/db/__init__.py +3 -3
- rclone_api/detail/walk.py +116 -116
- rclone_api/dir.py +113 -113
- rclone_api/install.py +90 -0
- rclone_api/log.py +44 -44
- rclone_api/s3/multipart/upload_parts_server_side_merge.py +546 -546
- rclone_api/scan_missing_folders.py +153 -153
- rclone_api/util.py +315 -307
- {rclone_api-1.4.33.dist-info → rclone_api-1.5.0.dist-info}/METADATA +574 -572
- {rclone_api-1.4.33.dist-info → rclone_api-1.5.0.dist-info}/RECORD +15 -14
- {rclone_api-1.4.33.dist-info → rclone_api-1.5.0.dist-info}/WHEEL +1 -1
- {rclone_api-1.4.33.dist-info → rclone_api-1.5.0.dist-info}/LICENSE +0 -0
- {rclone_api-1.4.33.dist-info → rclone_api-1.5.0.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.4.33.dist-info → rclone_api-1.5.0.dist-info}/top_level.txt +0 -0
rclone_api/__init__.py
CHANGED
@@ -1,523 +1,523 @@
|
|
1
|
-
# Import logging module to activate default configuration
|
2
|
-
|
3
|
-
from datetime import datetime
|
4
|
-
from pathlib import Path
|
5
|
-
from typing import Generator
|
6
|
-
|
7
|
-
from rclone_api import log
|
8
|
-
|
9
|
-
from .completed_process import CompletedProcess
|
10
|
-
from .config import Config, Parsed, Section
|
11
|
-
from .diff import DiffItem, DiffOption, DiffType
|
12
|
-
from .dir import Dir
|
13
|
-
from .dir_listing import DirListing
|
14
|
-
from .file import File, FileItem
|
15
|
-
from .file_stream import FilesStream
|
16
|
-
from .filelist import FileList
|
17
|
-
from .http_server import HttpFetcher, HttpServer, Range
|
18
|
-
|
19
|
-
# Import the configure_logging function to make it available at package level
|
20
|
-
from .log import configure_logging, setup_default_logging
|
21
|
-
from .mount import Mount
|
22
|
-
from .process import Process
|
23
|
-
from .remote import Remote
|
24
|
-
from .rpath import RPath
|
25
|
-
from .s3.types import MultiUploadResult
|
26
|
-
from .types import ListingOption, Order, PartInfo, SizeResult, SizeSuffix
|
27
|
-
|
28
|
-
setup_default_logging()
|
29
|
-
|
30
|
-
|
31
|
-
def rclone_verbose(val: bool | None) -> bool:
|
32
|
-
from rclone_api.rclone_impl import rclone_verbose as _rclone_verbose
|
33
|
-
|
34
|
-
return _rclone_verbose(val)
|
35
|
-
|
36
|
-
|
37
|
-
class Rclone:
|
38
|
-
def __init__(
|
39
|
-
self, rclone_conf: Path | Config, rclone_exe: Path | None = None
|
40
|
-
) -> None:
|
41
|
-
from rclone_api.rclone_impl import RcloneImpl
|
42
|
-
|
43
|
-
self.impl: RcloneImpl = RcloneImpl(rclone_conf, rclone_exe)
|
44
|
-
|
45
|
-
def webgui(self, other_args: list[str] | None = None) -> Process:
|
46
|
-
"""Launch the Rclone web GUI."""
|
47
|
-
return self.impl.webgui(other_args=other_args)
|
48
|
-
|
49
|
-
def launch_server(
|
50
|
-
self,
|
51
|
-
addr: str,
|
52
|
-
user: str | None = None,
|
53
|
-
password: str | None = None,
|
54
|
-
other_args: list[str] | None = None,
|
55
|
-
) -> Process:
|
56
|
-
"""Launch the Rclone server so it can receive commands"""
|
57
|
-
return self.impl.launch_server(
|
58
|
-
addr=addr, user=user, password=password, other_args=other_args
|
59
|
-
)
|
60
|
-
|
61
|
-
def remote_control(
|
62
|
-
self,
|
63
|
-
addr: str,
|
64
|
-
user: str | None = None,
|
65
|
-
password: str | None = None,
|
66
|
-
capture: bool | None = None,
|
67
|
-
other_args: list[str] | None = None,
|
68
|
-
) -> CompletedProcess:
|
69
|
-
return self.impl.remote_control(
|
70
|
-
addr=addr,
|
71
|
-
user=user,
|
72
|
-
password=password,
|
73
|
-
capture=capture,
|
74
|
-
other_args=other_args,
|
75
|
-
)
|
76
|
-
|
77
|
-
def obscure(self, password: str) -> str:
|
78
|
-
"""Obscure a password for use in rclone config files."""
|
79
|
-
return self.impl.obscure(password=password)
|
80
|
-
|
81
|
-
def ls_stream(
|
82
|
-
self,
|
83
|
-
path: str,
|
84
|
-
max_depth: int = -1,
|
85
|
-
fast_list: bool = False,
|
86
|
-
) -> FilesStream:
|
87
|
-
"""
|
88
|
-
List files in the given path
|
89
|
-
|
90
|
-
Args:
|
91
|
-
src: Remote path to list
|
92
|
-
max_depth: Maximum recursion depth (-1 for unlimited)
|
93
|
-
fast_list: Use fast list (only use when getting THE entire data repository from the root/bucket, or it's small)
|
94
|
-
"""
|
95
|
-
return self.impl.ls_stream(path=path, max_depth=max_depth, fast_list=fast_list)
|
96
|
-
|
97
|
-
def save_to_db(
|
98
|
-
self,
|
99
|
-
src: str,
|
100
|
-
db_url: str,
|
101
|
-
max_depth: int = -1,
|
102
|
-
fast_list: bool = False,
|
103
|
-
) -> None:
|
104
|
-
"""
|
105
|
-
Save files to a database (sqlite, mysql, postgres)
|
106
|
-
|
107
|
-
Args:
|
108
|
-
src: Remote path to list, this will be used to populate an entire table, so always use the root-most path.
|
109
|
-
db_url: Database URL, like sqlite:///data.db or mysql://user:pass@localhost/db or postgres://user:pass@localhost/db
|
110
|
-
max_depth: Maximum depth to traverse (-1 for unlimited)
|
111
|
-
fast_list: Use fast list (only use when getting THE entire data repository from the root/bucket)
|
112
|
-
|
113
|
-
"""
|
114
|
-
return self.impl.save_to_db(
|
115
|
-
src=src, db_url=db_url, max_depth=max_depth, fast_list=fast_list
|
116
|
-
)
|
117
|
-
|
118
|
-
def ls(
|
119
|
-
self,
|
120
|
-
path: Dir | Remote | str | None = None,
|
121
|
-
max_depth: int | None = None,
|
122
|
-
glob: str | None = None,
|
123
|
-
order: Order = Order.NORMAL,
|
124
|
-
listing_option: ListingOption = ListingOption.ALL,
|
125
|
-
) -> DirListing:
|
126
|
-
return self.impl.ls(
|
127
|
-
path=path,
|
128
|
-
max_depth=max_depth,
|
129
|
-
glob=glob,
|
130
|
-
order=order,
|
131
|
-
listing_option=listing_option,
|
132
|
-
)
|
133
|
-
|
134
|
-
def listremotes(self) -> list[Remote]:
|
135
|
-
return self.impl.listremotes()
|
136
|
-
|
137
|
-
def diff(
|
138
|
-
self,
|
139
|
-
src: str,
|
140
|
-
dst: str,
|
141
|
-
min_size: (
|
142
|
-
str | None
|
143
|
-
) = None, # e. g. "1MB" - see rclone documentation: https://rclone.org/commands/rclone_check/
|
144
|
-
max_size: (
|
145
|
-
str | None
|
146
|
-
) = None, # e. g. "1GB" - see rclone documentation: https://rclone.org/commands/rclone_check/
|
147
|
-
diff_option: DiffOption = DiffOption.COMBINED,
|
148
|
-
fast_list: bool = True,
|
149
|
-
size_only: bool | None = None,
|
150
|
-
checkers: int | None = None,
|
151
|
-
other_args: list[str] | None = None,
|
152
|
-
) -> Generator[DiffItem, None, None]:
|
153
|
-
"""Be extra careful with the src and dst values. If you are off by one
|
154
|
-
parent directory, you will get a huge amount of false diffs."""
|
155
|
-
return self.impl.diff(
|
156
|
-
src=src,
|
157
|
-
dst=dst,
|
158
|
-
min_size=min_size,
|
159
|
-
max_size=max_size,
|
160
|
-
diff_option=diff_option,
|
161
|
-
fast_list=fast_list,
|
162
|
-
size_only=size_only,
|
163
|
-
checkers=checkers,
|
164
|
-
other_args=other_args,
|
165
|
-
)
|
166
|
-
|
167
|
-
def walk(
|
168
|
-
self,
|
169
|
-
path: Dir | Remote | str,
|
170
|
-
max_depth: int = -1,
|
171
|
-
breadth_first: bool = True,
|
172
|
-
order: Order = Order.NORMAL,
|
173
|
-
) -> Generator[DirListing, None, None]:
|
174
|
-
"""Walk through the given path recursively.
|
175
|
-
|
176
|
-
Args:
|
177
|
-
path: Remote path or Remote object to walk through
|
178
|
-
max_depth: Maximum depth to traverse (-1 for unlimited)
|
179
|
-
|
180
|
-
Yields:
|
181
|
-
DirListing: Directory listing for each directory encountered
|
182
|
-
"""
|
183
|
-
return self.impl.walk(
|
184
|
-
path=path, max_depth=max_depth, breadth_first=breadth_first, order=order
|
185
|
-
)
|
186
|
-
|
187
|
-
def scan_missing_folders(
|
188
|
-
self,
|
189
|
-
src: Dir | Remote | str,
|
190
|
-
dst: Dir | Remote | str,
|
191
|
-
max_depth: int = -1,
|
192
|
-
order: Order = Order.NORMAL,
|
193
|
-
) -> Generator[Dir, None, None]:
|
194
|
-
"""Walk through the given path recursively.
|
195
|
-
|
196
|
-
WORK IN PROGRESS!!
|
197
|
-
|
198
|
-
Args:
|
199
|
-
src: Source directory or Remote to walk through
|
200
|
-
dst: Destination directory or Remote to walk through
|
201
|
-
max_depth: Maximum depth to traverse (-1 for unlimited)
|
202
|
-
|
203
|
-
Yields:
|
204
|
-
DirListing: Directory listing for each directory encountered
|
205
|
-
"""
|
206
|
-
return self.impl.scan_missing_folders(
|
207
|
-
src=src, dst=dst, max_depth=max_depth, order=order
|
208
|
-
)
|
209
|
-
|
210
|
-
def cleanup(
|
211
|
-
self, path: str, other_args: list[str] | None = None
|
212
|
-
) -> CompletedProcess:
|
213
|
-
"""Cleanup any resources used by the Rclone instance."""
|
214
|
-
return self.impl.cleanup(path=path, other_args=other_args)
|
215
|
-
|
216
|
-
def copy_to(
|
217
|
-
self,
|
218
|
-
src: File | str,
|
219
|
-
dst: File | str,
|
220
|
-
check: bool | None = None,
|
221
|
-
verbose: bool | None = None,
|
222
|
-
other_args: list[str] | None = None,
|
223
|
-
) -> CompletedProcess:
|
224
|
-
"""Copy one file from source to destination.
|
225
|
-
|
226
|
-
Warning - slow.
|
227
|
-
|
228
|
-
"""
|
229
|
-
return self.impl.copy_to(
|
230
|
-
src=src, dst=dst, check=check, verbose=verbose, other_args=other_args
|
231
|
-
)
|
232
|
-
|
233
|
-
def copy_files(
|
234
|
-
self,
|
235
|
-
src: str,
|
236
|
-
dst: str,
|
237
|
-
files: list[str] | Path,
|
238
|
-
check: bool | None = None,
|
239
|
-
max_backlog: int | None = None,
|
240
|
-
verbose: bool | None = None,
|
241
|
-
checkers: int | None = None,
|
242
|
-
transfers: int | None = None,
|
243
|
-
low_level_retries: int | None = None,
|
244
|
-
retries: int | None = None,
|
245
|
-
retries_sleep: str | None = None,
|
246
|
-
metadata: bool | None = None,
|
247
|
-
timeout: str | None = None,
|
248
|
-
max_partition_workers: int | None = None,
|
249
|
-
multi_thread_streams: int | None = None,
|
250
|
-
other_args: list[str] | None = None,
|
251
|
-
) -> list[CompletedProcess]:
|
252
|
-
"""Copy multiple files from source to destination.
|
253
|
-
|
254
|
-
Args:
|
255
|
-
payload: Dictionary of source and destination file paths
|
256
|
-
"""
|
257
|
-
return self.impl.copy_files(
|
258
|
-
src=src,
|
259
|
-
dst=dst,
|
260
|
-
files=files,
|
261
|
-
check=check,
|
262
|
-
max_backlog=max_backlog,
|
263
|
-
verbose=verbose,
|
264
|
-
checkers=checkers,
|
265
|
-
transfers=transfers,
|
266
|
-
low_level_retries=low_level_retries,
|
267
|
-
retries=retries,
|
268
|
-
retries_sleep=retries_sleep,
|
269
|
-
metadata=metadata,
|
270
|
-
timeout=timeout,
|
271
|
-
max_partition_workers=max_partition_workers,
|
272
|
-
multi_thread_streams=multi_thread_streams,
|
273
|
-
other_args=other_args,
|
274
|
-
)
|
275
|
-
|
276
|
-
def copy(
|
277
|
-
self,
|
278
|
-
src: Dir | str,
|
279
|
-
dst: Dir | str,
|
280
|
-
check: bool | None = None,
|
281
|
-
transfers: int | None = None,
|
282
|
-
checkers: int | None = None,
|
283
|
-
multi_thread_streams: int | None = None,
|
284
|
-
low_level_retries: int | None = None,
|
285
|
-
retries: int | None = None,
|
286
|
-
other_args: list[str] | None = None,
|
287
|
-
) -> CompletedProcess:
|
288
|
-
"""Copy files from source to destination.
|
289
|
-
|
290
|
-
Args:
|
291
|
-
src: Source directory
|
292
|
-
dst: Destination directory
|
293
|
-
"""
|
294
|
-
return self.impl.copy(
|
295
|
-
src=src,
|
296
|
-
dst=dst,
|
297
|
-
check=check,
|
298
|
-
transfers=transfers,
|
299
|
-
checkers=checkers,
|
300
|
-
multi_thread_streams=multi_thread_streams,
|
301
|
-
low_level_retries=low_level_retries,
|
302
|
-
retries=retries,
|
303
|
-
other_args=other_args,
|
304
|
-
)
|
305
|
-
|
306
|
-
def purge(self, path: Dir | str) -> CompletedProcess:
|
307
|
-
"""Purge a directory"""
|
308
|
-
return self.impl.purge(path=path)
|
309
|
-
|
310
|
-
def delete_files(
|
311
|
-
self,
|
312
|
-
files: str | File | list[str] | list[File],
|
313
|
-
check: bool | None = None,
|
314
|
-
rmdirs=False,
|
315
|
-
verbose: bool | None = None,
|
316
|
-
max_partition_workers: int | None = None,
|
317
|
-
other_args: list[str] | None = None,
|
318
|
-
) -> CompletedProcess:
|
319
|
-
"""Delete a directory"""
|
320
|
-
return self.impl.delete_files(
|
321
|
-
files=files,
|
322
|
-
check=check,
|
323
|
-
rmdirs=rmdirs,
|
324
|
-
verbose=verbose,
|
325
|
-
max_partition_workers=max_partition_workers,
|
326
|
-
other_args=other_args,
|
327
|
-
)
|
328
|
-
|
329
|
-
def exists(self, path: Dir | Remote | str | File) -> bool:
|
330
|
-
"""Check if a file or directory exists."""
|
331
|
-
return self.impl.exists(path=path)
|
332
|
-
|
333
|
-
def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
|
334
|
-
"""Check if two directories are in sync."""
|
335
|
-
return self.impl.is_synced(src=src, dst=dst)
|
336
|
-
|
337
|
-
def modtime(self, src: str) -> str | Exception:
|
338
|
-
"""Get the modification time of a file or directory."""
|
339
|
-
return self.impl.modtime(src=src)
|
340
|
-
|
341
|
-
def modtime_dt(self, src: str) -> datetime | Exception:
|
342
|
-
"""Get the modification time of a file or directory."""
|
343
|
-
return self.impl.modtime_dt(src=src)
|
344
|
-
|
345
|
-
def write_text(
|
346
|
-
self,
|
347
|
-
text: str,
|
348
|
-
dst: str,
|
349
|
-
) -> Exception | None:
|
350
|
-
"""Write text to a file."""
|
351
|
-
return self.impl.write_text(text=text, dst=dst)
|
352
|
-
|
353
|
-
def write_bytes(
|
354
|
-
self,
|
355
|
-
data: bytes,
|
356
|
-
dst: str,
|
357
|
-
) -> Exception | None:
|
358
|
-
"""Write bytes to a file."""
|
359
|
-
return self.impl.write_bytes(data=data, dst=dst)
|
360
|
-
|
361
|
-
def read_bytes(self, src: str) -> bytes | Exception:
|
362
|
-
"""Read bytes from a file."""
|
363
|
-
return self.impl.read_bytes(src=src)
|
364
|
-
|
365
|
-
def read_text(self, src: str) -> str | Exception:
|
366
|
-
"""Read text from a file."""
|
367
|
-
return self.impl.read_text(src=src)
|
368
|
-
|
369
|
-
def copy_bytes(
|
370
|
-
self,
|
371
|
-
src: str,
|
372
|
-
offset: int | SizeSuffix,
|
373
|
-
length: int | SizeSuffix,
|
374
|
-
outfile: Path,
|
375
|
-
other_args: list[str] | None = None,
|
376
|
-
) -> Exception | None:
|
377
|
-
"""Copy a slice of bytes from the src file to dst."""
|
378
|
-
return self.impl.copy_bytes(
|
379
|
-
src=src,
|
380
|
-
offset=offset,
|
381
|
-
length=length,
|
382
|
-
outfile=outfile,
|
383
|
-
other_args=other_args,
|
384
|
-
)
|
385
|
-
|
386
|
-
def copy_dir(
|
387
|
-
self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
|
388
|
-
) -> CompletedProcess:
|
389
|
-
"""Copy a directory from source to destination."""
|
390
|
-
# convert src to str, also dst
|
391
|
-
return self.impl.copy_dir(src=src, dst=dst, args=args)
|
392
|
-
|
393
|
-
def copy_remote(
|
394
|
-
self, src: Remote, dst: Remote, args: list[str] | None = None
|
395
|
-
) -> CompletedProcess:
|
396
|
-
"""Copy a remote to another remote."""
|
397
|
-
return self.impl.copy_remote(src=src, dst=dst, args=args)
|
398
|
-
|
399
|
-
def copy_file_s3_resumable(
|
400
|
-
self,
|
401
|
-
src: str, # src:/Bucket/path/myfile.large.zst
|
402
|
-
dst: str, # dst:/Bucket/path/myfile.large.zst
|
403
|
-
part_infos: list[PartInfo] | None = None,
|
404
|
-
upload_threads: int = 8, # Number of reader and writer threads to use
|
405
|
-
merge_threads: int = 4, # Number of threads to use for merging the parts
|
406
|
-
) -> Exception | None:
|
407
|
-
"""Copy a file in parts."""
|
408
|
-
return self.impl.copy_file_s3_resumable(
|
409
|
-
src=src,
|
410
|
-
dst=dst,
|
411
|
-
part_infos=part_infos,
|
412
|
-
upload_threads=upload_threads,
|
413
|
-
merge_threads=merge_threads,
|
414
|
-
)
|
415
|
-
|
416
|
-
def mount(
|
417
|
-
self,
|
418
|
-
src: Remote | Dir | str,
|
419
|
-
outdir: Path,
|
420
|
-
allow_writes: bool | None = False,
|
421
|
-
use_links: bool | None = None,
|
422
|
-
vfs_cache_mode: str | None = None,
|
423
|
-
verbose: bool | None = None,
|
424
|
-
cache_dir: Path | None = None,
|
425
|
-
cache_dir_delete_on_exit: bool | None = None,
|
426
|
-
log: Path | None = None,
|
427
|
-
other_args: list[str] | None = None,
|
428
|
-
) -> Mount:
|
429
|
-
"""Mount a remote or directory to a local path.
|
430
|
-
|
431
|
-
Args:
|
432
|
-
src: Remote or directory to mount
|
433
|
-
outdir: Local path to mount to
|
434
|
-
|
435
|
-
Returns:
|
436
|
-
CompletedProcess from the mount command execution
|
437
|
-
|
438
|
-
Raises:
|
439
|
-
subprocess.CalledProcessError: If the mount operation fails
|
440
|
-
"""
|
441
|
-
return self.impl.mount(
|
442
|
-
src=src,
|
443
|
-
outdir=outdir,
|
444
|
-
allow_writes=allow_writes,
|
445
|
-
use_links=use_links,
|
446
|
-
vfs_cache_mode=vfs_cache_mode,
|
447
|
-
verbose=verbose,
|
448
|
-
cache_dir=cache_dir,
|
449
|
-
cache_dir_delete_on_exit=cache_dir_delete_on_exit,
|
450
|
-
log=log,
|
451
|
-
other_args=other_args,
|
452
|
-
)
|
453
|
-
|
454
|
-
def serve_http(
|
455
|
-
self,
|
456
|
-
src: str,
|
457
|
-
addr: str = "localhost:8080",
|
458
|
-
other_args: list[str] | None = None,
|
459
|
-
) -> HttpServer:
|
460
|
-
"""Serve a remote or directory via HTTP. The returned HttpServer has a client which can be used to
|
461
|
-
fetch files or parts.
|
462
|
-
|
463
|
-
Args:
|
464
|
-
src: Remote or directory to serve
|
465
|
-
addr: Network address and port to serve on (default: localhost:8080)
|
466
|
-
"""
|
467
|
-
return self.impl.serve_http(src=src, addr=addr, other_args=other_args)
|
468
|
-
|
469
|
-
def size_files(
|
470
|
-
self,
|
471
|
-
src: str,
|
472
|
-
files: list[str],
|
473
|
-
fast_list: bool = False, # Recommend that this is False
|
474
|
-
other_args: list[str] | None = None,
|
475
|
-
check: bool | None = False,
|
476
|
-
verbose: bool | None = None,
|
477
|
-
) -> SizeResult | Exception:
|
478
|
-
"""Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
|
479
|
-
return self.impl.size_files(
|
480
|
-
src=src,
|
481
|
-
files=files,
|
482
|
-
fast_list=fast_list,
|
483
|
-
other_args=other_args,
|
484
|
-
check=check,
|
485
|
-
verbose=verbose,
|
486
|
-
)
|
487
|
-
|
488
|
-
def size_file(self, src: str) -> SizeSuffix | Exception:
|
489
|
-
"""Get the size of a file."""
|
490
|
-
return self.impl.size_file(src=src)
|
491
|
-
|
492
|
-
|
493
|
-
__all__ = [
|
494
|
-
"Rclone",
|
495
|
-
"File",
|
496
|
-
"Config",
|
497
|
-
"Remote",
|
498
|
-
"Dir",
|
499
|
-
"RPath",
|
500
|
-
"DirListing",
|
501
|
-
"FileList",
|
502
|
-
"FileItem",
|
503
|
-
"Process",
|
504
|
-
"DiffItem",
|
505
|
-
"DiffType",
|
506
|
-
"rclone_verbose",
|
507
|
-
"CompletedProcess",
|
508
|
-
"DiffOption",
|
509
|
-
"ListingOption",
|
510
|
-
"Order",
|
511
|
-
"ListingOption",
|
512
|
-
"SizeResult",
|
513
|
-
"Parsed",
|
514
|
-
"Section",
|
515
|
-
"MultiUploadResult",
|
516
|
-
"SizeSuffix",
|
517
|
-
"configure_logging",
|
518
|
-
"log",
|
519
|
-
"HttpServer",
|
520
|
-
"Range",
|
521
|
-
"HttpFetcher",
|
522
|
-
"PartInfo",
|
523
|
-
]
|
1
|
+
# Import logging module to activate default configuration
|
2
|
+
|
3
|
+
from datetime import datetime
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import Generator
|
6
|
+
|
7
|
+
from rclone_api import log
|
8
|
+
|
9
|
+
from .completed_process import CompletedProcess
|
10
|
+
from .config import Config, Parsed, Section
|
11
|
+
from .diff import DiffItem, DiffOption, DiffType
|
12
|
+
from .dir import Dir
|
13
|
+
from .dir_listing import DirListing
|
14
|
+
from .file import File, FileItem
|
15
|
+
from .file_stream import FilesStream
|
16
|
+
from .filelist import FileList
|
17
|
+
from .http_server import HttpFetcher, HttpServer, Range
|
18
|
+
|
19
|
+
# Import the configure_logging function to make it available at package level
|
20
|
+
from .log import configure_logging, setup_default_logging
|
21
|
+
from .mount import Mount
|
22
|
+
from .process import Process
|
23
|
+
from .remote import Remote
|
24
|
+
from .rpath import RPath
|
25
|
+
from .s3.types import MultiUploadResult
|
26
|
+
from .types import ListingOption, Order, PartInfo, SizeResult, SizeSuffix
|
27
|
+
|
28
|
+
setup_default_logging()
|
29
|
+
|
30
|
+
|
31
|
+
def rclone_verbose(val: bool | None) -> bool:
|
32
|
+
from rclone_api.rclone_impl import rclone_verbose as _rclone_verbose
|
33
|
+
|
34
|
+
return _rclone_verbose(val)
|
35
|
+
|
36
|
+
|
37
|
+
class Rclone:
|
38
|
+
def __init__(
|
39
|
+
self, rclone_conf: Path | Config, rclone_exe: Path | None = None
|
40
|
+
) -> None:
|
41
|
+
from rclone_api.rclone_impl import RcloneImpl
|
42
|
+
|
43
|
+
self.impl: RcloneImpl = RcloneImpl(rclone_conf, rclone_exe)
|
44
|
+
|
45
|
+
def webgui(self, other_args: list[str] | None = None) -> Process:
|
46
|
+
"""Launch the Rclone web GUI."""
|
47
|
+
return self.impl.webgui(other_args=other_args)
|
48
|
+
|
49
|
+
def launch_server(
|
50
|
+
self,
|
51
|
+
addr: str,
|
52
|
+
user: str | None = None,
|
53
|
+
password: str | None = None,
|
54
|
+
other_args: list[str] | None = None,
|
55
|
+
) -> Process:
|
56
|
+
"""Launch the Rclone server so it can receive commands"""
|
57
|
+
return self.impl.launch_server(
|
58
|
+
addr=addr, user=user, password=password, other_args=other_args
|
59
|
+
)
|
60
|
+
|
61
|
+
def remote_control(
|
62
|
+
self,
|
63
|
+
addr: str,
|
64
|
+
user: str | None = None,
|
65
|
+
password: str | None = None,
|
66
|
+
capture: bool | None = None,
|
67
|
+
other_args: list[str] | None = None,
|
68
|
+
) -> CompletedProcess:
|
69
|
+
return self.impl.remote_control(
|
70
|
+
addr=addr,
|
71
|
+
user=user,
|
72
|
+
password=password,
|
73
|
+
capture=capture,
|
74
|
+
other_args=other_args,
|
75
|
+
)
|
76
|
+
|
77
|
+
def obscure(self, password: str) -> str:
|
78
|
+
"""Obscure a password for use in rclone config files."""
|
79
|
+
return self.impl.obscure(password=password)
|
80
|
+
|
81
|
+
def ls_stream(
|
82
|
+
self,
|
83
|
+
path: str,
|
84
|
+
max_depth: int = -1,
|
85
|
+
fast_list: bool = False,
|
86
|
+
) -> FilesStream:
|
87
|
+
"""
|
88
|
+
List files in the given path
|
89
|
+
|
90
|
+
Args:
|
91
|
+
src: Remote path to list
|
92
|
+
max_depth: Maximum recursion depth (-1 for unlimited)
|
93
|
+
fast_list: Use fast list (only use when getting THE entire data repository from the root/bucket, or it's small)
|
94
|
+
"""
|
95
|
+
return self.impl.ls_stream(path=path, max_depth=max_depth, fast_list=fast_list)
|
96
|
+
|
97
|
+
def save_to_db(
|
98
|
+
self,
|
99
|
+
src: str,
|
100
|
+
db_url: str,
|
101
|
+
max_depth: int = -1,
|
102
|
+
fast_list: bool = False,
|
103
|
+
) -> None:
|
104
|
+
"""
|
105
|
+
Save files to a database (sqlite, mysql, postgres)
|
106
|
+
|
107
|
+
Args:
|
108
|
+
src: Remote path to list, this will be used to populate an entire table, so always use the root-most path.
|
109
|
+
db_url: Database URL, like sqlite:///data.db or mysql://user:pass@localhost/db or postgres://user:pass@localhost/db
|
110
|
+
max_depth: Maximum depth to traverse (-1 for unlimited)
|
111
|
+
fast_list: Use fast list (only use when getting THE entire data repository from the root/bucket)
|
112
|
+
|
113
|
+
"""
|
114
|
+
return self.impl.save_to_db(
|
115
|
+
src=src, db_url=db_url, max_depth=max_depth, fast_list=fast_list
|
116
|
+
)
|
117
|
+
|
118
|
+
def ls(
|
119
|
+
self,
|
120
|
+
path: Dir | Remote | str | None = None,
|
121
|
+
max_depth: int | None = None,
|
122
|
+
glob: str | None = None,
|
123
|
+
order: Order = Order.NORMAL,
|
124
|
+
listing_option: ListingOption = ListingOption.ALL,
|
125
|
+
) -> DirListing:
|
126
|
+
return self.impl.ls(
|
127
|
+
path=path,
|
128
|
+
max_depth=max_depth,
|
129
|
+
glob=glob,
|
130
|
+
order=order,
|
131
|
+
listing_option=listing_option,
|
132
|
+
)
|
133
|
+
|
134
|
+
def listremotes(self) -> list[Remote]:
|
135
|
+
return self.impl.listremotes()
|
136
|
+
|
137
|
+
def diff(
|
138
|
+
self,
|
139
|
+
src: str,
|
140
|
+
dst: str,
|
141
|
+
min_size: (
|
142
|
+
str | None
|
143
|
+
) = None, # e. g. "1MB" - see rclone documentation: https://rclone.org/commands/rclone_check/
|
144
|
+
max_size: (
|
145
|
+
str | None
|
146
|
+
) = None, # e. g. "1GB" - see rclone documentation: https://rclone.org/commands/rclone_check/
|
147
|
+
diff_option: DiffOption = DiffOption.COMBINED,
|
148
|
+
fast_list: bool = True,
|
149
|
+
size_only: bool | None = None,
|
150
|
+
checkers: int | None = None,
|
151
|
+
other_args: list[str] | None = None,
|
152
|
+
) -> Generator[DiffItem, None, None]:
|
153
|
+
"""Be extra careful with the src and dst values. If you are off by one
|
154
|
+
parent directory, you will get a huge amount of false diffs."""
|
155
|
+
return self.impl.diff(
|
156
|
+
src=src,
|
157
|
+
dst=dst,
|
158
|
+
min_size=min_size,
|
159
|
+
max_size=max_size,
|
160
|
+
diff_option=diff_option,
|
161
|
+
fast_list=fast_list,
|
162
|
+
size_only=size_only,
|
163
|
+
checkers=checkers,
|
164
|
+
other_args=other_args,
|
165
|
+
)
|
166
|
+
|
167
|
+
def walk(
|
168
|
+
self,
|
169
|
+
path: Dir | Remote | str,
|
170
|
+
max_depth: int = -1,
|
171
|
+
breadth_first: bool = True,
|
172
|
+
order: Order = Order.NORMAL,
|
173
|
+
) -> Generator[DirListing, None, None]:
|
174
|
+
"""Walk through the given path recursively.
|
175
|
+
|
176
|
+
Args:
|
177
|
+
path: Remote path or Remote object to walk through
|
178
|
+
max_depth: Maximum depth to traverse (-1 for unlimited)
|
179
|
+
|
180
|
+
Yields:
|
181
|
+
DirListing: Directory listing for each directory encountered
|
182
|
+
"""
|
183
|
+
return self.impl.walk(
|
184
|
+
path=path, max_depth=max_depth, breadth_first=breadth_first, order=order
|
185
|
+
)
|
186
|
+
|
187
|
+
def scan_missing_folders(
|
188
|
+
self,
|
189
|
+
src: Dir | Remote | str,
|
190
|
+
dst: Dir | Remote | str,
|
191
|
+
max_depth: int = -1,
|
192
|
+
order: Order = Order.NORMAL,
|
193
|
+
) -> Generator[Dir, None, None]:
|
194
|
+
"""Walk through the given path recursively.
|
195
|
+
|
196
|
+
WORK IN PROGRESS!!
|
197
|
+
|
198
|
+
Args:
|
199
|
+
src: Source directory or Remote to walk through
|
200
|
+
dst: Destination directory or Remote to walk through
|
201
|
+
max_depth: Maximum depth to traverse (-1 for unlimited)
|
202
|
+
|
203
|
+
Yields:
|
204
|
+
DirListing: Directory listing for each directory encountered
|
205
|
+
"""
|
206
|
+
return self.impl.scan_missing_folders(
|
207
|
+
src=src, dst=dst, max_depth=max_depth, order=order
|
208
|
+
)
|
209
|
+
|
210
|
+
def cleanup(
|
211
|
+
self, path: str, other_args: list[str] | None = None
|
212
|
+
) -> CompletedProcess:
|
213
|
+
"""Cleanup any resources used by the Rclone instance."""
|
214
|
+
return self.impl.cleanup(path=path, other_args=other_args)
|
215
|
+
|
216
|
+
def copy_to(
|
217
|
+
self,
|
218
|
+
src: File | str,
|
219
|
+
dst: File | str,
|
220
|
+
check: bool | None = None,
|
221
|
+
verbose: bool | None = None,
|
222
|
+
other_args: list[str] | None = None,
|
223
|
+
) -> CompletedProcess:
|
224
|
+
"""Copy one file from source to destination.
|
225
|
+
|
226
|
+
Warning - slow.
|
227
|
+
|
228
|
+
"""
|
229
|
+
return self.impl.copy_to(
|
230
|
+
src=src, dst=dst, check=check, verbose=verbose, other_args=other_args
|
231
|
+
)
|
232
|
+
|
233
|
+
def copy_files(
|
234
|
+
self,
|
235
|
+
src: str,
|
236
|
+
dst: str,
|
237
|
+
files: list[str] | Path,
|
238
|
+
check: bool | None = None,
|
239
|
+
max_backlog: int | None = None,
|
240
|
+
verbose: bool | None = None,
|
241
|
+
checkers: int | None = None,
|
242
|
+
transfers: int | None = None,
|
243
|
+
low_level_retries: int | None = None,
|
244
|
+
retries: int | None = None,
|
245
|
+
retries_sleep: str | None = None,
|
246
|
+
metadata: bool | None = None,
|
247
|
+
timeout: str | None = None,
|
248
|
+
max_partition_workers: int | None = None,
|
249
|
+
multi_thread_streams: int | None = None,
|
250
|
+
other_args: list[str] | None = None,
|
251
|
+
) -> list[CompletedProcess]:
|
252
|
+
"""Copy multiple files from source to destination.
|
253
|
+
|
254
|
+
Args:
|
255
|
+
payload: Dictionary of source and destination file paths
|
256
|
+
"""
|
257
|
+
return self.impl.copy_files(
|
258
|
+
src=src,
|
259
|
+
dst=dst,
|
260
|
+
files=files,
|
261
|
+
check=check,
|
262
|
+
max_backlog=max_backlog,
|
263
|
+
verbose=verbose,
|
264
|
+
checkers=checkers,
|
265
|
+
transfers=transfers,
|
266
|
+
low_level_retries=low_level_retries,
|
267
|
+
retries=retries,
|
268
|
+
retries_sleep=retries_sleep,
|
269
|
+
metadata=metadata,
|
270
|
+
timeout=timeout,
|
271
|
+
max_partition_workers=max_partition_workers,
|
272
|
+
multi_thread_streams=multi_thread_streams,
|
273
|
+
other_args=other_args,
|
274
|
+
)
|
275
|
+
|
276
|
+
def copy(
|
277
|
+
self,
|
278
|
+
src: Dir | str,
|
279
|
+
dst: Dir | str,
|
280
|
+
check: bool | None = None,
|
281
|
+
transfers: int | None = None,
|
282
|
+
checkers: int | None = None,
|
283
|
+
multi_thread_streams: int | None = None,
|
284
|
+
low_level_retries: int | None = None,
|
285
|
+
retries: int | None = None,
|
286
|
+
other_args: list[str] | None = None,
|
287
|
+
) -> CompletedProcess:
|
288
|
+
"""Copy files from source to destination.
|
289
|
+
|
290
|
+
Args:
|
291
|
+
src: Source directory
|
292
|
+
dst: Destination directory
|
293
|
+
"""
|
294
|
+
return self.impl.copy(
|
295
|
+
src=src,
|
296
|
+
dst=dst,
|
297
|
+
check=check,
|
298
|
+
transfers=transfers,
|
299
|
+
checkers=checkers,
|
300
|
+
multi_thread_streams=multi_thread_streams,
|
301
|
+
low_level_retries=low_level_retries,
|
302
|
+
retries=retries,
|
303
|
+
other_args=other_args,
|
304
|
+
)
|
305
|
+
|
306
|
+
def purge(self, path: Dir | str) -> CompletedProcess:
|
307
|
+
"""Purge a directory"""
|
308
|
+
return self.impl.purge(path=path)
|
309
|
+
|
310
|
+
def delete_files(
|
311
|
+
self,
|
312
|
+
files: str | File | list[str] | list[File],
|
313
|
+
check: bool | None = None,
|
314
|
+
rmdirs=False,
|
315
|
+
verbose: bool | None = None,
|
316
|
+
max_partition_workers: int | None = None,
|
317
|
+
other_args: list[str] | None = None,
|
318
|
+
) -> CompletedProcess:
|
319
|
+
"""Delete a directory"""
|
320
|
+
return self.impl.delete_files(
|
321
|
+
files=files,
|
322
|
+
check=check,
|
323
|
+
rmdirs=rmdirs,
|
324
|
+
verbose=verbose,
|
325
|
+
max_partition_workers=max_partition_workers,
|
326
|
+
other_args=other_args,
|
327
|
+
)
|
328
|
+
|
329
|
+
def exists(self, path: Dir | Remote | str | File) -> bool:
|
330
|
+
"""Check if a file or directory exists."""
|
331
|
+
return self.impl.exists(path=path)
|
332
|
+
|
333
|
+
def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
|
334
|
+
"""Check if two directories are in sync."""
|
335
|
+
return self.impl.is_synced(src=src, dst=dst)
|
336
|
+
|
337
|
+
def modtime(self, src: str) -> str | Exception:
|
338
|
+
"""Get the modification time of a file or directory."""
|
339
|
+
return self.impl.modtime(src=src)
|
340
|
+
|
341
|
+
def modtime_dt(self, src: str) -> datetime | Exception:
|
342
|
+
"""Get the modification time of a file or directory."""
|
343
|
+
return self.impl.modtime_dt(src=src)
|
344
|
+
|
345
|
+
def write_text(
|
346
|
+
self,
|
347
|
+
text: str,
|
348
|
+
dst: str,
|
349
|
+
) -> Exception | None:
|
350
|
+
"""Write text to a file."""
|
351
|
+
return self.impl.write_text(text=text, dst=dst)
|
352
|
+
|
353
|
+
def write_bytes(
|
354
|
+
self,
|
355
|
+
data: bytes,
|
356
|
+
dst: str,
|
357
|
+
) -> Exception | None:
|
358
|
+
"""Write bytes to a file."""
|
359
|
+
return self.impl.write_bytes(data=data, dst=dst)
|
360
|
+
|
361
|
+
def read_bytes(self, src: str) -> bytes | Exception:
|
362
|
+
"""Read bytes from a file."""
|
363
|
+
return self.impl.read_bytes(src=src)
|
364
|
+
|
365
|
+
def read_text(self, src: str) -> str | Exception:
|
366
|
+
"""Read text from a file."""
|
367
|
+
return self.impl.read_text(src=src)
|
368
|
+
|
369
|
+
def copy_bytes(
|
370
|
+
self,
|
371
|
+
src: str,
|
372
|
+
offset: int | SizeSuffix,
|
373
|
+
length: int | SizeSuffix,
|
374
|
+
outfile: Path,
|
375
|
+
other_args: list[str] | None = None,
|
376
|
+
) -> Exception | None:
|
377
|
+
"""Copy a slice of bytes from the src file to dst."""
|
378
|
+
return self.impl.copy_bytes(
|
379
|
+
src=src,
|
380
|
+
offset=offset,
|
381
|
+
length=length,
|
382
|
+
outfile=outfile,
|
383
|
+
other_args=other_args,
|
384
|
+
)
|
385
|
+
|
386
|
+
def copy_dir(
|
387
|
+
self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
|
388
|
+
) -> CompletedProcess:
|
389
|
+
"""Copy a directory from source to destination."""
|
390
|
+
# convert src to str, also dst
|
391
|
+
return self.impl.copy_dir(src=src, dst=dst, args=args)
|
392
|
+
|
393
|
+
def copy_remote(
|
394
|
+
self, src: Remote, dst: Remote, args: list[str] | None = None
|
395
|
+
) -> CompletedProcess:
|
396
|
+
"""Copy a remote to another remote."""
|
397
|
+
return self.impl.copy_remote(src=src, dst=dst, args=args)
|
398
|
+
|
399
|
+
def copy_file_s3_resumable(
|
400
|
+
self,
|
401
|
+
src: str, # src:/Bucket/path/myfile.large.zst
|
402
|
+
dst: str, # dst:/Bucket/path/myfile.large.zst
|
403
|
+
part_infos: list[PartInfo] | None = None,
|
404
|
+
upload_threads: int = 8, # Number of reader and writer threads to use
|
405
|
+
merge_threads: int = 4, # Number of threads to use for merging the parts
|
406
|
+
) -> Exception | None:
|
407
|
+
"""Copy a file in parts."""
|
408
|
+
return self.impl.copy_file_s3_resumable(
|
409
|
+
src=src,
|
410
|
+
dst=dst,
|
411
|
+
part_infos=part_infos,
|
412
|
+
upload_threads=upload_threads,
|
413
|
+
merge_threads=merge_threads,
|
414
|
+
)
|
415
|
+
|
416
|
+
def mount(
|
417
|
+
self,
|
418
|
+
src: Remote | Dir | str,
|
419
|
+
outdir: Path,
|
420
|
+
allow_writes: bool | None = False,
|
421
|
+
use_links: bool | None = None,
|
422
|
+
vfs_cache_mode: str | None = None,
|
423
|
+
verbose: bool | None = None,
|
424
|
+
cache_dir: Path | None = None,
|
425
|
+
cache_dir_delete_on_exit: bool | None = None,
|
426
|
+
log: Path | None = None,
|
427
|
+
other_args: list[str] | None = None,
|
428
|
+
) -> Mount:
|
429
|
+
"""Mount a remote or directory to a local path.
|
430
|
+
|
431
|
+
Args:
|
432
|
+
src: Remote or directory to mount
|
433
|
+
outdir: Local path to mount to
|
434
|
+
|
435
|
+
Returns:
|
436
|
+
CompletedProcess from the mount command execution
|
437
|
+
|
438
|
+
Raises:
|
439
|
+
subprocess.CalledProcessError: If the mount operation fails
|
440
|
+
"""
|
441
|
+
return self.impl.mount(
|
442
|
+
src=src,
|
443
|
+
outdir=outdir,
|
444
|
+
allow_writes=allow_writes,
|
445
|
+
use_links=use_links,
|
446
|
+
vfs_cache_mode=vfs_cache_mode,
|
447
|
+
verbose=verbose,
|
448
|
+
cache_dir=cache_dir,
|
449
|
+
cache_dir_delete_on_exit=cache_dir_delete_on_exit,
|
450
|
+
log=log,
|
451
|
+
other_args=other_args,
|
452
|
+
)
|
453
|
+
|
454
|
+
def serve_http(
|
455
|
+
self,
|
456
|
+
src: str,
|
457
|
+
addr: str = "localhost:8080",
|
458
|
+
other_args: list[str] | None = None,
|
459
|
+
) -> HttpServer:
|
460
|
+
"""Serve a remote or directory via HTTP. The returned HttpServer has a client which can be used to
|
461
|
+
fetch files or parts.
|
462
|
+
|
463
|
+
Args:
|
464
|
+
src: Remote or directory to serve
|
465
|
+
addr: Network address and port to serve on (default: localhost:8080)
|
466
|
+
"""
|
467
|
+
return self.impl.serve_http(src=src, addr=addr, other_args=other_args)
|
468
|
+
|
469
|
+
def size_files(
|
470
|
+
self,
|
471
|
+
src: str,
|
472
|
+
files: list[str],
|
473
|
+
fast_list: bool = False, # Recommend that this is False
|
474
|
+
other_args: list[str] | None = None,
|
475
|
+
check: bool | None = False,
|
476
|
+
verbose: bool | None = None,
|
477
|
+
) -> SizeResult | Exception:
|
478
|
+
"""Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
|
479
|
+
return self.impl.size_files(
|
480
|
+
src=src,
|
481
|
+
files=files,
|
482
|
+
fast_list=fast_list,
|
483
|
+
other_args=other_args,
|
484
|
+
check=check,
|
485
|
+
verbose=verbose,
|
486
|
+
)
|
487
|
+
|
488
|
+
def size_file(self, src: str) -> SizeSuffix | Exception:
|
489
|
+
"""Get the size of a file."""
|
490
|
+
return self.impl.size_file(src=src)
|
491
|
+
|
492
|
+
|
493
|
+
__all__ = [
|
494
|
+
"Rclone",
|
495
|
+
"File",
|
496
|
+
"Config",
|
497
|
+
"Remote",
|
498
|
+
"Dir",
|
499
|
+
"RPath",
|
500
|
+
"DirListing",
|
501
|
+
"FileList",
|
502
|
+
"FileItem",
|
503
|
+
"Process",
|
504
|
+
"DiffItem",
|
505
|
+
"DiffType",
|
506
|
+
"rclone_verbose",
|
507
|
+
"CompletedProcess",
|
508
|
+
"DiffOption",
|
509
|
+
"ListingOption",
|
510
|
+
"Order",
|
511
|
+
"ListingOption",
|
512
|
+
"SizeResult",
|
513
|
+
"Parsed",
|
514
|
+
"Section",
|
515
|
+
"MultiUploadResult",
|
516
|
+
"SizeSuffix",
|
517
|
+
"configure_logging",
|
518
|
+
"log",
|
519
|
+
"HttpServer",
|
520
|
+
"Range",
|
521
|
+
"HttpFetcher",
|
522
|
+
"PartInfo",
|
523
|
+
]
|