rclone-api 1.0.26__py2.py3-none-any.whl → 1.0.28__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/__init__.py +3 -0
- rclone_api/diff.py +119 -0
- rclone_api/dir_listing.py +16 -0
- rclone_api/exec.py +2 -1
- rclone_api/process.py +8 -1
- rclone_api/rclone.py +24 -2
- {rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/METADATA +3 -3
- {rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/RECORD +12 -11
- {rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/LICENSE +0 -0
- {rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/WHEEL +0 -0
- {rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/top_level.txt +0 -0
rclone_api/__init__.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
from .config import Config
|
2
|
+
from .diff import DiffItem, DiffType
|
2
3
|
from .dir import Dir
|
3
4
|
from .dir_listing import DirListing
|
4
5
|
from .file import File
|
@@ -18,4 +19,6 @@ __all__ = [
|
|
18
19
|
"DirListing",
|
19
20
|
"FileList",
|
20
21
|
"Process",
|
22
|
+
"DiffItem",
|
23
|
+
"DiffType",
|
21
24
|
]
|
rclone_api/diff.py
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
from dataclasses import dataclass
|
2
|
+
from enum import Enum
|
3
|
+
from queue import Queue
|
4
|
+
from threading import Thread
|
5
|
+
from typing import Generator
|
6
|
+
|
7
|
+
from rclone_api.process import Process
|
8
|
+
|
9
|
+
|
10
|
+
class DiffType(Enum):
|
11
|
+
EQUAL = "="
|
12
|
+
MISSING_ON_SRC = (
|
13
|
+
"-" # means path was missing on the source, so only in the destination
|
14
|
+
)
|
15
|
+
MISSING_ON_DST = (
|
16
|
+
"+" # means path was missing on the destination, so only in the source
|
17
|
+
)
|
18
|
+
DIFFERENT = "*" # means path was present in source and destination but different.
|
19
|
+
ERROR = "!" # means there was an error
|
20
|
+
|
21
|
+
|
22
|
+
@dataclass
|
23
|
+
class DiffItem:
|
24
|
+
type: DiffType
|
25
|
+
path: str
|
26
|
+
src_prefix: str
|
27
|
+
dst_prefix: str
|
28
|
+
|
29
|
+
def __str__(self) -> str:
|
30
|
+
return f"{self.type.value} {self.path}"
|
31
|
+
|
32
|
+
def __repr__(self) -> str:
|
33
|
+
return f"{self.type.name} {self.path}"
|
34
|
+
|
35
|
+
|
36
|
+
def _classify_diff(line: str, src_slug: str, dst_slug: str) -> DiffItem | None:
|
37
|
+
def _new(type: DiffType, path: str) -> DiffItem:
|
38
|
+
return DiffItem(type, path, src_prefix=src_slug, dst_prefix=dst_slug)
|
39
|
+
|
40
|
+
suffix = line[1:].strip() if len(line) > 0 else ""
|
41
|
+
if line.startswith(DiffType.EQUAL.value):
|
42
|
+
return _new(DiffType.EQUAL, suffix)
|
43
|
+
if line.startswith(DiffType.MISSING_ON_SRC.value):
|
44
|
+
return _new(DiffType.MISSING_ON_SRC, suffix)
|
45
|
+
if line.startswith(DiffType.MISSING_ON_DST.value):
|
46
|
+
# return DiffItem(DiffType.MISSING_ON_DST, f"{src_slug}/{suffix}")
|
47
|
+
return _new(DiffType.MISSING_ON_DST, suffix)
|
48
|
+
if line.startswith(DiffType.DIFFERENT.value):
|
49
|
+
# return DiffItem(DiffType.DIFFERENT, suffix)
|
50
|
+
return _new(DiffType.DIFFERENT, suffix)
|
51
|
+
if line.startswith(DiffType.ERROR.value):
|
52
|
+
# return DiffItem(DiffType.ERROR, suffix)
|
53
|
+
return _new(DiffType.ERROR, suffix)
|
54
|
+
return None
|
55
|
+
|
56
|
+
|
57
|
+
def _async_diff_stream_from_running_process(
|
58
|
+
running_process: Process,
|
59
|
+
src_slug: str,
|
60
|
+
dst_slug: str,
|
61
|
+
output: Queue[DiffItem | None],
|
62
|
+
) -> None:
|
63
|
+
count = 0
|
64
|
+
first_few_lines: list[str] = []
|
65
|
+
try:
|
66
|
+
assert running_process.stdout is not None
|
67
|
+
n_max = 10
|
68
|
+
for line in iter(running_process.stdout.readline, b""):
|
69
|
+
try:
|
70
|
+
line_str = line.decode("utf-8").strip()
|
71
|
+
if len(first_few_lines) < n_max:
|
72
|
+
first_few_lines.append(line_str)
|
73
|
+
# _classify_line_type
|
74
|
+
diff_item: DiffItem | None = _classify_diff(
|
75
|
+
line_str, src_slug, dst_slug
|
76
|
+
)
|
77
|
+
if diff_item is None:
|
78
|
+
# Some other output that we don't care about, debug print etc.
|
79
|
+
continue
|
80
|
+
output.put(diff_item)
|
81
|
+
count += 1
|
82
|
+
# print(f"unhandled: {line_str}")
|
83
|
+
except UnicodeDecodeError:
|
84
|
+
print("UnicodeDecodeError")
|
85
|
+
continue
|
86
|
+
output.put(None)
|
87
|
+
print("done")
|
88
|
+
except KeyboardInterrupt:
|
89
|
+
import _thread
|
90
|
+
|
91
|
+
print("KeyboardInterrupt")
|
92
|
+
output.put(None)
|
93
|
+
_thread.interrupt_main()
|
94
|
+
if count == 0:
|
95
|
+
first_lines_str = "\n".join(first_few_lines)
|
96
|
+
raise ValueError(
|
97
|
+
f"No output from rclone check, first few lines: {first_lines_str}"
|
98
|
+
)
|
99
|
+
|
100
|
+
|
101
|
+
def diff_stream_from_running_process(
|
102
|
+
running_process: Process,
|
103
|
+
src_slug: str,
|
104
|
+
dst_slug: str,
|
105
|
+
) -> Generator[DiffItem, None, None]:
|
106
|
+
output: Queue[DiffItem | None] = Queue()
|
107
|
+
# process_output_to_diff_stream(running_process, src_slug, dst_slug, output)
|
108
|
+
thread = Thread(
|
109
|
+
target=_async_diff_stream_from_running_process,
|
110
|
+
args=(running_process, src_slug, dst_slug, output),
|
111
|
+
daemon=True,
|
112
|
+
)
|
113
|
+
thread.start()
|
114
|
+
while True:
|
115
|
+
item = output.get()
|
116
|
+
if item is None:
|
117
|
+
break
|
118
|
+
yield item
|
119
|
+
thread.join(timeout=5)
|
rclone_api/dir_listing.py
CHANGED
@@ -1,8 +1,22 @@
|
|
1
1
|
import json
|
2
|
+
import warnings
|
2
3
|
|
3
4
|
from rclone_api.rpath import RPath
|
4
5
|
|
5
6
|
|
7
|
+
def _dedupe(items: list[RPath]) -> list[RPath]:
|
8
|
+
"""Remove duplicate items from a list of RPath objects."""
|
9
|
+
seen = set()
|
10
|
+
unique_items = []
|
11
|
+
for item in items:
|
12
|
+
if item not in seen:
|
13
|
+
seen.add(item)
|
14
|
+
unique_items.append(item)
|
15
|
+
else:
|
16
|
+
warnings.warn(f"Duplicate item found: {item}, filtered out.")
|
17
|
+
return unique_items
|
18
|
+
|
19
|
+
|
6
20
|
class DirListing:
|
7
21
|
"""Remote file dataclass."""
|
8
22
|
|
@@ -10,6 +24,8 @@ class DirListing:
|
|
10
24
|
from rclone_api.dir import Dir
|
11
25
|
from rclone_api.file import File
|
12
26
|
|
27
|
+
dirs_and_files = _dedupe(dirs_and_files)
|
28
|
+
|
13
29
|
self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
|
14
30
|
self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
|
15
31
|
|
rclone_api/exec.py
CHANGED
@@ -19,7 +19,7 @@ class RcloneExec:
|
|
19
19
|
|
20
20
|
return rclone_execute(cmd, self.rclone_config, self.rclone_exe, check=check)
|
21
21
|
|
22
|
-
def launch_process(self, cmd: list[str]) -> Process:
|
22
|
+
def launch_process(self, cmd: list[str], capture: bool | None) -> Process:
|
23
23
|
"""Launch rclone process."""
|
24
24
|
|
25
25
|
args: ProcessArgs = ProcessArgs(
|
@@ -27,6 +27,7 @@ class RcloneExec:
|
|
27
27
|
rclone_conf=self.rclone_config,
|
28
28
|
rclone_exe=self.rclone_exe,
|
29
29
|
cmd_list=cmd,
|
30
|
+
capture_stdout=capture,
|
30
31
|
)
|
31
32
|
process = Process(args)
|
32
33
|
return process
|
rclone_api/process.py
CHANGED
@@ -56,6 +56,7 @@ class ProcessArgs:
|
|
56
56
|
rclone_exe: Path
|
57
57
|
cmd_list: list[str]
|
58
58
|
verbose: bool | None = None
|
59
|
+
capture_stdout: bool | None = None
|
59
60
|
|
60
61
|
|
61
62
|
class Process:
|
@@ -84,7 +85,13 @@ class Process:
|
|
84
85
|
if verbose:
|
85
86
|
cmd_str = subprocess.list2cmdline(self.cmd)
|
86
87
|
print(f"Running: {cmd_str}")
|
87
|
-
|
88
|
+
kwargs: dict = {}
|
89
|
+
kwargs["shell"] = False
|
90
|
+
if args.capture_stdout:
|
91
|
+
kwargs["stdout"] = subprocess.PIPE
|
92
|
+
kwargs["stderr"] = subprocess.STDOUT
|
93
|
+
|
94
|
+
self.process = subprocess.Popen(self.cmd, **kwargs) # type: ignore
|
88
95
|
|
89
96
|
def cleanup(self) -> None:
|
90
97
|
if self.tempdir and self.needs_cleanup:
|
rclone_api/rclone.py
CHANGED
@@ -14,6 +14,7 @@ from typing import Generator
|
|
14
14
|
from rclone_api import Dir
|
15
15
|
from rclone_api.config import Config
|
16
16
|
from rclone_api.convert import convert_to_filestr_list, convert_to_str
|
17
|
+
from rclone_api.diff import DiffItem, diff_stream_from_running_process
|
17
18
|
from rclone_api.dir_listing import DirListing
|
18
19
|
from rclone_api.exec import RcloneExec
|
19
20
|
from rclone_api.file import File
|
@@ -41,8 +42,8 @@ class Rclone:
|
|
41
42
|
def _run(self, cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
|
42
43
|
return self._exec.execute(cmd, check=check)
|
43
44
|
|
44
|
-
def _launch_process(self, cmd: list[str]) -> Process:
|
45
|
-
return self._exec.launch_process(cmd)
|
45
|
+
def _launch_process(self, cmd: list[str], capture: bool | None = None) -> Process:
|
46
|
+
return self._exec.launch_process(cmd, capture=capture)
|
46
47
|
|
47
48
|
def obscure(self, password: str) -> str:
|
48
49
|
"""Obscure a password for use in rclone config files."""
|
@@ -108,6 +109,27 @@ class Rclone:
|
|
108
109
|
out = [Remote(name=t, rclone=self) for t in tmp]
|
109
110
|
return out
|
110
111
|
|
112
|
+
def diff(self, src: str, dst: str) -> Generator[DiffItem, None, None]:
|
113
|
+
"""Be extra careful with the src and dst values. If you are off by one
|
114
|
+
parent directory, you will get a huge amount of false diffs."""
|
115
|
+
cmd = [
|
116
|
+
"check",
|
117
|
+
src,
|
118
|
+
dst,
|
119
|
+
"--checkers",
|
120
|
+
"1000",
|
121
|
+
"--log-level",
|
122
|
+
"INFO",
|
123
|
+
"--combined",
|
124
|
+
"-",
|
125
|
+
]
|
126
|
+
proc = self._launch_process(cmd, capture=True)
|
127
|
+
item: DiffItem
|
128
|
+
for item in diff_stream_from_running_process(proc, src_slug=src, dst_slug=src):
|
129
|
+
if item is None:
|
130
|
+
break
|
131
|
+
yield item
|
132
|
+
|
111
133
|
def walk(
|
112
134
|
self, path: Dir | Remote | str, max_depth: int = -1, breadth_first: bool = True
|
113
135
|
) -> Generator[DirListing, None, None]:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: rclone_api
|
3
|
-
Version: 1.0.
|
3
|
+
Version: 1.0.28
|
4
4
|
Summary: rclone api in python
|
5
5
|
Home-page: https://github.com/zackees/rclone-api
|
6
6
|
Maintainer: Zachary Vorhies
|
@@ -22,11 +22,11 @@ Dynamic: maintainer
|
|
22
22
|
[](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
|
23
23
|
[](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
|
24
24
|
|
25
|
-
Api version of rclone. It's
|
25
|
+
Api version of rclone. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done. Unit tests up the wazoo. S3 Mounts on windows/linux are heavily optimized.
|
26
26
|
|
27
27
|
You will need to have rclone installed and on your path.
|
28
28
|
|
29
|
-
One of the benefits of this api is that it does not use
|
29
|
+
One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
|
30
30
|
|
31
31
|
# Install
|
32
32
|
|
@@ -1,23 +1,24 @@
|
|
1
|
-
rclone_api/__init__.py,sha256=
|
1
|
+
rclone_api/__init__.py,sha256=L_ulzYG9WmQqmrVANt-Omi1FJUQVTkXA7SVZx9QH_9M,457
|
2
2
|
rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
|
3
3
|
rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
|
4
4
|
rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
|
5
|
+
rclone_api/diff.py,sha256=yxg0e4142CKYe_S-RWYrTfgPIEFRr8p0f8_0dBrJapI,3822
|
5
6
|
rclone_api/dir.py,sha256=vV-bcI2ESijmwF5rPID5WO2K7soAfZa35wv4KRh_GIo,2154
|
6
|
-
rclone_api/dir_listing.py,sha256=
|
7
|
-
rclone_api/exec.py,sha256=
|
7
|
+
rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
|
8
|
+
rclone_api/exec.py,sha256=HWmnU2Jwb-3EttSbAJSaLloYA7YI2mHTzRJ5VEri9aM,941
|
8
9
|
rclone_api/file.py,sha256=D02iHJW1LhfOiM_R_yPHP8_ApnDiYrkuraVcrV8-qkw,1246
|
9
10
|
rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
|
10
|
-
rclone_api/process.py,sha256=
|
11
|
-
rclone_api/rclone.py,sha256=
|
11
|
+
rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
|
12
|
+
rclone_api/rclone.py,sha256=Yxuwz8cp-gcVJ45iyRadVDy8UfXzaWO_ynCJ86xhPyk,16578
|
12
13
|
rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
|
13
14
|
rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
|
14
15
|
rclone_api/util.py,sha256=AWTImA1liCMw52wtS-Gs8Abz96diQ39x6Tx56mvbrEo,3860
|
15
16
|
rclone_api/walk.py,sha256=kca0t1GAnF6FLclN01G8NG__Qe-ggodLtAbQSHyVPng,2968
|
16
17
|
rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
|
17
18
|
rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
|
18
|
-
rclone_api-1.0.
|
19
|
-
rclone_api-1.0.
|
20
|
-
rclone_api-1.0.
|
21
|
-
rclone_api-1.0.
|
22
|
-
rclone_api-1.0.
|
23
|
-
rclone_api-1.0.
|
19
|
+
rclone_api-1.0.28.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
20
|
+
rclone_api-1.0.28.dist-info/METADATA,sha256=-IsR17g5WUZqisSwegxyFInKQBcnTM3dXz-uEJt8Ssc,4488
|
21
|
+
rclone_api-1.0.28.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
|
22
|
+
rclone_api-1.0.28.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
|
23
|
+
rclone_api-1.0.28.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
24
|
+
rclone_api-1.0.28.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|