rclone-api 1.0.26__py2.py3-none-any.whl → 1.0.28__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rclone_api/__init__.py CHANGED
@@ -1,4 +1,5 @@
1
1
  from .config import Config
2
+ from .diff import DiffItem, DiffType
2
3
  from .dir import Dir
3
4
  from .dir_listing import DirListing
4
5
  from .file import File
@@ -18,4 +19,6 @@ __all__ = [
18
19
  "DirListing",
19
20
  "FileList",
20
21
  "Process",
22
+ "DiffItem",
23
+ "DiffType",
21
24
  ]
rclone_api/diff.py ADDED
@@ -0,0 +1,119 @@
1
+ from dataclasses import dataclass
2
+ from enum import Enum
3
+ from queue import Queue
4
+ from threading import Thread
5
+ from typing import Generator
6
+
7
+ from rclone_api.process import Process
8
+
9
+
10
+ class DiffType(Enum):
11
+ EQUAL = "="
12
+ MISSING_ON_SRC = (
13
+ "-" # means path was missing on the source, so only in the destination
14
+ )
15
+ MISSING_ON_DST = (
16
+ "+" # means path was missing on the destination, so only in the source
17
+ )
18
+ DIFFERENT = "*" # means path was present in source and destination but different.
19
+ ERROR = "!" # means there was an error
20
+
21
+
22
+ @dataclass
23
+ class DiffItem:
24
+ type: DiffType
25
+ path: str
26
+ src_prefix: str
27
+ dst_prefix: str
28
+
29
+ def __str__(self) -> str:
30
+ return f"{self.type.value} {self.path}"
31
+
32
+ def __repr__(self) -> str:
33
+ return f"{self.type.name} {self.path}"
34
+
35
+
36
+ def _classify_diff(line: str, src_slug: str, dst_slug: str) -> DiffItem | None:
37
+ def _new(type: DiffType, path: str) -> DiffItem:
38
+ return DiffItem(type, path, src_prefix=src_slug, dst_prefix=dst_slug)
39
+
40
+ suffix = line[1:].strip() if len(line) > 0 else ""
41
+ if line.startswith(DiffType.EQUAL.value):
42
+ return _new(DiffType.EQUAL, suffix)
43
+ if line.startswith(DiffType.MISSING_ON_SRC.value):
44
+ return _new(DiffType.MISSING_ON_SRC, suffix)
45
+ if line.startswith(DiffType.MISSING_ON_DST.value):
46
+ # return DiffItem(DiffType.MISSING_ON_DST, f"{src_slug}/{suffix}")
47
+ return _new(DiffType.MISSING_ON_DST, suffix)
48
+ if line.startswith(DiffType.DIFFERENT.value):
49
+ # return DiffItem(DiffType.DIFFERENT, suffix)
50
+ return _new(DiffType.DIFFERENT, suffix)
51
+ if line.startswith(DiffType.ERROR.value):
52
+ # return DiffItem(DiffType.ERROR, suffix)
53
+ return _new(DiffType.ERROR, suffix)
54
+ return None
55
+
56
+
57
+ def _async_diff_stream_from_running_process(
58
+ running_process: Process,
59
+ src_slug: str,
60
+ dst_slug: str,
61
+ output: Queue[DiffItem | None],
62
+ ) -> None:
63
+ count = 0
64
+ first_few_lines: list[str] = []
65
+ try:
66
+ assert running_process.stdout is not None
67
+ n_max = 10
68
+ for line in iter(running_process.stdout.readline, b""):
69
+ try:
70
+ line_str = line.decode("utf-8").strip()
71
+ if len(first_few_lines) < n_max:
72
+ first_few_lines.append(line_str)
73
+ # _classify_line_type
74
+ diff_item: DiffItem | None = _classify_diff(
75
+ line_str, src_slug, dst_slug
76
+ )
77
+ if diff_item is None:
78
+ # Some other output that we don't care about, debug print etc.
79
+ continue
80
+ output.put(diff_item)
81
+ count += 1
82
+ # print(f"unhandled: {line_str}")
83
+ except UnicodeDecodeError:
84
+ print("UnicodeDecodeError")
85
+ continue
86
+ output.put(None)
87
+ print("done")
88
+ except KeyboardInterrupt:
89
+ import _thread
90
+
91
+ print("KeyboardInterrupt")
92
+ output.put(None)
93
+ _thread.interrupt_main()
94
+ if count == 0:
95
+ first_lines_str = "\n".join(first_few_lines)
96
+ raise ValueError(
97
+ f"No output from rclone check, first few lines: {first_lines_str}"
98
+ )
99
+
100
+
101
+ def diff_stream_from_running_process(
102
+ running_process: Process,
103
+ src_slug: str,
104
+ dst_slug: str,
105
+ ) -> Generator[DiffItem, None, None]:
106
+ output: Queue[DiffItem | None] = Queue()
107
+ # process_output_to_diff_stream(running_process, src_slug, dst_slug, output)
108
+ thread = Thread(
109
+ target=_async_diff_stream_from_running_process,
110
+ args=(running_process, src_slug, dst_slug, output),
111
+ daemon=True,
112
+ )
113
+ thread.start()
114
+ while True:
115
+ item = output.get()
116
+ if item is None:
117
+ break
118
+ yield item
119
+ thread.join(timeout=5)
rclone_api/dir_listing.py CHANGED
@@ -1,8 +1,22 @@
1
1
  import json
2
+ import warnings
2
3
 
3
4
  from rclone_api.rpath import RPath
4
5
 
5
6
 
7
+ def _dedupe(items: list[RPath]) -> list[RPath]:
8
+ """Remove duplicate items from a list of RPath objects."""
9
+ seen = set()
10
+ unique_items = []
11
+ for item in items:
12
+ if item not in seen:
13
+ seen.add(item)
14
+ unique_items.append(item)
15
+ else:
16
+ warnings.warn(f"Duplicate item found: {item}, filtered out.")
17
+ return unique_items
18
+
19
+
6
20
  class DirListing:
7
21
  """Remote file dataclass."""
8
22
 
@@ -10,6 +24,8 @@ class DirListing:
10
24
  from rclone_api.dir import Dir
11
25
  from rclone_api.file import File
12
26
 
27
+ dirs_and_files = _dedupe(dirs_and_files)
28
+
13
29
  self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
14
30
  self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
15
31
 
rclone_api/exec.py CHANGED
@@ -19,7 +19,7 @@ class RcloneExec:
19
19
 
20
20
  return rclone_execute(cmd, self.rclone_config, self.rclone_exe, check=check)
21
21
 
22
- def launch_process(self, cmd: list[str]) -> Process:
22
+ def launch_process(self, cmd: list[str], capture: bool | None) -> Process:
23
23
  """Launch rclone process."""
24
24
 
25
25
  args: ProcessArgs = ProcessArgs(
@@ -27,6 +27,7 @@ class RcloneExec:
27
27
  rclone_conf=self.rclone_config,
28
28
  rclone_exe=self.rclone_exe,
29
29
  cmd_list=cmd,
30
+ capture_stdout=capture,
30
31
  )
31
32
  process = Process(args)
32
33
  return process
rclone_api/process.py CHANGED
@@ -56,6 +56,7 @@ class ProcessArgs:
56
56
  rclone_exe: Path
57
57
  cmd_list: list[str]
58
58
  verbose: bool | None = None
59
+ capture_stdout: bool | None = None
59
60
 
60
61
 
61
62
  class Process:
@@ -84,7 +85,13 @@ class Process:
84
85
  if verbose:
85
86
  cmd_str = subprocess.list2cmdline(self.cmd)
86
87
  print(f"Running: {cmd_str}")
87
- self.process = subprocess.Popen(self.cmd, shell=False)
88
+ kwargs: dict = {}
89
+ kwargs["shell"] = False
90
+ if args.capture_stdout:
91
+ kwargs["stdout"] = subprocess.PIPE
92
+ kwargs["stderr"] = subprocess.STDOUT
93
+
94
+ self.process = subprocess.Popen(self.cmd, **kwargs) # type: ignore
88
95
 
89
96
  def cleanup(self) -> None:
90
97
  if self.tempdir and self.needs_cleanup:
rclone_api/rclone.py CHANGED
@@ -14,6 +14,7 @@ from typing import Generator
14
14
  from rclone_api import Dir
15
15
  from rclone_api.config import Config
16
16
  from rclone_api.convert import convert_to_filestr_list, convert_to_str
17
+ from rclone_api.diff import DiffItem, diff_stream_from_running_process
17
18
  from rclone_api.dir_listing import DirListing
18
19
  from rclone_api.exec import RcloneExec
19
20
  from rclone_api.file import File
@@ -41,8 +42,8 @@ class Rclone:
41
42
  def _run(self, cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
42
43
  return self._exec.execute(cmd, check=check)
43
44
 
44
- def _launch_process(self, cmd: list[str]) -> Process:
45
- return self._exec.launch_process(cmd)
45
+ def _launch_process(self, cmd: list[str], capture: bool | None = None) -> Process:
46
+ return self._exec.launch_process(cmd, capture=capture)
46
47
 
47
48
  def obscure(self, password: str) -> str:
48
49
  """Obscure a password for use in rclone config files."""
@@ -108,6 +109,27 @@ class Rclone:
108
109
  out = [Remote(name=t, rclone=self) for t in tmp]
109
110
  return out
110
111
 
112
+ def diff(self, src: str, dst: str) -> Generator[DiffItem, None, None]:
113
+ """Be extra careful with the src and dst values. If you are off by one
114
+ parent directory, you will get a huge amount of false diffs."""
115
+ cmd = [
116
+ "check",
117
+ src,
118
+ dst,
119
+ "--checkers",
120
+ "1000",
121
+ "--log-level",
122
+ "INFO",
123
+ "--combined",
124
+ "-",
125
+ ]
126
+ proc = self._launch_process(cmd, capture=True)
127
+ item: DiffItem
128
+ for item in diff_stream_from_running_process(proc, src_slug=src, dst_slug=src):
129
+ if item is None:
130
+ break
131
+ yield item
132
+
111
133
  def walk(
112
134
  self, path: Dir | Remote | str, max_depth: int = -1, breadth_first: bool = True
113
135
  ) -> Generator[DirListing, None, None]:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: rclone_api
3
- Version: 1.0.26
3
+ Version: 1.0.28
4
4
  Summary: rclone api in python
5
5
  Home-page: https://github.com/zackees/rclone-api
6
6
  Maintainer: Zachary Vorhies
@@ -22,11 +22,11 @@ Dynamic: maintainer
22
22
  [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
23
23
  [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
24
24
 
25
- Api version of rclone. It's well tested. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done.
25
+ Api version of rclone. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done. Unit tests up the wazoo. S3 Mounts on windows/linux are heavily optimized.
26
26
 
27
27
  You will need to have rclone installed and on your path.
28
28
 
29
- One of the benefits of this api is that it does not use `shell=True`, which can keep `rclone` running in some instances even when try to kill the process.
29
+ One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
30
30
 
31
31
  # Install
32
32
 
@@ -1,23 +1,24 @@
1
- rclone_api/__init__.py,sha256=UWQMbhE4WOQ3Skfb0LagFKW8PUKUGOm9_T3z--5FHiY,388
1
+ rclone_api/__init__.py,sha256=L_ulzYG9WmQqmrVANt-Omi1FJUQVTkXA7SVZx9QH_9M,457
2
2
  rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
3
  rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
4
4
  rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
5
+ rclone_api/diff.py,sha256=yxg0e4142CKYe_S-RWYrTfgPIEFRr8p0f8_0dBrJapI,3822
5
6
  rclone_api/dir.py,sha256=vV-bcI2ESijmwF5rPID5WO2K7soAfZa35wv4KRh_GIo,2154
6
- rclone_api/dir_listing.py,sha256=8t5Jx9ZVOJPqGKTJbWaES6bjgogUT2bnpbPVWwK1Fcs,1124
7
- rclone_api/exec.py,sha256=9qSOpZo8YRYxv3hOvNr57ApnY2KbjxwT1QNr8OgcLM4,883
7
+ rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
8
+ rclone_api/exec.py,sha256=HWmnU2Jwb-3EttSbAJSaLloYA7YI2mHTzRJ5VEri9aM,941
8
9
  rclone_api/file.py,sha256=D02iHJW1LhfOiM_R_yPHP8_ApnDiYrkuraVcrV8-qkw,1246
9
10
  rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
10
- rclone_api/process.py,sha256=iVA8_qtJFgHtg92Yb4W7xv9jnyds_7kAdKeSiGCBIZ0,3952
11
- rclone_api/rclone.py,sha256=C5cbPahwJy0QuhQ3U8wH0DOY-SvKxPKKyogshXlp3P4,15755
11
+ rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
12
+ rclone_api/rclone.py,sha256=Yxuwz8cp-gcVJ45iyRadVDy8UfXzaWO_ynCJ86xhPyk,16578
12
13
  rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
13
14
  rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
14
15
  rclone_api/util.py,sha256=AWTImA1liCMw52wtS-Gs8Abz96diQ39x6Tx56mvbrEo,3860
15
16
  rclone_api/walk.py,sha256=kca0t1GAnF6FLclN01G8NG__Qe-ggodLtAbQSHyVPng,2968
16
17
  rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
17
18
  rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
18
- rclone_api-1.0.26.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
19
- rclone_api-1.0.26.dist-info/METADATA,sha256=CgaiMFCFjCPlnagzK7xi1wsLwB6YVl8fsEJii3HQbAM,4454
20
- rclone_api-1.0.26.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
21
- rclone_api-1.0.26.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
22
- rclone_api-1.0.26.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
23
- rclone_api-1.0.26.dist-info/RECORD,,
19
+ rclone_api-1.0.28.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
20
+ rclone_api-1.0.28.dist-info/METADATA,sha256=-IsR17g5WUZqisSwegxyFInKQBcnTM3dXz-uEJt8Ssc,4488
21
+ rclone_api-1.0.28.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
22
+ rclone_api-1.0.28.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
23
+ rclone_api-1.0.28.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
24
+ rclone_api-1.0.28.dist-info/RECORD,,