rclone-api 1.0.26__py2.py3-none-any.whl → 1.0.28__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -1,4 +1,5 @@
 from .config import Config
+from .diff import DiffItem, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
 from .file import File
@@ -18,4 +19,6 @@ __all__ = [
     "DirListing",
     "FileList",
     "Process",
+    "DiffItem",
+    "DiffType",
 ]

rclone_api/diff.py

@@ -0,0 +1,119 @@
+from dataclasses import dataclass
+from enum import Enum
+from queue import Queue
+from threading import Thread
+from typing import Generator
+
+from rclone_api.process import Process
+
+
+class DiffType(Enum):
+    EQUAL = "="
+    MISSING_ON_SRC = (
+        "-"  # means path was missing on the source, so only in the destination
+    )
+    MISSING_ON_DST = (
+        "+"  # means path was missing on the destination, so only in the source
+    )
+    DIFFERENT = "*"  # means path was present in source and destination but different.
+    ERROR = "!"  # means there was an error
+
+
+@dataclass
+class DiffItem:
+    type: DiffType
+    path: str
+    src_prefix: str
+    dst_prefix: str
+
+    def __str__(self) -> str:
+        return f"{self.type.value} {self.path}"
+
+    def __repr__(self) -> str:
+        return f"{self.type.name} {self.path}"
+
+
+def _classify_diff(line: str, src_slug: str, dst_slug: str) -> DiffItem | None:
+    def _new(type: DiffType, path: str) -> DiffItem:
+        return DiffItem(type, path, src_prefix=src_slug, dst_prefix=dst_slug)
+
+    suffix = line[1:].strip() if len(line) > 0 else ""
+    if line.startswith(DiffType.EQUAL.value):
+        return _new(DiffType.EQUAL, suffix)
+    if line.startswith(DiffType.MISSING_ON_SRC.value):
+        return _new(DiffType.MISSING_ON_SRC, suffix)
+    if line.startswith(DiffType.MISSING_ON_DST.value):
+        # return DiffItem(DiffType.MISSING_ON_DST, f"{src_slug}/{suffix}")
+        return _new(DiffType.MISSING_ON_DST, suffix)
+    if line.startswith(DiffType.DIFFERENT.value):
+        # return DiffItem(DiffType.DIFFERENT, suffix)
+        return _new(DiffType.DIFFERENT, suffix)
+    if line.startswith(DiffType.ERROR.value):
+        # return DiffItem(DiffType.ERROR, suffix)
+        return _new(DiffType.ERROR, suffix)
+    return None
+
+
+def _async_diff_stream_from_running_process(
+    running_process: Process,
+    src_slug: str,
+    dst_slug: str,
+    output: Queue[DiffItem | None],
+) -> None:
+    count = 0
+    first_few_lines: list[str] = []
+    try:
+        assert running_process.stdout is not None
+        n_max = 10
+        for line in iter(running_process.stdout.readline, b""):
+            try:
+                line_str = line.decode("utf-8").strip()
+                if len(first_few_lines) < n_max:
+                    first_few_lines.append(line_str)
+                # _classify_line_type
+                diff_item: DiffItem | None = _classify_diff(
+                    line_str, src_slug, dst_slug
+                )
+                if diff_item is None:
+                    # Some other output that we don't care about, debug print etc.
+                    continue
+                output.put(diff_item)
+                count += 1
+                # print(f"unhandled: {line_str}")
+            except UnicodeDecodeError:
+                print("UnicodeDecodeError")
+                continue
+        output.put(None)
+        print("done")
+    except KeyboardInterrupt:
+        import _thread
+
+        print("KeyboardInterrupt")
+        output.put(None)
+        _thread.interrupt_main()
+    if count == 0:
+        first_lines_str = "\n".join(first_few_lines)
+        raise ValueError(
+            f"No output from rclone check, first few lines: {first_lines_str}"
+        )
+
+
+def diff_stream_from_running_process(
+    running_process: Process,
+    src_slug: str,
+    dst_slug: str,
+) -> Generator[DiffItem, None, None]:
+    output: Queue[DiffItem | None] = Queue()
+    # process_output_to_diff_stream(running_process, src_slug, dst_slug, output)
+    thread = Thread(
+        target=_async_diff_stream_from_running_process,
+        args=(running_process, src_slug, dst_slug, output),
+        daemon=True,
+    )
+    thread.start()
+    while True:
+        item = output.get()
+        if item is None:
+            break
+        yield item
+    thread.join(timeout=5)

rclone_api/dir_listing.py

@@ -1,8 +1,22 @@
 import json
+import warnings
 
 from rclone_api.rpath import RPath
 
 
+def _dedupe(items: list[RPath]) -> list[RPath]:
+    """Remove duplicate items from a list of RPath objects."""
+    seen = set()
+    unique_items = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            unique_items.append(item)
+        else:
+            warnings.warn(f"Duplicate item found: {item}, filtered out.")
+    return unique_items
+
+
 class DirListing:
     """Remote file dataclass."""
 
@@ -10,6 +24,8 @@ class DirListing:
         from rclone_api.dir import Dir
         from rclone_api.file import File
 
+        dirs_and_files = _dedupe(dirs_and_files)
+
         self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
         self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
 

rclone_api/exec.py

@@ -19,7 +19,7 @@ class RcloneExec:
 
         return rclone_execute(cmd, self.rclone_config, self.rclone_exe, check=check)
 
-    def launch_process(self, cmd: list[str]) -> Process:
+    def launch_process(self, cmd: list[str], capture: bool | None) -> Process:
         """Launch rclone process."""
 
         args: ProcessArgs = ProcessArgs(
@@ -27,6 +27,7 @@ class RcloneExec:
             rclone_conf=self.rclone_config,
             rclone_exe=self.rclone_exe,
             cmd_list=cmd,
+            capture_stdout=capture,
         )
         process = Process(args)
         return process

rclone_api/process.py

@@ -56,6 +56,7 @@ class ProcessArgs:
     rclone_exe: Path
     cmd_list: list[str]
     verbose: bool | None = None
+    capture_stdout: bool | None = None
 
 
 class Process:
@@ -84,7 +85,13 @@ class Process:
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")
-        self.process = subprocess.Popen(self.cmd, shell=False)
+        kwargs: dict = {}
+        kwargs["shell"] = False
+        if args.capture_stdout:
+            kwargs["stdout"] = subprocess.PIPE
+            kwargs["stderr"] = subprocess.STDOUT
+
+        self.process = subprocess.Popen(self.cmd, **kwargs)  # type: ignore
 
     def cleanup(self) -> None:
         if self.tempdir and self.needs_cleanup:

rclone_api/rclone.py

@@ -14,6 +14,7 @@ from typing import Generator
 from rclone_api import Dir
 from rclone_api.config import Config
 from rclone_api.convert import convert_to_filestr_list, convert_to_str
+from rclone_api.diff import DiffItem, diff_stream_from_running_process
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
@@ -41,8 +42,8 @@ class Rclone:
     def _run(self, cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
         return self._exec.execute(cmd, check=check)
 
-    def _launch_process(self, cmd: list[str]) -> Process:
-        return self._exec.launch_process(cmd)
+    def _launch_process(self, cmd: list[str], capture: bool | None = None) -> Process:
+        return self._exec.launch_process(cmd, capture=capture)
 
     def obscure(self, password: str) -> str:
         """Obscure a password for use in rclone config files."""
@@ -108,6 +109,27 @@ class Rclone:
         out = [Remote(name=t, rclone=self) for t in tmp]
         return out
 
+    def diff(self, src: str, dst: str) -> Generator[DiffItem, None, None]:
+        """Be extra careful with the src and dst values. If you are off by one
+        parent directory, you will get a huge amount of false diffs."""
+        cmd = [
+            "check",
+            src,
+            dst,
+            "--checkers",
+            "1000",
+            "--log-level",
+            "INFO",
+            "--combined",
+            "-",
+        ]
+        proc = self._launch_process(cmd, capture=True)
+        item: DiffItem
+        for item in diff_stream_from_running_process(proc, src_slug=src, dst_slug=src):
+            if item is None:
+                break
+            yield item
+
     def walk(
         self, path: Dir | Remote | str, max_depth: int = -1, breadth_first: bool = True
     ) -> Generator[DirListing, None, None]:

{rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.26
+Version: 1.0.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies
@@ -22,11 +22,11 @@ Dynamic: maintainer
 [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
 [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
 
-Api version of rclone. It's well tested. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done.
+Api version of rclone. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done. Unit tests up the wazoo. S3 Mounts on windows/linux are heavily optimized.
 
 You will need to have rclone installed and on your path.
 
-One of the benefits of this api is that it does not use `shell=True`, which can keep `rclone` running in some instances even when try to kill the process.
+One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
 
 # Install
 

{rclone_api-1.0.26.dist-info → rclone_api-1.0.28.dist-info}/RECORD

@@ -1,23 +1,24 @@
-rclone_api/__init__.py,sha256=UWQMbhE4WOQ3Skfb0LagFKW8PUKUGOm9_T3z--5FHiY,388
+rclone_api/__init__.py,sha256=L_ulzYG9WmQqmrVANt-Omi1FJUQVTkXA7SVZx9QH_9M,457
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
 rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
+rclone_api/diff.py,sha256=yxg0e4142CKYe_S-RWYrTfgPIEFRr8p0f8_0dBrJapI,3822
 rclone_api/dir.py,sha256=vV-bcI2ESijmwF5rPID5WO2K7soAfZa35wv4KRh_GIo,2154
-rclone_api/dir_listing.py,sha256=8t5Jx9ZVOJPqGKTJbWaES6bjgogUT2bnpbPVWwK1Fcs,1124
-rclone_api/exec.py,sha256=9qSOpZo8YRYxv3hOvNr57ApnY2KbjxwT1QNr8OgcLM4,883
+rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
+rclone_api/exec.py,sha256=HWmnU2Jwb-3EttSbAJSaLloYA7YI2mHTzRJ5VEri9aM,941
 rclone_api/file.py,sha256=D02iHJW1LhfOiM_R_yPHP8_ApnDiYrkuraVcrV8-qkw,1246
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
-rclone_api/process.py,sha256=iVA8_qtJFgHtg92Yb4W7xv9jnyds_7kAdKeSiGCBIZ0,3952
-rclone_api/rclone.py,sha256=C5cbPahwJy0QuhQ3U8wH0DOY-SvKxPKKyogshXlp3P4,15755
+rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
+rclone_api/rclone.py,sha256=Yxuwz8cp-gcVJ45iyRadVDy8UfXzaWO_ynCJ86xhPyk,16578
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/util.py,sha256=AWTImA1liCMw52wtS-Gs8Abz96diQ39x6Tx56mvbrEo,3860
 rclone_api/walk.py,sha256=kca0t1GAnF6FLclN01G8NG__Qe-ggodLtAbQSHyVPng,2968
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.26.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.26.dist-info/METADATA,sha256=CgaiMFCFjCPlnagzK7xi1wsLwB6YVl8fsEJii3HQbAM,4454
-rclone_api-1.0.26.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.26.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.26.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.26.dist-info/RECORD,,
+rclone_api-1.0.28.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.28.dist-info/METADATA,sha256=-IsR17g5WUZqisSwegxyFInKQBcnTM3dXz-uEJt8Ssc,4488
+rclone_api-1.0.28.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.28.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.28.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.28.dist-info/RECORD,,