rclone-api 1.0.68__py2.py3-none-any.whl → 1.0.70__py2.py3-none-any.whl

rclone_api/diff_walk.py

@@ -0,0 +1,123 @@
+import time
+from concurrent.futures import ThreadPoolExecutor
+from queue import Empty, Queue
+from threading import Thread
+from typing import Generator
+
+from rclone_api import Dir
+from rclone_api.dir_listing import DirListing
+from rclone_api.types import ListingOption
+from rclone_api.walk import walk_runner_depth_first
+
+_MAX_OUT_QUEUE_SIZE = 50
+
+
+# ONLY Works from src -> dst diffing.
+def _async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse
+) -> None:
+    curr_src, curr_dst = src, dst
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        # src_dir_listing = src.ls(listing_option=ListingOption.DIRS_ONLY)
+        # dst_dir_listing = dst.ls(listing_option=ListingOption.DIRS_ONLY)
+        t1 = executor.submit(
+            src.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+        )
+        t2 = executor.submit(
+            dst.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+        )
+        src_dir_listing: DirListing = t1.result()
+        dst_dir_listing: DirListing = t2.result()
+    next_depth = max_depth - 1 if max_depth > 0 else max_depth
+    dst_files: list[str] = [d.name for d in dst_dir_listing.dirs]
+    src_files: list[str] = [d.name for d in src_dir_listing.dirs]
+    dst_files_set: set[str] = set(dst_files)
+    matching_dirs: list[str] = []
+    for file in src_files:
+        if file not in dst_files_set:
+            # print(f"missing dir on src: {file}")
+            queue_dir_listing: Queue[DirListing | None] = Queue()
+            if next_depth > 0 or next_depth == -1:
+                walk_runner_depth_first(
+                    dir=curr_src,
+                    out_queue=queue_dir_listing,
+                    reverse=reverse,
+                    max_depth=next_depth,
+                )
+            while dirlisting := queue_dir_listing.get():
+                if dirlisting is None:
+                    break
+                # print(f"dirlisting: {dirlisting}")
+                for d in dirlisting.dirs:
+                    out_queue.put(d)
+        else:
+            matching_dirs.append(file)
+
+    for matching_dir in matching_dirs:
+        # print(f"matching dir: {matching_dir}")
+        if next_depth > 0 or next_depth == -1:
+            src_next = curr_src / matching_dir
+            dst_next = curr_dst / matching_dir
+            _async_diff_dir_walk_task(
+                src=src_next,
+                dst=dst_next,
+                max_depth=next_depth,
+                out_queue=out_queue,
+                reverse=reverse,
+            )
+
+
+def async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse=False
+) -> None:
+    try:
+        _async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse)
+    except Exception:
+        import _thread
+
+        _thread.interrupt_main()
+        raise
+    finally:
+        out_queue.put(None)
+
+
+def diff_walk(
+    src: Dir,
+    dst: Dir,
+    max_depth: int = -1,
+    reverse: bool = False,
+) -> Generator[Dir, None, None]:
+    """Walk through the given directory recursively.
+
+    Args:
+        dir: Directory or Remote to walk through
+        max_depth: Maximum depth to traverse (-1 for unlimited)
+
+    Yields:
+        DirListing: Directory listing for each directory encountered
+    """
+
+    try:
+        out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
+
+        def task() -> None:
+            async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse=reverse)
+
+        worker = Thread(
+            target=task,
+            daemon=True,
+        )
+        worker.start()
+
+        while True:
+            try:
+                dirlisting = out_queue.get_nowait()
+                if dirlisting is None:
+                    break
+                yield dirlisting
+            except Empty:
+                time.sleep(0.1)
+
+        worker.join()
+    except KeyboardInterrupt:
+        pass

rclone_api/dir.py

@@ -1,9 +1,11 @@
 import json
+from pathlib import Path
 from typing import Generator
 
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.types import ListingOption
 
 
 class Dir:
@@ -41,11 +43,30 @@ class Dir:
         # self.path.set_rclone(self.path.remote.rclone)
         assert self.path.rclone is not None
 
-    def ls(self, max_depth: int = 0, reverse: bool = False) -> DirListing:
+    def ls(
+        self,
+        max_depth: int | None = None,
+        glob: str | None = None,
+        reverse: bool = False,
+        listing_option: ListingOption = ListingOption.ALL,
+    ) -> DirListing:
         """List files and directories in the given path."""
         assert self.path.rclone is not None
         dir = Dir(self.path)
-        return self.path.rclone.ls(dir, max_depth=max_depth, reverse=reverse)
+        return self.path.rclone.ls(
+            dir,
+            max_depth=max_depth,
+            glob=glob,
+            reverse=reverse,
+            listing_option=listing_option,
+        )
+
+    def relative_to(self, other: "Dir") -> str:
+        """Return the relative path to the other directory."""
+        self_path = Path(self.path.path)
+        other_path = Path(other.path.path)
+        rel_path = self_path.relative_to(other_path)
+        return str(rel_path)
 
     def walk(
         self, breadth_first: bool, max_depth: int = -1
@@ -67,3 +88,26 @@ class Dir:
         data = self.path.to_json()
         data_str = json.dumps(data)
         return data_str
+
+    def to_string(self, include_remote: bool = True) -> str:
+        """Convert the File to a string."""
+        out = str(self.path)
+        if not include_remote:
+            _, out = out.split(":", 1)
+        return out
+
+    # / operator
+    def __truediv__(self, other: str) -> "Dir":
+        """Join the current path with another path."""
+        path = Path(self.path.path) / other
+        rpath = RPath(
+            self.path.remote,
+            str(path.as_posix()),
+            name=other,
+            size=0,
+            mime_type="inode/directory",
+            mod_time="",
+            is_dir=True,
+        )
+        rpath.set_rclone(self.path.rclone)
+        return Dir(rpath)

rclone_api/rclone.py

@@ -7,7 +7,6 @@ import subprocess
 import time
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
-from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -26,6 +25,7 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.types import ListingOption, ModTimeStrategy
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -42,17 +42,6 @@ def rclone_verbose(verbose: bool | None) -> bool:
     return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
 
 
-class ModTimeStrategy(Enum):
-    USE_SERVER_MODTIME = "use-server-modtime"
-    NO_MODTIME = "no-modtime"
-
-
-class ListingOption(Enum):
-    DIRS_ONLY = "dirs-only"
-    FILES_ONLY = "files-only"
-    ALL = "all"
-
-
 class Rclone:
     def __init__(
         self, rclone_conf: Path | Config, rclone_exe: Path | None = None
@@ -203,17 +192,20 @@ class Rclone:
         diff_option: DiffOption = DiffOption.COMBINED,
         fast_list: bool = True,
         size_only: bool | None = None,
+        checkers: int | None = None,
         other_args: list[str] | None = None,
     ) -> Generator[DiffItem, None, None]:
         """Be extra careful with the src and dst values. If you are off by one
         parent directory, you will get a huge amount of false diffs."""
         other_args = other_args or []
+        if checkers is None or checkers < 1:
+            checkers = 1000
         cmd = [
             "check",
             src,
             dst,
             "--checkers",
-            "1000",
+            str(checkers),
             "--log-level",
             "INFO",
             f"--{diff_option.value}",
@@ -288,6 +280,33 @@ class Rclone:
             dir_obj, max_depth=max_depth, breadth_first=breadth_first, reverse=reverse
         )
 
+    def diff_walk(
+        self,
+        src: Dir | Remote | str,
+        dst: Dir | Remote | str,
+        max_depth: int = -1,
+        reverse: bool = False,
+    ) -> Generator[Dir, None, None]:
+        """Walk through the given path recursively.
+
+        WORK IN PROGRESS!!
+
+        Args:
+            src: Source directory or Remote to walk through
+            dst: Destination directory or Remote to walk through
+            max_depth: Maximum depth to traverse (-1 for unlimited)
+
+        Yields:
+            DirListing: Directory listing for each directory encountered
+        """
+        from rclone_api.diff_walk import diff_walk
+
+        src_dir = Dir(to_path(src, self))
+        dst_dir = Dir(to_path(dst, self))
+        yield from diff_walk(
+            src=src_dir, dst=dst_dir, max_depth=max_depth, reverse=reverse
+        )
+
     def cleanup(
         self, path: str, other_args: list[str] | None = None
     ) -> CompletedProcess:

rclone_api/types.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class ModTimeStrategy(Enum):
+    USE_SERVER_MODTIME = "use-server-modtime"
+    NO_MODTIME = "no-modtime"
+
+
+class ListingOption(Enum):
+    DIRS_ONLY = "dirs-only"
+    FILES_ONLY = "files-only"
+    ALL = "all"

rclone_api/walk.py

@@ -9,7 +9,7 @@ from rclone_api.remote import Remote
 _MAX_OUT_QUEUE_SIZE = 50
 
 
-def _walk_runner_breadth_first(
+def walk_runner_breadth_first(
     dir: Dir,
     max_depth: int,
     out_queue: Queue[DirListing | None],
@@ -40,7 +40,7 @@ def _walk_runner_breadth_first(
         _thread.interrupt_main()
 
 
-def _walk_runner_depth_first(
+def walk_runner_depth_first(
     dir: Dir, max_depth: int, out_queue: Queue[DirListing | None], reverse=False
 ) -> None:
     try:
@@ -54,7 +54,7 @@ def _walk_runner_depth_first(
                 for subdir in dirlisting.dirs:  # Process deeper directories first
                     # stack.append((child, depth - 1 if depth > 0 else depth))
                     next_depth = depth - 1 if depth > 0 else depth
-                    _walk_runner_depth_first(
+                    walk_runner_depth_first(
                         subdir, next_depth, out_queue, reverse=reverse
                     )
             out_queue.put(dirlisting)
@@ -85,16 +85,17 @@ def walk(
         # Convert Remote to Dir if needed
         if isinstance(dir, Remote):
             dir = Dir(dir)
-        out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
+        out_queue: Queue[DirListing | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
 
-        strategy = (
-            _walk_runner_breadth_first if breadth_first else _walk_runner_depth_first
-        )
+        def _task() -> None:
+            if breadth_first:
+                walk_runner_breadth_first(dir, max_depth, out_queue, reverse)
+            else:
+                walk_runner_depth_first(dir, max_depth, out_queue, reverse)
 
         # Start worker thread
         worker = Thread(
-            target=strategy,
-            args=(dir, max_depth, out_queue, reverse),
+            target=_task,
             daemon=True,
         )
         worker.start()

{rclone_api-1.0.68.dist-info → rclone_api-1.0.70.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.68
+Version: 1.0.70
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.68.dist-info → rclone_api-1.0.70.dist-info}/RECORD

@@ -5,23 +5,25 @@ rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
 rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
 rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
 rclone_api/diff.py,sha256=ggdDLUZxa13jMcPzKBcwAElmPCNWMOSR89D4yhpO74M,5264
-rclone_api/dir.py,sha256=2-PFaDpjEs28z82DQ-TyaCgrm_OgpwlkwfTnx1-Wwpk,2194
+rclone_api/diff_walk.py,sha256=fnCjzenKBBocu6sEd9CDEcwNqCUYwz3rZsxUuKNaYEg,4049
+rclone_api/dir.py,sha256=BuXPd-bAvZund8k7mXjvx_UylsPFwbWq2zaUdflTX04,3520
 rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
 rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
 rclone_api/file.py,sha256=YtR5Y6c0YfXTS-sReOy2UgiSnafcAeO6b2hnbojBQD4,1423
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=kOHh6ysFDkxjldSwvW6KqmiADUC1yFCdrZRY57TvbGY,5328
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=Hnk74ZyCnxSkeNIGRfZLqtGdWHTzPRye2lzRah6vLeI,28071
+rclone_api/rclone.py,sha256=nN1bqUmiopCKP6VNG4XpSfBfWx4EuAIyxB9NevYp-OA,28854
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
+rclone_api/types.py,sha256=VXA-56va2S41onAWM_UmhHlG4qeyMUEF5dsVFyfLlH8,232
 rclone_api/util.py,sha256=_cvmHcJPRl2yXw4zgZiop3z-riA_8Ek6S5NDPw8cqSY,4198
-rclone_api/walk.py,sha256=UaNOE3ICd8k5ouSFZvkVEH4r2GnnrD9TxfwkFcQnayo,3170
+rclone_api/walk.py,sha256=ndWV7WBVQLbpZu3HuJrAe1cXcmQVjT9_YPsbat158bQ,3231
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.68.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.68.dist-info/METADATA,sha256=p3oHKpaFeHKBgQhxdAqBHrHVBwzm7k_VuL8W8oYmx9U,4489
-rclone_api-1.0.68.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.68.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.68.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.68.dist-info/RECORD,,
+rclone_api-1.0.70.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.70.dist-info/METADATA,sha256=73129SIcLoTZ3R35zB0vuZZ65SC0RBttGk1LtPARAx8,4489
+rclone_api-1.0.70.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.70.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.70.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.70.dist-info/RECORD,,