rclone-api 1.0.69__py2.py3-none-any.whl → 1.0.72__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -1,14 +1,15 @@
 from .completed_process import CompletedProcess
 from .config import Config
-from .diff import DiffItem, DiffType
+from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
 from .file import File
 from .filelist import FileList
 from .process import Process
-from .rclone import DiffOption, ListingOption, Rclone, rclone_verbose
+from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .types import ListingOption, Order
 
 __all__ = [
     "Rclone",
@@ -26,4 +27,6 @@ __all__ = [
     "CompletedProcess",
     "DiffOption",
     "ListingOption",
+    "Order",
+    "ListingOption",
 ]

rclone_api/dir.py

@@ -5,7 +5,7 @@ from typing import Generator
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption
+from rclone_api.types import ListingOption, Order
 
 
 class Dir:
@@ -47,7 +47,7 @@ class Dir:
         self,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files and directories in the given path."""
@@ -57,7 +57,7 @@ class Dir:
             dir,
             max_depth=max_depth,
             glob=glob,
-            reverse=reverse,
+            order=order,
             listing_option=listing_option,
         )
 
@@ -102,7 +102,7 @@ class Dir:
         path = Path(self.path.path) / other
         rpath = RPath(
             self.path.remote,
-            str(path),
+            str(path.as_posix()),
             name=other,
             size=0,
             mime_type="inode/directory",

rclone_api/rclone.py

@@ -3,6 +3,7 @@ Unit test file.
 """
 
 import os
+import random
 import subprocess
 import time
 import warnings
@@ -25,7 +26,7 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption, ModTimeStrategy
+from rclone_api.types import ListingOption, ModTimeStrategy, Order
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -118,7 +119,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files in the given path.
@@ -164,8 +165,10 @@ class Rclone:
         if glob is not None:
             paths = [p for p in paths if fnmatch(p.path, glob)]
 
-        if reverse:
+        if order == Order.REVERSE:
             paths.reverse()
+        elif order == Order.RANDOM:
+            random.shuffle(paths)
         return DirListing(paths)
 
     def listremotes(self) -> list[Remote]:
@@ -242,7 +245,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int = -1,
         breadth_first: bool = True,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[DirListing, None, None]:
         """Walk through the given path recursively.
 
@@ -277,15 +280,15 @@ class Rclone:
             assert f"Invalid type for path: {type(path)}"
 
         yield from walk(
-            dir_obj, max_depth=max_depth, breadth_first=breadth_first, reverse=reverse
+            dir_obj, max_depth=max_depth, breadth_first=breadth_first, order=order
         )
 
-    def diff_walk(
+    def scan_missing_folders(
         self,
         src: Dir | Remote | str,
         dst: Dir | Remote | str,
         max_depth: int = -1,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[Dir, None, None]:
         """Walk through the given path recursively.
 
@@ -299,12 +302,12 @@ class Rclone:
         Yields:
             DirListing: Directory listing for each directory encountered
         """
-        from rclone_api.diff_walk import diff_walk
+        from rclone_api.scan_missing_folders import scan_missing_folders
 
         src_dir = Dir(to_path(src, self))
         dst_dir = Dir(to_path(dst, self))
-        yield from diff_walk(
-            src=src_dir, dst=dst_dir, max_depth=max_depth, reverse=reverse
+        yield from scan_missing_folders(
+            src=src_dir, dst=dst_dir, max_depth=max_depth, order=order
         )
 
     def cleanup(
@@ -426,10 +429,6 @@ class Rclone:
                                 chunk = files_fqdn[i : i + chunk_size]
                                 files_str = "\n".join(chunk)
                                 print(f"{files_str}")
-                            # files_str = "\n".join(files_fqdn)
-                            # print(f"Copying {nfiles} files: \n{files_str}")
-
-                        # print(include_files_txt)
                         cmd_list: list[str] = [
                             "copy",
                             src_path,

rclone_api/scan_missing_folders.py

@@ -0,0 +1,135 @@
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor
+from queue import Empty, Queue
+from threading import Thread
+from typing import Generator
+
+from rclone_api import Dir
+from rclone_api.dir_listing import DirListing
+from rclone_api.types import ListingOption, Order
+from rclone_api.walk import walk_runner_depth_first
+
+_MAX_OUT_QUEUE_SIZE = 50
+
+
+# ONLY Works from src -> dst diffing.
+def _async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
+) -> None:
+    curr_src, curr_dst = src, dst
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        t1 = executor.submit(
+            src.ls, listing_option=ListingOption.DIRS_ONLY, order=order
+        )
+        t2 = executor.submit(
+            dst.ls, listing_option=ListingOption.DIRS_ONLY, order=order
+        )
+        src_dir_listing: DirListing = t1.result()
+        dst_dir_listing: DirListing = t2.result()
+    next_depth = max_depth - 1 if max_depth > 0 else max_depth
+    dst_dirs: list[str] = [d.name for d in dst_dir_listing.dirs]
+    src_dirs: list[str] = [d.name for d in src_dir_listing.dirs]
+    dst_files_set: set[str] = set(dst_dirs)
+    matching_dirs: list[str] = []
+    if order == Order.REVERSE:
+        src_dirs.reverse()
+        dst_dirs.reverse()
+    elif order == Order.RANDOM:
+        random.shuffle(src_dirs)
+        random.shuffle(dst_dirs)
+    for file in src_dirs:
+        if file not in dst_files_set:
+            queue_dir_listing: Queue[DirListing | None] = Queue()
+            if next_depth > 0 or next_depth == -1:
+                walk_runner_depth_first(
+                    dir=curr_src,
+                    out_queue=queue_dir_listing,
+                    order=order,
+                    max_depth=next_depth,
+                )
+            while dirlisting := queue_dir_listing.get():
+                if dirlisting is None:
+                    break
+                # print(f"dirlisting: {dirlisting}")
+                for d in dirlisting.dirs:
+                    out_queue.put(d)
+        else:
+            matching_dirs.append(file)
+
+    for matching_dir in matching_dirs:
+        # print(f"matching dir: {matching_dir}")
+        if next_depth > 0 or next_depth == -1:
+            src_next = curr_src / matching_dir
+            dst_next = curr_dst / matching_dir
+            _async_diff_dir_walk_task(
+                src=src_next,
+                dst=dst_next,
+                max_depth=next_depth,
+                out_queue=out_queue,
+                order=order,
+            )
+
+
+def async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
+) -> None:
+    try:
+        _async_diff_dir_walk_task(
+            src=src, dst=dst, max_depth=max_depth, out_queue=out_queue, order=order
+        )
+    except Exception:
+        import _thread
+
+        _thread.interrupt_main()
+        raise
+    finally:
+        out_queue.put(None)
+
+
+def scan_missing_folders(
+    src: Dir,
+    dst: Dir,
+    max_depth: int = -1,
+    order: Order = Order.NORMAL,
+) -> Generator[Dir, None, None]:
+    """Walk through the given directory recursively.
+
+    Args:
+        dir: Directory or Remote to walk through
+        max_depth: Maximum depth to traverse (-1 for unlimited)
+
+    Yields:
+        DirListing: Directory listing for each directory encountered
+    """
+
+    try:
+        out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
+
+        def task() -> None:
+            async_diff_dir_walk_task(
+                src=src,
+                dst=dst,
+                max_depth=max_depth,
+                out_queue=out_queue,
+                order=order,
+            )
+
+        worker = Thread(
+            target=task,
+            daemon=True,
+        )
+        worker.start()
+
+        while True:
+            try:
+                dir = out_queue.get_nowait()
+                if dir is None:
+                    break
+                yield dir
+            except Empty:
+                time.sleep(0.1)
+
+        worker.join()
+    except KeyboardInterrupt:
+        pass

rclone_api/types.py

@@ -10,3 +10,9 @@ class ListingOption(Enum):
     DIRS_ONLY = "dirs-only"
     FILES_ONLY = "files-only"
     ALL = "all"
+
+
+class Order(Enum):
+    NORMAL = "normal"
+    REVERSE = "reverse"
+    RANDOM = "random"

rclone_api/walk.py

@@ -1,3 +1,4 @@
+import random
 from queue import Queue
 from threading import Thread
 from typing import Generator
@@ -5,6 +6,7 @@ from typing import Generator
 from rclone_api import Dir
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
+from rclone_api.types import Order
 
 _MAX_OUT_QUEUE_SIZE = 50
 
@@ -13,14 +15,14 @@ def walk_runner_breadth_first(
     dir: Dir,
     max_depth: int,
     out_queue: Queue[DirListing | None],
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> None:
     queue: Queue[Dir] = Queue()
     queue.put(dir)
     try:
         while not queue.empty():
             current_dir = queue.get()
-            dirlisting = current_dir.ls(max_depth=0, reverse=reverse)
+            dirlisting = current_dir.ls(max_depth=0, order=order)
             out_queue.put(dirlisting)
             dirs = dirlisting.dirs
 
@@ -41,22 +43,26 @@ def walk_runner_breadth_first(
 
 
 def walk_runner_depth_first(
-    dir: Dir, max_depth: int, out_queue: Queue[DirListing | None], reverse=False
+    dir: Dir,
+    max_depth: int,
+    out_queue: Queue[DirListing | None],
+    order: Order = Order.NORMAL,
 ) -> None:
     try:
         stack = [(dir, max_depth)]
         while stack:
             current_dir, depth = stack.pop()
             dirlisting = current_dir.ls()
-            if reverse:
+            if order == Order.REVERSE:
                 dirlisting.dirs.reverse()
+            if order == Order.RANDOM:
+
+                random.shuffle(dirlisting.dirs)
             if depth != 0:
                 for subdir in dirlisting.dirs:  # Process deeper directories first
                     # stack.append((child, depth - 1 if depth > 0 else depth))
                     next_depth = depth - 1 if depth > 0 else depth
-                    walk_runner_depth_first(
-                        subdir, next_depth, out_queue, reverse=reverse
-                    )
+                    walk_runner_depth_first(subdir, next_depth, out_queue, order=order)
             out_queue.put(dirlisting)
         out_queue.put(None)
     except KeyboardInterrupt:
@@ -70,7 +76,7 @@ def walk(
     dir: Dir | Remote,
     breadth_first: bool,
     max_depth: int = -1,
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> Generator[DirListing, None, None]:
     """Walk through the given directory recursively.
 
@@ -89,9 +95,9 @@ def walk(
 
         def _task() -> None:
             if breadth_first:
-                walk_runner_breadth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_breadth_first(dir, max_depth, out_queue, order)
             else:
-                walk_runner_depth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_depth_first(dir, max_depth, out_queue, order)
 
         # Start worker thread
         worker = Thread(

{rclone_api-1.0.69.dist-info → rclone_api-1.0.72.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.69
+Version: 1.0.72
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.69.dist-info → rclone_api-1.0.72.dist-info}/RECORD

@@ -1,29 +1,29 @@
-rclone_api/__init__.py,sha256=B9q_4JU5Myh0DN-wqr_IgF5hZeAvszrGcqs2x-3hq-Q,633
+rclone_api/__init__.py,sha256=kJUk9KAxQ2AEms9lM5yqw-dqlabarhU69b2FAxJVBlY,692
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=Pp-hXnLgej0IGO5ee9Fmx64dGzIofbQFEUyXdFCvO54,1371
 rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
 rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
 rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
 rclone_api/diff.py,sha256=ggdDLUZxa13jMcPzKBcwAElmPCNWMOSR89D4yhpO74M,5264
-rclone_api/diff_walk.py,sha256=V8U6qwCzG7_Z6-xFpB4zP6kCq_jN30OMYo7tT92OTd0,5061
-rclone_api/dir.py,sha256=8EwaGonkCeHhe-rsVl98PkEKvyf-3pFUJnXqkN6Fx24,3509
+rclone_api/dir.py,sha256=hL0i4zrNUbpvFWI3TvKyOyanJ2okcb1lz4r2kxlvSi4,3529
 rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
 rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
 rclone_api/file.py,sha256=YtR5Y6c0YfXTS-sReOy2UgiSnafcAeO6b2hnbojBQD4,1423
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=kOHh6ysFDkxjldSwvW6KqmiADUC1yFCdrZRY57TvbGY,5328
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=nN1bqUmiopCKP6VNG4XpSfBfWx4EuAIyxB9NevYp-OA,28854
+rclone_api/rclone.py,sha256=kIamoje3fUaWboMdF_d_a4WVaSa8BfK6zDFY8U6pNbs,28820
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
-rclone_api/types.py,sha256=VXA-56va2S41onAWM_UmhHlG4qeyMUEF5dsVFyfLlH8,232
+rclone_api/scan_missing_folders.py,sha256=ke2AVIdX-MiuJUi7poJBzP8GxjzzMhCsr1Mxbg0EIu4,4217
+rclone_api/types.py,sha256=DcbNw1R6j2f_1zHrhqEJcpCR-8kJfFJawMY0AmPsCnM,321
 rclone_api/util.py,sha256=_cvmHcJPRl2yXw4zgZiop3z-riA_8Ek6S5NDPw8cqSY,4198
-rclone_api/walk.py,sha256=ndWV7WBVQLbpZu3HuJrAe1cXcmQVjT9_YPsbat158bQ,3231
+rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.69.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.69.dist-info/METADATA,sha256=bl94_jCcD0a0buUsLVFIQ-MPQ5O2fy-hFrvK1EvU7Ls,4489
-rclone_api-1.0.69.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.69.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.69.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.69.dist-info/RECORD,,
+rclone_api-1.0.72.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.72.dist-info/METADATA,sha256=xzlW0jfYuqOPGSCI9k8MxNvp4BpI0Duojacxq_l-C-M,4489
+rclone_api-1.0.72.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.72.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.72.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.72.dist-info/RECORD,,

rclone_api/diff_walk.py

@@ -1,143 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-from queue import Queue
-from threading import Thread
-from typing import Generator
-
-from rclone_api import Dir
-from rclone_api.dir_listing import DirListing
-from rclone_api.types import ListingOption
-from rclone_api.walk import walk_runner_depth_first
-
-_MAX_OUT_QUEUE_SIZE = 50
-
-
-# ONLY Works from src -> dst diffing.
-def _async_diff_dir_walk_task(
-    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse=False
-) -> None:
-
-    try:
-        stack = [(src, dst)]
-        while stack:
-            curr_src, curr_dst = stack.pop()
-            curr_src = curr_src
-            curr_dst = curr_dst
-
-            with ThreadPoolExecutor(max_workers=2) as executor:
-                # src_dir_listing = src.ls(listing_option=ListingOption.DIRS_ONLY)
-                # dst_dir_listing = dst.ls(listing_option=ListingOption.DIRS_ONLY)
-                t1 = executor.submit(
-                    src.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
-                )
-                t2 = executor.submit(
-                    dst.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
-                )
-                src_dir_listing: DirListing = t1.result()
-                dst_dir_listing: DirListing = t2.result()
-
-            # dirlisting = current_dir.ls()
-            # if reverse:
-            #     dirlisting.dirs.reverse()
-            # if depth != 0:
-            #     for subdir in dirlisting.dirs:  # Process deeper directories first
-            #         # stack.append((child, depth - 1 if depth > 0 else depth))
-            #         next_depth = depth - 1 if depth > 0 else depth
-            #         _walk_runner_depth_first(
-            #             subdir, next_depth, out_queue, reverse=reverse
-            #         )
-            # out_queue.put(dirlisting)
-
-            # for subdir in dst_dir_listing.dirs:
-            #     subdir.to_string(include_remote=False)
-            #     walk_runner_depth_first()
-
-            # find elements missing on dst
-            # missing_on_dst: set[Dir] = set(src_dir_listing.dirs) - set(
-            #     dst_dir_listing.dirs
-            # )
-            # exists_on_dst: set[Dir] = set(src_dir_listing.dirs) - missing_on_dst
-
-            dst_files: list[str] = [d.name for d in dst_dir_listing.dirs]
-            src_files: list[str] = [d.name for d in src_dir_listing.dirs]
-
-            dst_files_set: set[str] = set(dst_files)
-            # src_files_set: set[str] = set(src_files)
-
-            # print(f"src_files: {src_files}")
-            # print(f"dst_files: {dst_files}")
-
-            matching_dirs: list[str] = []
-
-            for file in src_files:
-                if file not in dst_files_set:
-                    # print(f"missing dir on src: {file}")
-                    queue_dir_listing: Queue[DirListing | None] = Queue()
-                    walk_runner_depth_first(
-                        dir=curr_src,
-                        max_depth=max_depth,
-                        out_queue=queue_dir_listing,
-                        reverse=reverse,
-                    )
-                    while dirlisting := queue_dir_listing.get():
-                        if dirlisting is None:
-                            break
-                        # print(f"dirlisting: {dirlisting}")
-                        for d in dirlisting.dirs:
-                            out_queue.put(d)
-                else:
-                    matching_dirs.append(file)
-
-            for matching_dir in matching_dirs:
-                # print(f"matching dir: {matching_dir}")
-                _async_diff_dir_walk_task(
-                    src=curr_src / matching_dir,
-                    dst=curr_dst / matching_dir,
-                    max_depth=max_depth,
-                    out_queue=out_queue,
-                    reverse=reverse,
-                )
-
-        out_queue.put(None)
-    except KeyboardInterrupt:
-        import _thread
-
-        out_queue.put(None)
-        _thread.interrupt_main()
-
-
-def diff_walk(
-    src: Dir,
-    dst: Dir,
-    max_depth: int = -1,
-    reverse: bool = False,
-) -> Generator[Dir, None, None]:
-    """Walk through the given directory recursively.
-
-    Args:
-        dir: Directory or Remote to walk through
-        max_depth: Maximum depth to traverse (-1 for unlimited)
-
-    Yields:
-        DirListing: Directory listing for each directory encountered
-    """
-
-    try:
-        out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
-
-        def task() -> None:
-            _async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse=reverse)
-
-        worker = Thread(
-            target=task,
-            daemon=True,
-        )
-        worker.start()
-
-        while dirlisting := out_queue.get():
-            if dirlisting is None:
-                break
-            yield dirlisting
-
-        worker.join()
-    except KeyboardInterrupt:
-        pass