rclone-api 1.0.70__py2.py3-none-any.whl → 1.0.73__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -1,14 +1,15 @@
 from .completed_process import CompletedProcess
 from .config import Config
-from .diff import DiffItem, DiffType
+from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
 from .file import File
 from .filelist import FileList
 from .process import Process
-from .rclone import DiffOption, ListingOption, Rclone, rclone_verbose
+from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .types import ListingOption, Order
 
 __all__ = [
     "Rclone",
@@ -26,4 +27,6 @@ __all__ = [
     "CompletedProcess",
     "DiffOption",
     "ListingOption",
+    "Order",
+    "ListingOption",
 ]

rclone_api/dir.py

@@ -5,7 +5,7 @@ from typing import Generator
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption
+from rclone_api.types import ListingOption, Order
 
 
 class Dir:
@@ -47,7 +47,7 @@ class Dir:
         self,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files and directories in the given path."""
@@ -57,7 +57,7 @@ class Dir:
             dir,
             max_depth=max_depth,
             glob=glob,
-            reverse=reverse,
+            order=order,
             listing_option=listing_option,
         )
 

rclone_api/rclone.py

@@ -3,6 +3,7 @@ Unit test file.
 """
 
 import os
+import random
 import subprocess
 import time
 import warnings
@@ -25,7 +26,7 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption, ModTimeStrategy
+from rclone_api.types import ListingOption, ModTimeStrategy, Order
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -118,7 +119,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files in the given path.
@@ -164,8 +165,10 @@ class Rclone:
         if glob is not None:
             paths = [p for p in paths if fnmatch(p.path, glob)]
 
-        if reverse:
+        if order == Order.REVERSE:
             paths.reverse()
+        elif order == Order.RANDOM:
+            random.shuffle(paths)
         return DirListing(paths)
 
     def listremotes(self) -> list[Remote]:
@@ -242,7 +245,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int = -1,
         breadth_first: bool = True,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[DirListing, None, None]:
         """Walk through the given path recursively.
 
@@ -277,15 +280,15 @@ class Rclone:
             assert f"Invalid type for path: {type(path)}"
 
         yield from walk(
-            dir_obj, max_depth=max_depth, breadth_first=breadth_first, reverse=reverse
+            dir_obj, max_depth=max_depth, breadth_first=breadth_first, order=order
         )
 
-    def diff_walk(
+    def scan_missing_folders(
         self,
         src: Dir | Remote | str,
         dst: Dir | Remote | str,
         max_depth: int = -1,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[Dir, None, None]:
         """Walk through the given path recursively.
 
@@ -299,12 +302,12 @@ class Rclone:
         Yields:
             DirListing: Directory listing for each directory encountered
         """
-        from rclone_api.diff_walk import diff_walk
+        from rclone_api.scan_missing_folders import scan_missing_folders
 
         src_dir = Dir(to_path(src, self))
         dst_dir = Dir(to_path(dst, self))
-        yield from diff_walk(
-            src=src_dir, dst=dst_dir, max_depth=max_depth, reverse=reverse
+        yield from scan_missing_folders(
+            src=src_dir, dst=dst_dir, max_depth=max_depth, order=order
         )
 
     def cleanup(
@@ -426,10 +429,6 @@ class Rclone:
                                 chunk = files_fqdn[i : i + chunk_size]
                                 files_str = "\n".join(chunk)
                                 print(f"{files_str}")
-                            # files_str = "\n".join(files_fqdn)
-                            # print(f"Copying {nfiles} files: \n{files_str}")
-
-                        # print(include_files_txt)
                         cmd_list: list[str] = [
                             "copy",
                             src_path,

rclone_api/{diff_walk.py → scan_missing_folders.py}

@@ -1,3 +1,4 @@
+import random
 import time
 from concurrent.futures import ThreadPoolExecutor
 from queue import Empty, Queue
@@ -6,7 +7,7 @@ from typing import Generator
 
 from rclone_api import Dir
 from rclone_api.dir_listing import DirListing
-from rclone_api.types import ListingOption
+from rclone_api.types import ListingOption, Order
 from rclone_api.walk import walk_runner_depth_first
 
 _MAX_OUT_QUEUE_SIZE = 50
@@ -14,34 +15,45 @@ _MAX_OUT_QUEUE_SIZE = 50
 
 # ONLY Works from src -> dst diffing.
 def _async_diff_dir_walk_task(
-    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
 ) -> None:
     curr_src, curr_dst = src, dst
+    can_scan_two_deep = max_depth > 1 or max_depth == -1
+    ls_depth = 2 if can_scan_two_deep else 1
     with ThreadPoolExecutor(max_workers=2) as executor:
-        # src_dir_listing = src.ls(listing_option=ListingOption.DIRS_ONLY)
-        # dst_dir_listing = dst.ls(listing_option=ListingOption.DIRS_ONLY)
         t1 = executor.submit(
-            src.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+            src.ls,
+            listing_option=ListingOption.DIRS_ONLY,
+            order=order,
+            max_depth=ls_depth,
         )
         t2 = executor.submit(
-            dst.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+            dst.ls,
+            listing_option=ListingOption.DIRS_ONLY,
+            order=order,
+            max_depth=ls_depth,
         )
         src_dir_listing: DirListing = t1.result()
         dst_dir_listing: DirListing = t2.result()
     next_depth = max_depth - 1 if max_depth > 0 else max_depth
-    dst_files: list[str] = [d.name for d in dst_dir_listing.dirs]
-    src_files: list[str] = [d.name for d in src_dir_listing.dirs]
-    dst_files_set: set[str] = set(dst_files)
+    dst_dirs: list[str] = [d.name for d in dst_dir_listing.dirs]
+    src_dirs: list[str] = [d.name for d in src_dir_listing.dirs]
+    dst_files_set: set[str] = set(dst_dirs)
     matching_dirs: list[str] = []
-    for file in src_files:
+    if order == Order.REVERSE:
+        src_dirs.reverse()
+        dst_dirs.reverse()
+    elif order == Order.RANDOM:
+        random.shuffle(src_dirs)
+        random.shuffle(dst_dirs)
+    for file in src_dirs:
         if file not in dst_files_set:
-            # print(f"missing dir on src: {file}")
             queue_dir_listing: Queue[DirListing | None] = Queue()
             if next_depth > 0 or next_depth == -1:
                 walk_runner_depth_first(
                     dir=curr_src,
                     out_queue=queue_dir_listing,
-                    reverse=reverse,
+                    order=order,
                     max_depth=next_depth,
                 )
             while dirlisting := queue_dir_listing.get():
@@ -63,15 +75,17 @@ def _async_diff_dir_walk_task(
                 dst=dst_next,
                 max_depth=next_depth,
                 out_queue=out_queue,
-                reverse=reverse,
+                order=order,
             )
 
 
 def async_diff_dir_walk_task(
-    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse=False
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
 ) -> None:
     try:
-        _async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse)
+        _async_diff_dir_walk_task(
+            src=src, dst=dst, max_depth=max_depth, out_queue=out_queue, order=order
+        )
     except Exception:
         import _thread
 
@@ -81,11 +95,11 @@ def async_diff_dir_walk_task(
         out_queue.put(None)
 
 
-def diff_walk(
+def scan_missing_folders(
     src: Dir,
     dst: Dir,
     max_depth: int = -1,
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> Generator[Dir, None, None]:
     """Walk through the given directory recursively.
 
@@ -101,7 +115,13 @@ def diff_walk(
         out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
 
         def task() -> None:
-            async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse=reverse)
+            async_diff_dir_walk_task(
+                src=src,
+                dst=dst,
+                max_depth=max_depth,
+                out_queue=out_queue,
+                order=order,
+            )
 
         worker = Thread(
             target=task,
@@ -111,10 +131,10 @@ def diff_walk(
 
         while True:
             try:
-                dirlisting = out_queue.get_nowait()
-                if dirlisting is None:
+                dir = out_queue.get_nowait()
+                if dir is None:
                     break
-                yield dirlisting
+                yield dir
             except Empty:
                 time.sleep(0.1)
 

rclone_api/types.py

@@ -10,3 +10,9 @@ class ListingOption(Enum):
     DIRS_ONLY = "dirs-only"
     FILES_ONLY = "files-only"
     ALL = "all"
+
+
+class Order(Enum):
+    NORMAL = "normal"
+    REVERSE = "reverse"
+    RANDOM = "random"

rclone_api/walk.py

@@ -1,3 +1,4 @@
+import random
 from queue import Queue
 from threading import Thread
 from typing import Generator
@@ -5,6 +6,7 @@ from typing import Generator
 from rclone_api import Dir
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
+from rclone_api.types import Order
 
 _MAX_OUT_QUEUE_SIZE = 50
 
@@ -13,14 +15,14 @@ def walk_runner_breadth_first(
     dir: Dir,
     max_depth: int,
     out_queue: Queue[DirListing | None],
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> None:
     queue: Queue[Dir] = Queue()
     queue.put(dir)
     try:
         while not queue.empty():
             current_dir = queue.get()
-            dirlisting = current_dir.ls(max_depth=0, reverse=reverse)
+            dirlisting = current_dir.ls(max_depth=0, order=order)
             out_queue.put(dirlisting)
             dirs = dirlisting.dirs
 
@@ -41,22 +43,26 @@ def walk_runner_breadth_first(
 
 
 def walk_runner_depth_first(
-    dir: Dir, max_depth: int, out_queue: Queue[DirListing | None], reverse=False
+    dir: Dir,
+    max_depth: int,
+    out_queue: Queue[DirListing | None],
+    order: Order = Order.NORMAL,
 ) -> None:
     try:
         stack = [(dir, max_depth)]
         while stack:
             current_dir, depth = stack.pop()
             dirlisting = current_dir.ls()
-            if reverse:
+            if order == Order.REVERSE:
                 dirlisting.dirs.reverse()
+            if order == Order.RANDOM:
+
+                random.shuffle(dirlisting.dirs)
             if depth != 0:
                 for subdir in dirlisting.dirs:  # Process deeper directories first
                     # stack.append((child, depth - 1 if depth > 0 else depth))
                     next_depth = depth - 1 if depth > 0 else depth
-                    walk_runner_depth_first(
-                        subdir, next_depth, out_queue, reverse=reverse
-                    )
+                    walk_runner_depth_first(subdir, next_depth, out_queue, order=order)
             out_queue.put(dirlisting)
         out_queue.put(None)
     except KeyboardInterrupt:
@@ -70,7 +76,7 @@ def walk(
     dir: Dir | Remote,
     breadth_first: bool,
     max_depth: int = -1,
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> Generator[DirListing, None, None]:
     """Walk through the given directory recursively.
 
@@ -89,9 +95,9 @@ def walk(
 
         def _task() -> None:
             if breadth_first:
-                walk_runner_breadth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_breadth_first(dir, max_depth, out_queue, order)
             else:
-                walk_runner_depth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_depth_first(dir, max_depth, out_queue, order)
 
         # Start worker thread
         worker = Thread(

{rclone_api-1.0.70.dist-info → rclone_api-1.0.73.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.70
+Version: 1.0.73
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.70.dist-info → rclone_api-1.0.73.dist-info}/RECORD

@@ -1,29 +1,29 @@
-rclone_api/__init__.py,sha256=B9q_4JU5Myh0DN-wqr_IgF5hZeAvszrGcqs2x-3hq-Q,633
+rclone_api/__init__.py,sha256=kJUk9KAxQ2AEms9lM5yqw-dqlabarhU69b2FAxJVBlY,692
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=Pp-hXnLgej0IGO5ee9Fmx64dGzIofbQFEUyXdFCvO54,1371
 rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
 rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
 rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
 rclone_api/diff.py,sha256=ggdDLUZxa13jMcPzKBcwAElmPCNWMOSR89D4yhpO74M,5264
-rclone_api/diff_walk.py,sha256=fnCjzenKBBocu6sEd9CDEcwNqCUYwz3rZsxUuKNaYEg,4049
-rclone_api/dir.py,sha256=BuXPd-bAvZund8k7mXjvx_UylsPFwbWq2zaUdflTX04,3520
+rclone_api/dir.py,sha256=hL0i4zrNUbpvFWI3TvKyOyanJ2okcb1lz4r2kxlvSi4,3529
 rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
 rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
 rclone_api/file.py,sha256=YtR5Y6c0YfXTS-sReOy2UgiSnafcAeO6b2hnbojBQD4,1423
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=kOHh6ysFDkxjldSwvW6KqmiADUC1yFCdrZRY57TvbGY,5328
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=nN1bqUmiopCKP6VNG4XpSfBfWx4EuAIyxB9NevYp-OA,28854
+rclone_api/rclone.py,sha256=kIamoje3fUaWboMdF_d_a4WVaSa8BfK6zDFY8U6pNbs,28820
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
-rclone_api/types.py,sha256=VXA-56va2S41onAWM_UmhHlG4qeyMUEF5dsVFyfLlH8,232
+rclone_api/scan_missing_folders.py,sha256=eySQNFwsHxdhN6AAB4K94iH9rJDWUHQIJbddlZKvdFQ,4441
+rclone_api/types.py,sha256=DcbNw1R6j2f_1zHrhqEJcpCR-8kJfFJawMY0AmPsCnM,321
 rclone_api/util.py,sha256=_cvmHcJPRl2yXw4zgZiop3z-riA_8Ek6S5NDPw8cqSY,4198
-rclone_api/walk.py,sha256=ndWV7WBVQLbpZu3HuJrAe1cXcmQVjT9_YPsbat158bQ,3231
+rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.70.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.70.dist-info/METADATA,sha256=73129SIcLoTZ3R35zB0vuZZ65SC0RBttGk1LtPARAx8,4489
-rclone_api-1.0.70.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.70.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.70.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.70.dist-info/RECORD,,
+rclone_api-1.0.73.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.73.dist-info/METADATA,sha256=le9xg4WsNvEKk5SO_RVrqJ9kfIiMS7ASt6bhAJ68GpE,4489
+rclone_api-1.0.73.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.73.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.73.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.73.dist-info/RECORD,,