rclone-api 1.0.70__tar.gz → 1.0.73__tar.gz

{rclone_api-1.0.70 → rclone_api-1.0.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.70
+Version: 1.0.73
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.70 → rclone_api-1.0.73}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.70"
+version = "1.0.73"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api/__init__.py

@@ -1,14 +1,15 @@
 from .completed_process import CompletedProcess
 from .config import Config
-from .diff import DiffItem, DiffType
+from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
 from .file import File
 from .filelist import FileList
 from .process import Process
-from .rclone import DiffOption, ListingOption, Rclone, rclone_verbose
+from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .types import ListingOption, Order
 
 __all__ = [
     "Rclone",
@@ -26,4 +27,6 @@ __all__ = [
     "CompletedProcess",
     "DiffOption",
     "ListingOption",
+    "Order",
+    "ListingOption",
 ]

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api/dir.py

@@ -5,7 +5,7 @@ from typing import Generator
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption
+from rclone_api.types import ListingOption, Order
 
 
 class Dir:
@@ -47,7 +47,7 @@ class Dir:
         self,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files and directories in the given path."""
@@ -57,7 +57,7 @@ class Dir:
             dir,
             max_depth=max_depth,
             glob=glob,
-            reverse=reverse,
+            order=order,
             listing_option=listing_option,
         )
 

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api/rclone.py

@@ -3,6 +3,7 @@ Unit test file.
 """
 
 import os
+import random
 import subprocess
 import time
 import warnings
@@ -25,7 +26,7 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption, ModTimeStrategy
+from rclone_api.types import ListingOption, ModTimeStrategy, Order
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -118,7 +119,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files in the given path.
@@ -164,8 +165,10 @@ class Rclone:
         if glob is not None:
             paths = [p for p in paths if fnmatch(p.path, glob)]
 
-        if reverse:
+        if order == Order.REVERSE:
             paths.reverse()
+        elif order == Order.RANDOM:
+            random.shuffle(paths)
         return DirListing(paths)
 
     def listremotes(self) -> list[Remote]:
@@ -242,7 +245,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int = -1,
         breadth_first: bool = True,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[DirListing, None, None]:
         """Walk through the given path recursively.
 
@@ -277,15 +280,15 @@ class Rclone:
             assert f"Invalid type for path: {type(path)}"
 
         yield from walk(
-            dir_obj, max_depth=max_depth, breadth_first=breadth_first, reverse=reverse
+            dir_obj, max_depth=max_depth, breadth_first=breadth_first, order=order
         )
 
-    def diff_walk(
+    def scan_missing_folders(
         self,
         src: Dir | Remote | str,
         dst: Dir | Remote | str,
         max_depth: int = -1,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[Dir, None, None]:
         """Walk through the given path recursively.
 
@@ -299,12 +302,12 @@ class Rclone:
         Yields:
             DirListing: Directory listing for each directory encountered
         """
-        from rclone_api.diff_walk import diff_walk
+        from rclone_api.scan_missing_folders import scan_missing_folders
 
         src_dir = Dir(to_path(src, self))
         dst_dir = Dir(to_path(dst, self))
-        yield from diff_walk(
-            src=src_dir, dst=dst_dir, max_depth=max_depth, reverse=reverse
+        yield from scan_missing_folders(
+            src=src_dir, dst=dst_dir, max_depth=max_depth, order=order
         )
 
     def cleanup(
@@ -426,10 +429,6 @@ class Rclone:
                                 chunk = files_fqdn[i : i + chunk_size]
                                 files_str = "\n".join(chunk)
                                 print(f"{files_str}")
-                            # files_str = "\n".join(files_fqdn)
-                            # print(f"Copying {nfiles} files: \n{files_str}")
-
-                        # print(include_files_txt)
                         cmd_list: list[str] = [
                             "copy",
                             src_path,

rclone_api-1.0.70/src/rclone_api/diff_walk.py → rclone_api-1.0.73/src/rclone_api/scan_missing_folders.py

@@ -1,3 +1,4 @@
+import random
 import time
 from concurrent.futures import ThreadPoolExecutor
 from queue import Empty, Queue
@@ -6,7 +7,7 @@ from typing import Generator
 
 from rclone_api import Dir
 from rclone_api.dir_listing import DirListing
-from rclone_api.types import ListingOption
+from rclone_api.types import ListingOption, Order
 from rclone_api.walk import walk_runner_depth_first
 
 _MAX_OUT_QUEUE_SIZE = 50
@@ -14,34 +15,45 @@ _MAX_OUT_QUEUE_SIZE = 50
 
 # ONLY Works from src -> dst diffing.
 def _async_diff_dir_walk_task(
-    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
 ) -> None:
     curr_src, curr_dst = src, dst
+    can_scan_two_deep = max_depth > 1 or max_depth == -1
+    ls_depth = 2 if can_scan_two_deep else 1
     with ThreadPoolExecutor(max_workers=2) as executor:
-        # src_dir_listing = src.ls(listing_option=ListingOption.DIRS_ONLY)
-        # dst_dir_listing = dst.ls(listing_option=ListingOption.DIRS_ONLY)
         t1 = executor.submit(
-            src.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+            src.ls,
+            listing_option=ListingOption.DIRS_ONLY,
+            order=order,
+            max_depth=ls_depth,
         )
         t2 = executor.submit(
-            dst.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+            dst.ls,
+            listing_option=ListingOption.DIRS_ONLY,
+            order=order,
+            max_depth=ls_depth,
         )
         src_dir_listing: DirListing = t1.result()
         dst_dir_listing: DirListing = t2.result()
     next_depth = max_depth - 1 if max_depth > 0 else max_depth
-    dst_files: list[str] = [d.name for d in dst_dir_listing.dirs]
-    src_files: list[str] = [d.name for d in src_dir_listing.dirs]
-    dst_files_set: set[str] = set(dst_files)
+    dst_dirs: list[str] = [d.name for d in dst_dir_listing.dirs]
+    src_dirs: list[str] = [d.name for d in src_dir_listing.dirs]
+    dst_files_set: set[str] = set(dst_dirs)
     matching_dirs: list[str] = []
-    for file in src_files:
+    if order == Order.REVERSE:
+        src_dirs.reverse()
+        dst_dirs.reverse()
+    elif order == Order.RANDOM:
+        random.shuffle(src_dirs)
+        random.shuffle(dst_dirs)
+    for file in src_dirs:
         if file not in dst_files_set:
-            # print(f"missing dir on src: {file}")
             queue_dir_listing: Queue[DirListing | None] = Queue()
             if next_depth > 0 or next_depth == -1:
                 walk_runner_depth_first(
                     dir=curr_src,
                     out_queue=queue_dir_listing,
-                    reverse=reverse,
+                    order=order,
                     max_depth=next_depth,
                 )
             while dirlisting := queue_dir_listing.get():
@@ -63,15 +75,17 @@ def _async_diff_dir_walk_task(
                 dst=dst_next,
                 max_depth=next_depth,
                 out_queue=out_queue,
-                reverse=reverse,
+                order=order,
             )
 
 
 def async_diff_dir_walk_task(
-    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse=False
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
 ) -> None:
     try:
-        _async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse)
+        _async_diff_dir_walk_task(
+            src=src, dst=dst, max_depth=max_depth, out_queue=out_queue, order=order
+        )
     except Exception:
         import _thread
 
@@ -81,11 +95,11 @@ def async_diff_dir_walk_task(
         out_queue.put(None)
 
 
-def diff_walk(
+def scan_missing_folders(
     src: Dir,
     dst: Dir,
     max_depth: int = -1,
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> Generator[Dir, None, None]:
     """Walk through the given directory recursively.
 
@@ -101,7 +115,13 @@ def diff_walk(
         out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
 
         def task() -> None:
-            async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse=reverse)
+            async_diff_dir_walk_task(
+                src=src,
+                dst=dst,
+                max_depth=max_depth,
+                out_queue=out_queue,
+                order=order,
+            )
 
         worker = Thread(
             target=task,
@@ -111,10 +131,10 @@ def diff_walk(
 
         while True:
             try:
-                dirlisting = out_queue.get_nowait()
-                if dirlisting is None:
+                dir = out_queue.get_nowait()
+                if dir is None:
                     break
-                yield dirlisting
+                yield dir
             except Empty:
                 time.sleep(0.1)
 

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api/types.py

@@ -10,3 +10,9 @@ class ListingOption(Enum):
     DIRS_ONLY = "dirs-only"
     FILES_ONLY = "files-only"
     ALL = "all"
+
+
+class Order(Enum):
+    NORMAL = "normal"
+    REVERSE = "reverse"
+    RANDOM = "random"

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api/walk.py

@@ -1,3 +1,4 @@
+import random
 from queue import Queue
 from threading import Thread
 from typing import Generator
@@ -5,6 +6,7 @@ from typing import Generator
 from rclone_api import Dir
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
+from rclone_api.types import Order
 
 _MAX_OUT_QUEUE_SIZE = 50
 
@@ -13,14 +15,14 @@ def walk_runner_breadth_first(
     dir: Dir,
     max_depth: int,
     out_queue: Queue[DirListing | None],
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> None:
     queue: Queue[Dir] = Queue()
     queue.put(dir)
     try:
         while not queue.empty():
             current_dir = queue.get()
-            dirlisting = current_dir.ls(max_depth=0, reverse=reverse)
+            dirlisting = current_dir.ls(max_depth=0, order=order)
             out_queue.put(dirlisting)
             dirs = dirlisting.dirs
 
@@ -41,22 +43,26 @@ def walk_runner_breadth_first(
 
 
 def walk_runner_depth_first(
-    dir: Dir, max_depth: int, out_queue: Queue[DirListing | None], reverse=False
+    dir: Dir,
+    max_depth: int,
+    out_queue: Queue[DirListing | None],
+    order: Order = Order.NORMAL,
 ) -> None:
     try:
         stack = [(dir, max_depth)]
         while stack:
             current_dir, depth = stack.pop()
             dirlisting = current_dir.ls()
-            if reverse:
+            if order == Order.REVERSE:
                 dirlisting.dirs.reverse()
+            if order == Order.RANDOM:
+
+                random.shuffle(dirlisting.dirs)
             if depth != 0:
                 for subdir in dirlisting.dirs:  # Process deeper directories first
                     # stack.append((child, depth - 1 if depth > 0 else depth))
                     next_depth = depth - 1 if depth > 0 else depth
-                    walk_runner_depth_first(
-                        subdir, next_depth, out_queue, reverse=reverse
-                    )
+                    walk_runner_depth_first(subdir, next_depth, out_queue, order=order)
             out_queue.put(dirlisting)
         out_queue.put(None)
     except KeyboardInterrupt:
@@ -70,7 +76,7 @@ def walk(
     dir: Dir | Remote,
     breadth_first: bool,
     max_depth: int = -1,
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> Generator[DirListing, None, None]:
     """Walk through the given directory recursively.
 
@@ -89,9 +95,9 @@ def walk(
 
         def _task() -> None:
             if breadth_first:
-                walk_runner_breadth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_breadth_first(dir, max_depth, out_queue, order)
             else:
-                walk_runner_depth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_depth_first(dir, max_depth, out_queue, order)
 
         # Start worker thread
         worker = Thread(

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.70
+Version: 1.0.73
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.70 → rclone_api-1.0.73}/src/rclone_api.egg-info/SOURCES.txt

@@ -27,7 +27,6 @@ src/rclone_api/config.py
 src/rclone_api/convert.py
 src/rclone_api/deprecated.py
 src/rclone_api/diff.py
-src/rclone_api/diff_walk.py
 src/rclone_api/dir.py
 src/rclone_api/dir_listing.py
 src/rclone_api/exec.py
@@ -38,6 +37,7 @@ src/rclone_api/process.py
 src/rclone_api/rclone.py
 src/rclone_api/remote.py
 src/rclone_api/rpath.py
+src/rclone_api/scan_missing_folders.py
 src/rclone_api/types.py
 src/rclone_api/util.py
 src/rclone_api/walk.py
@@ -53,7 +53,6 @@ tests/test_cmd_list_files.py
 tests/test_copy.py
 tests/test_copy_files.py
 tests/test_diff.py
-tests/test_diff_walk.py
 tests/test_group_files.py
 tests/test_is_synced.py
 tests/test_ls.py
@@ -63,5 +62,6 @@ tests/test_mount_webdav.py
 tests/test_obscure.py
 tests/test_remote_control.py
 tests/test_remotes.py
+tests/test_scan_missing_folders.py
 tests/test_serve_webdav.py
 tests/test_walk.py

{rclone_api-1.0.70 → rclone_api-1.0.73}/tests/test_diff.py

@@ -7,8 +7,8 @@ import unittest
 
 from dotenv import load_dotenv
 
-from rclone_api import Config, DiffOption, Rclone
-from rclone_api.diff import DiffItem, DiffType
+from rclone_api import Config, Rclone
+from rclone_api.diff import DiffItem, DiffOption, DiffType
 
 load_dotenv()
 

rclone_api-1.0.70/tests/test_diff_walk.py → rclone_api-1.0.73/tests/test_scan_missing_folders.py

@@ -8,6 +8,7 @@ import unittest
 from dotenv import load_dotenv
 
 from rclone_api import Config, Dir, Rclone
+from rclone_api.types import Order
 
 load_dotenv()
 
@@ -37,7 +38,7 @@ endpoint = {BUCKET_URL}
     return out
 
 
-class RcloneDiffTests(unittest.TestCase):
+class RcloneScanMissingFoldersTests(unittest.TestCase):
     """Test rclone functionality."""
 
     def setUp(self) -> None:
@@ -55,13 +56,16 @@ class RcloneDiffTests(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
-    def test_diff_walk(self) -> None:
+    def test_scan_missing_folders(self) -> None:
         """Test copying a single file to remote storage."""
         rclone = Rclone(_generate_rclone_config())
         item: Dir
         all: list[Dir] = []
-        for item in rclone.diff_walk(
-            "dst:rclone-api-unit-test", "dst:rclone-api-unit-test"
+        for item in rclone.scan_missing_folders(
+            src="dst:rclone-api-unit-test",
+            dst="dst:rclone-api-unit-test",
+            max_depth=-1,
+            order=Order.NORMAL,
         ):
             all.append(item)
         self.assertEqual(len(all), 0)