rclone-api 1.0.69__tar.gz → 1.0.72__tar.gz

{rclone_api-1.0.69 → rclone_api-1.0.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.69
+Version: 1.0.72
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.69 → rclone_api-1.0.72}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.69"
+version = "1.0.72"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api/__init__.py

@@ -1,14 +1,15 @@
 from .completed_process import CompletedProcess
 from .config import Config
-from .diff import DiffItem, DiffType
+from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
 from .file import File
 from .filelist import FileList
 from .process import Process
-from .rclone import DiffOption, ListingOption, Rclone, rclone_verbose
+from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .types import ListingOption, Order
 
 __all__ = [
     "Rclone",
@@ -26,4 +27,6 @@ __all__ = [
     "CompletedProcess",
     "DiffOption",
     "ListingOption",
+    "Order",
+    "ListingOption",
 ]

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api/dir.py

@@ -5,7 +5,7 @@ from typing import Generator
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption
+from rclone_api.types import ListingOption, Order
 
 
 class Dir:
@@ -47,7 +47,7 @@ class Dir:
         self,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files and directories in the given path."""
@@ -57,7 +57,7 @@ class Dir:
             dir,
             max_depth=max_depth,
             glob=glob,
-            reverse=reverse,
+            order=order,
             listing_option=listing_option,
         )
 
@@ -102,7 +102,7 @@ class Dir:
         path = Path(self.path.path) / other
         rpath = RPath(
             self.path.remote,
-            str(path),
+            str(path.as_posix()),
             name=other,
             size=0,
             mime_type="inode/directory",

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api/rclone.py

@@ -3,6 +3,7 @@ Unit test file.
 """
 
 import os
+import random
 import subprocess
 import time
 import warnings
@@ -25,7 +26,7 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption, ModTimeStrategy
+from rclone_api.types import ListingOption, ModTimeStrategy, Order
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -118,7 +119,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int | None = None,
         glob: str | None = None,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
         listing_option: ListingOption = ListingOption.ALL,
     ) -> DirListing:
         """List files in the given path.
@@ -164,8 +165,10 @@ class Rclone:
         if glob is not None:
             paths = [p for p in paths if fnmatch(p.path, glob)]
 
-        if reverse:
+        if order == Order.REVERSE:
             paths.reverse()
+        elif order == Order.RANDOM:
+            random.shuffle(paths)
         return DirListing(paths)
 
     def listremotes(self) -> list[Remote]:
@@ -242,7 +245,7 @@ class Rclone:
         path: Dir | Remote | str,
         max_depth: int = -1,
         breadth_first: bool = True,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[DirListing, None, None]:
         """Walk through the given path recursively.
 
@@ -277,15 +280,15 @@ class Rclone:
             assert f"Invalid type for path: {type(path)}"
 
         yield from walk(
-            dir_obj, max_depth=max_depth, breadth_first=breadth_first, reverse=reverse
+            dir_obj, max_depth=max_depth, breadth_first=breadth_first, order=order
         )
 
-    def diff_walk(
+    def scan_missing_folders(
         self,
         src: Dir | Remote | str,
         dst: Dir | Remote | str,
         max_depth: int = -1,
-        reverse: bool = False,
+        order: Order = Order.NORMAL,
     ) -> Generator[Dir, None, None]:
         """Walk through the given path recursively.
 
@@ -299,12 +302,12 @@ class Rclone:
         Yields:
             DirListing: Directory listing for each directory encountered
         """
-        from rclone_api.diff_walk import diff_walk
+        from rclone_api.scan_missing_folders import scan_missing_folders
 
         src_dir = Dir(to_path(src, self))
         dst_dir = Dir(to_path(dst, self))
-        yield from diff_walk(
-            src=src_dir, dst=dst_dir, max_depth=max_depth, reverse=reverse
+        yield from scan_missing_folders(
+            src=src_dir, dst=dst_dir, max_depth=max_depth, order=order
         )
 
     def cleanup(
@@ -426,10 +429,6 @@ class Rclone:
                                 chunk = files_fqdn[i : i + chunk_size]
                                 files_str = "\n".join(chunk)
                                 print(f"{files_str}")
-                            # files_str = "\n".join(files_fqdn)
-                            # print(f"Copying {nfiles} files: \n{files_str}")
-
-                        # print(include_files_txt)
                         cmd_list: list[str] = [
                             "copy",
                             src_path,

rclone_api-1.0.72/src/rclone_api/scan_missing_folders.py

@@ -0,0 +1,135 @@
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor
+from queue import Empty, Queue
+from threading import Thread
+from typing import Generator
+
+from rclone_api import Dir
+from rclone_api.dir_listing import DirListing
+from rclone_api.types import ListingOption, Order
+from rclone_api.walk import walk_runner_depth_first
+
+_MAX_OUT_QUEUE_SIZE = 50
+
+
+# ONLY Works from src -> dst diffing.
+def _async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
+) -> None:
+    curr_src, curr_dst = src, dst
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        t1 = executor.submit(
+            src.ls, listing_option=ListingOption.DIRS_ONLY, order=order
+        )
+        t2 = executor.submit(
+            dst.ls, listing_option=ListingOption.DIRS_ONLY, order=order
+        )
+        src_dir_listing: DirListing = t1.result()
+        dst_dir_listing: DirListing = t2.result()
+    next_depth = max_depth - 1 if max_depth > 0 else max_depth
+    dst_dirs: list[str] = [d.name for d in dst_dir_listing.dirs]
+    src_dirs: list[str] = [d.name for d in src_dir_listing.dirs]
+    dst_files_set: set[str] = set(dst_dirs)
+    matching_dirs: list[str] = []
+    if order == Order.REVERSE:
+        src_dirs.reverse()
+        dst_dirs.reverse()
+    elif order == Order.RANDOM:
+        random.shuffle(src_dirs)
+        random.shuffle(dst_dirs)
+    for file in src_dirs:
+        if file not in dst_files_set:
+            queue_dir_listing: Queue[DirListing | None] = Queue()
+            if next_depth > 0 or next_depth == -1:
+                walk_runner_depth_first(
+                    dir=curr_src,
+                    out_queue=queue_dir_listing,
+                    order=order,
+                    max_depth=next_depth,
+                )
+            while dirlisting := queue_dir_listing.get():
+                if dirlisting is None:
+                    break
+                # print(f"dirlisting: {dirlisting}")
+                for d in dirlisting.dirs:
+                    out_queue.put(d)
+        else:
+            matching_dirs.append(file)
+
+    for matching_dir in matching_dirs:
+        # print(f"matching dir: {matching_dir}")
+        if next_depth > 0 or next_depth == -1:
+            src_next = curr_src / matching_dir
+            dst_next = curr_dst / matching_dir
+            _async_diff_dir_walk_task(
+                src=src_next,
+                dst=dst_next,
+                max_depth=next_depth,
+                out_queue=out_queue,
+                order=order,
+            )
+
+
+def async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], order: Order
+) -> None:
+    try:
+        _async_diff_dir_walk_task(
+            src=src, dst=dst, max_depth=max_depth, out_queue=out_queue, order=order
+        )
+    except Exception:
+        import _thread
+
+        _thread.interrupt_main()
+        raise
+    finally:
+        out_queue.put(None)
+
+
+def scan_missing_folders(
+    src: Dir,
+    dst: Dir,
+    max_depth: int = -1,
+    order: Order = Order.NORMAL,
+) -> Generator[Dir, None, None]:
+    """Walk through the given directory recursively.
+
+    Args:
+        dir: Directory or Remote to walk through
+        max_depth: Maximum depth to traverse (-1 for unlimited)
+
+    Yields:
+        DirListing: Directory listing for each directory encountered
+    """
+
+    try:
+        out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
+
+        def task() -> None:
+            async_diff_dir_walk_task(
+                src=src,
+                dst=dst,
+                max_depth=max_depth,
+                out_queue=out_queue,
+                order=order,
+            )
+
+        worker = Thread(
+            target=task,
+            daemon=True,
+        )
+        worker.start()
+
+        while True:
+            try:
+                dir = out_queue.get_nowait()
+                if dir is None:
+                    break
+                yield dir
+            except Empty:
+                time.sleep(0.1)
+
+        worker.join()
+    except KeyboardInterrupt:
+        pass

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api/types.py

@@ -10,3 +10,9 @@ class ListingOption(Enum):
     DIRS_ONLY = "dirs-only"
     FILES_ONLY = "files-only"
     ALL = "all"
+
+
+class Order(Enum):
+    NORMAL = "normal"
+    REVERSE = "reverse"
+    RANDOM = "random"

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api/walk.py

@@ -1,3 +1,4 @@
+import random
 from queue import Queue
 from threading import Thread
 from typing import Generator
@@ -5,6 +6,7 @@ from typing import Generator
 from rclone_api import Dir
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
+from rclone_api.types import Order
 
 _MAX_OUT_QUEUE_SIZE = 50
 
@@ -13,14 +15,14 @@ def walk_runner_breadth_first(
     dir: Dir,
     max_depth: int,
     out_queue: Queue[DirListing | None],
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> None:
     queue: Queue[Dir] = Queue()
     queue.put(dir)
     try:
         while not queue.empty():
             current_dir = queue.get()
-            dirlisting = current_dir.ls(max_depth=0, reverse=reverse)
+            dirlisting = current_dir.ls(max_depth=0, order=order)
             out_queue.put(dirlisting)
             dirs = dirlisting.dirs
 
@@ -41,22 +43,26 @@ def walk_runner_breadth_first(
 
 
 def walk_runner_depth_first(
-    dir: Dir, max_depth: int, out_queue: Queue[DirListing | None], reverse=False
+    dir: Dir,
+    max_depth: int,
+    out_queue: Queue[DirListing | None],
+    order: Order = Order.NORMAL,
 ) -> None:
     try:
         stack = [(dir, max_depth)]
         while stack:
             current_dir, depth = stack.pop()
             dirlisting = current_dir.ls()
-            if reverse:
+            if order == Order.REVERSE:
                 dirlisting.dirs.reverse()
+            if order == Order.RANDOM:
+
+                random.shuffle(dirlisting.dirs)
             if depth != 0:
                 for subdir in dirlisting.dirs:  # Process deeper directories first
                     # stack.append((child, depth - 1 if depth > 0 else depth))
                     next_depth = depth - 1 if depth > 0 else depth
-                    walk_runner_depth_first(
-                        subdir, next_depth, out_queue, reverse=reverse
-                    )
+                    walk_runner_depth_first(subdir, next_depth, out_queue, order=order)
             out_queue.put(dirlisting)
         out_queue.put(None)
     except KeyboardInterrupt:
@@ -70,7 +76,7 @@ def walk(
     dir: Dir | Remote,
     breadth_first: bool,
     max_depth: int = -1,
-    reverse: bool = False,
+    order: Order = Order.NORMAL,
 ) -> Generator[DirListing, None, None]:
     """Walk through the given directory recursively.
 
@@ -89,9 +95,9 @@ def walk(
 
         def _task() -> None:
             if breadth_first:
-                walk_runner_breadth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_breadth_first(dir, max_depth, out_queue, order)
             else:
-                walk_runner_depth_first(dir, max_depth, out_queue, reverse)
+                walk_runner_depth_first(dir, max_depth, out_queue, order)
 
         # Start worker thread
         worker = Thread(

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.69
+Version: 1.0.72
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.69 → rclone_api-1.0.72}/src/rclone_api.egg-info/SOURCES.txt

@@ -27,7 +27,6 @@ src/rclone_api/config.py
 src/rclone_api/convert.py
 src/rclone_api/deprecated.py
 src/rclone_api/diff.py
-src/rclone_api/diff_walk.py
 src/rclone_api/dir.py
 src/rclone_api/dir_listing.py
 src/rclone_api/exec.py
@@ -38,6 +37,7 @@ src/rclone_api/process.py
 src/rclone_api/rclone.py
 src/rclone_api/remote.py
 src/rclone_api/rpath.py
+src/rclone_api/scan_missing_folders.py
 src/rclone_api/types.py
 src/rclone_api/util.py
 src/rclone_api/walk.py
@@ -53,7 +53,6 @@ tests/test_cmd_list_files.py
 tests/test_copy.py
 tests/test_copy_files.py
 tests/test_diff.py
-tests/test_diff_walk.py
 tests/test_group_files.py
 tests/test_is_synced.py
 tests/test_ls.py
@@ -63,5 +62,6 @@ tests/test_mount_webdav.py
 tests/test_obscure.py
 tests/test_remote_control.py
 tests/test_remotes.py
+tests/test_scan_missing_folders.py
 tests/test_serve_webdav.py
 tests/test_walk.py

{rclone_api-1.0.69 → rclone_api-1.0.72}/tests/test_diff.py

@@ -7,8 +7,8 @@ import unittest
 
 from dotenv import load_dotenv
 
-from rclone_api import Config, DiffOption, Rclone
-from rclone_api.diff import DiffItem, DiffType
+from rclone_api import Config, Rclone
+from rclone_api.diff import DiffItem, DiffOption, DiffType
 
 load_dotenv()
 

rclone_api-1.0.69/tests/test_diff_walk.py → rclone_api-1.0.72/tests/test_scan_missing_folders.py

@@ -8,6 +8,7 @@ import unittest
 from dotenv import load_dotenv
 
 from rclone_api import Config, Dir, Rclone
+from rclone_api.types import Order
 
 load_dotenv()
 
@@ -37,7 +38,7 @@ endpoint = {BUCKET_URL}
     return out
 
 
-class RcloneDiffTests(unittest.TestCase):
+class RcloneScanMissingFoldersTests(unittest.TestCase):
     """Test rclone functionality."""
 
     def setUp(self) -> None:
@@ -55,13 +56,16 @@ class RcloneDiffTests(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
-    def test_diff_walk(self) -> None:
+    def test_scan_missing_folders(self) -> None:
         """Test copying a single file to remote storage."""
         rclone = Rclone(_generate_rclone_config())
         item: Dir
         all: list[Dir] = []
-        for item in rclone.diff_walk(
-            "dst:rclone-api-unit-test", "dst:rclone-api-unit-test"
+        for item in rclone.scan_missing_folders(
+            src="dst:rclone-api-unit-test",
+            dst="dst:rclone-api-unit-test",
+            max_depth=-1,
+            order=Order.NORMAL,
         ):
             all.append(item)
         self.assertEqual(len(all), 0)

rclone_api-1.0.69/src/rclone_api/diff_walk.py

@@ -1,143 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-from queue import Queue
-from threading import Thread
-from typing import Generator
-
-from rclone_api import Dir
-from rclone_api.dir_listing import DirListing
-from rclone_api.types import ListingOption
-from rclone_api.walk import walk_runner_depth_first
-
-_MAX_OUT_QUEUE_SIZE = 50
-
-
-# ONLY Works from src -> dst diffing.
-def _async_diff_dir_walk_task(
-    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse=False
-) -> None:
-
-    try:
-        stack = [(src, dst)]
-        while stack:
-            curr_src, curr_dst = stack.pop()
-            curr_src = curr_src
-            curr_dst = curr_dst
-
-            with ThreadPoolExecutor(max_workers=2) as executor:
-                # src_dir_listing = src.ls(listing_option=ListingOption.DIRS_ONLY)
-                # dst_dir_listing = dst.ls(listing_option=ListingOption.DIRS_ONLY)
-                t1 = executor.submit(
-                    src.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
-                )
-                t2 = executor.submit(
-                    dst.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
-                )
-                src_dir_listing: DirListing = t1.result()
-                dst_dir_listing: DirListing = t2.result()
-
-            # dirlisting = current_dir.ls()
-            # if reverse:
-            #     dirlisting.dirs.reverse()
-            # if depth != 0:
-            #     for subdir in dirlisting.dirs:  # Process deeper directories first
-            #         # stack.append((child, depth - 1 if depth > 0 else depth))
-            #         next_depth = depth - 1 if depth > 0 else depth
-            #         _walk_runner_depth_first(
-            #             subdir, next_depth, out_queue, reverse=reverse
-            #         )
-            # out_queue.put(dirlisting)
-
-            # for subdir in dst_dir_listing.dirs:
-            #     subdir.to_string(include_remote=False)
-            #     walk_runner_depth_first()
-
-            # find elements missing on dst
-            # missing_on_dst: set[Dir] = set(src_dir_listing.dirs) - set(
-            #     dst_dir_listing.dirs
-            # )
-            # exists_on_dst: set[Dir] = set(src_dir_listing.dirs) - missing_on_dst
-
-            dst_files: list[str] = [d.name for d in dst_dir_listing.dirs]
-            src_files: list[str] = [d.name for d in src_dir_listing.dirs]
-
-            dst_files_set: set[str] = set(dst_files)
-            # src_files_set: set[str] = set(src_files)
-
-            # print(f"src_files: {src_files}")
-            # print(f"dst_files: {dst_files}")
-
-            matching_dirs: list[str] = []
-
-            for file in src_files:
-                if file not in dst_files_set:
-                    # print(f"missing dir on src: {file}")
-                    queue_dir_listing: Queue[DirListing | None] = Queue()
-                    walk_runner_depth_first(
-                        dir=curr_src,
-                        max_depth=max_depth,
-                        out_queue=queue_dir_listing,
-                        reverse=reverse,
-                    )
-                    while dirlisting := queue_dir_listing.get():
-                        if dirlisting is None:
-                            break
-                        # print(f"dirlisting: {dirlisting}")
-                        for d in dirlisting.dirs:
-                            out_queue.put(d)
-                else:
-                    matching_dirs.append(file)
-
-            for matching_dir in matching_dirs:
-                # print(f"matching dir: {matching_dir}")
-                _async_diff_dir_walk_task(
-                    src=curr_src / matching_dir,
-                    dst=curr_dst / matching_dir,
-                    max_depth=max_depth,
-                    out_queue=out_queue,
-                    reverse=reverse,
-                )
-
-        out_queue.put(None)
-    except KeyboardInterrupt:
-        import _thread
-
-        out_queue.put(None)
-        _thread.interrupt_main()
-
-
-def diff_walk(
-    src: Dir,
-    dst: Dir,
-    max_depth: int = -1,
-    reverse: bool = False,
-) -> Generator[Dir, None, None]:
-    """Walk through the given directory recursively.
-
-    Args:
-        dir: Directory or Remote to walk through
-        max_depth: Maximum depth to traverse (-1 for unlimited)
-
-    Yields:
-        DirListing: Directory listing for each directory encountered
-    """
-
-    try:
-        out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
-
-        def task() -> None:
-            _async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse=reverse)
-
-        worker = Thread(
-            target=task,
-            daemon=True,
-        )
-        worker.start()
-
-        while dirlisting := out_queue.get():
-            if dirlisting is None:
-                break
-            yield dirlisting
-
-        worker.join()
-    except KeyboardInterrupt:
-        pass