rclone-api 1.0.68__tar.gz → 1.0.69__tar.gz

{rclone_api-1.0.68 → rclone_api-1.0.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.68
+Version: 1.0.69
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.68 → rclone_api-1.0.69}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.68"
+version = "1.0.69"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.0.69/src/rclone_api/diff_walk.py

@@ -0,0 +1,143 @@
+from concurrent.futures import ThreadPoolExecutor
+from queue import Queue
+from threading import Thread
+from typing import Generator
+
+from rclone_api import Dir
+from rclone_api.dir_listing import DirListing
+from rclone_api.types import ListingOption
+from rclone_api.walk import walk_runner_depth_first
+
+_MAX_OUT_QUEUE_SIZE = 50
+
+
+# ONLY Works from src -> dst diffing.
+def _async_diff_dir_walk_task(
+    src: Dir, dst: Dir, max_depth: int, out_queue: Queue[Dir | None], reverse=False
+) -> None:
+
+    try:
+        stack = [(src, dst)]
+        while stack:
+            curr_src, curr_dst = stack.pop()
+            curr_src = curr_src
+            curr_dst = curr_dst
+
+            with ThreadPoolExecutor(max_workers=2) as executor:
+                # src_dir_listing = src.ls(listing_option=ListingOption.DIRS_ONLY)
+                # dst_dir_listing = dst.ls(listing_option=ListingOption.DIRS_ONLY)
+                t1 = executor.submit(
+                    src.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+                )
+                t2 = executor.submit(
+                    dst.ls, listing_option=ListingOption.DIRS_ONLY, reverse=reverse
+                )
+                src_dir_listing: DirListing = t1.result()
+                dst_dir_listing: DirListing = t2.result()
+
+            # dirlisting = current_dir.ls()
+            # if reverse:
+            #     dirlisting.dirs.reverse()
+            # if depth != 0:
+            #     for subdir in dirlisting.dirs:  # Process deeper directories first
+            #         # stack.append((child, depth - 1 if depth > 0 else depth))
+            #         next_depth = depth - 1 if depth > 0 else depth
+            #         _walk_runner_depth_first(
+            #             subdir, next_depth, out_queue, reverse=reverse
+            #         )
+            # out_queue.put(dirlisting)
+
+            # for subdir in dst_dir_listing.dirs:
+            #     subdir.to_string(include_remote=False)
+            #     walk_runner_depth_first()
+
+            # find elements missing on dst
+            # missing_on_dst: set[Dir] = set(src_dir_listing.dirs) - set(
+            #     dst_dir_listing.dirs
+            # )
+            # exists_on_dst: set[Dir] = set(src_dir_listing.dirs) - missing_on_dst
+
+            dst_files: list[str] = [d.name for d in dst_dir_listing.dirs]
+            src_files: list[str] = [d.name for d in src_dir_listing.dirs]
+
+            dst_files_set: set[str] = set(dst_files)
+            # src_files_set: set[str] = set(src_files)
+
+            # print(f"src_files: {src_files}")
+            # print(f"dst_files: {dst_files}")
+
+            matching_dirs: list[str] = []
+
+            for file in src_files:
+                if file not in dst_files_set:
+                    # print(f"missing dir on src: {file}")
+                    queue_dir_listing: Queue[DirListing | None] = Queue()
+                    walk_runner_depth_first(
+                        dir=curr_src,
+                        max_depth=max_depth,
+                        out_queue=queue_dir_listing,
+                        reverse=reverse,
+                    )
+                    while dirlisting := queue_dir_listing.get():
+                        if dirlisting is None:
+                            break
+                        # print(f"dirlisting: {dirlisting}")
+                        for d in dirlisting.dirs:
+                            out_queue.put(d)
+                else:
+                    matching_dirs.append(file)
+
+            for matching_dir in matching_dirs:
+                # print(f"matching dir: {matching_dir}")
+                _async_diff_dir_walk_task(
+                    src=curr_src / matching_dir,
+                    dst=curr_dst / matching_dir,
+                    max_depth=max_depth,
+                    out_queue=out_queue,
+                    reverse=reverse,
+                )
+
+        out_queue.put(None)
+    except KeyboardInterrupt:
+        import _thread
+
+        out_queue.put(None)
+        _thread.interrupt_main()
+
+
+def diff_walk(
+    src: Dir,
+    dst: Dir,
+    max_depth: int = -1,
+    reverse: bool = False,
+) -> Generator[Dir, None, None]:
+    """Walk through the given directory recursively.
+
+    Args:
+        dir: Directory or Remote to walk through
+        max_depth: Maximum depth to traverse (-1 for unlimited)
+
+    Yields:
+        DirListing: Directory listing for each directory encountered
+    """
+
+    try:
+        out_queue: Queue[Dir | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
+
+        def task() -> None:
+            _async_diff_dir_walk_task(src, dst, max_depth, out_queue, reverse=reverse)
+
+        worker = Thread(
+            target=task,
+            daemon=True,
+        )
+        worker.start()
+
+        while dirlisting := out_queue.get():
+            if dirlisting is None:
+                break
+            yield dirlisting
+
+        worker.join()
+    except KeyboardInterrupt:
+        pass

{rclone_api-1.0.68 → rclone_api-1.0.69}/src/rclone_api/dir.py

@@ -1,9 +1,11 @@
 import json
+from pathlib import Path
 from typing import Generator
 
 from rclone_api.dir_listing import DirListing
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.types import ListingOption
 
 
 class Dir:
@@ -41,11 +43,30 @@ class Dir:
         # self.path.set_rclone(self.path.remote.rclone)
         assert self.path.rclone is not None
 
-    def ls(self, max_depth: int = 0, reverse: bool = False) -> DirListing:
+    def ls(
+        self,
+        max_depth: int | None = None,
+        glob: str | None = None,
+        reverse: bool = False,
+        listing_option: ListingOption = ListingOption.ALL,
+    ) -> DirListing:
         """List files and directories in the given path."""
         assert self.path.rclone is not None
         dir = Dir(self.path)
-        return self.path.rclone.ls(dir, max_depth=max_depth, reverse=reverse)
+        return self.path.rclone.ls(
+            dir,
+            max_depth=max_depth,
+            glob=glob,
+            reverse=reverse,
+            listing_option=listing_option,
+        )
+
+    def relative_to(self, other: "Dir") -> str:
+        """Return the relative path to the other directory."""
+        self_path = Path(self.path.path)
+        other_path = Path(other.path.path)
+        rel_path = self_path.relative_to(other_path)
+        return str(rel_path)
 
     def walk(
         self, breadth_first: bool, max_depth: int = -1
@@ -67,3 +88,26 @@ class Dir:
         data = self.path.to_json()
         data_str = json.dumps(data)
         return data_str
+
+    def to_string(self, include_remote: bool = True) -> str:
+        """Convert the File to a string."""
+        out = str(self.path)
+        if not include_remote:
+            _, out = out.split(":", 1)
+        return out
+
+    # / operator
+    def __truediv__(self, other: str) -> "Dir":
+        """Join the current path with another path."""
+        path = Path(self.path.path) / other
+        rpath = RPath(
+            self.path.remote,
+            str(path),
+            name=other,
+            size=0,
+            mime_type="inode/directory",
+            mod_time="",
+            is_dir=True,
+        )
+        rpath.set_rclone(self.path.rclone)
+        return Dir(rpath)

{rclone_api-1.0.68 → rclone_api-1.0.69}/src/rclone_api/rclone.py

@@ -7,7 +7,6 @@ import subprocess
 import time
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
-from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -26,6 +25,7 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.types import ListingOption, ModTimeStrategy
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -42,17 +42,6 @@ def rclone_verbose(verbose: bool | None) -> bool:
     return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
 
 
-class ModTimeStrategy(Enum):
-    USE_SERVER_MODTIME = "use-server-modtime"
-    NO_MODTIME = "no-modtime"
-
-
-class ListingOption(Enum):
-    DIRS_ONLY = "dirs-only"
-    FILES_ONLY = "files-only"
-    ALL = "all"
-
-
 class Rclone:
     def __init__(
         self, rclone_conf: Path | Config, rclone_exe: Path | None = None
@@ -203,17 +192,20 @@ class Rclone:
         diff_option: DiffOption = DiffOption.COMBINED,
         fast_list: bool = True,
         size_only: bool | None = None,
+        checkers: int | None = None,
         other_args: list[str] | None = None,
     ) -> Generator[DiffItem, None, None]:
         """Be extra careful with the src and dst values. If you are off by one
         parent directory, you will get a huge amount of false diffs."""
         other_args = other_args or []
+        if checkers is None or checkers < 1:
+            checkers = 1000
         cmd = [
             "check",
             src,
             dst,
             "--checkers",
-            "1000",
+            str(checkers),
             "--log-level",
             "INFO",
             f"--{diff_option.value}",
@@ -288,6 +280,33 @@ class Rclone:
             dir_obj, max_depth=max_depth, breadth_first=breadth_first, reverse=reverse
         )
 
+    def diff_walk(
+        self,
+        src: Dir | Remote | str,
+        dst: Dir | Remote | str,
+        max_depth: int = -1,
+        reverse: bool = False,
+    ) -> Generator[Dir, None, None]:
+        """Walk through the given path recursively.
+
+        WORK IN PROGRESS!!
+
+        Args:
+            src: Source directory or Remote to walk through
+            dst: Destination directory or Remote to walk through
+            max_depth: Maximum depth to traverse (-1 for unlimited)
+
+        Yields:
+            DirListing: Directory listing for each directory encountered
+        """
+        from rclone_api.diff_walk import diff_walk
+
+        src_dir = Dir(to_path(src, self))
+        dst_dir = Dir(to_path(dst, self))
+        yield from diff_walk(
+            src=src_dir, dst=dst_dir, max_depth=max_depth, reverse=reverse
+        )
+
     def cleanup(
         self, path: str, other_args: list[str] | None = None
     ) -> CompletedProcess:

rclone_api-1.0.69/src/rclone_api/types.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class ModTimeStrategy(Enum):
+    USE_SERVER_MODTIME = "use-server-modtime"
+    NO_MODTIME = "no-modtime"
+
+
+class ListingOption(Enum):
+    DIRS_ONLY = "dirs-only"
+    FILES_ONLY = "files-only"
+    ALL = "all"

{rclone_api-1.0.68 → rclone_api-1.0.69}/src/rclone_api/walk.py

@@ -9,7 +9,7 @@ from rclone_api.remote import Remote
 _MAX_OUT_QUEUE_SIZE = 50
 
 
-def _walk_runner_breadth_first(
+def walk_runner_breadth_first(
     dir: Dir,
     max_depth: int,
     out_queue: Queue[DirListing | None],
@@ -40,7 +40,7 @@ def _walk_runner_breadth_first(
         _thread.interrupt_main()
 
 
-def _walk_runner_depth_first(
+def walk_runner_depth_first(
     dir: Dir, max_depth: int, out_queue: Queue[DirListing | None], reverse=False
 ) -> None:
     try:
@@ -54,7 +54,7 @@ def _walk_runner_depth_first(
                 for subdir in dirlisting.dirs:  # Process deeper directories first
                     # stack.append((child, depth - 1 if depth > 0 else depth))
                     next_depth = depth - 1 if depth > 0 else depth
-                    _walk_runner_depth_first(
+                    walk_runner_depth_first(
                         subdir, next_depth, out_queue, reverse=reverse
                     )
             out_queue.put(dirlisting)
@@ -85,16 +85,17 @@ def walk(
         # Convert Remote to Dir if needed
         if isinstance(dir, Remote):
             dir = Dir(dir)
-        out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
+        out_queue: Queue[DirListing | None] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
 
-        strategy = (
-            _walk_runner_breadth_first if breadth_first else _walk_runner_depth_first
-        )
+        def _task() -> None:
+            if breadth_first:
+                walk_runner_breadth_first(dir, max_depth, out_queue, reverse)
+            else:
+                walk_runner_depth_first(dir, max_depth, out_queue, reverse)
 
         # Start worker thread
         worker = Thread(
-            target=strategy,
-            args=(dir, max_depth, out_queue, reverse),
+            target=_task,
             daemon=True,
         )
         worker.start()

{rclone_api-1.0.68 → rclone_api-1.0.69}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.68
+Version: 1.0.69
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.68 → rclone_api-1.0.69}/src/rclone_api.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ src/rclone_api/config.py
 src/rclone_api/convert.py
 src/rclone_api/deprecated.py
 src/rclone_api/diff.py
+src/rclone_api/diff_walk.py
 src/rclone_api/dir.py
 src/rclone_api/dir_listing.py
 src/rclone_api/exec.py
@@ -37,6 +38,7 @@ src/rclone_api/process.py
 src/rclone_api/rclone.py
 src/rclone_api/remote.py
 src/rclone_api/rpath.py
+src/rclone_api/types.py
 src/rclone_api/util.py
 src/rclone_api/walk.py
 src/rclone_api.egg-info/PKG-INFO
@@ -51,6 +53,7 @@ tests/test_cmd_list_files.py
 tests/test_copy.py
 tests/test_copy_files.py
 tests/test_diff.py
+tests/test_diff_walk.py
 tests/test_group_files.py
 tests/test_is_synced.py
 tests/test_ls.py

rclone_api-1.0.69/tests/test_diff_walk.py

@@ -0,0 +1,73 @@
+"""
+Unit test file.
+"""
+
+import os
+import unittest
+
+from dotenv import load_dotenv
+
+from rclone_api import Config, Dir, Rclone
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config() -> Config:
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+"""
+
+    out = Config(config_text)
+    return out
+
+
+class RcloneDiffTests(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_diff_walk(self) -> None:
+        """Test copying a single file to remote storage."""
+        rclone = Rclone(_generate_rclone_config())
+        item: Dir
+        all: list[Dir] = []
+        for item in rclone.diff_walk(
+            "dst:rclone-api-unit-test", "dst:rclone-api-unit-test"
+        ):
+            all.append(item)
+        self.assertEqual(len(all), 0)
+        msg = "\n".join([str(item) for item in all])
+        print(msg)
+
+
+if __name__ == "__main__":
+    unittest.main()