rclone-api 1.0.26__tar.gz → 1.0.28__tar.gz

{rclone_api-1.0.26 → rclone_api-1.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.26
+Version: 1.0.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies
@@ -22,11 +22,11 @@ Dynamic: maintainer
 [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
 [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
 
-Api version of rclone. It's well tested. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done.
+Api version of rclone. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done. Unit tests up the wazoo. S3 Mounts on windows/linux are heavily optimized.
 
 You will need to have rclone installed and on your path.
 
-One of the benefits of this api is that it does not use `shell=True`, which can keep `rclone` running in some instances even when try to kill the process.
+One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
 
 # Install
 

{rclone_api-1.0.26 → rclone_api-1.0.28}/README.md

@@ -5,11 +5,11 @@
 [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
 [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
 
-Api version of rclone. It's well tested. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done.
+Api version of rclone. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done. Unit tests up the wazoo. S3 Mounts on windows/linux are heavily optimized.
 
 You will need to have rclone installed and on your path.
 
-One of the benefits of this api is that it does not use `shell=True`, which can keep `rclone` running in some instances even when try to kill the process.
+One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
 
 # Install
 

{rclone_api-1.0.26 → rclone_api-1.0.28}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.26"
+version = "1.0.28"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api/__init__.py

@@ -1,4 +1,5 @@
 from .config import Config
+from .diff import DiffItem, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
 from .file import File
@@ -18,4 +19,6 @@ __all__ = [
     "DirListing",
     "FileList",
     "Process",
+    "DiffItem",
+    "DiffType",
 ]

rclone_api-1.0.28/src/rclone_api/diff.py

@@ -0,0 +1,119 @@
+from dataclasses import dataclass
+from enum import Enum
+from queue import Queue
+from threading import Thread
+from typing import Generator
+
+from rclone_api.process import Process
+
+
+class DiffType(Enum):
+    EQUAL = "="
+    MISSING_ON_SRC = (
+        "-"  # means path was missing on the source, so only in the destination
+    )
+    MISSING_ON_DST = (
+        "+"  # means path was missing on the destination, so only in the source
+    )
+    DIFFERENT = "*"  # means path was present in source and destination but different.
+    ERROR = "!"  # means there was an error
+
+
+@dataclass
+class DiffItem:
+    type: DiffType
+    path: str
+    src_prefix: str
+    dst_prefix: str
+
+    def __str__(self) -> str:
+        return f"{self.type.value} {self.path}"
+
+    def __repr__(self) -> str:
+        return f"{self.type.name} {self.path}"
+
+
+def _classify_diff(line: str, src_slug: str, dst_slug: str) -> DiffItem | None:
+    def _new(type: DiffType, path: str) -> DiffItem:
+        return DiffItem(type, path, src_prefix=src_slug, dst_prefix=dst_slug)
+
+    suffix = line[1:].strip() if len(line) > 0 else ""
+    if line.startswith(DiffType.EQUAL.value):
+        return _new(DiffType.EQUAL, suffix)
+    if line.startswith(DiffType.MISSING_ON_SRC.value):
+        return _new(DiffType.MISSING_ON_SRC, suffix)
+    if line.startswith(DiffType.MISSING_ON_DST.value):
+        # return DiffItem(DiffType.MISSING_ON_DST, f"{src_slug}/{suffix}")
+        return _new(DiffType.MISSING_ON_DST, suffix)
+    if line.startswith(DiffType.DIFFERENT.value):
+        # return DiffItem(DiffType.DIFFERENT, suffix)
+        return _new(DiffType.DIFFERENT, suffix)
+    if line.startswith(DiffType.ERROR.value):
+        # return DiffItem(DiffType.ERROR, suffix)
+        return _new(DiffType.ERROR, suffix)
+    return None
+
+
+def _async_diff_stream_from_running_process(
+    running_process: Process,
+    src_slug: str,
+    dst_slug: str,
+    output: Queue[DiffItem | None],
+) -> None:
+    count = 0
+    first_few_lines: list[str] = []
+    try:
+        assert running_process.stdout is not None
+        n_max = 10
+        for line in iter(running_process.stdout.readline, b""):
+            try:
+                line_str = line.decode("utf-8").strip()
+                if len(first_few_lines) < n_max:
+                    first_few_lines.append(line_str)
+                # _classify_line_type
+                diff_item: DiffItem | None = _classify_diff(
+                    line_str, src_slug, dst_slug
+                )
+                if diff_item is None:
+                    # Some other output that we don't care about, debug print etc.
+                    continue
+                output.put(diff_item)
+                count += 1
+                # print(f"unhandled: {line_str}")
+            except UnicodeDecodeError:
+                print("UnicodeDecodeError")
+                continue
+        output.put(None)
+        print("done")
+    except KeyboardInterrupt:
+        import _thread
+
+        print("KeyboardInterrupt")
+        output.put(None)
+        _thread.interrupt_main()
+    if count == 0:
+        first_lines_str = "\n".join(first_few_lines)
+        raise ValueError(
+            f"No output from rclone check, first few lines: {first_lines_str}"
+        )
+
+
+def diff_stream_from_running_process(
+    running_process: Process,
+    src_slug: str,
+    dst_slug: str,
+) -> Generator[DiffItem, None, None]:
+    output: Queue[DiffItem | None] = Queue()
+    # process_output_to_diff_stream(running_process, src_slug, dst_slug, output)
+    thread = Thread(
+        target=_async_diff_stream_from_running_process,
+        args=(running_process, src_slug, dst_slug, output),
+        daemon=True,
+    )
+    thread.start()
+    while True:
+        item = output.get()
+        if item is None:
+            break
+        yield item
+    thread.join(timeout=5)

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api/dir_listing.py

@@ -1,8 +1,22 @@
 import json
+import warnings
 
 from rclone_api.rpath import RPath
 
 
+def _dedupe(items: list[RPath]) -> list[RPath]:
+    """Remove duplicate items from a list of RPath objects."""
+    seen = set()
+    unique_items = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            unique_items.append(item)
+        else:
+            warnings.warn(f"Duplicate item found: {item}, filtered out.")
+    return unique_items
+
+
 class DirListing:
     """Remote file dataclass."""
 
@@ -10,6 +24,8 @@ class DirListing:
         from rclone_api.dir import Dir
         from rclone_api.file import File
 
+        dirs_and_files = _dedupe(dirs_and_files)
+
         self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
         self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
 

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api/exec.py

@@ -19,7 +19,7 @@ class RcloneExec:
 
         return rclone_execute(cmd, self.rclone_config, self.rclone_exe, check=check)
 
-    def launch_process(self, cmd: list[str]) -> Process:
+    def launch_process(self, cmd: list[str], capture: bool | None) -> Process:
         """Launch rclone process."""
 
         args: ProcessArgs = ProcessArgs(
@@ -27,6 +27,7 @@ class RcloneExec:
             rclone_conf=self.rclone_config,
             rclone_exe=self.rclone_exe,
             cmd_list=cmd,
+            capture_stdout=capture,
         )
         process = Process(args)
         return process

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api/process.py

@@ -56,6 +56,7 @@ class ProcessArgs:
     rclone_exe: Path
     cmd_list: list[str]
     verbose: bool | None = None
+    capture_stdout: bool | None = None
 
 
 class Process:
@@ -84,7 +85,13 @@ class Process:
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")
-        self.process = subprocess.Popen(self.cmd, shell=False)
+        kwargs: dict = {}
+        kwargs["shell"] = False
+        if args.capture_stdout:
+            kwargs["stdout"] = subprocess.PIPE
+            kwargs["stderr"] = subprocess.STDOUT
+
+        self.process = subprocess.Popen(self.cmd, **kwargs)  # type: ignore
 
     def cleanup(self) -> None:
         if self.tempdir and self.needs_cleanup:

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api/rclone.py

@@ -14,6 +14,7 @@ from typing import Generator
 from rclone_api import Dir
 from rclone_api.config import Config
 from rclone_api.convert import convert_to_filestr_list, convert_to_str
+from rclone_api.diff import DiffItem, diff_stream_from_running_process
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
@@ -41,8 +42,8 @@ class Rclone:
     def _run(self, cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
         return self._exec.execute(cmd, check=check)
 
-    def _launch_process(self, cmd: list[str]) -> Process:
-        return self._exec.launch_process(cmd)
+    def _launch_process(self, cmd: list[str], capture: bool | None = None) -> Process:
+        return self._exec.launch_process(cmd, capture=capture)
 
     def obscure(self, password: str) -> str:
         """Obscure a password for use in rclone config files."""
@@ -108,6 +109,27 @@ class Rclone:
         out = [Remote(name=t, rclone=self) for t in tmp]
         return out
 
+    def diff(self, src: str, dst: str) -> Generator[DiffItem, None, None]:
+        """Be extra careful with the src and dst values. If you are off by one
+        parent directory, you will get a huge amount of false diffs."""
+        cmd = [
+            "check",
+            src,
+            dst,
+            "--checkers",
+            "1000",
+            "--log-level",
+            "INFO",
+            "--combined",
+            "-",
+        ]
+        proc = self._launch_process(cmd, capture=True)
+        item: DiffItem
+        for item in diff_stream_from_running_process(proc, src_slug=src, dst_slug=src):
+            if item is None:
+                break
+            yield item
+
     def walk(
         self, path: Dir | Remote | str, max_depth: int = -1, breadth_first: bool = True
     ) -> Generator[DirListing, None, None]:

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.26
+Version: 1.0.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies
@@ -22,11 +22,11 @@ Dynamic: maintainer
 [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
 [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
 
-Api version of rclone. It's well tested. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done.
+Api version of rclone. It's a pretty low level api without the bells and whistles of other apis, but it will get the job done. Unit tests up the wazoo. S3 Mounts on windows/linux are heavily optimized.
 
 You will need to have rclone installed and on your path.
 
-One of the benefits of this api is that it does not use `shell=True`, which can keep `rclone` running in some instances even when try to kill the process.
+One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
 
 # Install
 

{rclone_api-1.0.26 → rclone_api-1.0.28}/src/rclone_api.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ src/rclone_api/__init__.py
 src/rclone_api/cli.py
 src/rclone_api/config.py
 src/rclone_api/convert.py
+src/rclone_api/diff.py
 src/rclone_api/dir.py
 src/rclone_api/dir_listing.py
 src/rclone_api/exec.py
@@ -45,6 +46,7 @@ src/rclone_api/assets/example.txt
 src/rclone_api/cmd/list_files.py
 tests/test_cmd_list_files.py
 tests/test_copy.py
+tests/test_diff.py
 tests/test_is_synced.py
 tests/test_ls.py
 tests/test_mount.py

rclone_api-1.0.28/tests/test_diff.py

@@ -0,0 +1,87 @@
+"""
+Unit test file.
+"""
+
+import os
+import unittest
+
+from dotenv import load_dotenv
+
+from rclone_api import Config, Rclone
+from rclone_api.diff import DiffItem, DiffType
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config() -> Config:
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+"""
+
+    out = Config(config_text)
+    return out
+
+
+class RcloneDiffTests(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_diff(self) -> None:
+        """Test copying a single file to remote storage."""
+        rclone = Rclone(_generate_rclone_config())
+        item: DiffItem
+        all: list[DiffItem] = []
+        for item in rclone.diff("dst:rclone-api-unit-test", "dst:rclone-api-unit-test"):
+            self.assertEqual(
+                item.type, DiffType.EQUAL
+            )  # should be equal because same repo
+            all.append(item)
+        self.assertGreater(len(all), 10)
+        msg = "\n".join([str(item) for item in all])
+        print(msg)
+
+        all.clear()
+        for item in rclone.diff(
+            "dst:rclone-api-unit-test/test", "dst:rclone-api-unit-test/test"
+        ):
+            self.assertEqual(item.type, DiffType.EQUAL)
+            print(item)
+            all.append(item)
+
+        msg = "\n".join([str(item) for item in all])
+        print(msg)
+        print("done")
+
+
+if __name__ == "__main__":
+    unittest.main()

{rclone_api-1.0.26 → rclone_api-1.0.28}/tests/test_mount_s3.py

@@ -13,6 +13,8 @@ from rclone_api import Config, Process, Rclone
 
 load_dotenv()
 
+_ENABLED = False
+
 
 def _generate_rclone_config() -> Config:
     # Load environment variables
@@ -60,6 +62,7 @@ class RcloneMountS3Tests(unittest.TestCase):
         os.environ["RCLONE_API_VERBOSE"] = "1"
         self.rclone = Rclone(_generate_rclone_config())
 
+    @unittest.skipUnless(_ENABLED, "Test is disabled by default")
     def test_mount(self) -> None:
         """Test mounting a remote bucket."""
         remote_path = f"dst:{self.bucket_name}"

{rclone_api-1.0.26 → rclone_api-1.0.28}/tests/test_mount_webdav.py

@@ -10,6 +10,8 @@ from dotenv import load_dotenv
 
 from rclone_api import Config, Process, Rclone, Remote
 
+_ENABLED = False
+
 load_dotenv()
 
 BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
@@ -66,6 +68,7 @@ class RcloneMountWebdavTester(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
+    @unittest.skipIf(not _ENABLED, "Test not enabled")
     def test_serve_webdav_and_mount(self) -> None:
         """Test basic NFS serve functionality."""
         port = 8090