rclone-api 1.0.17__tar.gz → 1.0.19__tar.gz

{rclone_api-1.0.17 → rclone_api-1.0.19}/.gitignore

@@ -144,4 +144,4 @@ uv.lock
 !.aider.conf.yml
 !.aiderignore
 
-rclone.conf
+rclone*.conf

{rclone_api-1.0.17 → rclone_api-1.0.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.17
+Version: 1.0.19
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pyright>=1.1.393
 Requires-Dist: python-dotenv>=1.0.0
 Dynamic: home-page
 Dynamic: maintainer

{rclone_api-1.0.17 → rclone_api-1.0.19}/lint

@@ -11,5 +11,7 @@ echo Running isort src tests
 uv run isort --profile black src tests
 echo Running mypy src
 uv run mypy src tests
+echo Running pyright src tests
+uv run pyright src tests --threads
 echo Linting complete!
 exit 0

{rclone_api-1.0.17 → rclone_api-1.0.19}/pyproject.toml

@@ -11,10 +11,11 @@ keywords = ["template-python-cmd"]
 license = { text = "BSD 3-Clause License" }
 classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
-    "python-dotenv>=1.0.0"
+    "pyright>=1.1.393",
+    "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.17"
+version = "1.0.19"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.17 → rclone_api-1.0.19}/requirements.testing.txt

@@ -5,3 +5,5 @@ pytest
 tox
 ruff
 pytest-xdist
+httpx
+pyright

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api/dir.py

@@ -47,12 +47,14 @@ class Dir:
         dir = Dir(self.path)
         return self.path.rclone.ls(dir, max_depth=max_depth)
 
-    def walk(self, max_depth: int = -1) -> Generator[DirListing, None, None]:
+    def walk(
+        self, breadth_first: bool, max_depth: int = -1
+    ) -> Generator[DirListing, None, None]:
         """List files and directories in the given path."""
         from rclone_api.walk import walk
 
         assert self.path.rclone is not None
-        return walk(self, max_depth=max_depth)
+        return walk(self, breadth_first=breadth_first, max_depth=max_depth)
 
     def to_json(self) -> dict:
         """Convert the Dir to a JSON serializable dictionary."""

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api/process.py

@@ -69,8 +69,10 @@ class Process:
             tmpfile = Path(self.tempdir.name) / "rclone.conf"
             tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
             rclone_conf = tmpfile
+            self.needs_cleanup = True
         else:
             rclone_conf = args.rclone_conf
+            self.needs_cleanup = False
 
         assert rclone_conf.exists()
 
@@ -85,7 +87,7 @@ class Process:
         self.process = subprocess.Popen(self.cmd, shell=False)
 
     def cleanup(self) -> None:
-        if self.tempdir:
+        if self.tempdir and self.needs_cleanup:
             try:
                 self.tempdir.cleanup()
             except Exception as e:

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api/rclone.py

@@ -38,6 +38,12 @@ class Rclone:
     def _launch_process(self, cmd: list[str]) -> Process:
         return self._exec.launch_process(cmd)
 
+    def obscure(self, password: str) -> str:
+        """Obscure a password for use in rclone config files."""
+        cmd_list: list[str] = ["obscure", password]
+        cp = self._run(cmd_list)
+        return cp.stdout.strip()
+
     def ls(
         self,
         path: Dir | Remote | str,
@@ -61,8 +67,9 @@ class Rclone:
 
         cmd = ["lsjson"]
         if max_depth is not None:
-            cmd.append("--recursive")
-            if max_depth > -1:
+            if max_depth < 0:
+                cmd.append("--recursive")
+            if max_depth > 0:
                 cmd.append("--max-depth")
                 cmd.append(str(max_depth))
         cmd.append(str(path))
@@ -96,7 +103,7 @@ class Rclone:
         return out
 
     def walk(
-        self, path: Dir | Remote | str, max_depth: int = -1
+        self, path: Dir | Remote | str, max_depth: int = -1, breadth_first: bool = True
     ) -> Generator[DirListing, None, None]:
         """Walk through the given path recursively.
 
@@ -107,6 +114,7 @@ class Rclone:
         Yields:
             DirListing: Directory listing for each directory encountered
         """
+        dir_obj: Dir
         if isinstance(path, Dir):
             # Create a Remote object for the path
             remote = path.remote
@@ -126,9 +134,10 @@ class Rclone:
         elif isinstance(path, Remote):
             dir_obj = Dir(path)
         else:
+            dir_obj = Dir(path)  # shut up pyright
             assert f"Invalid type for path: {type(path)}"
 
-        yield from walk(dir_obj, max_depth=max_depth)
+        yield from walk(dir_obj, max_depth=max_depth, breadth_first=breadth_first)
 
     def copyfile(self, src: File | str, dst: File | str) -> None:
         """Copy a single file from source to destination.
@@ -239,7 +248,13 @@ class Rclone:
         return self._run(cmd_list)
 
     def mount(
-        self, src: Remote | Dir | str, outdir: Path, allow_writes=False, use_links=True
+        self,
+        src: Remote | Dir | str,
+        outdir: Path,
+        allow_writes=False,
+        use_links=True,
+        vfs_cache_mode="full",
+        other_cmds: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -266,8 +281,85 @@ class Rclone:
             cmd_list.append("--read-only")
         if use_links:
             cmd_list.append("--links")
+        if vfs_cache_mode:
+            cmd_list.append("--vfs-cache-mode")
+            cmd_list.append(vfs_cache_mode)
+        if other_cmds:
+            cmd_list += other_cmds
         proc = self._launch_process(cmd_list)
         time.sleep(2)  # give it a moment to mount
         if proc.poll() is not None:
             raise ValueError("Mount process failed to start")
         return proc
+
+    def mount_webdav(
+        self,
+        url: str,
+        outdir: Path,
+        vfs_cache_mode="full",
+        other_cmds: list[str] | None = None,
+    ) -> Process:
+        """Mount a remote or directory to a local path.
+
+        Args:
+            src: Remote or directory to mount
+            outdir: Local path to mount to
+
+        Returns:
+            CompletedProcess from the mount command execution
+
+        Raises:
+            subprocess.CalledProcessError: If the mount operation fails
+        """
+        if outdir.exists():
+            is_empty = not list(outdir.iterdir())
+            if not is_empty:
+                raise ValueError(
+                    f"Mount directory already exists and is not empty: {outdir}"
+                )
+            outdir.rmdir()
+
+        src_str = url
+        cmd_list: list[str] = ["mount", src_str, str(outdir)]
+        cmd_list.append("--vfs-cache-mode")
+        cmd_list.append(vfs_cache_mode)
+        if other_cmds:
+            cmd_list += other_cmds
+        proc = self._launch_process(cmd_list)
+        # proc = rclone_exec.launch_process(cmd_list)
+        time.sleep(2)
+        if proc.poll() is not None:
+            raise ValueError("Mount process failed to start")
+        return proc
+
+    def serve_webdav(
+        self,
+        src: Remote | Dir | str,
+        user: str,
+        password: str,
+        addr: str = "localhost:2049",
+        allow_other: bool = False,
+    ) -> Process:
+        """Serve a remote or directory via NFS.
+
+        Args:
+            src: Remote or directory to serve
+            addr: Network address and port to serve on (default: localhost:2049)
+            allow_other: Allow other users to access the share
+
+        Returns:
+            Process: The running NFS server process
+
+        Raises:
+            ValueError: If the NFS server fails to start
+        """
+        src_str = convert_to_str(src)
+        cmd_list: list[str] = ["serve", "webdav", "--addr", addr, src_str]
+        cmd_list.extend(["--user", user, "--pass", password])
+        if allow_other:
+            cmd_list.append("--allow-other")
+        proc = self._launch_process(cmd_list)
+        time.sleep(2)  # give it a moment to start
+        if proc.poll() is not None:
+            raise ValueError("NFS serve process failed to start")
+        return proc

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api/walk.py

@@ -9,9 +9,11 @@ from rclone_api.remote import Remote
 _MAX_OUT_QUEUE_SIZE = 50
 
 
-def _walk_runner(
-    queue: Queue[Dir], max_depth: int, out_queue: Queue[DirListing | None]
+def _walk_runner_breadth_first(
+    dir: Dir, max_depth: int, out_queue: Queue[DirListing | None]
 ) -> None:
+    queue: Queue[Dir] = Queue()
+    queue.put(dir)
     try:
         while not queue.empty():
             current_dir = queue.get()
@@ -35,7 +37,35 @@ def _walk_runner(
         _thread.interrupt_main()
 
 
-def walk(dir: Dir | Remote, max_depth: int = -1) -> Generator[DirListing, None, None]:
+def _walk_runner_depth_first(
+    dir: Dir, max_depth: int, out_queue: Queue[DirListing | None]
+) -> None:
+    try:
+        stack = [(dir, max_depth)]
+        while stack:
+            current_dir, depth = stack.pop()
+            dirlisting = current_dir.ls()
+            if depth != 0:
+                for subdir in reversed(
+                    dirlisting.dirs
+                ):  # Process deeper directories first
+                    # stack.append((child, depth - 1 if depth > 0 else depth))
+                    next_depth = depth - 1 if depth > 0 else depth
+                    _walk_runner_depth_first(subdir, next_depth, out_queue)
+            out_queue.put(dirlisting)
+        out_queue.put(None)
+    except KeyboardInterrupt:
+        import _thread
+
+        out_queue.put(None)
+        _thread.interrupt_main()
+
+
+def walk(
+    dir: Dir | Remote,
+    breadth_first: bool,
+    max_depth: int = -1,
+) -> Generator[DirListing, None, None]:
     """Walk through the given directory recursively.
 
     Args:
@@ -49,14 +79,17 @@ def walk(dir: Dir | Remote, max_depth: int = -1) -> Generator[DirListing, None,
         # Convert Remote to Dir if needed
         if isinstance(dir, Remote):
             dir = Dir(dir)
-
-        in_queue: Queue[Dir] = Queue()
         out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
-        in_queue.put(dir)
+
+        strategy = (
+            _walk_runner_breadth_first if breadth_first else _walk_runner_depth_first
+        )
 
         # Start worker thread
         worker = Thread(
-            target=_walk_runner, args=(in_queue, max_depth, out_queue), daemon=True
+            target=strategy,
+            args=(dir, max_depth, out_queue),
+            daemon=True,
         )
         worker.start()
 

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.17
+Version: 1.0.19
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pyright>=1.1.393
 Requires-Dist: python-dotenv>=1.0.0
 Dynamic: home-page
 Dynamic: maintainer

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api.egg-info/SOURCES.txt

@@ -45,5 +45,8 @@ tests/test_copy.py
 tests/test_is_synced.py
 tests/test_ls.py
 tests/test_mount.py
+tests/test_mount_webdav.py
+tests/test_obscure.py
 tests/test_remotes.py
+tests/test_serve_webdav.py
 tests/test_walk.py

{rclone_api-1.0.17 → rclone_api-1.0.19}/src/rclone_api.egg-info/requires.txt

@@ -1 +1,2 @@
+pyright>=1.1.393
 python-dotenv>=1.0.0

{rclone_api-1.0.17 → rclone_api-1.0.19}/tests/test_ls.py

@@ -31,6 +31,7 @@ provider = DigitalOcean
 access_key_id = {BUCKET_KEY_PUBLIC}
 secret_access_key = {BUCKET_KEY_SECRET}
 endpoint = {BUCKET_URL}
+bucket = {BUCKET_NAME}
 """
 
     out = Config(config_text)

rclone_api-1.0.19/tests/test_mount_webdav.py

@@ -0,0 +1,114 @@
+"""
+Unit test file.
+"""
+
+import os
+import unittest
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from rclone_api import Config, Process, Rclone, Remote
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config(port: int) -> Config:
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+bucket = {BUCKET_NAME}
+
+
+[webdav]
+type = webdav
+user = guest
+# obscured password for "1234", use Rclone.obscure("1234") to generate
+pass = d4IbQLV9W0JhI2tm5Zp88hpMtEg
+url = http://localhost:{port}
+vendor = rclone
+"""
+
+    out = Config(config_text)
+    return out
+
+
+class RcloneMountWebdavTester(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_serve_webdav_and_mount(self) -> None:
+        """Test basic NFS serve functionality."""
+        port = 8090
+        config = _generate_rclone_config(port)
+        rclone = Rclone(config)
+
+        # Start NFS server for the remote
+        remote = Remote("dst", rclone=rclone)
+        # serve = Remote("webdav", rclone=rclone)
+        test_addr = f"localhost:{port}"
+        user = "guest"
+        password = "1234"
+
+        process = rclone.serve_webdav(
+            f"{remote.name}:{BUCKET_NAME}", addr=test_addr, user=user, password=password
+        )
+        mount_proc: Process | None = None
+
+        try:
+            # Verify process is running
+            self.assertIsNone(process.poll())
+            mount_point = Path("test_mount2")
+            mount_proc = rclone.mount_webdav("webdav:", mount_point)
+            # test that the mount point exists
+            self.assertTrue(mount_point.exists())
+            # test the folder is not empty
+            next_path = next(mount_point.iterdir())
+            self.assertIsNotNone(next_path)
+
+        finally:
+            # Clean up
+            if mount_proc:
+                mount_proc.terminate()
+                mount_proc.wait()
+            process.terminate()
+            process.wait()
+            if mount_proc:
+                mount_proc.terminate()
+                mount_proc.wait()
+
+        # Verify process terminated
+        self.assertIsNotNone(process.poll())
+
+
+if __name__ == "__main__":
+    unittest.main()

rclone_api-1.0.17/tests/test_walk.py → rclone_api-1.0.19/tests/test_obscure.py

@@ -7,7 +7,7 @@ import unittest
 
 from dotenv import load_dotenv
 
-from rclone_api import Config, DirListing, Rclone
+from rclone_api import Config, Rclone
 
 load_dotenv()
 
@@ -31,13 +31,14 @@ provider = DigitalOcean
 access_key_id = {BUCKET_KEY_PUBLIC}
 secret_access_key = {BUCKET_KEY_SECRET}
 endpoint = {BUCKET_URL}
+bucket = {BUCKET_NAME}
 """
 
     out = Config(config_text)
     return out
 
 
-class RcloneWalkTest(unittest.TestCase):
+class RcloneLsTests(unittest.TestCase):
     """Test rclone functionality."""
 
     def setUp(self) -> None:
@@ -55,13 +56,11 @@ class RcloneWalkTest(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
-    def test_walk(self) -> None:
+    def test_list_remotes(self) -> None:
         rclone = Rclone(_generate_rclone_config())
-        # rclone.walk
-        dirlisting: DirListing
-        for dirlisting in rclone.walk(f"dst:{BUCKET_NAME}", max_depth=1):
-            print(dirlisting)
-        print("done")
+        obscured = rclone.obscure("1234")
+        self.assertNotEqual(obscured, "1234")
+        print()
 
 
 if __name__ == "__main__":

rclone_api-1.0.19/tests/test_serve_webdav.py

@@ -0,0 +1,108 @@
+"""
+Unit test file.
+"""
+
+import os
+import unittest
+
+import httpx
+from dotenv import load_dotenv
+
+from rclone_api import Config, Process, Rclone, Remote
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config(port: int) -> Config:
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+bucket = {BUCKET_NAME}
+
+
+[webdav]
+type = webdav
+user = guest
+# obscured password for "1234", use Rclone.obscure("1234") to generate
+pass = d4IbQLV9W0JhI2tm5Zp88hpMtEg
+url = http://localhost:{port}
+vendor = rclone
+"""
+
+    out = Config(config_text)
+    return out
+
+
+class RcloneServeWebdavTester(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_serve_webdav(self) -> None:
+        """Test basic NFS serve functionality."""
+        port = 8089
+        config = _generate_rclone_config(port)
+        rclone = Rclone(config)
+
+        # Start NFS server for the remote
+        remote = Remote("dst", rclone=rclone)
+        # serve = Remote("webdav", rclone=rclone)
+        test_addr = f"localhost:{port}"
+        user = "guest"
+        password = "1234"
+
+        process = rclone.serve_webdav(
+            f"{remote.name}:{BUCKET_NAME}", addr=test_addr, user=user, password=password
+        )
+        mount_proc: Process | None = None
+
+        try:
+            # Verify process is running
+            self.assertIsNone(process.poll())
+            response = httpx.get(f"http://{test_addr}/", auth=(user, password))
+            # Note that windows is kinda broken and returns internal server error
+            is_serving = response.status_code == 200
+            self.assertTrue(is_serving)
+
+        finally:
+            # Clean up
+            process.terminate()
+            process.wait()
+            if mount_proc:
+                mount_proc.terminate()
+                mount_proc.wait()
+
+        # Verify process terminated
+        self.assertIsNotNone(process.poll())
+
+
+if __name__ == "__main__":
+    unittest.main()

rclone_api-1.0.19/tests/test_walk.py

@@ -0,0 +1,86 @@
+"""
+Unit test file.
+"""
+
+import os
+import unittest
+
+from dotenv import load_dotenv
+
+from rclone_api import Config, DirListing, Rclone
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config() -> Config:
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+"""
+
+    out = Config(config_text)
+    return out
+
+
+class RcloneWalkTest(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_walk(self) -> None:
+        rclone = Rclone(_generate_rclone_config())
+        # rclone.walk
+        dirlisting: DirListing
+        is_first = True
+        for dirlisting in rclone.walk(f"dst:{BUCKET_NAME}", max_depth=1):
+            if is_first:
+                # assert just one file
+                # assert len(dirlisting.files) == 1
+                self.assertEqual(len(dirlisting.files), 1)
+                # assert it's first.txt
+                self.assertEqual(dirlisting.files[0].name, "first.txt")
+                is_first = False
+            print(dirlisting)
+        print("done")
+
+    def test_walk_depth_first(self) -> None:
+        rclone = Rclone(_generate_rclone_config())
+        # rclone.walk
+        dirlisting: DirListing
+        for dirlisting in rclone.walk(
+            f"dst:{BUCKET_NAME}", max_depth=1, breadth_first=False
+        ):
+            print(dirlisting)
+        print("done")
+
+
+if __name__ == "__main__":
+    unittest.main()