rclone-api 1.0.40__tar.gz → 1.0.42__tar.gz

{rclone_api-1.0.40 → rclone_api-1.0.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.40
+Version: 1.0.42
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.40 → rclone_api-1.0.42}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.40"
+version = "1.0.42"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.0.42/src/rclone_api/group_files.py

@@ -0,0 +1,145 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class FilePathParts:
+    """File path dataclass."""
+
+    remote: str
+    parents: list[str]
+    name: str
+
+
+def parse_file(file_path: str) -> FilePathParts:
+    """Parse file path into parts."""
+    assert not file_path.endswith("/"), "This looks like a directory path"
+    parts = file_path.split(":")
+    remote = parts[0]
+    path = parts[1]
+    if path.startswith("/"):
+        path = path[1:]
+    parents = path.split("/")
+    if len(parents) == 1:
+        return FilePathParts(remote=remote, parents=[], name=parents[0])
+    name = parents.pop()
+    return FilePathParts(remote=remote, parents=parents, name=name)
+
+
+class TreeNode:
+    def __init__(
+        self,
+        name: str,
+        child_nodes: dict[str, "TreeNode"] | None = None,
+        files: list[str] | None = None,
+        parent: "TreeNode | None" = None,
+    ):
+        self.name = name
+        self.child_nodes = child_nodes or {}
+        self.files = files or []
+        self.count = 0
+        self.parent = parent
+
+    def add_count(self):
+        self.count += 1
+        if self.parent:
+            self.parent.add_count()
+
+    def get_path(self) -> str:
+        paths_reversed: list[str] = [self.name]
+        node: TreeNode | None = self
+        assert node is not None
+        while node := node.parent:
+            paths_reversed.append(node.name)
+        return "/".join(reversed(paths_reversed))
+
+    def get_child_subpaths(self, parent_path: str | None = None) -> list[str]:
+        paths: list[str] = []
+        for child in self.child_nodes.values():
+            child_paths = child.get_child_subpaths(parent_path=child.name)
+            paths.extend(child_paths)
+        for file in self.files:
+            if parent_path:
+                file = f"{parent_path}/{file}"
+            paths.append(file)
+        return paths
+
+    def __repr__(self, indent: int = 0) -> str:
+        # return f"{self.name}: {self.count}, {len(self.children)}"
+        leftpad = " " * indent
+        msg = f"{leftpad}{self.name}: {self.count}"
+        if self.child_nodes:
+            # msg += f"\n   {len(self.children)} children"
+            msg += "\n"
+            for child in self.child_nodes.values():
+                if isinstance(child, TreeNode):
+                    msg += child.__repr__(indent + 2)
+                else:
+                    msg += f"{leftpad}  {child}\n"
+        return msg
+
+
+def _merge(node: TreeNode, parent_path: str, out: dict[str, list[str]]) -> None:
+    parent_path = parent_path + "/" + node.name
+    this_count = node.count
+    child_count = 0
+    children_has_files = False
+    if not node.child_nodes and not node.files:
+        return  # done
+
+    if node.files:
+        children_has_files = True
+        filelist = out.setdefault(parent_path, [])
+        # for file in node.files:
+        #    filelist.append(file)
+        # out[parent_path] = filelist
+        paths = node.get_child_subpaths()
+        for path in paths:
+            filelist.append(path)
+        out[parent_path] = filelist
+        return
+
+    for child in node.child_nodes.values():
+        child_count += child.count
+    child_count += len(node.files)
+    for file in node.files:
+        child_count += 1
+
+    if child_count != this_count or children_has_files:
+        # print(
+        #    f"Cannot merge {node.name} because different counts or has children with files"
+        # )
+        filelist = out.setdefault(parent_path, [])
+        for child in node.child_nodes.values():
+            subpaths = child.get_child_subpaths()
+            filelist.extend(subpaths)
+        out[parent_path] = filelist
+    else:
+        for child in node.child_nodes.values():
+            _merge(child, parent_path, out)
+
+
+def group_files(files: list[str]) -> dict[str, list[str]]:
+    """split between filename and parent directory path"""
+    tree: dict[str, TreeNode] = {}
+    for file in files:
+        parts = parse_file(file)
+        remote = parts.remote
+        node: TreeNode = tree.setdefault(remote, TreeNode(remote))
+        for parent in parts.parents:
+            is_last = parent == parts.parents[-1]
+            node = node.child_nodes.setdefault(parent, TreeNode(parent, parent=node))
+            if is_last:
+                node.files.append(parts.name)
+                node.add_count()
+    outpaths: dict[str, list[str]] = {}
+    for _, node in tree.items():
+        _merge(node, "", outpaths)
+    out: dict[str, list[str]] = {}
+    for path, files in outpaths.items():
+        # fixup path
+        assert path.startswith("/"), "Path should start with /"
+        path = path[1:]
+        # replace the first / with :
+        path = path.replace("/", ":", 1)
+        out[path] = files
+    return out

{rclone_api-1.0.40 → rclone_api-1.0.42}/src/rclone_api/rclone.py

@@ -5,6 +5,7 @@ Unit test file.
 import subprocess
 import time
 import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
 from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
@@ -20,12 +21,20 @@ from rclone_api.diff import DiffItem, diff_stream_from_running_process
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
+from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.util import get_rclone_exe, partition_files, to_path, wait_for_mount
+from rclone_api.util import (
+    get_rclone_exe,
+    get_verbose,
+    to_path,
+    wait_for_mount,
+)
 from rclone_api.walk import walk
 
+EXECUTOR = ThreadPoolExecutor(16)
+
 
 class ModTimeStrategy(Enum):
     USE_SERVER_MODTIME = "use-server-modtime"
@@ -197,7 +206,7 @@ class Rclone:
         cmd_list: list[str] = ["copyto", src, dst]
         self._run(cmd_list)
 
-    def copyfiles(self, files: str | File | list[str] | list[File]) -> None:
+    def copyfiles(self, files: str | File | list[str] | list[File], check=True) -> None:
         """Copy multiple files from source to destination.
 
         Warning - slow.
@@ -209,31 +218,42 @@ class Rclone:
         if len(payload) == 0:
             return
 
-        datalists: dict[str, list[str]] = partition_files(payload)
+        datalists: dict[str, list[str]] = group_files(payload)
         out: subprocess.CompletedProcess | None = None
 
+        futures: list[Future] = []
+
         for remote, files in datalists.items():
-            with TemporaryDirectory() as tmpdir:
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                # print(include_files_txt)
-                cmd_list: list[str] = [
-                    "delete",
-                    remote,
-                    "--files-from",
-                    str(include_files_txt),
-                    "--checkers",
-                    "1000",
-                    "--transfers",
-                    "1000",
-                ]
-                out = self._run(cmd_list)
-                if out.returncode != 0:
-                    print(out)
-                    raise ValueError(f"Error deleting files: {out.stderr}")
 
-        assert out is not None
+            def _task(files=files) -> subprocess.CompletedProcess:
+                with TemporaryDirectory() as tmpdir:
+                    include_files_txt = Path(tmpdir) / "include_files.txt"
+                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                    # print(include_files_txt)
+                    cmd_list: list[str] = [
+                        "delete",
+                        remote,
+                        "--files-from",
+                        str(include_files_txt),
+                        "--checkers",
+                        "1000",
+                        "--transfers",
+                        "1000",
+                    ]
+                    out = self._run(cmd_list)
+                    return out
+
+            fut: Future = EXECUTOR.submit(_task)
+            futures.append(fut)
+        for fut in futures:
+            out = fut.result()
+            assert out is not None
+            if out.returncode != 0:
+                if check:
+                    raise ValueError(f"Error deleting files: {out.stderr}")
+                else:
+                    warnings.warn(f"Error deleting files: {out.stderr}")
 
     def copy(self, src: Dir | str, dst: Dir | str) -> CompletedProcess:
         """Copy files from source to destination.
@@ -263,6 +283,7 @@ class Rclone:
         files: str | File | list[str] | list[File],
         check=True,
         rmdirs=False,
+        verbose: bool | None = None,
         other_args: list[str] | None = None,
     ) -> CompletedProcess:
         """Delete a directory"""
@@ -276,41 +297,53 @@ class Rclone:
             )
             return CompletedProcess.from_subprocess(cp)
 
-        datalists: dict[str, list[str]] = partition_files(payload)
-        out: subprocess.CompletedProcess | None = None
-
+        datalists: dict[str, list[str]] = group_files(payload)
         completed_processes: list[subprocess.CompletedProcess] = []
+        verbose = get_verbose(verbose)
+
+        futures: list[Future] = []
 
         for remote, files in datalists.items():
-            with TemporaryDirectory() as tmpdir:
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                # print(include_files_txt)
-                cmd_list: list[str] = [
-                    "delete",
-                    remote,
-                    "--files-from",
-                    str(include_files_txt),
-                    "--checkers",
-                    "1000",
-                    "--transfers",
-                    "1000",
-                ]
-                if rmdirs:
-                    cmd_list.append("--rmdirs")
-                if other_args:
-                    cmd_list += other_args
-                out = self._run(cmd_list)
-                completed_processes.append(out)
+
+            def _task(files=files, check=check) -> subprocess.CompletedProcess:
+                with TemporaryDirectory() as tmpdir:
+                    include_files_txt = Path(tmpdir) / "include_files.txt"
+                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                    # print(include_files_txt)
+                    cmd_list: list[str] = [
+                        "delete",
+                        remote,
+                        "--files-from",
+                        str(include_files_txt),
+                        "--checkers",
+                        "1000",
+                        "--transfers",
+                        "1000",
+                    ]
+                    if verbose:
+                        cmd_list.append("-vvvv")
+                    if rmdirs:
+                        cmd_list.append("--rmdirs")
+                    if other_args:
+                        cmd_list += other_args
+                    out = self._run(cmd_list, check=check)
                 if out.returncode != 0:
                     if check:
                         completed_processes.append(out)
                         raise ValueError(f"Error deleting files: {out}")
                     else:
                         warnings.warn(f"Error deleting files: {out}")
+                return out
+
+            fut: Future = EXECUTOR.submit(_task)
+            futures.append(fut)
+
+        for fut in futures:
+            out = fut.result()
+            assert out is not None
+            completed_processes.append(out)
 
-        assert out is not None
         return CompletedProcess(completed_processes)
 
     @deprecated("delete_files")

{rclone_api-1.0.40 → rclone_api-1.0.42}/src/rclone_api/util.py

@@ -129,15 +129,3 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
         if path.exists():
             return
     raise TimeoutError(f"Path {path} did not exist after {timeout} seconds")
-
-
-def partition_files(files: list[str]) -> dict[str, list[str]]:
-    """split between filename and parent directory path"""
-    datalists: dict[str, list[str]] = {}
-    for f in files:
-        base = os.path.basename(f)
-        parent_path = os.path.dirname(f)
-        if parent_path not in datalists:
-            datalists[parent_path] = []
-        datalists[parent_path].append(base)
-    return datalists

{rclone_api-1.0.40 → rclone_api-1.0.42}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.40
+Version: 1.0.42
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.40 → rclone_api-1.0.42}/src/rclone_api.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ src/rclone_api/dir_listing.py
 src/rclone_api/exec.py
 src/rclone_api/file.py
 src/rclone_api/filelist.py
+src/rclone_api/group_files.py
 src/rclone_api/process.py
 src/rclone_api/rclone.py
 src/rclone_api/remote.py
@@ -49,6 +50,7 @@ src/rclone_api/cmd/list_files.py
 tests/test_cmd_list_files.py
 tests/test_copy.py
 tests/test_diff.py
+tests/test_group_files.py
 tests/test_is_synced.py
 tests/test_ls.py
 tests/test_mount.py

rclone_api-1.0.42/tests/test_group_files.py

@@ -0,0 +1,81 @@
+"""
+Unit test file.
+"""
+
+import unittest
+
+from rclone_api.group_files import group_files
+
+
+class GroupFilestest(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def test_simple_group_files(self) -> None:
+        files = [
+            "dst:Bucket/subdir/file1.txt",
+            "dst:Bucket/subdir/file2.txt",
+        ]
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 1)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:Bucket/subdir", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir"]), 2)
+        expected_files = [
+            "file1.txt",
+            "file2.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir"])
+        self.assertIn(expected_files[1], groups["dst:Bucket/subdir"])
+        print("done")
+
+    def test_different_paths(self) -> None:
+        files = [
+            "dst:Bucket/subdir/file1.txt",
+            "dst:Bucket/subdir2/file2.txt",
+        ]
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 2)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:Bucket/subdir", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir"]), 1)
+        expected_files = [
+            "file1.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir"])
+        # dst:/Bucket/subdir2 should be the key
+        self.assertIn("dst:Bucket/subdir2", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir2"]), 1)
+
+    def test_two_big_directories(self) -> None:
+        files = [
+            "dst:Bucket/subdir/file1.txt",
+            "dst:Bucket/subdir/file2.txt",
+            "dst:Bucket/subdir2/file3.txt",
+            "dst:Bucket/subdir2/file4.txt",
+        ]
+
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 2)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:Bucket/subdir", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir"]), 2)
+        expected_files = [
+            "file1.txt",
+            "file2.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir"])
+        self.assertIn(expected_files[1], groups["dst:Bucket/subdir"])
+        # dst:/Bucket/subdir2 should be the key
+        self.assertIn("dst:Bucket/subdir2", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir2"]), 2)
+        expected_files = [
+            "file3.txt",
+            "file4.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir2"])
+        self.assertIn(expected_files[1], groups["dst:Bucket/subdir2"])
+        print("done")
+
+
+if __name__ == "__main__":
+    unittest.main()