rclone-api 1.0.41__tar.gz → 1.0.43__tar.gz

{rclone_api-1.0.41 → rclone_api-1.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.41
+Version: 1.0.43
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.41 → rclone_api-1.0.43}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.41"
+version = "1.0.43"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.0.43/src/rclone_api/group_files.py

@@ -0,0 +1,147 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class FilePathParts:
+    """File path dataclass."""
+
+    remote: str
+    parents: list[str]
+    name: str
+
+
+def parse_file(file_path: str) -> FilePathParts:
+    """Parse file path into parts."""
+    assert not file_path.endswith("/"), "This looks like a directory path"
+    parts = file_path.split(":")
+    remote = parts[0]
+    path = parts[1]
+    if path.startswith("/"):
+        path = path[1:]
+    parents = path.split("/")
+    if len(parents) == 1:
+        return FilePathParts(remote=remote, parents=[], name=parents[0])
+    name = parents.pop()
+    return FilePathParts(remote=remote, parents=parents, name=name)
+
+
+class TreeNode:
+    def __init__(
+        self,
+        name: str,
+        child_nodes: dict[str, "TreeNode"] | None = None,
+        files: list[str] | None = None,
+        parent: "TreeNode | None" = None,
+    ):
+        self.name = name
+        self.child_nodes = child_nodes or {}
+        self.files = files or []
+        self.count = 0
+        self.parent = parent
+
+    def add_count(self):
+        self.count += 1
+        if self.parent:
+            self.parent.add_count()
+
+    def get_path(self) -> str:
+        paths_reversed: list[str] = [self.name]
+        node: TreeNode | None = self
+        assert node is not None
+        while node := node.parent:
+            paths_reversed.append(node.name)
+        return "/".join(reversed(paths_reversed))
+
+    def get_child_subpaths(self, parent_path: str | None = None) -> list[str]:
+        paths: list[str] = []
+        for child in self.child_nodes.values():
+            child_paths = child.get_child_subpaths(parent_path=child.name)
+            paths.extend(child_paths)
+        for file in self.files:
+            if parent_path:
+                file = f"{parent_path}/{file}"
+            paths.append(file)
+        return paths
+
+    def __repr__(self, indent: int = 0) -> str:
+        # return f"{self.name}: {self.count}, {len(self.children)}"
+        leftpad = " " * indent
+        msg = f"{leftpad}{self.name}: {self.count}"
+        if self.child_nodes:
+            # msg += f"\n   {len(self.children)} children"
+            msg += "\n"
+            for child in self.child_nodes.values():
+                if isinstance(child, TreeNode):
+                    msg += child.__repr__(indent + 2)
+                else:
+                    msg += f"{leftpad}  {child}\n"
+        return msg
+
+
+def _merge(node: TreeNode, parent_path: str, out: dict[str, list[str]]) -> None:
+    parent_path = parent_path + "/" + node.name
+    if not node.child_nodes and not node.files:
+        return  # done
+    if node.files:
+        # we saw files, to don't try to go any deeper.
+        filelist = out.setdefault(parent_path, [])
+        # for file in node.files:
+        #    filelist.append(file)
+        # out[parent_path] = filelist
+        paths = node.get_child_subpaths()
+        for path in paths:
+            filelist.append(path)
+        out[parent_path] = filelist
+        return
+
+    n_child_nodes = len(node.child_nodes)
+
+    if n_child_nodes < 4:
+        # child = list(node.child_nodes.values())[0]
+        # _merge(child, parent_path, out)
+        # return
+        for child in node.child_nodes.values():
+            _merge(child, parent_path, out)
+        return
+
+    filelist = out.setdefault(parent_path, [])
+    # for file in node.files:
+    #    filelist.append(file)
+    # out[parent_path] = filelist
+    paths = node.get_child_subpaths()
+    for path in paths:
+        filelist.append(path)
+    out[parent_path] = filelist
+    return
+
+
+def _make_tree(files: list[str]) -> dict[str, TreeNode]:
+    tree: dict[str, TreeNode] = {}
+    for file in files:
+        parts = parse_file(file)
+        remote = parts.remote
+        node: TreeNode = tree.setdefault(remote, TreeNode(remote))
+        for parent in parts.parents:
+            is_last = parent == parts.parents[-1]
+            node = node.child_nodes.setdefault(parent, TreeNode(parent, parent=node))
+            if is_last:
+                node.files.append(parts.name)
+                node.add_count()
+    return tree
+
+
+def group_files(files: list[str]) -> dict[str, list[str]]:
+    """split between filename and parent directory path"""
+    tree: dict[str, TreeNode] = _make_tree(files)
+    outpaths: dict[str, list[str]] = {}
+    for _, node in tree.items():
+        _merge(node, "", outpaths)
+    out: dict[str, list[str]] = {}
+    for path, files in outpaths.items():
+        # fixup path
+        assert path.startswith("/"), "Path should start with /"
+        path = path[1:]
+        # replace the first / with :
+        path = path.replace("/", ":", 1)
+        out[path] = files
+    return out

{rclone_api-1.0.41 → rclone_api-1.0.43}/src/rclone_api/rclone.py

@@ -21,13 +21,13 @@ from rclone_api.diff import DiffItem, diff_stream_from_running_process
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
+from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
 from rclone_api.util import (
     get_rclone_exe,
     get_verbose,
-    partition_files,
     to_path,
     wait_for_mount,
 )
@@ -218,7 +218,7 @@ class Rclone:
         if len(payload) == 0:
             return
 
-        datalists: dict[str, list[str]] = partition_files(payload)
+        datalists: dict[str, list[str]] = group_files(payload)
         out: subprocess.CompletedProcess | None = None
 
         futures: list[Future] = []
@@ -297,7 +297,7 @@ class Rclone:
             )
             return CompletedProcess.from_subprocess(cp)
 
-        datalists: dict[str, list[str]] = partition_files(payload)
+        datalists: dict[str, list[str]] = group_files(payload)
         completed_processes: list[subprocess.CompletedProcess] = []
         verbose = get_verbose(verbose)
 

{rclone_api-1.0.41 → rclone_api-1.0.43}/src/rclone_api/util.py

@@ -129,15 +129,3 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
         if path.exists():
             return
     raise TimeoutError(f"Path {path} did not exist after {timeout} seconds")
-
-
-def partition_files(files: list[str]) -> dict[str, list[str]]:
-    """split between filename and parent directory path"""
-    datalists: dict[str, list[str]] = {}
-    for f in files:
-        base = os.path.basename(f)
-        parent_path = os.path.dirname(f)
-        if parent_path not in datalists:
-            datalists[parent_path] = []
-        datalists[parent_path].append(base)
-    return datalists

{rclone_api-1.0.41 → rclone_api-1.0.43}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.41
+Version: 1.0.43
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.41 → rclone_api-1.0.43}/src/rclone_api.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ src/rclone_api/dir_listing.py
 src/rclone_api/exec.py
 src/rclone_api/file.py
 src/rclone_api/filelist.py
+src/rclone_api/group_files.py
 src/rclone_api/process.py
 src/rclone_api/rclone.py
 src/rclone_api/remote.py
@@ -49,6 +50,7 @@ src/rclone_api/cmd/list_files.py
 tests/test_cmd_list_files.py
 tests/test_copy.py
 tests/test_diff.py
+tests/test_group_files.py
 tests/test_is_synced.py
 tests/test_ls.py
 tests/test_mount.py

rclone_api-1.0.43/tests/test_group_files.py

@@ -0,0 +1,101 @@
+"""
+Unit test file.
+"""
+
+import unittest
+
+from rclone_api.group_files import group_files
+
+
+class GroupFilestest(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def test_simple_group_files(self) -> None:
+        files = [
+            "dst:Bucket/subdir/file1.txt",
+            "dst:Bucket/subdir/file2.txt",
+        ]
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 1)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:Bucket/subdir", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir"]), 2)
+        expected_files = [
+            "file1.txt",
+            "file2.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir"])
+        self.assertIn(expected_files[1], groups["dst:Bucket/subdir"])
+        print("done")
+
+    def test_different_paths(self) -> None:
+        files = [
+            "dst:Bucket/subdir/file1.txt",
+            "dst:Bucket/subdir2/file2.txt",
+        ]
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 2)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:Bucket/subdir", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir"]), 1)
+        expected_files = [
+            "file1.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir"])
+        # dst:/Bucket/subdir2 should be the key
+        self.assertIn("dst:Bucket/subdir2", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir2"]), 1)
+
+    def test_two_big_directories(self) -> None:
+        files = [
+            "dst:Bucket/subdir/file1.txt",
+            "dst:Bucket/subdir/file2.txt",
+            "dst:Bucket/subdir2/file3.txt",
+            "dst:Bucket/subdir2/file4.txt",
+        ]
+
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 2)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:Bucket/subdir", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir"]), 2)
+        expected_files = [
+            "file1.txt",
+            "file2.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir"])
+        self.assertIn(expected_files[1], groups["dst:Bucket/subdir"])
+        # dst:/Bucket/subdir2 should be the key
+        self.assertIn("dst:Bucket/subdir2", groups)
+        self.assertEqual(len(groups["dst:Bucket/subdir2"]), 2)
+        expected_files = [
+            "file3.txt",
+            "file4.txt",
+        ]
+        self.assertIn(expected_files[0], groups["dst:Bucket/subdir2"])
+        self.assertIn(expected_files[1], groups["dst:Bucket/subdir2"])
+        print("done")
+
+    def test_two_fine_grained(self) -> None:
+        files = [
+            "dst:TorrentBooks/libgenrs_nonfiction/204000/a2b20b2c89240ce81dec16091e18113e",
+            "dst:TorrentBooks/libgenrs_nonfiction/208000/155fe185bc03048b003a8e145ed097c8",
+            "dst:TorrentBooks/libgenrs_nonfiction/208001/155fe185bc03048b003a8e145ed097c8",
+            "dst:TorrentBooks/libgenrs_nonfiction/208002/155fe185bc03048b003a8e145ed097c8",
+            "dst:TorrentBooks/libgenrs_nonfiction/2080054/155fe185bc03048b003a8e145ed097c4",
+        ]
+        # expect that this all goes under the same parent
+        groups: dict[str, list[str]] = group_files(files)
+        self.assertEqual(len(groups), 1)
+        # dst:/Bucket/subdir should be the key
+        self.assertIn("dst:TorrentBooks/libgenrs_nonfiction", groups)
+        self.assertEqual(len(groups["dst:TorrentBooks/libgenrs_nonfiction"]), 5)
+        expected_files = [
+            "204000/a2b20b2c89240ce81dec16091e18113e",
+            "208000/155fe185bc03048b003a8e145ed097c8",
+        ]
+        self.assertIn(expected_files[0], groups["dst:TorrentBooks/libgenrs_nonfiction"])
+
+
+if __name__ == "__main__":
+    unittest.main()