rclone-api 1.0.41__py2.py3-none-any.whl → 1.0.42__py2.py3-none-any.whl

rclone_api/group_files.py

@@ -0,0 +1,145 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class FilePathParts:
+    """File path dataclass."""
+
+    remote: str
+    parents: list[str]
+    name: str
+
+
+def parse_file(file_path: str) -> FilePathParts:
+    """Parse file path into parts."""
+    assert not file_path.endswith("/"), "This looks like a directory path"
+    parts = file_path.split(":")
+    remote = parts[0]
+    path = parts[1]
+    if path.startswith("/"):
+        path = path[1:]
+    parents = path.split("/")
+    if len(parents) == 1:
+        return FilePathParts(remote=remote, parents=[], name=parents[0])
+    name = parents.pop()
+    return FilePathParts(remote=remote, parents=parents, name=name)
+
+
+class TreeNode:
+    def __init__(
+        self,
+        name: str,
+        child_nodes: dict[str, "TreeNode"] | None = None,
+        files: list[str] | None = None,
+        parent: "TreeNode | None" = None,
+    ):
+        self.name = name
+        self.child_nodes = child_nodes or {}
+        self.files = files or []
+        self.count = 0
+        self.parent = parent
+
+    def add_count(self):
+        self.count += 1
+        if self.parent:
+            self.parent.add_count()
+
+    def get_path(self) -> str:
+        paths_reversed: list[str] = [self.name]
+        node: TreeNode | None = self
+        assert node is not None
+        while node := node.parent:
+            paths_reversed.append(node.name)
+        return "/".join(reversed(paths_reversed))
+
+    def get_child_subpaths(self, parent_path: str | None = None) -> list[str]:
+        paths: list[str] = []
+        for child in self.child_nodes.values():
+            child_paths = child.get_child_subpaths(parent_path=child.name)
+            paths.extend(child_paths)
+        for file in self.files:
+            if parent_path:
+                file = f"{parent_path}/{file}"
+            paths.append(file)
+        return paths
+
+    def __repr__(self, indent: int = 0) -> str:
+        # return f"{self.name}: {self.count}, {len(self.children)}"
+        leftpad = " " * indent
+        msg = f"{leftpad}{self.name}: {self.count}"
+        if self.child_nodes:
+            # msg += f"\n   {len(self.children)} children"
+            msg += "\n"
+            for child in self.child_nodes.values():
+                if isinstance(child, TreeNode):
+                    msg += child.__repr__(indent + 2)
+                else:
+                    msg += f"{leftpad}  {child}\n"
+        return msg
+
+
+def _merge(node: TreeNode, parent_path: str, out: dict[str, list[str]]) -> None:
+    parent_path = parent_path + "/" + node.name
+    this_count = node.count
+    child_count = 0
+    children_has_files = False
+    if not node.child_nodes and not node.files:
+        return  # done
+
+    if node.files:
+        children_has_files = True
+        filelist = out.setdefault(parent_path, [])
+        # for file in node.files:
+        #    filelist.append(file)
+        # out[parent_path] = filelist
+        paths = node.get_child_subpaths()
+        for path in paths:
+            filelist.append(path)
+        out[parent_path] = filelist
+        return
+
+    for child in node.child_nodes.values():
+        child_count += child.count
+    child_count += len(node.files)
+    for file in node.files:
+        child_count += 1
+
+    if child_count != this_count or children_has_files:
+        # print(
+        #    f"Cannot merge {node.name} because different counts or has children with files"
+        # )
+        filelist = out.setdefault(parent_path, [])
+        for child in node.child_nodes.values():
+            subpaths = child.get_child_subpaths()
+            filelist.extend(subpaths)
+        out[parent_path] = filelist
+    else:
+        for child in node.child_nodes.values():
+            _merge(child, parent_path, out)
+
+
+def group_files(files: list[str]) -> dict[str, list[str]]:
+    """split between filename and parent directory path"""
+    tree: dict[str, TreeNode] = {}
+    for file in files:
+        parts = parse_file(file)
+        remote = parts.remote
+        node: TreeNode = tree.setdefault(remote, TreeNode(remote))
+        for parent in parts.parents:
+            is_last = parent == parts.parents[-1]
+            node = node.child_nodes.setdefault(parent, TreeNode(parent, parent=node))
+            if is_last:
+                node.files.append(parts.name)
+                node.add_count()
+    outpaths: dict[str, list[str]] = {}
+    for _, node in tree.items():
+        _merge(node, "", outpaths)
+    out: dict[str, list[str]] = {}
+    for path, files in outpaths.items():
+        # fixup path
+        assert path.startswith("/"), "Path should start with /"
+        path = path[1:]
+        # replace the first / with :
+        path = path.replace("/", ":", 1)
+        out[path] = files
+    return out

rclone_api/rclone.py

@@ -21,13 +21,13 @@ from rclone_api.diff import DiffItem, diff_stream_from_running_process
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
+from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
 from rclone_api.util import (
     get_rclone_exe,
     get_verbose,
-    partition_files,
     to_path,
     wait_for_mount,
 )
@@ -218,7 +218,7 @@ class Rclone:
         if len(payload) == 0:
             return
 
-        datalists: dict[str, list[str]] = partition_files(payload)
+        datalists: dict[str, list[str]] = group_files(payload)
         out: subprocess.CompletedProcess | None = None
 
         futures: list[Future] = []
@@ -297,7 +297,7 @@ class Rclone:
             )
             return CompletedProcess.from_subprocess(cp)
 
-        datalists: dict[str, list[str]] = partition_files(payload)
+        datalists: dict[str, list[str]] = group_files(payload)
         completed_processes: list[subprocess.CompletedProcess] = []
         verbose = get_verbose(verbose)
 

rclone_api/util.py

@@ -129,15 +129,3 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
         if path.exists():
             return
     raise TimeoutError(f"Path {path} did not exist after {timeout} seconds")
-
-
-def partition_files(files: list[str]) -> dict[str, list[str]]:
-    """split between filename and parent directory path"""
-    datalists: dict[str, list[str]] = {}
-    for f in files:
-        base = os.path.basename(f)
-        parent_path = os.path.dirname(f)
-        if parent_path not in datalists:
-            datalists[parent_path] = []
-        datalists[parent_path].append(base)
-    return datalists

{rclone_api-1.0.41.dist-info → rclone_api-1.0.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.41
+Version: 1.0.42
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.41.dist-info → rclone_api-1.0.42.dist-info}/RECORD

@@ -10,17 +10,18 @@ rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,157
 rclone_api/exec.py,sha256=HWmnU2Jwb-3EttSbAJSaLloYA7YI2mHTzRJ5VEri9aM,941
 rclone_api/file.py,sha256=D02iHJW1LhfOiM_R_yPHP8_ApnDiYrkuraVcrV8-qkw,1246
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
+rclone_api/group_files.py,sha256=xHE3_gvvMWgvLk33TyTNAa0GRFvIRo2qkwsEKA-yujU,4730
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=hPR1dV4s-ekruVBbLHpojNvNLreJVdm3tWVOHsSDh2c,20986
+rclone_api/rclone.py,sha256=zKLlHykEs43kt-FB3_YFvwPmRcHY2a3-fShMfGEjYH0,21004
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
-rclone_api/util.py,sha256=yUN7afH4PuuUNcrXwdtyDTIluvkNOw-c5v5VPKDXoBM,4325
+rclone_api/util.py,sha256=sUjH5NmsawmNbPMY7V6hD8vFJXCwbl44XM1kuij3tA0,3918
 rclone_api/walk.py,sha256=kca0t1GAnF6FLclN01G8NG__Qe-ggodLtAbQSHyVPng,2968
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.41.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.41.dist-info/METADATA,sha256=GnbL4D29d2oXbfexuymEQT9P1-PVEQJtoxxAd8YCa3A,4489
-rclone_api-1.0.41.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.41.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.41.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.41.dist-info/RECORD,,
+rclone_api-1.0.42.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.42.dist-info/METADATA,sha256=f5ePfkFNusMVvM9SSIdDsLXuZOUFKYF_ak8yAKnuzc4,4489
+rclone_api-1.0.42.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.42.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.42.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.42.dist-info/RECORD,,