rclone-api 1.0.78__tar.gz → 1.0.80__tar.gz

{rclone_api-1.0.78 → rclone_api-1.0.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.78
+Version: 1.0.80
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.78 → rclone_api-1.0.80}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.78"
+version = "1.0.80"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.78 → rclone_api-1.0.80}/src/rclone_api/group_files.py

@@ -1,4 +1,11 @@
 from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class PrefixResult:
+    prefix: str
+    files: list[str]
 
 
 @dataclass
@@ -178,23 +185,6 @@ def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, lis
     return out
 
 
-def group_under_remote(
-    files: list[str], fully_qualified: bool = True
-) -> dict[str, list[str]]:
-    """split between filename and remote"""
-
-    #### DOE STHIS NEED TO BE REMOVEDD????? #####
-
-    assert fully_qualified is True, "Not implemented for fully_qualified=False"
-    out: dict[str, list[str]] = {}
-    for file in files:
-        parsed = parse_file(file)
-        remote = f"{parsed.remote}:"
-        file_list = out.setdefault(remote, [])
-        file_list.append(parsed.to_string(include_remote=False, include_bucket=True))
-    return out
-
-
 def group_under_remote_bucket(
     files: list[str], fully_qualified: bool = True
 ) -> dict[str, list[str]]:
@@ -212,4 +202,52 @@ def group_under_remote_bucket(
     return out
 
 
-__all__ = ["group_files", "group_under_remote", "group_under_remote_bucket"]
+def _get_prefix(path: str) -> tuple[str, str] | None:
+    path_path: Path = Path(path)
+    parts = path_path.parts
+    if len(parts) == 1:
+        return None
+    return parts[0], "/".join(parts[1:])
+
+
+def _common_prefix(prefix: str, files: list[str]) -> PrefixResult:
+    if not files:
+        return PrefixResult(prefix=prefix, files=[])
+    prefix = prefix
+    tmp: list[str] = list(files)
+    while True:
+        if not tmp:
+            break
+        prefix_set: set[str | None] = set()
+        for file in tmp:
+            pair = _get_prefix(file)
+            if pair is None:
+                break
+            _prefix, _ = pair
+            prefix_set.add(_prefix)
+        if len(prefix_set) > 1 or len(prefix_set) == 0:
+            break
+        next_prefix: str | None = prefix_set.pop()
+        if next_prefix is None:
+            break
+        prefix += f"/{next_prefix}"
+        new_tmp: list[str] = []
+        for file in tmp:
+            pair = _get_prefix(file)
+            assert pair is not None
+            _, path = pair
+            new_tmp.append(path)
+        tmp = new_tmp
+    return PrefixResult(prefix=prefix, files=tmp)
+
+
+def group_under_one_prefix(prefix: str, files: list[str]) -> tuple[str, list[str]]:
+    """Group files under one prefix."""
+    if not files:
+        return prefix, []
+    prefix = prefix
+    result = _common_prefix(prefix, files)
+    return result.prefix, result.files
+
+
+__all__ = ["group_files", "group_under_remote_bucket", "group_under_one_prefix"]

{rclone_api-1.0.78 → rclone_api-1.0.80}/src/rclone_api/rclone.py

@@ -22,10 +22,7 @@ from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_proce
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
-from rclone_api.group_files import (
-    group_files,
-    group_under_remote_bucket,
-)
+from rclone_api.group_files import group_files, group_under_one_prefix
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
@@ -849,44 +846,39 @@ class Rclone:
         verbose = get_verbose(verbose)
         check = get_check(check)
         files = list(files)
-        prefix = src if src.endswith(":") else f"{src}/"
-        if src:
-            files = [f"{prefix}{f}" for f in files]
-        file_list: dict[str, list[str]]
-        file_list = group_under_remote_bucket(files)
         all_files: list[File] = []
-        for src_path, files in file_list.items():
-            cmd = ["lsjson", src_path, "--files-only", "-R"]
-            with TemporaryDirectory() as tmpdir:
-                # print("files: " + ",".join(files))
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-                cmd += ["--files-from", str(include_files_txt)]
-                if fast_list:
-                    cmd.append("--fast-list")
-                if other_args:
-                    cmd += other_args
-                cp = self._run(cmd, check=check)
-
-                if cp.returncode != 0:
-                    if check:
-                        raise ValueError(f"Error getting file sizes: {cp.stderr}")
-                    else:
-                        warnings.warn(f"Error getting file sizes: {cp.stderr}")
-                stdout = cp.stdout
-                pieces = src_path.split(":", 1)
-                remote_name = pieces[0]
-                parent_path: str | None
-                if len(pieces) > 1:
-                    parent_path = pieces[1]
+        src, files = group_under_one_prefix(src, files)
+        cmd = ["lsjson", src, "--files-only", "-R"]
+        with TemporaryDirectory() as tmpdir:
+            # print("files: " + ",".join(files))
+            include_files_txt = Path(tmpdir) / "include_files.txt"
+            include_files_txt.write_text("\n".join(files), encoding="utf-8")
+            cmd += ["--files-from", str(include_files_txt)]
+            if fast_list:
+                cmd.append("--fast-list")
+            if other_args:
+                cmd += other_args
+            cp = self._run(cmd, check=check)
+
+            if cp.returncode != 0:
+                if check:
+                    raise ValueError(f"Error getting file sizes: {cp.stderr}")
                 else:
-                    parent_path = None
-                remote = Remote(name=remote_name, rclone=self)
-                paths: list[RPath] = RPath.from_json_str(
-                    stdout, remote, parent_path=parent_path
-                )
-                # print(paths)
-                all_files += [File(p) for p in paths]
+                    warnings.warn(f"Error getting file sizes: {cp.stderr}")
+            stdout = cp.stdout
+            pieces = src.split(":", 1)
+            remote_name = pieces[0]
+            parent_path: str | None
+            if len(pieces) > 1:
+                parent_path = pieces[1]
+            else:
+                parent_path = None
+            remote = Remote(name=remote_name, rclone=self)
+            paths: list[RPath] = RPath.from_json_str(
+                stdout, remote, parent_path=parent_path
+            )
+            # print(paths)
+            all_files += [File(p) for p in paths]
         file_sizes: dict[str, int] = {}
         f: File
         for f in all_files:
@@ -903,9 +895,9 @@ class Rclone:
         for path, size in file_sizes.items():
             # remove the prefix
             path_path = Path(path)
-            path_str = path_path.relative_to(prefix).as_posix()
+            path_str = path_path.relative_to(src).as_posix()
             file_sizes_path_corrected[path_str] = size
         out: SizeResult = SizeResult(
-            prefix=prefix, total_size=total_size, file_sizes=file_sizes_path_corrected
+            prefix=src, total_size=total_size, file_sizes=file_sizes_path_corrected
         )
         return out

{rclone_api-1.0.78 → rclone_api-1.0.80}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.78
+Version: 1.0.80
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.78 → rclone_api-1.0.80}/tests/test_size_files.py

@@ -64,18 +64,11 @@ class RcloneSizeFilesTester(unittest.TestCase):
         src = f"dst:{BUCKET_NAME}"
         for dirlisting in rclone.walk(src, max_depth=1):
             if is_first:
-                # assert just one file
-                # assert len(dirlisting.files) == 1
                 self.assertEqual(len(dirlisting.files), 1)
-                # assert it's first.txt
                 self.assertEqual(dirlisting.files[0].name, "first.txt")
                 is_first = False
-            # print(dirlisting)
             for file in dirlisting.files_relative(src):
                 files.append(file)
-
-        # print(files)
-
         size_map: SizeResult = rclone.size_files(src=src, files=files, check=True)
         print(size_map)
         print("done")