rclone-api 1.0.76__py2.py3-none-any.whl → 1.0.77__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -9,7 +9,7 @@ from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
-from .types import ListingOption, Order
+from .types import GroupingOption, ListingOption, Order, SizeResult
 
 __all__ = [
     "Rclone",
@@ -29,4 +29,6 @@ __all__ = [
     "ListingOption",
     "Order",
     "ListingOption",
+    "GroupingOption",
+    "SizeResult",
 ]

rclone_api/completed_process.py

@@ -47,3 +47,14 @@ class CompletedProcess:
             if rtn != 0:
                 return rtn
         return 0
+
+    def __str__(self) -> str:
+
+        cmd_strs: list[str] = []
+        rtn_cods: list[int] = []
+        for cp in self.completed:
+            cmd_strs.append(subprocess.list2cmdline(cp.args))
+            rtn_cods.append(cp.returncode)
+        msg = f"CompletedProcess: {len(cmd_strs)} commands\n"
+        msg += "\n".join([f"{cmd} -> {rtn}" for cmd, rtn in zip(cmd_strs, rtn_cods)])
+        return msg

rclone_api/file.py

@@ -40,10 +40,17 @@ class File:
 
     def to_string(self, include_remote: bool = True) -> str:
         """Convert the File to a string."""
-        out = str(self.path)
-        if not include_remote:
-            _, out = out.split(":", 1)
-        return out
+        # out = str(self.path)
+        remote = self.path.remote
+        rest = self.path.path
+        if include_remote:
+            return f"{remote.name}:{rest}"
+        return rest
+
+    @property
+    def size(self) -> int:
+        """Get the size of the file."""
+        return self.path.size
 
     def __str__(self) -> str:
         return str(self.path)

rclone_api/group_files.py

@@ -9,6 +9,19 @@ class FilePathParts:
     parents: list[str]
     name: str
 
+    def to_string(self, include_remote: bool, include_bucket: bool) -> str:
+        """Convert to string, may throw for not include_bucket=False."""
+        parents = list(self.parents)
+        if not include_bucket:
+            parents.pop(0)
+        path = "/".join(parents)
+        if path:
+            path += "/"
+        path += self.name
+        if include_remote:
+            return f"{self.remote}{path}"
+        return path
+
 
 def parse_file(file_path: str) -> FilePathParts:
     """Parse file path into parts."""
@@ -165,4 +178,35 @@ def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, lis
     return out
 
 
-__all__ = ["group_files"]
+def group_under_remote(
+    files: list[str], fully_qualified: bool = True
+) -> dict[str, list[str]]:
+    """split between filename and remote"""
+    assert fully_qualified is True, "Not implemented for fully_qualified=False"
+    out: dict[str, list[str]] = {}
+    for file in files:
+        parsed = parse_file(file)
+        remote = f"{parsed.remote}:"
+        file_list = out.setdefault(remote, [])
+        file_list.append(parsed.to_string(include_remote=False, include_bucket=True))
+    return out
+
+
+def group_under_remote_bucket(
+    files: list[str], fully_qualified: bool = True
+) -> dict[str, list[str]]:
+    """split between filename and bucket"""
+    assert fully_qualified is True, "Not implemented for fully_qualified=False"
+    out: dict[str, list[str]] = {}
+    for file in files:
+        parsed = parse_file(file)
+        remote = f"{parsed.remote}:"
+        parts = parsed.parents
+        bucket = parts[0]
+        remote_bucket = f"{remote}{bucket}"
+        file_list = out.setdefault(remote_bucket, [])
+        file_list.append(parsed.to_string(include_remote=False, include_bucket=False))
+    return out
+
+
+__all__ = ["group_files", "group_under_remote", "group_under_remote_bucket"]

rclone_api/rclone.py

@@ -22,11 +22,21 @@ from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_proce
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
-from rclone_api.group_files import group_files
+from rclone_api.group_files import (
+    group_files,
+    group_under_remote,
+    group_under_remote_bucket,
+)
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption, ModTimeStrategy, Order
+from rclone_api.types import (
+    GroupingOption,
+    ListingOption,
+    ModTimeStrategy,
+    Order,
+    SizeResult,
+)
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -827,3 +837,68 @@ class Rclone:
         if proc.poll() is not None:
             raise ValueError("NFS serve process failed to start")
         return proc
+
+    def size_files(
+        self,
+        files: list[str],
+        fast_list: bool = True,
+        other_args: list[str] | None = None,
+        grouping: GroupingOption = GroupingOption.BUCKET,
+        check: bool | None = False,
+        verbose: bool | None = None,
+    ) -> SizeResult:
+        """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
+        verbose = get_verbose(verbose)
+        check = get_check(check)
+        file_list: dict[str, list[str]]
+        if grouping == GroupingOption.BUCKET:
+            file_list = group_under_remote_bucket(files)
+        elif grouping == GroupingOption.REMOTE:
+            file_list = group_under_remote(files)
+        all_files: list[File] = []
+        for src_path, files in file_list.items():
+            cmd = ["lsjson", src_path, "--files-only", "-R"]
+            with TemporaryDirectory() as tmpdir:
+                # print("files: " + ",".join(files))
+                include_files_txt = Path(tmpdir) / "include_files.txt"
+                include_files_txt.write_text("\n".join(files), encoding="utf-8")
+                cmd += ["--files-from", str(include_files_txt)]
+                if fast_list:
+                    cmd.append("--fast-list")
+                if other_args:
+                    cmd += other_args
+                cp = self._run(cmd, check=check)
+
+                if cp.returncode != 0:
+                    if check:
+                        raise ValueError(f"Error getting file sizes: {cp.stderr}")
+                    else:
+                        warnings.warn(f"Error getting file sizes: {cp.stderr}")
+                stdout = cp.stdout
+                pieces = src_path.split(":", 1)
+                remote_name = pieces[0]
+                parent_path: str | None
+                if len(pieces) > 1:
+                    parent_path = pieces[1]
+                else:
+                    parent_path = None
+                remote = Remote(name=remote_name, rclone=self)
+                paths: list[RPath] = RPath.from_json_str(
+                    stdout, remote, parent_path=parent_path
+                )
+                # print(paths)
+                all_files += [File(p) for p in paths]
+        file_sizes: dict[str, int] = {}
+        f: File
+        for f in all_files:
+            p = f.to_string(include_remote=True)
+            if p in file_sizes:
+                warnings.warn(f"Duplicate file found: {p}")
+                continue
+            size = f.size
+            if size == 0:
+                warnings.warn(f"File size is 0: {p}")
+            file_sizes[p] = f.size
+        total_size = sum(file_sizes.values())
+        out: SizeResult = SizeResult(total_size=total_size, file_sizes=file_sizes)
+        return out

rclone_api/remote.py

@@ -7,6 +7,9 @@ class Remote:
     def __init__(self, name: str, rclone: Any) -> None:
         from rclone_api.rclone import Rclone
 
+        if ":" in name:
+            raise ValueError("Remote name cannot contain ':'")
+
         assert isinstance(rclone, Rclone)
         self.name = name
         self.rclone: Rclone = rclone

rclone_api/types.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from enum import Enum
 
 
@@ -16,3 +17,16 @@ class Order(Enum):
     NORMAL = "normal"
     REVERSE = "reverse"
     RANDOM = "random"
+
+
+class GroupingOption(Enum):
+    BUCKET = "bucket"
+    REMOTE = "remote"
+
+
+@dataclass
+class SizeResult:
+    """Size result dataclass."""
+
+    total_size: int
+    file_sizes: dict[str, int]

rclone_api/util.py

@@ -104,7 +104,7 @@ def rclone_execute(
         )
         if verbose:
             cmd_str = subprocess.list2cmdline(cmd)
-            print(f"Running: {cmd_str}")
+            print(f"\nRunning: {cmd_str}")
         cp = subprocess.run(
             cmd, capture_output=capture, encoding="utf-8", check=False, shell=False
         )

{rclone_api-1.0.76.dist-info → rclone_api-1.0.77.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.76
+Version: 1.0.77
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.76.dist-info → rclone_api-1.0.77.dist-info}/RECORD

@@ -1,6 +1,6 @@
-rclone_api/__init__.py,sha256=kJUk9KAxQ2AEms9lM5yqw-dqlabarhU69b2FAxJVBlY,692
+rclone_api/__init__.py,sha256=5KkqA2OnclssekOdx9ZdT1fFRlxzhpB9zBvXhpG44Ag,760
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
-rclone_api/completed_process.py,sha256=Pp-hXnLgej0IGO5ee9Fmx64dGzIofbQFEUyXdFCvO54,1371
+rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
 rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
 rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
@@ -8,22 +8,22 @@ rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
 rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
 rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
 rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
-rclone_api/file.py,sha256=YtR5Y6c0YfXTS-sReOy2UgiSnafcAeO6b2hnbojBQD4,1423
+rclone_api/file.py,sha256=1rrZk_L2mdkgsvhr57dbs89wUaDb2SZBSwJg4e13xrk,1602
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
-rclone_api/group_files.py,sha256=kOHh6ysFDkxjldSwvW6KqmiADUC1yFCdrZRY57TvbGY,5328
+rclone_api/group_files.py,sha256=pjZAjy01rh5zQ9fExXHDQ4mxmSClVVp_L_3uUWqrXsE,6939
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=JeeqJsJH8rBZW895rkT3HMCkaWeKllnG-6xj_PzLwXY,29738
-rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
+rclone_api/rclone.py,sha256=ptozXdIK9H3wEjcVBjZLLYh2sU9rTbrAc0SQfvKON-k,32598
+rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=DcbNw1R6j2f_1zHrhqEJcpCR-8kJfFJawMY0AmPsCnM,321
-rclone_api/util.py,sha256=_cvmHcJPRl2yXw4zgZiop3z-riA_8Ek6S5NDPw8cqSY,4198
+rclone_api/types.py,sha256=yysVEP4xxmYWl5DoLoRzBFaMgmJXFxRvQF34FqMtxGE,545
+rclone_api/util.py,sha256=XMrA2m_di4h8JTM-qyx2iyrFZn-l-or_SJOa5tEsDsI,4200
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.76.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.76.dist-info/METADATA,sha256=13_xQBLBt8rYCmVYCXjHahKRMdC8p1Ao0lKOY8xhlVU,4489
-rclone_api-1.0.76.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.76.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.76.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.76.dist-info/RECORD,,
+rclone_api-1.0.77.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.77.dist-info/METADATA,sha256=8SsedCvu64dTTGi_C-uaTKrDOnKZ5NMzUM9y1WvzoAE,4489
+rclone_api-1.0.77.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.77.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.77.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.77.dist-info/RECORD,,