rclone-api 1.0.76__tar.gz → 1.0.78__tar.gz

{rclone_api-1.0.76 → rclone_api-1.0.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.76
+Version: 1.0.78
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.76 → rclone_api-1.0.78}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.76"
+version = "1.0.78"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/__init__.py

@@ -9,7 +9,7 @@ from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
-from .types import ListingOption, Order
+from .types import ListingOption, Order, SizeResult
 
 __all__ = [
     "Rclone",
@@ -29,4 +29,5 @@ __all__ = [
     "ListingOption",
     "Order",
     "ListingOption",
+    "SizeResult",
 ]

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/completed_process.py

@@ -47,3 +47,14 @@ class CompletedProcess:
             if rtn != 0:
                 return rtn
         return 0
+
+    def __str__(self) -> str:
+
+        cmd_strs: list[str] = []
+        rtn_cods: list[int] = []
+        for cp in self.completed:
+            cmd_strs.append(subprocess.list2cmdline(cp.args))
+            rtn_cods.append(cp.returncode)
+        msg = f"CompletedProcess: {len(cmd_strs)} commands\n"
+        msg += "\n".join([f"{cmd} -> {rtn}" for cmd, rtn in zip(cmd_strs, rtn_cods)])
+        return msg

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/dir_listing.py

@@ -29,6 +29,16 @@ class DirListing:
         self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
         self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
 
+    def files_relative(self, prefix: str) -> list[str]:
+        """Return a list of file paths relative to the root directory."""
+        from rclone_api.file import File
+
+        out: list[str] = []
+        f: File
+        for f in self.files:
+            out.append(f.relative_to(prefix))
+        return out
+
     def __str__(self) -> str:
         n_files = len(self.files)
         n_dirs = len(self.dirs)

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/file.py

@@ -1,4 +1,5 @@
 import json
+from pathlib import Path
 
 from rclone_api.rpath import RPath
 
@@ -40,10 +41,23 @@ class File:
 
     def to_string(self, include_remote: bool = True) -> str:
         """Convert the File to a string."""
-        out = str(self.path)
-        if not include_remote:
-            _, out = out.split(":", 1)
-        return out
+        # out = str(self.path)
+        remote = self.path.remote
+        rest = self.path.path
+        if include_remote:
+            return f"{remote.name}:{rest}"
+        return rest
+
+    def relative_to(self, prefix: str) -> str:
+        """Return the relative path to the other directory."""
+        self_path = Path(str(self))
+        rel_path = self_path.relative_to(prefix)
+        return str(rel_path.as_posix())
+
+    @property
+    def size(self) -> int:
+        """Get the size of the file."""
+        return self.path.size
 
     def __str__(self) -> str:
         return str(self.path)

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/group_files.py

@@ -9,6 +9,19 @@ class FilePathParts:
     parents: list[str]
     name: str
 
+    def to_string(self, include_remote: bool, include_bucket: bool) -> str:
+        """Convert to string, may throw for not include_bucket=False."""
+        parents = list(self.parents)
+        if not include_bucket:
+            parents.pop(0)
+        path = "/".join(parents)
+        if path:
+            path += "/"
+        path += self.name
+        if include_remote:
+            return f"{self.remote}{path}"
+        return path
+
 
 def parse_file(file_path: str) -> FilePathParts:
     """Parse file path into parts."""
@@ -165,4 +178,38 @@ def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, lis
     return out
 
 
-__all__ = ["group_files"]
+def group_under_remote(
+    files: list[str], fully_qualified: bool = True
+) -> dict[str, list[str]]:
+    """split between filename and remote"""
+
+    #### DOE STHIS NEED TO BE REMOVEDD????? #####
+
+    assert fully_qualified is True, "Not implemented for fully_qualified=False"
+    out: dict[str, list[str]] = {}
+    for file in files:
+        parsed = parse_file(file)
+        remote = f"{parsed.remote}:"
+        file_list = out.setdefault(remote, [])
+        file_list.append(parsed.to_string(include_remote=False, include_bucket=True))
+    return out
+
+
+def group_under_remote_bucket(
+    files: list[str], fully_qualified: bool = True
+) -> dict[str, list[str]]:
+    """split between filename and bucket"""
+    assert fully_qualified is True, "Not implemented for fully_qualified=False"
+    out: dict[str, list[str]] = {}
+    for file in files:
+        parsed = parse_file(file)
+        remote = f"{parsed.remote}:"
+        parts = parsed.parents
+        bucket = parts[0]
+        remote_bucket = f"{remote}{bucket}"
+        file_list = out.setdefault(remote_bucket, [])
+        file_list.append(parsed.to_string(include_remote=False, include_bucket=False))
+    return out
+
+
+__all__ = ["group_files", "group_under_remote", "group_under_remote_bucket"]

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/rclone.py

@@ -22,11 +22,19 @@ from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_proce
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
-from rclone_api.group_files import group_files
+from rclone_api.group_files import (
+    group_files,
+    group_under_remote_bucket,
+)
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import ListingOption, ModTimeStrategy, Order
+from rclone_api.types import (
+    ListingOption,
+    ModTimeStrategy,
+    Order,
+    SizeResult,
+)
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -827,3 +835,77 @@ class Rclone:
         if proc.poll() is not None:
             raise ValueError("NFS serve process failed to start")
         return proc
+
+    def size_files(
+        self,
+        src: str,
+        files: list[str],
+        fast_list: bool = True,
+        other_args: list[str] | None = None,
+        check: bool | None = False,
+        verbose: bool | None = None,
+    ) -> SizeResult:
+        """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
+        verbose = get_verbose(verbose)
+        check = get_check(check)
+        files = list(files)
+        prefix = src if src.endswith(":") else f"{src}/"
+        if src:
+            files = [f"{prefix}{f}" for f in files]
+        file_list: dict[str, list[str]]
+        file_list = group_under_remote_bucket(files)
+        all_files: list[File] = []
+        for src_path, files in file_list.items():
+            cmd = ["lsjson", src_path, "--files-only", "-R"]
+            with TemporaryDirectory() as tmpdir:
+                # print("files: " + ",".join(files))
+                include_files_txt = Path(tmpdir) / "include_files.txt"
+                include_files_txt.write_text("\n".join(files), encoding="utf-8")
+                cmd += ["--files-from", str(include_files_txt)]
+                if fast_list:
+                    cmd.append("--fast-list")
+                if other_args:
+                    cmd += other_args
+                cp = self._run(cmd, check=check)
+
+                if cp.returncode != 0:
+                    if check:
+                        raise ValueError(f"Error getting file sizes: {cp.stderr}")
+                    else:
+                        warnings.warn(f"Error getting file sizes: {cp.stderr}")
+                stdout = cp.stdout
+                pieces = src_path.split(":", 1)
+                remote_name = pieces[0]
+                parent_path: str | None
+                if len(pieces) > 1:
+                    parent_path = pieces[1]
+                else:
+                    parent_path = None
+                remote = Remote(name=remote_name, rclone=self)
+                paths: list[RPath] = RPath.from_json_str(
+                    stdout, remote, parent_path=parent_path
+                )
+                # print(paths)
+                all_files += [File(p) for p in paths]
+        file_sizes: dict[str, int] = {}
+        f: File
+        for f in all_files:
+            p = f.to_string(include_remote=True)
+            if p in file_sizes:
+                warnings.warn(f"Duplicate file found: {p}")
+                continue
+            size = f.size
+            if size == 0:
+                warnings.warn(f"File size is 0: {p}")
+            file_sizes[p] = f.size
+        total_size = sum(file_sizes.values())
+        file_sizes_path_corrected: dict[str, int] = {}
+        for path, size in file_sizes.items():
+            # remove the prefix
+            path_path = Path(path)
+            path_str = path_path.relative_to(prefix).as_posix()
+            file_sizes_path_corrected[path_str] = size
+        out: SizeResult = SizeResult(
+            prefix=prefix, total_size=total_size, file_sizes=file_sizes_path_corrected
+        )
+        return out

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/remote.py

@@ -7,6 +7,9 @@ class Remote:
     def __init__(self, name: str, rclone: Any) -> None:
         from rclone_api.rclone import Rclone
 
+        if ":" in name:
+            raise ValueError("Remote name cannot contain ':'")
+
         assert isinstance(rclone, Rclone)
         self.name = name
         self.rclone: Rclone = rclone

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/types.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from enum import Enum
 
 
@@ -16,3 +17,17 @@ class Order(Enum):
     NORMAL = "normal"
     REVERSE = "reverse"
     RANDOM = "random"
+
+
+# class GroupingOption(Enum):
+#     BUCKET = "bucket"
+#     REMOTE = "remote"
+
+
+@dataclass
+class SizeResult:
+    """Size result dataclass."""
+
+    prefix: str
+    total_size: int
+    file_sizes: dict[str, int]

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api/util.py

@@ -104,7 +104,7 @@ def rclone_execute(
         )
         if verbose:
             cmd_str = subprocess.list2cmdline(cmd)
-            print(f"Running: {cmd_str}")
+            print(f"\nRunning: {cmd_str}")
         cp = subprocess.run(
             cmd, capture_output=capture, encoding="utf-8", check=False, shell=False
         )

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.76
+Version: 1.0.78
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.76 → rclone_api-1.0.78}/src/rclone_api.egg-info/SOURCES.txt

@@ -64,4 +64,5 @@ tests/test_remote_control.py
 tests/test_remotes.py
 tests/test_scan_missing_folders.py
 tests/test_serve_webdav.py
+tests/test_size_files.py
 tests/test_walk.py

{rclone_api-1.0.76 → rclone_api-1.0.78}/tests/test_diff.py

@@ -70,6 +70,29 @@ class RcloneDiffTests(unittest.TestCase):
         msg = "\n".join([str(item) for item in all])
         print(msg)
 
+    def test_min_max_size(self) -> None:
+        rclone = Rclone(_generate_rclone_config())
+        item: DiffItem
+        all: list[DiffItem] = list(
+            rclone.diff(
+                "dst:rclone-api-unit-test", "dst:rclone-api-unit-test", min_size="70M"
+            )
+        )
+        for item in all:
+            if "internaly_ai_alignment.mp4" in item.path:
+                break
+        else:
+            self.fail("internaly_ai_alignment.mp4 not found")
+        all.clear()
+        all = list(
+            rclone.diff(
+                "dst:rclone-api-unit-test", "dst:rclone-api-unit-test", max_size="70M"
+            )
+        )
+        for item in all:
+            if "internaly_ai_alignment.mp4" in item.path:
+                self.fail("internaly_ai_alignment.mp4 not filtered")
+
     def test_diff_missing_on_dst(self) -> None:
         rclone = Rclone(_generate_rclone_config())
         item: DiffItem

rclone_api-1.0.78/tests/test_size_files.py

@@ -0,0 +1,85 @@
+"""
+Unit test file.
+"""
+
+import os
+import unittest
+
+from dotenv import load_dotenv
+
+from rclone_api import Config, DirListing, Rclone, SizeResult
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config() -> Config:
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+"""
+
+    out = Config(config_text)
+    return out
+
+
+class RcloneSizeFilesTester(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_size(self) -> None:
+        rclone = Rclone(_generate_rclone_config())
+        # rclone.walk
+        dirlisting: DirListing
+        is_first = True
+        files: list[str] = []
+        src = f"dst:{BUCKET_NAME}"
+        for dirlisting in rclone.walk(src, max_depth=1):
+            if is_first:
+                # assert just one file
+                # assert len(dirlisting.files) == 1
+                self.assertEqual(len(dirlisting.files), 1)
+                # assert it's first.txt
+                self.assertEqual(dirlisting.files[0].name, "first.txt")
+                is_first = False
+            # print(dirlisting)
+            for file in dirlisting.files_relative(src):
+                files.append(file)
+
+        # print(files)
+
+        size_map: SizeResult = rclone.size_files(src=src, files=files, check=True)
+        print(size_map)
+        print("done")
+
+
+if __name__ == "__main__":
+    unittest.main()