rclone-api 1.5.8__py3-none-any.whl

rclone_api/dir.py

@@ -0,0 +1,113 @@
+import json
+from pathlib import Path
+from typing import Generator
+
+from rclone_api.dir_listing import DirListing
+from rclone_api.remote import Remote
+from rclone_api.rpath import RPath
+from rclone_api.types import ListingOption, Order
+
+
+class Dir:
+    """Remote file dataclass."""
+
+    @property
+    def remote(self) -> Remote:
+        return self.path.remote
+
+    @property
+    def name(self) -> str:
+        return self.path.name
+
+    def __init__(self, path: RPath | Remote) -> None:
+        """Initialize Dir with either an RPath or Remote.
+
+        Args:
+            path: Either an RPath object or a Remote object
+        """
+        if isinstance(path, Remote):
+            # Need to create an RPath for the Remote's root
+            self.path = RPath(
+                remote=path,
+                path=str(path),
+                name=str(path),
+                size=0,
+                mime_type="inode/directory",
+                mod_time="",
+                is_dir=True,
+            )
+            # Ensure the RPath has the same rclone instance as the Remote
+            self.path.set_rclone(path.rclone)
+        else:
+            self.path = path
+        # self.path.set_rclone(self.path.remote.rclone)
+        assert self.path.rclone is not None
+
+    def ls(
+        self,
+        max_depth: int | None = None,
+        glob: str | None = None,
+        order: Order = Order.NORMAL,
+        listing_option: ListingOption = ListingOption.ALL,
+    ) -> DirListing:
+        """List files and directories in the given path."""
+        assert self.path.rclone is not None
+        dir = Dir(self.path)
+        return self.path.rclone.ls(
+            dir,
+            max_depth=max_depth,
+            glob=glob,
+            order=order,
+            listing_option=listing_option,
+        )
+
+    def relative_to(self, other: "Dir") -> str:
+        """Return the relative path to the other directory."""
+        self_path = Path(self.path.path)
+        other_path = Path(other.path.path)
+        rel_path = self_path.relative_to(other_path)
+        return str(rel_path.as_posix())
+
+    def walk(
+        self, breadth_first: bool, max_depth: int = -1
+    ) -> Generator[DirListing, None, None]:
+        """List files and directories in the given path."""
+        from rclone_api.detail.walk import walk
+
+        assert self.path.rclone is not None
+        return walk(self, breadth_first=breadth_first, max_depth=max_depth)
+
+    def to_json(self) -> dict:
+        """Convert the Dir to a JSON serializable dictionary."""
+        return self.path.to_json()
+
+    def __str__(self) -> str:
+        return str(self.path)
+
+    def __repr__(self) -> str:
+        data = self.path.to_json()
+        data_str = json.dumps(data)
+        return data_str
+
+    def to_string(self, include_remote: bool = True) -> str:
+        """Convert the File to a string."""
+        out = str(self.path)
+        if not include_remote:
+            _, out = out.split(":", 1)
+        return out
+
+    # / operator
+    def __truediv__(self, other: str) -> "Dir":
+        """Join the current path with another path."""
+        path = Path(self.path.path) / other
+        rpath = RPath(
+            self.path.remote,
+            str(path.as_posix()),
+            name=other,
+            size=0,
+            mime_type="inode/directory",
+            mod_time="",
+            is_dir=True,
+        )
+        rpath.set_rclone(self.path.rclone)
+        return Dir(rpath)

rclone_api/dir_listing.py

@@ -0,0 +1,66 @@
+import json
+import warnings
+
+from rclone_api.rpath import RPath
+
+
+def _dedupe(items: list[RPath]) -> list[RPath]:
+    """Remove duplicate items from a list of RPath objects."""
+    seen = set()
+    unique_items = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            unique_items.append(item)
+        else:
+            warnings.warn(f"Duplicate item found: {item}, filtered out.")
+    return unique_items
+
+
+class DirListing:
+    """Remote file dataclass."""
+
+    def __init__(self, dirs_and_files: list[RPath]) -> None:
+        from rclone_api.dir import Dir
+        from rclone_api.file import File
+
+        dirs_and_files = _dedupe(dirs_and_files)
+
+        self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
+        self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
+
+    def files_relative(self, prefix: str) -> list[str]:
+        """Return a list of file paths relative to the root directory."""
+        from rclone_api.file import File
+
+        out: list[str] = []
+        f: File
+        for f in self.files:
+            out.append(f.relative_to(prefix))
+        return out
+
+    def __str__(self) -> str:
+        n_files = len(self.files)
+        n_dirs = len(self.dirs)
+        msg = f"\nFiles: {n_files}\n"
+        if n_files > 0:
+            for f in self.files:
+                msg += f"  {f}\n"
+        msg += f"Dirs: {n_dirs}\n"
+        if n_dirs > 0:
+            for d in self.dirs:
+                msg += f"  {d}\n"
+        return msg
+
+    def __repr__(self) -> str:
+        dirs: list = []
+        files: list = []
+        for d in self.dirs:
+            dirs.append(d.path.to_json())
+        for f in self.files:
+            files.append(f.path.to_json())
+        json_obj = {
+            "dirs": dirs,
+            "files": files,
+        }
+        return json.dumps(json_obj, indent=2)

rclone_api/exec.py

@@ -0,0 +1,40 @@
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api.config import Config
+from rclone_api.process import Process, ProcessArgs
+
+
+@dataclass
+class RcloneExec:
+    """Rclone execution dataclass."""
+
+    rclone_config: Path | Config
+    rclone_exe: Path
+
+    def execute(
+        self, cmd: list[str], check: bool, capture: bool | Path | None = None
+    ) -> subprocess.CompletedProcess:
+        """Execute rclone command."""
+        from rclone_api.util import rclone_execute
+
+        return rclone_execute(
+            cmd, self.rclone_config, self.rclone_exe, check=check, capture=capture
+        )
+
+    def launch_process(
+        self, cmd: list[str], capture: bool | None, log: Path | None
+    ) -> Process:
+        """Launch rclone process."""
+
+        args: ProcessArgs = ProcessArgs(
+            cmd=cmd,
+            rclone_conf=self.rclone_config,
+            rclone_exe=self.rclone_exe,
+            cmd_list=cmd,
+            capture_stdout=capture,
+            log=log,
+        )
+        process = Process(args)
+        return process

rclone_api/experimental/flags.py

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+
+from rclone_api.experimental.flags_base import BaseFlags, merge_flags
+from rclone_api.types import SizeSuffix
+
+
+@dataclass
+class CopyFlags(BaseFlags):
+    check_first: bool | None = None
+    checksum: bool | None = False
+    compare_dest: list[str] | None = None
+    copy_dest: list[str] | None = None
+    cutoff_mode: str | None = None
+    ignore_case_sync: bool | None = None
+    ignore_checksum: bool | None = None
+    ignore_existing: bool | None = None
+    ignore_size: bool | None = None
+    ignore_times: bool | None = None
+    immutable: bool | None = None
+    inplace: bool | None = None
+    links: bool | None = None
+    max_backlog: int | None = None
+    max_duration: str | None = None
+    max_transfer: SizeSuffix | None = None
+    metadata: bool | None = None
+    modify_window: str | None = None
+    multi_thread_chunk_size: str | None = None
+    multi_thread_cutoff: str | None = None
+    multi_thread_streams: int | None = None
+    multi_thread_write_buffer_size: str | None = None
+    no_check_dest: bool | None = None
+    no_traverse: bool | None = None
+    no_update_dir_modtime: bool | None = None
+    no_update_modtime: bool | None = None
+    order_by: str | None = None
+    partial_suffix: str | None = None
+    refresh_times: bool | None = None
+    server_side_across_configs: bool | None = None
+    size_only: bool | None = None
+    streaming_upload_cutoff: str | None = None
+    update: bool | None = None
+
+    def to_args(self) -> list[str]:
+        return super().to_args()
+
+    def merge(self, other: "CopyFlags") -> "CopyFlags":
+        return merge_flags(CopyFlags, self, other)
+
+    def __repr__(self):
+        return super().__repr__()
+
+
+@dataclass
+class Flags(BaseFlags):
+    copy: CopyFlags | None = None
+
+    def to_args(self) -> list[str]:
+        return super().to_args()
+
+    def merge(self, other: "Flags") -> "Flags":
+        return merge_flags(Flags, self, other)
+
+    def __repr__(self):
+        return super().__repr__()
+
+
+def unit_test() -> None:
+    copy_flags_a = CopyFlags(compare_dest=["a", "b"])
+    copy_flags_b = CopyFlags(checksum=False)
+    flags_a = copy_flags_a.merge(copy_flags_b)
+    print("A:", flags_a)
+
+    copy_flags_c = CopyFlags(checksum=True)
+    copy_flags_d = CopyFlags(checksum=False)
+
+    merged_c_d = copy_flags_c.merge(copy_flags_d)
+    print("B:", merged_c_d)
+    merged_d_c = copy_flags_d.merge(copy_flags_c)
+    print("C:", merged_d_c)
+
+    # now do the one with the SizeSuffix type
+    copy_flags_e = CopyFlags(max_transfer=SizeSuffix("128M"))
+    copy_flags_f = CopyFlags(max_transfer=SizeSuffix("256M"))
+    merged_e_f = copy_flags_e.merge(copy_flags_f)
+    print("D:", merged_e_f)
+
+
+if __name__ == "__main__":
+    unit_test()

rclone_api/experimental/flags_base.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass, fields, is_dataclass
+from typing import Type, TypeVar
+
+T = TypeVar("T")
+
+
+def merge_flags(cls: Type[T], dataclass_a: T, dataclass_b: T) -> T:
+    if not is_dataclass(dataclass_a) or not is_dataclass(dataclass_b):
+        raise ValueError("Both inputs must be dataclass instances")
+    if type(dataclass_a) is not type(dataclass_b):
+        raise ValueError("Dataclass instances must be of the same type")
+
+    merged_kwargs = {}
+    for field in fields(dataclass_a):
+        a_value = getattr(dataclass_a, field.name)
+        b_value = getattr(dataclass_b, field.name)
+
+        if is_dataclass(a_value) and is_dataclass(b_value):
+            merged_kwargs[field.name] = merge_flags(type(a_value), a_value, b_value)
+        else:
+            merged_kwargs[field.name] = b_value if b_value is not None else a_value
+
+    return cls(**merged_kwargs)
+
+
+def _field_name_to_flag(field_name: str) -> str:
+    return f"--{field_name.replace('_', '-')}"
+
+
+@dataclass
+class BaseFlags:
+    """provides to_args(), merge() and __repr__ methods for flags dataclasses"""
+
+    def to_args(self) -> list[str]:
+        args = []
+        for field in fields(self):
+            value = getattr(self, field.name)
+            if value is None:
+                continue
+            # If the field value is a nested dataclass that supports to_args, use it.
+            if is_dataclass(value) and hasattr(value, "to_args"):
+                to_args = getattr(value, "to_args")
+                args.extend(to_args())
+            elif isinstance(value, bool):
+                # Only include the flag if the boolean is True.
+                if value:
+                    args.append(_field_name_to_flag(field.name))
+            else:
+                args.append(_field_name_to_flag(field.name))
+                if isinstance(value, list):
+                    # Join list values with a comma.
+                    args.append(",".join(map(str, value)))
+                else:
+                    args.append(str(value))
+        return args
+
+    def __repr__(self):
+        return str(self.to_args())

rclone_api/file.py

@@ -0,0 +1,205 @@
+import json
+import warnings
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+from rclone_api.rpath import RPath
+
+_STRING_INTERNER: dict[str, str] = {}
+
+
+def _intern(s: str) -> str:
+    return _STRING_INTERNER.setdefault(s, s)
+
+
+_SUFFIX_LARGEST_SIZE = len("torrents") + 2
+
+
+def _suffix_clean_bad_parts(suffix: list[str]) -> list[str]:
+    """Remove any bad parts from the suffix list."""
+    out = []
+    for part in suffix:
+        if part in ["", ""]:
+            continue
+        if " " in part:
+            # split on spaces
+            continue
+        if "--" in part:
+            # split on --
+            parts = part.split("--")
+            parts = [x.strip() for x in parts if x.strip()]
+            out.extend(parts)
+        out.append(part)
+
+    out, tmp = [], out
+    for part in tmp:
+        if len(part) > _SUFFIX_LARGEST_SIZE:
+            continue
+        out.append(part)
+    return out
+
+
+def _get_suffix(name: str, chop_compressed_suffixes: bool = True) -> str:
+    # name.sql.gz -> sql.gz
+    try:
+        parts = name.split(".")
+        if len(parts) == 1:
+            return ""
+        parts = _suffix_clean_bad_parts(parts)
+        last_part = parts[-1]
+        if chop_compressed_suffixes:
+            if last_part == "gz" and len(parts) > 2:
+                parts = parts[:-1]
+        return ".".join(parts[-1:])
+    except IndexError:
+        warnings.warn(f"Invalid name: {name} for normal suffix extraction")
+        suffix = Path(name).suffix
+        if suffix.startswith("."):
+            return suffix[1:]
+        return suffix
+
+
+# File is too complex, this is a simple dataclass that can be streamed out.
+@dataclass
+class FileItem:
+    """Remote file dataclass."""
+
+    remote: str
+    parent: str
+    name: str
+    size: int
+    mime_type: str
+    mod_time: str
+    hash: str | None = None
+    id: int | None = None
+
+    @property
+    def path(self) -> str:
+        if self.parent == ".":
+            return f"{self.remote}/{self.name}"
+        else:
+            return f"{self.remote}/{self.parent}/{self.name}"
+
+    @property
+    def path_no_remote(self) -> str:
+        if self.parent == ".":
+            return f"{self.name}"
+        else:
+            return f"{self.parent}/{self.name}"
+
+    @property
+    def real_suffix(self) -> str:
+        return self._suffix
+
+    def __post_init__(self):
+        self.parent = _intern(self.parent)
+        self.mime_type = _intern(self.mime_type)
+        self.remote = _intern(self.remote)
+        self._suffix = _intern(_get_suffix(self.name))
+
+    @staticmethod
+    def from_json(remote: str, data: dict) -> "FileItem | None":
+        try:
+            path_str: str = data["Path"]
+            parent_path = Path(path_str).parent.as_posix()
+            name = data["Name"]
+            size = data["Size"]
+            mime_type = data["MimeType"]
+            mod_time = data["ModTime"]
+
+            return FileItem(
+                remote=remote,
+                parent=parent_path,
+                name=name,
+                size=size,
+                mime_type=mime_type,
+                mod_time=mod_time,
+            )
+        except KeyError:
+            warnings.warn(f"Invalid data: {data}")
+            return None
+
+    @staticmethod
+    def from_json_str(remote: str, data: str) -> "FileItem | None":
+        try:
+            data_dict = json.loads(data)
+            return FileItem.from_json(remote, data_dict)
+        except json.JSONDecodeError:
+            warnings.warn(f"Invalid JSON data: {data}")
+            return None
+
+    # hasher for set membership
+    def __hash__(self) -> int:
+        return hash(self.path_no_remote)
+
+
+class File:
+    """Remote file dataclass."""
+
+    def __init__(
+        self,
+        path: RPath,
+    ) -> None:
+        self.path = path
+
+    @property
+    def name(self) -> str:
+        return self.path.name
+
+    def mod_time(self) -> str:
+        return self.path.mod_time
+
+    def mod_time_dt(self) -> datetime:
+        """Return the modification time as a datetime object."""
+        return self.path.mod_time_dt()
+
+    def read_text(self) -> str:
+        """Read the file contents as bytes.
+
+        Returns:
+            bytes: The file contents
+
+        Raises:
+            RuntimeError: If no rclone instance is associated with this file
+            RuntimeError: If the path represents a directory
+        """
+        if self.path.rclone is None:
+            raise RuntimeError("No rclone instance associated with this file")
+        if self.path.is_dir:
+            raise RuntimeError("Cannot read a directory as bytes")
+
+        result = self.path.rclone._run(["cat", self.path.path], check=True)
+        return result.stdout
+
+    def to_json(self) -> dict:
+        """Convert the File to a JSON serializable dictionary."""
+        return self.path.to_json()
+
+    def to_string(self, include_remote: bool = True) -> str:
+        """Convert the File to a string."""
+        # out = str(self.path)
+        remote = self.path.remote
+        rest = self.path.path
+        if include_remote:
+            return f"{remote.name}:{rest}"
+        return rest
+
+    def relative_to(self, prefix: str) -> str:
+        """Return the relative path to the other directory."""
+        self_path = Path(str(self))
+        rel_path = self_path.relative_to(prefix)
+        return str(rel_path.as_posix())
+
+    @property
+    def size(self) -> int:
+        """Get the size of the file."""
+        return self.path.size
+
+    def __str__(self) -> str:
+        return str(self.path)
+
+    def __repr__(self) -> str:
+        data = self.path.to_json()
+        data_str = json.dumps(data)
+        return data_str

rclone_api/file_item.py

@@ -0,0 +1,68 @@
+import json
+import warnings
+from dataclasses import dataclass
+from pathlib import Path
+
+_STRING_INTERNER: dict[str, str] = {}
+
+
+def _intern(s: str) -> str:
+    return _STRING_INTERNER.setdefault(s, s)
+
+
+# File is too complex, this is a simple dataclass that can be streamed out.
+@dataclass
+class FileItem:
+    """Remote file dataclass."""
+
+    remote: str
+    parent: str
+    name: str
+    size: int
+    mime_type: str
+    mod_time: str
+
+    @property
+    def path(self) -> str:
+        return f"{self.remote}/{self.parent}/{self.name}"
+
+    @property
+    def suffix(self) -> str:
+        return self._suffix
+
+    def __post_init__(self):
+        self.parent = _intern(self.parent)
+        self.mime_type = _intern(self.mime_type)
+        suffix = Path(self.name).suffix
+        self._suffix = _intern(suffix)
+
+    @staticmethod
+    def from_json(data: dict) -> "FileItem | None":
+        try:
+            path_str: str = data["Path"]
+            parent_path = Path(path_str).parent.as_posix()
+            name = data["Name"]
+            size = data["Size"]
+            mime_type = data["MimeType"]
+            mod_time = data["ModTime"]
+
+            return FileItem(
+                remote="DUMMY",
+                parent=parent_path,
+                name=name,
+                size=size,
+                mime_type=mime_type,
+                mod_time=mod_time,
+            )
+        except KeyError:
+            warnings.warn(f"Invalid data: {data}")
+            return None
+
+    @staticmethod
+    def from_json_str(data: str) -> "FileItem | None":
+        try:
+            data_dict = json.loads(data)
+            return FileItem.from_json(data_dict)
+        except json.JSONDecodeError:
+            warnings.warn(f"Invalid JSON data: {data}")
+            return None