rclone-api 1.0.88__py2.py3-none-any.whl → 1.0.90__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -1,5 +1,5 @@
 from .completed_process import CompletedProcess
-from .config import Config
+from .config import Config, Parsed, Section
 from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
@@ -9,6 +9,7 @@ from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .s3.types import MultiUploadResult
 from .types import ListingOption, Order, SizeResult
 
 __all__ = [
@@ -30,4 +31,7 @@ __all__ = [
     "Order",
     "ListingOption",
     "SizeResult",
+    "Parsed",
+    "Section",
+    "MultiUploadResult",
 ]

rclone_api/cmd/copy_large_s3.py

@@ -0,0 +1,99 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import MultiUploadResult, Rclone
+
+_1MB = 1024 * 1024
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str
+    dst: str
+    chunk_size_mb: int
+    read_concurrent_chunks: int
+    retries: int
+    save_state_json: Path
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=True
+    )
+    parser.add_argument(
+        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+    )
+    parser.add_argument(
+        "--read-concurrent-chunks",
+        help="Maximum number of chunks to read",
+        type=int,
+        default=4,
+    )
+    parser.add_argument("--retries", help="Number of retries", type=int, default=3)
+    parser.add_argument(
+        "--resumable-json",
+        help="Path to resumable JSON file",
+        type=Path,
+        default="resume.json",
+    )
+
+    args = parser.parse_args()
+    out = Args(
+        config_path=Path(args.config),
+        src=args.src,
+        dst=args.dst,
+        chunk_size_mb=args.chunk_size_mb,
+        read_concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.resumable_json,
+    )
+    return out
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
+        src=args.src,
+        dst=args.dst,
+        chunk_size=args.chunk_size_mb * _1MB,
+        concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.save_state_json,
+    )
+    print(rslt)
+    return 0
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+
+    here = Path(__file__).parent
+    project_root = here.parent.parent.parent
+    print(f"project_root: {project_root}")
+    os.chdir(str(project_root))
+    cwd = Path(__file__).parent
+    print(f"cwd: {cwd}")
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    main()

rclone_api/config.py

@@ -1,4 +1,48 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+
+@dataclass
+class Section:
+    name: str
+    data: Dict[str, str] = field(default_factory=dict)
+
+    def add(self, key: str, value: str) -> None:
+        self.data[key] = value
+
+    def type(self) -> str:
+        return self.data["type"]
+
+    def provider(self) -> str | None:
+        return self.data.get("provider")
+
+    def access_key_id(self) -> str:
+        if "access_key_id" in self.data:
+            return self.data["access_key_id"]
+        elif "account" in self.data:
+            return self.data["account"]
+        raise KeyError("No access key found")
+
+    def secret_access_key(self) -> str:
+        # return self.data["secret_access_key"]
+        if "secret_access_key" in self.data:
+            return self.data["secret_access_key"]
+        elif "key" in self.data:
+            return self.data["key"]
+        raise KeyError("No secret access key found")
+
+    def endpoint(self) -> str | None:
+        return self.data.get("endpoint")
+
+
+@dataclass
+class Parsed:
+    # sections: List[ParsedSection]
+    sections: dict[str, Section]
+
+    @staticmethod
+    def parse(content: str) -> "Parsed":
+        return parse_rclone_config(content)
 
 
 @dataclass
@@ -6,3 +50,38 @@ class Config:
     """Rclone configuration dataclass."""
 
     text: str
+
+    def parse(self) -> Parsed:
+        return Parsed.parse(self.text)
+
+
+def parse_rclone_config(content: str) -> Parsed:
+    """
+    Parses an rclone configuration file and returns a list of RcloneConfigSection objects.
+
+    Each section in the file starts with a line like [section_name]
+    followed by key=value pairs.
+    """
+    sections: List[Section] = []
+    current_section: Section | None = None
+
+    lines = content.splitlines()
+    for line in lines:
+        line = line.strip()
+        # Skip empty lines and comments (assumed to start with '#' or ';')
+        if not line or line.startswith(("#", ";")):
+            continue
+        # New section header detected
+        if line.startswith("[") and line.endswith("]"):
+            section_name = line[1:-1].strip()
+            current_section = Section(name=section_name)
+            sections.append(current_section)
+        elif "=" in line and current_section is not None:
+            # Parse key and value, splitting only on the first '=' found
+            key, value = line.split("=", 1)
+            current_section.add(key.strip(), value.strip())
+
+    data: dict[str, Section] = {}
+    for section in sections:
+        data[section.name] = section
+    return Parsed(sections=data)

rclone_api/group_files.py

@@ -68,7 +68,10 @@ class TreeNode:
         paths_reversed: list[str] = [self.name]
         node: TreeNode | None = self
         assert node is not None
-        while node := node.parent:
+        while True:
+            node = node.parent
+            if node is None:
+                break
             paths_reversed.append(node.name)
         return "/".join(reversed(paths_reversed))
 

rclone_api/rclone.py

@@ -6,8 +6,10 @@ import os
 import random
 import subprocess
 import time
+import traceback
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import contextmanager
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -15,7 +17,7 @@ from typing import Generator
 
 from rclone_api import Dir
 from rclone_api.completed_process import CompletedProcess
-from rclone_api.config import Config
+from rclone_api.config import Config, Parsed, Section
 from rclone_api.convert import convert_to_filestr_list, convert_to_str
 from rclone_api.deprecated import deprecated
 from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_process
@@ -26,12 +28,13 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import (
-    ListingOption,
-    ModTimeStrategy,
-    Order,
-    SizeResult,
+from rclone_api.s3.types import (
+    MultiUploadResult,
+    S3MutliPartUploadConfig,
+    S3Provider,
+    S3UploadTarget,
 )
+from rclone_api.types import ListingOption, ModTimeStrategy, Order, SizeResult
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -41,6 +44,8 @@ from rclone_api.util import (
 )
 from rclone_api.walk import walk
 
+_IS_WINDOWS = os.name == "nt"
+
 
 def rclone_verbose(verbose: bool | None) -> bool:
     if verbose is not None:
@@ -48,6 +53,14 @@ def rclone_verbose(verbose: bool | None) -> bool:
     return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
 
 
+def _to_rclone_conf(config: Config | Path) -> Config:
+    if isinstance(config, Path):
+        content = config.read_text(encoding="utf-8")
+        return Config(content)
+    else:
+        return config
+
+
 class Rclone:
     def __init__(
         self, rclone_conf: Path | Config, rclone_exe: Path | None = None
@@ -56,6 +69,7 @@ class Rclone:
             if not rclone_conf.exists():
                 raise ValueError(f"Rclone config file not found: {rclone_conf}")
         self._exec = RcloneExec(rclone_conf, get_rclone_exe(rclone_exe))
+        self.config: Config = _to_rclone_conf(rclone_conf)
 
     def _run(
         self, cmd: list[str], check: bool = False, capture: bool | None = None
@@ -399,7 +413,7 @@ class Rclone:
         using_fast_list = "--fast-list" in other_args
         if using_fast_list:
             warnings.warn(
-                "It's not recommended to use --fast-list with copy_files as the entire repository has to be listed"
+                "It's not recommended to use --fast-list with copy_files as this will perform poorly on large repositories since the entire repository has to be scanned."
             )
 
         if max_partition_workers > 1:
@@ -654,6 +668,127 @@ class Rclone:
         except subprocess.CalledProcessError:
             return False
 
+    def copy_file_resumable_s3(
+        self,
+        src: str,
+        dst: str,
+        save_state_json: Path,
+        chunk_size: int = 16
+        * 1024
+        * 1024,  # This setting will scale the performance of the upload
+        concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
+        retries: int = 3,
+        max_chunks_before_suspension: int | None = None,
+    ) -> MultiUploadResult:
+        """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        from rclone_api.s3.api import S3Client
+        from rclone_api.s3.create import S3Credentials
+        from rclone_api.util import S3PathInfo, split_s3_path
+
+        other_args: list[str] = [
+            "--no-modtime",
+            "--vfs-read-wait",
+            "1s",
+            "--vfs-disk-space-total-size",
+            str(2 * chunk_size * concurrent_chunks),  # purge quickly.
+            "--vfs-read-chunk-size",
+            str(chunk_size),
+            "--vfs-read-chunk-size-limit",
+            str(chunk_size * concurrent_chunks),
+            "--vfs-read-chunk-streams",
+            str(concurrent_chunks),
+            "--vfs-fast-fingerprint",
+        ]
+        mount_path = Path("rclone_api_upload_mount")
+        src_path = Path(src)
+        name = src_path.name
+
+        parent_path = str(src_path.parent.as_posix())
+        with self.scoped_mount(
+            parent_path,
+            mount_path,
+            use_links=True,
+            vfs_cache_mode="minimal",
+            other_args=other_args,
+        ):
+            # raise NotImplementedError("Not implemented yet")
+
+            path_info: S3PathInfo = split_s3_path(dst)
+            remote = path_info.remote
+            bucket_name = path_info.bucket
+            s3_key = path_info.key
+            parsed: Parsed = self.config.parse()
+            sections: dict[str, Section] = parsed.sections
+            if remote not in sections:
+                raise ValueError(
+                    f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
+                )
+
+            section: Section = sections[remote]
+            dst_type = section.type()
+            if dst_type != "s3" and dst_type != "b2":
+                raise ValueError(
+                    f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
+                )
+
+            def get_provider_str(section=section) -> str | None:
+                type: str = section.type()
+                provider: str | None = section.provider()
+                if provider is not None:
+                    return provider
+                if type == "b2":
+                    return S3Provider.BACKBLAZE.value
+                if type != "s3":
+                    raise ValueError(f"Remote {remote} is not an S3 remote")
+                return S3Provider.S3.value
+
+            provider: str = get_provider_str() or S3Provider.S3.value
+            provider_enum = S3Provider.from_str(provider)
+
+            s3_creds: S3Credentials = S3Credentials(
+                provider=provider_enum,
+                access_key_id=section.access_key_id(),
+                secret_access_key=section.secret_access_key(),
+                endpoint_url=section.endpoint(),
+            )
+
+            client = S3Client(s3_creds)
+            config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
+                chunk_size=chunk_size,
+                retries=retries,
+                resume_path_json=save_state_json,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+
+            src_file = mount_path / name
+
+            print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
+            print(f"Source: {src_path}")
+            print(f"bucket_name: {bucket_name}")
+            print(f"upload_config: {config}")
+
+            upload_target: S3UploadTarget
+            upload_config: S3MutliPartUploadConfig
+
+            upload_target = S3UploadTarget(
+                bucket_name=bucket_name,
+                src_file=src_file,
+                s3_key=s3_key,
+            )
+
+            upload_config = S3MutliPartUploadConfig(
+                chunk_size=chunk_size,
+                retries=retries,
+                resume_path_json=save_state_json,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+
+            out: MultiUploadResult = client.upload_file_multipart(
+                upload_target=upload_target,
+                upload_config=upload_config
+            )
+            return out
+
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
     ) -> CompletedProcess:
@@ -682,10 +817,10 @@ class Rclone:
         self,
         src: Remote | Dir | str,
         outdir: Path,
-        allow_writes=False,
-        use_links=True,
-        vfs_cache_mode="full",
-        other_cmds: list[str] | None = None,
+        allow_writes: bool | None = False,
+        use_links: bool | None = None,
+        vfs_cache_mode: str | None = None,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -699,6 +834,9 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        allow_writes = allow_writes or False
+        use_links = use_links or True
+        vfs_cache_mode = vfs_cache_mode or "full"
         if outdir.exists():
             is_empty = not list(outdir.iterdir())
             if not is_empty:
@@ -706,12 +844,13 @@ class Rclone:
                     f"Mount directory already exists and is not empty: {outdir}"
                 )
             outdir.rmdir()
-        try:
+
+        if _IS_WINDOWS:
+            # Windows -> Must create parent directories only if they don't exist
             outdir.parent.mkdir(parents=True, exist_ok=True)
-        except PermissionError:
-            warnings.warn(
-                f"Permission error creating parent directory: {outdir.parent}"
-            )
+        else:
+            # Linux -> Must create parent directories and the directory itself
+            outdir.mkdir(parents=True, exist_ok=True)
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:
@@ -721,19 +860,50 @@ class Rclone:
         if vfs_cache_mode:
             cmd_list.append("--vfs-cache-mode")
             cmd_list.append(vfs_cache_mode)
-        if other_cmds:
-            cmd_list += other_cmds
+        if other_args:
+            cmd_list += other_args
         proc = self._launch_process(cmd_list)
         wait_for_mount(outdir, proc)
         return proc
 
+    @contextmanager
+    def scoped_mount(
+        self,
+        src: Remote | Dir | str,
+        outdir: Path,
+        allow_writes: bool | None = None,
+        use_links: bool | None = None,
+        vfs_cache_mode: str | None = None,
+        other_args: list[str] | None = None,
+    ) -> Generator[Process, None, None]:
+        """Like mount, but can be used in a context manager."""
+        proc = self.mount(
+            src,
+            outdir,
+            allow_writes=allow_writes,
+            use_links=use_links,
+            vfs_cache_mode=vfs_cache_mode,
+            other_args=other_args,
+        )
+        try:
+            yield proc
+        except Exception as e:
+            stack_trace = traceback.format_exc()
+            warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
+            raise
+        finally:
+            if proc.poll() is None:
+                proc.terminate()
+            proc.wait()
+
+    @deprecated("mount")
     def mount_webdav(
         self,
         url: str,
         outdir: Path,
-        vfs_cache_mode="full",
+        vfs_cache_mode: str | None = None,
         vfs_disk_space_total_size: str | None = "10G",
-        other_cmds: list[str] | None = None,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -747,6 +917,20 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        other_args = other_args or []
+        if vfs_cache_mode is None:
+            if "--vfs-cache-mode" in other_args:
+                pass
+            else:
+                vfs_cache_mode = "full"
+        elif "--vfs-cache-mode" in other_args:
+            warnings.warn(
+                f"vfs_cache_mode is set to {vfs_cache_mode} but --vfs-cache-mode is already in other_args"
+            )
+            idx = other_args.index("--vfs-cache-mode")
+            other_args.pop(idx)
+            other_args.pop(idx)  # also the next value which will be the cache mode.
+
         if outdir.exists():
             is_empty = not list(outdir.iterdir())
             if not is_empty:
@@ -757,10 +941,11 @@ class Rclone:
 
         src_str = url
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
-        cmd_list.append("--vfs-cache-mode")
-        cmd_list.append(vfs_cache_mode)
-        if other_cmds:
-            cmd_list += other_cmds
+        if vfs_cache_mode:
+            cmd_list.append("--vfs-cache-mode")
+            cmd_list.append(vfs_cache_mode)
+        if other_args:
+            cmd_list += other_args
         if vfs_disk_space_total_size is not None:
             cmd_list.append("--vfs-cache-max-size")
             cmd_list.append(vfs_disk_space_total_size)
@@ -768,6 +953,7 @@ class Rclone:
         wait_for_mount(outdir, proc)
         return proc
 
+    # Settings optimized for s3.
     def mount_s3(
         self,
         url: str,
@@ -789,7 +975,7 @@ class Rclone:
         vfs_fast_fingerprint: bool = True,
         # vfs-refresh
         vfs_refresh: bool = True,
-        other_cmds: list[str] | None = None,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -797,44 +983,44 @@ class Rclone:
             src: Remote or directory to mount
             outdir: Local path to mount to
         """
-        other_cmds = other_cmds or []
+        other_args = other_args or []
         if modtime_strategy is not None:
-            other_cmds.append(f"--{modtime_strategy.value}")
+            other_args.append(f"--{modtime_strategy.value}")
         if (vfs_cache_mode == "full" or vfs_cache_mode == "writes") and (
-            transfers is not None and "--transfers" not in other_cmds
+            transfers is not None and "--transfers" not in other_args
         ):
-            other_cmds.append("--transfers")
-            other_cmds.append(str(transfers))
-        if dir_cache_time is not None and "--dir-cache-time" not in other_cmds:
-            other_cmds.append("--dir-cache-time")
-            other_cmds.append(dir_cache_time)
+            other_args.append("--transfers")
+            other_args.append(str(transfers))
+        if dir_cache_time is not None and "--dir-cache-time" not in other_args:
+            other_args.append("--dir-cache-time")
+            other_args.append(dir_cache_time)
         if (
             vfs_disk_space_total_size is not None
-            and "--vfs-cache-max-size" not in other_cmds
+            and "--vfs-cache-max-size" not in other_args
         ):
-            other_cmds.append("--vfs-cache-max-size")
-            other_cmds.append(vfs_disk_space_total_size)
-        if vfs_refresh and "--vfs-refresh" not in other_cmds:
-            other_cmds.append("--vfs-refresh")
-        if attribute_timeout is not None and "--attr-timeout" not in other_cmds:
-            other_cmds.append("--attr-timeout")
-            other_cmds.append(attribute_timeout)
+            other_args.append("--vfs-cache-max-size")
+            other_args.append(vfs_disk_space_total_size)
+        if vfs_refresh and "--vfs-refresh" not in other_args:
+            other_args.append("--vfs-refresh")
+        if attribute_timeout is not None and "--attr-timeout" not in other_args:
+            other_args.append("--attr-timeout")
+            other_args.append(attribute_timeout)
         if vfs_read_chunk_streams:
-            other_cmds.append("--vfs-read-chunk-streams")
-            other_cmds.append(str(vfs_read_chunk_streams))
+            other_args.append("--vfs-read-chunk-streams")
+            other_args.append(str(vfs_read_chunk_streams))
         if vfs_read_chunk_size:
-            other_cmds.append("--vfs-read-chunk-size")
-            other_cmds.append(vfs_read_chunk_size)
+            other_args.append("--vfs-read-chunk-size")
+            other_args.append(vfs_read_chunk_size)
         if vfs_fast_fingerprint:
-            other_cmds.append("--vfs-fast-fingerprint")
+            other_args.append("--vfs-fast-fingerprint")
 
-        other_cmds = other_cmds if other_cmds else None
+        other_args = other_args if other_args else None
         return self.mount(
             url,
             outdir,
             allow_writes=allow_writes,
             vfs_cache_mode=vfs_cache_mode,
-            other_cmds=other_cmds,
+            other_args=other_args,
         )
 
     def serve_webdav(
@@ -844,6 +1030,7 @@ class Rclone:
         password: str,
         addr: str = "localhost:2049",
         allow_other: bool = False,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Serve a remote or directory via NFS.
 
@@ -863,6 +1050,8 @@ class Rclone:
         cmd_list.extend(["--user", user, "--pass", password])
         if allow_other:
             cmd_list.append("--allow-other")
+        if other_args:
+            cmd_list += other_args
         proc = self._launch_process(cmd_list)
         time.sleep(2)  # give it a moment to start
         if proc.poll() is not None:
@@ -883,7 +1072,7 @@ class Rclone:
         check = get_check(check)
         if fast_list or (other_args and "--fast-list" in other_args):
             warnings.warn(
-                "It's not recommended to use --fast-list with size_files as the entire repository has to be listed"
+                "It's not recommended to use --fast-list with size_files as this will perform poorly on large repositories since the entire repository has to be scanned."
             )
         files = list(files)
         all_files: list[File] = []