rclone-api 1.0.88__tar.gz → 1.0.89__tar.gz

{rclone_api-1.0.88 → rclone_api-1.0.89}/.gitignore

@@ -145,3 +145,8 @@ uv.lock
 !.aiderignore
 
 rclone*.conf
+test_mount2
+t.py
+mount
+t2.py
+chunk_store

{rclone_api-1.0.88 → rclone_api-1.0.89}/PKG-INFO

@@ -1,9 +1,8 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.88
+Version: 1.0.89
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
-Maintainer: Zachary Vorhies
 License: BSD 3-Clause License
 Keywords: template-python-cmd
 Classifier: Programming Language :: Python :: 3
@@ -12,8 +11,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyright>=1.1.393
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: boto3<=1.35.99,>=1.20.1
 Dynamic: home-page
-Dynamic: maintainer
 
 # rclone-api
 

{rclone_api-1.0.88 → rclone_api-1.0.89}/pyproject.toml

@@ -13,9 +13,15 @@ classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
     "pyright>=1.1.393",
     "python-dotenv>=1.0.0",
+
+    # BOTO3 Library needs to be pinned to a specific version
+    # BackBlaze S3 fails with checksum header which it doesn't support after 1.35.99
+    # The 1.20.1 was the earliest one I checked that worked and is not the true lower bound.
+    "boto3>=1.20.1,<=1.35.99",
 ]
+
 # Change this with the version number bump.
-version = "1.0.88"
+version = "1.0.89"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.88 → rclone_api-1.0.89}/setup.py

@@ -15,7 +15,6 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 
 if __name__ == "__main__":
     setup(
-        maintainer="Zachary Vorhies",
         keywords=KEYWORDS,
         url=URL,
         package_data={"": ["assets/example.txt"]},

{rclone_api-1.0.88 → rclone_api-1.0.89}/src/rclone_api/__init__.py

@@ -1,5 +1,5 @@
 from .completed_process import CompletedProcess
-from .config import Config
+from .config import Config, Parsed, Section
 from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
@@ -9,6 +9,7 @@ from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .s3.types import MultiUploadResult
 from .types import ListingOption, Order, SizeResult
 
 __all__ = [
@@ -30,4 +31,7 @@ __all__ = [
     "Order",
     "ListingOption",
     "SizeResult",
+    "Parsed",
+    "Section",
+    "MultiUploadResult",
 ]

rclone_api-1.0.89/src/rclone_api/config.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+
+@dataclass
+class Section:
+    name: str
+    data: Dict[str, str] = field(default_factory=dict)
+
+    def add(self, key: str, value: str) -> None:
+        self.data[key] = value
+
+    def provider(self) -> str:
+        return self.data["provider"]
+
+    def access_key_id(self) -> str:
+        return self.data["access_key_id"]
+
+    def secret_access_key(self) -> str:
+        return self.data["secret_access_key"]
+
+    def endpoint(self) -> str | None:
+        return self.data.get("endpoint")
+
+
+@dataclass
+class Parsed:
+    # sections: List[ParsedSection]
+    sections: dict[str, Section]
+
+    @staticmethod
+    def parse(content: str) -> "Parsed":
+        return parse_rclone_config(content)
+
+
+@dataclass
+class Config:
+    """Rclone configuration dataclass."""
+
+    text: str
+
+    def parse(self) -> Parsed:
+        return Parsed.parse(self.text)
+
+
+def parse_rclone_config(content: str) -> Parsed:
+    """
+    Parses an rclone configuration file and returns a list of RcloneConfigSection objects.
+
+    Each section in the file starts with a line like [section_name]
+    followed by key=value pairs.
+    """
+    sections: List[Section] = []
+    current_section: Section | None = None
+
+    lines = content.splitlines()
+    for line in lines:
+        line = line.strip()
+        # Skip empty lines and comments (assumed to start with '#' or ';')
+        if not line or line.startswith(("#", ";")):
+            continue
+        # New section header detected
+        if line.startswith("[") and line.endswith("]"):
+            section_name = line[1:-1].strip()
+            current_section = Section(name=section_name)
+            sections.append(current_section)
+        elif "=" in line and current_section is not None:
+            # Parse key and value, splitting only on the first '=' found
+            key, value = line.split("=", 1)
+            current_section.add(key.strip(), value.strip())
+
+    data: dict[str, Section] = {}
+    for section in sections:
+        data[section.name] = section
+    return Parsed(sections=data)

{rclone_api-1.0.88 → rclone_api-1.0.89}/src/rclone_api/group_files.py

@@ -68,7 +68,10 @@ class TreeNode:
         paths_reversed: list[str] = [self.name]
         node: TreeNode | None = self
         assert node is not None
-        while node := node.parent:
+        while True:
+            node = node.parent
+            if node is None:
+                break
             paths_reversed.append(node.name)
         return "/".join(reversed(paths_reversed))
 

{rclone_api-1.0.88 → rclone_api-1.0.89}/src/rclone_api/rclone.py

@@ -6,8 +6,10 @@ import os
 import random
 import subprocess
 import time
+import traceback
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import contextmanager
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -15,7 +17,7 @@ from typing import Generator
 
 from rclone_api import Dir
 from rclone_api.completed_process import CompletedProcess
-from rclone_api.config import Config
+from rclone_api.config import Config, Parsed, Section
 from rclone_api.convert import convert_to_filestr_list, convert_to_str
 from rclone_api.deprecated import deprecated
 from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_process
@@ -26,12 +28,13 @@ from rclone_api.group_files import group_files
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.types import (
-    ListingOption,
-    ModTimeStrategy,
-    Order,
-    SizeResult,
+from rclone_api.s3.types import (
+    MultiUploadResult,
+    S3MutliPartUploadConfig,
+    S3Provider,
+    S3UploadTarget,
 )
+from rclone_api.types import ListingOption, ModTimeStrategy, Order, SizeResult
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -41,6 +44,8 @@ from rclone_api.util import (
 )
 from rclone_api.walk import walk
 
+_IS_WINDOWS = os.name == "nt"
+
 
 def rclone_verbose(verbose: bool | None) -> bool:
     if verbose is not None:
@@ -48,6 +53,14 @@ def rclone_verbose(verbose: bool | None) -> bool:
     return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
 
 
+def _to_rclone_conf(config: Config | Path) -> Config:
+    if isinstance(config, Path):
+        content = config.read_text(encoding="utf-8")
+        return Config(content)
+    else:
+        return config
+
+
 class Rclone:
     def __init__(
         self, rclone_conf: Path | Config, rclone_exe: Path | None = None
@@ -56,6 +69,7 @@ class Rclone:
             if not rclone_conf.exists():
                 raise ValueError(f"Rclone config file not found: {rclone_conf}")
         self._exec = RcloneExec(rclone_conf, get_rclone_exe(rclone_exe))
+        self.config: Config = _to_rclone_conf(rclone_conf)
 
     def _run(
         self, cmd: list[str], check: bool = False, capture: bool | None = None
@@ -399,7 +413,7 @@ class Rclone:
         using_fast_list = "--fast-list" in other_args
         if using_fast_list:
             warnings.warn(
-                "It's not recommended to use --fast-list with copy_files as the entire repository has to be listed"
+                "It's not recommended to use --fast-list with copy_files as this will perform poorly on large repositories since the entire repository has to be scanned."
             )
 
         if max_partition_workers > 1:
@@ -654,6 +668,114 @@ class Rclone:
         except subprocess.CalledProcessError:
             return False
 
+    def copy_file_resumable_s3(
+        self,
+        src: str,
+        dst: str,
+        save_state_json: Path,
+        chunk_size: int = 16
+        * 1024
+        * 1024,  # This setting will scale the performance of the upload
+        concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
+        retries: int = 3,
+        max_chunks_before_suspension: int | None = None,
+    ) -> MultiUploadResult:
+        """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        other_args: list[str] = [
+            "--no-modtime",
+            "--vfs-read-wait",
+            "1s",
+            "--vfs-disk-space-total-size",
+            str(2 * chunk_size * concurrent_chunks),  # purge quickly.
+            "--vfs-read-chunk-size",
+            str(chunk_size),
+            "--vfs-read-chunk-size-limit",
+            str(chunk_size * concurrent_chunks),
+            "--vfs-read-chunk-streams",
+            str(concurrent_chunks),
+            "--vfs-fast-fingerprint",
+        ]
+        mount_path = Path("rclone_api_upload_mount")
+        src_path = Path(src)
+        name = src_path.name
+
+        parent_path = str(src_path.parent.as_posix())
+        with self.scoped_mount(
+            parent_path,
+            mount_path,
+            use_links=True,
+            vfs_cache_mode="minimal",
+            other_args=other_args,
+        ):
+            # raise NotImplementedError("Not implemented yet")
+            from rclone_api.s3.create import S3Credentials
+            from rclone_api.util import S3PathInfo, split_s3_path
+
+            path_info: S3PathInfo = split_s3_path(dst)
+            remote = path_info.remote
+            bucket_name = path_info.bucket
+            s3_key = path_info.key
+            parsed: Parsed = self.config.parse()
+            sections: dict[str, Section] = parsed.sections
+            if remote not in sections:
+                raise ValueError(
+                    f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
+                )
+
+            section: Section = sections[remote]
+            provider: str = section.provider()
+            provider_enum = S3Provider.from_str(provider)
+
+            s3_creds: S3Credentials = S3Credentials(
+                provider=provider_enum,
+                access_key_id=section.access_key_id(),
+                secret_access_key=section.secret_access_key(),
+                endpoint_url=section.endpoint(),
+            )
+            print(s3_creds)
+            # create_s3_client
+
+            print(f"Info: {section}")
+            from rclone_api.s3.api import S3Client
+
+            client = S3Client(s3_creds)
+            print(f"Client: {client}")
+
+            config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
+                chunk_size=chunk_size,
+                retries=retries,
+                resume_path_json=save_state_json,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+
+            src_file = mount_path / name
+
+            print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
+            print(f"Source: {src_path}")
+            print(f"bucket_name: {bucket_name}")
+            print(f"upload_config: {config}")
+
+            upload_target: S3UploadTarget
+            upload_config: S3MutliPartUploadConfig
+
+            upload_target = S3UploadTarget(
+                bucket_name=bucket_name,
+                src_file=src_file,
+                s3_key=s3_key,
+            )
+
+            upload_config = S3MutliPartUploadConfig(
+                chunk_size=chunk_size,
+                retries=retries,
+                resume_path_json=save_state_json,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+
+            out: MultiUploadResult = client.upload_file_multipart(
+                upload_target=upload_target, upload_config=upload_config
+            )
+            return out
+
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
     ) -> CompletedProcess:
@@ -682,10 +804,10 @@ class Rclone:
         self,
         src: Remote | Dir | str,
         outdir: Path,
-        allow_writes=False,
-        use_links=True,
-        vfs_cache_mode="full",
-        other_cmds: list[str] | None = None,
+        allow_writes: bool | None = False,
+        use_links: bool | None = None,
+        vfs_cache_mode: str | None = None,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -699,6 +821,9 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        allow_writes = allow_writes or False
+        use_links = use_links or True
+        vfs_cache_mode = vfs_cache_mode or "full"
         if outdir.exists():
             is_empty = not list(outdir.iterdir())
             if not is_empty:
@@ -706,12 +831,13 @@ class Rclone:
                     f"Mount directory already exists and is not empty: {outdir}"
                 )
             outdir.rmdir()
-        try:
+
+        if _IS_WINDOWS:
+            # Windows -> Must create parent directories only if they don't exist
             outdir.parent.mkdir(parents=True, exist_ok=True)
-        except PermissionError:
-            warnings.warn(
-                f"Permission error creating parent directory: {outdir.parent}"
-            )
+        else:
+            # Linux -> Must create parent directories and the directory itself
+            outdir.mkdir(parents=True, exist_ok=True)
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:
@@ -721,19 +847,50 @@ class Rclone:
         if vfs_cache_mode:
             cmd_list.append("--vfs-cache-mode")
             cmd_list.append(vfs_cache_mode)
-        if other_cmds:
-            cmd_list += other_cmds
+        if other_args:
+            cmd_list += other_args
         proc = self._launch_process(cmd_list)
         wait_for_mount(outdir, proc)
         return proc
 
+    @contextmanager
+    def scoped_mount(
+        self,
+        src: Remote | Dir | str,
+        outdir: Path,
+        allow_writes: bool | None = None,
+        use_links: bool | None = None,
+        vfs_cache_mode: str | None = None,
+        other_args: list[str] | None = None,
+    ) -> Generator[Process, None, None]:
+        """Like mount, but can be used in a context manager."""
+        proc = self.mount(
+            src,
+            outdir,
+            allow_writes=allow_writes,
+            use_links=use_links,
+            vfs_cache_mode=vfs_cache_mode,
+            other_args=other_args,
+        )
+        try:
+            yield proc
+        except Exception as e:
+            stack_trace = traceback.format_exc()
+            warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
+            raise
+        finally:
+            if proc.poll() is None:
+                proc.terminate()
+            proc.wait()
+
+    @deprecated("mount")
     def mount_webdav(
         self,
         url: str,
         outdir: Path,
-        vfs_cache_mode="full",
+        vfs_cache_mode: str | None = None,
         vfs_disk_space_total_size: str | None = "10G",
-        other_cmds: list[str] | None = None,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -747,6 +904,20 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        other_args = other_args or []
+        if vfs_cache_mode is None:
+            if "--vfs-cache-mode" in other_args:
+                pass
+            else:
+                vfs_cache_mode = "full"
+        elif "--vfs-cache-mode" in other_args:
+            warnings.warn(
+                f"vfs_cache_mode is set to {vfs_cache_mode} but --vfs-cache-mode is already in other_args"
+            )
+            idx = other_args.index("--vfs-cache-mode")
+            other_args.pop(idx)
+            other_args.pop(idx)  # also the next value which will be the cache mode.
+
         if outdir.exists():
             is_empty = not list(outdir.iterdir())
             if not is_empty:
@@ -757,10 +928,11 @@ class Rclone:
 
         src_str = url
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
-        cmd_list.append("--vfs-cache-mode")
-        cmd_list.append(vfs_cache_mode)
-        if other_cmds:
-            cmd_list += other_cmds
+        if vfs_cache_mode:
+            cmd_list.append("--vfs-cache-mode")
+            cmd_list.append(vfs_cache_mode)
+        if other_args:
+            cmd_list += other_args
         if vfs_disk_space_total_size is not None:
             cmd_list.append("--vfs-cache-max-size")
             cmd_list.append(vfs_disk_space_total_size)
@@ -768,6 +940,7 @@ class Rclone:
         wait_for_mount(outdir, proc)
         return proc
 
+    # Settings optimized for s3.
     def mount_s3(
         self,
         url: str,
@@ -789,7 +962,7 @@ class Rclone:
         vfs_fast_fingerprint: bool = True,
         # vfs-refresh
         vfs_refresh: bool = True,
-        other_cmds: list[str] | None = None,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Mount a remote or directory to a local path.
 
@@ -797,44 +970,44 @@ class Rclone:
             src: Remote or directory to mount
             outdir: Local path to mount to
         """
-        other_cmds = other_cmds or []
+        other_args = other_args or []
         if modtime_strategy is not None:
-            other_cmds.append(f"--{modtime_strategy.value}")
+            other_args.append(f"--{modtime_strategy.value}")
         if (vfs_cache_mode == "full" or vfs_cache_mode == "writes") and (
-            transfers is not None and "--transfers" not in other_cmds
+            transfers is not None and "--transfers" not in other_args
         ):
-            other_cmds.append("--transfers")
-            other_cmds.append(str(transfers))
-        if dir_cache_time is not None and "--dir-cache-time" not in other_cmds:
-            other_cmds.append("--dir-cache-time")
-            other_cmds.append(dir_cache_time)
+            other_args.append("--transfers")
+            other_args.append(str(transfers))
+        if dir_cache_time is not None and "--dir-cache-time" not in other_args:
+            other_args.append("--dir-cache-time")
+            other_args.append(dir_cache_time)
         if (
             vfs_disk_space_total_size is not None
-            and "--vfs-cache-max-size" not in other_cmds
+            and "--vfs-cache-max-size" not in other_args
         ):
-            other_cmds.append("--vfs-cache-max-size")
-            other_cmds.append(vfs_disk_space_total_size)
-        if vfs_refresh and "--vfs-refresh" not in other_cmds:
-            other_cmds.append("--vfs-refresh")
-        if attribute_timeout is not None and "--attr-timeout" not in other_cmds:
-            other_cmds.append("--attr-timeout")
-            other_cmds.append(attribute_timeout)
+            other_args.append("--vfs-cache-max-size")
+            other_args.append(vfs_disk_space_total_size)
+        if vfs_refresh and "--vfs-refresh" not in other_args:
+            other_args.append("--vfs-refresh")
+        if attribute_timeout is not None and "--attr-timeout" not in other_args:
+            other_args.append("--attr-timeout")
+            other_args.append(attribute_timeout)
         if vfs_read_chunk_streams:
-            other_cmds.append("--vfs-read-chunk-streams")
-            other_cmds.append(str(vfs_read_chunk_streams))
+            other_args.append("--vfs-read-chunk-streams")
+            other_args.append(str(vfs_read_chunk_streams))
         if vfs_read_chunk_size:
-            other_cmds.append("--vfs-read-chunk-size")
-            other_cmds.append(vfs_read_chunk_size)
+            other_args.append("--vfs-read-chunk-size")
+            other_args.append(vfs_read_chunk_size)
         if vfs_fast_fingerprint:
-            other_cmds.append("--vfs-fast-fingerprint")
+            other_args.append("--vfs-fast-fingerprint")
 
-        other_cmds = other_cmds if other_cmds else None
+        other_args = other_args if other_args else None
         return self.mount(
             url,
             outdir,
             allow_writes=allow_writes,
             vfs_cache_mode=vfs_cache_mode,
-            other_cmds=other_cmds,
+            other_args=other_args,
         )
 
     def serve_webdav(
@@ -844,6 +1017,7 @@ class Rclone:
         password: str,
         addr: str = "localhost:2049",
         allow_other: bool = False,
+        other_args: list[str] | None = None,
     ) -> Process:
         """Serve a remote or directory via NFS.
 
@@ -863,6 +1037,8 @@ class Rclone:
         cmd_list.extend(["--user", user, "--pass", password])
         if allow_other:
             cmd_list.append("--allow-other")
+        if other_args:
+            cmd_list += other_args
         proc = self._launch_process(cmd_list)
         time.sleep(2)  # give it a moment to start
         if proc.poll() is not None:
@@ -883,7 +1059,7 @@ class Rclone:
         check = get_check(check)
         if fast_list or (other_args and "--fast-list" in other_args):
             warnings.warn(
-                "It's not recommended to use --fast-list with size_files as the entire repository has to be listed"
+                "It's not recommended to use --fast-list with size_files as this will perform poorly on large repositories since the entire repository has to be scanned."
             )
         files = list(files)
         all_files: list[File] = []

rclone_api-1.0.89/src/rclone_api/s3/api.py

@@ -0,0 +1,72 @@
+import warnings
+
+from botocore.client import BaseClient
+
+from rclone_api.s3.basic_ops import (
+    download_file,
+    head,
+    list_bucket_contents,
+    upload_file,
+)
+from rclone_api.s3.chunk_uploader import MultiUploadResult, upload_file_multipart
+from rclone_api.s3.create import create_s3_client
+from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
+
+_MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
+
+
+class S3Client:
+    def __init__(self, credentials: S3Credentials):
+        self.credentials: S3Credentials = credentials
+        self.client: BaseClient = create_s3_client(credentials)
+
+    def list_bucket_contents(self, bucket_name: str) -> None:
+        list_bucket_contents(self.client, bucket_name)
+
+    def upload_file(self, target: S3UploadTarget) -> Exception | None:
+        bucket_name = target.bucket_name
+        file_path = target.src_file
+        object_name = target.s3_key
+        return upload_file(
+            s3_client=self.client,
+            bucket_name=bucket_name,
+            file_path=file_path,
+            object_name=object_name,
+        )
+
+    def download_file(self, bucket_name: str, object_name: str, file_path: str) -> None:
+        download_file(self.client, bucket_name, object_name, file_path)
+
+    def head(self, bucket_name: str, object_name: str) -> dict | None:
+        return head(self.client, bucket_name, object_name)
+
+    def upload_file_multipart(
+        self,
+        upload_target: S3UploadTarget,
+        upload_config: S3MutliPartUploadConfig,
+    ) -> MultiUploadResult:
+        filesize = upload_target.src_file.stat().st_size
+        if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
+            warnings.warn(
+                f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
+            )
+            err = self.upload_file(upload_target)
+            if err:
+                raise err
+            return MultiUploadResult.UPLOADED_FRESH
+        chunk_size = upload_config.chunk_size
+        retries = upload_config.retries
+        resume_path_json = upload_config.resume_path_json
+        max_chunks_before_suspension = upload_config.max_chunks_before_suspension
+        bucket_name = upload_target.bucket_name
+        out = upload_file_multipart(
+            s3_client=self.client,
+            bucket_name=bucket_name,
+            file_path=upload_target.src_file,
+            object_name=upload_target.s3_key,
+            resumable_info_path=resume_path_json,
+            chunk_size=chunk_size,
+            retries=retries,
+            max_chunks_before_suspension=max_chunks_before_suspension,
+        )
+        return out