rclone-api 1.1.4__tar.gz → 1.1.6__tar.gz

{rclone_api-1.1.4 → rclone_api-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.4
+Version: 1.1.6
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.4 → rclone_api-1.1.6}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.4"
+version = "1.1.6"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.4 → rclone_api-1.1.6}/src/rclone_api/__init__.py

@@ -10,7 +10,7 @@ from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
 from .s3.types import MultiUploadResult
-from .types import ListingOption, Order, SizeResult
+from .types import ListingOption, Order, SizeResult, SizeSuffix
 
 __all__ = [
     "Rclone",
@@ -34,4 +34,5 @@ __all__ = [
     "Parsed",
     "Section",
     "MultiUploadResult",
+    "SizeSuffix",
 ]

{rclone_api-1.1.4 → rclone_api-1.1.6}/src/rclone_api/cmd/copy_large_s3.py

@@ -2,7 +2,7 @@ import argparse
 from dataclasses import dataclass
 from pathlib import Path
 
-from rclone_api import MultiUploadResult, Rclone
+from rclone_api import MultiUploadResult, Rclone, SizeSuffix
 
 _1MB = 1024 * 1024
 
@@ -12,7 +12,7 @@ class Args:
     config_path: Path
     src: str
     dst: str
-    chunk_size_mb: int
+    chunk_size_mb: SizeSuffix
     read_concurrent_chunks: int
     retries: int
     save_state_json: Path
@@ -35,17 +35,20 @@ def _parse_args() -> Args:
         "--config", help="Path to rclone config file", type=Path, required=True
     )
     parser.add_argument(
-        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
+        type=str,
+        default="128M",
     )
     parser.add_argument(
         "--read-concurrent-chunks",
-        help="Maximum number of chunks to read",
+        help="Maximum number of chunks to read in a look ahead cache",
         type=int,
-        default=4,
+        default=1,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
-        "--resumable-json",
+        "--resume-json",
         help="Path to resumable JSON file",
         type=Path,
         default="resume.json",
@@ -56,7 +59,7 @@ def _parse_args() -> Args:
         config_path=Path(args.config),
         src=args.src,
         dst=args.dst,
-        chunk_size_mb=args.chunk_size_mb,
+        chunk_size_mb=SizeSuffix(args.chunk_size),
         read_concurrent_chunks=args.read_concurrent_chunks,
         retries=args.retries,
         save_state_json=args.resumable_json,

rclone_api-1.1.6/src/rclone_api/mount.py

@@ -0,0 +1,115 @@
+import platform
+import subprocess
+import time
+import warnings
+from pathlib import Path
+
+_SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
+
+
+def run_command(cmd: str, verbose: bool) -> int:
+    """Run a shell command and print its output if verbose is True."""
+    if verbose:
+        print(f"Executing: {cmd}")
+    try:
+        result = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, check=False
+        )
+        if result.returncode != 0 and verbose:
+            print(f"Command failed: {cmd}\nStdErr: {result.stderr.strip()}")
+        return result.returncode
+    except Exception as e:
+        warnings.warn(f"Error running command '{cmd}': {e}")
+        return -1
+
+
+def prepare_mount(outdir: Path, verbose: bool) -> None:
+    if _SYSTEM == "Windows":
+        # Windows -> Must create parent directories only if they don't exist
+        if verbose:
+            print(f"Creating parent directories for {outdir}")
+        outdir.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # Linux -> Must create parent directories and the directory itself
+        if verbose:
+            print(f"Creating directories for {outdir}")
+        outdir.mkdir(parents=True, exist_ok=True)
+
+
+def clean_mount(mount_path: Path, verbose: bool = False) -> None:
+    """
+    Clean up a mount path across Linux, macOS, and Windows.
+
+    The function attempts to unmount the mount at mount_path, then, if the
+    directory is empty, removes it. On Linux it uses 'fusermount -u' (for FUSE mounts)
+    and 'umount'. On macOS it uses 'umount' (and optionally 'diskutil unmount'),
+    while on Windows it attempts to remove the mount point via 'mountvol /D'.
+    """
+    # Check if the mount path exists; if an OSError occurs, assume it exists.
+    try:
+        mount_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error checking {mount_path}: {e}")
+        mount_exists = True
+
+    # Give the system a moment (if unmount is in progress, etc.)
+    time.sleep(2)
+
+    if not mount_exists:
+        if verbose:
+            print(f"{mount_path} does not exist; nothing to clean up.")
+        return
+
+    if verbose:
+        print(f"{mount_path} still exists, attempting to unmount and remove.")
+
+    # Platform-specific unmount procedures
+    if _SYSTEM == "Linux":
+        # Try FUSE unmount first (if applicable), then the regular umount.
+        run_command(f"fusermount -u {mount_path}", verbose)
+        run_command(f"umount {mount_path}", verbose)
+    elif _SYSTEM == "Darwin":
+        # On macOS, use umount; optionally try diskutil for stubborn mounts.
+        run_command(f"umount {mount_path}", verbose)
+        # Optionally: uncomment the next line if diskutil unmount is preferred.
+        # run_command(f"diskutil unmount {mount_path}", verbose)
+    elif _SYSTEM == "Windows":
+        # On Windows, remove the mount point using mountvol.
+        run_command(f"mountvol {mount_path} /D", verbose)
+        # If that does not work, try to remove the directory directly.
+        try:
+            mount_path.rmdir()
+            if verbose:
+                print(f"Successfully removed mount directory {mount_path}")
+        except Exception as e:
+            warnings.warn(f"Failed to remove mount {mount_path}: {e}")
+    else:
+        warnings.warn(f"Unsupported platform: {_SYSTEM}")
+
+    # Allow some time for the unmount commands to take effect.
+    time.sleep(2)
+
+    # Re-check if the mount path still exists.
+    try:
+        still_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error re-checking {mount_path}: {e}")
+        still_exists = True
+
+    if still_exists:
+        if verbose:
+            print(f"{mount_path} still exists after unmount attempt.")
+        # Attempt to remove the directory if it is empty.
+        try:
+            # Only remove if the directory is empty.
+            if not any(mount_path.iterdir()):
+                mount_path.rmdir()
+                if verbose:
+                    print(f"Removed empty mount directory {mount_path}")
+            else:
+                warnings.warn(f"{mount_path} is not empty; cannot remove.")
+        except Exception as e:
+            warnings.warn(f"Failed during cleanup of {mount_path}: {e}")
+    else:
+        if verbose:
+            print(f"{mount_path} successfully cleaned up.")

{rclone_api-1.1.4 → rclone_api-1.1.6}/src/rclone_api/rclone.py

@@ -34,7 +34,13 @@ from rclone_api.s3.types import (
     S3Provider,
     S3UploadTarget,
 )
-from rclone_api.types import ListingOption, ModTimeStrategy, Order, SizeResult
+from rclone_api.types import (
+    ListingOption,
+    ModTimeStrategy,
+    Order,
+    SizeResult,
+    SizeSuffix,
+)
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -673,10 +679,13 @@ class Rclone:
         src: str,
         dst: str,
         save_state_json: Path,
-        chunk_size: int = 16
-        * 1024
-        * 1024,  # This setting will scale the performance of the upload
-        concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
+        chunk_size: SizeSuffix | None = None,
+        # 16
+        # * 1024
+        # * 1024,  # This setting will scale the performance of the upload
+        concurrent_chunks: (
+            int | None
+        ) = None,  # This setting will scale the performance of the upload
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -687,16 +696,22 @@ class Rclone:
         from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, random_str, split_s3_path
 
+        _tmp: SizeSuffix | str = chunk_size or "16MiB"
+        chunk_size = SizeSuffix(_tmp)
+        assert chunk_size is not None
+        concurrent_chunks = concurrent_chunks or 4
+        size_limit = SizeSuffix(chunk_size * concurrent_chunks)
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
             "1s",
             "--vfs-disk-space-total-size",
-            str(2 * chunk_size * concurrent_chunks),  # purge quickly.
+            size_limit.as_str(),  # purge quickly.
             "--vfs-read-chunk-size",
-            str(chunk_size),
+            chunk_size.as_str(),
             "--vfs-read-chunk-size-limit",
-            str(chunk_size * concurrent_chunks),
+            size_limit.as_str(),
             "--vfs-read-chunk-streams",
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
@@ -765,7 +780,7 @@ class Rclone:
 
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -788,7 +803,7 @@ class Rclone:
             )
 
             upload_config = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -845,24 +860,14 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        from rclone_api.mount import clean_mount, prepare_mount
+
         allow_writes = allow_writes or False
         use_links = use_links or True
         verbose = get_verbose(verbose)
         vfs_cache_mode = vfs_cache_mode or "full"
-        if outdir.exists():
-            is_empty = not list(outdir.iterdir())
-            if not is_empty:
-                raise ValueError(
-                    f"Mount directory already exists and is not empty: {outdir}"
-                )
-            outdir.rmdir()
-
-        if _IS_WINDOWS:
-            # Windows -> Must create parent directories only if they don't exist
-            outdir.parent.mkdir(parents=True, exist_ok=True)
-        else:
-            # Linux -> Must create parent directories and the directory itself
-            outdir.mkdir(parents=True, exist_ok=True)
+        clean_mount(outdir, verbose=verbose)
+        prepare_mount(outdir, verbose=verbose)
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:
@@ -914,38 +919,10 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
-            if not error_happened and outdir.exists():
-                time.sleep(2)
-                if outdir.exists():
-                    print(f"{outdir} mount still exists, attempting to remove")
-                    if not _IS_WINDOWS:
-
-                        def exec(cmd: str) -> int:
-                            if verbose:
-                                print(f"Executing: {cmd}")
-                            rtn = os.system(cmd)
-                            if rtn != 0 and verbose:
-                                print(f"Failed to execute: {cmd}")
-                            return rtn
-
-                        exec(f"fusermount -u {outdir}")
-                        exec(f"umount {outdir}")
-                        time.sleep(2)
-                        if outdir.exists():
-                            is_empty = True
-                            try:
-                                is_empty = not list(outdir.iterdir())
-                                if not is_empty:
-                                    warnings.warn(f"Failed to unmount {outdir}")
-                                else:
-                                    try:
-                                        outdir.rmdir()
-                                    except Exception as e:
-                                        warnings.warn(f"Failed to remove {outdir}: {e}")
-                            except Exception as e:
-                                warnings.warn(
-                                    f"Failed during mount cleanup of {outdir}: because {e}"
-                                )
+            if not error_happened:
+                from rclone_api.mount import clean_mount
+
+                clean_mount(outdir, verbose=verbose)
 
     @deprecated("mount")
     def mount_webdav(

rclone_api-1.1.6/src/rclone_api/s3/chunk_file.py

@@ -0,0 +1,102 @@
+import time
+import warnings
+from pathlib import Path
+from queue import Queue
+
+from rclone_api.s3.chunk_types import FileChunk, UploadState
+from rclone_api.util import locked_print
+
+
+def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+    sleep_time = timeout / 60 if timeout > 0 else 1
+    start = time.time()
+    while True:
+        expired = time.time() - start > timeout
+        try:
+            time.sleep(sleep_time)
+            if file_path.exists():
+                return file_path.stat().st_size
+        except FileNotFoundError as e:
+            if expired:
+                print(f"File not found: {file_path}, exception is {e}")
+                raise
+        if expired:
+            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+
+
+def file_chunker(
+    upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
+) -> None:
+    count = 0
+
+    def should_stop() -> bool:
+        nonlocal count
+        if max_chunks is None:
+            return False
+        if count >= max_chunks:
+            return True
+        count += 1
+        if count > 10 and count % 10 == 0:
+            # recheck that the file size has not changed
+            file_size = _get_file_size(upload_state.upload_info.src_file_path)
+            if file_size != upload_state.upload_info.file_size:
+                locked_print(
+                    f"File size changed, cannot resume, expected {upload_state.upload_info.file_size}, got {file_size}"
+                )
+                raise ValueError("File size changed, cannot resume")
+        return False
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+    src = Path(file_path)
+    # Mounted files may take a while to appear, so keep retrying.
+
+    try:
+        file_size = _get_file_size(src, timeout=60)
+        part_number = 1
+        done_part_numbers: set[int] = {
+            p.part_number for p in upload_state.parts if p is not None
+        }
+        num_parts = upload_info.total_chunks()
+
+        def next_part_number() -> int | None:
+            nonlocal part_number
+            while part_number in done_part_numbers:
+                part_number += 1
+            if part_number > num_parts:
+                return None
+            return part_number
+
+        while not should_stop():
+            curr_parth_num = next_part_number()
+            if curr_parth_num is None:
+                locked_print(f"File {file_path} has completed chunking all parts")
+                break
+            assert curr_parth_num is not None
+            offset = (curr_parth_num - 1) * chunk_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+
+            # Open the file, seek, read the chunk, and close immediately.
+            with open(file_path, "rb") as f:
+                f.seek(offset)
+                data = f.read(chunk_size)
+
+            if not data:
+                warnings.warn(f"Empty data for part {part_number} of {file_path}")
+
+            file_chunk = FileChunk(
+                src,
+                upload_id=upload_info.upload_id,
+                part_number=part_number,
+                data=data,  # After this, data should not be reused.
+            )
+            done_part_numbers.add(part_number)
+            output.put(file_chunk)
+            part_number += 1
+    except Exception as e:
+
+        warnings.warn(f"Error reading file: {e}")
+    finally:
+        output.put(None)