rclone-api 1.1.4__py2.py3-none-any.whl → 1.1.6__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -10,7 +10,7 @@ from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
 from .s3.types import MultiUploadResult
-from .types import ListingOption, Order, SizeResult
+from .types import ListingOption, Order, SizeResult, SizeSuffix
 
 __all__ = [
     "Rclone",
@@ -34,4 +34,5 @@ __all__ = [
     "Parsed",
     "Section",
     "MultiUploadResult",
+    "SizeSuffix",
 ]

rclone_api/cmd/copy_large_s3.py

@@ -2,7 +2,7 @@ import argparse
 from dataclasses import dataclass
 from pathlib import Path
 
-from rclone_api import MultiUploadResult, Rclone
+from rclone_api import MultiUploadResult, Rclone, SizeSuffix
 
 _1MB = 1024 * 1024
 
@@ -12,7 +12,7 @@ class Args:
     config_path: Path
     src: str
     dst: str
-    chunk_size_mb: int
+    chunk_size_mb: SizeSuffix
     read_concurrent_chunks: int
     retries: int
     save_state_json: Path
@@ -35,17 +35,20 @@ def _parse_args() -> Args:
         "--config", help="Path to rclone config file", type=Path, required=True
     )
     parser.add_argument(
-        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
+        type=str,
+        default="128M",
     )
     parser.add_argument(
         "--read-concurrent-chunks",
-        help="Maximum number of chunks to read",
+        help="Maximum number of chunks to read in a look ahead cache",
         type=int,
-        default=4,
+        default=1,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
-        "--resumable-json",
+        "--resume-json",
         help="Path to resumable JSON file",
         type=Path,
         default="resume.json",
@@ -56,7 +59,7 @@ def _parse_args() -> Args:
         config_path=Path(args.config),
         src=args.src,
         dst=args.dst,
-        chunk_size_mb=args.chunk_size_mb,
+        chunk_size_mb=SizeSuffix(args.chunk_size),
         read_concurrent_chunks=args.read_concurrent_chunks,
         retries=args.retries,
         save_state_json=args.resumable_json,

rclone_api/mount.py

@@ -0,0 +1,115 @@
+import platform
+import subprocess
+import time
+import warnings
+from pathlib import Path
+
+_SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
+
+
+def run_command(cmd: str, verbose: bool) -> int:
+    """Run a shell command and print its output if verbose is True."""
+    if verbose:
+        print(f"Executing: {cmd}")
+    try:
+        result = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, check=False
+        )
+        if result.returncode != 0 and verbose:
+            print(f"Command failed: {cmd}\nStdErr: {result.stderr.strip()}")
+        return result.returncode
+    except Exception as e:
+        warnings.warn(f"Error running command '{cmd}': {e}")
+        return -1
+
+
+def prepare_mount(outdir: Path, verbose: bool) -> None:
+    if _SYSTEM == "Windows":
+        # Windows -> Must create parent directories only if they don't exist
+        if verbose:
+            print(f"Creating parent directories for {outdir}")
+        outdir.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # Linux -> Must create parent directories and the directory itself
+        if verbose:
+            print(f"Creating directories for {outdir}")
+        outdir.mkdir(parents=True, exist_ok=True)
+
+
+def clean_mount(mount_path: Path, verbose: bool = False) -> None:
+    """
+    Clean up a mount path across Linux, macOS, and Windows.
+
+    The function attempts to unmount the mount at mount_path, then, if the
+    directory is empty, removes it. On Linux it uses 'fusermount -u' (for FUSE mounts)
+    and 'umount'. On macOS it uses 'umount' (and optionally 'diskutil unmount'),
+    while on Windows it attempts to remove the mount point via 'mountvol /D'.
+    """
+    # Check if the mount path exists; if an OSError occurs, assume it exists.
+    try:
+        mount_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error checking {mount_path}: {e}")
+        mount_exists = True
+
+    # Give the system a moment (if unmount is in progress, etc.)
+    time.sleep(2)
+
+    if not mount_exists:
+        if verbose:
+            print(f"{mount_path} does not exist; nothing to clean up.")
+        return
+
+    if verbose:
+        print(f"{mount_path} still exists, attempting to unmount and remove.")
+
+    # Platform-specific unmount procedures
+    if _SYSTEM == "Linux":
+        # Try FUSE unmount first (if applicable), then the regular umount.
+        run_command(f"fusermount -u {mount_path}", verbose)
+        run_command(f"umount {mount_path}", verbose)
+    elif _SYSTEM == "Darwin":
+        # On macOS, use umount; optionally try diskutil for stubborn mounts.
+        run_command(f"umount {mount_path}", verbose)
+        # Optionally: uncomment the next line if diskutil unmount is preferred.
+        # run_command(f"diskutil unmount {mount_path}", verbose)
+    elif _SYSTEM == "Windows":
+        # On Windows, remove the mount point using mountvol.
+        run_command(f"mountvol {mount_path} /D", verbose)
+        # If that does not work, try to remove the directory directly.
+        try:
+            mount_path.rmdir()
+            if verbose:
+                print(f"Successfully removed mount directory {mount_path}")
+        except Exception as e:
+            warnings.warn(f"Failed to remove mount {mount_path}: {e}")
+    else:
+        warnings.warn(f"Unsupported platform: {_SYSTEM}")
+
+    # Allow some time for the unmount commands to take effect.
+    time.sleep(2)
+
+    # Re-check if the mount path still exists.
+    try:
+        still_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error re-checking {mount_path}: {e}")
+        still_exists = True
+
+    if still_exists:
+        if verbose:
+            print(f"{mount_path} still exists after unmount attempt.")
+        # Attempt to remove the directory if it is empty.
+        try:
+            # Only remove if the directory is empty.
+            if not any(mount_path.iterdir()):
+                mount_path.rmdir()
+                if verbose:
+                    print(f"Removed empty mount directory {mount_path}")
+            else:
+                warnings.warn(f"{mount_path} is not empty; cannot remove.")
+        except Exception as e:
+            warnings.warn(f"Failed during cleanup of {mount_path}: {e}")
+    else:
+        if verbose:
+            print(f"{mount_path} successfully cleaned up.")

rclone_api/rclone.py

@@ -34,7 +34,13 @@ from rclone_api.s3.types import (
     S3Provider,
     S3UploadTarget,
 )
-from rclone_api.types import ListingOption, ModTimeStrategy, Order, SizeResult
+from rclone_api.types import (
+    ListingOption,
+    ModTimeStrategy,
+    Order,
+    SizeResult,
+    SizeSuffix,
+)
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -673,10 +679,13 @@ class Rclone:
         src: str,
         dst: str,
         save_state_json: Path,
-        chunk_size: int = 16
-        * 1024
-        * 1024,  # This setting will scale the performance of the upload
-        concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
+        chunk_size: SizeSuffix | None = None,
+        # 16
+        # * 1024
+        # * 1024,  # This setting will scale the performance of the upload
+        concurrent_chunks: (
+            int | None
+        ) = None,  # This setting will scale the performance of the upload
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -687,16 +696,22 @@ class Rclone:
         from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, random_str, split_s3_path
 
+        _tmp: SizeSuffix | str = chunk_size or "16MiB"
+        chunk_size = SizeSuffix(_tmp)
+        assert chunk_size is not None
+        concurrent_chunks = concurrent_chunks or 4
+        size_limit = SizeSuffix(chunk_size * concurrent_chunks)
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
             "1s",
             "--vfs-disk-space-total-size",
-            str(2 * chunk_size * concurrent_chunks),  # purge quickly.
+            size_limit.as_str(),  # purge quickly.
             "--vfs-read-chunk-size",
-            str(chunk_size),
+            chunk_size.as_str(),
             "--vfs-read-chunk-size-limit",
-            str(chunk_size * concurrent_chunks),
+            size_limit.as_str(),
             "--vfs-read-chunk-streams",
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
@@ -765,7 +780,7 @@ class Rclone:
 
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -788,7 +803,7 @@ class Rclone:
             )
 
             upload_config = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -845,24 +860,14 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        from rclone_api.mount import clean_mount, prepare_mount
+
         allow_writes = allow_writes or False
         use_links = use_links or True
         verbose = get_verbose(verbose)
         vfs_cache_mode = vfs_cache_mode or "full"
-        if outdir.exists():
-            is_empty = not list(outdir.iterdir())
-            if not is_empty:
-                raise ValueError(
-                    f"Mount directory already exists and is not empty: {outdir}"
-                )
-            outdir.rmdir()
-
-        if _IS_WINDOWS:
-            # Windows -> Must create parent directories only if they don't exist
-            outdir.parent.mkdir(parents=True, exist_ok=True)
-        else:
-            # Linux -> Must create parent directories and the directory itself
-            outdir.mkdir(parents=True, exist_ok=True)
+        clean_mount(outdir, verbose=verbose)
+        prepare_mount(outdir, verbose=verbose)
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:
@@ -914,38 +919,10 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
-            if not error_happened and outdir.exists():
-                time.sleep(2)
-                if outdir.exists():
-                    print(f"{outdir} mount still exists, attempting to remove")
-                    if not _IS_WINDOWS:
-
-                        def exec(cmd: str) -> int:
-                            if verbose:
-                                print(f"Executing: {cmd}")
-                            rtn = os.system(cmd)
-                            if rtn != 0 and verbose:
-                                print(f"Failed to execute: {cmd}")
-                            return rtn
-
-                        exec(f"fusermount -u {outdir}")
-                        exec(f"umount {outdir}")
-                        time.sleep(2)
-                        if outdir.exists():
-                            is_empty = True
-                            try:
-                                is_empty = not list(outdir.iterdir())
-                                if not is_empty:
-                                    warnings.warn(f"Failed to unmount {outdir}")
-                                else:
-                                    try:
-                                        outdir.rmdir()
-                                    except Exception as e:
-                                        warnings.warn(f"Failed to remove {outdir}: {e}")
-                            except Exception as e:
-                                warnings.warn(
-                                    f"Failed during mount cleanup of {outdir}: because {e}"
-                                )
+            if not error_happened:
+                from rclone_api.mount import clean_mount
+
+                clean_mount(outdir, verbose=verbose)
 
     @deprecated("mount")
     def mount_webdav(

rclone_api/s3/chunk_file.py

@@ -0,0 +1,102 @@
+import time
+import warnings
+from pathlib import Path
+from queue import Queue
+
+from rclone_api.s3.chunk_types import FileChunk, UploadState
+from rclone_api.util import locked_print
+
+
+def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+    sleep_time = timeout / 60 if timeout > 0 else 1
+    start = time.time()
+    while True:
+        expired = time.time() - start > timeout
+        try:
+            time.sleep(sleep_time)
+            if file_path.exists():
+                return file_path.stat().st_size
+        except FileNotFoundError as e:
+            if expired:
+                print(f"File not found: {file_path}, exception is {e}")
+                raise
+        if expired:
+            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+
+
+def file_chunker(
+    upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
+) -> None:
+    count = 0
+
+    def should_stop() -> bool:
+        nonlocal count
+        if max_chunks is None:
+            return False
+        if count >= max_chunks:
+            return True
+        count += 1
+        if count > 10 and count % 10 == 0:
+            # recheck that the file size has not changed
+            file_size = _get_file_size(upload_state.upload_info.src_file_path)
+            if file_size != upload_state.upload_info.file_size:
+                locked_print(
+                    f"File size changed, cannot resume, expected {upload_state.upload_info.file_size}, got {file_size}"
+                )
+                raise ValueError("File size changed, cannot resume")
+        return False
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+    src = Path(file_path)
+    # Mounted files may take a while to appear, so keep retrying.
+
+    try:
+        file_size = _get_file_size(src, timeout=60)
+        part_number = 1
+        done_part_numbers: set[int] = {
+            p.part_number for p in upload_state.parts if p is not None
+        }
+        num_parts = upload_info.total_chunks()
+
+        def next_part_number() -> int | None:
+            nonlocal part_number
+            while part_number in done_part_numbers:
+                part_number += 1
+            if part_number > num_parts:
+                return None
+            return part_number
+
+        while not should_stop():
+            curr_parth_num = next_part_number()
+            if curr_parth_num is None:
+                locked_print(f"File {file_path} has completed chunking all parts")
+                break
+            assert curr_parth_num is not None
+            offset = (curr_parth_num - 1) * chunk_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+
+            # Open the file, seek, read the chunk, and close immediately.
+            with open(file_path, "rb") as f:
+                f.seek(offset)
+                data = f.read(chunk_size)
+
+            if not data:
+                warnings.warn(f"Empty data for part {part_number} of {file_path}")
+
+            file_chunk = FileChunk(
+                src,
+                upload_id=upload_info.upload_id,
+                part_number=part_number,
+                data=data,  # After this, data should not be reused.
+            )
+            done_part_numbers.add(part_number)
+            output.put(file_chunk)
+            part_number += 1
+    except Exception as e:
+
+        warnings.warn(f"Error reading file: {e}")
+    finally:
+        output.put(None)

rclone_api/s3/chunk_types.py

@@ -0,0 +1,254 @@
+import json
+import os
+import time
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from threading import Lock
+
+from botocore.client import BaseClient
+
+from rclone_api.util import locked_print
+
+_MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
+_SAVE_STATE_LOCK = Lock()
+
+_TMP_DIR_ACCESS_LOCK = Lock()
+
+
+def _clean_old_files(out: Path) -> None:
+    # clean up files older than 1 day
+
+    now = time.time()
+    # Erase all stale files and then purge empty directories.
+    for root, dirs, files in os.walk(out):
+        for name in files:
+            f = Path(root) / name
+            filemod = f.stat().st_mtime
+            diff_secs = now - filemod
+            diff_days = diff_secs / (60 * 60 * 24)
+            if diff_days > 1:
+                locked_print(f"Removing old file: {f}")
+                f.unlink()
+
+    for root, dirs, _ in os.walk(out):
+        for dir in dirs:
+            d = Path(root) / dir
+            if not list(d.iterdir()):
+                locked_print(f"Removing empty directory: {d}")
+                d.rmdir()
+
+
+def _get_chunk_tmpdir() -> Path:
+    with _TMP_DIR_ACCESS_LOCK:
+        dat = _get_chunk_tmpdir.__dict__
+        if "out" in dat:
+            return dat["out"]  # Folder already validated.
+        out = Path("chunk_store")
+        if out.exists():
+            # first access, clean up directory
+            _clean_old_files(out)
+        out.mkdir(exist_ok=True, parents=True)
+        dat["out"] = out
+        return out
+
+
+class FileChunk:
+    def __init__(self, src: Path, upload_id: str, part_number: int, data: bytes):
+        assert data is not None, f"{src}: Data must not be None"
+        self.upload_id = upload_id
+        self.src = src
+        self.part_number = part_number
+        name = src.name
+        self.tmpdir = _get_chunk_tmpdir()
+        self.filepart = self.tmpdir / f"{name}_{upload_id}.part_{part_number}.tmp"
+        self.filepart.write_bytes(data)
+        del data  # free up memory
+
+    @property
+    def data(self) -> bytes:
+        assert self.filepart is not None
+        with open(self.filepart, "rb") as f:
+            return f.read()
+        return b""
+
+    def close(self):
+        if self.filepart.exists():
+            self.filepart.unlink()
+
+    def __del__(self):
+        self.close()
+
+
+@dataclass
+class UploadInfo:
+    s3_client: BaseClient
+    bucket_name: str
+    object_name: str
+    src_file_path: Path
+    upload_id: str
+    retries: int
+    chunk_size: int
+    file_size: int
+    _total_chunks: int | None = None
+
+    def total_chunks(self) -> int:
+        out = self.file_size // self.chunk_size
+        if self.file_size % self.chunk_size:
+            return out + 1
+        return out
+
+    def __post_init__(self):
+        if self._total_chunks is not None:
+            return
+        self._total_chunks = self.total_chunks()
+
+    def to_json(self) -> dict:
+        json_dict = {}
+        for f in fields(self):
+            value = getattr(self, f.name)
+            # Convert non-serializable objects (like s3_client) to a string representation.
+            if f.name == "s3_client":
+                json_dict[f.name] = "RUNTIME OBJECT"
+            else:
+                if isinstance(value, Path):
+                    value = str(value)
+                json_dict[f.name] = value
+        return json_dict
+
+    @staticmethod
+    def from_json(s3_client: BaseClient, json_dict: dict) -> "UploadInfo":
+        json_dict.pop("s3_client")  # Remove the placeholder string
+        return UploadInfo(s3_client=s3_client, **json_dict)
+
+
+@dataclass
+class FinishedPiece:
+    part_number: int
+    etag: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "etag": self.etag}
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_json(), indent=0)
+
+    @staticmethod
+    def to_json_array(parts: list["FinishedPiece | None"]) -> list[dict | None]:
+        non_none: list[FinishedPiece] = [p for p in parts if p is not None]
+        non_none.sort(key=lambda x: x.part_number)
+        all_nones: list[None] = [None for p in parts if p is None]
+        assert len(all_nones) <= 1, "Only one None should be present"
+        return [p.to_json() for p in non_none]
+
+    @staticmethod
+    def from_json(json: dict | None) -> "FinishedPiece | None":
+        if json is None:
+            return None
+        return FinishedPiece(**json)
+
+
+@dataclass
+class UploadState:
+    upload_info: UploadInfo
+    # finished_parts: Queue[FinishedPiece | None]
+    peristant: Path | None
+    lock: Lock = Lock()
+    parts: list[FinishedPiece | None] = field(default_factory=list)
+
+    def update_source_file(self, src_file: Path) -> None:
+        new_file_size = os.path.getsize(src_file)
+        if new_file_size != self.upload_info.file_size:
+            raise ValueError("File size changed, cannot resume")
+        self.upload_info.src_file_path = src_file
+        self.save()
+
+    def is_done(self) -> bool:
+        return self.remaining() == 0
+
+    def count(self) -> tuple[int, int]:  # count, num_chunks
+        num_chunks = self.upload_info.total_chunks()
+        count = 0
+        for p in self.parts:
+            if p is not None:
+                count += 1
+        return count, num_chunks
+
+    def finished(self) -> int:
+        count, _ = self.count()
+        return count
+
+    def remaining(self) -> int:
+        count, num_chunks = self.count()
+        assert (
+            count <= num_chunks
+        ), f"Count {count} is greater than num_chunks {num_chunks}"
+        return num_chunks - count
+
+    def add_finished(self, part: FinishedPiece | None) -> None:
+        with self.lock:
+            self.parts.append(part)
+            self._save_no_lock()
+
+    def __post_init__(self):
+        if self.peristant is None:
+            # upload_id = self.upload_info.upload_id
+            object_name = self.upload_info.object_name
+            chunk_size = self.upload_info.chunk_size
+            parent = _get_chunk_tmpdir()
+            self.peristant = parent / f"{object_name}_chunk_size_{chunk_size}_.json"
+
+    def save(self) -> None:
+        with _SAVE_STATE_LOCK:
+            self._save_no_lock()
+
+    def _save_no_lock(self) -> None:
+        assert self.peristant is not None, "No path to save to"
+        self.peristant.write_text(self.to_json_str(), encoding="utf-8")
+
+    @staticmethod
+    def load(s3_client: BaseClient, path: Path) -> "UploadState":
+        with _SAVE_STATE_LOCK:
+            return UploadState.from_json(s3_client, path)
+
+    def to_json(self) -> dict:
+        # queue -> list
+        # parts: list[dict] = [f.to_json() for f in self.parts]
+        parts: list[FinishedPiece | None] = list(self.parts)
+
+        parts_json = FinishedPiece.to_json_array(parts)
+        is_done = self.is_done()
+        count_non_none: int = 0
+        for p in parts:
+            if p is not None:
+                count_non_none += 1
+
+        # self.count()
+        finished_count, total = self.count()
+
+        # parts.sort(key=lambda x: x.part_number)  # Some backends need this.
+        out_json = {
+            "upload_info": self.upload_info.to_json(),
+            "finished_parts": parts_json,
+            "is_done": is_done,
+            "finished_count": finished_count,
+            "total_parts": total,
+        }
+
+        # check that we can sererialize
+        # json.dumps(out_json)
+        return out_json
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_json(), indent=4)
+
+    @staticmethod
+    def from_json(s3_client: BaseClient, json_file: Path) -> "UploadState":
+        json_str = json_file.read_text(encoding="utf-8")
+        data = json.loads(json_str)
+        upload_info_json = data["upload_info"]
+        finished_parts_json = data["finished_parts"]
+        upload_info = UploadInfo.from_json(s3_client, upload_info_json)
+        finished_parts = [FinishedPiece.from_json(p) for p in finished_parts_json]
+        return UploadState(
+            peristant=json_file, upload_info=upload_info, parts=finished_parts
+        )

rclone_api/s3/chunk_uploader.py

@@ -1,359 +1,19 @@
 import _thread
-import json
 import os
-import time
 import warnings
 from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass, field, fields
 from pathlib import Path
 from queue import Queue
-from threading import Lock, Thread
+from threading import Thread
 
 from botocore.client import BaseClient
 
+from rclone_api.s3.chunk_file import file_chunker
+from rclone_api.s3.chunk_types import FileChunk, FinishedPiece, UploadInfo, UploadState
 from rclone_api.s3.types import MultiUploadResult
+from rclone_api.util import locked_print
 
 _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
-_SAVE_STATE_LOCK = Lock()
-
-_PRINT_LOCK = Lock()
-
-
-def locked_print(*args, **kwargs):
-    with _PRINT_LOCK:
-        print(*args, **kwargs)
-
-
-class FileChunk:
-    def __init__(self, src: Path, upload_id: str, part_number: int, data: bytes):
-        assert data is not None, f"{src}: Data must not be None"
-        self.upload_id = upload_id
-        self.src = src
-        self.part_number = part_number
-        name = src.name
-        self.tmpdir = _get_chunk_tmpdir()
-        self.filepart = self.tmpdir / f"{name}_{upload_id}.part_{part_number}.tmp"
-        self.filepart.write_bytes(data)
-        del data  # free up memory
-
-    @property
-    def data(self) -> bytes:
-        assert self.filepart is not None
-        with open(self.filepart, "rb") as f:
-            return f.read()
-        return b""
-
-    def close(self):
-        if self.filepart.exists():
-            self.filepart.unlink()
-
-    def __del__(self):
-        self.close()
-
-
-@dataclass
-class UploadInfo:
-    s3_client: BaseClient
-    bucket_name: str
-    object_name: str
-    src_file_path: Path
-    upload_id: str
-    retries: int
-    chunk_size: int
-    file_size: int
-    _total_chunks: int | None = None
-
-    def total_chunks(self) -> int:
-        out = self.file_size // self.chunk_size
-        if self.file_size % self.chunk_size:
-            return out + 1
-        return out
-
-    def __post_init__(self):
-        if self._total_chunks is not None:
-            return
-        self._total_chunks = self.total_chunks()
-
-    def to_json(self) -> dict:
-        json_dict = {}
-        for f in fields(self):
-            value = getattr(self, f.name)
-            # Convert non-serializable objects (like s3_client) to a string representation.
-            if f.name == "s3_client":
-                json_dict[f.name] = "RUNTIME OBJECT"
-            else:
-                if isinstance(value, Path):
-                    value = str(value)
-                json_dict[f.name] = value
-        return json_dict
-
-    @staticmethod
-    def from_json(s3_client: BaseClient, json_dict: dict) -> "UploadInfo":
-        json_dict.pop("s3_client")  # Remove the placeholder string
-        return UploadInfo(s3_client=s3_client, **json_dict)
-
-
-@dataclass
-class FinishedPiece:
-    part_number: int
-    etag: str
-
-    def to_json(self) -> dict:
-        return {"part_number": self.part_number, "etag": self.etag}
-
-    def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=0)
-
-    @staticmethod
-    def to_json_array(parts: list["FinishedPiece | None"]) -> list[dict | None]:
-        non_none: list[FinishedPiece] = [p for p in parts if p is not None]
-        non_none.sort(key=lambda x: x.part_number)
-        all_nones: list[None] = [None for p in parts if p is None]
-        assert len(all_nones) <= 1, "Only one None should be present"
-        return [p.to_json() for p in non_none]
-
-    @staticmethod
-    def from_json(json: dict | None) -> "FinishedPiece | None":
-        if json is None:
-            return None
-        return FinishedPiece(**json)
-
-
-@dataclass
-class UploadState:
-    upload_info: UploadInfo
-    # finished_parts: Queue[FinishedPiece | None]
-    peristant: Path | None
-    lock: Lock = Lock()
-    parts: list[FinishedPiece | None] = field(default_factory=list)
-
-    def update_source_file(self, src_file: Path) -> None:
-        new_file_size = os.path.getsize(src_file)
-        if new_file_size != self.upload_info.file_size:
-            raise ValueError("File size changed, cannot resume")
-        self.upload_info.src_file_path = src_file
-        self.save()
-
-    def is_done(self) -> bool:
-        return self.remaining() == 0
-
-    def count(self) -> tuple[int, int]:  # count, num_chunks
-        num_chunks = self.upload_info.total_chunks()
-        count = 0
-        for p in self.parts:
-            if p is not None:
-                count += 1
-        return count, num_chunks
-
-    def finished(self) -> int:
-        count, _ = self.count()
-        return count
-
-    def remaining(self) -> int:
-        count, num_chunks = self.count()
-        assert (
-            count <= num_chunks
-        ), f"Count {count} is greater than num_chunks {num_chunks}"
-        return num_chunks - count
-
-    def add_finished(self, part: FinishedPiece | None) -> None:
-        with self.lock:
-            self.parts.append(part)
-            self._save_no_lock()
-
-    def __post_init__(self):
-        if self.peristant is None:
-            # upload_id = self.upload_info.upload_id
-            object_name = self.upload_info.object_name
-            chunk_size = self.upload_info.chunk_size
-            parent = _get_chunk_tmpdir()
-            self.peristant = parent / f"{object_name}_chunk_size_{chunk_size}_.json"
-
-    def save(self) -> None:
-        with _SAVE_STATE_LOCK:
-            self._save_no_lock()
-
-    def _save_no_lock(self) -> None:
-        assert self.peristant is not None, "No path to save to"
-        self.peristant.write_text(self.to_json_str(), encoding="utf-8")
-
-    @staticmethod
-    def load(s3_client: BaseClient, path: Path) -> "UploadState":
-        with _SAVE_STATE_LOCK:
-            return UploadState.from_json(s3_client, path)
-
-    def to_json(self) -> dict:
-        # queue -> list
-        # parts: list[dict] = [f.to_json() for f in self.parts]
-        parts: list[FinishedPiece | None] = list(self.parts)
-
-        parts_json = FinishedPiece.to_json_array(parts)
-        is_done = self.is_done()
-        count_non_none: int = 0
-        for p in parts:
-            if p is not None:
-                count_non_none += 1
-
-        # self.count()
-        finished_count, total = self.count()
-
-        # parts.sort(key=lambda x: x.part_number)  # Some backends need this.
-        out_json = {
-            "upload_info": self.upload_info.to_json(),
-            "finished_parts": parts_json,
-            "is_done": is_done,
-            "finished_count": finished_count,
-            "total_parts": total,
-        }
-
-        # check that we can sererialize
-        # json.dumps(out_json)
-        return out_json
-
-    def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=4)
-
-    @staticmethod
-    def from_json(s3_client: BaseClient, json_file: Path) -> "UploadState":
-        json_str = json_file.read_text(encoding="utf-8")
-        data = json.loads(json_str)
-        upload_info_json = data["upload_info"]
-        finished_parts_json = data["finished_parts"]
-        upload_info = UploadInfo.from_json(s3_client, upload_info_json)
-        finished_parts = [FinishedPiece.from_json(p) for p in finished_parts_json]
-        return UploadState(
-            peristant=json_file, upload_info=upload_info, parts=finished_parts
-        )
-
-
-# lock
-
-_TMP_DIR_ACCESS_LOCK = Lock()
-
-
-def clean_old_files(out: Path) -> None:
-    # clean up files older than 1 day
-
-    now = time.time()
-    # Erase all stale files and then purge empty directories.
-    for root, dirs, files in os.walk(out):
-        for name in files:
-            f = Path(root) / name
-            filemod = f.stat().st_mtime
-            diff_secs = now - filemod
-            diff_days = diff_secs / (60 * 60 * 24)
-            if diff_days > 1:
-                locked_print(f"Removing old file: {f}")
-                f.unlink()
-
-    for root, dirs, _ in os.walk(out):
-        for dir in dirs:
-            d = Path(root) / dir
-            if not list(d.iterdir()):
-                locked_print(f"Removing empty directory: {d}")
-                d.rmdir()
-
-
-def _get_chunk_tmpdir() -> Path:
-    with _TMP_DIR_ACCESS_LOCK:
-        dat = _get_chunk_tmpdir.__dict__
-        if "out" in dat:
-            return dat["out"]  # Folder already validated.
-        out = Path("chunk_store")
-        if out.exists():
-            # first access, clean up directory
-            clean_old_files(out)
-        out.mkdir(exist_ok=True, parents=True)
-        dat["out"] = out
-        return out
-
-
-def _get_file_size(file_path: Path, timeout: int = 60) -> int:
-    sleep_time = timeout / 60 if timeout > 0 else 1
-    start = time.time()
-    while True:
-        expired = time.time() - start > timeout
-        try:
-            time.sleep(sleep_time)
-            if file_path.exists():
-                return file_path.stat().st_size
-        except FileNotFoundError as e:
-            if expired:
-                print(f"File not found: {file_path}, exception is {e}")
-                raise
-        if expired:
-            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
-
-
-def file_chunker(
-    upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
-) -> None:
-
-    count = 0
-
-    def should_stop() -> bool:
-        nonlocal count
-        if max_chunks is None:
-            return False
-        if count >= max_chunks:
-            return True
-        count += 1
-        return False
-
-    upload_info = upload_state.upload_info
-    file_path = upload_info.src_file_path
-    chunk_size = upload_info.chunk_size
-    src = Path(file_path)
-    # Mounted files may take a while to appear, so keep retrying.
-
-    try:
-        file_size = _get_file_size(src, timeout=60)
-        part_number = 1
-        done_part_numbers: set[int] = {
-            p.part_number for p in upload_state.parts if p is not None
-        }
-        num_parts = upload_info.total_chunks()
-
-        def next_part_number() -> int | None:
-            nonlocal part_number
-            while part_number in done_part_numbers:
-                part_number += 1
-            if part_number > num_parts:
-                return None
-            return part_number
-
-        while not should_stop():
-            curr_parth_num = next_part_number()
-            if curr_parth_num is None:
-                locked_print(f"File {file_path} has completed chunking all parts")
-                break
-            assert curr_parth_num is not None
-            offset = (curr_parth_num - 1) * chunk_size
-
-            assert offset < file_size, f"Offset {offset} is greater than file size"
-
-            # Open the file, seek, read the chunk, and close immediately.
-            with open(file_path, "rb") as f:
-                f.seek(offset)
-                data = f.read(chunk_size)
-
-            if not data:
-                warnings.warn(f"Empty data for part {part_number} of {file_path}")
-
-            file_chunk = FileChunk(
-                src,
-                upload_id=upload_info.upload_id,
-                part_number=part_number,
-                data=data,  # After this, data should not be reused.
-            )
-            done_part_numbers.add(part_number)
-            output.put(file_chunk)
-            part_number += 1
-    except Exception as e:
-
-        warnings.warn(f"Error reading file: {e}")
-    finally:
-        output.put(None)
 
 
 def upload_task(

rclone_api/types.py

@@ -1,3 +1,4 @@
+import re
 from dataclasses import dataclass
 from enum import Enum
 
@@ -33,3 +34,92 @@ class SizeResult:
     prefix: str
     total_size: int
     file_sizes: dict[str, int]
+
+
+def _to_size_suffix(size: int) -> str:
+    if size < 1024:
+        return f"{size}B"
+    if size < 1024 * 1024:
+        return f"{size // 1024}K"
+    if size < 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024)}M"
+    if size < 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024)}G"
+    if size < 1024 * 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024 * 1024)}T"
+    if size < 1024 * 1024 * 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024 * 1024 * 1024)}P"
+    raise ValueError(f"Invalid size: {size}")
+
+
+_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+)([A-Za-z]+)$")
+
+
+def _from_size_suffix(size: str) -> int:
+    # 16MiB
+    # parse out number and suffix
+    match = _PATTERN_SIZE_SUFFIX.match(size)
+    if match is None:
+        raise ValueError(f"Invalid size suffix: {size}")
+    size = match.group(1)
+    suffix = match.group(2)[0:1].upper()
+    n = int(size)
+    if suffix == "B":
+        return n
+    if suffix == "K":
+        return n * 1024
+    if suffix == "M":
+        return n * 1024 * 1024
+    if suffix == "G":
+        return n * 1024 * 1024 * 1024
+    if suffix == "T":
+        return n * 1024 * 1024 * 1024 * 1024
+    if suffix == "P":
+        return n * 1024 * 1024 * 1024 * 1024 * 1024
+    raise ValueError(f"Invalid size suffix: {size}")
+
+
+class SizeSuffix:
+    def __init__(self, size: "int | str | SizeSuffix"):
+        self._size: int
+        if isinstance(size, SizeSuffix):
+            self._size = size._size
+        elif isinstance(size, int):
+            self._size = size
+        elif isinstance(size, str):
+            self._size = _from_size_suffix(size)
+        else:
+            raise ValueError(f"Invalid type for size: {type(size)}")
+
+    def as_int(self) -> int:
+        return self._size
+
+    def as_str(self) -> str:
+        return _to_size_suffix(self._size)
+
+    def __repr__(self) -> str:
+        return self.as_str()
+
+    def __str__(self) -> str:
+        return self.as_str()
+
+    @staticmethod
+    def _to_size(size: "int | SizeSuffix") -> int:
+        if isinstance(size, int):
+            return size
+        elif isinstance(size, SizeSuffix):
+            return size._size
+        else:
+            raise ValueError(f"Invalid type for size: {type(size)}")
+
+    def __mul__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size * other_int._size)
+
+    def __add__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size + other_int._size)
+
+    def __sub__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size - other_int._size)

rclone_api/util.py

@@ -5,6 +5,7 @@ import time
 import warnings
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from threading import Lock
 from typing import Any
 
 from rclone_api.config import Config
@@ -15,6 +16,13 @@ from rclone_api.types import S3PathInfo
 
 # from .rclone import Rclone
 
+_PRINT_LOCK = Lock()
+
+
+def locked_print(*args, **kwargs):
+    with _PRINT_LOCK:
+        print(*args, **kwargs)
+
 
 def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
     from rclone_api.rclone import Rclone

{rclone_api-1.1.4.dist-info → rclone_api-1.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.4
+Version: 1.1.6
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.4.dist-info → rclone_api-1.1.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=m7hQkaIsn9grAlziu9nvDlOIQY6VflYFwC1c_g0qCJo,833
+rclone_api/__init__.py,sha256=6TVA3JUVxr76wzy29XRTX_xTdQ5JBGwuheNvQc8EgsU,863
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -11,27 +11,30 @@ rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
+rclone_api/mount.py,sha256=Zf41gGfTXtSB-KkQoXCq_dpqBvezgz_dPBzk_aDk5Dc,4354
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=_CejQQ8jF-VYc_rO2u5i9qr8jmyksxyuoO4ZdHLW6tg,43540
+rclone_api/rclone.py,sha256=yI_8QdDXppcLO-KbRVGteu-onTa4fTd4ojy_N1HYxx4,42029
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=NC3e78aXCx-sEQ-FqEaC9KzaJDdJhJrKa4Nwum_-Db0,563
-rclone_api/util.py,sha256=efck9W0rw5wfeRI35iiEz4dy2cMkNpVXrQ9zzynkBks,5185
+rclone_api/types.py,sha256=e7VmqA2U6MkTT9VDHeP2qwRw_lUUFbaiN6RO-R8B1oo,3336
+rclone_api/util.py,sha256=ujinqW4xUkZAHBCL1VMhGu88LMdUFIu1ApF8rZEH8rQ,5324
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/copy_large_s3.py,sha256=33KFvCrh5uk-rdRtkREdEs2WNwxGgTdCAWDLCE4dm0A,2855
+rclone_api/cmd/copy_large_s3.py,sha256=EfikmB7l1qqOE9rNEKJ-8S8mx5fY0wNXUABDtOlYqfs,3002
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=0-mtXg9J4MoMm2uBKbsMLj4pSGRLQUAqNRDJWGttnAQ,2443
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
 rclone_api/s3/api.py,sha256=VstlaEnBjO2JDQuCRLdTfUGvQLbfshlXXhAzimFv4Vc,3763
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_uploader.py,sha256=k7491vxvCX2KDz-nMQ45PDeHpaKPKmDtxl-URynLdeo,19736
+rclone_api/s3/chunk_file.py,sha256=XPoDl7DJMJIGBMRoPO2wqwqCMT7ZrIsEkDqlbMH8jzs,3506
+rclone_api/s3/chunk_types.py,sha256=Fq0IlhZ0IftuFQFkbICmmrOonII0BNzuY4CIKNC4wB0,8006
+rclone_api/s3/chunk_uploader.py,sha256=KO8834Gery9HKWSqjQTNW0pbBbVoGrza9gj-1OaNLQQ,9130
 rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
 rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
-rclone_api-1.1.4.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.4.dist-info/METADATA,sha256=DTOOicQKR2vGSu2zZCKLOwj8uR1u7jh8sst9VOevGQ0,4478
-rclone_api-1.1.4.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.4.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.1.4.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.4.dist-info/RECORD,,
+rclone_api-1.1.6.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.6.dist-info/METADATA,sha256=gxcApsFnHMer-mQnReTCQ6_G7kKli0bIN63uderDBDQ,4478
+rclone_api-1.1.6.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.6.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.1.6.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.6.dist-info/RECORD,,