rclone-api 1.1.4__tar.gz → 1.1.5__tar.gz

{rclone_api-1.1.4 → rclone_api-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.4
+Version: 1.1.5
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.4 → rclone_api-1.1.5}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.4"
+version = "1.1.5"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.1.5/src/rclone_api/mount.py

@@ -0,0 +1,102 @@
+import platform
+import subprocess
+import time
+import warnings
+from pathlib import Path
+
+_SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
+
+
+def run_command(cmd: str, verbose: bool) -> int:
+    """Run a shell command and print its output if verbose is True."""
+    if verbose:
+        print(f"Executing: {cmd}")
+    try:
+        result = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, check=False
+        )
+        if result.returncode != 0 and verbose:
+            print(f"Command failed: {cmd}\nStdErr: {result.stderr.strip()}")
+        return result.returncode
+    except Exception as e:
+        warnings.warn(f"Error running command '{cmd}': {e}")
+        return -1
+
+
+def clean_mount(mount_path: Path, verbose: bool = False) -> None:
+    """
+    Clean up a mount path across Linux, macOS, and Windows.
+
+    The function attempts to unmount the mount at mount_path, then, if the
+    directory is empty, removes it. On Linux it uses 'fusermount -u' (for FUSE mounts)
+    and 'umount'. On macOS it uses 'umount' (and optionally 'diskutil unmount'),
+    while on Windows it attempts to remove the mount point via 'mountvol /D'.
+    """
+    # Check if the mount path exists; if an OSError occurs, assume it exists.
+    try:
+        mount_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error checking {mount_path}: {e}")
+        mount_exists = True
+
+    # Give the system a moment (if unmount is in progress, etc.)
+    time.sleep(2)
+
+    if not mount_exists:
+        if verbose:
+            print(f"{mount_path} does not exist; nothing to clean up.")
+        return
+
+    if verbose:
+        print(f"{mount_path} still exists, attempting to unmount and remove.")
+
+    # Platform-specific unmount procedures
+    if _SYSTEM == "Linux":
+        # Try FUSE unmount first (if applicable), then the regular umount.
+        run_command(f"fusermount -u {mount_path}", verbose)
+        run_command(f"umount {mount_path}", verbose)
+    elif _SYSTEM == "Darwin":
+        # On macOS, use umount; optionally try diskutil for stubborn mounts.
+        run_command(f"umount {mount_path}", verbose)
+        # Optionally: uncomment the next line if diskutil unmount is preferred.
+        # run_command(f"diskutil unmount {mount_path}", verbose)
+    elif _SYSTEM == "Windows":
+        # On Windows, remove the mount point using mountvol.
+        run_command(f"mountvol {mount_path} /D", verbose)
+        # If that does not work, try to remove the directory directly.
+        try:
+            mount_path.rmdir()
+            if verbose:
+                print(f"Successfully removed mount directory {mount_path}")
+        except Exception as e:
+            warnings.warn(f"Failed to remove mount {mount_path}: {e}")
+    else:
+        warnings.warn(f"Unsupported platform: {_SYSTEM}")
+
+    # Allow some time for the unmount commands to take effect.
+    time.sleep(2)
+
+    # Re-check if the mount path still exists.
+    try:
+        still_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error re-checking {mount_path}: {e}")
+        still_exists = True
+
+    if still_exists:
+        if verbose:
+            print(f"{mount_path} still exists after unmount attempt.")
+        # Attempt to remove the directory if it is empty.
+        try:
+            # Only remove if the directory is empty.
+            if not any(mount_path.iterdir()):
+                mount_path.rmdir()
+                if verbose:
+                    print(f"Removed empty mount directory {mount_path}")
+            else:
+                warnings.warn(f"{mount_path} is not empty; cannot remove.")
+        except Exception as e:
+            warnings.warn(f"Failed during cleanup of {mount_path}: {e}")
+    else:
+        if verbose:
+            print(f"{mount_path} successfully cleaned up.")

{rclone_api-1.1.4 → rclone_api-1.1.5}/src/rclone_api/rclone.py

@@ -914,38 +914,10 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
-            if not error_happened and outdir.exists():
-                time.sleep(2)
-                if outdir.exists():
-                    print(f"{outdir} mount still exists, attempting to remove")
-                    if not _IS_WINDOWS:
-
-                        def exec(cmd: str) -> int:
-                            if verbose:
-                                print(f"Executing: {cmd}")
-                            rtn = os.system(cmd)
-                            if rtn != 0 and verbose:
-                                print(f"Failed to execute: {cmd}")
-                            return rtn
-
-                        exec(f"fusermount -u {outdir}")
-                        exec(f"umount {outdir}")
-                        time.sleep(2)
-                        if outdir.exists():
-                            is_empty = True
-                            try:
-                                is_empty = not list(outdir.iterdir())
-                                if not is_empty:
-                                    warnings.warn(f"Failed to unmount {outdir}")
-                                else:
-                                    try:
-                                        outdir.rmdir()
-                                    except Exception as e:
-                                        warnings.warn(f"Failed to remove {outdir}: {e}")
-                            except Exception as e:
-                                warnings.warn(
-                                    f"Failed during mount cleanup of {outdir}: because {e}"
-                                )
+            if not error_happened:
+                from rclone_api.mount import clean_mount
+
+                clean_mount(outdir, verbose=verbose)
 
     @deprecated("mount")
     def mount_webdav(

rclone_api-1.1.5/src/rclone_api/s3/chunk_file.py

@@ -0,0 +1,102 @@
+import time
+import warnings
+from pathlib import Path
+from queue import Queue
+
+from rclone_api.s3.chunk_types import FileChunk, UploadState
+from rclone_api.util import locked_print
+
+
+def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+    sleep_time = timeout / 60 if timeout > 0 else 1
+    start = time.time()
+    while True:
+        expired = time.time() - start > timeout
+        try:
+            time.sleep(sleep_time)
+            if file_path.exists():
+                return file_path.stat().st_size
+        except FileNotFoundError as e:
+            if expired:
+                print(f"File not found: {file_path}, exception is {e}")
+                raise
+        if expired:
+            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+
+
+def file_chunker(
+    upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
+) -> None:
+    count = 0
+
+    def should_stop() -> bool:
+        nonlocal count
+        if max_chunks is None:
+            return False
+        if count >= max_chunks:
+            return True
+        count += 1
+        if count > 10 and count % 10 == 0:
+            # recheck that the file size has not changed
+            file_size = _get_file_size(upload_state.upload_info.src_file_path)
+            if file_size != upload_state.upload_info.file_size:
+                locked_print(
+                    f"File size changed, cannot resume, expected {upload_state.upload_info.file_size}, got {file_size}"
+                )
+                raise ValueError("File size changed, cannot resume")
+        return False
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+    src = Path(file_path)
+    # Mounted files may take a while to appear, so keep retrying.
+
+    try:
+        file_size = _get_file_size(src, timeout=60)
+        part_number = 1
+        done_part_numbers: set[int] = {
+            p.part_number for p in upload_state.parts if p is not None
+        }
+        num_parts = upload_info.total_chunks()
+
+        def next_part_number() -> int | None:
+            nonlocal part_number
+            while part_number in done_part_numbers:
+                part_number += 1
+            if part_number > num_parts:
+                return None
+            return part_number
+
+        while not should_stop():
+            curr_parth_num = next_part_number()
+            if curr_parth_num is None:
+                locked_print(f"File {file_path} has completed chunking all parts")
+                break
+            assert curr_parth_num is not None
+            offset = (curr_parth_num - 1) * chunk_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+
+            # Open the file, seek, read the chunk, and close immediately.
+            with open(file_path, "rb") as f:
+                f.seek(offset)
+                data = f.read(chunk_size)
+
+            if not data:
+                warnings.warn(f"Empty data for part {part_number} of {file_path}")
+
+            file_chunk = FileChunk(
+                src,
+                upload_id=upload_info.upload_id,
+                part_number=part_number,
+                data=data,  # After this, data should not be reused.
+            )
+            done_part_numbers.add(part_number)
+            output.put(file_chunk)
+            part_number += 1
+    except Exception as e:
+
+        warnings.warn(f"Error reading file: {e}")
+    finally:
+        output.put(None)

rclone_api-1.1.5/src/rclone_api/s3/chunk_types.py

@@ -0,0 +1,254 @@
+import json
+import os
+import time
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from threading import Lock
+
+from botocore.client import BaseClient
+
+from rclone_api.util import locked_print
+
+_MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
+_SAVE_STATE_LOCK = Lock()
+
+_TMP_DIR_ACCESS_LOCK = Lock()
+
+
+def _clean_old_files(out: Path) -> None:
+    # clean up files older than 1 day
+
+    now = time.time()
+    # Erase all stale files and then purge empty directories.
+    for root, dirs, files in os.walk(out):
+        for name in files:
+            f = Path(root) / name
+            filemod = f.stat().st_mtime
+            diff_secs = now - filemod
+            diff_days = diff_secs / (60 * 60 * 24)
+            if diff_days > 1:
+                locked_print(f"Removing old file: {f}")
+                f.unlink()
+
+    for root, dirs, _ in os.walk(out):
+        for dir in dirs:
+            d = Path(root) / dir
+            if not list(d.iterdir()):
+                locked_print(f"Removing empty directory: {d}")
+                d.rmdir()
+
+
+def _get_chunk_tmpdir() -> Path:
+    with _TMP_DIR_ACCESS_LOCK:
+        dat = _get_chunk_tmpdir.__dict__
+        if "out" in dat:
+            return dat["out"]  # Folder already validated.
+        out = Path("chunk_store")
+        if out.exists():
+            # first access, clean up directory
+            _clean_old_files(out)
+        out.mkdir(exist_ok=True, parents=True)
+        dat["out"] = out
+        return out
+
+
+class FileChunk:
+    def __init__(self, src: Path, upload_id: str, part_number: int, data: bytes):
+        assert data is not None, f"{src}: Data must not be None"
+        self.upload_id = upload_id
+        self.src = src
+        self.part_number = part_number
+        name = src.name
+        self.tmpdir = _get_chunk_tmpdir()
+        self.filepart = self.tmpdir / f"{name}_{upload_id}.part_{part_number}.tmp"
+        self.filepart.write_bytes(data)
+        del data  # free up memory
+
+    @property
+    def data(self) -> bytes:
+        assert self.filepart is not None
+        with open(self.filepart, "rb") as f:
+            return f.read()
+        return b""
+
+    def close(self):
+        if self.filepart.exists():
+            self.filepart.unlink()
+
+    def __del__(self):
+        self.close()
+
+
+@dataclass
+class UploadInfo:
+    s3_client: BaseClient
+    bucket_name: str
+    object_name: str
+    src_file_path: Path
+    upload_id: str
+    retries: int
+    chunk_size: int
+    file_size: int
+    _total_chunks: int | None = None
+
+    def total_chunks(self) -> int:
+        out = self.file_size // self.chunk_size
+        if self.file_size % self.chunk_size:
+            return out + 1
+        return out
+
+    def __post_init__(self):
+        if self._total_chunks is not None:
+            return
+        self._total_chunks = self.total_chunks()
+
+    def to_json(self) -> dict:
+        json_dict = {}
+        for f in fields(self):
+            value = getattr(self, f.name)
+            # Convert non-serializable objects (like s3_client) to a string representation.
+            if f.name == "s3_client":
+                json_dict[f.name] = "RUNTIME OBJECT"
+            else:
+                if isinstance(value, Path):
+                    value = str(value)
+                json_dict[f.name] = value
+        return json_dict
+
+    @staticmethod
+    def from_json(s3_client: BaseClient, json_dict: dict) -> "UploadInfo":
+        json_dict.pop("s3_client")  # Remove the placeholder string
+        return UploadInfo(s3_client=s3_client, **json_dict)
+
+
+@dataclass
+class FinishedPiece:
+    part_number: int
+    etag: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "etag": self.etag}
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_json(), indent=0)
+
+    @staticmethod
+    def to_json_array(parts: list["FinishedPiece | None"]) -> list[dict | None]:
+        non_none: list[FinishedPiece] = [p for p in parts if p is not None]
+        non_none.sort(key=lambda x: x.part_number)
+        all_nones: list[None] = [None for p in parts if p is None]
+        assert len(all_nones) <= 1, "Only one None should be present"
+        return [p.to_json() for p in non_none]
+
+    @staticmethod
+    def from_json(json: dict | None) -> "FinishedPiece | None":
+        if json is None:
+            return None
+        return FinishedPiece(**json)
+
+
+@dataclass
+class UploadState:
+    upload_info: UploadInfo
+    # finished_parts: Queue[FinishedPiece | None]
+    peristant: Path | None
+    lock: Lock = Lock()
+    parts: list[FinishedPiece | None] = field(default_factory=list)
+
+    def update_source_file(self, src_file: Path) -> None:
+        new_file_size = os.path.getsize(src_file)
+        if new_file_size != self.upload_info.file_size:
+            raise ValueError("File size changed, cannot resume")
+        self.upload_info.src_file_path = src_file
+        self.save()
+
+    def is_done(self) -> bool:
+        return self.remaining() == 0
+
+    def count(self) -> tuple[int, int]:  # count, num_chunks
+        num_chunks = self.upload_info.total_chunks()
+        count = 0
+        for p in self.parts:
+            if p is not None:
+                count += 1
+        return count, num_chunks
+
+    def finished(self) -> int:
+        count, _ = self.count()
+        return count
+
+    def remaining(self) -> int:
+        count, num_chunks = self.count()
+        assert (
+            count <= num_chunks
+        ), f"Count {count} is greater than num_chunks {num_chunks}"
+        return num_chunks - count
+
+    def add_finished(self, part: FinishedPiece | None) -> None:
+        with self.lock:
+            self.parts.append(part)
+            self._save_no_lock()
+
+    def __post_init__(self):
+        if self.peristant is None:
+            # upload_id = self.upload_info.upload_id
+            object_name = self.upload_info.object_name
+            chunk_size = self.upload_info.chunk_size
+            parent = _get_chunk_tmpdir()
+            self.peristant = parent / f"{object_name}_chunk_size_{chunk_size}_.json"
+
+    def save(self) -> None:
+        with _SAVE_STATE_LOCK:
+            self._save_no_lock()
+
+    def _save_no_lock(self) -> None:
+        assert self.peristant is not None, "No path to save to"
+        self.peristant.write_text(self.to_json_str(), encoding="utf-8")
+
+    @staticmethod
+    def load(s3_client: BaseClient, path: Path) -> "UploadState":
+        with _SAVE_STATE_LOCK:
+            return UploadState.from_json(s3_client, path)
+
+    def to_json(self) -> dict:
+        # queue -> list
+        # parts: list[dict] = [f.to_json() for f in self.parts]
+        parts: list[FinishedPiece | None] = list(self.parts)
+
+        parts_json = FinishedPiece.to_json_array(parts)
+        is_done = self.is_done()
+        count_non_none: int = 0
+        for p in parts:
+            if p is not None:
+                count_non_none += 1
+
+        # self.count()
+        finished_count, total = self.count()
+
+        # parts.sort(key=lambda x: x.part_number)  # Some backends need this.
+        out_json = {
+            "upload_info": self.upload_info.to_json(),
+            "finished_parts": parts_json,
+            "is_done": is_done,
+            "finished_count": finished_count,
+            "total_parts": total,
+        }
+
+        # check that we can sererialize
+        # json.dumps(out_json)
+        return out_json
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_json(), indent=4)
+
+    @staticmethod
+    def from_json(s3_client: BaseClient, json_file: Path) -> "UploadState":
+        json_str = json_file.read_text(encoding="utf-8")
+        data = json.loads(json_str)
+        upload_info_json = data["upload_info"]
+        finished_parts_json = data["finished_parts"]
+        upload_info = UploadInfo.from_json(s3_client, upload_info_json)
+        finished_parts = [FinishedPiece.from_json(p) for p in finished_parts_json]
+        return UploadState(
+            peristant=json_file, upload_info=upload_info, parts=finished_parts
+        )