rclone-api 1.5.8__py3-none-any.whl

rclone_api/http_server.py

@@ -0,0 +1,244 @@
+"""
+Unit test file for testing rclone mount functionality.
+"""
+
+import tempfile
+import time
+import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from threading import Semaphore
+from typing import Any
+
+import httpx
+
+from rclone_api.file_part import FilePart
+from rclone_api.process import Process
+from rclone_api.types import Range, SizeSuffix, get_chunk_tmpdir
+
+_TIMEOUT = 10 * 60  # 10 minutes
+
+
+_range = range
+
+
+class HttpServer:
+    """HTTP server configuration."""
+
+    def __init__(self, url: str, subpath: str, process: Process) -> None:
+        self.url = url
+        self.subpath = subpath
+        self.process: Process | None = process
+
+    def _get_file_url(self, path: str | Path) -> str:
+        # if self.subpath == "":
+        path = Path(path).as_posix()
+        return f"{self.url}/{path}"
+        # return f"{self.url}/{self.subpath}/{path}"
+
+    def get_fetcher(self, path: str, n_threads: int = 16) -> "HttpFetcher":
+        return HttpFetcher(self, path, n_threads=n_threads)
+
+    def get(self, path: str, range: Range | None = None) -> bytes | Exception:
+        """Get bytes from the server."""
+        with tempfile.TemporaryFile() as file:
+            self.download(path, Path(file.name), range)
+            file.seek(0)
+            return file.read()
+
+    def size(self, path: str) -> int | Exception:
+        """Get size of the file from the server."""
+        try:
+            assert self.process is not None
+            # response = httpx.head(f"{self.url}/{path}")
+            url = self._get_file_url(path)
+            response = httpx.head(url)
+            response.raise_for_status()
+            size = int(response.headers["Content-Length"])
+            return size
+        except Exception as e:
+            warnings.warn(f"Failed to get size of {self.url}/{path}: {e}")
+            return e
+
+    def download(
+        self, path: str, dst: Path, range: Range | None = None
+    ) -> Path | Exception:
+        """Get bytes from the server."""
+
+        def task() -> Path | Exception:
+
+            if not dst.parent.exists():
+                dst.parent.mkdir(parents=True, exist_ok=True)
+            headers: dict[str, str] = {}
+            if range:
+                headers.update(range.to_header())
+            url = self._get_file_url(path)
+            try:
+                with httpx.stream(
+                    "GET", url, headers=headers, timeout=_TIMEOUT
+                ) as response:
+                    response.raise_for_status()
+                    with open(dst, "wb") as file:
+                        for chunk in response.iter_bytes(chunk_size=8192):
+                            if chunk:
+                                file.write(chunk)
+                            else:
+                                assert response.is_closed
+                    # print(f"Downloaded bytes {start}-{end} to {dst}")
+                    if range:
+                        length = range.end - range.start
+                        print(
+                            f"Downloaded bytes starting at {range.start} with size {length} to {dst}"
+                        )
+                    else:
+                        size = dst.stat().st_size
+                        print(f"Downloaded {size} bytes to {dst}")
+                    return dst
+            except Exception as e:
+                warnings.warn(f"Failed to download {url} to {dst}: {e}")
+                return e
+
+        retries = 3
+        for i in _range(retries):
+            out = task()
+            if not isinstance(out, Exception):
+                return out
+            warnings.warn(f"Failed to download {path} to {dst}: {out}, retrying ({i})")
+            time.sleep(10)
+        else:
+            return Exception(f"Failed to download {path} to {dst}")
+
+    def download_multi_threaded(
+        self,
+        src_path: str,
+        dst_path: Path,
+        chunk_size: int = 32 * 1024 * 1024,
+        n_threads: int = 16,
+        range: Range | None = None,
+    ) -> Path | Exception:
+        """Copy file from src to dst."""
+
+        finished: list[Path] = []
+        errors: list[Exception] = []
+
+        if range is None:
+            sz = self.size(src_path)
+            if isinstance(sz, Exception):
+                return sz
+            range = Range(0, sz)
+
+        with ThreadPoolExecutor(max_workers=n_threads) as executor:
+            try:
+                futures: list[Future[Path | Exception]] = []
+                start: int
+                for start in _range(
+                    range.start.as_int(), range.end.as_int(), chunk_size
+                ):
+                    end = min(
+                        SizeSuffix(start + chunk_size).as_int(), range.end.as_int()
+                    )
+                    r = Range(start=start, end=end)
+
+                    def task(r: Range = r) -> Path | Exception:
+                        dst = dst_path.with_suffix(f".{r.start}")
+                        out = self.download(src_path, dst, r)
+                        if isinstance(out, Exception):
+                            warnings.warn(f"Failed to download chunked: {out}")
+                        return out
+
+                    fut = executor.submit(task, r)
+                    futures.append(fut)
+                for fut in futures:
+                    result = fut.result()
+                    if isinstance(result, Exception):
+                        errors.append(result)
+                    else:
+                        finished.append(result)
+                if errors:
+                    for finished_file in finished:
+                        try:
+                            finished_file.unlink()
+                        except Exception as e:
+                            warnings.warn(f"Failed to delete file {finished_file}: {e}")
+                    return Exception(f"Failed to download chunked: {errors}")
+
+                if not dst_path.parent.exists():
+                    dst_path.parent.mkdir(parents=True, exist_ok=True)
+
+                count = 0
+                with open(dst_path, "wb") as file:
+                    for f in finished:
+                        print(f"Appending {f} to {dst_path}")
+                        with open(f, "rb") as part:
+                            # chunk = part.read(8192 * 4)
+                            while chunk := part.read(8192 * 4):
+                                if not chunk:
+                                    break
+                                count += len(chunk)
+                                file.write(chunk)
+                        print(f"Removing {f}")
+                        f.unlink()
+                # print(f"Downloaded {count} bytes to {dst_path}")
+                return dst_path
+            except Exception as e:
+                warnings.warn(f"Failed to copy chunked: {e}")
+                for f in finished:
+                    try:
+                        if f.exists():
+                            f.unlink()
+                    except Exception as ee:
+                        warnings.warn(f"Failed to delete file {f}: {ee}")
+                return e
+
+    def __enter__(self) -> "HttpServer":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.shutdown()
+
+    def shutdown(self) -> None:
+        """Shutdown the server."""
+        if self.process:
+            self.process.terminate()
+            if self.process.stdout:
+                self.process.stdout.close()
+            if self.process.stderr:
+                self.process.stderr.close()
+
+
+class HttpFetcher:
+    def __init__(self, server: "HttpServer", path: str, n_threads: int) -> None:
+        self.server = server
+        self.path = path
+        self.executor = ThreadPoolExecutor(max_workers=n_threads)
+        # Semaphore throttles the number of concurrent fetches
+        # TODO this is kind of a hack.
+        self.semaphore = Semaphore(n_threads)
+
+    def bytes_fetcher(
+        self, offset: int | SizeSuffix, size: int | SizeSuffix, extra: Any
+    ) -> Future[FilePart]:
+        if isinstance(offset, SizeSuffix):
+            offset = offset.as_int()
+        if isinstance(size, SizeSuffix):
+            size = size.as_int()
+
+        def task() -> FilePart:
+            from rclone_api.util import random_str
+
+            try:
+                range = Range(offset, offset + size)
+                dst = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+                out = self.server.download(self.path, dst, range)
+                if isinstance(out, Exception):
+                    raise out
+                return FilePart(payload=dst, extra=extra)
+            finally:
+                self.semaphore.release()
+
+        self.semaphore.acquire()
+        fut = self.executor.submit(task)
+        return fut
+
+    def shutdown(self) -> None:
+        self.executor.shutdown(wait=True)

rclone_api/install.py

@@ -0,0 +1,95 @@
+import os
+import platform
+import shutil
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from warnings import warn
+
+from download import download
+
+URL_WINDOWS = "https://downloads.rclone.org/rclone-current-windows-amd64.zip"
+URL_LINUX = "https://downloads.rclone.org/rclone-current-linux-amd64.zip"
+URL_MACOS_ARM = "https://downloads.rclone.org/rclone-current-osx-arm64.zip"
+URL_MACOS_X86 = "https://downloads.rclone.org/rclone-current-osx-amd64.zip"
+
+
+def rclone_download_url() -> str:
+    system = platform.system()
+    arch = platform.machine()
+    if system == "Windows":
+        assert "arm" not in arch, f"Unsupported arch: {arch}"
+        return URL_WINDOWS
+    elif system == "Linux":
+        assert "arm" not in arch, f"Unsupported arch: {arch}"
+        return URL_LINUX
+    elif system == "Darwin":
+        if "x86" in arch:
+            return URL_MACOS_X86
+        elif "arm" in arch:
+            return URL_MACOS_ARM
+        else:
+            raise Exception(f"Unsupported arch: {arch}")
+    else:
+        raise Exception("Unsupported system")
+
+
+def _remove_signed_binary_requirements(out: Path) -> None:
+    if platform.system() == "Windows":
+        return
+    # mac os
+    if platform.system() == "Darwin":
+        # remove signed binary requirements
+        #
+        # xattr -d com.apple.quarantine rclone
+        import subprocess
+
+        subprocess.run(
+            ["xattr", "-d", "com.apple.quarantine", str(out)],
+            capture_output=True,
+            check=False,
+        )
+        return
+
+
+def _make_executable(out: Path) -> None:
+    if platform.system() == "Windows":
+        return
+    # linux and mac os
+    os.chmod(out, 0o755)
+
+
+def _find_rclone_exe(start: Path) -> Path | None:
+    for root, dirs, files in os.walk(start):
+        if platform.system() == "Windows":
+            if "rclone.exe" in files:
+                return Path(root) / "rclone.exe"
+        else:
+            if "rclone" in files:
+                return Path(root) / "rclone"
+    return None
+
+
+def rclone_download(out: Path, replace=False) -> Exception | None:
+    if out.exists() and not replace:
+        return None
+    try:
+        url = rclone_download_url()
+        with TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            download(url, tmp, kind="zip", replace=True)
+            exe = _find_rclone_exe(tmp)
+            if exe is None:
+                raise FileNotFoundError("rclone executable not found")
+            if os.path.exists(out):
+                os.remove(out)
+            out.parent.mkdir(parents=True, exist_ok=True)
+            shutil.move(exe, out)
+        _remove_signed_binary_requirements(out)
+        _make_executable(out)
+        return None
+    except Exception as e:
+        import traceback
+
+        stacktrace = traceback.format_exc()
+        warn(f"Failed to download rclone: {e}\n{stacktrace}")
+        return e

rclone_api/log.py

@@ -0,0 +1,44 @@
+import logging
+import sys
+
+_INITIALISED = False
+
+
+def setup_default_logging():
+    """Set up default logging configuration if none exists."""
+    global _INITIALISED
+    if _INITIALISED:
+        return
+    if not logging.root.handlers:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            handlers=[
+                logging.StreamHandler(sys.stdout),
+                # Uncomment to add file logging
+                # logging.FileHandler('rclone_api.log')
+            ],
+        )
+
+
+def configure_logging(level=logging.INFO, log_file=None):
+    """Configure logging for the rclone_api package.
+
+    Args:
+        level: The logging level (default: logging.INFO)
+        log_file: Optional path to a log file
+    """
+    handlers = [logging.StreamHandler(sys.stdout)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file))
+
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=handlers,
+        force=True,  # Override any existing configuration
+    )
+
+
+# Call setup_default_logging when this module is imported
+setup_default_logging()

rclone_api/mount.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api.process import Process
+
+
+@dataclass
+class Mount:
+    """Mount information."""
+
+    src: str
+    mount_path: Path
+    process: Process
+    read_only: bool
+    cache_dir: Path | None = None
+    cache_dir_delete_on_exit: bool | None = None
+    _closed: bool = False
+
+    def __post_init__(self):
+        from rclone_api.mount_util import add_mount_for_gc, wait_for_mount
+
+        assert isinstance(self.mount_path, Path)
+        assert self.process is not None
+        wait_for_mount(self)
+        add_mount_for_gc(self)
+
+    def close(self, wait=True) -> None:
+        """Clean up the mount."""
+        from rclone_api.mount_util import (
+            cache_dir_delete_on_exit,
+            clean_mount,
+            remove_mount_for_gc,
+        )
+
+        if self._closed:
+            return
+        self._closed = True
+        self.process.terminate()
+        clean_mount(self, verbose=False, wait=wait)
+        if self.cache_dir and self.cache_dir_delete_on_exit:
+            cache_dir_delete_on_exit(self.cache_dir)
+        remove_mount_for_gc(self)
+
+    def __enter__(self) -> "Mount":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close(wait=True)
+
+    def __del__(self):
+        self.close(wait=False)
+
+    # make this a hashable object
+    def __hash__(self):
+        return hash(self.mount_path)

rclone_api/mount_util.py

@@ -0,0 +1,247 @@
+import atexit
+import os
+import platform
+import shutil
+import subprocess
+import time
+import warnings
+import weakref
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from rclone_api.mount import Mount
+from rclone_api.process import Process
+
+_SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
+
+
+_MOUNTS_FOR_GC: weakref.WeakSet = weakref.WeakSet()
+
+
+def _cleanup_mounts() -> None:
+    with ThreadPoolExecutor() as executor:
+        mount: Mount
+        for mount in _MOUNTS_FOR_GC:
+            executor.submit(mount.close)
+
+
+def _run_command(cmd: str, verbose: bool) -> int:
+    """Run a shell command and print its output if verbose is True."""
+    if verbose:
+        print(f"Executing: {cmd}")
+    try:
+        result = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, check=False
+        )
+        if result.returncode != 0 and verbose:
+            print(f"Command failed: {cmd}\nStdErr: {result.stderr.strip()}")
+        return result.returncode
+    except Exception as e:
+        warnings.warn(f"Error running command '{cmd}': {e}")
+        return -1
+
+
+atexit.register(_cleanup_mounts)
+
+
+def cache_dir_delete_on_exit(cache_dir: Path) -> None:
+    if cache_dir.exists():
+        try:
+            shutil.rmtree(cache_dir)
+        except Exception as e:
+            warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
+
+
+def add_mount_for_gc(mount: Mount) -> None:
+    # weak reference to avoid circular references
+    _MOUNTS_FOR_GC.add(mount)
+
+
+def remove_mount_for_gc(mount: Mount) -> None:
+    _MOUNTS_FOR_GC.discard(mount)
+
+
+def prepare_mount(outdir: Path, verbose: bool) -> None:
+    if _SYSTEM == "Windows":
+        # Windows -> Must create parent directories only if they don't exist
+        if verbose:
+            print(f"Creating parent directories for {outdir}")
+        outdir.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # Linux -> Must create parent directories and the directory itself
+        if verbose:
+            print(f"Creating directories for {outdir}")
+        outdir.mkdir(parents=True, exist_ok=True)
+
+
+def wait_for_mount(
+    mount: Mount,
+    timeout: int = 20,
+    post_mount_delay: int = 5,
+    poll_interval: float = 1.0,
+    check_mount_flag: bool = False,
+) -> None:
+    """
+    Wait for a mount point to become available by checking if the directory exists,
+    optionally verifying that it is a mount point, and confirming that it contains files.
+    This function periodically polls for the mount status, ensures the mount process
+    is still running, and applies an extra delay after detecting content for stabilization.
+
+    Args:
+        src (Path): The mount point directory to check.
+        mount_process (Any): A Process instance handling the mount (must be an instance of Process).
+        timeout (int): Maximum time in seconds to wait for the mount to become available.
+        post_mount_delay (int): Additional seconds to wait after detecting files.
+        poll_interval (float): Seconds between each poll iteration.
+        check_mount_flag (bool): If True, verifies that the path is recognized as a mount point.
+
+    Raises:
+        subprocess.CalledProcessError: If the mount_process exits unexpectedly.
+        TimeoutError: If the mount is not available within the timeout period.
+        TypeError: If mount_process is not an instance of Process.
+    """
+
+    mount_process = mount.process
+    src = mount.mount_path
+
+    if not isinstance(mount_process, Process):
+        raise TypeError("mount_process must be an instance of Process")
+
+    expire_time = time.time() + timeout
+    last_error = None
+
+    while time.time() < expire_time:
+        # Check if the mount process has terminated unexpectedly.
+        rtn = mount_process.poll()
+        if rtn is not None:
+            cmd_str = subprocess.list2cmdline(mount_process.cmd)
+            print(f"Mount process terminated unexpectedly: {cmd_str}")
+            raise subprocess.CalledProcessError(rtn, cmd_str)
+
+        # Check if the mount path exists.
+        if src.exists():
+            # Optionally check if path is a mount point.
+            if check_mount_flag:
+                try:
+                    if not os.path.ismount(str(src)):
+                        print(
+                            f"{src} exists but is not recognized as a mount point yet."
+                        )
+                        time.sleep(poll_interval)
+                        continue
+                except Exception as e:
+                    print(f"Could not verify mount point status for {src}: {e}")
+
+            try:
+                # Check for at least one entry in the directory.
+                if any(src.iterdir()):
+                    print(
+                        f"Mount point {src} appears available with files. Waiting {post_mount_delay} seconds for stabilization."
+                    )
+                    time.sleep(post_mount_delay)
+                    return
+                else:
+                    print(f"Mount point {src} is empty. Waiting for files to appear.")
+            except Exception as e:
+                last_error = e
+                print(f"Error accessing {src}: {e}")
+        else:
+            print(f"Mount point {src} does not exist yet.")
+
+        time.sleep(poll_interval)
+
+    # raise TimeoutError(
+    #     f"Mount point {src} did not become available within {timeout} seconds. Last error: {last_error}"
+    # )
+    if last_error is not None:
+        raise last_error
+
+
+def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
+    """
+    Clean up a mount path across Linux, macOS, and Windows.
+
+    The function attempts to unmount the mount at mount_path, then, if the
+    directory is empty, removes it. On Linux it uses 'fusermount -u' (for FUSE mounts)
+    and 'umount'. On macOS it uses 'umount' (and optionally 'diskutil unmount'),
+    while on Windows it attempts to remove the mount point via 'mountvol /D'.
+    """
+
+    def verbose_print(msg: str):
+        if verbose:
+            print(msg)
+
+    proc = mount.process if isinstance(mount, Mount) else None
+    if proc is not None and proc.poll() is None:
+        verbose_print(f"Terminating mount process {proc.pid}")
+        proc.kill()
+
+    # Check if the mount path exists; if an OSError occurs, assume it exists.
+    mount_path = mount.mount_path if isinstance(mount, Mount) else mount
+    try:
+        mount_exists = mount_path.exists()
+    except OSError:
+        # warnings.warn(f"Error checking {mount_path}: {e}")
+        mount_exists = True
+
+    # Give the system a moment (if unmount is in progress, etc.)
+    if wait:
+        time.sleep(2)
+
+    if not mount_exists:
+        verbose_print(f"{mount_path} does not exist; nothing to clean up.")
+        return
+
+    verbose_print(f"{mount_path} still exists, attempting to unmount and remove.")
+
+    # Platform-specific unmount procedures
+    if _SYSTEM == "Linux":
+        # Try FUSE unmount first (if applicable), then the regular umount.
+        _run_command(f"fusermount -u {mount_path}", verbose)
+        _run_command(f"umount {mount_path}", verbose)
+    elif _SYSTEM == "Darwin":
+        # On macOS, use umount; optionally try diskutil for stubborn mounts.
+        _run_command(f"umount {mount_path}", verbose)
+        # Optionally: uncomment the next line if diskutil unmount is preferred.
+        # _run_command(f"diskutil unmount {mount_path}", verbose)
+    elif _SYSTEM == "Windows":
+        # On Windows, remove the mount point using mountvol.
+        _run_command(f"mountvol {mount_path} /D", verbose)
+        # If that does not work, try to remove the directory directly.
+        try:
+            mount_path.rmdir()
+            if verbose:
+                print(f"Successfully removed mount directory {mount_path}")
+        except Exception:
+            # warnings.warn(f"Failed to remove mount {mount_path}: {e}")
+            pass
+    else:
+        warnings.warn(f"Unsupported platform: {_SYSTEM}")
+
+    # Allow some time for the unmount commands to take effect.
+    if wait:
+        time.sleep(2)
+
+    # Re-check if the mount path still exists.
+    try:
+        still_exists = mount_path.exists()
+    except OSError as e:
+        warnings.warn(f"Error re-checking {mount_path}: {e}")
+        still_exists = True
+
+    if still_exists:
+        verbose_print(f"{mount_path} still exists after unmount attempt.")
+        # Attempt to remove the directory if it is empty.
+        try:
+            # Only remove if the directory is empty.
+            if not any(mount_path.iterdir()):
+                mount_path.rmdir()
+                if verbose:
+                    verbose_print(f"Removed empty mount directory {mount_path}")
+            else:
+                warnings.warn(f"{mount_path} is not empty; cannot remove.")
+                raise OSError(f"{mount_path} is not empty")
+        except Exception as e:
+            warnings.warn(f"Failed during cleanup of {mount_path}: {e}")
+    else:
+        verbose_print(f"{mount_path} successfully cleaned up.")