rclone-api 1.3.19__tar.gz → 1.3.22__tar.gz

{rclone_api-1.3.19 → rclone_api-1.3.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.19
+Version: 1.3.22
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License
@@ -16,6 +16,7 @@ Requires-Dist: psutil
 Requires-Dist: boto3<=1.35.99,>=1.20.1
 Requires-Dist: sqlmodel>=0.0.23
 Requires-Dist: psycopg2-binary>=2.9.10
+Requires-Dist: httpx>=0.28.1
 Dynamic: home-page
 
 # rclone-api

{rclone_api-1.3.19 → rclone_api-1.3.22}/pyproject.toml

@@ -21,10 +21,11 @@ dependencies = [
     "boto3>=1.20.1,<=1.35.99",
     "sqlmodel>=0.0.23",
     "psycopg2-binary>=2.9.10",
+    "httpx>=0.28.1",
 ]
 
 # Change this with the version number bump.
-version = "1.3.19"
+version = "1.3.22"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/__init__.py

@@ -9,6 +9,7 @@ from .dir import Dir
 from .dir_listing import DirListing
 from .file import File, FileItem
 from .filelist import FileList
+from .http_server import HttpFetcher, HttpServer, Range
 
 # Import the configure_logging function to make it available at package level
 from .log import configure_logging, setup_default_logging
@@ -45,6 +46,9 @@ __all__ = [
     "SizeSuffix",
     "configure_logging",
     "log",
+    "HttpServer",
+    "Range",
+    "HttpFetcher",
 ]
 
 setup_default_logging()

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/cmd/copy_large_s3.py

@@ -31,7 +31,7 @@ def _parse_args() -> Args:
     parser.add_argument("dst", help="Destination file")
     parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
     parser.add_argument(
-        "--config", help="Path to rclone config file", type=Path, required=True
+        "--config", help="Path to rclone config file", type=Path, required=False
     )
     parser.add_argument(
         "--chunk-size",
@@ -60,8 +60,14 @@ def _parse_args() -> Args:
     )
 
     args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
     out = Args(
-        config_path=Path(args.config),
+        config_path=config,
         src=args.src,
         dst=args.dst,
         chunk_size=SizeSuffix(args.chunk_size),
@@ -94,21 +100,20 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    import os
     import sys
 
-    here = Path(__file__).parent
-    project_root = here.parent.parent.parent
-    print(f"project_root: {project_root}")
-    os.chdir(str(project_root))
-    cwd = Path(__file__).parent
-    print(f"cwd: {cwd}")
+    # here = Path(__file__).parent
+    # project_root = here.parent.parent.parent
+    # print(f"project_root: {project_root}")
+    # os.chdir(str(project_root))
+    # cwd = Path(__file__).parent
+    # print(f"cwd: {cwd}")
     sys.argv.append("--config")
     sys.argv.append("rclone.conf")
     sys.argv.append(
-        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
     )
     sys.argv.append(
-        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
     )
     main()

rclone_api-1.3.22/src/rclone_api/http_server.py

@@ -0,0 +1,233 @@
+"""
+Unit test file for testing rclone mount functionality.
+"""
+
+import tempfile
+import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from rclone_api.process import Process
+from rclone_api.types import FilePart, SizeSuffix, get_chunk_tmpdir
+
+_TIMEOUT = 10 * 60  # 10 minutes
+
+
+@dataclass
+class Range:
+    start: int
+    end: int
+
+    def to_header(self) -> dict[str, str]:
+        val = f"bytes={self.start}-{self.end-1}"
+        return {
+            "Range": val,
+        }
+
+
+_range = range
+
+
+class HttpServer:
+    """HTTP server configuration."""
+
+    def __init__(self, url: str, subpath: str, process: Process) -> None:
+        self.url = url
+        self.subpath = subpath
+        self.process: Process | None = process
+
+    def _get_file_url(self, path: str | Path) -> str:
+        # if self.subpath == "":
+        path = Path(path).as_posix()
+        return f"{self.url}/{path}"
+        # return f"{self.url}/{self.subpath}/{path}"
+
+    def get_fetcher(self, path: str, n_threads: int = 16) -> "HttpFetcher":
+        return HttpFetcher(self, path, n_threads=n_threads)
+
+    def get(self, path: str) -> bytes | Exception:
+        """Get bytes from the server."""
+        with tempfile.TemporaryFile() as file:
+            self.download(path, Path(file.name), None)
+            file.seek(0)
+            return file.read()
+
+    def size(self, path: str) -> int | Exception:
+        """Get size of the file from the server."""
+        try:
+            assert self.process is not None
+            # response = httpx.head(f"{self.url}/{path}")
+            url = self._get_file_url(path)
+            response = httpx.head(url)
+            response.raise_for_status()
+            size = int(response.headers["Content-Length"])
+            return size
+        except Exception as e:
+            warnings.warn(f"Failed to get size of {self.url}/{path}: {e}")
+            return e
+
+    def download(
+        self, path: str, dst: Path, range: Range | None = None
+    ) -> Path | Exception:
+        """Get bytes from the server."""
+        if not dst.parent.exists():
+            dst.parent.mkdir(parents=True, exist_ok=True)
+        headers: dict[str, str] = {}
+        if range:
+            headers.update(range.to_header())
+        url = self._get_file_url(path)
+        try:
+            with httpx.stream(
+                "GET", url, headers=headers, timeout=_TIMEOUT
+            ) as response:
+                response.raise_for_status()
+                with open(dst, "wb") as file:
+                    for chunk in response.iter_bytes(chunk_size=8192):
+                        if chunk:
+                            file.write(chunk)
+                        else:
+                            assert response.is_closed
+                # print(f"Downloaded bytes {start}-{end} to {dst}")
+                if range:
+                    print(f"Downloaded bytes {range.start}-{range.end} to {dst}")
+                else:
+                    size = dst.stat().st_size
+                    print(f"Downloaded {size} bytes to {dst}")
+                return dst
+        except Exception as e:
+            warnings.warn(f"Failed to download {url} to {dst}: {e}")
+            return e
+
+    def download_multi_threaded(
+        self,
+        src_path: str,
+        dst_path: Path,
+        chunk_size: int = 32 * 1024 * 1024,
+        n_threads: int = 16,
+        range: Range | None = None,
+    ) -> Path | Exception:
+        """Copy file from src to dst."""
+
+        finished: list[Path] = []
+        errors: list[Exception] = []
+
+        if range is None:
+            sz = self.size(src_path)
+            if isinstance(sz, Exception):
+                return sz
+            range = Range(0, sz)
+
+        with ThreadPoolExecutor(max_workers=n_threads) as executor:
+            try:
+                futures: list[Future[Path | Exception]] = []
+                for start in _range(range.start, range.end, chunk_size):
+                    end = min(start + chunk_size, range.end)
+                    r = Range(start=start, end=end)
+
+                    def task(r: Range = r) -> Path | Exception:
+                        dst = dst_path.with_suffix(f".{r.start}")
+                        out = self.download(src_path, dst, r)
+                        if isinstance(out, Exception):
+                            warnings.warn(f"Failed to download chunked: {out}")
+                        return out
+
+                    fut = executor.submit(task, r)
+                    futures.append(fut)
+                for fut in futures:
+                    result = fut.result()
+                    if isinstance(result, Exception):
+                        errors.append(result)
+                    else:
+                        finished.append(result)
+                if errors:
+                    for finished_file in finished:
+                        try:
+                            finished_file.unlink()
+                        except Exception as e:
+                            warnings.warn(f"Failed to delete file {finished_file}: {e}")
+                    return Exception(f"Failed to download chunked: {errors}")
+
+                if not dst_path.parent.exists():
+                    dst_path.parent.mkdir(parents=True, exist_ok=True)
+
+                count = 0
+                with open(dst_path, "wb") as file:
+                    for f in finished:
+                        print(f"Appending {f} to {dst_path}")
+                        with open(f, "rb") as part:
+                            # chunk = part.read(8192 * 4)
+                            while chunk := part.read(8192 * 4):
+                                if not chunk:
+                                    break
+                                count += len(chunk)
+                                file.write(chunk)
+                        print(f"Removing {f}")
+                        f.unlink()
+                # print(f"Downloaded {count} bytes to {dst_path}")
+                return dst_path
+            except Exception as e:
+                warnings.warn(f"Failed to copy chunked: {e}")
+                for f in finished:
+                    try:
+                        if f.exists():
+                            f.unlink()
+                    except Exception as ee:
+                        warnings.warn(f"Failed to delete file {f}: {ee}")
+                return e
+
+    def __enter__(self) -> "HttpServer":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.shutdown()
+
+    def shutdown(self) -> None:
+        """Shutdown the server."""
+        if self.process:
+            self.process.terminate()
+            if self.process.stdout:
+                self.process.stdout.close()
+            if self.process.stderr:
+                self.process.stderr.close()
+
+
+class HttpFetcher:
+    def __init__(self, server: "HttpServer", path: str, n_threads: int) -> None:
+        self.server = server
+        self.path = path
+        self.executor = ThreadPoolExecutor(max_workers=n_threads)
+        from threading import Semaphore
+
+        self.semaphore = Semaphore(n_threads)
+
+    def fetch(
+        self, offset: int | SizeSuffix, size: int | SizeSuffix, extra: Any
+    ) -> Future[FilePart]:
+        if isinstance(offset, SizeSuffix):
+            offset = offset.as_int()
+        if isinstance(size, SizeSuffix):
+            size = size.as_int()
+
+        def task() -> FilePart:
+            from rclone_api.util import random_str
+
+            try:
+                range = Range(offset, offset + size)
+                dst = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+                out = self.server.download(self.path, dst, range)
+                if isinstance(out, Exception):
+                    raise out
+                return FilePart(payload=dst, extra=extra)
+            finally:
+                self.semaphore.release()
+
+        self.semaphore.acquire()
+        fut = self.executor.submit(task)
+        return fut
+
+    def shutdown(self) -> None:
+        self.executor.shutdown(wait=True)

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/profile/mount_copy_bytes.py

@@ -142,7 +142,7 @@ def _run_profile(
         filepart_or_err = future.result()
         if isinstance(filepart_or_err, Exception):
             assert False, f"Error: {filepart_or_err}"
-        filepart_or_err.close()
+        filepart_or_err.dispose()
     futures.clear()
 
     start = time.time()

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/rclone.py

@@ -11,10 +11,11 @@ import traceback
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from contextlib import contextmanager
+from dataclasses import dataclass
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Generator
+from typing import Any, Callable, Generator
 
 from rclone_api import Dir
 from rclone_api.completed_process import CompletedProcess
@@ -26,6 +27,7 @@ from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File, FileItem
 from rclone_api.group_files import group_files
+from rclone_api.http_server import HttpServer
 from rclone_api.mount import Mount, clean_mount, prepare_mount
 from rclone_api.mount_read_chunker import MultiMountFileChunker
 from rclone_api.process import Process
@@ -38,6 +40,7 @@ from rclone_api.s3.types import (
     S3UploadTarget,
 )
 from rclone_api.types import (
+    FilePart,
     ListingOption,
     ModTimeStrategy,
     Order,
@@ -242,7 +245,7 @@ class Rclone:
 
     def ls(
         self,
-        path: Dir | Remote | str,
+        path: Dir | Remote | str | None = None,
         max_depth: int | None = None,
         glob: str | None = None,
         order: Order = Order.NORMAL,
@@ -258,6 +261,15 @@ class Rclone:
             List of File objects found at the path
         """
 
+        if path is None:
+            # list remotes instead
+            list_remotes: list[Remote] = self.listremotes()
+            dirs: list[Dir] = [Dir(remote) for remote in list_remotes]
+            for d in dirs:
+                d.path.path = ""
+            rpaths = [d.path for d in dirs]
+            return DirListing(rpaths)
+
         if isinstance(path, str):
             path = Dir(
                 to_path(path, self)
@@ -786,14 +798,33 @@ class Rclone:
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
         mount_log: Path | None = None,
+        use_http_fetcher: bool = True,  # else use mount fetcher
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        from rclone_api.http_server import HttpFetcher, HttpServer
         from rclone_api.s3.api import S3Client
         from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, split_s3_path
 
-        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
+        src_path = Path(src)
+        name = src_path.name
+        src_parent_path = Path(src).parent.as_posix()
+
+        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        target_size = SizeSuffix(size_result.total_size)
+
         chunk_size = chunk_size or SizeSuffix("64M")
+        MAX_CHUNKS = 10000
+        min_chunk_size = SizeSuffix(size_result.total_size // (MAX_CHUNKS - 1))
+        if min_chunk_size > chunk_size:
+            warnings.warn(
+                f"Chunk size {chunk_size} is too small for file size {size_result.total_size}, setting to {min_chunk_size}"
+            )
+            chunk_size = SizeSuffix(min_chunk_size)
+
+        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
+
+        # BEGIN MOUNT SPECIFIC CONFIG
         unit_chunk_size = chunk_size / read_threads
         tmp_mount_dir = self._get_tmp_mount_dir()
         vfs_read_chunk_size = unit_chunk_size
@@ -818,18 +849,16 @@ class Rclone:
         # --vfs-cache-max-size
         other_args += ["--vfs-cache-max-size", vfs_disk_space_total_size.as_str()]
         mount_path = tmp_mount_dir / "RCLONE_API_DYNAMIC_MOUNT"
-        src_path = Path(src)
-        name = src_path.name
+        ## END MOUNT SPECIFIC CONFIG
 
-        src_parent_path = Path(src).parent.as_posix()
-        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        # size_result: SizeResult = self.size_files(os.path.dirname(src), [name])
 
-        target_size = SizeSuffix(size_result.total_size)
         if target_size < SizeSuffix("5M"):
             # fallback to normal copy
             completed_proc = self.copy_to(src, dst, check=True)
             if completed_proc.ok:
                 return MultiUploadResult.UPLOADED_FRESH
+
         if size_result.total_size <= 0:
             raise ValueError(
                 f"File {src} has size {size_result.total_size}, is this a directory?"
@@ -882,18 +911,46 @@ class Rclone:
             endpoint_url=section.endpoint(),
         )
 
-        chunk_fetcher: MultiMountFileChunker = self.get_multi_mount_file_chunker(
-            src=src_path.as_posix(),
-            chunk_size=chunk_size,
-            threads=read_threads,
-            mount_log=mount_log,
-            direct_io=True,
-        )
+        @dataclass
+        class Fetcher:
+            fetch: Callable[[int, int, Any], Future[FilePart]]
+            shutdown: Callable[[], None]
+
+        def get_fetcher() -> Fetcher:
+            if use_http_fetcher:
+                import random
 
+                port = random.randint(10000, 20000)
+                http_server: HttpServer = self.serve_http(
+                    src=src_path.parent.as_posix(), addr=f"localhost:{port}"
+                )
+                chunk_fetcher: HttpFetcher = http_server.get_fetcher(
+                    path=src_path.name,
+                    n_threads=read_threads,
+                )
+                # return chunk_fetcher.fetch
+                return Fetcher(fetch=chunk_fetcher.fetch, shutdown=http_server.shutdown)
+            else:
+                # Use the mount fetcher, which relies on FUSE which has problems in Docker/Windows/MacOS
+                mount_fetcher: MultiMountFileChunker = (
+                    self.get_multi_mount_file_chunker(
+                        src=src_path.as_posix(),
+                        chunk_size=chunk_size,
+                        threads=read_threads,
+                        mount_log=mount_log,
+                        direct_io=True,
+                    )
+                )
+                # return chunk_fetcher.fetch
+                return Fetcher(
+                    fetch=mount_fetcher.fetch, shutdown=mount_fetcher.shutdown
+                )
+
+        fetcher = get_fetcher()
         client = S3Client(s3_creds)
         upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
             chunk_size=chunk_size.as_int(),
-            chunk_fetcher=chunk_fetcher.fetch,
+            chunk_fetcher=fetcher.fetch,
             max_write_threads=write_threads,
             retries=retries,
             resume_path_json=save_state_json,
@@ -927,7 +984,8 @@ class Rclone:
             traceback.print_exc()
             raise
         finally:
-            chunk_fetcher.shutdown()
+            fetcher.shutdown()
+            fetcher.shutdown()
 
     def get_multi_mount_file_chunker(
         self,
@@ -1095,7 +1153,7 @@ class Rclone:
                 shutil.move(payload, outfile)
                 return bytes(0)
             finally:
-                fp.close()
+                fp.dispose()
 
         except Exception as e:
             warnings.warn(f"Error copying bytes: {e}")
@@ -1334,6 +1392,31 @@ class Rclone:
             raise ValueError("NFS serve process failed to start")
         return proc
 
+    def serve_http(
+        self,
+        src: str,
+        addr: str = "localhost:8080",
+        other_args: list[str] | None = None,
+    ) -> HttpServer:
+        """Serve a remote or directory via HTTP.
+
+        Args:
+            src: Remote or directory to serve
+            addr: Network address and port to serve on (default: localhost:8080)
+        """
+        _, subpath = src.split(":", 1)  # might not work on local paths.
+        cmd_list: list[str] = ["serve", "http", "--addr", addr, src]
+        if other_args:
+            cmd_list += other_args
+        proc = self._launch_process(cmd_list)
+        time.sleep(2)
+        if proc.poll() is not None:
+            raise ValueError("HTTP serve process failed to start")
+        out: HttpServer = HttpServer(
+            url=f"http://{addr}", subpath=subpath, process=proc
+        )
+        return out
+
     def size_files(
         self,
         src: str,

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/remote.py

@@ -16,3 +16,6 @@ class Remote:
 
     def __str__(self) -> str:
         return f"{self.name}:"
+
+    def __repr__(self) -> str:
+        return f"Remote({self.name!r})"

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/s3/upload_file_multipart.py

@@ -25,31 +25,39 @@ _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
 
 
 def upload_task(
-    info: UploadInfo, chunk: bytes, part_number: int, retries: int
+    info: UploadInfo, chunk: FilePart, part_number: int, retries: int
 ) -> FinishedPiece:
-    assert len(chunk) > 0
+    file_or_err: Path | Exception = chunk.get_file()
+    if isinstance(file_or_err, Exception):
+        raise file_or_err
+    file: Path = file_or_err
+    size = os.path.getsize(file)
     retries = retries + 1  # Add one for the initial attempt
     for retry in range(retries):
         try:
             if retry > 0:
                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
             locked_print(
-                f"Uploading part {part_number} for {info.src_file_path} of size {len(chunk)}"
-            )
-            part = info.s3_client.upload_part(
-                Bucket=info.bucket_name,
-                Key=info.object_name,
-                PartNumber=part_number,
-                UploadId=info.upload_id,
-                Body=chunk,
-            )
-            out: FinishedPiece = FinishedPiece(
-                etag=part["ETag"], part_number=part_number
+                f"Uploading part {part_number} for {info.src_file_path} of size {size}"
             )
+
+            with open(file, "rb") as f:
+                part = info.s3_client.upload_part(
+                    Bucket=info.bucket_name,
+                    Key=info.object_name,
+                    PartNumber=part_number,
+                    UploadId=info.upload_id,
+                    Body=f,
+                )
+                out: FinishedPiece = FinishedPiece(
+                    etag=part["ETag"], part_number=part_number
+                )
+            chunk.dispose()
             return out
         except Exception as e:
             if retry == retries - 1:
                 locked_print(f"Error uploading part {part_number}: {e}")
+                chunk.dispose()
                 raise e
             else:
                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
@@ -72,7 +80,7 @@ def handle_upload(
 
         part: FinishedPiece = upload_task(
             info=upload_info,
-            chunk=fp.load(),
+            chunk=fp,
             part_number=part_number,
             retries=upload_info.retries,
         )
@@ -83,7 +91,7 @@ def handle_upload(
         warnings.warn(msg)
         return e
     finally:
-        fp.close()
+        fp.dispose()
 
 
 def prepare_upload_file_multipart(

{rclone_api-1.3.19 → rclone_api-1.3.22}/src/rclone_api/types.py

@@ -289,7 +289,7 @@ atexit.register(_on_exit_cleanup)
 
 
 class FilePart:
-    def __init__(self, payload: bytes | Exception, extra: Any) -> None:
+    def __init__(self, payload: Path | bytes | Exception, extra: Any) -> None:
         from rclone_api.util import random_str
 
         self.extra = extra
@@ -298,12 +298,18 @@ class FilePart:
         if isinstance(payload, Exception):
             self.payload = payload
             return
-        self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
-        with _TMP_DIR_ACCESS_LOCK:
-            if not self.payload.parent.exists():
-                self.payload.parent.mkdir(parents=True, exist_ok=True)
-            self.payload.write_bytes(payload)
-        _add_for_cleanup(self.payload)
+        if isinstance(payload, bytes):
+            self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+            with _TMP_DIR_ACCESS_LOCK:
+                if not self.payload.parent.exists():
+                    self.payload.parent.mkdir(parents=True, exist_ok=True)
+                self.payload.write_bytes(payload)
+            _add_for_cleanup(self.payload)
+        if isinstance(payload, Path):
+            self.payload = payload
+
+    def get_file(self) -> Path | Exception:
+        return self.payload
 
     @property
     def size(self) -> int:
@@ -337,7 +343,7 @@ class FilePart:
     def is_error(self) -> bool:
         return isinstance(self.payload, Exception)
 
-    def close(self) -> None:
+    def dispose(self) -> None:
         with self._lock:
             if isinstance(self.payload, Exception):
                 warnings.warn(
@@ -352,4 +358,4 @@ class FilePart:
                     warnings.warn(f"Cannot close file part because of error: {e}")
 
     def __del__(self):
-        self.close()
+        self.dispose()