rclone-api 1.3.27__py2.py3-none-any.whl → 1.4.1__py2.py3-none-any.whl

rclone_api/{rclone.py → rclone_impl.py}

@@ -4,32 +4,31 @@ Unit test file.
 
 import os
 import random
-import shutil
 import subprocess
 import time
 import traceback
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
-from contextlib import contextmanager
-from dataclasses import dataclass
+from datetime import datetime
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Any, Callable, Generator
+from typing import Generator
 
 from rclone_api import Dir
 from rclone_api.completed_process import CompletedProcess
 from rclone_api.config import Config, Parsed, Section
 from rclone_api.convert import convert_to_filestr_list, convert_to_str
 from rclone_api.deprecated import deprecated
+from rclone_api.detail.walk import walk
 from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_process
 from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
-from rclone_api.file import File, FileItem
+from rclone_api.file import File
+from rclone_api.file_stream import FilesStream
 from rclone_api.group_files import group_files
 from rclone_api.http_server import HttpServer
 from rclone_api.mount import Mount, clean_mount, prepare_mount
-from rclone_api.mount_read_chunker import MultiMountFileChunker
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
@@ -40,20 +39,20 @@ from rclone_api.s3.types import (
     S3UploadTarget,
 )
 from rclone_api.types import (
-    FilePart,
     ListingOption,
     ModTimeStrategy,
     Order,
+    PartInfo,
     SizeResult,
     SizeSuffix,
 )
 from rclone_api.util import (
+    find_free_port,
     get_check,
     get_rclone_exe,
     get_verbose,
     to_path,
 )
-from rclone_api.walk import walk
 
 
 def rclone_verbose(verbose: bool | None) -> bool:
@@ -70,51 +69,7 @@ def _to_rclone_conf(config: Config | Path) -> Config:
         return config
 
 
-class FilesStream:
-
-    def __init__(self, path: str, process: Process) -> None:
-        self.path = path
-        self.process = process
-
-    def __enter__(self) -> "FilesStream":
-        self.process.__enter__()
-        return self
-
-    def __exit__(self, *exc_info):
-        self.process.__exit__(*exc_info)
-
-    def files(self) -> Generator[FileItem, None, None]:
-        line: bytes
-        for line in self.process.stdout:
-            linestr: str = line.decode("utf-8").strip()
-            if linestr.startswith("["):
-                continue
-            if linestr.endswith(","):
-                linestr = linestr[:-1]
-            if linestr.endswith("]"):
-                continue
-            fileitem: FileItem | None = FileItem.from_json_str(self.path, linestr)
-            if fileitem is None:
-                continue
-            yield fileitem
-
-    def files_paged(
-        self, page_size: int = 1000
-    ) -> Generator[list[FileItem], None, None]:
-        page: list[FileItem] = []
-        for fileitem in self.files():
-            page.append(fileitem)
-            if len(page) >= page_size:
-                yield page
-                page = []
-        if len(page) > 0:
-            yield page
-
-    def __iter__(self) -> Generator[FileItem, None, None]:
-        return self.files()
-
-
-class Rclone:
+class RcloneImpl:
     def __init__(
         self, rclone_conf: Path | Config, rclone_exe: Path | None = None
     ) -> None:
@@ -309,6 +264,46 @@ class Rclone:
             random.shuffle(paths)
         return DirListing(paths)
 
+    def print(self, path: str) -> Exception | None:
+        """Print the contents of a file."""
+        try:
+            text_or_err = self.read_text(path)
+            if isinstance(text_or_err, Exception):
+                return text_or_err
+            print(text_or_err)
+        except Exception as e:
+            return e
+        return None
+
+    def stat(self, src: str) -> File | Exception:
+        """Get the status of a file or directory."""
+        dirlist: DirListing = self.ls(src)
+        if len(dirlist.files) == 0:
+            # raise FileNotFoundError(f"File not found: {src}")
+            return FileNotFoundError(f"File not found: {src}")
+        try:
+            file: File = dirlist.files[0]
+            return file
+        except Exception as e:
+            return e
+
+    def modtime(self, src: str) -> str | Exception:
+        """Get the modification time of a file or directory."""
+        try:
+            file: File | Exception = self.stat(src)
+            if isinstance(file, Exception):
+                return file
+            return file.mod_time()
+        except Exception as e:
+            return e
+
+    def modtime_dt(self, src: str) -> datetime | Exception:
+        """Get the modification time of a file or directory."""
+        modtime: str | Exception = self.modtime(src)
+        if isinstance(modtime, Exception):
+            return modtime
+        return datetime.fromisoformat(modtime)
+
     def listremotes(self) -> list[Remote]:
         cmd = ["listremotes"]
         cp = self._run(cmd)
@@ -476,7 +471,7 @@ class Rclone:
         verbose = get_verbose(verbose)
         src = src if isinstance(src, str) else str(src.path)
         dst = dst if isinstance(dst, str) else str(dst.path)
-        cmd_list: list[str] = ["copyto", src, dst]
+        cmd_list: list[str] = ["copyto", src, dst, "--s3-no-check-bucket"]
         if other_args is not None:
             cmd_list += other_args
         cp = self._run(cmd_list, check=check)
@@ -511,6 +506,7 @@ class Rclone:
         low_level_retries = low_level_retries or 10
         retries = retries or 3
         other_args = other_args or []
+        other_args.append("--s3-no-check-bucket")
         checkers = checkers or 1000
         transfers = transfers or 32
         verbose = get_verbose(verbose)
@@ -666,6 +662,7 @@ class Rclone:
         cmd_list += ["--checkers", str(checkers)]
         cmd_list += ["--transfers", str(transfers)]
         cmd_list += ["--low-level-retries", str(low_level_retries)]
+        cmd_list.append("--s3-no-check-bucket")
         if multi_thread_streams is not None:
             cmd_list += ["--multi-thread-streams", str(multi_thread_streams)]
         if other_args:
@@ -786,6 +783,85 @@ class Rclone:
         except subprocess.CalledProcessError:
             return False
 
+    def copy_file_parts(
+        self,
+        src: str,  # src:/Bucket/path/myfile.large.zst
+        dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
+        part_infos: list[PartInfo] | None = None,
+        threads: int = 1,
+    ) -> Exception | None:
+        """Copy parts of a file from source to destination."""
+        from rclone_api.detail.copy_file_parts import copy_file_parts
+
+        out = copy_file_parts(
+            self=self,
+            src=src,
+            dst_dir=dst_dir,
+            part_infos=part_infos,
+            threads=threads,
+        )
+        return out
+
+    def write_text(
+        self,
+        dst: str,
+        text: str,
+    ) -> Exception | None:
+        """Write text to a file."""
+        data = text.encode("utf-8")
+        return self.write_bytes(dst=dst, data=data)
+
+    def write_bytes(
+        self,
+        dst: str,
+        data: bytes,
+    ) -> Exception | None:
+        """Write bytes to a file."""
+        with TemporaryDirectory() as tmpdir:
+            tmpfile = Path(tmpdir) / "file.bin"
+            tmpfile.write_bytes(data)
+            completed_proc = self.copy_to(str(tmpfile), dst, check=True)
+            if completed_proc.returncode != 0:
+                return Exception(f"Failed to write bytes to {dst}", completed_proc)
+        return None
+
+    def read_bytes(self, src: str) -> bytes | Exception:
+        """Read bytes from a file."""
+        with TemporaryDirectory() as tmpdir:
+            tmpfile = Path(tmpdir) / "file.bin"
+            completed_proc = self.copy_to(src, str(tmpfile), check=True)
+            if completed_proc.returncode != 0:
+                return Exception(f"Failed to read bytes from {src}", completed_proc)
+
+            if not tmpfile.exists():
+                return Exception(f"Failed to read bytes from {src}, file not found")
+            try:
+                return tmpfile.read_bytes()
+            except Exception as e:
+                return Exception(f"Failed to read bytes from {src}", e)
+
+    def read_text(self, src: str) -> str | Exception:
+        """Read text from a file."""
+        data = self.read_bytes(src)
+        if isinstance(data, Exception):
+            return data
+        try:
+            return data.decode("utf-8")
+        except UnicodeDecodeError as e:
+            return Exception(f"Failed to decode text from {src}", e)
+
+    def size_file(self, src: str) -> SizeSuffix | Exception:
+        """Get the size of a file or directory."""
+        src_parent = os.path.dirname(src)
+        src_name = os.path.basename(src)
+        out: SizeResult = self.size_files(src_parent, [src_name])
+        one_file = len(out.file_sizes) == 1
+        if not one_file:
+            return Exception(
+                f"More than one result returned, is this is a directory? {out}"
+            )
+        return SizeSuffix(out.total_size)
+
     def copy_file_resumable_s3(
         self,
         src: str,
@@ -797,8 +873,7 @@ class Rclone:
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
-        mount_log: Path | None = None,
-        use_http_fetcher: bool = True,  # else use mount fetcher
+        backend_log: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.http_server import HttpFetcher, HttpServer
@@ -822,37 +897,6 @@ class Rclone:
             )
             chunk_size = SizeSuffix(min_chunk_size)
 
-        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
-
-        # BEGIN MOUNT SPECIFIC CONFIG
-        unit_chunk_size = chunk_size / read_threads
-        tmp_mount_dir = self._get_tmp_mount_dir()
-        vfs_read_chunk_size = unit_chunk_size
-        vfs_read_chunk_size_limit = chunk_size
-        vfs_read_chunk_streams = read_threads
-        vfs_disk_space_total_size = chunk_size
-        # assert (
-        #     chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
-        # ), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
-        other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
-        other_args += [
-            "--vfs-read-chunk-size-limit",
-            vfs_read_chunk_size_limit.as_str(),
-        ]
-        other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
-        other_args += [
-            "--vfs-disk-space-total-size",
-            vfs_disk_space_total_size.as_str(),
-        ]
-        other_args += ["--read-only"]
-        other_args += ["--direct-io"]
-        # --vfs-cache-max-size
-        other_args += ["--vfs-cache-max-size", vfs_disk_space_total_size.as_str()]
-        mount_path = tmp_mount_dir / "RCLONE_API_DYNAMIC_MOUNT"
-        ## END MOUNT SPECIFIC CONFIG
-
-        # size_result: SizeResult = self.size_files(os.path.dirname(src), [name])
-
         if target_size < SizeSuffix("5M"):
             # fallback to normal copy
             completed_proc = self.copy_to(src, dst, check=True)
@@ -911,63 +955,34 @@ class Rclone:
             endpoint_url=section.endpoint(),
         )
 
-        @dataclass
-        class Fetcher:
-            fetch: Callable[[int, int, Any], Future[FilePart]]
-            shutdown: Callable[[], None]
-
-        def get_fetcher() -> Fetcher:
-            if use_http_fetcher:
-                import random
-
-                port = random.randint(10000, 20000)
-                http_server: HttpServer = self.serve_http(
-                    src=src_path.parent.as_posix(), addr=f"localhost:{port}"
-                )
-                chunk_fetcher: HttpFetcher = http_server.get_fetcher(
-                    path=src_path.name,
-                    n_threads=read_threads,
-                )
-                # return chunk_fetcher.fetch
-                return Fetcher(fetch=chunk_fetcher.fetch, shutdown=http_server.shutdown)
-            else:
-                # Use the mount fetcher, which relies on FUSE which has problems in Docker/Windows/MacOS
-                mount_fetcher: MultiMountFileChunker = (
-                    self.get_multi_mount_file_chunker(
-                        src=src_path.as_posix(),
-                        chunk_size=chunk_size,
-                        threads=read_threads,
-                        mount_log=mount_log,
-                        direct_io=True,
-                    )
-                )
-                # return chunk_fetcher.fetch
-                return Fetcher(
-                    fetch=mount_fetcher.fetch, shutdown=mount_fetcher.shutdown
-                )
+        port = random.randint(10000, 20000)
+        http_server: HttpServer = self.serve_http(
+            src=src_path.parent.as_posix(),
+            addr=f"localhost:{port}",
+            serve_http_log=backend_log,
+        )
+        chunk_fetcher: HttpFetcher = http_server.get_fetcher(
+            path=src_path.name,
+            n_threads=read_threads,
+        )
 
-        fetcher = get_fetcher()
         client = S3Client(s3_creds)
         upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
             chunk_size=chunk_size.as_int(),
-            chunk_fetcher=fetcher.fetch,
+            chunk_fetcher=chunk_fetcher.bytes_fetcher,
             max_write_threads=write_threads,
             retries=retries,
             resume_path_json=save_state_json,
             max_chunks_before_suspension=max_chunks_before_suspension,
         )
 
-        src_file = mount_path / name
-
         print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
         print(f"Source: {src_path}")
         print(f"bucket_name: {bucket_name}")
         print(f"upload_config: {upload_config}")
 
-        # get the file size
-
         upload_target = S3UploadTarget(
-            src_file=src_file,
+            src_file=src_path,
             src_file_size=size_result.total_size,
             bucket_name=bucket_name,
             s3_key=s3_key,
@@ -984,106 +999,7 @@ class Rclone:
             traceback.print_exc()
             raise
         finally:
-            fetcher.shutdown()
-            fetcher.shutdown()
-
-    def get_multi_mount_file_chunker(
-        self,
-        src: str,
-        chunk_size: SizeSuffix,
-        threads: int,
-        mount_log: Path | None,
-        direct_io: bool,
-    ) -> MultiMountFileChunker:
-        from rclone_api.util import random_str
-
-        mounts: list[Mount] = []
-        vfs_read_chunk_size = chunk_size
-        vfs_read_chunk_size_limit = chunk_size
-        vfs_read_chunk_streams = 0
-        vfs_disk_space_total_size = chunk_size
-        other_args: list[str] = []
-        other_args += ["--no-modtime"]
-        other_args += ["--vfs-read-chunk-size", vfs_read_chunk_size.as_str()]
-        other_args += [
-            "--vfs-read-chunk-size-limit",
-            vfs_read_chunk_size_limit.as_str(),
-        ]
-        other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
-        other_args += [
-            "--vfs-disk-space-total-size",
-            vfs_disk_space_total_size.as_str(),
-        ]
-        other_args += ["--read-only"]
-        if direct_io:
-            other_args += ["--direct-io"]
-
-        base_mount_dir = self._get_tmp_mount_dir()
-        base_cache_dir = self._get_cache_dir()
-
-        filename = Path(src).name
-        with ThreadPoolExecutor(max_workers=threads) as executor:
-            futures: list[Future] = []
-            try:
-                for i in range(threads):
-                    tmp_mnts = base_mount_dir / random_str(12)
-                    verbose = mount_log is not None
-
-                    src_parent_path = Path(src).parent.as_posix()
-                    cache_dir = base_cache_dir / random_str(12)
-
-                    def task(
-                        src_parent_path=src_parent_path,
-                        tmp_mnts=tmp_mnts,
-                        cache_dir=cache_dir,
-                    ):
-                        clean_mount(tmp_mnts, verbose=verbose)
-                        prepare_mount(tmp_mnts, verbose=verbose)
-                        return self.mount(
-                            src=src_parent_path,
-                            outdir=tmp_mnts,
-                            allow_writes=False,
-                            use_links=True,
-                            vfs_cache_mode="minimal",
-                            verbose=False,
-                            cache_dir=cache_dir,
-                            cache_dir_delete_on_exit=True,
-                            log=mount_log,
-                            other_args=other_args,
-                        )
-
-                    futures.append(executor.submit(task))
-                mount_errors: list[Exception] = []
-                for fut in futures:
-                    try:
-                        mount = fut.result()
-                        mounts.append(mount)
-                    except Exception as er:
-                        warnings.warn(f"Error mounting: {er}")
-                        mount_errors.append(er)
-                if mount_errors:
-                    warnings.warn(f"Error mounting: {mount_errors}")
-                    raise Exception(mount_errors)
-            except Exception:
-                for mount in mounts:
-                    mount.close()
-                raise
-
-        src_path: Path = Path(src)
-        src_parent_path = src_path.parent.as_posix()
-        name = src_path.name
-        size_result: SizeResult = self.size_files(src_parent_path, [name])
-        filesize = size_result.total_size
-
-        executor = ThreadPoolExecutor(max_workers=threads)
-        filechunker: MultiMountFileChunker = MultiMountFileChunker(
-            filename=filename,
-            filesize=filesize,
-            mounts=mounts,
-            executor=executor,
-            verbose=mount_log is not None,
-        )
-        return filechunker
+            chunk_fetcher.shutdown()
 
     def copy_bytes(
         self,
@@ -1114,56 +1030,6 @@ class Rclone:
         except subprocess.CalledProcessError as e:
             return e
 
-    def copy_bytes_mount(
-        self,
-        src: str,
-        offset: int | SizeSuffix,
-        length: int | SizeSuffix,
-        chunk_size: SizeSuffix,
-        max_threads: int = 1,
-        # If outfile is supplied then bytes are written to this file and success returns bytes(0)
-        outfile: Path | None = None,
-        mount_log: Path | None = None,
-        direct_io: bool = True,
-    ) -> bytes | Exception:
-        """Copy a slice of bytes from the src file to dst. Parallelism is achieved through multiple mounted files."""
-        from rclone_api.types import FilePart
-
-        offset = SizeSuffix(offset).as_int()
-        length = SizeSuffix(length).as_int()
-        # determine number of threads from chunk size
-        threads = max(1, min(max_threads, length // chunk_size.as_int()))
-        # todo - implement max threads.
-        filechunker = self.get_multi_mount_file_chunker(
-            src=src,
-            chunk_size=chunk_size,
-            threads=threads,
-            mount_log=mount_log,
-            direct_io=direct_io,
-        )
-        try:
-            fut = filechunker.fetch(offset, length, extra=None)
-            fp: FilePart = fut.result()
-            payload = fp.payload
-            if isinstance(payload, Exception):
-                return payload
-            try:
-                if outfile is None:
-                    return payload.read_bytes()
-                shutil.move(payload, outfile)
-                return bytes(0)
-            finally:
-                fp.dispose()
-
-        except Exception as e:
-            warnings.warn(f"Error copying bytes: {e}")
-            return e
-        finally:
-            try:
-                filechunker.shutdown()
-            except Exception as e:
-                warnings.warn(f"Error closing filechunker: {e}")
-
     def copy_dir(
         self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
     ) -> CompletedProcess:
@@ -1171,7 +1037,7 @@ class Rclone:
         # convert src to str, also dst
         src = convert_to_str(src)
         dst = convert_to_str(dst)
-        cmd_list: list[str] = ["copy", src, dst]
+        cmd_list: list[str] = ["copy", src, dst, "--s3-no-check-bucket"]
         if args is not None:
             cmd_list += args
         cp = self._run(cmd_list)
@@ -1181,7 +1047,7 @@ class Rclone:
         self, src: Remote, dst: Remote, args: list[str] | None = None
     ) -> CompletedProcess:
         """Copy a remote to another remote."""
-        cmd_list: list[str] = ["copy", str(src), str(dst)]
+        cmd_list: list[str] = ["copy", str(src), str(dst), "--s3-no-check-bucket"]
         if args is not None:
             cmd_list += args
         # return self._run(cmd_list)
@@ -1251,45 +1117,6 @@ class Rclone:
         )
         return mount
 
-    @contextmanager
-    def scoped_mount(
-        self,
-        src: Remote | Dir | str,
-        outdir: Path,
-        allow_writes: bool | None = None,
-        use_links: bool | None = None,
-        vfs_cache_mode: str | None = None,
-        verbose: bool | None = None,
-        log: Path | None = None,
-        cache_dir: Path | None = None,
-        cache_dir_delete_on_exit: bool | None = None,
-        other_args: list[str] | None = None,
-    ) -> Generator[Mount, None, None]:
-        """Like mount, but can be used in a context manager."""
-        error_happened = False
-        mount: Mount = self.mount(
-            src,
-            outdir,
-            allow_writes=allow_writes,
-            use_links=use_links,
-            vfs_cache_mode=vfs_cache_mode,
-            verbose=verbose,
-            cache_dir=cache_dir,
-            cache_dir_delete_on_exit=cache_dir_delete_on_exit,
-            log=log,
-            other_args=other_args,
-        )
-        try:
-            yield mount
-        except Exception as e:
-            error_happened = True
-            stack_trace = traceback.format_exc()
-            warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
-            raise
-        finally:
-            if not error_happened or (not allow_writes):
-                mount.close()
-
     # Settings optimized for s3.
     def mount_s3(
         self,
@@ -1395,7 +1222,8 @@ class Rclone:
     def serve_http(
         self,
         src: str,
-        addr: str = "localhost:8080",
+        addr: str | None = None,
+        serve_http_log: Path | None = None,
         other_args: list[str] | None = None,
     ) -> HttpServer:
         """Serve a remote or directory via HTTP.
@@ -1404,11 +1232,15 @@ class Rclone:
             src: Remote or directory to serve
             addr: Network address and port to serve on (default: localhost:8080)
         """
+        addr = addr or f"localhost:{find_free_port()}"
         _, subpath = src.split(":", 1)  # might not work on local paths.
         cmd_list: list[str] = ["serve", "http", "--addr", addr, src]
+        if serve_http_log:
+            cmd_list += ["--log-file", str(serve_http_log)]
+            cmd_list += ["-vvvv"]
         if other_args:
             cmd_list += other_args
-        proc = self._launch_process(cmd_list)
+        proc = self._launch_process(cmd_list, log=serve_http_log)
         time.sleep(2)
         if proc.poll() is not None:
             raise ValueError("HTTP serve process failed to start")

rclone_api/remote.py

@@ -5,14 +5,14 @@ class Remote:
     """Remote (root) directory."""
 
     def __init__(self, name: str, rclone: Any) -> None:
-        from rclone_api.rclone import Rclone
+        from rclone_api.rclone_impl import RcloneImpl
 
         if ":" in name:
             raise ValueError("Remote name cannot contain ':'")
 
-        assert isinstance(rclone, Rclone)
+        assert isinstance(rclone, RcloneImpl)
         self.name = name
-        self.rclone: Rclone = rclone
+        self.rclone: RcloneImpl = rclone
 
     def __str__(self) -> str:
         return f"{self.name}:"

rclone_api/rpath.py

@@ -1,4 +1,5 @@
 import json
+from datetime import datetime
 from typing import Any
 
 from rclone_api.remote import Remote
@@ -17,8 +18,10 @@ class RPath:
         mod_time: str,
         is_dir: bool,
     ) -> None:
-        from rclone_api.rclone import Rclone
+        from rclone_api.rclone_impl import RcloneImpl
 
+        if path.endswith("/"):
+            path = path[:-1]
         self.remote = remote
         self.path = path
         self.name = name
@@ -26,13 +29,17 @@ class RPath:
         self.mime_type = mime_type
         self.mod_time = mod_time
         self.is_dir = is_dir
-        self.rclone: Rclone | None = None
+        self.rclone: RcloneImpl | None = None
+
+    def mod_time_dt(self) -> datetime:
+        """Return the modification time as a datetime object."""
+        return datetime.fromisoformat(self.mod_time)
 
     def set_rclone(self, rclone: Any) -> None:
         """Set the rclone object."""
-        from rclone_api.rclone import Rclone
+        from rclone_api.rclone_impl import RcloneImpl
 
-        assert isinstance(rclone, Rclone)
+        assert isinstance(rclone, RcloneImpl)
         self.rclone = rclone
 
     @staticmethod