rclone-api 1.4.19__py2.py3-none-any.whl → 1.4.22__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -428,11 +428,16 @@ class Rclone:
         src: str,  # src:/Bucket/path/myfile.large.zst
         dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/part.{part_number:05d}.start-end
         part_infos: list[PartInfo] | None = None,
-        threads: int = 1,  # Number of reader and writer threads to use
+        upload_threads: int = 8,  # Number of reader and writer threads to use
+        merge_threads: int = 4,  # Number of threads to use for merging the parts
     ) -> Exception | None:
         """Copy a file in parts."""
         return self.impl.copy_file_parts(
-            src=src, dst_dir=dst_dir, part_infos=part_infos, threads=threads
+            src=src,
+            dst_dir=dst_dir,
+            part_infos=part_infos,
+            upload_threads=upload_threads,
+            merge_threads=merge_threads,
         )
 
     def mount(
@@ -496,7 +501,7 @@ class Rclone:
         other_args: list[str] | None = None,
         check: bool | None = False,
         verbose: bool | None = None,
-    ) -> SizeResult:
+    ) -> SizeResult | Exception:
         """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
         return self.impl.size_files(
             src=src,

rclone_api/cmd/copy_large_s3.py

@@ -90,8 +90,6 @@ def main() -> int:
     err: Exception | None = rclone.copy_file_parts(
         src=args.src,
         dst_dir=args.dst,
-        threads=8,
-        # verbose=args.verbose,
     )
     if err is not None:
         print(f"Error: {err}")

rclone_api/cmd/copy_large_s3_finish.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from rclone_api import Rclone
-from rclone_api.s3.s3_multipart_uploader_by_copy import (
+from rclone_api.s3.multipart.upload_parts_server_side_merge import (
     s3_server_side_multi_part_merge,
 )
 
@@ -57,7 +57,7 @@ def main() -> int:
     rclone = Rclone(rclone_conf=args.config_path)
     info_path = _get_info_path(src=args.src)
     s3_server_side_multi_part_merge(
-        rclone=rclone.impl, info_path=info_path, max_workers=1
+        rclone=rclone.impl, info_path=info_path, max_workers=5
     )
     return 0
 

rclone_api/detail/copy_file_parts_resumable.py

@@ -0,0 +1,42 @@
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.types import (
+    PartInfo,
+)
+
+
+def copy_file_parts_resumable(
+    self: RcloneImpl,
+    src: str,  # src:/Bucket/path/myfile.large.zst
+    dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
+    part_infos: list[PartInfo] | None = None,
+    upload_threads: int = 10,
+    merge_threads: int = 5,
+    verbose: bool | None = None,
+) -> Exception | None:
+    # _upload_parts
+    from rclone_api.s3.multipart.upload_parts_resumable import upload_parts_resumable
+    from rclone_api.s3.multipart.upload_parts_server_side_merge import (
+        s3_server_side_multi_part_merge,
+    )
+
+    if verbose is None:
+        verbose = self.get_verbose()
+
+    err: Exception | None = upload_parts_resumable(
+        self=self,
+        src=src,
+        dst_dir=dst_dir,
+        part_infos=part_infos,
+        threads=upload_threads,
+    )
+    if isinstance(err, Exception):
+        return err
+    if dst_dir.endswith("/"):
+        dst_dir = dst_dir[:-1]
+    dst_info = f"{dst_dir}/info.json"
+    err = s3_server_side_multi_part_merge(
+        rclone=self, info_path=dst_info, max_workers=merge_threads, verbose=verbose
+    )
+    if isinstance(err, Exception):
+        return err
+    return None

rclone_api/http_server.py

@@ -86,7 +86,10 @@ class HttpServer:
                                 assert response.is_closed
                     # print(f"Downloaded bytes {start}-{end} to {dst}")
                     if range:
-                        print(f"Downloaded bytes {range.start}-{range.end} to {dst}")
+                        length = range.end - range.start
+                        print(
+                            f"Downloaded bytes starting at {range.start} with size {length} to {dst}"
+                        )
                     else:
                         size = dst.stat().st_size
                         print(f"Downloaded {size} bytes to {dst}")

rclone_api/rclone_impl.py

@@ -455,6 +455,9 @@ class RcloneImpl:
         out = self._run(cmd)
         return CompletedProcess.from_subprocess(out)
 
+    def get_verbose(self) -> bool:
+        return get_verbose(None)
+
     def copy_to(
         self,
         src: File | str,
@@ -789,17 +792,21 @@ class RcloneImpl:
         src: str,  # src:/Bucket/path/myfile.large.zst
         dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
         part_infos: list[PartInfo] | None = None,
-        threads: int = 1,
+        upload_threads: int = 8,
+        merge_threads: int = 4,
     ) -> Exception | None:
         """Copy parts of a file from source to destination."""
-        from rclone_api.detail.copy_file_parts import copy_file_parts
+        from rclone_api.detail.copy_file_parts_resumable import (
+            copy_file_parts_resumable,
+        )
 
-        out = copy_file_parts(
+        out = copy_file_parts_resumable(
             self=self,
             src=src,
             dst_dir=dst_dir,
             part_infos=part_infos,
-            threads=threads,
+            upload_threads=upload_threads,
+            merge_threads=merge_threads,
         )
         return out
 
@@ -853,15 +860,25 @@ class RcloneImpl:
 
     def size_file(self, src: str) -> SizeSuffix | Exception:
         """Get the size of a file or directory."""
-        src_parent = os.path.dirname(src)
-        src_name = os.path.basename(src)
-        out: SizeResult = self.size_files(src_parent, [src_name])
-        one_file = len(out.file_sizes) == 1
-        if not one_file:
-            return Exception(
-                f"More than one result returned, is this is a directory? {out}"
-            )
-        return SizeSuffix(out.total_size)
+        # src_parent = os.path.dirname(src)
+        # src_name = os.path.basename(src)
+        # can't use this because it's only one file.
+        # out: SizeResult = self.size_files(src_parent, [src_name])
+        # one_file = len(out.file_sizes) == 1
+        # if not one_file:
+        #     return Exception(
+        #         f"More than one result returned, is this is a directory? {out}"
+        #     )
+        # return SizeSuffix(out.total_size)
+        dirlist: DirListing = self.ls(
+            src, listing_option=ListingOption.FILES_ONLY, max_depth=0
+        )
+        if len(dirlist.files) == 0:
+            return FileNotFoundError(f"File not found: {src}")
+        if len(dirlist.files) > 1:
+            return Exception(f"More than one file found: {src}")
+        file: File = dirlist.files[0]
+        return SizeSuffix(file.size)
 
     def get_s3_credentials(
         self, remote: str, verbose: bool | None = None
@@ -943,7 +960,9 @@ class RcloneImpl:
         name = src_path.name
         src_parent_path = Path(src).parent.as_posix()
 
-        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        size_result: SizeResult | Exception = self.size_files(src_parent_path, [name])
+        if isinstance(size_result, Exception):
+            raise size_result
         target_size = SizeSuffix(size_result.total_size)
 
         chunk_size = chunk_size or SizeSuffix("64M")
@@ -1286,10 +1305,18 @@ class RcloneImpl:
         other_args: list[str] | None = None,
         check: bool | None = False,
         verbose: bool | None = None,
-    ) -> SizeResult:
+    ) -> SizeResult | Exception:
         """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
         verbose = get_verbose(verbose)
         check = get_check(check)
+        if len(files) < 2:
+            tmp = self.size_file(files[0])
+            if isinstance(tmp, Exception):
+                return tmp
+            assert isinstance(tmp, SizeSuffix)
+            return SizeResult(
+                prefix=src, total_size=tmp.as_int(), file_sizes={files[0]: tmp.as_int()}
+            )
         if fast_list or (other_args and "--fast-list" in other_args):
             warnings.warn(
                 "It's not recommended to use --fast-list with size_files as this will perform poorly on large repositories since the entire repository has to be scanned."

rclone_api/s3/api.py

@@ -10,11 +10,11 @@ from rclone_api.s3.basic_ops import (
     upload_file,
 )
 from rclone_api.s3.create import S3Config, create_s3_client
-from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
-from rclone_api.s3.upload_file_multipart import (
+from rclone_api.s3.multipart.upload_parts_inline import (
     MultiUploadResult,
     upload_file_multipart,
 )
+from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
 
 _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 

rclone_api/s3/multipart/finished_piece.py

@@ -10,7 +10,17 @@ class FinishedPiece:
     etag: str
 
     def to_json(self) -> dict:
-        return {"part_number": self.part_number, "etag": self.etag}
+        # return {"part_number": self.part_number, "etag": self.etag}
+        # amazon s3 style dict
+        tag = self.etag
+        if not tag.startswith('"'):
+            tag = f'"{tag}"'
+        out = {"PartNumber": self.part_number, "ETag": self.etag}
+        return out
+
+    def __post_init__(self):
+        assert isinstance(self.part_number, int)
+        assert isinstance(self.etag, str)
 
     @staticmethod
     def to_json_array(
@@ -30,13 +40,21 @@ class FinishedPiece:
         # assert count_eos <= 1, "Only one EndOfStream should be present"
         if count_eos > 1:
             warnings.warn(f"Only one EndOfStream should be present, found {count_eos}")
-        return [p.to_json() for p in non_none]
+        out = [p.to_json() for p in non_none]
+        return out
 
     @staticmethod
     def from_json(json: dict | None) -> "FinishedPiece | EndOfStream":
         if json is None:
             return EndOfStream()
-        return FinishedPiece(**json)
+        part_number = json.get("PartNumber")
+        etag = json.get("ETag")
+        assert isinstance(etag, str)
+        # handle the double quotes around the etag
+        etag = etag.replace('"', "")
+        assert isinstance(part_number, int)
+        assert isinstance(etag, str)
+        return FinishedPiece(part_number=part_number, etag=etag)
 
     @staticmethod
     def from_json_array(json: dict) -> list["FinishedPiece"]:

rclone_api/s3/multipart/info_json.py

@@ -0,0 +1,239 @@
+import hashlib
+import json
+import os
+import warnings
+from datetime import datetime
+
+from rclone_api.dir_listing import DirListing
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.types import (
+    PartInfo,
+    SizeSuffix,
+)
+
+
+def _fetch_all_names(
+    self: RcloneImpl,
+    src: str,
+) -> list[str]:
+    dl: DirListing = self.ls(src)
+    files = dl.files
+    filenames: list[str] = [f.name for f in files]
+    filtered: list[str] = [f for f in filenames if f.startswith("part.")]
+    return filtered
+
+
+def _get_info_json(self: RcloneImpl, src: str | None, src_info: str) -> dict:
+    from rclone_api.file import File
+
+    data: dict
+    text: str
+    if src is None:
+        # just try to load the file
+        text_or_err = self.read_text(src_info)
+        if isinstance(text_or_err, Exception):
+            raise FileNotFoundError(f"Could not load {src_info}: {text_or_err}")
+        assert isinstance(text_or_err, str)
+        text = text_or_err
+        data = json.loads(text)
+        return data
+
+    src_stat: File | Exception = self.stat(src)
+    if isinstance(src_stat, Exception):
+        # just try to load the file
+        raise FileNotFoundError(f"Failed to stat {src}: {src_stat}")
+
+    now: datetime = datetime.now()
+    new_data = {
+        "new": True,
+        "created": now.isoformat(),
+        "src": src,
+        "src_modtime": src_stat.mod_time(),
+        "size": src_stat.size,
+        "chunksize": None,
+        "chunksize_int": None,
+        "first_part": None,
+        "last_part": None,
+        "hash": None,
+    }
+
+    text_or_err = self.read_text(src_info)
+    err: Exception | None = text_or_err if isinstance(text_or_err, Exception) else None
+    if isinstance(text_or_err, Exception):
+        warnings.warn(f"Failed to read {src_info}: {text_or_err}")
+        return new_data
+    assert isinstance(text_or_err, str)
+    text = text_or_err
+
+    if err is not None:
+        return new_data
+
+    try:
+        data = json.loads(text)
+        return data
+    except Exception as e:
+        warnings.warn(f"Failed to parse JSON: {e} at {src_info}")
+        return new_data
+
+
+def _save_info_json(self: RcloneImpl, src: str, data: dict) -> None:
+    data = data.copy()
+    data["new"] = False
+    # hash
+
+    h = hashlib.md5()
+    tmp = [
+        data.get("src"),
+        data.get("src_modtime"),
+        data.get("size"),
+        data.get("chunksize_int"),
+    ]
+    data_vals: list[str] = [str(v) for v in tmp]
+    str_data = "".join(data_vals)
+    h.update(str_data.encode("utf-8"))
+    data["hash"] = h.hexdigest()
+    json_str = json.dumps(data, indent=0)
+    self.write_text(dst=src, text=json_str)
+
+
+class InfoJson:
+    def __init__(self, rclone: RcloneImpl, src: str | None, src_info: str) -> None:
+        self.rclone = rclone
+        self.src = src
+        self.src_info = src_info
+        self.data: dict = {}
+
+    def load(self) -> bool:
+        """Returns true if the file exist and is now loaded."""
+        self.data = _get_info_json(self.rclone, self.src, self.src_info)
+        return not self.data.get("new", False)
+
+    def save(self) -> None:
+        _save_info_json(self.rclone, self.src_info, self.data)
+
+    def print(self) -> None:
+        self.rclone.print(self.src_info)
+
+    def fetch_all_finished(self) -> list[str]:
+        parent_path = os.path.dirname(self.src_info)
+        out = _fetch_all_names(self.rclone, parent_path)
+        return out
+
+    def fetch_all_finished_part_numbers(self) -> list[int]:
+        names = self.fetch_all_finished()
+        part_numbers = [int(name.split("_")[0].split(".")[1]) for name in names]
+        return part_numbers
+
+    @property
+    def parts_dir(self) -> str:
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
+
+    @property
+    def dst(self) -> str:
+        parts_dir = self.parts_dir
+        assert parts_dir.endswith("-parts")
+        out = parts_dir[:-6]
+        return out
+
+    @property
+    def dst_name(self) -> str:
+        return os.path.basename(self.dst)
+
+    def compute_all_parts(self) -> list[PartInfo] | Exception:
+        # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
+        # src_size, SizeSuffix("96MB")
+        try:
+
+            src_size = self.size
+            chunk_size = self.chunksize
+            assert isinstance(src_size, SizeSuffix)
+            assert isinstance(chunk_size, SizeSuffix)
+            first_part = self.data["first_part"]
+            last_part = self.data["last_part"]
+            full_part_infos: list[PartInfo] = PartInfo.split_parts(src_size, chunk_size)
+            return full_part_infos[first_part : last_part + 1]
+        except Exception as e:
+            return e
+
+    def compute_all_part_numbers(self) -> list[int] | Exception:
+        all_parts: list[PartInfo] | Exception = self.compute_all_parts()
+        if isinstance(all_parts, Exception):
+            raise all_parts
+
+        all_part_nums: list[int] = [p.part_number for p in all_parts]
+        return all_part_nums
+
+    def fetch_remaining_part_numbers(self) -> list[int] | Exception:
+        all_part_nums: list[int] | Exception = self.compute_all_part_numbers()
+        if isinstance(all_part_nums, Exception):
+            return all_part_nums
+        finished_part_nums: list[int] = self.fetch_all_finished_part_numbers()
+        remaining_part_nums: list[int] = list(
+            set(all_part_nums) - set(finished_part_nums)
+        )
+        return sorted(remaining_part_nums)
+
+    def fetch_is_done(self) -> bool:
+        remaining_part_nums: list[int] | Exception = self.fetch_remaining_part_numbers()
+        if isinstance(remaining_part_nums, Exception):
+            return False
+        return len(remaining_part_nums) == 0
+
+    @property
+    def new(self) -> bool:
+        return self.data.get("new", False)
+
+    @property
+    def chunksize(self) -> SizeSuffix | None:
+        chunksize_int: int | None = self.data.get("chunksize_int")
+        if chunksize_int is None:
+            return None
+        return SizeSuffix(chunksize_int)
+
+    @chunksize.setter
+    def chunksize(self, value: SizeSuffix) -> None:
+        self.data["chunksize"] = str(value)
+        self.data["chunksize_int"] = value.as_int()
+
+    @property
+    def src_modtime(self) -> datetime:
+        return datetime.fromisoformat(self.data["src_modtime"])
+
+    @src_modtime.setter
+    def src_modtime(self, value: datetime) -> None:
+        self.data["src_modtime"] = value.isoformat()
+
+    @property
+    def size(self) -> SizeSuffix:
+        return SizeSuffix(self.data["size"])
+
+    def _get_first_part(self) -> int | None:
+        return self.data.get("first_part")
+
+    def _set_first_part(self, value: int) -> None:
+        self.data["first_part"] = value
+
+    def _get_last_part(self) -> int | None:
+        return self.data.get("last_part")
+
+    def _set_last_part(self, value: int) -> None:
+        self.data["last_part"] = value
+
+    first_part: int | None = property(_get_first_part, _set_first_part)  # type: ignore
+    last_part: int | None = property(_get_last_part, _set_last_part)  # type: ignore
+
+    @property
+    def hash(self) -> str | None:
+        return self.data.get("hash")
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.data)
+
+    def __repr__(self):
+        return f"InfoJson({self.src}, {self.src_info}, {self.data})"
+
+    def __str__(self):
+        return self.to_json_str()

rclone_api/{detail/copy_file_parts.py → s3/multipart/upload_parts_resumable.py}

@@ -1,26 +1,22 @@
 import _thread
-import hashlib
-import json
+import atexit
 import os
+import shutil
 import threading
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
-from datetime import datetime
 from pathlib import Path
 
-from rclone_api import rclone_verbose
-from rclone_api.dir_listing import DirListing
 from rclone_api.http_server import HttpServer
 from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.s3.multipart.info_json import InfoJson
 from rclone_api.types import (
     PartInfo,
     Range,
     SizeSuffix,
 )
 
-rclone_verbose(True)
-
 
 @dataclass
 class UploadPart:
@@ -95,233 +91,6 @@ def read_task(
         return UploadPart(chunk=outchunk, dst_part=part_dst, exception=e)
 
 
-def _fetch_all_names(
-    self: RcloneImpl,
-    src: str,
-) -> list[str]:
-    dl: DirListing = self.ls(src)
-    files = dl.files
-    filenames: list[str] = [f.name for f in files]
-    filtered: list[str] = [f for f in filenames if f.startswith("part.")]
-    return filtered
-
-
-def _get_info_json(self: RcloneImpl, src: str | None, src_info: str) -> dict:
-    from rclone_api.file import File
-
-    data: dict
-    text: str
-    if src is None:
-        # just try to load the file
-        text_or_err = self.read_text(src_info)
-        if isinstance(text_or_err, Exception):
-            raise FileNotFoundError(f"Could not load {src_info}: {text_or_err}")
-        assert isinstance(text_or_err, str)
-        text = text_or_err
-        data = json.loads(text)
-        return data
-
-    src_stat: File | Exception = self.stat(src)
-    if isinstance(src_stat, Exception):
-        # just try to load the file
-        raise FileNotFoundError(f"Failed to stat {src}: {src_stat}")
-
-    now: datetime = datetime.now()
-    new_data = {
-        "new": True,
-        "created": now.isoformat(),
-        "src": src,
-        "src_modtime": src_stat.mod_time(),
-        "size": src_stat.size,
-        "chunksize": None,
-        "chunksize_int": None,
-        "first_part": None,
-        "last_part": None,
-        "hash": None,
-    }
-
-    text_or_err = self.read_text(src_info)
-    err: Exception | None = text_or_err if isinstance(text_or_err, Exception) else None
-    if isinstance(text_or_err, Exception):
-        warnings.warn(f"Failed to read {src_info}: {text_or_err}")
-        return new_data
-    assert isinstance(text_or_err, str)
-    text = text_or_err
-
-    if err is not None:
-        return new_data
-
-    try:
-        data = json.loads(text)
-        return data
-    except Exception as e:
-        warnings.warn(f"Failed to parse JSON: {e} at {src_info}")
-        return new_data
-
-
-def _save_info_json(self: RcloneImpl, src: str, data: dict) -> None:
-    data = data.copy()
-    data["new"] = False
-    # hash
-
-    h = hashlib.md5()
-    tmp = [
-        data.get("src"),
-        data.get("src_modtime"),
-        data.get("size"),
-        data.get("chunksize_int"),
-    ]
-    data_vals: list[str] = [str(v) for v in tmp]
-    str_data = "".join(data_vals)
-    h.update(str_data.encode("utf-8"))
-    data["hash"] = h.hexdigest()
-    json_str = json.dumps(data, indent=0)
-    self.write_text(dst=src, text=json_str)
-
-
-class InfoJson:
-    def __init__(self, rclone: RcloneImpl, src: str | None, src_info: str) -> None:
-        self.rclone = rclone
-        self.src = src
-        self.src_info = src_info
-        self.data: dict = {}
-
-    def load(self) -> bool:
-        """Returns true if the file exist and is now loaded."""
-        self.data = _get_info_json(self.rclone, self.src, self.src_info)
-        return not self.data.get("new", False)
-
-    def save(self) -> None:
-        _save_info_json(self.rclone, self.src_info, self.data)
-
-    def print(self) -> None:
-        self.rclone.print(self.src_info)
-
-    def fetch_all_finished(self) -> list[str]:
-        parent_path = os.path.dirname(self.src_info)
-        out = _fetch_all_names(self.rclone, parent_path)
-        return out
-
-    def fetch_all_finished_part_numbers(self) -> list[int]:
-        names = self.fetch_all_finished()
-        part_numbers = [int(name.split("_")[0].split(".")[1]) for name in names]
-        return part_numbers
-
-    @property
-    def parts_dir(self) -> str:
-        parts_dir = os.path.dirname(self.src_info)
-        if parts_dir.endswith("/"):
-            parts_dir = parts_dir[:-1]
-        return parts_dir
-
-    @property
-    def dst(self) -> str:
-        parts_dir = self.parts_dir
-        assert parts_dir.endswith("-parts")
-        out = parts_dir[:-6]
-        return out
-
-    @property
-    def dst_name(self) -> str:
-        return os.path.basename(self.dst)
-
-    def compute_all_parts(self) -> list[PartInfo] | Exception:
-        # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
-        # src_size, SizeSuffix("96MB")
-        try:
-
-            src_size = self.size
-            chunk_size = self.chunksize
-            assert isinstance(src_size, SizeSuffix)
-            assert isinstance(chunk_size, SizeSuffix)
-            first_part = self.data["first_part"]
-            last_part = self.data["last_part"]
-            full_part_infos: list[PartInfo] = PartInfo.split_parts(src_size, chunk_size)
-            return full_part_infos[first_part : last_part + 1]
-        except Exception as e:
-            return e
-
-    def compute_all_part_numbers(self) -> list[int] | Exception:
-        all_parts: list[PartInfo] | Exception = self.compute_all_parts()
-        if isinstance(all_parts, Exception):
-            raise all_parts
-
-        all_part_nums: list[int] = [p.part_number for p in all_parts]
-        return all_part_nums
-
-    def fetch_remaining_part_numbers(self) -> list[int] | Exception:
-        all_part_nums: list[int] | Exception = self.compute_all_part_numbers()
-        if isinstance(all_part_nums, Exception):
-            return all_part_nums
-        finished_part_nums: list[int] = self.fetch_all_finished_part_numbers()
-        remaining_part_nums: list[int] = list(
-            set(all_part_nums) - set(finished_part_nums)
-        )
-        return sorted(remaining_part_nums)
-
-    def fetch_is_done(self) -> bool:
-        remaining_part_nums: list[int] | Exception = self.fetch_remaining_part_numbers()
-        if isinstance(remaining_part_nums, Exception):
-            return False
-        return len(remaining_part_nums) == 0
-
-    @property
-    def new(self) -> bool:
-        return self.data.get("new", False)
-
-    @property
-    def chunksize(self) -> SizeSuffix | None:
-        chunksize_int: int | None = self.data.get("chunksize_int")
-        if chunksize_int is None:
-            return None
-        return SizeSuffix(chunksize_int)
-
-    @chunksize.setter
-    def chunksize(self, value: SizeSuffix) -> None:
-        self.data["chunksize"] = str(value)
-        self.data["chunksize_int"] = value.as_int()
-
-    @property
-    def src_modtime(self) -> datetime:
-        return datetime.fromisoformat(self.data["src_modtime"])
-
-    @src_modtime.setter
-    def src_modtime(self, value: datetime) -> None:
-        self.data["src_modtime"] = value.isoformat()
-
-    @property
-    def size(self) -> SizeSuffix:
-        return SizeSuffix(self.data["size"])
-
-    def _get_first_part(self) -> int | None:
-        return self.data.get("first_part")
-
-    def _set_first_part(self, value: int) -> None:
-        self.data["first_part"] = value
-
-    def _get_last_part(self) -> int | None:
-        return self.data.get("last_part")
-
-    def _set_last_part(self, value: int) -> None:
-        self.data["last_part"] = value
-
-    first_part: int | None = property(_get_first_part, _set_first_part)  # type: ignore
-    last_part: int | None = property(_get_last_part, _set_last_part)  # type: ignore
-
-    @property
-    def hash(self) -> str | None:
-        return self.data.get("hash")
-
-    def to_json_str(self) -> str:
-        return json.dumps(self.data)
-
-    def __repr__(self):
-        return f"InfoJson({self.src}, {self.src_info}, {self.data})"
-
-    def __str__(self):
-        return self.to_json_str()
-
-
 def collapse_runs(numbers: list[int]) -> list[str]:
     if not numbers:
         return []
@@ -352,19 +121,40 @@ def collapse_runs(numbers: list[int]) -> list[str]:
     return runs
 
 
-def copy_file_parts(
+_MIN_PART_UPLOAD_SIZE = SizeSuffix("5MB")
+
+
+def _check_part_size(parts: list[PartInfo]) -> Exception | None:
+    if len(parts) == 0:
+        return Exception("No parts to upload")
+    part = parts[0]
+    chunk = part.range.end - part.range.start
+    if chunk < _MIN_PART_UPLOAD_SIZE:
+        return Exception(
+            f"Part size {chunk} is too small to upload. Minimum size for server side merge is {_MIN_PART_UPLOAD_SIZE}"
+        )
+    return None
+
+
+def upload_parts_resumable(
     self: RcloneImpl,
     src: str,  # src:/Bucket/path/myfile.large.zst
     dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
     part_infos: list[PartInfo] | None = None,
     threads: int = 1,
+    verbose: bool | None = None,
 ) -> Exception | None:
     """Copy parts of a file from source to destination."""
     from rclone_api.util import random_str
 
+    def verbose_print(*args, **kwargs):
+        if verbose:
+            print(*args, **kwargs)
+
     if dst_dir.endswith("/"):
         dst_dir = dst_dir[:-1]
     src_size = self.size_file(src)
+
     if isinstance(src_size, Exception):
         return src_size
 
@@ -386,12 +176,16 @@ def copy_file_parts(
             return src_size
         part_infos = full_part_infos.copy()
 
+    err = _check_part_size(part_infos)
+    if err:
+        return err
+
     all_part_numbers: list[int] = [p.part_number for p in part_infos]
     src_info_json = f"{dst_dir}/info.json"
     info_json = InfoJson(self, src, src_info_json)
 
     if not info_json.load():
-        print(f"New: {src_info_json}")
+        verbose_print(f"New: {src_info_json}")
         # info_json.save()
 
     all_numbers_already_done: set[int] = set(
@@ -401,7 +195,7 @@ def copy_file_parts(
     first_part_number = part_infos[0].part_number
     last_part_number = part_infos[-1].part_number
 
-    print(
+    verbose_print(
         f"all_numbers_already_done: {collapse_runs(sorted(list(all_numbers_already_done)))}"
     )
 
@@ -410,12 +204,15 @@ def copy_file_parts(
         if part_info.part_number not in all_numbers_already_done:
             filtered_part_infos.append(part_info)
     part_infos = filtered_part_infos
-
     remaining_part_numbers: list[int] = [p.part_number for p in part_infos]
-    print(f"remaining_part_numbers: {collapse_runs(remaining_part_numbers)}")
+    verbose_print(f"remaining_part_numbers: {collapse_runs(remaining_part_numbers)}")
+    num_remaining_to_upload = len(part_infos)
+    verbose_print(
+        f"num_remaining_to_upload: {num_remaining_to_upload} / {len(full_part_infos)}"
+    )
 
-    if len(part_infos) == 0:
-        return Exception(f"No parts to copy for {src}")
+    if num_remaining_to_upload == 0:
+        return None
     chunk_size = SizeSuffix(part_infos[0].range.end - part_infos[0].range.start)
 
     info_json.chunksize = chunk_size
@@ -432,8 +229,6 @@ def copy_file_parts(
 
     finished_tasks: list[UploadPart] = []
     tmp_dir = str(Path("chunks") / random_str(12))
-    import atexit
-    import shutil
 
     atexit.register(lambda: shutil.rmtree(tmp_dir, ignore_errors=True))
 

rclone_api/s3/{s3_multipart_uploader_by_copy.py → multipart/upload_parts_server_side_merge.py}

@@ -13,17 +13,17 @@ import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from queue import Queue
 from threading import Semaphore, Thread
-from typing import Callable
+from typing import Any, Callable
 
-from rclone_api.detail.copy_file_parts import InfoJson
 from rclone_api.rclone_impl import RcloneImpl
 from rclone_api.s3.create import (
     BaseClient,
     S3Config,
     create_s3_client,
 )
-from rclone_api.s3.merge_state import MergeState, Part
 from rclone_api.s3.multipart.finished_piece import FinishedPiece
+from rclone_api.s3.multipart.info_json import InfoJson
+from rclone_api.s3.multipart.merge_state import MergeState, Part
 from rclone_api.types import EndOfStream
 from rclone_api.util import locked_print
 
@@ -110,7 +110,7 @@ def _upload_part_copy_task(
 
 def _complete_multipart_upload_from_parts(
     s3_client: BaseClient, state: MergeState, finished_parts: list[FinishedPiece]
-) -> str:
+) -> Exception | None:
     """
     Complete a multipart upload using the provided parts.
 
@@ -124,17 +124,28 @@ def _complete_multipart_upload_from_parts(
     # Sort parts by part number to ensure correct order
     finished_parts.sort(key=lambda x: x.part_number)
     multipart_parts = FinishedPiece.to_json_array(finished_parts)
+    multipart_upload: dict = {
+        "Parts": multipart_parts,
+    }
+    response: Any = None
+    try:
+        # Complete the multipart upload
+        response = s3_client.complete_multipart_upload(
+            Bucket=state.bucket,
+            Key=state.dst_key,
+            UploadId=state.upload_id,
+            MultipartUpload=multipart_upload,
+        )
+    except Exception as e:
+        import traceback
 
-    # Complete the multipart upload
-    response = s3_client.complete_multipart_upload(
-        Bucket=state.bucket,
-        Key=state.dst_key,
-        UploadId=state.upload_id,
-        MultipartUpload={"Parts": multipart_parts},
-    )
+        stacktrace = traceback.format_exc()
+        warnings.warn(
+            f"Error completing multipart upload: {e}\n\n{response}\n\n{stacktrace}"
+        )
+        return e
 
-    # Return the URL of the completed object
-    return response.get("Location", f"s3://{state.bucket}/{state.dst_key}")
+    return None
 
 
 def _do_upload_task(
@@ -178,17 +189,22 @@ def _do_upload_task(
             while not semaphore.acquire(blocking=False):
                 time.sleep(0.1)
 
-        # Upload parts by copying from source objects
-        finished_parts: list[FinishedPiece] = []
+        final_fut = executor.submit(lambda: on_finished(EndOfStream()))
 
         for fut in futures:
             finished_part = fut.result()
             if isinstance(finished_part, Exception):
                 executor.shutdown(wait=True, cancel_futures=True)
                 return finished_part
-            finished_parts.append(finished_part)
+        final_fut.result()
 
-        on_finished(EndOfStream())
+        finished_parts = merge_state.finished
+        try:
+            assert len(finished_parts) == len(merge_state.all_parts)
+        except Exception:
+            return ValueError(
+                f"Finished parts mismatch: {len(finished_parts)} != {len(parts)}"
+            )
 
         try:
             # Complete the multipart upload
@@ -243,9 +259,10 @@ def _begin_upload(
 
 
 class WriteMergeStateThread(Thread):
-    def __init__(self, rclone_impl: RcloneImpl, merge_state: MergeState):
+    def __init__(self, rclone_impl: RcloneImpl, merge_state: MergeState, verbose: bool):
         super().__init__(daemon=True)
         assert isinstance(merge_state, MergeState)
+        self.verbose = verbose
         self.merge_state = merge_state
         self.merge_path = merge_state.merge_path
         self.rclone_impl = rclone_impl
@@ -265,11 +282,15 @@ class WriteMergeStateThread(Thread):
                 return item
         return item
 
+    def verbose_print(self, msg: str) -> None:
+        if self.verbose:
+            locked_print(msg)
+
     def run(self):
         while True:
             item = self._get_next()
             if isinstance(item, EndOfStream):
-                warnings.warn("End of stream")
+                self.verbose_print("WriteMergeStateThread: End of stream")
                 break
 
             assert isinstance(item, FinishedPiece)
@@ -315,13 +336,14 @@ def _get_merge_path(info_path: str) -> str:
 def _begin_or_resume_merge(
     rclone: RcloneImpl,
     info: InfoJson,
+    verbose: bool = False,
     max_workers: int = DEFAULT_MAX_WORKERS,
 ) -> "S3MultiPartMerger | Exception":
     try:
         merger: S3MultiPartMerger = S3MultiPartMerger(
             rclone_impl=rclone,
             info=info,
-            verbose=True,
+            verbose=verbose,
             max_workers=max_workers,
         )
 
@@ -412,9 +434,11 @@ class S3MultiPartMerger:
 
     @staticmethod
     def create(
-        rclone: RcloneImpl, info: InfoJson, max_workers: int
+        rclone: RcloneImpl, info: InfoJson, max_workers: int, verbose: bool
     ) -> "S3MultiPartMerger | Exception":
-        return _begin_or_resume_merge(rclone=rclone, info=info, max_workers=max_workers)
+        return _begin_or_resume_merge(
+            rclone=rclone, info=info, max_workers=max_workers, verbose=verbose
+        )
 
     @property
     def bucket(self) -> str:
@@ -426,6 +450,7 @@ class S3MultiPartMerger:
         self.write_thread = WriteMergeStateThread(
             rclone_impl=self.rclone_impl,
             merge_state=self.state,
+            verbose=self.verbose,
         )
 
     def _begin_new_merge(
@@ -494,7 +519,10 @@ class S3MultiPartMerger:
 
 
 def s3_server_side_multi_part_merge(
-    rclone: RcloneImpl, info_path: str, max_workers: int = DEFAULT_MAX_WORKERS
+    rclone: RcloneImpl,
+    info_path: str,
+    max_workers: int = DEFAULT_MAX_WORKERS,
+    verbose: bool = False,
 ) -> Exception | None:
     info = InfoJson(rclone, src=None, src_info=info_path)
     loaded = info.load()
@@ -503,7 +531,7 @@ def s3_server_side_multi_part_merge(
             f"Info file not found, has the upload finished? {info_path}"
         )
     merger: S3MultiPartMerger | Exception = S3MultiPartMerger.create(
-        rclone=rclone, info=info, max_workers=max_workers
+        rclone=rclone, info=info, max_workers=max_workers, verbose=verbose
     )
     if isinstance(merger, Exception):
         return merger
@@ -514,5 +542,5 @@ def s3_server_side_multi_part_merge(
 
     err = merger.cleanup()
     if isinstance(err, Exception):
-        err
+        return err
     return None

rclone_api/types.py

@@ -215,7 +215,7 @@ class SizeSuffix:
         return self._size == SizeSuffix(other)._size
 
     def __ne__(self, other: object) -> bool:
-        return not self.__ne__(other)
+        return not self.__eq__(other)
 
     def __lt__(self, other: "int | SizeSuffix") -> bool:
         # if not isinstance(other, SizeSuffix):

{rclone_api-1.4.19.dist-info → rclone_api-1.4.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.19
+Version: 1.4.22
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.19.dist-info → rclone_api-1.4.22.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=gsYL3jaqMgHbJ6mnLxNttuaKltUBXNuvpz7C6fF8_8w,17678
+rclone_api/__init__.py,sha256=wvLRTEYvwLfkBx_LuWI62cZQA-pEz0xo0oVh_9DCOuM,17876
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -14,44 +14,46 @@ rclone_api/file_part.py,sha256=i6ByS5_sae8Eba-4imBVTxd-xKC8ExWy7NR8QGr0ors,6155
 rclone_api/file_stream.py,sha256=_W3qnwCuigqA0hzXl2q5pAxSZDRaUSwet4BkT0lpnzs,1431
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/http_server.py,sha256=3fPBV6l50erTe32DyeJBNmsDrn5KuujsbmEAbx13T-c,8720
+rclone_api/http_server.py,sha256=LhovQu2AI-Z7zQIWflWelCiCDLnWzisL32Rs5350kxE,8850
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
 rclone_api/process.py,sha256=tGooS5NLdPuqHh7hCH8SfK44A6LGftPQCPQUNgSo0a0,5714
-rclone_api/rclone_impl.py,sha256=xTTriz6-zn_aSrkY8B7wzT-zRXax7Og7ns6xu6-7O6g,48769
+rclone_api/rclone_impl.py,sha256=WBLkQpQq4lGPla1uJBzpp1yf4kS3ub7fxpbU6SdJyZY,49873
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
-rclone_api/types.py,sha256=HkpEZgZWhr5Gb04iHq5NxMRXxieWoN-PKmOfJFrg5Qg,12155
+rclone_api/types.py,sha256=2ngxwpdNy88y0teeYJ5Vz5NiLK1rfaFx8Xf99i0J-Js,12155
 rclone_api/util.py,sha256=yY72YKpmpT_ZM7AleVtPpl0YZZYQPTwTdqKn9qPwm8Y,9290
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
-rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
-rclone_api/cmd/copy_large_s3_finish.py,sha256=Q_8uvC2J3V_jGQj8ualMlejWEPpkFn267bUKLC3Sp2M,2050
+rclone_api/cmd/copy_large_s3.py,sha256=yhPwbtGz9MmlronB-biiYUfNVclOsxfX9GIhe3ai3g4,3463
+rclone_api/cmd/copy_large_s3_finish.py,sha256=k9-LC5kr20JMdHtELqFBrxClNLOEUsfys4Sx6v_n23c,2061
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
 rclone_api/db/db.py,sha256=YRnYrCaXHwytQt07uEZ_mMpvPHo9-0IWcOb95fVOOfs,10086
 rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
-rclone_api/detail/copy_file_parts.py,sha256=1h-5JJmZdB0_TuVcuYMIClHqAgCXUI4eLyZHbdRiCHg,16280
+rclone_api/detail/copy_file_parts_resumable.py,sha256=RoUWV2eBWEvuuTfsvrz5BhtvX3BmX-DVmQKdARhRF80,1248
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/s3/api.py,sha256=owoQ1H-R0hXcUozxC6sl53D7NmMOewHk2pUxK-ye8ms,4061
+rclone_api/s3/api.py,sha256=6E4xEOxtpP6niiAFEpgB1-ckWJclNyRsJ3D11Qm4RwU,4069
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
 rclone_api/s3/create.py,sha256=_Q-faQ4Zl8XKTB28gireRxVXWP-YNxoAK4bligxDtiI,3998
-rclone_api/s3/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
-rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=iuxnRaRDHvIMCu-9YDZzirRMc1R4gZgjigkM5u_MFJg,17188
 rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
-rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
 rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaRCmkYLTQ,105
-rclone_api/s3/multipart/finished_piece.py,sha256=LtlX_mm6_hsADR8FxgfC2_pcO5Wou_20-jE34IcRXew,1633
+rclone_api/s3/multipart/finished_piece.py,sha256=XiFf-TXczCnMwg9OEy8xVY9uR-ytX6QvuFwywiewdQQ,2290
+rclone_api/s3/multipart/info_json.py,sha256=-e8UCwrqjAP64U8PmH-o2ciJ6TN48DwHktJfpFV_3wU,7450
+rclone_api/s3/multipart/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
+rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
+rclone_api/s3/multipart/upload_parts_resumable.py,sha256=diJoUpVYow6No_dNgOZIYVsv43k4evb6zixqpzWJaUk,9771
+rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.19.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.19.dist-info/METADATA,sha256=RpqVhvMVSL_isjprjoRJJWMStau5WxyiiyO1OkHBydo,4628
-rclone_api-1.4.19.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.19.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.19.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.19.dist-info/RECORD,,
+rclone_api-1.4.22.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.22.dist-info/METADATA,sha256=eqenqWL8LT3t-zpC0Qy-OgRQSYyC1Shg8dC8xwDsnuo,4628
+rclone_api-1.4.22.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.22.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.22.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.22.dist-info/RECORD,,