rclone-api 1.4.18__py2.py3-none-any.whl → 1.4.20__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3_finish.py

@@ -1,25 +1,17 @@
 import argparse
-import os
 from dataclasses import dataclass
 from pathlib import Path
 
 from rclone_api import Rclone
-from rclone_api.detail.copy_file_parts import InfoJson
 from rclone_api.s3.s3_multipart_uploader_by_copy import (
-    Part,
-    S3MultiPartUploader,
+    s3_server_side_multi_part_merge,
 )
 
-_TIMEOUT_READ = 900
-_TIMEOUT_CONNECTION = 900
-_MAX_WORKERS = 10
-
 
 @dataclass
 class Args:
     config_path: Path
     src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
-    dst: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst
     verbose: bool
 
 
@@ -32,8 +24,7 @@ def list_files(rclone: Rclone, path: str):
 
 def _parse_args() -> Args:
     parser = argparse.ArgumentParser(description="List files in a remote path.")
-    parser.add_argument("src", help="File to copy")
-    parser.add_argument("dst", help="Destination file")
+    parser.add_argument("src", help="Directory that holds the info.json file")
     parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
     parser.add_argument(
         "--config", help="Path to rclone config file", type=Path, required=False
@@ -48,166 +39,26 @@ def _parse_args() -> Args:
     out = Args(
         config_path=config,
         src=args.src,
-        dst=args.dst,
         verbose=args.verbose,
     )
     return out
 
 
-# def finish_multipart_upload_from_keys(
-#     s3_client: BaseClient,
-#     source_bucket: str,
-#     parts: list[Part],
-#     destination_bucket: str,
-#     destination_key: str,
-#     chunk_size: int,  # 5MB default
-#     max_workers: int = 100,
-#     retries: int = 3,
-# ) -> str | Exception:
-#     """
-#     Finish a multipart upload by copying parts from existing S3 objects.
-
-#     Args:
-#         s3_client: Boto3 S3 client
-#         source_bucket: Source bucket name
-#         source_keys: List of source object keys to copy from
-#         destination_bucket: Destination bucket name
-#         destination_key: Destination object key
-#         chunk_size: Size of each part in bytes
-#         retries: Number of retry attempts
-#         byte_ranges: Optional list of byte ranges corresponding to source_keys
-
-#     Returns:
-#         The URL of the completed object
-#     """
-
-#     # Create upload info
-#     info = begin_upload(
-#         s3_client=s3_client,
-#         parts=parts,
-#         destination_bucket=destination_bucket,
-#         destination_key=destination_key,
-#         chunk_size=chunk_size,
-#         retries=retries,
-#     )
-
-#     out = do_body_work(
-#         info=info,
-#         source_bucket=source_bucket,
-#         parts=parts,
-#         max_workers=max_workers,
-#         retries=retries,
-#     )
-
-#     return out
-
-
-def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> Exception | None:
-    from rclone_api.s3.create import (
-        BaseClient,
-        S3Config,
-        S3Credentials,
-        create_s3_client,
-    )
-
-    s3_config = S3Config(
-        verbose=False,
-        timeout_read=_TIMEOUT_READ,
-        timeout_connection=_TIMEOUT_CONNECTION,
-    )
-
-    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(s3_creds=s3_creds, s3_config=s3_config)
-    s3_bucket = s3_creds.bucket_name
-    is_done = info.fetch_is_done()
-    size = info.size
-    assert is_done, f"Upload is not done: {info}"
-
-    parts_dir = info.parts_dir
-    if parts_dir.endswith("/"):
-        parts_dir = parts_dir[:-1]
-    source_keys = info.fetch_all_finished()
-    # print(parts_dir)
-    # print(source_keys)
-
-    parts_path = parts_dir.split(s3_bucket)[1]
-    if parts_path.startswith("/"):
-        parts_path = parts_path[1:]
-
-    first_part: int | None = info.first_part
-    last_part: int | None = info.last_part
-
-    assert first_part is not None
-    assert last_part is not None
-
-    def _to_s3_key(name: str | None) -> str:
-        if name:
-            out = f"{parts_path}/{name}"
-            return out
-        out = f"{parts_path}"
-        return out
-
-    parts: list[Part] = []
-    part_num = 1
-    for part_key in source_keys:
-        s3_key = _to_s3_key(name=part_key)
-        part = Part(part_number=part_num, s3_key=s3_key)
-        parts.append(part)
-        part_num += 1
-
-    chunksize = info.chunksize
-    assert chunksize is not None
-
-    dst_name = info.dst_name
-    dst_dir = os.path.dirname(parts_path)
-    # dst_key =
-    dst_key = f"{dst_dir}/{dst_name}"
-
-    uploader: S3MultiPartUploader = S3MultiPartUploader(
-        s3_client=s3_client,
-        verbose=True,
-    )
-
-    from rclone_api.s3.s3_multipart_uploader_by_copy import MultipartUploadInfo
-
-    state: MultipartUploadInfo = uploader.begin_new_upload(
-        parts=parts,
-        destination_bucket=s3_creds.bucket_name,
-        destination_key=dst_key,
-        chunk_size=chunksize.as_int(),
-    )
-
-    uploader.start_upload(info=state, parts=parts, max_workers=_MAX_WORKERS)
-
-    # now check if the dst now exists, if so, delete the parts folder.
-    # if rclone.exists(dst):
-    #     rclone.purge(parts_dir)
-
-    if not rclone.exists(dst):
-        return FileNotFoundError(f"Destination file not found: {dst}")
-
-    write_size = rclone.size_file(dst)
-    if write_size != size:
-        return ValueError(f"Size mismatch: {write_size} != {size}")
-
-    print(f"Upload complete: {dst}")
-    rclone.purge(parts_dir)
-    return None
+def _get_info_path(src: str) -> str:
+    if src.endswith("/"):
+        src = src[:-1]
+    info_path = f"{src}/info.json"
+    return info_path
 
 
 def main() -> int:
     """Main entry point."""
     args = _parse_args()
     rclone = Rclone(rclone_conf=args.config_path)
-    info_json = f"{args.src}/info.json".replace("//", "/")
-    info = InfoJson(rclone.impl, src=None, src_info=info_json)
-    loaded = info.load()
-    if not loaded:
-        raise FileNotFoundError(
-            f"Info file not found, has the upload finished? {info_json}"
-        )
-    print(info)
-    do_finish_part(rclone=rclone, info=info, dst=args.dst)
+    info_path = _get_info_path(src=args.src)
+    s3_server_side_multi_part_merge(
+        rclone=rclone.impl, info_path=info_path, max_workers=5
+    )
     return 0
 
 
@@ -219,7 +70,4 @@ if __name__ == "__main__":
     sys.argv.append(
         "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
     )
-    sys.argv.append(
-        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
-    )
     main()

rclone_api/detail/copy_file_parts.py

@@ -209,7 +209,10 @@ class InfoJson:
 
     @property
     def parts_dir(self) -> str:
-        return os.path.dirname(self.src_info)
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
 
     @property
     def dst(self) -> str:

rclone_api/s3/merge_state.py

@@ -8,8 +8,9 @@ from existing S3 objects using upload_part_copy.
 
 import json
 from dataclasses import dataclass
-from typing import Any, Callable
+from typing import Any
 
+from rclone_api.rclone_impl import RcloneImpl
 from rclone_api.s3.multipart.finished_piece import FinishedPiece
 
 
@@ -46,34 +47,59 @@ class Part:
 
 class MergeState:
 
-    def __init__(self, finished: list[FinishedPiece], all_parts: list[Part]) -> None:
+    def __init__(
+        self,
+        rclone_impl: RcloneImpl,
+        merge_path: str,
+        upload_id: str,
+        bucket: str,
+        dst_key: str,
+        finished: list[FinishedPiece],
+        all_parts: list[Part],
+    ) -> None:
+        self.rclone_impl: RcloneImpl = rclone_impl
+        self.merge_path: str = merge_path
+        self.merge_parts_path: str = f"{merge_path}/merge"  # future use?
+        self.upload_id: str = upload_id
+        self.bucket: str = bucket
+        self.dst_key: str = dst_key
         self.finished: list[FinishedPiece] = list(finished)
         self.all_parts: list[Part] = list(all_parts)
-        self.callbacks: list[Callable[[FinishedPiece], None]] = []
-
-    def add_callback(self, callback: Callable[[FinishedPiece], None]) -> None:
-        self.callbacks.append(callback)
 
     def on_finished(self, finished_piece: FinishedPiece) -> None:
-        for callback in list(self.callbacks):
-            callback(finished_piece)
+        self.finished.append(finished_piece)
+
+    def remaining_parts(self) -> list[Part]:
+        finished_parts: set[int] = set([p.part_number for p in self.finished])
+        remaining = [p for p in self.all_parts if p.part_number not in finished_parts]
+        return remaining
 
     @staticmethod
-    def from_json_array(json_array: dict) -> "MergeState | Exception":
+    def from_json(rclone_impl: RcloneImpl, json: dict) -> "MergeState | Exception":
         try:
+            merge_path = json["merge_path"]
+            bucket = json["bucket"]
+            dst_key = json["dst_key"]
             finished: list[FinishedPiece] = FinishedPiece.from_json_array(
-                json_array["finished"]
+                json["finished"]
             )
-            all_parts: list[Part | Exception] = [
-                Part.from_json(j) for j in json_array["all"]
-            ]
+            all_parts: list[Part | Exception] = [Part.from_json(j) for j in json["all"]]
             all_parts_no_err: list[Part] = [
                 p for p in all_parts if not isinstance(p, Exception)
             ]
+            upload_id: str = json["upload_id"]
             errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
             if len(errs):
                 return Exception(f"Errors in parts: {errs}")
-            return MergeState(finished=finished, all_parts=all_parts_no_err)
+            return MergeState(
+                rclone_impl=rclone_impl,
+                merge_path=merge_path,
+                upload_id=upload_id,
+                bucket=bucket,
+                dst_key=dst_key,
+                finished=finished,
+                all_parts=all_parts_no_err,
+            )
         except Exception as e:
             return e
 
@@ -81,12 +107,18 @@ class MergeState:
         finished = self.finished.copy()
         all_parts = self.all_parts.copy()
         return {
+            "merge_path": self.merge_path,
+            "bucket": self.bucket,
+            "dst_key": self.dst_key,
+            "upload_id": self.upload_id,
             "finished": FinishedPiece.to_json_array(finished),
             "all": [part.to_json() for part in all_parts],
         }
 
     def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=1)
+        data = self.to_json()
+        out = json.dumps(data, indent=2)
+        return out
 
     def __str__(self):
         return self.to_json_str()

rclone_api/s3/multipart/finished_piece.py

@@ -1,4 +1,3 @@
-import json
 import warnings
 from dataclasses import dataclass
 
@@ -13,9 +12,6 @@ class FinishedPiece:
     def to_json(self) -> dict:
         return {"part_number": self.part_number, "etag": self.etag}
 
-    def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=0)
-
     @staticmethod
     def to_json_array(
         parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],

rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -6,33 +6,36 @@ This module provides functionality for S3 multipart uploads, including copying p
 from existing S3 objects using upload_part_copy.
 """
 
+import json
+import os
+import time
+import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
-from dataclasses import dataclass
-from pathlib import Path
-from threading import Semaphore
-from typing import Optional
-
-from botocore.client import BaseClient
-
+from queue import Queue
+from threading import Semaphore, Thread
+from typing import Callable
+
+from rclone_api.detail.copy_file_parts import InfoJson
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.s3.create import (
+    BaseClient,
+    S3Config,
+    create_s3_client,
+)
 from rclone_api.s3.merge_state import MergeState, Part
 from rclone_api.s3.multipart.finished_piece import FinishedPiece
+from rclone_api.types import EndOfStream
 from rclone_api.util import locked_print
 
+DEFAULT_MAX_WORKERS = 5  # Backblaze can do 10 with exponential backoff, so let's try 5
 
-@dataclass
-class MultipartUploadInfo:
-    """Simplified upload information for multipart uploads."""
-
-    s3_client: BaseClient
-    bucket_name: str
-    object_name: str
-    upload_id: str
-    chunk_size: int
-    src_file_path: Optional[Path] = None
+_TIMEOUT_READ = 900
+_TIMEOUT_CONNECTION = 900
 
 
-def upload_part_copy_task(
-    info: MultipartUploadInfo,
+def _upload_part_copy_task(
+    s3_client: BaseClient,
+    state: MergeState,
     source_bucket: str,
     source_key: str,
     part_number: int,
@@ -60,32 +63,34 @@ def upload_part_copy_task(
         params: dict = {}
         try:
             if retry > 0:
-                locked_print(f"Retrying part copy {part_number} for {info.object_name}")
+                locked_print(f"Retrying part copy {part_number} for {state.dst_key}")
 
             locked_print(
-                f"Copying part {part_number} for {info.object_name} from {source_bucket}/{source_key}"
+                f"Copying part {part_number} for {state.dst_key} from {source_bucket}/{source_key}"
             )
 
             # Prepare the upload_part_copy parameters
             params = {
-                "Bucket": info.bucket_name,
+                "Bucket": state.bucket,
                 "CopySource": copy_source,
-                "Key": info.object_name,
+                "Key": state.dst_key,
                 "PartNumber": part_number,
-                "UploadId": info.upload_id,
+                "UploadId": state.upload_id,
             }
 
             # Execute the copy operation
-            part = info.s3_client.upload_part_copy(**params)
+            part = s3_client.upload_part_copy(**params)
 
             # Extract ETag from the response
             etag = part["CopyPartResult"]["ETag"]
             out = FinishedPiece(etag=etag, part_number=part_number)
-            locked_print(f"Finished part {part_number} for {info.object_name}")
+            locked_print(f"Finished part {part_number} for {state.dst_key}")
             return out
 
         except Exception as e:
-            msg = f"Error copying {copy_source} -> {info.object_name}: {e}, params={params}"
+            msg = (
+                f"Error copying {copy_source} -> {state.dst_key}: {e}, params={params}"
+            )
             if "An error occurred (InternalError)" in str(e):
                 locked_print(msg)
             elif "NoSuchKey" in str(e):
@@ -103,8 +108,8 @@ def upload_part_copy_task(
     return Exception("Should not reach here")
 
 
-def complete_multipart_upload_from_parts(
-    info: MultipartUploadInfo, parts: list[FinishedPiece]
+def _complete_multipart_upload_from_parts(
+    s3_client: BaseClient, state: MergeState, finished_parts: list[FinishedPiece]
 ) -> str:
     """
     Complete a multipart upload using the provided parts.
@@ -117,61 +122,61 @@ def complete_multipart_upload_from_parts(
         The URL of the completed object
     """
     # Sort parts by part number to ensure correct order
-    parts.sort(key=lambda x: x.part_number)
-
-    # Prepare the parts list for the complete_multipart_upload call
-    multipart_parts = [
-        {"ETag": part.etag, "PartNumber": part.part_number} for part in parts
-    ]
+    finished_parts.sort(key=lambda x: x.part_number)
+    multipart_parts = FinishedPiece.to_json_array(finished_parts)
 
     # Complete the multipart upload
-    response = info.s3_client.complete_multipart_upload(
-        Bucket=info.bucket_name,
-        Key=info.object_name,
-        UploadId=info.upload_id,
+    response = s3_client.complete_multipart_upload(
+        Bucket=state.bucket,
+        Key=state.dst_key,
+        UploadId=state.upload_id,
         MultipartUpload={"Parts": multipart_parts},
     )
 
     # Return the URL of the completed object
-    return response.get("Location", f"s3://{info.bucket_name}/{info.object_name}")
+    return response.get("Location", f"s3://{state.bucket}/{state.dst_key}")
 
 
-def do_body_work(
-    info: MultipartUploadInfo,
-    source_bucket: str,
+def _do_upload_task(
+    s3_client: BaseClient,
     max_workers: int,
     merge_state: MergeState,
-) -> str | Exception:
-
+    on_finished: Callable[[FinishedPiece | EndOfStream], None],
+) -> Exception | None:
     futures: list[Future[FinishedPiece | Exception]] = []
-    parts = list(merge_state.all_parts)
-
+    parts = merge_state.remaining_parts()
+    source_bucket = merge_state.bucket
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
         semaphore = Semaphore(max_workers)
         for part in parts:
             part_number, s3_key = part.part_number, part.s3_key
 
             def task(
-                info=info,
+                s3_client=s3_client,
+                state=merge_state,
                 source_bucket=source_bucket,
                 s3_key=s3_key,
                 part_number=part_number,
             ):
-                out = upload_part_copy_task(
-                    info=info,
+                out = _upload_part_copy_task(
+                    s3_client=s3_client,
+                    state=state,
                     source_bucket=source_bucket,
                     source_key=s3_key,
                     part_number=part_number,
                 )
                 if isinstance(out, Exception):
                     return out
-                merge_state.on_finished(out)
+                # merge_state.on_finished(out)
+                on_finished(out)
                 return out
 
             fut = executor.submit(task)
             fut.add_done_callback(lambda x: semaphore.release())
             futures.append(fut)
-            semaphore.acquire()
+
+            while not semaphore.acquire(blocking=False):
+                time.sleep(0.1)
 
         # Upload parts by copying from source objects
         finished_parts: list[FinishedPiece] = []
@@ -183,17 +188,26 @@ def do_body_work(
                 return finished_part
             finished_parts.append(finished_part)
 
-        # Complete the multipart upload
-        return complete_multipart_upload_from_parts(info, finished_parts)
+        on_finished(EndOfStream())
 
+        try:
+            # Complete the multipart upload
+            _complete_multipart_upload_from_parts(
+                s3_client=s3_client, state=merge_state, finished_parts=finished_parts
+            )
+        except Exception as e:
+            warnings.warn(f"Error completing multipart upload: {e}")
+            return e
+        return None
 
-def begin_upload(
+
+def _begin_upload(
     s3_client: BaseClient,
     parts: list[Part],
-    destination_bucket: str,
-    destination_key: str,
-    chunk_size: int,
-) -> MultipartUploadInfo:
+    bucket: str,
+    dst_key: str,
+    verbose: bool,
+) -> str:
     """
     Finish a multipart upload by copying parts from existing S3 objects.
 
@@ -201,127 +215,310 @@ def begin_upload(
         s3_client: Boto3 S3 client
         source_bucket: Source bucket name
         source_keys: List of source object keys to copy from
-        destination_bucket: Destination bucket name
-        destination_key: Destination object key
-        chunk_size: Size of each part in bytes
+        bucket: Destination bucket name
+        dst_key: Destination object key
         retries: Number of retry attempts
         byte_ranges: Optional list of byte ranges corresponding to source_keys
 
     Returns:
-        The URL of the completed object
+        The upload id of the multipart upload
     """
 
     # Initiate multipart upload
-    locked_print(
-        f"Creating multipart upload for {destination_bucket}/{destination_key} from {len(parts)} source objects"
-    )
+    if verbose:
+        locked_print(
+            f"Creating multipart upload for {bucket}/{dst_key} from {len(parts)} source objects"
+        )
     create_params: dict[str, str] = {
-        "Bucket": destination_bucket,
-        "Key": destination_key,
+        "Bucket": bucket,
+        "Key": dst_key,
     }
-    print(f"Creating multipart upload with {create_params}")
+    if verbose:
+        locked_print(f"Creating multipart upload with {create_params}")
     mpu = s3_client.create_multipart_upload(**create_params)
-    print(f"Created multipart upload: {mpu}")
+    if verbose:
+        locked_print(f"Created multipart upload: {mpu}")
     upload_id = mpu["UploadId"]
+    return upload_id
+
+
+class WriteMergeStateThread(Thread):
+    def __init__(self, rclone_impl: RcloneImpl, merge_state: MergeState):
+        super().__init__(daemon=True)
+        assert isinstance(merge_state, MergeState)
+        self.merge_state = merge_state
+        self.merge_path = merge_state.merge_path
+        self.rclone_impl = rclone_impl
+        self.queue: Queue[FinishedPiece | EndOfStream] = Queue()
+        self.start()
+
+    def _get_next(self) -> FinishedPiece | EndOfStream:
+        item = self.queue.get()
+        if isinstance(item, EndOfStream):
+            return item
+        # see if there are more items in the queue, only write the last one
+        while not self.queue.empty():
+            item = self.queue.get()
+            if isinstance(item, EndOfStream):
+                # put it back in for next time
+                self.queue.put(item)
+                return item
+        return item
+
+    def run(self):
+        while True:
+            item = self._get_next()
+            if isinstance(item, EndOfStream):
+                warnings.warn("End of stream")
+                break
+
+            assert isinstance(item, FinishedPiece)
+            # piece: FinishedPiece = item
+            # at this point just write out the whole json str
+            json_str = self.merge_state.to_json_str()
+            err = self.rclone_impl.write_text(self.merge_path, json_str)
+            if isinstance(err, Exception):
+                warnings.warn(f"Error writing merge state: {err}")
+                break
+
+    def add_finished(self, finished: FinishedPiece) -> None:
+        self.queue.put(finished)
+
+    def add_eos(self) -> None:
+        self.queue.put(EndOfStream())
+
+
+def _cleanup_merge(rclone: RcloneImpl, info: InfoJson) -> Exception | None:
+    size = info.size
+    dst = info.dst
+    parts_dir = info.parts_dir
+    if not rclone.exists(dst):
+        return FileNotFoundError(f"Destination file not found: {dst}")
+
+    write_size = rclone.size_file(dst)
+    if write_size != size:
+        return ValueError(f"Size mismatch: {write_size} != {size}")
+
+    print(f"Upload complete: {dst}")
+    cp = rclone.purge(parts_dir)
+    if cp.failed():
+        return Exception(f"Failed to purge parts dir: {cp}")
+    return None
+
+
+def _get_merge_path(info_path: str) -> str:
+    par_dir = os.path.dirname(info_path)
+    merge_path = f"{par_dir}/merge.json"
+    return merge_path
+
+
+def _begin_or_resume_merge(
+    rclone: RcloneImpl,
+    info: InfoJson,
+    verbose: bool = False,
+    max_workers: int = DEFAULT_MAX_WORKERS,
+) -> "S3MultiPartMerger | Exception":
+    try:
+        merger: S3MultiPartMerger = S3MultiPartMerger(
+            rclone_impl=rclone,
+            info=info,
+            verbose=verbose,
+            max_workers=max_workers,
+        )
 
-    # Create upload info
-    info = MultipartUploadInfo(
-        s3_client=s3_client,
-        bucket_name=destination_bucket,
-        object_name=destination_key,
-        upload_id=upload_id,
-        chunk_size=chunk_size,
-    )
-    return info
-
-
-def finish_multipart_upload_from_keys(
-    s3_client: BaseClient,
-    source_bucket: str,
-    parts: list[Part],
-    destination_bucket: str,
-    destination_key: str,
-    chunk_size: int,  # 5MB default
-    max_workers: int = 100,
-) -> str | Exception:
-    """
-    Finish a multipart upload by copying parts from existing S3 objects.
-
-    Args:
-        s3_client: Boto3 S3 client
-        source_bucket: Source bucket name
-        source_keys: List of source object keys to copy from
-        destination_bucket: Destination bucket name
-        destination_key: Destination object key
-        chunk_size: Size of each part in bytes
-        retries: Number of retry attempts
-        byte_ranges: Optional list of byte ranges corresponding to source_keys
-
-    Returns:
-        The URL of the completed object
-    """
-
-    merge_state = MergeState(finished=[], all_parts=parts)
-
-    # Create upload info
-    info = begin_upload(
-        s3_client=s3_client,
-        parts=merge_state.all_parts,
-        destination_bucket=destination_bucket,
-        destination_key=destination_key,
-        chunk_size=chunk_size,
-    )
-
-    out = do_body_work(
-        info=info,
-        source_bucket=source_bucket,
-        max_workers=max_workers,
-        merge_state=merge_state,
-    )
+        s3_bucket = merger.bucket
+        is_done = info.fetch_is_done()
+        assert is_done, f"Upload is not done: {info}"
+
+        merge_path = _get_merge_path(info_path=info.src_info)
+        merge_json_text = rclone.read_text(merge_path)
+        if isinstance(merge_json_text, str):
+            # Attempt to do a resume
+            merge_data = json.loads(merge_json_text)
+            merge_state = MergeState.from_json(rclone_impl=rclone, json=merge_data)
+            if isinstance(merge_state, MergeState):
+                merger._begin_resume_merge(merge_state=merge_state)
+                return merger
+            warnings.warn(f"Failed to resume merge: {merge_state}, starting new merge")
+
+        parts_dir = info.parts_dir
+        source_keys = info.fetch_all_finished()
+
+        parts_path = parts_dir.split(s3_bucket)[1]
+        if parts_path.startswith("/"):
+            parts_path = parts_path[1:]
+
+        first_part: int | None = info.first_part
+        last_part: int | None = info.last_part
+
+        assert first_part is not None
+        assert last_part is not None
+
+        def _to_s3_key(name: str | None) -> str:
+            if name:
+                out = f"{parts_path}/{name}"
+                return out
+            out = f"{parts_path}"
+            return out
 
-    return out
+        parts: list[Part] = []
+        part_num = first_part
+        for part_key in source_keys:
+            assert part_num <= last_part and part_num >= first_part
+            s3_key = _to_s3_key(name=part_key)
+            part = Part(part_number=part_num, s3_key=s3_key)
+            parts.append(part)
+            part_num += 1
+
+        dst_name = info.dst_name
+        dst_dir = os.path.dirname(parts_path)
+        dst_key = f"{dst_dir}/{dst_name}"
+
+        err = merger._begin_new_merge(
+            merge_path=merge_path,
+            parts=parts,
+            bucket=merger.bucket,
+            dst_key=dst_key,
+        )
+        if isinstance(err, Exception):
+            return err
+        return merger
+    except Exception as e:
+        return e
 
 
-_DEFAULT_MAX_WORKERS = 10
+class S3MultiPartMerger:
+    def __init__(
+        self,
+        rclone_impl: RcloneImpl,
+        info: InfoJson,
+        s3_config: S3Config | None = None,
+        verbose: bool = False,
+        max_workers: int = DEFAULT_MAX_WORKERS,
+    ) -> None:
+        self.rclone_impl: RcloneImpl = rclone_impl
+        self.info = info
+        self.s3_creds = rclone_impl.get_s3_credentials(remote=info.dst)
+        self.verbose = verbose
+        s3_config = s3_config or S3Config(
+            verbose=verbose,
+            timeout_read=_TIMEOUT_READ,
+            timeout_connection=_TIMEOUT_CONNECTION,
+            max_pool_connections=max_workers,
+        )
+        self.max_workers = s3_config.max_pool_connections or DEFAULT_MAX_WORKERS
+        self.client = create_s3_client(s3_creds=self.s3_creds, s3_config=s3_config)
+        self.state: MergeState | None = None
+        self.write_thread: WriteMergeStateThread | None = None
+
+    @staticmethod
+    def create(
+        rclone: RcloneImpl, info: InfoJson, max_workers: int, verbose: bool
+    ) -> "S3MultiPartMerger | Exception":
+        return _begin_or_resume_merge(
+            rclone=rclone, info=info, max_workers=max_workers, verbose=verbose
+        )
 
+    @property
+    def bucket(self) -> str:
+        return self.s3_creds.bucket_name
 
-class S3MultiPartUploader:
-    def __init__(self, s3_client: BaseClient, verbose: bool = False) -> None:
-        self.verbose = verbose
-        self.client: BaseClient = s3_client
+    def start_write_thread(self) -> None:
+        assert self.state is not None
+        assert self.write_thread is None
+        self.write_thread = WriteMergeStateThread(
+            rclone_impl=self.rclone_impl,
+            merge_state=self.state,
+        )
 
-    def begin_new_upload(
+    def _begin_new_merge(
         self,
         parts: list[Part],
-        destination_bucket: str,
-        destination_key: str,
-        chunk_size: int,
-    ) -> MultipartUploadInfo:
-        return begin_upload(
-            s3_client=self.client,
-            parts=parts,
-            destination_bucket=destination_bucket,
-            destination_key=destination_key,
-            chunk_size=chunk_size,
-        )
+        merge_path: str,
+        bucket: str,
+        dst_key: str,
+    ) -> Exception | None:
+        try:
+            upload_id: str = _begin_upload(
+                s3_client=self.client,
+                parts=parts,
+                bucket=bucket,
+                dst_key=dst_key,
+                verbose=self.verbose,
+            )
+            merge_state = MergeState(
+                rclone_impl=self.rclone_impl,
+                merge_path=merge_path,
+                upload_id=upload_id,
+                bucket=bucket,
+                dst_key=dst_key,
+                finished=[],
+                all_parts=parts,
+            )
+            self.state = merge_state
+            return None
+        except Exception as e:
+            return e
 
-    def start_upload_resume(
+    def _begin_resume_merge(
         self,
-        info: MultipartUploadInfo,
-        state: MergeState,
-        max_workers: int = _DEFAULT_MAX_WORKERS,
-    ) -> MultipartUploadInfo | Exception:
-        return Exception("Not implemented")
-
-    def start_upload(
+        merge_state: MergeState,
+    ) -> None:
+        self.state = merge_state
+
+    def _on_piece_finished(self, finished_piece: FinishedPiece | EndOfStream) -> None:
+        assert self.write_thread is not None
+        assert self.state is not None
+        if isinstance(finished_piece, EndOfStream):
+            self.write_thread.add_eos()
+        else:
+            self.state.on_finished(finished_piece)
+            self.write_thread.add_finished(finished_piece)
+
+    def merge(
         self,
-        info: MultipartUploadInfo,
-        parts: list[Part],
-        max_workers: int = _DEFAULT_MAX_WORKERS,
-    ) -> str | Exception:
-        return do_body_work(
-            info=info,
-            source_bucket=info.bucket_name,
-            max_workers=max_workers,
-            merge_state=MergeState(finished=[], all_parts=parts),
+    ) -> Exception | None:
+        state = self.state
+        if state is None:
+            return Exception("No merge state loaded")
+        self.start_write_thread()
+        err = _do_upload_task(
+            s3_client=self.client,
+            merge_state=state,
+            max_workers=self.max_workers,
+            on_finished=self._on_piece_finished,
         )
+        if isinstance(err, Exception):
+            return err
+        return None
+
+    def cleanup(self) -> Exception | None:
+        return _cleanup_merge(rclone=self.rclone_impl, info=self.info)
+
+
+def s3_server_side_multi_part_merge(
+    rclone: RcloneImpl,
+    info_path: str,
+    max_workers: int = DEFAULT_MAX_WORKERS,
+    verbose: bool = False,
+) -> Exception | None:
+    info = InfoJson(rclone, src=None, src_info=info_path)
+    loaded = info.load()
+    if not loaded:
+        return FileNotFoundError(
+            f"Info file not found, has the upload finished? {info_path}"
+        )
+    merger: S3MultiPartMerger | Exception = S3MultiPartMerger.create(
+        rclone=rclone, info=info, max_workers=max_workers, verbose=verbose
+    )
+    if isinstance(merger, Exception):
+        return merger
+
+    err = merger.merge()
+    if isinstance(err, Exception):
+        return err
+
+    err = merger.cleanup()
+    if isinstance(err, Exception):
+        err
+    return None

{rclone_api-1.4.18.dist-info → rclone_api-1.4.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.18
+Version: 1.4.20
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.18.dist-info → rclone_api-1.4.20.dist-info}/RECORD

@@ -27,13 +27,13 @@ rclone_api/util.py,sha256=yY72YKpmpT_ZM7AleVtPpl0YZZYQPTwTdqKn9qPwm8Y,9290
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
 rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
-rclone_api/cmd/copy_large_s3_finish.py,sha256=FeeFZsuc1Gh7Xpso3alvirTX1_eXvHoAIsddMrVdCyw,6521
+rclone_api/cmd/copy_large_s3_finish.py,sha256=EQjynfT1Iwyh7eObO-sNL26QtMmtd4GI6LkSFnGIHMo,2050
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
 rclone_api/db/db.py,sha256=YRnYrCaXHwytQt07uEZ_mMpvPHo9-0IWcOb95fVOOfs,10086
 rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
-rclone_api/detail/copy_file_parts.py,sha256=dpqZ0d7l195dZg6Vob2Ty43Uah1v0ozQu5kMtblGqYc,16175
+rclone_api/detail/copy_file_parts.py,sha256=1h-5JJmZdB0_TuVcuYMIClHqAgCXUI4eLyZHbdRiCHg,16280
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
@@ -41,17 +41,17 @@ rclone_api/s3/api.py,sha256=owoQ1H-R0hXcUozxC6sl53D7NmMOewHk2pUxK-ye8ms,4061
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
 rclone_api/s3/create.py,sha256=_Q-faQ4Zl8XKTB28gireRxVXWP-YNxoAK4bligxDtiI,3998
-rclone_api/s3/merge_state.py,sha256=FJOQvOVLvAe9E_M_UgPffrTSN2YzRYfBMnzkOtBTqwg,3939
-rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=Rww9S81pbCUUu_f72xkxU4HQ_xOf69_C5MyMmmNchcw,10339
+rclone_api/s3/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
+rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=qBUZlNooqA151u9sV6hrZKXIZbpT68kKzXMuuwJuv3s,17331
 rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
 rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
 rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaRCmkYLTQ,105
-rclone_api/s3/multipart/finished_piece.py,sha256=6ev7MFOV3dWqylJFEttOIeoaEA74RMqNWh258L_ENnY,1732
+rclone_api/s3/multipart/finished_piece.py,sha256=LtlX_mm6_hsADR8FxgfC2_pcO5Wou_20-jE34IcRXew,1633
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.18.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.18.dist-info/METADATA,sha256=Ym9QwhV3G0YNmRao9cluDShuY7AEwclfGTDEz-B2xio,4628
-rclone_api-1.4.18.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.18.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.18.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.18.dist-info/RECORD,,
+rclone_api-1.4.20.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.20.dist-info/METADATA,sha256=vd-0g-1byVjdVbQX_fN4AsQB-xA4Z1pI8vjj8RVF3N0,4628
+rclone_api-1.4.20.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.20.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.20.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.20.dist-info/RECORD,,