rclone-api 1.4.7__py2.py3-none-any.whl → 1.4.8__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3.py

@@ -11,8 +11,7 @@ class Args:
     src: str
     dst: str
     chunk_size: SizeSuffix
-    read_threads: int
-    write_threads: int
+    threads: int
     retries: int
     save_state_json: Path
     verbose: bool
@@ -40,16 +39,10 @@ def _parse_args() -> Args:
         default="128MB",  # if this is too low or too high an s3 service
     )
     parser.add_argument(
-        "--read-threads",
-        help="Number of concurrent read threads per chunk, only one chunk will be read at a time",
-        type=int,
-        default=8,
-    )
-    parser.add_argument(
-        "--write-threads",
+        "--threads",
         help="Max number of chunks to upload in parallel to the destination, each chunk is uploaded in a separate thread",
         type=int,
-        default=16,
+        default=8,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
@@ -70,9 +63,8 @@ def _parse_args() -> Args:
         config_path=config,
         src=args.src,
         dst=args.dst,
+        threads=args.threads,
         chunk_size=SizeSuffix(args.chunk_size),
-        read_threads=args.read_threads,
-        write_threads=args.write_threads,
         retries=args.retries,
         save_state_json=args.resume_json,
         verbose=args.verbose,
@@ -122,6 +114,6 @@ if __name__ == "__main__":
         "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
     )
     sys.argv.append(
-        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts"
     )
     main()

rclone_api/cmd/copy_large_s3_finish.py

@@ -0,0 +1,217 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+from rclone_api.detail.copy_file_parts import InfoJson
+from rclone_api.s3.s3_multipart_uploader_by_copy import (
+    finish_multipart_upload_from_keys,
+)
+from rclone_api.types import SizeSuffix
+
+DATA_SOURCE = (
+    "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+)
+
+
+# response = client.upload_part_copy(
+#     Bucket='string',
+#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
+#     CopySourceIfMatch='string',
+#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
+#     CopySourceIfNoneMatch='string',
+#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
+#     CopySourceRange='string',
+#     Key='string',
+#     PartNumber=123,
+#     UploadId='string',
+#     SSECustomerAlgorithm='string',
+#     SSECustomerKey='string',
+#     CopySourceSSECustomerAlgorithm='string',
+#     CopySourceSSECustomerKey='string',
+#     RequestPayer='requester',
+#     ExpectedBucketOwner='string',
+#     ExpectedSourceBucketOwner='string'
+# )
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
+    dst: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst
+    verbose: bool
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=False
+    )
+    parser.add_argument(
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
+        type=str,
+        default="128MB",  # if this is too low or too high an s3 service
+    )
+
+    args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
+    out = Args(
+        config_path=config,
+        src=args.src,
+        dst=args.dst,
+        verbose=args.verbose,
+    )
+    return out
+
+
+# from dataclasses import dataclass
+
+# def parse_info_json(text: str) -> UploadInfo:
+#     import json
+#     data = json.loads(text)
+#     chunk_size = data["chunksize_int"]
+#     first_part = data["first_part"]
+#     last_part = data["last_part"]
+#     assert isinstance(chunk_size, int)
+#     assert isinstance(first_part, int)
+#     assert isinstance(last_part, int)
+#     assert first_part <= last_part
+#     parts: list[str] = []
+#     fmt = "part.{:05d}_{}-{}"
+#     for i in range(first_part, last_part + 1):
+#         offset: int = i * chunk_size
+#         end: int = (i + 1) * chunk_size
+#         part = fmt.format(i, offset, end)
+#         parts.append(part)
+#     return UploadInfo(chunk_size=chunk_size, parts=parts)
+
+
+def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
+    from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
+
+    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
+    s3_client: BaseClient = create_s3_client(s3_creds)
+    s3_bucket = s3_creds.bucket_name
+    is_done = info.fetch_is_done()
+    assert is_done, f"Upload is not done: {info}"
+
+    parts_dir = info.parts_dir
+    if parts_dir.endswith("/"):
+        parts_dir = parts_dir[:-1]
+    source_keys = info.fetch_all_finished()
+
+    print(parts_dir)
+    print(source_keys)
+
+    parent_path = parts_dir.split(s3_bucket)[1]
+    if parent_path.startswith("/"):
+        parent_path = parent_path[1:]
+
+    first_part: int | None = info.first_part
+    last_part: int | None = info.last_part
+    size: SizeSuffix | None = info.size
+
+    assert first_part is not None
+    assert last_part is not None
+    assert size is not None
+
+    def _to_s3_key(name: str) -> str:
+        out = f"{parent_path}/{name}"
+        return out
+
+    # s3_keys: list[str] = [_to_s3_key(name=p) for p in source_keys]
+    parts: list[tuple[int, str]] = []
+    for i in range(first_part, last_part + 1):
+        part_name = f"part.{i:05d}"
+        s3_key = _to_s3_key(name=part_name)
+        parts.append((i, s3_key))
+
+    # for key in parts:
+    #     print(key)
+
+    chunksize = info.chunksize
+    assert chunksize is not None
+
+    finish_multipart_upload_from_keys(
+        s3_client=s3_client,
+        source_bucket=s3_creds.bucket_name,
+        parts=parts,
+        destination_bucket=s3_creds.bucket_name,
+        destination_key=dst,
+        chunk_size=chunksize.as_int(),
+        final_size=size.as_int(),
+        retries=3,
+    )
+
+    if False:
+        print(finish_multipart_upload_from_keys)
+        print(s3_client)
+    print("done")
+
+    # def finish_multipart_upload_from_keys(
+    # s3_client: BaseClient,
+    # source_bucket: str,
+    # source_keys: list[str],
+    # destination_bucket: str,
+    # destination_key: str,
+    # chunk_size: int = 5 * 1024 * 1024,  # 5MB default
+    # retries: int = 3,
+    # byte_ranges: list[str] | None = None,
+
+    # if False:
+    #     finish_multipart_upload_from_keys(
+    #         s3_client=s3_client,
+    #         source_bucket="TODO",
+    #         source_keys=[p.key for p in all_parts],
+    #         destination_bucket=info.dst_bucket,
+    #         destination_key=info.dst_key,
+    #         chunk_size=5 * 1024 * 1024,
+    #         retries=3,
+    #         byte_ranges=None,
+    #     )
+
+    # print(all_parts)
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    info_json = f"{args.src}/info.json".replace("//", "/")
+    info = InfoJson(rclone.impl, src=None, src_info=info_json)
+    loaded = info.load()
+    assert loaded
+    print(info)
+    do_finish_part(rclone=rclone, info=info, dst=args.dst)
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+    )
+    main()

rclone_api/detail/copy_file_parts.py

@@ -53,7 +53,7 @@ def upload_task(self: RcloneImpl, upload_part: UploadPart) -> UploadPart:
         msg = "\n#########################################\n"
         msg += f"# Uploading {upload_part.chunk} to {upload_part.dst_part}\n"
         msg += "#########################################\n"
-        print
+        print(msg)
         self.copy_to(upload_part.chunk.as_posix(), upload_part.dst_part)
         return upload_part
     except Exception as e:
@@ -106,11 +106,24 @@ def _fetch_all_names(
     return filtered
 
 
-def _get_info_json(self: RcloneImpl, src: str, src_info: str) -> dict:
+def _get_info_json(self: RcloneImpl, src: str | None, src_info: str) -> dict:
     from rclone_api.file import File
 
+    data: dict
+    text: str
+    if src is None:
+        # just try to load the file
+        text_or_err = self.read_text(src_info)
+        if isinstance(text_or_err, Exception):
+            raise FileNotFoundError(f"Could not load {src_info}: {text_or_err}")
+        assert isinstance(text_or_err, str)
+        text = text_or_err
+        data = json.loads(text)
+        return data
+
     src_stat: File | Exception = self.stat(src)
     if isinstance(src_stat, Exception):
+        # just try to load the file
         raise FileNotFoundError(f"Failed to stat {src}: {src_stat}")
 
     now: datetime = datetime.now()
@@ -133,12 +146,11 @@ def _get_info_json(self: RcloneImpl, src: str, src_info: str) -> dict:
         warnings.warn(f"Failed to read {src_info}: {text_or_err}")
         return new_data
     assert isinstance(text_or_err, str)
-    text: str = text_or_err
+    text = text_or_err
 
     if err is not None:
         return new_data
 
-    data: dict = {}
     try:
         data = json.loads(text)
         return data
@@ -168,13 +180,14 @@ def _save_info_json(self: RcloneImpl, src: str, data: dict) -> None:
 
 
 class InfoJson:
-    def __init__(self, rclone: RcloneImpl, src: str, src_info: str) -> None:
+    def __init__(self, rclone: RcloneImpl, src: str | None, src_info: str) -> None:
         self.rclone = rclone
         self.src = src
         self.src_info = src_info
         self.data: dict = {}
 
     def load(self) -> bool:
+        """Returns true if the file exist and is now loaded."""
         self.data = _get_info_json(self.rclone, self.src, self.src_info)
         return not self.data.get("new", False)
 
@@ -194,16 +207,60 @@ class InfoJson:
         part_numbers = [int(name.split("_")[0].split(".")[1]) for name in names]
         return part_numbers
 
+    @property
+    def parts_dir(self) -> str:
+        return os.path.dirname(self.src_info)
+
+    def compute_all_parts(self) -> list[PartInfo] | Exception:
+        # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
+        # src_size, SizeSuffix("96MB")
+        try:
+
+            src_size = self.size
+            chunk_size = self.chunksize
+            assert isinstance(src_size, SizeSuffix)
+            assert isinstance(chunk_size, SizeSuffix)
+            first_part = self.data["first_part"]
+            last_part = self.data["last_part"]
+            full_part_infos: list[PartInfo] = PartInfo.split_parts(src_size, chunk_size)
+            return full_part_infos[first_part : last_part + 1]
+        except Exception as e:
+            return e
+
+    def compute_all_part_numbers(self) -> list[int] | Exception:
+        all_parts: list[PartInfo] | Exception = self.compute_all_parts()
+        if isinstance(all_parts, Exception):
+            raise all_parts
+
+        all_part_nums: list[int] = [p.part_number for p in all_parts]
+        return all_part_nums
+
+    def fetch_remaining_part_numbers(self) -> list[int] | Exception:
+        all_part_nums: list[int] | Exception = self.compute_all_part_numbers()
+        if isinstance(all_part_nums, Exception):
+            return all_part_nums
+        finished_part_nums: list[int] = self.fetch_all_finished_part_numbers()
+        remaining_part_nums: list[int] = list(
+            set(all_part_nums) - set(finished_part_nums)
+        )
+        return sorted(remaining_part_nums)
+
+    def fetch_is_done(self) -> bool:
+        remaining_part_nums: list[int] | Exception = self.fetch_remaining_part_numbers()
+        if isinstance(remaining_part_nums, Exception):
+            return False
+        return len(remaining_part_nums) == 0
+
     @property
     def new(self) -> bool:
         return self.data.get("new", False)
 
     @property
     def chunksize(self) -> SizeSuffix | None:
-        chunksize: str | None = self.data.get("chunksize")
-        if chunksize is None:
+        chunksize_int: int | None = self.data.get("chunksize_int")
+        if chunksize_int is None:
             return None
-        return SizeSuffix(chunksize)
+        return SizeSuffix(chunksize_int)
 
     @chunksize.setter
     def chunksize(self, value: SizeSuffix) -> None:
@@ -219,21 +276,24 @@ class InfoJson:
         self.data["src_modtime"] = value.isoformat()
 
     @property
-    def first_part(self) -> int | None:
+    def size(self) -> SizeSuffix:
+        return SizeSuffix(self.data["size"])
+
+    def _get_first_part(self) -> int | None:
         return self.data.get("first_part")
 
-    @first_part.setter
-    def first_part(self, value: int) -> None:
+    def _set_first_part(self, value: int) -> None:
         self.data["first_part"] = value
 
-    @property
-    def last_part(self) -> int | None:
+    def _get_last_part(self) -> int | None:
         return self.data.get("last_part")
 
-    @last_part.setter
-    def last_part(self, value: int) -> None:
+    def _set_last_part(self, value: int) -> None:
         self.data["last_part"] = value
 
+    first_part: int | None = property(_get_first_part, _set_first_part)  # type: ignore
+    last_part: int | None = property(_get_last_part, _set_last_part)  # type: ignore
+
     @property
     def hash(self) -> str | None:
         return self.data.get("hash")

rclone_api/process.py

@@ -5,11 +5,10 @@ import time
 import weakref
 from dataclasses import dataclass
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from typing import Any
 
 from rclone_api.config import Config
-from rclone_api.util import get_verbose
+from rclone_api.util import clear_temp_config_file, get_verbose, make_temp_config_file
 
 
 @dataclass
@@ -28,17 +27,14 @@ class Process:
         assert args.rclone_exe.exists()
         self.args = args
         self.log = args.log
-        self.tempdir: TemporaryDirectory | None = None
+        self.tempfile: Path | None = None
         verbose = get_verbose(args.verbose)
         if isinstance(args.rclone_conf, Config):
-            self.tempdir = TemporaryDirectory()
-            tmpfile = Path(self.tempdir.name) / "rclone.conf"
-            tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
-            rclone_conf = tmpfile
-            self.needs_cleanup = True
+            self.tmpfile = make_temp_config_file()
+            self.tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
+            rclone_conf = self.tmpfile
         else:
             rclone_conf = args.rclone_conf
-            self.needs_cleanup = False
 
         assert rclone_conf.exists()
 
@@ -81,11 +77,7 @@ class Process:
         self.cleanup()
 
     def cleanup(self) -> None:
-        if self.tempdir and self.needs_cleanup:
-            try:
-                self.tempdir.cleanup()
-            except Exception as e:
-                print(f"Error cleaning up tempdir: {e}")
+        clear_temp_config_file(self.tempfile)
 
     def _atexit_terminate(self) -> None:
         """

rclone_api/rclone_impl.py

@@ -32,6 +32,7 @@ from rclone_api.mount import Mount, clean_mount, prepare_mount
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.s3.create import S3Credentials
 from rclone_api.s3.types import (
     MultiUploadResult,
     S3MutliPartUploadConfig,
@@ -862,6 +863,64 @@ class RcloneImpl:
             )
         return SizeSuffix(out.total_size)
 
+    def get_s3_credentials(
+        self, remote: str, verbose: bool | None = None
+    ) -> S3Credentials:
+        from rclone_api.util import S3PathInfo, split_s3_path
+
+        verbose = get_verbose(verbose)
+        path_info: S3PathInfo = split_s3_path(remote)
+
+        # path_info: S3PathInfo = split_s3_path(remote)
+        remote = path_info.remote
+        bucket_name = path_info.bucket
+
+        remote = path_info.remote
+        parsed: Parsed = self.config.parse()
+        sections: dict[str, Section] = parsed.sections
+        if remote not in sections:
+            raise ValueError(
+                f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
+            )
+
+        section: Section = sections[remote]
+        dst_type = section.type()
+        if dst_type != "s3" and dst_type != "b2":
+            raise ValueError(
+                f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
+            )
+
+        def get_provider_str(section=section) -> str | None:
+            type: str = section.type()
+            provider: str | None = section.provider()
+            if provider is not None:
+                return provider
+            if type == "b2":
+                return S3Provider.BACKBLAZE.value
+            if type != "s3":
+                raise ValueError(f"Remote {remote} is not an S3 remote")
+            return S3Provider.S3.value
+
+        provider: str
+        if provided_provider_str := get_provider_str():
+            if verbose:
+                print(f"Using provided provider: {provided_provider_str}")
+            provider = provided_provider_str
+        else:
+            if verbose:
+                print(f"Using default provider: {S3Provider.S3.value}")
+            provider = S3Provider.S3.value
+        provider_enum = S3Provider.from_str(provider)
+
+        s3_creds: S3Credentials = S3Credentials(
+            bucket_name=bucket_name,
+            provider=provider_enum,
+            access_key_id=section.access_key_id(),
+            secret_access_key=section.secret_access_key(),
+            endpoint_url=section.endpoint(),
+        )
+        return s3_creds
+
     def copy_file_resumable_s3(
         self,
         src: str,
@@ -878,7 +937,6 @@ class RcloneImpl:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.http_server import HttpFetcher, HttpServer
         from rclone_api.s3.api import S3Client
-        from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, split_s3_path
 
         src_path = Path(src)
@@ -909,51 +967,10 @@ class RcloneImpl:
             )
 
         path_info: S3PathInfo = split_s3_path(dst)
-        remote = path_info.remote
+        # remote = path_info.remote
         bucket_name = path_info.bucket
         s3_key = path_info.key
-        parsed: Parsed = self.config.parse()
-        sections: dict[str, Section] = parsed.sections
-        if remote not in sections:
-            raise ValueError(
-                f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
-            )
-
-        section: Section = sections[remote]
-        dst_type = section.type()
-        if dst_type != "s3" and dst_type != "b2":
-            raise ValueError(
-                f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
-            )
-
-        def get_provider_str(section=section) -> str | None:
-            type: str = section.type()
-            provider: str | None = section.provider()
-            if provider is not None:
-                return provider
-            if type == "b2":
-                return S3Provider.BACKBLAZE.value
-            if type != "s3":
-                raise ValueError(f"Remote {remote} is not an S3 remote")
-            return S3Provider.S3.value
-
-        provider: str
-        if provided_provider_str := get_provider_str():
-            if verbose:
-                print(f"Using provided provider: {provided_provider_str}")
-            provider = provided_provider_str
-        else:
-            if verbose:
-                print(f"Using default provider: {S3Provider.S3.value}")
-            provider = S3Provider.S3.value
-        provider_enum = S3Provider.from_str(provider)
-
-        s3_creds: S3Credentials = S3Credentials(
-            provider=provider_enum,
-            access_key_id=section.access_key_id(),
-            secret_access_key=section.secret_access_key(),
-            endpoint_url=section.endpoint(),
-        )
+        s3_creds: S3Credentials = self.get_s3_credentials(dst, verbose=verbose)
 
         port = random.randint(10000, 20000)
         http_server: HttpServer = self.serve_http(

rclone_api/s3/multipart/finished_piece.py

@@ -1,4 +1,5 @@
 import json
+import warnings
 from dataclasses import dataclass
 
 from rclone_api.types import EndOfStream
@@ -28,7 +29,9 @@ class FinishedPiece:
         for p in parts:
             if p is EndOfStream:
                 count_eos += 1
-        assert count_eos <= 1, "Only one EndOfStream should be present"
+        # assert count_eos <= 1, "Only one EndOfStream should be present"
+        if count_eos > 1:
+            warnings.warn(f"Only one EndOfStream should be present, found {count_eos}")
         return [p.to_json() for p in non_none]
 
     @staticmethod

rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -0,0 +1,395 @@
+"""
+https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
+  *  client.upload_part_copy
+
+This module provides functionality for S3 multipart uploads, including copying parts
+from existing S3 objects using upload_part_copy.
+"""
+
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from botocore.client import BaseClient
+
+from rclone_api.s3.multipart.finished_piece import FinishedPiece
+from rclone_api.util import locked_print
+
+
+@dataclass
+class MultipartUploadInfo:
+    """Simplified upload information for multipart uploads."""
+
+    s3_client: BaseClient
+    bucket_name: str
+    object_name: str
+    upload_id: str
+    chunk_size: int
+    retries: int
+    file_size: Optional[int] = None
+    src_file_path: Optional[Path] = None
+
+
+# response = client.upload_part_copy(
+#     Bucket='string',
+#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
+#     CopySourceIfMatch='string',
+#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
+#     CopySourceIfNoneMatch='string',
+#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
+#     CopySourceRange='string',
+#     Key='string',
+#     PartNumber=123,
+#     UploadId='string',
+#     SSECustomerAlgorithm='string',
+#     SSECustomerKey='string',
+#     CopySourceSSECustomerAlgorithm='string',
+#     CopySourceSSECustomerKey='string',
+#     RequestPayer='requester',
+#     ExpectedBucketOwner='string',
+#     ExpectedSourceBucketOwner='string'
+# )
+
+# import _thread
+# import os
+# import traceback
+# import warnings
+# from concurrent.futures import Future, ThreadPoolExecutor
+# from pathlib import Path
+# from queue import Queue
+# from threading import Event, Thread
+# from typing import Any, Callable
+
+# from botocore.client import BaseClient
+
+# from rclone_api.mount_read_chunker import FilePart
+# from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
+# from rclone_api.s3.chunk_types import (
+#     FinishedPiece,
+#     UploadInfo,
+#     UploadState,
+# )
+# from rclone_api.s3.types import MultiUploadResult
+# from rclone_api.types import EndOfStream
+# from rclone_api.util import locked_print
+
+
+# This is how you upload large parts through multi part upload, then the final call
+# is to assemble the parts that have already been uploaded through a multi part uploader
+# and then call complete_multipart_upload to finish the upload
+# response = (
+#  client.upload_part_copy(
+#     Bucket='string',
+#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
+#     CopySourceIfMatch='string',
+#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
+#     CopySourceIfNoneMatch='string',
+#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
+#     CopySourceRange='string',
+#     Key='string',
+#     PartNumber=123,
+#     UploadId='string',
+#     SSECustomerAlgorithm='string',
+#     SSECustomerKey='string',
+#     CopySourceSSECustomerAlgorithm='string',
+#     CopySourceSSECustomerKey='string',
+#     RequestPayer='requester',
+#     ExpectedBucketOwner='string',
+#     ExpectedSourceBucketOwner='string'
+# )
+
+
+# def upload_task(
+#     info: UploadInfo,
+#     chunk: FilePart,
+#     part_number: int,
+#     retries: int,
+# ) -> FinishedPiece:
+#     file_or_err: Path | Exception = chunk.get_file()
+#     if isinstance(file_or_err, Exception):
+#         raise file_or_err
+#     file: Path = file_or_err
+#     size = os.path.getsize(file)
+#     retries = retries + 1  # Add one for the initial attempt
+#     for retry in range(retries):
+#         try:
+#             if retry > 0:
+#                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
+#             locked_print(
+#                 f"Uploading part {part_number} for {info.src_file_path} of size {size}"
+#             )
+
+#             with open(file, "rb") as f:
+#                 part = info.s3_client.upload_part(
+#                     Bucket=info.bucket_name,
+#                     Key=info.object_name,
+#                     PartNumber=part_number,
+#                     UploadId=info.upload_id,
+#                     Body=f,
+#                 )
+#                 out: FinishedPiece = FinishedPiece(
+#                     etag=part["ETag"], part_number=part_number
+#                 )
+#             chunk.dispose()
+#             return out
+#         except Exception as e:
+#             if retry == retries - 1:
+#                 locked_print(f"Error uploading part {part_number}: {e}")
+#                 chunk.dispose()
+#                 raise e
+#             else:
+#                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
+#                 continue
+#     raise Exception("Should not reach here")
+
+
+# def prepare_upload_file_multipart(
+#     s3_client: BaseClient,
+#     bucket_name: str,
+#     file_path: Path,
+#     file_size: int | None,
+#     object_name: str,
+#     chunk_size: int,
+#     retries: int,
+# ) -> UploadInfo:
+#     """Upload a file to the bucket using multipart upload with customizable chunk size."""
+
+#     # Initiate multipart upload
+#     locked_print(
+#         f"Creating multipart upload for {file_path} to {bucket_name}/{object_name}"
+#     )
+#     mpu = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)
+#     upload_id = mpu["UploadId"]
+
+#     file_size = file_size if file_size is not None else os.path.getsize(file_path)
+
+#     upload_info: UploadInfo = UploadInfo(
+#         s3_client=s3_client,
+#         bucket_name=bucket_name,
+#         object_name=object_name,
+#         src_file_path=file_path,
+#         upload_id=upload_id,
+#         retries=retries,
+#         chunk_size=chunk_size,
+#         file_size=file_size,
+#     )
+#     return upload_info
+
+
+def upload_part_copy_task(
+    info: MultipartUploadInfo,
+    source_bucket: str,
+    source_key: str,
+    part_number: int,
+    retries: int = 3,
+) -> FinishedPiece:
+    """
+    Upload a part by copying from an existing S3 object.
+
+    Args:
+        info: Upload information
+        source_bucket: Source bucket name
+        source_key: Source object key
+        part_number: Part number (1-10000)
+        byte_range: Optional byte range in format 'bytes=start-end'
+        retries: Number of retry attempts
+
+    Returns:
+        FinishedPiece with ETag and part number
+    """
+    copy_source = {"Bucket": source_bucket, "Key": source_key}
+
+    retries = retries + 1  # Add one for the initial attempt
+    for retry in range(retries):
+        try:
+            if retry > 0:
+                locked_print(f"Retrying part copy {part_number} for {info.object_name}")
+
+            locked_print(
+                f"Copying part {part_number} for {info.object_name} from {source_bucket}/{source_key}"
+            )
+
+            # Prepare the upload_part_copy parameters
+            params = {
+                "Bucket": info.bucket_name,
+                "CopySource": copy_source,
+                "Key": info.object_name,
+                "PartNumber": part_number,
+                "UploadId": info.upload_id,
+            }
+
+            # Execute the copy operation
+            part = info.s3_client.upload_part_copy(**params)
+
+            # Extract ETag from the response
+            etag = part["CopyPartResult"]["ETag"]
+
+            return FinishedPiece(etag=etag, part_number=part_number)
+
+        except Exception as e:
+            if retry == retries - 1:
+                locked_print(f"Error copying part {part_number}: {e}")
+                raise e
+            else:
+                locked_print(f"Error copying part {part_number}: {e}, retrying")
+                continue
+
+    raise Exception("Should not reach here")
+
+
+def complete_multipart_upload_from_parts(
+    info: MultipartUploadInfo, parts: list[FinishedPiece]
+) -> str:
+    """
+    Complete a multipart upload using the provided parts.
+
+    Args:
+        info: Upload information
+        parts: List of finished pieces with ETags
+
+    Returns:
+        The URL of the completed object
+    """
+    # Sort parts by part number to ensure correct order
+    parts.sort(key=lambda x: x.part_number)
+
+    # Prepare the parts list for the complete_multipart_upload call
+    multipart_parts = [
+        {"ETag": part.etag, "PartNumber": part.part_number} for part in parts
+    ]
+
+    # Complete the multipart upload
+    response = info.s3_client.complete_multipart_upload(
+        Bucket=info.bucket_name,
+        Key=info.object_name,
+        UploadId=info.upload_id,
+        MultipartUpload={"Parts": multipart_parts},
+    )
+
+    # Return the URL of the completed object
+    return response.get("Location", f"s3://{info.bucket_name}/{info.object_name}")
+
+
+def finish_multipart_upload_from_keys(
+    s3_client: BaseClient,
+    source_bucket: str,
+    parts: list[tuple[int, str]],
+    final_size: int,
+    destination_bucket: str,
+    destination_key: str,
+    chunk_size: int,  # 5MB default
+    max_workers: int = 100,
+    retries: int = 3,
+) -> str:
+    """
+    Finish a multipart upload by copying parts from existing S3 objects.
+
+    Args:
+        s3_client: Boto3 S3 client
+        source_bucket: Source bucket name
+        source_keys: List of source object keys to copy from
+        destination_bucket: Destination bucket name
+        destination_key: Destination object key
+        chunk_size: Size of each part in bytes
+        retries: Number of retry attempts
+        byte_ranges: Optional list of byte ranges corresponding to source_keys
+
+    Returns:
+        The URL of the completed object
+    """
+
+    # Initiate multipart upload
+    locked_print(
+        f"Creating multipart upload for {destination_bucket}/{destination_key} from {len(parts)} source objects"
+    )
+    mpu = s3_client.create_multipart_upload(
+        Bucket=destination_bucket, Key=destination_key
+    )
+    upload_id = mpu["UploadId"]
+
+    # Create upload info
+    upload_info = MultipartUploadInfo(
+        s3_client=s3_client,
+        bucket_name=destination_bucket,
+        object_name=destination_key,
+        upload_id=upload_id,
+        retries=retries,
+        chunk_size=chunk_size,
+        file_size=final_size,
+    )
+
+    futures: list[Future[FinishedPiece]] = []
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        for part_number, source_key in parts:
+
+            def task(
+                info=upload_info,
+                source_bucket=source_bucket,
+                source_key=source_key,
+                part_number=part_number,
+                retries=retries,
+            ):
+                return upload_part_copy_task(
+                    info=info,
+                    source_bucket=source_bucket,
+                    source_key=source_key,
+                    part_number=part_number,
+                    retries=retries,
+                )
+
+            fut = executor.submit(task)
+            futures.append(fut)
+
+    # Upload parts by copying from source objects
+    finished_parts = []
+
+    for fut in futures:
+        finished_part = fut.result()
+        finished_parts.append(finished_part)
+
+    # Complete the multipart upload
+    return complete_multipart_upload_from_parts(upload_info, finished_parts)
+
+
+class S3MultiPartUploader:
+    def __init__(self, s3_client: BaseClient, verbose: bool) -> None:
+        self.s3_client = s3_client
+        self.verbose = verbose
+
+    def finish_from_keys(
+        self,
+        source_bucket: str,
+        parts: list[tuple[int, str]],
+        destination_bucket: str,
+        destination_key: str,
+        chunk_size: int,
+        final_size: int,
+        retries: int = 3,
+    ) -> str:
+        """
+        Finish a multipart upload by copying parts from existing S3 objects.
+
+        Args:
+            source_bucket: Source bucket name
+            source_keys: List of source object keys to copy from
+            destination_bucket: Destination bucket name
+            destination_key: Destination object key
+            chunk_size: Size of each part in bytes
+            retries: Number of retry attempts
+            byte_ranges: Optional list of byte ranges corresponding to source_keys
+
+        Returns:
+            The URL of the completed object
+        """
+        return finish_multipart_upload_from_keys(
+            s3_client=self.s3_client,
+            source_bucket=source_bucket,
+            parts=parts,
+            destination_bucket=destination_bucket,
+            destination_key=destination_key,
+            chunk_size=chunk_size,
+            final_size=final_size,
+            retries=retries,
+        )

rclone_api/s3/types.py

@@ -26,6 +26,7 @@ class S3Provider(Enum):
 class S3Credentials:
     """Credentials for accessing S3."""
 
+    bucket_name: str
     provider: S3Provider
     access_key_id: str
     secret_access_key: str

rclone_api/types.py

@@ -309,6 +309,13 @@ class Range:
         val = f"bytes={self.start.as_int()}-{last.as_int()}"
         return {"Range": val}
 
+    def __repr__(self) -> str:
+        length = self.end - self.start
+        return f"Range(start={self.start}, length={length})"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
 
 _MAX_PART_NUMBER = 10000
 
@@ -387,3 +394,9 @@ class PartInfo:
         end = SizeSuffix(self.range.end._size).as_int()
         dst_name = f"part.{partnumber}_{offset}-{end}"
         return dst_name
+
+    def __repr__(self) -> str:
+        return f"PartInfo(part_number={self.part_number}, range={self.range})"
+
+    def __str__(self) -> str:
+        return self.__repr__()

rclone_api/util.py

@@ -1,10 +1,11 @@
+import atexit
 import os
 import random
 import shutil
+import signal
 import subprocess
 import warnings
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from threading import Lock
 from typing import Any
 
@@ -18,6 +19,53 @@ from rclone_api.types import S3PathInfo
 
 _PRINT_LOCK = Lock()
 
+_TMP_CONFIG_DIR = Path(".") / ".rclone" / "tmp_config"
+_RCLONE_CONFIGS_LIST: list[Path] = []
+_DO_CLEANUP = os.getenv("RCLONE_API_CLEANUP", "1") == "1"
+
+
+def _clean_configs(signum=None, frame=None) -> None:
+    if not _DO_CLEANUP:
+        return
+    for config in _RCLONE_CONFIGS_LIST:
+        try:
+            config.unlink()
+        except Exception as e:
+            print(f"Error deleting config file: {config}, {e}")
+    _RCLONE_CONFIGS_LIST.clear()
+    if signum is not None:
+        signal.signal(signum, signal.SIG_DFL)
+        os.kill(os.getpid(), signum)
+
+
+def _init_cleanup() -> None:
+    atexit.register(_clean_configs)
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        signal.signal(sig, _clean_configs)
+
+
+_init_cleanup()
+
+
+def make_temp_config_file() -> Path:
+    from rclone_api.util import random_str
+
+    tmpdir = _TMP_CONFIG_DIR / random_str(32)
+    tmpdir.mkdir(parents=True, exist_ok=True)
+    tmpfile = tmpdir / "rclone.conf"
+    _RCLONE_CONFIGS_LIST.append(tmpfile)
+    return tmpfile
+
+
+def clear_temp_config_file(path: Path | None) -> None:
+    if (path is None) or (not path.exists()) or (not _DO_CLEANUP):
+        return
+    try:
+        path.unlink()
+    except Exception as e:
+        print(f"Error deleting config file: {path}, {e}")
+
 
 def locked_print(*args, **kwargs):
     with _PRINT_LOCK:
@@ -116,7 +164,7 @@ def rclone_execute(
     capture: bool | Path | None = None,
     verbose: bool | None = None,
 ) -> subprocess.CompletedProcess:
-    tempdir: TemporaryDirectory | None = None
+    tmpfile: Path | None = None
     verbose = get_verbose(verbose)
 
     # Handle the Path case for capture
@@ -131,8 +179,7 @@ def rclone_execute(
 
     try:
         if isinstance(rclone_conf, Config):
-            tempdir = TemporaryDirectory()
-            tmpfile = Path(tempdir.name) / "rclone.conf"
+            tmpfile = make_temp_config_file()
             tmpfile.write_text(rclone_conf.text, encoding="utf-8")
             rclone_conf = tmpfile
         cmd = (
@@ -168,11 +215,7 @@ def rclone_execute(
                 )
         return cp
     finally:
-        if tempdir:
-            try:
-                tempdir.cleanup()
-            except Exception as e:
-                print(f"Error cleaning up tempdir: {e}")
+        clear_temp_config_file(tmpfile)
 
 
 def split_s3_path(path: str) -> S3PathInfo:

{rclone_api-1.4.7.dist-info → rclone_api-1.4.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.7
+Version: 1.4.8
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.7.dist-info → rclone_api-1.4.8.dist-info}/RECORD

@@ -17,22 +17,23 @@ rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,803
 rclone_api/http_server.py,sha256=3fPBV6l50erTe32DyeJBNmsDrn5KuujsbmEAbx13T-c,8720
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
-rclone_api/process.py,sha256=BGXJTZVT__jeaDyjN8_kRycliOhkBErMPdHO1hKRvJE,5271
-rclone_api/rclone_impl.py,sha256=HTvWbD0cWhqrIfUPzqCDp3Av1uCEBOoVO0BQrBYmjQM,48205
+rclone_api/process.py,sha256=I7B4arAIbFcTBsek27cZ0t-l5YRWVHJJPji7G6ZLGjQ,4989
+rclone_api/rclone_impl.py,sha256=xTTriz6-zn_aSrkY8B7wzT-zRXax7Og7ns6xu6-7O6g,48769
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
-rclone_api/types.py,sha256=gpEYVHNkxB7X3p_B4nEc1ls1kF_ykjNuUyZXCsvU-Cs,11788
-rclone_api/util.py,sha256=j252WPB-UMiz6zQcLvsZe9XvMq-Z2FIeZrjb-y01eL4,5947
+rclone_api/types.py,sha256=HkpEZgZWhr5Gb04iHq5NxMRXxieWoN-PKmOfJFrg5Qg,12155
+rclone_api/util.py,sha256=9w_m6W62l_X42Jw5q8p_p30h-QoxAqufvnCLI4PTMOE,7056
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
-rclone_api/cmd/copy_large_s3.py,sha256=w1aCaq9EQ84aRoMj9mHtdr0Svjflkx6KTKcoldzRSo8,3788
+rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
+rclone_api/cmd/copy_large_s3_finish.py,sha256=ybZWaRa_89lyq7WpXUjpbhc_TQctUC-fKCZmdffPpKo,6731
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
 rclone_api/db/db.py,sha256=YRnYrCaXHwytQt07uEZ_mMpvPHo9-0IWcOb95fVOOfs,10086
 rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
-rclone_api/detail/copy_file_parts.py,sha256=CmkMqGTYe2F7MQ4scHy-ZJoaKKILVIaqAUmidHV3adE,13412
+rclone_api/detail/copy_file_parts.py,sha256=CXiFuCJKSAdopDzigdO0j8uYrNl1N2Y9X9sbeGffqDU,15919
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
@@ -40,16 +41,16 @@ rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
-rclone_api/s3/s3_multipart_uploader.py,sha256=KY9k585Us0VW8uqgS5jdSVrYynxlnO8Wzb52pcvR06M,4570
-rclone_api/s3/types.py,sha256=VqnvH0qhvb3_4wngqk0vSSStH-TgVQxNNxo9slz9-p8,1595
+rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=kXKAqpxi6WuyTjJ4BsBQ0wH3Px8aJEc8CH9Ugcni82A,12944
+rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
 rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
 rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaRCmkYLTQ,105
-rclone_api/s3/multipart/finished_piece.py,sha256=TcwA58-qgKBiskfHrePoCWaSSep6Za9psZEpzrLUUhE,1199
+rclone_api/s3/multipart/finished_piece.py,sha256=9nMWnVZ8S99wi2VFQsm1h1ZHqmebkhMGgd2s56wNj9w,1331
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.7.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.7.dist-info/METADATA,sha256=kFj6G3KlaQXxERlakrTl-RW0KUJHwSWckEKwJ-1iVb8,4627
-rclone_api-1.4.7.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.7.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.7.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.7.dist-info/RECORD,,
+rclone_api-1.4.8.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.8.dist-info/METADATA,sha256=GfvhP_JxvsCbW0r6NVyiEHzkac5QhkVjasG7vyP8IMo,4627
+rclone_api-1.4.8.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.8.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.8.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.8.dist-info/RECORD,,

rclone_api/s3/s3_multipart_uploader.py

@@ -1,138 +0,0 @@
-"""
-https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
-  *  client.upload_part_copy
-
-
-"""
-
-# import _thread
-# import os
-# import traceback
-# import warnings
-# from concurrent.futures import Future, ThreadPoolExecutor
-# from pathlib import Path
-# from queue import Queue
-# from threading import Event, Thread
-# from typing import Any, Callable
-
-# from botocore.client import BaseClient
-
-# from rclone_api.mount_read_chunker import FilePart
-# from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
-# from rclone_api.s3.chunk_types import (
-#     FinishedPiece,
-#     UploadInfo,
-#     UploadState,
-# )
-# from rclone_api.s3.types import MultiUploadResult
-# from rclone_api.types import EndOfStream
-# from rclone_api.util import locked_print
-
-
-# This is how you upload large parts through multi part upload, then the final call
-# is to assemble the parts that have already been uploaded through a multi part uploader
-# and then call complete_multipart_upload to finish the upload
-# response = (
-#  client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-
-# def upload_task(
-#     info: UploadInfo,
-#     chunk: FilePart,
-#     part_number: int,
-#     retries: int,
-# ) -> FinishedPiece:
-#     file_or_err: Path | Exception = chunk.get_file()
-#     if isinstance(file_or_err, Exception):
-#         raise file_or_err
-#     file: Path = file_or_err
-#     size = os.path.getsize(file)
-#     retries = retries + 1  # Add one for the initial attempt
-#     for retry in range(retries):
-#         try:
-#             if retry > 0:
-#                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
-#             locked_print(
-#                 f"Uploading part {part_number} for {info.src_file_path} of size {size}"
-#             )
-
-#             with open(file, "rb") as f:
-#                 part = info.s3_client.upload_part(
-#                     Bucket=info.bucket_name,
-#                     Key=info.object_name,
-#                     PartNumber=part_number,
-#                     UploadId=info.upload_id,
-#                     Body=f,
-#                 )
-#                 out: FinishedPiece = FinishedPiece(
-#                     etag=part["ETag"], part_number=part_number
-#                 )
-#             chunk.dispose()
-#             return out
-#         except Exception as e:
-#             if retry == retries - 1:
-#                 locked_print(f"Error uploading part {part_number}: {e}")
-#                 chunk.dispose()
-#                 raise e
-#             else:
-#                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
-#                 continue
-#     raise Exception("Should not reach here")
-
-
-# def prepare_upload_file_multipart(
-#     s3_client: BaseClient,
-#     bucket_name: str,
-#     file_path: Path,
-#     file_size: int | None,
-#     object_name: str,
-#     chunk_size: int,
-#     retries: int,
-# ) -> UploadInfo:
-#     """Upload a file to the bucket using multipart upload with customizable chunk size."""
-
-#     # Initiate multipart upload
-#     locked_print(
-#         f"Creating multipart upload for {file_path} to {bucket_name}/{object_name}"
-#     )
-#     mpu = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)
-#     upload_id = mpu["UploadId"]
-
-#     file_size = file_size if file_size is not None else os.path.getsize(file_path)
-
-#     upload_info: UploadInfo = UploadInfo(
-#         s3_client=s3_client,
-#         bucket_name=bucket_name,
-#         object_name=object_name,
-#         src_file_path=file_path,
-#         upload_id=upload_id,
-#         retries=retries,
-#         chunk_size=chunk_size,
-#         file_size=file_size,
-#     )
-#     return upload_info
-
-# class S3MultiPartUploader:
-#     def __init__(self, s3_client: BaseClient, verbose: bool) -> None:
-#         self.s3_client = s3_client
-#         self.verbose = verbose
-
-#     def prepare(self) -> UploadInfo: