rclone-api 1.4.8__tar.gz → 1.4.10__tar.gz

{rclone_api-1.4.8 → rclone_api-1.4.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.8
+Version: 1.4.10
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.8 → rclone_api-1.4.10}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.8"
+version = "1.4.10"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.8 → rclone_api-1.4.10}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -116,13 +116,12 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     if parts_dir.endswith("/"):
         parts_dir = parts_dir[:-1]
     source_keys = info.fetch_all_finished()
+    # print(parts_dir)
+    # print(source_keys)
 
-    print(parts_dir)
-    print(source_keys)
-
-    parent_path = parts_dir.split(s3_bucket)[1]
-    if parent_path.startswith("/"):
-        parent_path = parent_path[1:]
+    parts_path = parts_dir.split(s3_bucket)[1]
+    if parts_path.startswith("/"):
+        parts_path = parts_path[1:]
 
     first_part: int | None = info.first_part
     last_part: int | None = info.last_part
@@ -132,16 +131,20 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     assert last_part is not None
     assert size is not None
 
-    def _to_s3_key(name: str) -> str:
-        out = f"{parent_path}/{name}"
+    def _to_s3_key(name: str | None) -> str:
+        if name:
+            out = f"{parts_path}/{name}"
+            return out
+        out = f"{parts_path}"
         return out
 
     # s3_keys: list[str] = [_to_s3_key(name=p) for p in source_keys]
     parts: list[tuple[int, str]] = []
-    for i in range(first_part, last_part + 1):
-        part_name = f"part.{i:05d}"
-        s3_key = _to_s3_key(name=part_name)
-        parts.append((i, s3_key))
+    part_num = 1
+    for part_key in source_keys:
+        s3_key = _to_s3_key(name=part_key)
+        parts.append((part_num, s3_key))
+        part_num += 1
 
     # for key in parts:
     #     print(key)
@@ -149,14 +152,22 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     chunksize = info.chunksize
     assert chunksize is not None
 
+    import os
+
+    dst_name = info.dst_name
+    dst_dir = os.path.dirname(parts_path)
+    # dst_key =
+    dst_key = f"{dst_dir}/{dst_name}"
+
     finish_multipart_upload_from_keys(
         s3_client=s3_client,
         source_bucket=s3_creds.bucket_name,
         parts=parts,
         destination_bucket=s3_creds.bucket_name,
-        destination_key=dst,
+        destination_key=dst_key,
         chunk_size=chunksize.as_int(),
         final_size=size.as_int(),
+        max_workers=100,
         retries=3,
     )
 

{rclone_api-1.4.8 → rclone_api-1.4.10}/src/rclone_api/detail/copy_file_parts.py

@@ -211,6 +211,17 @@ class InfoJson:
     def parts_dir(self) -> str:
         return os.path.dirname(self.src_info)
 
+    @property
+    def dst(self) -> str:
+        parts_dir = self.parts_dir
+        assert parts_dir.endswith("-parts")
+        out = parts_dir[:-6]
+        return out
+
+    @property
+    def dst_name(self) -> str:
+        return os.path.basename(self.dst)
+
     def compute_all_parts(self) -> list[PartInfo] | Exception:
         # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
         # src_size, SizeSuffix("96MB")

{rclone_api-1.4.8 → rclone_api-1.4.10}/src/rclone_api/s3/create.py

@@ -25,7 +25,7 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
         aws_access_key_id=access_key,
         aws_secret_access_key=secret_key,
         endpoint_url=endpoint_url,
-        verify=False,  # Disables SSL certificate verification
+        # verify=False,  # Disables SSL certificate verification
         config=Config(
             signature_version="s3v4",
             region_name=region_name,

{rclone_api-1.4.8 → rclone_api-1.4.10}/src/rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -9,6 +9,7 @@ from existing S3 objects using upload_part_copy.
 from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
+from threading import Semaphore
 from typing import Optional
 
 from botocore.client import BaseClient
@@ -183,7 +184,7 @@ def upload_part_copy_task(
     source_key: str,
     part_number: int,
     retries: int = 3,
-) -> FinishedPiece:
+) -> FinishedPiece | Exception:
     """
     Upload a part by copying from an existing S3 object.
 
@@ -200,8 +201,11 @@ def upload_part_copy_task(
     """
     copy_source = {"Bucket": source_bucket, "Key": source_key}
 
+    # from botocore.exceptions import NoSuchKey
+
     retries = retries + 1  # Add one for the initial attempt
     for retry in range(retries):
+        params: dict = {}
         try:
             if retry > 0:
                 locked_print(f"Retrying part copy {part_number} for {info.object_name}")
@@ -226,16 +230,23 @@ def upload_part_copy_task(
             etag = part["CopyPartResult"]["ETag"]
 
             return FinishedPiece(etag=etag, part_number=part_number)
+        # except NoSuchKey as e:
+        #     locked_print(f"Error copying part {part_number}: {e}")
+        #     return e
 
         except Exception as e:
+            msg = f"Error copying {copy_source} -> {info.object_name}: {e}, params={params}"
+            if "NoSuchKey" in str(e):
+                locked_print(msg)
+                return e
             if retry == retries - 1:
-                locked_print(f"Error copying part {part_number}: {e}")
-                raise e
+                locked_print(msg)
+                return e
             else:
-                locked_print(f"Error copying part {part_number}: {e}, retrying")
+                locked_print(f"{msg}, retrying")
                 continue
 
-    raise Exception("Should not reach here")
+    return Exception("Should not reach here")
 
 
 def complete_multipart_upload_from_parts(
@@ -303,9 +314,14 @@ def finish_multipart_upload_from_keys(
     locked_print(
         f"Creating multipart upload for {destination_bucket}/{destination_key} from {len(parts)} source objects"
     )
-    mpu = s3_client.create_multipart_upload(
-        Bucket=destination_bucket, Key=destination_key
-    )
+
+    create_params: dict[str, str] = {
+        "Bucket": destination_bucket,
+        "Key": destination_key,
+    }
+    print(f"Creating multipart upload with {create_params}")
+    mpu = s3_client.create_multipart_upload(**create_params)
+    print(f"Created multipart upload: {mpu}")
     upload_id = mpu["UploadId"]
 
     # Create upload info
@@ -319,9 +335,12 @@ def finish_multipart_upload_from_keys(
         file_size=final_size,
     )
 
-    futures: list[Future[FinishedPiece]] = []
+    futures: list[Future[FinishedPiece | Exception]] = []
 
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # semaphore
+
+        semaphore = Semaphore(max_workers * 2)
         for part_number, source_key in parts:
 
             def task(
@@ -340,17 +359,22 @@ def finish_multipart_upload_from_keys(
                 )
 
             fut = executor.submit(task)
+            fut.add_done_callback(lambda x: semaphore.release())
             futures.append(fut)
+            semaphore.acquire()
 
-    # Upload parts by copying from source objects
-    finished_parts = []
+        # Upload parts by copying from source objects
+        finished_parts: list[FinishedPiece] = []
 
-    for fut in futures:
-        finished_part = fut.result()
-        finished_parts.append(finished_part)
+        for fut in futures:
+            finished_part = fut.result()
+            if isinstance(finished_part, Exception):
+                executor.shutdown(wait=True, cancel_futures=True)
+                raise finished_part
+            finished_parts.append(finished_part)
 
-    # Complete the multipart upload
-    return complete_multipart_upload_from_parts(upload_info, finished_parts)
+        # Complete the multipart upload
+        return complete_multipart_upload_from_parts(upload_info, finished_parts)
 
 
 class S3MultiPartUploader:

{rclone_api-1.4.8 → rclone_api-1.4.10}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.8
+Version: 1.4.10
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License