rclone-api 1.4.11__tar.gz → 1.4.12__tar.gz

{rclone_api-1.4.11 → rclone_api-1.4.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.11
+Version: 1.4.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.11 → rclone_api-1.4.12}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.11"
+version = "1.4.12"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.11 → rclone_api-1.4.12}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -9,31 +10,6 @@ from rclone_api.s3.s3_multipart_uploader_by_copy import (
 )
 from rclone_api.types import SizeSuffix
 
-DATA_SOURCE = (
-    "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
-)
-
-
-# response = client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
 
 @dataclass
 class Args:
@@ -58,13 +34,6 @@ def _parse_args() -> Args:
     parser.add_argument(
         "--config", help="Path to rclone config file", type=Path, required=False
     )
-    parser.add_argument(
-        "--chunk-size",
-        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
-        type=str,
-        default="128MB",  # if this is too low or too high an s3 service
-    )
-
     args = parser.parse_args()
     config: Path | None = args.config
     if config is None:
@@ -81,28 +50,6 @@ def _parse_args() -> Args:
     return out
 
 
-# from dataclasses import dataclass
-
-# def parse_info_json(text: str) -> UploadInfo:
-#     import json
-#     data = json.loads(text)
-#     chunk_size = data["chunksize_int"]
-#     first_part = data["first_part"]
-#     last_part = data["last_part"]
-#     assert isinstance(chunk_size, int)
-#     assert isinstance(first_part, int)
-#     assert isinstance(last_part, int)
-#     assert first_part <= last_part
-#     parts: list[str] = []
-#     fmt = "part.{:05d}_{}-{}"
-#     for i in range(first_part, last_part + 1):
-#         offset: int = i * chunk_size
-#         end: int = (i + 1) * chunk_size
-#         part = fmt.format(i, offset, end)
-#         parts.append(part)
-#     return UploadInfo(chunk_size=chunk_size, parts=parts)
-
-
 def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
 
@@ -138,7 +85,6 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         out = f"{parts_path}"
         return out
 
-    # s3_keys: list[str] = [_to_s3_key(name=p) for p in source_keys]
     parts: list[tuple[int, str]] = []
     part_num = 1
     for part_key in source_keys:
@@ -146,14 +92,9 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         parts.append((part_num, s3_key))
         part_num += 1
 
-    # for key in parts:
-    #     print(key)
-
     chunksize = info.chunksize
     assert chunksize is not None
 
-    import os
-
     dst_name = info.dst_name
     dst_dir = os.path.dirname(parts_path)
     # dst_key =
@@ -167,7 +108,7 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
         final_size=size.as_int(),
-        max_workers=100,
+        max_workers=50,
         retries=3,
     )
 

{rclone_api-1.4.11 → rclone_api-1.4.12}/src/rclone_api/s3/create.py

@@ -7,7 +7,7 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 500
+_MAX_CONNECTIONS = 50
 
 
 # Create a Boto3 session and S3 client, this is back blaze specific.

{rclone_api-1.4.11 → rclone_api-1.4.12}/src/rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -32,152 +32,6 @@ class MultipartUploadInfo:
     src_file_path: Optional[Path] = None
 
 
-# response = client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-# import _thread
-# import os
-# import traceback
-# import warnings
-# from concurrent.futures import Future, ThreadPoolExecutor
-# from pathlib import Path
-# from queue import Queue
-# from threading import Event, Thread
-# from typing import Any, Callable
-
-# from botocore.client import BaseClient
-
-# from rclone_api.mount_read_chunker import FilePart
-# from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
-# from rclone_api.s3.chunk_types import (
-#     FinishedPiece,
-#     UploadInfo,
-#     UploadState,
-# )
-# from rclone_api.s3.types import MultiUploadResult
-# from rclone_api.types import EndOfStream
-# from rclone_api.util import locked_print
-
-
-# This is how you upload large parts through multi part upload, then the final call
-# is to assemble the parts that have already been uploaded through a multi part uploader
-# and then call complete_multipart_upload to finish the upload
-# response = (
-#  client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-
-# def upload_task(
-#     info: UploadInfo,
-#     chunk: FilePart,
-#     part_number: int,
-#     retries: int,
-# ) -> FinishedPiece:
-#     file_or_err: Path | Exception = chunk.get_file()
-#     if isinstance(file_or_err, Exception):
-#         raise file_or_err
-#     file: Path = file_or_err
-#     size = os.path.getsize(file)
-#     retries = retries + 1  # Add one for the initial attempt
-#     for retry in range(retries):
-#         try:
-#             if retry > 0:
-#                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
-#             locked_print(
-#                 f"Uploading part {part_number} for {info.src_file_path} of size {size}"
-#             )
-
-#             with open(file, "rb") as f:
-#                 part = info.s3_client.upload_part(
-#                     Bucket=info.bucket_name,
-#                     Key=info.object_name,
-#                     PartNumber=part_number,
-#                     UploadId=info.upload_id,
-#                     Body=f,
-#                 )
-#                 out: FinishedPiece = FinishedPiece(
-#                     etag=part["ETag"], part_number=part_number
-#                 )
-#             chunk.dispose()
-#             return out
-#         except Exception as e:
-#             if retry == retries - 1:
-#                 locked_print(f"Error uploading part {part_number}: {e}")
-#                 chunk.dispose()
-#                 raise e
-#             else:
-#                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
-#                 continue
-#     raise Exception("Should not reach here")
-
-
-# def prepare_upload_file_multipart(
-#     s3_client: BaseClient,
-#     bucket_name: str,
-#     file_path: Path,
-#     file_size: int | None,
-#     object_name: str,
-#     chunk_size: int,
-#     retries: int,
-# ) -> UploadInfo:
-#     """Upload a file to the bucket using multipart upload with customizable chunk size."""
-
-#     # Initiate multipart upload
-#     locked_print(
-#         f"Creating multipart upload for {file_path} to {bucket_name}/{object_name}"
-#     )
-#     mpu = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)
-#     upload_id = mpu["UploadId"]
-
-#     file_size = file_size if file_size is not None else os.path.getsize(file_path)
-
-#     upload_info: UploadInfo = UploadInfo(
-#         s3_client=s3_client,
-#         bucket_name=bucket_name,
-#         object_name=object_name,
-#         src_file_path=file_path,
-#         upload_id=upload_id,
-#         retries=retries,
-#         chunk_size=chunk_size,
-#         file_size=file_size,
-#     )
-#     return upload_info
-
-
 def upload_part_copy_task(
     info: MultipartUploadInfo,
     source_bucket: str,
@@ -228,11 +82,7 @@ def upload_part_copy_task(
 
             # Extract ETag from the response
             etag = part["CopyPartResult"]["ETag"]
-
             return FinishedPiece(etag=etag, part_number=part_number)
-        # except NoSuchKey as e:
-        #     locked_print(f"Error copying part {part_number}: {e}")
-        #     return e
 
         except Exception as e:
             msg = f"Error copying {copy_source} -> {info.object_name}: {e}, params={params}"

{rclone_api-1.4.11 → rclone_api-1.4.12}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.11
+Version: 1.4.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License