rclone-api 1.4.11__py2.py3-none-any.whl → 1.4.12__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3_finish.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -9,31 +10,6 @@ from rclone_api.s3.s3_multipart_uploader_by_copy import (
 )
 from rclone_api.types import SizeSuffix
 
-DATA_SOURCE = (
-    "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
-)
-
-
-# response = client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
 
 @dataclass
 class Args:
@@ -58,13 +34,6 @@ def _parse_args() -> Args:
     parser.add_argument(
         "--config", help="Path to rclone config file", type=Path, required=False
     )
-    parser.add_argument(
-        "--chunk-size",
-        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
-        type=str,
-        default="128MB",  # if this is too low or too high an s3 service
-    )
-
     args = parser.parse_args()
     config: Path | None = args.config
     if config is None:
@@ -81,28 +50,6 @@ def _parse_args() -> Args:
     return out
 
 
-# from dataclasses import dataclass
-
-# def parse_info_json(text: str) -> UploadInfo:
-#     import json
-#     data = json.loads(text)
-#     chunk_size = data["chunksize_int"]
-#     first_part = data["first_part"]
-#     last_part = data["last_part"]
-#     assert isinstance(chunk_size, int)
-#     assert isinstance(first_part, int)
-#     assert isinstance(last_part, int)
-#     assert first_part <= last_part
-#     parts: list[str] = []
-#     fmt = "part.{:05d}_{}-{}"
-#     for i in range(first_part, last_part + 1):
-#         offset: int = i * chunk_size
-#         end: int = (i + 1) * chunk_size
-#         part = fmt.format(i, offset, end)
-#         parts.append(part)
-#     return UploadInfo(chunk_size=chunk_size, parts=parts)
-
-
 def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
 
@@ -138,7 +85,6 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         out = f"{parts_path}"
         return out
 
-    # s3_keys: list[str] = [_to_s3_key(name=p) for p in source_keys]
     parts: list[tuple[int, str]] = []
     part_num = 1
     for part_key in source_keys:
@@ -146,14 +92,9 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         parts.append((part_num, s3_key))
         part_num += 1
 
-    # for key in parts:
-    #     print(key)
-
     chunksize = info.chunksize
     assert chunksize is not None
 
-    import os
-
     dst_name = info.dst_name
     dst_dir = os.path.dirname(parts_path)
     # dst_key =
@@ -167,7 +108,7 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
         final_size=size.as_int(),
-        max_workers=100,
+        max_workers=50,
         retries=3,
     )
 

rclone_api/s3/create.py

@@ -7,7 +7,7 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 500
+_MAX_CONNECTIONS = 50
 
 
 # Create a Boto3 session and S3 client, this is back blaze specific.

rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -32,152 +32,6 @@ class MultipartUploadInfo:
     src_file_path: Optional[Path] = None
 
 
-# response = client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-# import _thread
-# import os
-# import traceback
-# import warnings
-# from concurrent.futures import Future, ThreadPoolExecutor
-# from pathlib import Path
-# from queue import Queue
-# from threading import Event, Thread
-# from typing import Any, Callable
-
-# from botocore.client import BaseClient
-
-# from rclone_api.mount_read_chunker import FilePart
-# from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
-# from rclone_api.s3.chunk_types import (
-#     FinishedPiece,
-#     UploadInfo,
-#     UploadState,
-# )
-# from rclone_api.s3.types import MultiUploadResult
-# from rclone_api.types import EndOfStream
-# from rclone_api.util import locked_print
-
-
-# This is how you upload large parts through multi part upload, then the final call
-# is to assemble the parts that have already been uploaded through a multi part uploader
-# and then call complete_multipart_upload to finish the upload
-# response = (
-#  client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-
-# def upload_task(
-#     info: UploadInfo,
-#     chunk: FilePart,
-#     part_number: int,
-#     retries: int,
-# ) -> FinishedPiece:
-#     file_or_err: Path | Exception = chunk.get_file()
-#     if isinstance(file_or_err, Exception):
-#         raise file_or_err
-#     file: Path = file_or_err
-#     size = os.path.getsize(file)
-#     retries = retries + 1  # Add one for the initial attempt
-#     for retry in range(retries):
-#         try:
-#             if retry > 0:
-#                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
-#             locked_print(
-#                 f"Uploading part {part_number} for {info.src_file_path} of size {size}"
-#             )
-
-#             with open(file, "rb") as f:
-#                 part = info.s3_client.upload_part(
-#                     Bucket=info.bucket_name,
-#                     Key=info.object_name,
-#                     PartNumber=part_number,
-#                     UploadId=info.upload_id,
-#                     Body=f,
-#                 )
-#                 out: FinishedPiece = FinishedPiece(
-#                     etag=part["ETag"], part_number=part_number
-#                 )
-#             chunk.dispose()
-#             return out
-#         except Exception as e:
-#             if retry == retries - 1:
-#                 locked_print(f"Error uploading part {part_number}: {e}")
-#                 chunk.dispose()
-#                 raise e
-#             else:
-#                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
-#                 continue
-#     raise Exception("Should not reach here")
-
-
-# def prepare_upload_file_multipart(
-#     s3_client: BaseClient,
-#     bucket_name: str,
-#     file_path: Path,
-#     file_size: int | None,
-#     object_name: str,
-#     chunk_size: int,
-#     retries: int,
-# ) -> UploadInfo:
-#     """Upload a file to the bucket using multipart upload with customizable chunk size."""
-
-#     # Initiate multipart upload
-#     locked_print(
-#         f"Creating multipart upload for {file_path} to {bucket_name}/{object_name}"
-#     )
-#     mpu = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)
-#     upload_id = mpu["UploadId"]
-
-#     file_size = file_size if file_size is not None else os.path.getsize(file_path)
-
-#     upload_info: UploadInfo = UploadInfo(
-#         s3_client=s3_client,
-#         bucket_name=bucket_name,
-#         object_name=object_name,
-#         src_file_path=file_path,
-#         upload_id=upload_id,
-#         retries=retries,
-#         chunk_size=chunk_size,
-#         file_size=file_size,
-#     )
-#     return upload_info
-
-
 def upload_part_copy_task(
     info: MultipartUploadInfo,
     source_bucket: str,
@@ -228,11 +82,7 @@ def upload_part_copy_task(
 
             # Extract ETag from the response
             etag = part["CopyPartResult"]["ETag"]
-
             return FinishedPiece(etag=etag, part_number=part_number)
-        # except NoSuchKey as e:
-        #     locked_print(f"Error copying part {part_number}: {e}")
-        #     return e
 
         except Exception as e:
             msg = f"Error copying {copy_source} -> {info.object_name}: {e}, params={params}"

{rclone_api-1.4.11.dist-info → rclone_api-1.4.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.11
+Version: 1.4.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.11.dist-info → rclone_api-1.4.12.dist-info}/RECORD

@@ -27,7 +27,7 @@ rclone_api/util.py,sha256=9w_m6W62l_X42Jw5q8p_p30h-QoxAqufvnCLI4PTMOE,7056
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
 rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
-rclone_api/cmd/copy_large_s3_finish.py,sha256=2L2kw206rJ-p3zQZkqiixt1tz5vXaw9SOfji6dFZFOs,6138
+rclone_api/cmd/copy_large_s3_finish.py,sha256=O7ezviM460ivprF5v9bx_ztqhg0_jnRmvzYK8n6R9IY,4212
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
@@ -40,17 +40,17 @@ rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
-rclone_api/s3/create.py,sha256=eoC-AFN-LNS0L6HoOnVYJtW7a5OaJWSkP9Vx7AvOX4U,2972
-rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=rivcMn9TgHCZIhq7w2lCtas6O-QM08-uCEX2ry8ZOhs,13893
+rclone_api/s3/create.py,sha256=Ao79MENZM5Bvi9SXwo6yng88Vkmn75rZElHKQrGckeo,2971
+rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=KNEWG3CrM8lVWd2b0dA6WT5LO3gEEXNk2tA6Uk9Vi7I,8701
 rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
 rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
 rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaRCmkYLTQ,105
 rclone_api/s3/multipart/finished_piece.py,sha256=9nMWnVZ8S99wi2VFQsm1h1ZHqmebkhMGgd2s56wNj9w,1331
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.11.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.11.dist-info/METADATA,sha256=ijUgYWrOnz6yxDexwljro-qm8ZZK3mN2HyreI6a5TxQ,4628
-rclone_api-1.4.11.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.11.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.11.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.11.dist-info/RECORD,,
+rclone_api-1.4.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.12.dist-info/METADATA,sha256=i7GFP4S5sHfeiTlh9sKl_CXd7jVyXqldB2jvd_rLWPc,4628
+rclone_api-1.4.12.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.12.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.12.dist-info/RECORD,,