rclone-api 1.4.11__tar.gz → 1.4.13__tar.gz

{rclone_api-1.4.11 → rclone_api-1.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.11
+Version: 1.4.13
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.11 → rclone_api-1.4.13}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.11"
+version = "1.4.13"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.11 → rclone_api-1.4.13}/src/rclone_api/__init__.py

@@ -507,6 +507,10 @@ class Rclone:
             verbose=verbose,
         )
 
+    def size_file(self, src: str) -> SizeSuffix | Exception:
+        """Get the size of a file."""
+        return self.impl.size_file(src=src)
+
 
 __all__ = [
     "Rclone",

{rclone_api-1.4.11 → rclone_api-1.4.13}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -9,31 +10,6 @@ from rclone_api.s3.s3_multipart_uploader_by_copy import (
 )
 from rclone_api.types import SizeSuffix
 
-DATA_SOURCE = (
-    "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
-)
-
-
-# response = client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
 
 @dataclass
 class Args:
@@ -58,13 +34,6 @@ def _parse_args() -> Args:
     parser.add_argument(
         "--config", help="Path to rclone config file", type=Path, required=False
     )
-    parser.add_argument(
-        "--chunk-size",
-        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
-        type=str,
-        default="128MB",  # if this is too low or too high an s3 service
-    )
-
     args = parser.parse_args()
     config: Path | None = args.config
     if config is None:
@@ -81,33 +50,18 @@ def _parse_args() -> Args:
     return out
 
 
-# from dataclasses import dataclass
-
-# def parse_info_json(text: str) -> UploadInfo:
-#     import json
-#     data = json.loads(text)
-#     chunk_size = data["chunksize_int"]
-#     first_part = data["first_part"]
-#     last_part = data["last_part"]
-#     assert isinstance(chunk_size, int)
-#     assert isinstance(first_part, int)
-#     assert isinstance(last_part, int)
-#     assert first_part <= last_part
-#     parts: list[str] = []
-#     fmt = "part.{:05d}_{}-{}"
-#     for i in range(first_part, last_part + 1):
-#         offset: int = i * chunk_size
-#         end: int = (i + 1) * chunk_size
-#         part = fmt.format(i, offset, end)
-#         parts.append(part)
-#     return UploadInfo(chunk_size=chunk_size, parts=parts)
-
-
 def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
-    from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
+    from rclone_api.s3.create import (
+        BaseClient,
+        S3Config,
+        S3Credentials,
+        create_s3_client,
+    )
 
     s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(s3_creds)
+    s3_client: BaseClient = create_s3_client(
+        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    )
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
     assert is_done, f"Upload is not done: {info}"
@@ -138,7 +92,6 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         out = f"{parts_path}"
         return out
 
-    # s3_keys: list[str] = [_to_s3_key(name=p) for p in source_keys]
     parts: list[tuple[int, str]] = []
     part_num = 1
     for part_key in source_keys:
@@ -146,14 +99,9 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         parts.append((part_num, s3_key))
         part_num += 1
 
-    # for key in parts:
-    #     print(key)
-
     chunksize = info.chunksize
     assert chunksize is not None
 
-    import os
-
     dst_name = info.dst_name
     dst_dir = os.path.dirname(parts_path)
     # dst_key =
@@ -167,10 +115,24 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
         final_size=size.as_int(),
-        max_workers=100,
+        max_workers=50,
         retries=3,
     )
 
+    # now check if the dst now exists, if so, delete the parts folder.
+    # if rclone.exists(dst):
+    #     rclone.purge(parts_dir)
+
+    if not rclone.exists(dst):
+        raise FileNotFoundError(f"Destination file not found: {dst}")
+
+    write_size = rclone.size_file(dst)
+    if write_size != size:
+        raise ValueError(f"Size mismatch: {write_size} != {size}")
+
+    print(f"Upload complete: {dst}")
+    rclone.purge(parts_dir)
+
 
 def main() -> int:
     """Main entry point."""
@@ -179,7 +141,10 @@ def main() -> int:
     info_json = f"{args.src}/info.json".replace("//", "/")
     info = InfoJson(rclone.impl, src=None, src_info=info_json)
     loaded = info.load()
-    assert loaded
+    if not loaded:
+        raise FileNotFoundError(
+            f"Info file not found, has the upload finished? {info_json}"
+        )
     print(info)
     do_finish_part(rclone=rclone, info=info, dst=args.dst)
     return 0

{rclone_api-1.4.11 → rclone_api-1.4.13}/src/rclone_api/s3/api.py

@@ -9,7 +9,7 @@ from rclone_api.s3.basic_ops import (
     list_bucket_contents,
     upload_file,
 )
-from rclone_api.s3.create import create_s3_client
+from rclone_api.s3.create import S3Config, create_s3_client
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
 from rclone_api.s3.upload_file_multipart import (
     MultiUploadResult,
@@ -23,7 +23,9 @@ class S3Client:
     def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
         self.credentials: S3Credentials = credentials
-        self.client: BaseClient = create_s3_client(credentials, verbose=verbose)
+        self.client: BaseClient = create_s3_client(
+            credentials, config=S3Config(verbose=verbose)
+        )
 
     def list_bucket_contents(self, bucket_name: str) -> None:
         list_bucket_contents(self.client, bucket_name)

{rclone_api-1.4.11 → rclone_api-1.4.13}/src/rclone_api/s3/create.py

@@ -1,4 +1,5 @@
 import warnings
+from dataclasses import dataclass
 
 import boto3
 from botocore.client import BaseClient
@@ -7,19 +8,41 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 500
+_MAX_CONNECTIONS = 50
+_TIMEOUT_READ = 120
+_TIMEOUT_CONNECT = 60
+
+
+@dataclass
+class S3Config:
+    max_pool_connections: int | None = None
+    max_connections: int | None = None
+    timeout_connection: int | None = None
+    timeout_read: int | None = None
+    verbose: bool | None = None
+
+    def resolve_defaults(self) -> None:
+        if self.max_pool_connections is None:
+            self.max_pool_connections = _MAX_CONNECTIONS
+        if self.timeout_connection is None:
+            self.timeout_connection = _TIMEOUT_CONNECT
+        if self.timeout_read is None:
+            self.timeout_read = _TIMEOUT_READ
+        if self.verbose is None:
+            self.verbose = False
 
 
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient:
+def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
     """Create and return an S3 client."""
     region_name = creds.region_name
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
+    config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -30,7 +53,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=_MAX_CONNECTIONS,
+            max_pool_connections=config.max_connections,
+            read_timeout=config.timeout_read,
+            connect_timeout=config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -39,18 +64,18 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient:
+def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
     """Create and return an S3 client."""
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if verbose:
+        if config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-
+    config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -60,19 +85,24 @@ def _create_unknown_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient
         config=Config(
             signature_version="s3v4",
             region_name=creds.region_name,
-            max_pool_connections=_MAX_CONNECTIONS,
+            max_pool_connections=config.max_connections,
+            read_timeout=config.timeout_read,
+            connect_timeout=config.timeout_connection,
         ),
     )
 
 
-def create_s3_client(credentials: S3Credentials, verbose=False) -> BaseClient:
+def create_s3_client(
+    credentials: S3Credentials, config: S3Config | None = None
+) -> BaseClient:
     """Create and return an S3 client."""
+    config = config or S3Config()
     provider = credentials.provider
     if provider == S3Provider.BACKBLAZE:
-        if verbose:
+        if config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, verbose=verbose)
+        return _create_backblaze_s3_client(creds=credentials, config=config)
     else:
-        if verbose:
+        if config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, verbose=verbose)
+        return _create_unknown_s3_client(creds=credentials, config=config)

{rclone_api-1.4.11 → rclone_api-1.4.13}/src/rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -32,152 +32,6 @@ class MultipartUploadInfo:
     src_file_path: Optional[Path] = None
 
 
-# response = client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-# import _thread
-# import os
-# import traceback
-# import warnings
-# from concurrent.futures import Future, ThreadPoolExecutor
-# from pathlib import Path
-# from queue import Queue
-# from threading import Event, Thread
-# from typing import Any, Callable
-
-# from botocore.client import BaseClient
-
-# from rclone_api.mount_read_chunker import FilePart
-# from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
-# from rclone_api.s3.chunk_types import (
-#     FinishedPiece,
-#     UploadInfo,
-#     UploadState,
-# )
-# from rclone_api.s3.types import MultiUploadResult
-# from rclone_api.types import EndOfStream
-# from rclone_api.util import locked_print
-
-
-# This is how you upload large parts through multi part upload, then the final call
-# is to assemble the parts that have already been uploaded through a multi part uploader
-# and then call complete_multipart_upload to finish the upload
-# response = (
-#  client.upload_part_copy(
-#     Bucket='string',
-#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
-#     CopySourceIfMatch='string',
-#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
-#     CopySourceIfNoneMatch='string',
-#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
-#     CopySourceRange='string',
-#     Key='string',
-#     PartNumber=123,
-#     UploadId='string',
-#     SSECustomerAlgorithm='string',
-#     SSECustomerKey='string',
-#     CopySourceSSECustomerAlgorithm='string',
-#     CopySourceSSECustomerKey='string',
-#     RequestPayer='requester',
-#     ExpectedBucketOwner='string',
-#     ExpectedSourceBucketOwner='string'
-# )
-
-
-# def upload_task(
-#     info: UploadInfo,
-#     chunk: FilePart,
-#     part_number: int,
-#     retries: int,
-# ) -> FinishedPiece:
-#     file_or_err: Path | Exception = chunk.get_file()
-#     if isinstance(file_or_err, Exception):
-#         raise file_or_err
-#     file: Path = file_or_err
-#     size = os.path.getsize(file)
-#     retries = retries + 1  # Add one for the initial attempt
-#     for retry in range(retries):
-#         try:
-#             if retry > 0:
-#                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
-#             locked_print(
-#                 f"Uploading part {part_number} for {info.src_file_path} of size {size}"
-#             )
-
-#             with open(file, "rb") as f:
-#                 part = info.s3_client.upload_part(
-#                     Bucket=info.bucket_name,
-#                     Key=info.object_name,
-#                     PartNumber=part_number,
-#                     UploadId=info.upload_id,
-#                     Body=f,
-#                 )
-#                 out: FinishedPiece = FinishedPiece(
-#                     etag=part["ETag"], part_number=part_number
-#                 )
-#             chunk.dispose()
-#             return out
-#         except Exception as e:
-#             if retry == retries - 1:
-#                 locked_print(f"Error uploading part {part_number}: {e}")
-#                 chunk.dispose()
-#                 raise e
-#             else:
-#                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
-#                 continue
-#     raise Exception("Should not reach here")
-
-
-# def prepare_upload_file_multipart(
-#     s3_client: BaseClient,
-#     bucket_name: str,
-#     file_path: Path,
-#     file_size: int | None,
-#     object_name: str,
-#     chunk_size: int,
-#     retries: int,
-# ) -> UploadInfo:
-#     """Upload a file to the bucket using multipart upload with customizable chunk size."""
-
-#     # Initiate multipart upload
-#     locked_print(
-#         f"Creating multipart upload for {file_path} to {bucket_name}/{object_name}"
-#     )
-#     mpu = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)
-#     upload_id = mpu["UploadId"]
-
-#     file_size = file_size if file_size is not None else os.path.getsize(file_path)
-
-#     upload_info: UploadInfo = UploadInfo(
-#         s3_client=s3_client,
-#         bucket_name=bucket_name,
-#         object_name=object_name,
-#         src_file_path=file_path,
-#         upload_id=upload_id,
-#         retries=retries,
-#         chunk_size=chunk_size,
-#         file_size=file_size,
-#     )
-#     return upload_info
-
-
 def upload_part_copy_task(
     info: MultipartUploadInfo,
     source_bucket: str,
@@ -228,11 +82,7 @@ def upload_part_copy_task(
 
             # Extract ETag from the response
             etag = part["CopyPartResult"]["ETag"]
-
             return FinishedPiece(etag=etag, part_number=part_number)
-        # except NoSuchKey as e:
-        #     locked_print(f"Error copying part {part_number}: {e}")
-        #     return e
 
         except Exception as e:
             msg = f"Error copying {copy_source} -> {info.object_name}: {e}, params={params}"

{rclone_api-1.4.11 → rclone_api-1.4.13}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.11
+Version: 1.4.13
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License