rclone-api 1.4.12__tar.gz → 1.4.13__tar.gz

{rclone_api-1.4.12 → rclone_api-1.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.12
+Version: 1.4.13
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.12 → rclone_api-1.4.13}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.12"
+version = "1.4.13"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.12 → rclone_api-1.4.13}/src/rclone_api/__init__.py

@@ -507,6 +507,10 @@ class Rclone:
             verbose=verbose,
         )
 
+    def size_file(self, src: str) -> SizeSuffix | Exception:
+        """Get the size of a file."""
+        return self.impl.size_file(src=src)
+
 
 __all__ = [
     "Rclone",

{rclone_api-1.4.12 → rclone_api-1.4.13}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -51,10 +51,17 @@ def _parse_args() -> Args:
 
 
 def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
-    from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
+    from rclone_api.s3.create import (
+        BaseClient,
+        S3Config,
+        S3Credentials,
+        create_s3_client,
+    )
 
     s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(s3_creds)
+    s3_client: BaseClient = create_s3_client(
+        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    )
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
     assert is_done, f"Upload is not done: {info}"
@@ -112,6 +119,20 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         retries=3,
     )
 
+    # now check if the dst now exists, if so, delete the parts folder.
+    # if rclone.exists(dst):
+    #     rclone.purge(parts_dir)
+
+    if not rclone.exists(dst):
+        raise FileNotFoundError(f"Destination file not found: {dst}")
+
+    write_size = rclone.size_file(dst)
+    if write_size != size:
+        raise ValueError(f"Size mismatch: {write_size} != {size}")
+
+    print(f"Upload complete: {dst}")
+    rclone.purge(parts_dir)
+
 
 def main() -> int:
     """Main entry point."""
@@ -120,7 +141,10 @@ def main() -> int:
     info_json = f"{args.src}/info.json".replace("//", "/")
     info = InfoJson(rclone.impl, src=None, src_info=info_json)
     loaded = info.load()
-    assert loaded
+    if not loaded:
+        raise FileNotFoundError(
+            f"Info file not found, has the upload finished? {info_json}"
+        )
     print(info)
     do_finish_part(rclone=rclone, info=info, dst=args.dst)
     return 0

{rclone_api-1.4.12 → rclone_api-1.4.13}/src/rclone_api/s3/api.py

@@ -9,7 +9,7 @@ from rclone_api.s3.basic_ops import (
     list_bucket_contents,
     upload_file,
 )
-from rclone_api.s3.create import create_s3_client
+from rclone_api.s3.create import S3Config, create_s3_client
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
 from rclone_api.s3.upload_file_multipart import (
     MultiUploadResult,
@@ -23,7 +23,9 @@ class S3Client:
     def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
         self.credentials: S3Credentials = credentials
-        self.client: BaseClient = create_s3_client(credentials, verbose=verbose)
+        self.client: BaseClient = create_s3_client(
+            credentials, config=S3Config(verbose=verbose)
+        )
 
     def list_bucket_contents(self, bucket_name: str) -> None:
         list_bucket_contents(self.client, bucket_name)

{rclone_api-1.4.12 → rclone_api-1.4.13}/src/rclone_api/s3/create.py

@@ -1,4 +1,5 @@
 import warnings
+from dataclasses import dataclass
 
 import boto3
 from botocore.client import BaseClient
@@ -8,18 +9,40 @@ from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
 _MAX_CONNECTIONS = 50
+_TIMEOUT_READ = 120
+_TIMEOUT_CONNECT = 60
+
+
+@dataclass
+class S3Config:
+    max_pool_connections: int | None = None
+    max_connections: int | None = None
+    timeout_connection: int | None = None
+    timeout_read: int | None = None
+    verbose: bool | None = None
+
+    def resolve_defaults(self) -> None:
+        if self.max_pool_connections is None:
+            self.max_pool_connections = _MAX_CONNECTIONS
+        if self.timeout_connection is None:
+            self.timeout_connection = _TIMEOUT_CONNECT
+        if self.timeout_read is None:
+            self.timeout_read = _TIMEOUT_READ
+        if self.verbose is None:
+            self.verbose = False
 
 
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient:
+def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
     """Create and return an S3 client."""
     region_name = creds.region_name
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
+    config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -30,7 +53,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=_MAX_CONNECTIONS,
+            max_pool_connections=config.max_connections,
+            read_timeout=config.timeout_read,
+            connect_timeout=config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -39,18 +64,18 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient:
+def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
     """Create and return an S3 client."""
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if verbose:
+        if config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-
+    config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -60,19 +85,24 @@ def _create_unknown_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient
         config=Config(
             signature_version="s3v4",
             region_name=creds.region_name,
-            max_pool_connections=_MAX_CONNECTIONS,
+            max_pool_connections=config.max_connections,
+            read_timeout=config.timeout_read,
+            connect_timeout=config.timeout_connection,
         ),
     )
 
 
-def create_s3_client(credentials: S3Credentials, verbose=False) -> BaseClient:
+def create_s3_client(
+    credentials: S3Credentials, config: S3Config | None = None
+) -> BaseClient:
     """Create and return an S3 client."""
+    config = config or S3Config()
     provider = credentials.provider
     if provider == S3Provider.BACKBLAZE:
-        if verbose:
+        if config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, verbose=verbose)
+        return _create_backblaze_s3_client(creds=credentials, config=config)
     else:
-        if verbose:
+        if config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, verbose=verbose)
+        return _create_unknown_s3_client(creds=credentials, config=config)

{rclone_api-1.4.12 → rclone_api-1.4.13}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.12
+Version: 1.4.13
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License