rclone-api 1.4.14__tar.gz → 1.4.15__tar.gz

{rclone_api-1.4.14 → rclone_api-1.4.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.14
+Version: 1.4.15
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.14 → rclone_api-1.4.15}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.14"
+version = "1.4.15"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.14 → rclone_api-1.4.15}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -10,6 +10,9 @@ from rclone_api.s3.s3_multipart_uploader_by_copy import (
 )
 from rclone_api.types import SizeSuffix
 
+_TIMEOUT_READ = 900
+_TIMEOUT_CONNECTION = 900
+
 
 @dataclass
 class Args:
@@ -58,10 +61,14 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         create_s3_client,
     )
 
-    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(
-        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    s3_config = S3Config(
+        verbose=False,
+        timeout_read=_TIMEOUT_READ,
+        timeout_connection=_TIMEOUT_CONNECTION,
     )
+
+    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
+    s3_client: BaseClient = create_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
     assert is_done, f"Upload is not done: {info}"
@@ -115,8 +122,7 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
         final_size=size.as_int(),
-        max_workers=50,
-        retries=3,
+        max_workers=10,
     )
 
     # now check if the dst now exists, if so, delete the parts folder.

{rclone_api-1.4.14 → rclone_api-1.4.15}/src/rclone_api/s3/api.py

@@ -20,11 +20,11 @@ _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 
 
 class S3Client:
-    def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
+    def __init__(self, s3_creds: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
-        self.credentials: S3Credentials = credentials
+        self.credentials: S3Credentials = s3_creds
         self.client: BaseClient = create_s3_client(
-            credentials, config=S3Config(verbose=verbose)
+            s3_creds=s3_creds, s3_config=S3Config(verbose=verbose)
         )
 
     def list_bucket_contents(self, bucket_name: str) -> None:

{rclone_api-1.4.14 → rclone_api-1.4.15}/src/rclone_api/s3/create.py

@@ -8,7 +8,7 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 50
+_MAX_CONNECTIONS = 10
 _TIMEOUT_READ = 120
 _TIMEOUT_CONNECT = 60
 
@@ -30,14 +30,16 @@ class S3Config:
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_backblaze_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    region_name = creds.region_name
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    region_name = s3_creds.region_name
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -48,9 +50,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=config.max_pool_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -59,18 +61,20 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_unknown_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if config.verbose:
+        if s3_config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -79,25 +83,25 @@ def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseCli
         endpoint_url=endpoint_url,
         config=Config(
             signature_version="s3v4",
-            region_name=creds.region_name,
-            max_pool_connections=config.max_pool_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            region_name=s3_creds.region_name,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
         ),
     )
 
 
 def create_s3_client(
-    credentials: S3Credentials, config: S3Config | None = None
+    s3_creds: S3Credentials, s3_config: S3Config | None = None
 ) -> BaseClient:
     """Create and return an S3 client."""
-    config = config or S3Config()
-    provider = credentials.provider
+    s3_config = s3_config or S3Config()
+    provider = s3_creds.provider
     if provider == S3Provider.BACKBLAZE:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, config=config)
+        return _create_backblaze_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     else:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, config=config)
+        return _create_unknown_s3_client(s3_creds=s3_creds, s3_config=s3_config)

{rclone_api-1.4.14 → rclone_api-1.4.15}/src/rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -240,7 +240,7 @@ class S3MultiPartUploader:
         destination_key: str,
         chunk_size: int,
         final_size: int,
-        retries: int = 3,
+        retries: int = 100,
     ) -> str:
         """
         Finish a multipart upload by copying parts from existing S3 objects.

{rclone_api-1.4.14 → rclone_api-1.4.15}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.14
+Version: 1.4.15
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License