rclone-api 1.4.13__py2.py3-none-any.whl → 1.4.15__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3_finish.py

@@ -10,6 +10,9 @@ from rclone_api.s3.s3_multipart_uploader_by_copy import (
 )
 from rclone_api.types import SizeSuffix
 
+_TIMEOUT_READ = 900
+_TIMEOUT_CONNECTION = 900
+
 
 @dataclass
 class Args:
@@ -58,10 +61,14 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         create_s3_client,
     )
 
-    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(
-        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    s3_config = S3Config(
+        verbose=False,
+        timeout_read=_TIMEOUT_READ,
+        timeout_connection=_TIMEOUT_CONNECTION,
     )
+
+    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
+    s3_client: BaseClient = create_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
     assert is_done, f"Upload is not done: {info}"
@@ -115,8 +122,7 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
         final_size=size.as_int(),
-        max_workers=50,
-        retries=3,
+        max_workers=10,
     )
 
     # now check if the dst now exists, if so, delete the parts folder.

rclone_api/s3/api.py

@@ -20,11 +20,11 @@ _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 
 
 class S3Client:
-    def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
+    def __init__(self, s3_creds: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
-        self.credentials: S3Credentials = credentials
+        self.credentials: S3Credentials = s3_creds
         self.client: BaseClient = create_s3_client(
-            credentials, config=S3Config(verbose=verbose)
+            s3_creds=s3_creds, s3_config=S3Config(verbose=verbose)
         )
 
     def list_bucket_contents(self, bucket_name: str) -> None:

rclone_api/s3/create.py

@@ -8,7 +8,7 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 50
+_MAX_CONNECTIONS = 10
 _TIMEOUT_READ = 120
 _TIMEOUT_CONNECT = 60
 
@@ -16,33 +16,30 @@ _TIMEOUT_CONNECT = 60
 @dataclass
 class S3Config:
     max_pool_connections: int | None = None
-    max_connections: int | None = None
     timeout_connection: int | None = None
     timeout_read: int | None = None
     verbose: bool | None = None
 
     def resolve_defaults(self) -> None:
-        if self.max_pool_connections is None:
-            self.max_pool_connections = _MAX_CONNECTIONS
-        if self.timeout_connection is None:
-            self.timeout_connection = _TIMEOUT_CONNECT
-        if self.timeout_read is None:
-            self.timeout_read = _TIMEOUT_READ
-        if self.verbose is None:
-            self.verbose = False
+        self.max_pool_connections = self.max_pool_connections or _MAX_CONNECTIONS
+        self.timeout_connection = self.timeout_connection or _TIMEOUT_CONNECT
+        self.timeout_read = self.timeout_read or _TIMEOUT_READ
+        self.verbose = self.verbose or False
 
 
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_backblaze_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    region_name = creds.region_name
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    region_name = s3_creds.region_name
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -53,9 +50,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=config.max_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -64,18 +61,20 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_unknown_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if config.verbose:
+        if s3_config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -84,25 +83,25 @@ def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseCli
         endpoint_url=endpoint_url,
         config=Config(
             signature_version="s3v4",
-            region_name=creds.region_name,
-            max_pool_connections=config.max_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            region_name=s3_creds.region_name,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
         ),
     )
 
 
 def create_s3_client(
-    credentials: S3Credentials, config: S3Config | None = None
+    s3_creds: S3Credentials, s3_config: S3Config | None = None
 ) -> BaseClient:
     """Create and return an S3 client."""
-    config = config or S3Config()
-    provider = credentials.provider
+    s3_config = s3_config or S3Config()
+    provider = s3_creds.provider
     if provider == S3Provider.BACKBLAZE:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, config=config)
+        return _create_backblaze_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     else:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, config=config)
+        return _create_unknown_s3_client(s3_creds=s3_creds, s3_config=s3_config)

rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -240,7 +240,7 @@ class S3MultiPartUploader:
         destination_key: str,
         chunk_size: int,
         final_size: int,
-        retries: int = 3,
+        retries: int = 100,
     ) -> str:
         """
         Finish a multipart upload by copying parts from existing S3 objects.

{rclone_api-1.4.13.dist-info → rclone_api-1.4.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.13
+Version: 1.4.15
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.13.dist-info → rclone_api-1.4.15.dist-info}/RECORD

@@ -27,7 +27,7 @@ rclone_api/util.py,sha256=9w_m6W62l_X42Jw5q8p_p30h-QoxAqufvnCLI4PTMOE,7056
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
 rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
-rclone_api/cmd/copy_large_s3_finish.py,sha256=7bDaUZ0forOqR4JOT1eoMBF_2qtifM9GcLK_he53cw4,4877
+rclone_api/cmd/copy_large_s3_finish.py,sha256=6fP95UU4ueBkGmkhX4YDu1ze0HIJlgYhCGF5z9IFc3k,5015
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
@@ -37,20 +37,20 @@ rclone_api/detail/copy_file_parts.py,sha256=dpqZ0d7l195dZg6Vob2Ty43Uah1v0ozQu5kM
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/s3/api.py,sha256=nIjOskTgTlbksbBHgF27ExFEYheCT_OvcdYl6FqRAto,4058
+rclone_api/s3/api.py,sha256=owoQ1H-R0hXcUozxC6sl53D7NmMOewHk2pUxK-ye8ms,4061
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
-rclone_api/s3/create.py,sha256=NsJtyOSTR_4kvwdojrzrg8LIjMVRuOsTGt4KkUrV0OM,4015
-rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=KNEWG3CrM8lVWd2b0dA6WT5LO3gEEXNk2tA6Uk9Vi7I,8701
+rclone_api/s3/create.py,sha256=_Q-faQ4Zl8XKTB28gireRxVXWP-YNxoAK4bligxDtiI,3998
+rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=JjMSG9XHiBlKnihPKw43SBYsh-NBqg5FZ4rnGU7j48s,8703
 rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
 rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
 rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaRCmkYLTQ,105
 rclone_api/s3/multipart/finished_piece.py,sha256=9nMWnVZ8S99wi2VFQsm1h1ZHqmebkhMGgd2s56wNj9w,1331
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.13.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.13.dist-info/METADATA,sha256=UFK0tBZDL5_d8X-NpHskBsPk7YxWHo5ZqMI0Quqwsic,4628
-rclone_api-1.4.13.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.13.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.13.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.13.dist-info/RECORD,,
+rclone_api-1.4.15.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.15.dist-info/METADATA,sha256=yC0hOJ09OMTVwf98k6NEfy7-gmlwu4IqWmWdnfUP67Q,4628
+rclone_api-1.4.15.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.15.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.15.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.15.dist-info/RECORD,,