rclone-api 1.4.12__py2.py3-none-any.whl → 1.4.14__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -507,6 +507,10 @@ class Rclone:
             verbose=verbose,
         )
 
+    def size_file(self, src: str) -> SizeSuffix | Exception:
+        """Get the size of a file."""
+        return self.impl.size_file(src=src)
+
 
 __all__ = [
     "Rclone",

rclone_api/cmd/copy_large_s3_finish.py

@@ -51,10 +51,17 @@ def _parse_args() -> Args:
 
 
 def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
-    from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
+    from rclone_api.s3.create import (
+        BaseClient,
+        S3Config,
+        S3Credentials,
+        create_s3_client,
+    )
 
     s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(s3_creds)
+    s3_client: BaseClient = create_s3_client(
+        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    )
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
     assert is_done, f"Upload is not done: {info}"
@@ -112,6 +119,20 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         retries=3,
     )
 
+    # now check if the dst now exists, if so, delete the parts folder.
+    # if rclone.exists(dst):
+    #     rclone.purge(parts_dir)
+
+    if not rclone.exists(dst):
+        raise FileNotFoundError(f"Destination file not found: {dst}")
+
+    write_size = rclone.size_file(dst)
+    if write_size != size:
+        raise ValueError(f"Size mismatch: {write_size} != {size}")
+
+    print(f"Upload complete: {dst}")
+    rclone.purge(parts_dir)
+
 
 def main() -> int:
     """Main entry point."""
@@ -120,7 +141,10 @@ def main() -> int:
     info_json = f"{args.src}/info.json".replace("//", "/")
     info = InfoJson(rclone.impl, src=None, src_info=info_json)
     loaded = info.load()
-    assert loaded
+    if not loaded:
+        raise FileNotFoundError(
+            f"Info file not found, has the upload finished? {info_json}"
+        )
     print(info)
     do_finish_part(rclone=rclone, info=info, dst=args.dst)
     return 0

rclone_api/s3/api.py

@@ -9,7 +9,7 @@ from rclone_api.s3.basic_ops import (
     list_bucket_contents,
     upload_file,
 )
-from rclone_api.s3.create import create_s3_client
+from rclone_api.s3.create import S3Config, create_s3_client
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
 from rclone_api.s3.upload_file_multipart import (
     MultiUploadResult,
@@ -23,7 +23,9 @@ class S3Client:
     def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
         self.credentials: S3Credentials = credentials
-        self.client: BaseClient = create_s3_client(credentials, verbose=verbose)
+        self.client: BaseClient = create_s3_client(
+            credentials, config=S3Config(verbose=verbose)
+        )
 
     def list_bucket_contents(self, bucket_name: str) -> None:
         list_bucket_contents(self.client, bucket_name)

rclone_api/s3/create.py

@@ -1,4 +1,5 @@
 import warnings
+from dataclasses import dataclass
 
 import boto3
 from botocore.client import BaseClient
@@ -8,18 +9,35 @@ from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
 _MAX_CONNECTIONS = 50
+_TIMEOUT_READ = 120
+_TIMEOUT_CONNECT = 60
+
+
+@dataclass
+class S3Config:
+    max_pool_connections: int | None = None
+    timeout_connection: int | None = None
+    timeout_read: int | None = None
+    verbose: bool | None = None
+
+    def resolve_defaults(self) -> None:
+        self.max_pool_connections = self.max_pool_connections or _MAX_CONNECTIONS
+        self.timeout_connection = self.timeout_connection or _TIMEOUT_CONNECT
+        self.timeout_read = self.timeout_read or _TIMEOUT_READ
+        self.verbose = self.verbose or False
 
 
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient:
+def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
     """Create and return an S3 client."""
     region_name = creds.region_name
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
+    config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -30,7 +48,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=_MAX_CONNECTIONS,
+            max_pool_connections=config.max_pool_connections,
+            read_timeout=config.timeout_read,
+            connect_timeout=config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -39,18 +59,18 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient:
+def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
     """Create and return an S3 client."""
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if verbose:
+        if config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-
+    config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -60,19 +80,24 @@ def _create_unknown_s3_client(creds: S3Credentials, verbose: bool) -> BaseClient
         config=Config(
             signature_version="s3v4",
             region_name=creds.region_name,
-            max_pool_connections=_MAX_CONNECTIONS,
+            max_pool_connections=config.max_pool_connections,
+            read_timeout=config.timeout_read,
+            connect_timeout=config.timeout_connection,
         ),
     )
 
 
-def create_s3_client(credentials: S3Credentials, verbose=False) -> BaseClient:
+def create_s3_client(
+    credentials: S3Credentials, config: S3Config | None = None
+) -> BaseClient:
     """Create and return an S3 client."""
+    config = config or S3Config()
     provider = credentials.provider
     if provider == S3Provider.BACKBLAZE:
-        if verbose:
+        if config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, verbose=verbose)
+        return _create_backblaze_s3_client(creds=credentials, config=config)
     else:
-        if verbose:
+        if config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, verbose=verbose)
+        return _create_unknown_s3_client(creds=credentials, config=config)

{rclone_api-1.4.12.dist-info → rclone_api-1.4.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.12
+Version: 1.4.14
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.12.dist-info → rclone_api-1.4.14.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=cJpn62AmlcCXdbRIMIls4yEek6SLWJ-ONlr2mDnkIFk,17530
+rclone_api/__init__.py,sha256=gsYL3jaqMgHbJ6mnLxNttuaKltUBXNuvpz7C6fF8_8w,17678
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -27,7 +27,7 @@ rclone_api/util.py,sha256=9w_m6W62l_X42Jw5q8p_p30h-QoxAqufvnCLI4PTMOE,7056
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
 rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
-rclone_api/cmd/copy_large_s3_finish.py,sha256=O7ezviM460ivprF5v9bx_ztqhg0_jnRmvzYK8n6R9IY,4212
+rclone_api/cmd/copy_large_s3_finish.py,sha256=7bDaUZ0forOqR4JOT1eoMBF_2qtifM9GcLK_he53cw4,4877
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
@@ -37,10 +37,10 @@ rclone_api/detail/copy_file_parts.py,sha256=dpqZ0d7l195dZg6Vob2Ty43Uah1v0ozQu5kM
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
+rclone_api/s3/api.py,sha256=nIjOskTgTlbksbBHgF27ExFEYheCT_OvcdYl6FqRAto,4058
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
-rclone_api/s3/create.py,sha256=Ao79MENZM5Bvi9SXwo6yng88Vkmn75rZElHKQrGckeo,2971
+rclone_api/s3/create.py,sha256=NinodMsVPtNXWOM_hlMJWWl-tOj6gV4BcOxEnXFGRWE,3902
 rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=KNEWG3CrM8lVWd2b0dA6WT5LO3gEEXNk2tA6Uk9Vi7I,8701
 rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
 rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
@@ -48,9 +48,9 @@ rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaR
 rclone_api/s3/multipart/finished_piece.py,sha256=9nMWnVZ8S99wi2VFQsm1h1ZHqmebkhMGgd2s56wNj9w,1331
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.12.dist-info/METADATA,sha256=i7GFP4S5sHfeiTlh9sKl_CXd7jVyXqldB2jvd_rLWPc,4628
-rclone_api-1.4.12.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.12.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.12.dist-info/RECORD,,
+rclone_api-1.4.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.14.dist-info/METADATA,sha256=w_EeXXSG4Z-yWrXBFz3kKPRtXh4oMZH2kTrrfko7eos,4628
+rclone_api-1.4.14.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.14.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.14.dist-info/RECORD,,