rclone-api 1.0.90__py2.py3-none-any.whl → 1.0.93__py2.py3-none-any.whl

rclone_api/rclone.py

@@ -679,11 +679,12 @@ class Rclone:
         concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
         retries: int = 3,
         max_chunks_before_suspension: int | None = None,
+        mount_path: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.s3.api import S3Client
         from rclone_api.s3.create import S3Credentials
-        from rclone_api.util import S3PathInfo, split_s3_path
+        from rclone_api.util import S3PathInfo, random_str, split_s3_path
 
         other_args: list[str] = [
             "--no-modtime",
@@ -699,7 +700,7 @@ class Rclone:
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
         ]
-        mount_path = Path("rclone_api_upload_mount")
+        mount_path = mount_path or Path("tmp_mnts") / random_str(12)
         src_path = Path(src)
         name = src_path.name
 
@@ -784,8 +785,7 @@ class Rclone:
             )
 
             out: MultiUploadResult = client.upload_file_multipart(
-                upload_target=upload_target,
-                upload_config=upload_config
+                upload_target=upload_target, upload_config=upload_config
             )
             return out
 
@@ -877,6 +877,7 @@ class Rclone:
         other_args: list[str] | None = None,
     ) -> Generator[Process, None, None]:
         """Like mount, but can be used in a context manager."""
+        error_happened = False
         proc = self.mount(
             src,
             outdir,
@@ -888,6 +889,7 @@ class Rclone:
         try:
             yield proc
         except Exception as e:
+            error_happened = True
             stack_trace = traceback.format_exc()
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
@@ -895,6 +897,24 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
+            if not error_happened and outdir.exists():
+                time.sleep(2)
+                if outdir.exists():
+                    print(f"{outdir} mount still exists, attempting to remove")
+                    if not _IS_WINDOWS:
+                        # attempt
+                        os.system(f"fusermount -u {outdir}")
+                        os.system(f"umount {outdir}")
+                        time.sleep(2)
+                        if outdir.exists():
+                            is_empty = not list(outdir.iterdir())
+                            if not is_empty:
+                                warnings.warn(f"Failed to unmount {outdir}")
+                            else:
+                                try:
+                                    outdir.rmdir()
+                                except Exception as e:
+                                    warnings.warn(f"Failed to remove {outdir}: {e}")
 
     @deprecated("mount")
     def mount_webdav(

rclone_api/s3/api.py

@@ -1,5 +1,5 @@
+import json
 import warnings
-from pathlib import Path
 
 from botocore.client import BaseClient
 
@@ -46,28 +46,50 @@ class S3Client:
         upload_target: S3UploadTarget,
         upload_config: S3MutliPartUploadConfig,
     ) -> MultiUploadResult:
-        filesize = upload_target.src_file.stat().st_size
-        if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
-            warnings.warn(
-                f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
-            )
-            err = self.upload_file(upload_target)
-            if err:
-                raise err
-            return MultiUploadResult.UPLOADED_FRESH
+
         chunk_size = upload_config.chunk_size
         retries = upload_config.retries
         resume_path_json = upload_config.resume_path_json
         max_chunks_before_suspension = upload_config.max_chunks_before_suspension
         bucket_name = upload_target.bucket_name
-        out = upload_file_multipart(
-            s3_client=self.client,
-            bucket_name=bucket_name,
-            file_path=upload_target.src_file,
-            object_name=upload_target.s3_key,
-            resumable_info_path=resume_path_json,
-            chunk_size=chunk_size,
-            retries=retries,
-            max_chunks_before_suspension=max_chunks_before_suspension,
-        )
-        return out
+
+        try:
+            filesize = upload_target.src_file.stat().st_size
+            if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
+                warnings.warn(
+                    f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
+                )
+                err = self.upload_file(upload_target)
+                if err:
+                    raise err
+                return MultiUploadResult.UPLOADED_FRESH
+
+            out = upload_file_multipart(
+                s3_client=self.client,
+                bucket_name=bucket_name,
+                file_path=upload_target.src_file,
+                object_name=upload_target.s3_key,
+                resumable_info_path=resume_path_json,
+                chunk_size=chunk_size,
+                retries=retries,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+            return out
+        except Exception as e:
+            key = upload_target.s3_key
+            access_key_id = self.credentials.access_key_id[:4] + "..."
+            secret = self.credentials.secret_access_key[:4] + "..."
+            endpoint_url = self.credentials.endpoint_url
+            provider = self.credentials.provider
+            region_name = self.credentials.region_name
+            info_json = {
+                "key": key,
+                "access_key_id": access_key_id[:4] + "...",
+                "secret": secret[:4] + "...",
+                "endpoint_url": endpoint_url,
+                "provider": provider,
+                "region": region_name,
+            }
+            info_json_str = json.dumps(info_json, indent=2)
+            warnings.warn(f"Error uploading file: {e}\nInfo:\n\n{info_json_str}")
+            raise

rclone_api/s3/types.py

@@ -47,8 +47,9 @@ class S3MutliPartUploadConfig:
     retries: int
     resume_path_json: Path
     max_chunks_before_suspension: int | None = None
-    mount_path: Path | None = None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
-
+    mount_path: Path | None = (
+        None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
+    )
 
 
 class MultiUploadResult(Enum):

rclone_api/util.py

@@ -166,3 +166,10 @@ def split_s3_path(path: str) -> S3PathInfo:
     assert bucket
     assert key
     return S3PathInfo(remote=remote, bucket=bucket, key=key)
+
+
+def random_str(length: int) -> str:
+    import random
+    import string
+
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))

{rclone_api-1.0.90.dist-info → rclone_api-1.0.93.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.90
+Version: 1.0.93
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.90.dist-info → rclone_api-1.0.93.dist-info}/RECORD

@@ -12,24 +12,24 @@ rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=W4CIhDEPJqY5X9-HXKp-DTVLhU5H8YUOjGYAHDVy-yU,41207
+rclone_api/rclone.py,sha256=tXVZv9N83DVVxVwevOo7FyzHCuUGjWx_KP51VwQ0QIY,42246
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
 rclone_api/types.py,sha256=NC3e78aXCx-sEQ-FqEaC9KzaJDdJhJrKa4Nwum_-Db0,563
-rclone_api/util.py,sha256=4bJF1CC8J2NW6UbJU3oSxmTJgN04uTovABZJn7c3-zk,5025
+rclone_api/util.py,sha256=efck9W0rw5wfeRI35iiEz4dy2cMkNpVXrQ9zzynkBks,5185
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/copy_large_s3.py,sha256=FPU0S1Y9pApVLmWcdMT_3llywjEtOtwobXTnMXE-FhY,2690
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api/s3/api.py,sha256=Ju9A6uNHON2by2jJPtH-awe-3d588QLh5E-vjdV4kOY,2729
+rclone_api/s3/api.py,sha256=sks9QLErHsLJ06Af7zJSa5-E05_vHqi9YJQxomtx3D8,3639
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_uploader.py,sha256=Wcm4h_A6ORaJvT9P9qSeknbJYch22RKZ4FGo1iePlAw,17385
 rclone_api/s3/create.py,sha256=X4mgjekgAoyHNeXnznHNS63DFViAhI0p7-fXDxml0y4,2701
-rclone_api/s3/types.py,sha256=BKvNkAFjLJK7Zd9xQya8gdjqJPLqdCPZSXD8GeFyP1A,1346
-rclone_api-1.0.90.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.90.dist-info/METADATA,sha256=7Y-NCkQi9yBFGqd90T-j9oA2rvA0rEu_-znW5LW_dZY,4479
-rclone_api-1.0.90.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.0.90.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.0.90.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.90.dist-info/RECORD,,
+rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
+rclone_api-1.0.93.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.93.dist-info/METADATA,sha256=T1_yOU3wV_gKRNH_bNGH9YSvOaj2rwyy8eIpQT1cw0k,4479
+rclone_api-1.0.93.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.0.93.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.0.93.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.93.dist-info/RECORD,,