rclone-api 1.0.90__tar.gz → 1.0.93__tar.gz

{rclone_api-1.0.90 → rclone_api-1.0.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.90
+Version: 1.0.93
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.90 → rclone_api-1.0.93}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.0.90"
+version = "1.0.93"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.90 → rclone_api-1.0.93}/src/rclone_api/rclone.py

@@ -679,11 +679,12 @@ class Rclone:
         concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
         retries: int = 3,
         max_chunks_before_suspension: int | None = None,
+        mount_path: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.s3.api import S3Client
         from rclone_api.s3.create import S3Credentials
-        from rclone_api.util import S3PathInfo, split_s3_path
+        from rclone_api.util import S3PathInfo, random_str, split_s3_path
 
         other_args: list[str] = [
             "--no-modtime",
@@ -699,7 +700,7 @@ class Rclone:
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
         ]
-        mount_path = Path("rclone_api_upload_mount")
+        mount_path = mount_path or Path("tmp_mnts") / random_str(12)
         src_path = Path(src)
         name = src_path.name
 
@@ -784,8 +785,7 @@ class Rclone:
             )
 
             out: MultiUploadResult = client.upload_file_multipart(
-                upload_target=upload_target,
-                upload_config=upload_config
+                upload_target=upload_target, upload_config=upload_config
             )
             return out
 
@@ -877,6 +877,7 @@ class Rclone:
         other_args: list[str] | None = None,
     ) -> Generator[Process, None, None]:
         """Like mount, but can be used in a context manager."""
+        error_happened = False
         proc = self.mount(
             src,
             outdir,
@@ -888,6 +889,7 @@ class Rclone:
         try:
             yield proc
         except Exception as e:
+            error_happened = True
             stack_trace = traceback.format_exc()
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
@@ -895,6 +897,24 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
+            if not error_happened and outdir.exists():
+                time.sleep(2)
+                if outdir.exists():
+                    print(f"{outdir} mount still exists, attempting to remove")
+                    if not _IS_WINDOWS:
+                        # attempt
+                        os.system(f"fusermount -u {outdir}")
+                        os.system(f"umount {outdir}")
+                        time.sleep(2)
+                        if outdir.exists():
+                            is_empty = not list(outdir.iterdir())
+                            if not is_empty:
+                                warnings.warn(f"Failed to unmount {outdir}")
+                            else:
+                                try:
+                                    outdir.rmdir()
+                                except Exception as e:
+                                    warnings.warn(f"Failed to remove {outdir}: {e}")
 
     @deprecated("mount")
     def mount_webdav(

{rclone_api-1.0.90 → rclone_api-1.0.93}/src/rclone_api/s3/api.py

@@ -1,5 +1,5 @@
+import json
 import warnings
-from pathlib import Path
 
 from botocore.client import BaseClient
 
@@ -46,28 +46,50 @@ class S3Client:
         upload_target: S3UploadTarget,
         upload_config: S3MutliPartUploadConfig,
     ) -> MultiUploadResult:
-        filesize = upload_target.src_file.stat().st_size
-        if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
-            warnings.warn(
-                f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
-            )
-            err = self.upload_file(upload_target)
-            if err:
-                raise err
-            return MultiUploadResult.UPLOADED_FRESH
+
         chunk_size = upload_config.chunk_size
         retries = upload_config.retries
         resume_path_json = upload_config.resume_path_json
         max_chunks_before_suspension = upload_config.max_chunks_before_suspension
         bucket_name = upload_target.bucket_name
-        out = upload_file_multipart(
-            s3_client=self.client,
-            bucket_name=bucket_name,
-            file_path=upload_target.src_file,
-            object_name=upload_target.s3_key,
-            resumable_info_path=resume_path_json,
-            chunk_size=chunk_size,
-            retries=retries,
-            max_chunks_before_suspension=max_chunks_before_suspension,
-        )
-        return out
+
+        try:
+            filesize = upload_target.src_file.stat().st_size
+            if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
+                warnings.warn(
+                    f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
+                )
+                err = self.upload_file(upload_target)
+                if err:
+                    raise err
+                return MultiUploadResult.UPLOADED_FRESH
+
+            out = upload_file_multipart(
+                s3_client=self.client,
+                bucket_name=bucket_name,
+                file_path=upload_target.src_file,
+                object_name=upload_target.s3_key,
+                resumable_info_path=resume_path_json,
+                chunk_size=chunk_size,
+                retries=retries,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+            return out
+        except Exception as e:
+            key = upload_target.s3_key
+            access_key_id = self.credentials.access_key_id[:4] + "..."
+            secret = self.credentials.secret_access_key[:4] + "..."
+            endpoint_url = self.credentials.endpoint_url
+            provider = self.credentials.provider
+            region_name = self.credentials.region_name
+            info_json = {
+                "key": key,
+                "access_key_id": access_key_id[:4] + "...",
+                "secret": secret[:4] + "...",
+                "endpoint_url": endpoint_url,
+                "provider": provider,
+                "region": region_name,
+            }
+            info_json_str = json.dumps(info_json, indent=2)
+            warnings.warn(f"Error uploading file: {e}\nInfo:\n\n{info_json_str}")
+            raise

{rclone_api-1.0.90 → rclone_api-1.0.93}/src/rclone_api/s3/types.py

@@ -47,8 +47,9 @@ class S3MutliPartUploadConfig:
     retries: int
     resume_path_json: Path
     max_chunks_before_suspension: int | None = None
-    mount_path: Path | None = None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
-
+    mount_path: Path | None = (
+        None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
+    )
 
 
 class MultiUploadResult(Enum):

{rclone_api-1.0.90 → rclone_api-1.0.93}/src/rclone_api/util.py

@@ -166,3 +166,10 @@ def split_s3_path(path: str) -> S3PathInfo:
     assert bucket
     assert key
     return S3PathInfo(remote=remote, bucket=bucket, key=key)
+
+
+def random_str(length: int) -> str:
+    import random
+    import string
+
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))

{rclone_api-1.0.90 → rclone_api-1.0.93}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.90
+Version: 1.0.93
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License