rclone-api 1.1.8__py2.py3-none-any.whl → 1.1.11__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3.py

@@ -38,13 +38,13 @@ def _parse_args() -> Args:
         "--chunk-size",
         help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
         type=str,
-        default="512M",
+        default="1G",
     )
     parser.add_argument(
         "--threads",
         help="Number of threads to use per chunk",
         type=int,
-        default=16,
+        default=64,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(

rclone_api/mount.py

@@ -18,12 +18,23 @@ class Mount:
 
     mount_path: Path
     process: Process
+    _closed: bool = False
 
     def __post_init__(self):
         assert isinstance(self.mount_path, Path)
         assert self.process is not None
         wait_for_mount(self.mount_path, self.process)
 
+    def close(self, wait=True) -> None:
+        """Clean up the mount."""
+        if self._closed:
+            return
+        self._closed = True
+        clean_mount(self, verbose=False)
+
+    def __del__(self):
+        self.close(wait=False)
+
 
 def run_command(cmd: str, verbose: bool) -> int:
     """Run a shell command and print its output if verbose is True."""
@@ -74,7 +85,7 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
         time.sleep(1)
 
 
-def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
+def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
     """
     Clean up a mount path across Linux, macOS, and Windows.
 
@@ -98,7 +109,8 @@ def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
         mount_exists = True
 
     # Give the system a moment (if unmount is in progress, etc.)
-    time.sleep(2)
+    if wait:
+        time.sleep(2)
 
     if not mount_exists:
         if verbose:
@@ -132,7 +144,8 @@ def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
         warnings.warn(f"Unsupported platform: {_SYSTEM}")
 
     # Allow some time for the unmount commands to take effect.
-    time.sleep(2)
+    if wait:
+        time.sleep(2)
 
     # Re-check if the mount path still exists.
     try:

rclone_api/rclone.py

@@ -695,6 +695,7 @@ class Rclone:
         vfs_read_chunk_size = unit_chunk_size
         vfs_read_chunk_size_limit = chunk_size
         vfs_read_chunk_streams = threads
+        vfs_disk_space_total_size = chunk_size
         assert (
             chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
         ), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
@@ -704,6 +705,10 @@ class Rclone:
             vfs_read_chunk_size_limit.as_str(),
         ]
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
+        other_args += [
+            "--vfs-disk-space-total-size",
+            vfs_disk_space_total_size.as_str(),
+        ]
         mount_path = mount_path or Path("tmp_mnts") / random_str(12)
         src_path = Path(src)
         name = src_path.name
@@ -903,11 +908,8 @@ class Rclone:
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
         finally:
-
             if not error_happened:
-                from rclone_api.mount import clean_mount
-
-                clean_mount(mount, verbose=verbose)
+                mount.close()
 
     # Settings optimized for s3.
     def mount_s3(

rclone_api/s3/chunk_types.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import time
@@ -103,6 +104,19 @@ class UploadInfo:
             return
         self._total_chunks = self.total_chunks()
 
+    def fingerprint(self) -> str:
+        # hash the attributes that are used to identify the upload
+        hasher = hashlib.sha256()
+        # first is file size
+        hasher.update(str(self.file_size).encode("utf-8"))
+        # second is the file path
+        hasher.update(str(self.src_file_path).encode("utf-8"))
+        # next is chunk size
+        hasher.update(str(self.chunk_size).encode("utf-8"))
+        # next is the number of parts
+        hasher.update(str(self._total_chunks).encode("utf-8"))
+        return hasher.hexdigest()
+
     def to_json(self) -> dict:
         json_dict = {}
         for f in fields(self):

rclone_api/s3/chunk_uploader.py

@@ -162,7 +162,25 @@ def upload_file_multipart(
         return upload_state
 
     filechunks: Queue[FileChunk | None] = Queue(10)
-    upload_state = get_upload_state() or make_new_state()
+    new_state = make_new_state()
+    loaded_state = get_upload_state()
+
+    if loaded_state is None:
+        upload_state = new_state
+    else:
+        # if the file size has changed, we cannot resume
+        if (
+            loaded_state.upload_info.fingerprint()
+            != new_state.upload_info.fingerprint()
+        ):
+            locked_print(
+                f"Cannot resume upload: file size changed, starting over for {file_path}"
+            )
+            _abort_previous_upload(loaded_state)
+            upload_state = new_state
+        else:
+            upload_state = loaded_state
+
     try:
         upload_state.update_source_file(file_path)
     except ValueError as e:

rclone_api/types.py

@@ -52,33 +52,33 @@ def _to_size_suffix(size: int) -> str:
     raise ValueError(f"Invalid size: {size}")
 
 
-_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+)([A-Za-z]+)$")
+# Update regex to allow decimals (e.g., 16.5MB)
+_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+(?:\.\d+)?)([A-Za-z]+)$")
 
 
 def _from_size_suffix(size: str) -> int:
-    # 16MiB
-    # parse out number and suffix
     if size == "0":
         return 0
     match = _PATTERN_SIZE_SUFFIX.match(size)
     if match is None:
         raise ValueError(f"Invalid size suffix: {size}")
-    size = match.group(1)
-    suffix = match.group(2)[0:1].upper()
-    n = int(size)
-    if suffix == "B":
-        return n
-    if suffix == "K":
-        return n * 1024
-    if suffix == "M":
-        return n * 1024 * 1024
-    if suffix == "G":
-        return n * 1024 * 1024 * 1024
-    if suffix == "T":
-        return n * 1024 * 1024 * 1024 * 1024
-    if suffix == "P":
-        return n * 1024 * 1024 * 1024 * 1024 * 1024
-    raise ValueError(f"Invalid size suffix: {size}")
+    num_str, suffix = match.group(1), match.group(2)
+    n = float(num_str)
+    # Determine the unit from the first letter (e.g., "M" from "MB")
+    unit = suffix[0].upper()
+    if unit == "B":
+        return int(n)
+    if unit == "K":
+        return int(n * 1024)
+    if unit == "M":
+        return int(n * 1024 * 1024)
+    if unit == "G":
+        return int(n * 1024 * 1024 * 1024)
+    if unit == "T":
+        return int(n * 1024**4)
+    if unit == "P":
+        return int(n * 1024**5)
+    raise ValueError(f"Invalid size suffix: {suffix}")
 
 
 class SizeSuffix:

{rclone_api-1.1.8.dist-info → rclone_api-1.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.8
+Version: 1.1.11
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.8.dist-info → rclone_api-1.1.11.dist-info}/RECORD

@@ -11,30 +11,30 @@ rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/mount.py,sha256=8vSXIaVwj6RNwPvk0N4IA6pPBXkI3XRnO4W0JcrAbDc,5759
+rclone_api/mount.py,sha256=Xj3BMVSDEwUbtMi8ycn5mUom-fAf-F9lJOjR_BzVllw,6073
 rclone_api/process.py,sha256=xYUgU17txkZfZdr4vtRfvD8YjvSfdrbjM7PYW1npAMI,4264
-rclone_api/rclone.py,sha256=-WJseVj--ko_D4dLcEqtfG6i7G_U3fu4YL95XCWH1wE,39718
+rclone_api/rclone.py,sha256=BA3DJvCraLIrglDGSi1mF-wCH8rsqpBzc-5O-IWQpq8,39809
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=OWj9mUFaehC3PMAyt-uBYB1V2MfQ-ugJfUlIVjZ09cw,3775
+rclone_api/types.py,sha256=zfTb0iM6mhfqgaYS6j6T0NIOA4e9GymNOXLPhVELe4A,3853
 rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/copy_large_s3.py,sha256=DNALWMwinu7I03nBhao-7Huo7OoS5k1HcqiaHTM0QZU,3071
+rclone_api/cmd/copy_large_s3.py,sha256=UR5qkW9vLtGLC951Om8UWFcQKApZUF-SzUhlIo3YIGU,3069
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
 rclone_api/s3/api.py,sha256=VstlaEnBjO2JDQuCRLdTfUGvQLbfshlXXhAzimFv4Vc,3763
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=XPoDl7DJMJIGBMRoPO2wqwqCMT7ZrIsEkDqlbMH8jzs,3506
-rclone_api/s3/chunk_types.py,sha256=6_ythhT7EcchJ9MI8qBKVKh-SVOe1Rf15t_JepHPilA,8587
-rclone_api/s3/chunk_uploader.py,sha256=KO8834Gery9HKWSqjQTNW0pbBbVoGrza9gj-1OaNLQQ,9130
+rclone_api/s3/chunk_types.py,sha256=pZUKc3L418XTwET1rLm4eRo_2swKQckSqlc4XjYmhDo,9147
+rclone_api/s3/chunk_uploader.py,sha256=Y4I208YoXTJQQke1qRsXARHxBLjJAz9nt_r1mKkplUQ,9670
 rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
 rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
-rclone_api-1.1.8.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.8.dist-info/METADATA,sha256=o2HlbSg87QWtMTq0983EdoivnDCTUsPI4tN5Cowi5Tc,4478
-rclone_api-1.1.8.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.8.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.1.8.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.8.dist-info/RECORD,,
+rclone_api-1.1.11.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.11.dist-info/METADATA,sha256=LnxU7g0Hzoe_gvGmz1F0MKU67cRtsJEPk1Gkxe6SQVQ,4479
+rclone_api-1.1.11.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.11.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.1.11.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.11.dist-info/RECORD,,