rclone-api 1.1.8__tar.gz → 1.1.11__tar.gz

{rclone_api-1.1.8 → rclone_api-1.1.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.8
+Version: 1.1.11
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.8 → rclone_api-1.1.11}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.8"
+version = "1.1.11"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api/cmd/copy_large_s3.py

@@ -38,13 +38,13 @@ def _parse_args() -> Args:
         "--chunk-size",
         help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
         type=str,
-        default="512M",
+        default="1G",
     )
     parser.add_argument(
         "--threads",
         help="Number of threads to use per chunk",
         type=int,
-        default=16,
+        default=64,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api/mount.py

@@ -18,12 +18,23 @@ class Mount:
 
     mount_path: Path
     process: Process
+    _closed: bool = False
 
     def __post_init__(self):
         assert isinstance(self.mount_path, Path)
         assert self.process is not None
         wait_for_mount(self.mount_path, self.process)
 
+    def close(self, wait=True) -> None:
+        """Clean up the mount."""
+        if self._closed:
+            return
+        self._closed = True
+        clean_mount(self, verbose=False)
+
+    def __del__(self):
+        self.close(wait=False)
+
 
 def run_command(cmd: str, verbose: bool) -> int:
     """Run a shell command and print its output if verbose is True."""
@@ -74,7 +85,7 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
         time.sleep(1)
 
 
-def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
+def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
     """
     Clean up a mount path across Linux, macOS, and Windows.
 
@@ -98,7 +109,8 @@ def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
         mount_exists = True
 
     # Give the system a moment (if unmount is in progress, etc.)
-    time.sleep(2)
+    if wait:
+        time.sleep(2)
 
     if not mount_exists:
         if verbose:
@@ -132,7 +144,8 @@ def clean_mount(mount: Mount | Path, verbose: bool = False) -> None:
         warnings.warn(f"Unsupported platform: {_SYSTEM}")
 
     # Allow some time for the unmount commands to take effect.
-    time.sleep(2)
+    if wait:
+        time.sleep(2)
 
     # Re-check if the mount path still exists.
     try:

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api/rclone.py

@@ -695,6 +695,7 @@ class Rclone:
         vfs_read_chunk_size = unit_chunk_size
         vfs_read_chunk_size_limit = chunk_size
         vfs_read_chunk_streams = threads
+        vfs_disk_space_total_size = chunk_size
         assert (
             chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
         ), f"chunk_size {chunk_size} must be a multiple of vfs_read_chunk_size {vfs_read_chunk_size}"
@@ -704,6 +705,10 @@ class Rclone:
             vfs_read_chunk_size_limit.as_str(),
         ]
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
+        other_args += [
+            "--vfs-disk-space-total-size",
+            vfs_disk_space_total_size.as_str(),
+        ]
         mount_path = mount_path or Path("tmp_mnts") / random_str(12)
         src_path = Path(src)
         name = src_path.name
@@ -903,11 +908,8 @@ class Rclone:
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
         finally:
-
             if not error_happened:
-                from rclone_api.mount import clean_mount
-
-                clean_mount(mount, verbose=verbose)
+                mount.close()
 
     # Settings optimized for s3.
     def mount_s3(

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api/s3/chunk_types.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import time
@@ -103,6 +104,19 @@ class UploadInfo:
             return
         self._total_chunks = self.total_chunks()
 
+    def fingerprint(self) -> str:
+        # hash the attributes that are used to identify the upload
+        hasher = hashlib.sha256()
+        # first is file size
+        hasher.update(str(self.file_size).encode("utf-8"))
+        # second is the file path
+        hasher.update(str(self.src_file_path).encode("utf-8"))
+        # next is chunk size
+        hasher.update(str(self.chunk_size).encode("utf-8"))
+        # next is the number of parts
+        hasher.update(str(self._total_chunks).encode("utf-8"))
+        return hasher.hexdigest()
+
     def to_json(self) -> dict:
         json_dict = {}
         for f in fields(self):

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api/s3/chunk_uploader.py

@@ -162,7 +162,25 @@ def upload_file_multipart(
         return upload_state
 
     filechunks: Queue[FileChunk | None] = Queue(10)
-    upload_state = get_upload_state() or make_new_state()
+    new_state = make_new_state()
+    loaded_state = get_upload_state()
+
+    if loaded_state is None:
+        upload_state = new_state
+    else:
+        # if the file size has changed, we cannot resume
+        if (
+            loaded_state.upload_info.fingerprint()
+            != new_state.upload_info.fingerprint()
+        ):
+            locked_print(
+                f"Cannot resume upload: file size changed, starting over for {file_path}"
+            )
+            _abort_previous_upload(loaded_state)
+            upload_state = new_state
+        else:
+            upload_state = loaded_state
+
     try:
         upload_state.update_source_file(file_path)
     except ValueError as e:

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api/types.py

@@ -52,33 +52,33 @@ def _to_size_suffix(size: int) -> str:
     raise ValueError(f"Invalid size: {size}")
 
 
-_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+)([A-Za-z]+)$")
+# Update regex to allow decimals (e.g., 16.5MB)
+_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+(?:\.\d+)?)([A-Za-z]+)$")
 
 
 def _from_size_suffix(size: str) -> int:
-    # 16MiB
-    # parse out number and suffix
     if size == "0":
         return 0
     match = _PATTERN_SIZE_SUFFIX.match(size)
     if match is None:
         raise ValueError(f"Invalid size suffix: {size}")
-    size = match.group(1)
-    suffix = match.group(2)[0:1].upper()
-    n = int(size)
-    if suffix == "B":
-        return n
-    if suffix == "K":
-        return n * 1024
-    if suffix == "M":
-        return n * 1024 * 1024
-    if suffix == "G":
-        return n * 1024 * 1024 * 1024
-    if suffix == "T":
-        return n * 1024 * 1024 * 1024 * 1024
-    if suffix == "P":
-        return n * 1024 * 1024 * 1024 * 1024 * 1024
-    raise ValueError(f"Invalid size suffix: {size}")
+    num_str, suffix = match.group(1), match.group(2)
+    n = float(num_str)
+    # Determine the unit from the first letter (e.g., "M" from "MB")
+    unit = suffix[0].upper()
+    if unit == "B":
+        return int(n)
+    if unit == "K":
+        return int(n * 1024)
+    if unit == "M":
+        return int(n * 1024 * 1024)
+    if unit == "G":
+        return int(n * 1024 * 1024 * 1024)
+    if unit == "T":
+        return int(n * 1024**4)
+    if unit == "P":
+        return int(n * 1024**5)
+    raise ValueError(f"Invalid size suffix: {suffix}")
 
 
 class SizeSuffix:

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.8
+Version: 1.1.11
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.8 → rclone_api-1.1.11}/src/rclone_api.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ clean
 install
 lint
 pyproject.toml
+rclone-mounted-ranged-download.conf.old3393025366
 requirements.testing.txt
 setup.py
 test

{rclone_api-1.1.8 → rclone_api-1.1.11}/tests/test_size_suffix.py

@@ -10,12 +10,17 @@ from rclone_api import SizeSuffix
 class RcloneSuffixSize(unittest.TestCase):
     """Test rclone functionality."""
 
-    def test_list_remotes(self) -> None:
+    def test_simple_suffix(self) -> None:
         size_suffix = SizeSuffix(1024)
         size_suffix = SizeSuffix("16MB")
         size_int = size_suffix.as_int()
         self.assertEqual(size_int, 16 * 1024 * 1024)
 
+    def test_float_suffix(self) -> None:
+        size_suffix = SizeSuffix("16.5MB")
+        size_int = size_suffix.as_int()
+        self.assertEqual(size_int, int(16.5 * 1024 * 1024))
+
 
 if __name__ == "__main__":
     unittest.main()