rclone-api 1.1.83__tar.gz → 1.1.85__tar.gz

{rclone_api-1.1.83 → rclone_api-1.1.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.83
+Version: 1.1.85
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.83 → rclone_api-1.1.85}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.83"
+version = "1.1.85"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.83 → rclone_api-1.1.85}/src/rclone_api/cmd/copy_large_s3.py

@@ -45,13 +45,13 @@ def _parse_args() -> Args:
         "--read-threads",
         help="Number of concurrent read threads per chunk, only one chunk will be read at a time",
         type=int,
-        default=32,
+        default=1,
     )
     parser.add_argument(
         "--write-threads",
         help="Max number of chunks to upload in parallel to the destination, each chunk is uploaded in a separate thread",
         type=int,
-        default=64,
+        default=4,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(

{rclone_api-1.1.83 → rclone_api-1.1.85}/src/rclone_api/mount.py

@@ -313,7 +313,7 @@ class MultiMountFileChunker:
             for mount in self.mounts_processing:
                 executor.submit(lambda: mount.close())
 
-    def fetch(self, offset: int, size: int) -> bytes | Exception:
+    def fetch(self, offset: int, size: int) -> Future[bytes | Exception]:
         if self.verbose:
             print(f"Fetching data range: offset={offset}, size={size}")
         try:
@@ -325,7 +325,10 @@ class MultiMountFileChunker:
                 assert offset >= 0, f"Invalid offset: {offset}"
             except AssertionError as e:
                 warnings.warn(f"Invalid chunk request: {e}")
-                return ValueError(e)
+                # return ValueError(e)
+                # return self.executor.submit(lambda: Exception(e))
+                err = Exception(e)
+                return self.executor.submit(lambda: err)
 
             chunks: list[tuple[int, int]] = []
             start = offset
@@ -375,15 +378,19 @@ class MultiMountFileChunker:
                 fut = self.executor.submit(task)
                 futures.append(fut)
 
-            out: bytes = b""
-            for fut in futures:
-                chunk: bytes | Exception = fut.result()
-                if isinstance(chunk, Exception):
-                    return chunk
-                else:
-                    out += chunk
-                    del chunk
-            return out
+            def combine(futs: list[Future[bytes | Exception]]) -> bytes | Exception:
+                finished_list: list[bytes | Exception] = [f.result() for f in futs]
+                bytes_list = [f for f in finished_list if isinstance(f, bytes)]
+                if len(bytes_list) != len(finished_list):
+                    exceptions = [f for f in finished_list if isinstance(f, Exception)]
+                    return Exception(f"Error fetching file chunk: {exceptions}")
+                return b"".join(bytes_list)
+
+            if len(futures) == 1:
+                return futures[0]
+            fut = self.executor.submit(combine, futures)
+            return fut
         except Exception as e:
             warnings.warn(f"Error fetching file chunk: {e}")
-            return e
+            err = Exception(e)
+            return self.executor.submit(lambda: err)

{rclone_api-1.1.83 → rclone_api-1.1.85}/src/rclone_api/profile/mount_copy_bytes.py

@@ -6,6 +6,7 @@ import argparse
 import os
 import shutil
 import time
+from concurrent.futures import Future
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -117,7 +118,7 @@ def _run_profile(
     print("#" * 80)
     print(f"# Started test download of {SizeSuffix(size)} with {transfers} transfers")
     print("#" * 80)
-    net_io_start = psutil.net_io_counters()
+
     chunk_size = size // transfers
 
     filechunker: MultiMountFileChunker = rclone.get_multi_mount_file_chunker(
@@ -128,9 +129,10 @@ def _run_profile(
         mount_log=mount_log,
     )
 
-    bytes_or_err = filechunker.fetch(offset.as_int(), size.as_int())
-    
+    bytes_or_err = filechunker.fetch(offset.as_int(), size.as_int()).result()
+
     start = time.time()
+    net_io_start = psutil.net_io_counters()
 
     # bytes_or_err: bytes | Exception = rclone.copy_bytes(
     #     src=src_file,
@@ -142,7 +144,22 @@ def _run_profile(
     #     mount_log=mount_log,
     # )
 
-    bytes_or_err = filechunker.fetch((offset + SizeSuffix("1G")).as_int(), size.as_int())
+    # bytes_or_err = filechunker.fetch(
+    #     (offset + SizeSuffix("1G")).as_int(), size.as_int()
+    # )
+
+    offset = SizeSuffix("1G")
+
+    futures: list[Future[bytes | Exception]] = []
+    for i in range(num):
+        offset = SizeSuffix(i * chunk_size.as_int()) + offset
+        future = filechunker.fetch(offset.as_int(), size.as_int())
+        futures.append(future)
+
+    for future in futures:
+        bytes_or_err = future.result()
+        if isinstance(bytes_or_err, Exception):
+            assert False, f"Error: {bytes_or_err}"
 
     diff = time.time() - start
     net_io_end = psutil.net_io_counters()

{rclone_api-1.1.83 → rclone_api-1.1.85}/src/rclone_api/rclone.py

@@ -993,7 +993,8 @@ class Rclone:
             direct_io=direct_io,
         )
         try:
-            data = filechunker.fetch(offset, length)
+            fut = filechunker.fetch(offset, length)
+            data = fut.result()
             if isinstance(data, Exception):
                 warnings.warn(f"Error copying bytes: {data}")
                 raise data

{rclone_api-1.1.83 → rclone_api-1.1.85}/src/rclone_api/types.py

@@ -178,40 +178,39 @@ class SizeSuffix:
             raise ZeroDivisionError("Division by zero is undefined")
         # Use floor division to maintain integer arithmetic.
         return SizeSuffix(self._size // other_int._size)
-    
+
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, SizeSuffix):
             return False
         return self._size == other._size
-    
+
     def __ne__(self, other: object) -> bool:
         if not isinstance(other, SizeSuffix):
             return True
         return self._size != other._size
-    
+
     def __lt__(self, other: object) -> bool:
         if not isinstance(other, SizeSuffix):
             return False
         return self._size < other._size
-    
+
     def __le__(self, other: object) -> bool:
         if not isinstance(other, SizeSuffix):
             return False
         return self._size <= other._size
-    
+
     def __gt__(self, other: object) -> bool:
         if not isinstance(other, SizeSuffix):
             return False
         return self._size > other._size
-    
+
     def __ge__(self, other: object) -> bool:
         if not isinstance(other, SizeSuffix):
             return False
         return self._size >= other._size
-    
+
     def __hash__(self) -> int:
         return hash(self._size)
-    
+
     def __int__(self) -> int:
         return self._size
-    

{rclone_api-1.1.83 → rclone_api-1.1.85}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.83
+Version: 1.1.85
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License