rclone-api 1.1.25__tar.gz → 1.1.28__tar.gz

{rclone_api-1.1.25 → rclone_api-1.1.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.25
+Version: 1.1.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.25 → rclone_api-1.1.28}/pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.25"
+version = "1.1.28"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.25 → rclone_api-1.1.28}/src/rclone_api/cmd/copy_large_s3.py

@@ -13,7 +13,7 @@ class Args:
     src: str
     dst: str
     chunk_size: SizeSuffix
-    threads: int
+    read_threads: int
     write_threads: int
     retries: int
     save_state_json: Path
@@ -37,19 +37,19 @@ def _parse_args() -> Args:
     )
     parser.add_argument(
         "--chunk-size",
-        help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
+        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
         type=str,
-        default="64MB",
+        default="64MB",  # if this is too low or too high an s3 service
     )
     parser.add_argument(
-        "--threads",
-        help="Number of threads to use per chunk",
+        "--read-threads",
+        help="Number of concurrent read threads per chunk, only one chunk will be read at a time",
         type=int,
-        default=8,
+        default=32,
     )
     parser.add_argument(
         "--write-threads",
-        help="Max number of chunks to upload in parallel to the destination",
+        help="Max number of chunks to upload in parallel to the destination, each chunk is uploaded in a separate thread",
         type=int,
         default=64,
     )
@@ -67,7 +67,7 @@ def _parse_args() -> Args:
         src=args.src,
         dst=args.dst,
         chunk_size=SizeSuffix(args.chunk_size),
-        threads=args.threads,
+        read_threads=args.read_threads,
         write_threads=args.write_threads,
         retries=args.retries,
         save_state_json=args.resume_json,
@@ -85,11 +85,8 @@ def main() -> int:
         src=args.src,
         dst=args.dst,
         chunk_size=args.chunk_size,
-        read_threads=args.threads,
+        read_threads=args.read_threads,
         write_threads=args.write_threads,
-        # vfs_read_chunk_size=unit_chunk,
-        # vfs_read_chunk_size_limit=args.chunk_size,
-        # vfs_read_chunk_streams=args.threads,
         retries=args.retries,
         save_state_json=args.save_state_json,
         verbose=args.verbose,

{rclone_api-1.1.25 → rclone_api-1.1.28}/src/rclone_api/s3/api.py

@@ -9,9 +9,9 @@ from rclone_api.s3.basic_ops import (
     list_bucket_contents,
     upload_file,
 )
-from rclone_api.s3.chunk_uploader import MultiUploadResult, upload_file_multipart
 from rclone_api.s3.create import create_s3_client
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
+from rclone_api.s3.upload_file_multipart import MultiUploadResult, upload_file_multipart
 
 _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 

{rclone_api-1.1.25 → rclone_api-1.1.28}/src/rclone_api/types.py

@@ -37,41 +37,68 @@ class SizeResult:
 
 
 def _to_size_suffix(size: int) -> str:
-    if size < 1024:
-        return f"{size}B"
-    elif size < 1024**2:
-        val = size / 1024
-        unit = "K"
-    elif size < 1024**3:
-        val = size / (1024**2)
-        unit = "M"
-    elif size < 1024**4:
-        val = size / (1024**3)
-        unit = "G"
-    elif size < 1024**5:
-        val = size / (1024**4)
-        unit = "T"
-    elif size < 1024**6:
-        val = size / (1024**5)
-        unit = "P"
-    else:
-        raise ValueError(f"Invalid size: {size}")
-
-    # If the float is an integer, drop the decimal, otherwise format with one decimal.
-    return f"{int(val) if val.is_integer() else f'{val:.1f}'}{unit}"
+    def _convert(size: int) -> tuple[float, str]:
+        val: float
+        unit: str
+        if size < 1024:
+            val = size
+            unit = "B"
+        elif size < 1024**2:
+            val = size / 1024
+            unit = "K"
+        elif size < 1024**3:
+            val = size / (1024**2)
+            unit = "M"
+        elif size < 1024**4:
+            val = size / (1024**3)
+            unit = "G"
+        elif size < 1024**5:
+            val = size / (1024**4)
+            unit = "T"
+        elif size < 1024**6:
+            val = size / (1024**5)
+            unit = "P"
+        else:
+            raise ValueError(f"Invalid size: {size}")
+
+        return val, unit
+
+    def _fmt(_val: float | int, _unit: str) -> str:
+        # If the float is an integer, drop the decimal, otherwise format with one decimal.
+        val_str: str = str(_val)
+        if not val_str.endswith(".0"):
+            first_str: str = f"{_val:.1f}"
+        else:
+            first_str = str(int(_val))
+        return first_str + _unit
+
+    val, unit = _convert(size)
+    out = _fmt(val, unit)
+    # Now round trip the value to fix floating point issues via rounding.
+    int_val = _from_size_suffix(out)
+    val, unit = _convert(int_val)
+    out = _fmt(val, unit)
+    return out
 
 
 # Update regex to allow decimals (e.g., 16.5MB)
 _PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+(?:\.\d+)?)([A-Za-z]+)$")
 
 
+def _parse_elements(value: str) -> tuple[str, str] | None:
+    match = _PATTERN_SIZE_SUFFIX.match(value)
+    if match is None:
+        return None
+    return match.group(1), match.group(2)
+
+
 def _from_size_suffix(size: str) -> int:
     if size == "0":
         return 0
-    match = _PATTERN_SIZE_SUFFIX.match(size)
-    if match is None:
+    pair = _parse_elements(size)
+    if pair is None:
         raise ValueError(f"Invalid size suffix: {size}")
-    num_str, suffix = match.group(1), match.group(2)
+    num_str, suffix = pair
     n = float(num_str)
     # Determine the unit from the first letter (e.g., "M" from "MB")
     unit = suffix[0].upper()

{rclone_api-1.1.25 → rclone_api-1.1.28}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.25
+Version: 1.1.28
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.25 → rclone_api-1.1.28}/src/rclone_api.egg-info/SOURCES.txt

@@ -57,9 +57,9 @@ src/rclone_api/s3/api.py
 src/rclone_api/s3/basic_ops.py
 src/rclone_api/s3/chunk_file.py
 src/rclone_api/s3/chunk_types.py
-src/rclone_api/s3/chunk_uploader.py
 src/rclone_api/s3/create.py
 src/rclone_api/s3/types.py
+src/rclone_api/s3/upload_file_multipart.py
 tests/test_cmd_list_files.py
 tests/test_copy.py
 tests/test_copy_file_resumable_s3.py

{rclone_api-1.1.25 → rclone_api-1.1.28}/tests/test_s3.py

@@ -6,9 +6,9 @@ from pathlib import Path
 from dotenv import load_dotenv
 
 from rclone_api.s3.api import S3Client
-from rclone_api.s3.chunk_uploader import MultiUploadResult
 from rclone_api.s3.create import S3Provider
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
+from rclone_api.s3.upload_file_multipart import MultiUploadResult
 
 load_dotenv()
 

{rclone_api-1.1.25 → rclone_api-1.1.28}/tests/test_size_suffix.py

@@ -24,6 +24,15 @@ class RcloneSuffixSize(unittest.TestCase):
         out_str = str(size_suffix)
         self.assertEqual(out_str, "16.5M")
 
+    def test_float_suffix_border(self) -> None:
+        size_suffix = SizeSuffix("1M")
+        size_int = size_suffix.as_int()
+        size_int -= 1
+        # now assert that the string value is the same as the input
+        tmp = SizeSuffix(size_int)
+        out_str = tmp.as_str()
+        self.assertEqual(out_str, "1M")
+
 
 if __name__ == "__main__":
     unittest.main()