rclone-api 1.1.5__py2.py3-none-any.whl → 1.1.7__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -10,7 +10,7 @@ from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
 from .s3.types import MultiUploadResult
-from .types import ListingOption, Order, SizeResult
+from .types import ListingOption, Order, SizeResult, SizeSuffix
 
 __all__ = [
     "Rclone",
@@ -34,4 +34,5 @@ __all__ = [
     "Parsed",
     "Section",
     "MultiUploadResult",
+    "SizeSuffix",
 ]

rclone_api/cmd/copy_large_s3.py

@@ -2,7 +2,7 @@ import argparse
 from dataclasses import dataclass
 from pathlib import Path
 
-from rclone_api import MultiUploadResult, Rclone
+from rclone_api import MultiUploadResult, Rclone, SizeSuffix
 
 _1MB = 1024 * 1024
 
@@ -12,7 +12,7 @@ class Args:
     config_path: Path
     src: str
     dst: str
-    chunk_size_mb: int
+    chunk_size_mb: SizeSuffix
     read_concurrent_chunks: int
     retries: int
     save_state_json: Path
@@ -35,17 +35,20 @@ def _parse_args() -> Args:
         "--config", help="Path to rclone config file", type=Path, required=True
     )
     parser.add_argument(
-        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
+        type=str,
+        default="128M",
     )
     parser.add_argument(
         "--read-concurrent-chunks",
-        help="Maximum number of chunks to read",
+        help="Maximum number of chunks to read in a look ahead cache",
         type=int,
-        default=4,
+        default=1,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
-        "--resumable-json",
+        "--resume-json",
         help="Path to resumable JSON file",
         type=Path,
         default="resume.json",
@@ -56,10 +59,10 @@ def _parse_args() -> Args:
         config_path=Path(args.config),
         src=args.src,
         dst=args.dst,
-        chunk_size_mb=args.chunk_size_mb,
+        chunk_size_mb=SizeSuffix(args.chunk_size),
         read_concurrent_chunks=args.read_concurrent_chunks,
         retries=args.retries,
-        save_state_json=args.resumable_json,
+        save_state_json=args.resume_json,
         verbose=args.verbose,
     )
     return out

rclone_api/mount.py

@@ -23,6 +23,19 @@ def run_command(cmd: str, verbose: bool) -> int:
         return -1
 
 
+def prepare_mount(outdir: Path, verbose: bool) -> None:
+    if _SYSTEM == "Windows":
+        # Windows -> Must create parent directories only if they don't exist
+        if verbose:
+            print(f"Creating parent directories for {outdir}")
+        outdir.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # Linux -> Must create parent directories and the directory itself
+        if verbose:
+            print(f"Creating directories for {outdir}")
+        outdir.mkdir(parents=True, exist_ok=True)
+
+
 def clean_mount(mount_path: Path, verbose: bool = False) -> None:
     """
     Clean up a mount path across Linux, macOS, and Windows.

rclone_api/rclone.py

@@ -34,7 +34,13 @@ from rclone_api.s3.types import (
     S3Provider,
     S3UploadTarget,
 )
-from rclone_api.types import ListingOption, ModTimeStrategy, Order, SizeResult
+from rclone_api.types import (
+    ListingOption,
+    ModTimeStrategy,
+    Order,
+    SizeResult,
+    SizeSuffix,
+)
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -673,10 +679,13 @@ class Rclone:
         src: str,
         dst: str,
         save_state_json: Path,
-        chunk_size: int = 16
-        * 1024
-        * 1024,  # This setting will scale the performance of the upload
-        concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
+        chunk_size: SizeSuffix | None = None,
+        # 16
+        # * 1024
+        # * 1024,  # This setting will scale the performance of the upload
+        concurrent_chunks: (
+            int | None
+        ) = None,  # This setting will scale the performance of the upload
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -687,16 +696,22 @@ class Rclone:
         from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, random_str, split_s3_path
 
+        _tmp: SizeSuffix | str = chunk_size or "16MiB"
+        chunk_size = SizeSuffix(_tmp)
+        assert chunk_size is not None
+        concurrent_chunks = concurrent_chunks or 4
+        size_limit = SizeSuffix(chunk_size * concurrent_chunks)
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
             "1s",
             "--vfs-disk-space-total-size",
-            str(2 * chunk_size * concurrent_chunks),  # purge quickly.
+            size_limit.as_str(),  # purge quickly.
             "--vfs-read-chunk-size",
-            str(chunk_size),
+            chunk_size.as_str(),
             "--vfs-read-chunk-size-limit",
-            str(chunk_size * concurrent_chunks),
+            size_limit.as_str(),
             "--vfs-read-chunk-streams",
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
@@ -765,7 +780,7 @@ class Rclone:
 
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -788,7 +803,7 @@ class Rclone:
             )
 
             upload_config = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -845,24 +860,14 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        from rclone_api.mount import clean_mount, prepare_mount
+
         allow_writes = allow_writes or False
         use_links = use_links or True
         verbose = get_verbose(verbose)
         vfs_cache_mode = vfs_cache_mode or "full"
-        if outdir.exists():
-            is_empty = not list(outdir.iterdir())
-            if not is_empty:
-                raise ValueError(
-                    f"Mount directory already exists and is not empty: {outdir}"
-                )
-            outdir.rmdir()
-
-        if _IS_WINDOWS:
-            # Windows -> Must create parent directories only if they don't exist
-            outdir.parent.mkdir(parents=True, exist_ok=True)
-        else:
-            # Linux -> Must create parent directories and the directory itself
-            outdir.mkdir(parents=True, exist_ok=True)
+        clean_mount(outdir, verbose=verbose)
+        prepare_mount(outdir, verbose=verbose)
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:

rclone_api/types.py

@@ -1,3 +1,4 @@
+import re
 from dataclasses import dataclass
 from enum import Enum
 
@@ -33,3 +34,92 @@ class SizeResult:
     prefix: str
     total_size: int
     file_sizes: dict[str, int]
+
+
+def _to_size_suffix(size: int) -> str:
+    if size < 1024:
+        return f"{size}B"
+    if size < 1024 * 1024:
+        return f"{size // 1024}K"
+    if size < 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024)}M"
+    if size < 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024)}G"
+    if size < 1024 * 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024 * 1024)}T"
+    if size < 1024 * 1024 * 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024 * 1024 * 1024)}P"
+    raise ValueError(f"Invalid size: {size}")
+
+
+_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+)([A-Za-z]+)$")
+
+
+def _from_size_suffix(size: str) -> int:
+    # 16MiB
+    # parse out number and suffix
+    match = _PATTERN_SIZE_SUFFIX.match(size)
+    if match is None:
+        raise ValueError(f"Invalid size suffix: {size}")
+    size = match.group(1)
+    suffix = match.group(2)[0:1].upper()
+    n = int(size)
+    if suffix == "B":
+        return n
+    if suffix == "K":
+        return n * 1024
+    if suffix == "M":
+        return n * 1024 * 1024
+    if suffix == "G":
+        return n * 1024 * 1024 * 1024
+    if suffix == "T":
+        return n * 1024 * 1024 * 1024 * 1024
+    if suffix == "P":
+        return n * 1024 * 1024 * 1024 * 1024 * 1024
+    raise ValueError(f"Invalid size suffix: {size}")
+
+
+class SizeSuffix:
+    def __init__(self, size: "int | str | SizeSuffix"):
+        self._size: int
+        if isinstance(size, SizeSuffix):
+            self._size = size._size
+        elif isinstance(size, int):
+            self._size = size
+        elif isinstance(size, str):
+            self._size = _from_size_suffix(size)
+        else:
+            raise ValueError(f"Invalid type for size: {type(size)}")
+
+    def as_int(self) -> int:
+        return self._size
+
+    def as_str(self) -> str:
+        return _to_size_suffix(self._size)
+
+    def __repr__(self) -> str:
+        return self.as_str()
+
+    def __str__(self) -> str:
+        return self.as_str()
+
+    @staticmethod
+    def _to_size(size: "int | SizeSuffix") -> int:
+        if isinstance(size, int):
+            return size
+        elif isinstance(size, SizeSuffix):
+            return size._size
+        else:
+            raise ValueError(f"Invalid type for size: {type(size)}")
+
+    def __mul__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size * other_int._size)
+
+    def __add__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size + other_int._size)
+
+    def __sub__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size - other_int._size)

{rclone_api-1.1.5.dist-info → rclone_api-1.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.5
+Version: 1.1.7
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.5.dist-info → rclone_api-1.1.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=m7hQkaIsn9grAlziu9nvDlOIQY6VflYFwC1c_g0qCJo,833
+rclone_api/__init__.py,sha256=6TVA3JUVxr76wzy29XRTX_xTdQ5JBGwuheNvQc8EgsU,863
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -11,17 +11,17 @@ rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/mount.py,sha256=OJGiZLhtvBJ8CTHSYUoKnYblCff3cD8_q4JTgTIVJHQ,3839
+rclone_api/mount.py,sha256=Zf41gGfTXtSB-KkQoXCq_dpqBvezgz_dPBzk_aDk5Dc,4354
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=TQuZiUGjVLr8cYVEZQ1m9fyujbASvNs5XFB9EBJOpuc,42115
+rclone_api/rclone.py,sha256=yI_8QdDXppcLO-KbRVGteu-onTa4fTd4ojy_N1HYxx4,42029
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=NC3e78aXCx-sEQ-FqEaC9KzaJDdJhJrKa4Nwum_-Db0,563
+rclone_api/types.py,sha256=e7VmqA2U6MkTT9VDHeP2qwRw_lUUFbaiN6RO-R8B1oo,3336
 rclone_api/util.py,sha256=ujinqW4xUkZAHBCL1VMhGu88LMdUFIu1ApF8rZEH8rQ,5324
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/copy_large_s3.py,sha256=33KFvCrh5uk-rdRtkREdEs2WNwxGgTdCAWDLCE4dm0A,2855
+rclone_api/cmd/copy_large_s3.py,sha256=kF0wRVRGjIB2CxuwN4z1WnfYjiux7fHkY9FLCw3EJsI,2999
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=0-mtXg9J4MoMm2uBKbsMLj4pSGRLQUAqNRDJWGttnAQ,2443
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
@@ -32,9 +32,9 @@ rclone_api/s3/chunk_types.py,sha256=Fq0IlhZ0IftuFQFkbICmmrOonII0BNzuY4CIKNC4wB0,
 rclone_api/s3/chunk_uploader.py,sha256=KO8834Gery9HKWSqjQTNW0pbBbVoGrza9gj-1OaNLQQ,9130
 rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
 rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
-rclone_api-1.1.5.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.5.dist-info/METADATA,sha256=5gWOijGTvfWbWh8q_PkuzrRXqqA-hFK-hTpq-rYoDNE,4478
-rclone_api-1.1.5.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.5.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.1.5.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.5.dist-info/RECORD,,
+rclone_api-1.1.7.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.7.dist-info/METADATA,sha256=HujPmSO7dGtf6yoYvjQ3ZkVHfezRF9sTXx1ZXLcaCO8,4478
+rclone_api-1.1.7.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.7.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.1.7.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.7.dist-info/RECORD,,