rclone-api 1.1.5__tar.gz → 1.1.6__tar.gz

{rclone_api-1.1.5 → rclone_api-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.5
+Version: 1.1.6
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.5 → rclone_api-1.1.6}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.5"
+version = "1.1.6"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.5 → rclone_api-1.1.6}/src/rclone_api/__init__.py

@@ -10,7 +10,7 @@ from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
 from .s3.types import MultiUploadResult
-from .types import ListingOption, Order, SizeResult
+from .types import ListingOption, Order, SizeResult, SizeSuffix
 
 __all__ = [
     "Rclone",
@@ -34,4 +34,5 @@ __all__ = [
     "Parsed",
     "Section",
     "MultiUploadResult",
+    "SizeSuffix",
 ]

{rclone_api-1.1.5 → rclone_api-1.1.6}/src/rclone_api/cmd/copy_large_s3.py

@@ -2,7 +2,7 @@ import argparse
 from dataclasses import dataclass
 from pathlib import Path
 
-from rclone_api import MultiUploadResult, Rclone
+from rclone_api import MultiUploadResult, Rclone, SizeSuffix
 
 _1MB = 1024 * 1024
 
@@ -12,7 +12,7 @@ class Args:
     config_path: Path
     src: str
     dst: str
-    chunk_size_mb: int
+    chunk_size_mb: SizeSuffix
     read_concurrent_chunks: int
     retries: int
     save_state_json: Path
@@ -35,17 +35,20 @@ def _parse_args() -> Args:
         "--config", help="Path to rclone config file", type=Path, required=True
     )
     parser.add_argument(
-        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
+        type=str,
+        default="128M",
     )
     parser.add_argument(
         "--read-concurrent-chunks",
-        help="Maximum number of chunks to read",
+        help="Maximum number of chunks to read in a look ahead cache",
         type=int,
-        default=4,
+        default=1,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
-        "--resumable-json",
+        "--resume-json",
         help="Path to resumable JSON file",
         type=Path,
         default="resume.json",
@@ -56,7 +59,7 @@ def _parse_args() -> Args:
         config_path=Path(args.config),
         src=args.src,
         dst=args.dst,
-        chunk_size_mb=args.chunk_size_mb,
+        chunk_size_mb=SizeSuffix(args.chunk_size),
         read_concurrent_chunks=args.read_concurrent_chunks,
         retries=args.retries,
         save_state_json=args.resumable_json,

{rclone_api-1.1.5 → rclone_api-1.1.6}/src/rclone_api/mount.py

@@ -23,6 +23,19 @@ def run_command(cmd: str, verbose: bool) -> int:
         return -1
 
 
+def prepare_mount(outdir: Path, verbose: bool) -> None:
+    if _SYSTEM == "Windows":
+        # Windows -> Must create parent directories only if they don't exist
+        if verbose:
+            print(f"Creating parent directories for {outdir}")
+        outdir.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # Linux -> Must create parent directories and the directory itself
+        if verbose:
+            print(f"Creating directories for {outdir}")
+        outdir.mkdir(parents=True, exist_ok=True)
+
+
 def clean_mount(mount_path: Path, verbose: bool = False) -> None:
     """
     Clean up a mount path across Linux, macOS, and Windows.

{rclone_api-1.1.5 → rclone_api-1.1.6}/src/rclone_api/rclone.py

@@ -34,7 +34,13 @@ from rclone_api.s3.types import (
     S3Provider,
     S3UploadTarget,
 )
-from rclone_api.types import ListingOption, ModTimeStrategy, Order, SizeResult
+from rclone_api.types import (
+    ListingOption,
+    ModTimeStrategy,
+    Order,
+    SizeResult,
+    SizeSuffix,
+)
 from rclone_api.util import (
     get_check,
     get_rclone_exe,
@@ -673,10 +679,13 @@ class Rclone:
         src: str,
         dst: str,
         save_state_json: Path,
-        chunk_size: int = 16
-        * 1024
-        * 1024,  # This setting will scale the performance of the upload
-        concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
+        chunk_size: SizeSuffix | None = None,
+        # 16
+        # * 1024
+        # * 1024,  # This setting will scale the performance of the upload
+        concurrent_chunks: (
+            int | None
+        ) = None,  # This setting will scale the performance of the upload
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -687,16 +696,22 @@ class Rclone:
         from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, random_str, split_s3_path
 
+        _tmp: SizeSuffix | str = chunk_size or "16MiB"
+        chunk_size = SizeSuffix(_tmp)
+        assert chunk_size is not None
+        concurrent_chunks = concurrent_chunks or 4
+        size_limit = SizeSuffix(chunk_size * concurrent_chunks)
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
             "1s",
             "--vfs-disk-space-total-size",
-            str(2 * chunk_size * concurrent_chunks),  # purge quickly.
+            size_limit.as_str(),  # purge quickly.
             "--vfs-read-chunk-size",
-            str(chunk_size),
+            chunk_size.as_str(),
             "--vfs-read-chunk-size-limit",
-            str(chunk_size * concurrent_chunks),
+            size_limit.as_str(),
             "--vfs-read-chunk-streams",
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
@@ -765,7 +780,7 @@ class Rclone:
 
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -788,7 +803,7 @@ class Rclone:
             )
 
             upload_config = S3MutliPartUploadConfig(
-                chunk_size=chunk_size,
+                chunk_size=chunk_size.as_int(),
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -845,24 +860,14 @@ class Rclone:
         Raises:
             subprocess.CalledProcessError: If the mount operation fails
         """
+        from rclone_api.mount import clean_mount, prepare_mount
+
         allow_writes = allow_writes or False
         use_links = use_links or True
         verbose = get_verbose(verbose)
         vfs_cache_mode = vfs_cache_mode or "full"
-        if outdir.exists():
-            is_empty = not list(outdir.iterdir())
-            if not is_empty:
-                raise ValueError(
-                    f"Mount directory already exists and is not empty: {outdir}"
-                )
-            outdir.rmdir()
-
-        if _IS_WINDOWS:
-            # Windows -> Must create parent directories only if they don't exist
-            outdir.parent.mkdir(parents=True, exist_ok=True)
-        else:
-            # Linux -> Must create parent directories and the directory itself
-            outdir.mkdir(parents=True, exist_ok=True)
+        clean_mount(outdir, verbose=verbose)
+        prepare_mount(outdir, verbose=verbose)
         src_str = convert_to_str(src)
         cmd_list: list[str] = ["mount", src_str, str(outdir)]
         if not allow_writes:

rclone_api-1.1.6/src/rclone_api/types.py

@@ -0,0 +1,125 @@
+import re
+from dataclasses import dataclass
+from enum import Enum
+
+
+class ModTimeStrategy(Enum):
+    USE_SERVER_MODTIME = "use-server-modtime"
+    NO_MODTIME = "no-modtime"
+
+
+class ListingOption(Enum):
+    DIRS_ONLY = "dirs-only"
+    FILES_ONLY = "files-only"
+    ALL = "all"
+
+
+class Order(Enum):
+    NORMAL = "normal"
+    REVERSE = "reverse"
+    RANDOM = "random"
+
+
+@dataclass
+class S3PathInfo:
+    remote: str
+    bucket: str
+    key: str
+
+
+@dataclass
+class SizeResult:
+    """Size result dataclass."""
+
+    prefix: str
+    total_size: int
+    file_sizes: dict[str, int]
+
+
+def _to_size_suffix(size: int) -> str:
+    if size < 1024:
+        return f"{size}B"
+    if size < 1024 * 1024:
+        return f"{size // 1024}K"
+    if size < 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024)}M"
+    if size < 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024)}G"
+    if size < 1024 * 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024 * 1024)}T"
+    if size < 1024 * 1024 * 1024 * 1024 * 1024 * 1024:
+        return f"{size // (1024 * 1024 * 1024 * 1024 * 1024)}P"
+    raise ValueError(f"Invalid size: {size}")
+
+
+_PATTERN_SIZE_SUFFIX = re.compile(r"^(\d+)([A-Za-z]+)$")
+
+
+def _from_size_suffix(size: str) -> int:
+    # 16MiB
+    # parse out number and suffix
+    match = _PATTERN_SIZE_SUFFIX.match(size)
+    if match is None:
+        raise ValueError(f"Invalid size suffix: {size}")
+    size = match.group(1)
+    suffix = match.group(2)[0:1].upper()
+    n = int(size)
+    if suffix == "B":
+        return n
+    if suffix == "K":
+        return n * 1024
+    if suffix == "M":
+        return n * 1024 * 1024
+    if suffix == "G":
+        return n * 1024 * 1024 * 1024
+    if suffix == "T":
+        return n * 1024 * 1024 * 1024 * 1024
+    if suffix == "P":
+        return n * 1024 * 1024 * 1024 * 1024 * 1024
+    raise ValueError(f"Invalid size suffix: {size}")
+
+
+class SizeSuffix:
+    def __init__(self, size: "int | str | SizeSuffix"):
+        self._size: int
+        if isinstance(size, SizeSuffix):
+            self._size = size._size
+        elif isinstance(size, int):
+            self._size = size
+        elif isinstance(size, str):
+            self._size = _from_size_suffix(size)
+        else:
+            raise ValueError(f"Invalid type for size: {type(size)}")
+
+    def as_int(self) -> int:
+        return self._size
+
+    def as_str(self) -> str:
+        return _to_size_suffix(self._size)
+
+    def __repr__(self) -> str:
+        return self.as_str()
+
+    def __str__(self) -> str:
+        return self.as_str()
+
+    @staticmethod
+    def _to_size(size: "int | SizeSuffix") -> int:
+        if isinstance(size, int):
+            return size
+        elif isinstance(size, SizeSuffix):
+            return size._size
+        else:
+            raise ValueError(f"Invalid type for size: {type(size)}")
+
+    def __mul__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size * other_int._size)
+
+    def __add__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size + other_int._size)
+
+    def __sub__(self, other: "int | SizeSuffix") -> "SizeSuffix":
+        other_int = SizeSuffix(other)
+        return SizeSuffix(self._size - other_int._size)

{rclone_api-1.1.5 → rclone_api-1.1.6}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.5
+Version: 1.1.6
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.5 → rclone_api-1.1.6}/src/rclone_api.egg-info/SOURCES.txt

@@ -78,5 +78,6 @@ tests/test_remotes.py
 tests/test_s3.py
 tests/test_scan_missing_folders.py
 tests/test_size_files.py
+tests/test_size_suffix.py
 tests/test_walk.py
 tests/archive/test_paramiko.py.disabled

{rclone_api-1.1.5 → rclone_api-1.1.6}/tests/test_mounted_ranged_download.py

@@ -37,7 +37,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from rclone_api import Rclone
+from rclone_api import Rclone, SizeSuffix
 
 _HERE = Path(__file__).parent
 _PROJECT_ROOT = _HERE.parent
@@ -139,7 +139,7 @@ class RcloneCopyResumableFileToS3(unittest.TestCase):
         rclone.copy_file_resumable_s3(
             src="src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent",
             dst="dst:rclone-api-unit-test/test_data/test.torrent.testwrite",
-            chunk_size=16 * 1024 * 1024,
+            chunk_size=SizeSuffix("16MB"),
             retries=0,
             save_state_json=save_state_json,
             max_chunks_before_suspension=1,

rclone_api-1.1.6/tests/test_size_suffix.py

@@ -0,0 +1,21 @@
+"""
+Unit test file.
+"""
+
+import unittest
+
+from rclone_api import SizeSuffix
+
+
+class RcloneSuffixSize(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def test_list_remotes(self) -> None:
+        size_suffix = SizeSuffix(1024)
+        size_suffix = SizeSuffix("16MB")
+        size_int = size_suffix.as_int()
+        self.assertEqual(size_int, 16 * 1024 * 1024)
+
+
+if __name__ == "__main__":
+    unittest.main()

rclone_api-1.1.5/src/rclone_api/types.py

@@ -1,35 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-
-class ModTimeStrategy(Enum):
-    USE_SERVER_MODTIME = "use-server-modtime"
-    NO_MODTIME = "no-modtime"
-
-
-class ListingOption(Enum):
-    DIRS_ONLY = "dirs-only"
-    FILES_ONLY = "files-only"
-    ALL = "all"
-
-
-class Order(Enum):
-    NORMAL = "normal"
-    REVERSE = "reverse"
-    RANDOM = "random"
-
-
-@dataclass
-class S3PathInfo:
-    remote: str
-    bucket: str
-    key: str
-
-
-@dataclass
-class SizeResult:
-    """Size result dataclass."""
-
-    prefix: str
-    total_size: int
-    file_sizes: dict[str, int]