rclone-api 1.0.98__py2.py3-none-any.whl → 1.0.100__py2.py3-none-any.whl

rclone_api/experimental/flags.py

@@ -1,93 +1,121 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass, fields, is_dataclass
+from typing import Type, TypeVar
 
+T = TypeVar("T")
 
-@dataclass
-class CopyFlags:
-    check_first: bool = False
-    checksum: bool = False
-    ignore_existing: bool = False
-    ignore_times: bool = False
-    immutable: bool = False
-    inplace: bool = False
-    links: bool = False
-    metadata: bool = False
-
-
-@dataclass
-class SyncFlags:
-    backup_dir: str | None = None
-    delete_after: bool = False
-    delete_before: bool = False
-    delete_during: bool = False
-    ignore_errors: bool = False
-    track_renames: bool = False
-
-
-@dataclass
-class ImportantFlags:
-    dry_run: bool = False
-    interactive: bool = False
-    verbose: int = 0  # 0 = default, 1 = -v, 2 = -vv, etc.
-
-
-@dataclass
-class NetworkingFlags:
-    bwlimit: str | None = None
-    timeout: str | None = "5m0s"
-    tpslimit: float | None = None
-    user_agent: str | None = "rclone/v1.69.1"
 
+def _merge(cls: Type[T], dataclass_a: T, dataclass_b: T) -> T:
+    if not is_dataclass(dataclass_a) or not is_dataclass(dataclass_b):
+        raise ValueError("Both inputs must be dataclass instances")
+    if type(dataclass_a) is not type(dataclass_b):
+        raise ValueError("Dataclass instances must be of the same type")
 
-@dataclass
-class PerformanceFlags:
-    buffer_size: str | None = "16MiB"
-    checkers: int = 8
-    transfers: int = 4
+    merged_kwargs = {}
+    for field in fields(dataclass_a):
+        a_value = getattr(dataclass_a, field.name)
+        b_value = getattr(dataclass_b, field.name)
 
+        if is_dataclass(a_value) and is_dataclass(b_value):
+            merged_kwargs[field.name] = _merge(type(a_value), a_value, b_value)
+        else:
+            merged_kwargs[field.name] = b_value if b_value is not None else a_value
 
-@dataclass
-class ConfigFlags:
-    config: str | None = None
-    ask_password: bool = True
-    auto_confirm: bool = False
+    return cls(**merged_kwargs)
 
 
-@dataclass
-class DebuggingFlags:
-    cpuprofile: str | None = None
-    memprofile: str | None = None
+def _field_name_to_flag(field_name: str) -> str:
+    return f"--{field_name.replace('_', '-')}"
 
 
 @dataclass
-class FilterFlags:
-    exclude: list[str] = field(default_factory=list)
-    include: list[str] = field(default_factory=list)
-    max_age: str | None = None
-    min_size: str | None = None
+class BaseFlags:
+    def to_args(self) -> list[str]:
+        args = []
+        for field in fields(self):
+            value = getattr(self, field.name)
+            if value is None:
+                continue
+            # If the field value is a nested dataclass that supports to_args, use it.
+            if is_dataclass(value) and hasattr(value, "to_args"):
+                to_args = getattr(value, "to_args")
+                args.extend(to_args())
+            elif isinstance(value, bool):
+                # Only include the flag if the boolean is True.
+                if value:
+                    args.append(_field_name_to_flag(field.name))
+            else:
+                args.append(_field_name_to_flag(field.name))
+                if isinstance(value, list):
+                    # Join list values with a comma.
+                    args.append(",".join(map(str, value)))
+                else:
+                    args.append(str(value))
+        return args
+
+    def merge(self, other: "BaseFlags") -> "BaseFlags":
+        # Use the type of self, so merging CopyFlags returns a CopyFlags instance.
+        return _merge(type(self), self, other)
+
+    def __repr__(self):
+        return str(self.to_args())
+
+
+@dataclass(repr=False)
+class CopyFlags(BaseFlags):
+    check_first: bool | None = None
+    checksum: bool | None = False
+    compare_dest: list[str] | None = None
+    copy_dest: list[str] | None = None
+    cutoff_mode: str | None = None
+    ignore_case_sync: bool | None = None
+    ignore_checksum: bool | None = None
+    ignore_existing: bool | None = None
+    ignore_size: bool | None = None
+    ignore_times: bool | None = None
+    immutable: bool | None = None
+    inplace: bool | None = None
+    links: bool | None = None
+    max_backlog: int | None = None
+    max_duration: str | None = None
+    max_transfer: str | None = None
+    metadata: bool | None = None
+    modify_window: str | None = None
+    multi_thread_chunk_size: str | None = None
+    multi_thread_cutoff: str | None = None
+    multi_thread_streams: int | None = None
+    multi_thread_write_buffer_size: str | None = None
+    no_check_dest: bool | None = None
+    no_traverse: bool | None = None
+    no_update_dir_modtime: bool | None = None
+    no_update_modtime: bool | None = None
+    order_by: str | None = None
+    partial_suffix: str | None = None
+    refresh_times: bool | None = None
+    server_side_across_configs: bool | None = None
+    size_only: bool | None = None
+    streaming_upload_cutoff: str | None = None
+    update: bool | None = None
+
+
+@dataclass(repr=False)
+class Flags(BaseFlags):
+    copy: CopyFlags | None = None
 
 
-@dataclass
-class ListingFlags:
-    fast_list: bool = False
+def unit_test() -> None:
+    copy_flags_a = CopyFlags(compare_dest=["a", "b"])
+    copy_flags_b = CopyFlags(checksum=False)
+    flags_a = copy_flags_a.merge(copy_flags_b)
+    print("A:", flags_a)
 
+    copy_flags_c = CopyFlags(checksum=True)
+    copy_flags_d = CopyFlags(checksum=False)
 
-@dataclass
-class LoggingFlags:
-    log_file: str | None = None
-    log_level: str = "NOTICE"  # Options: DEBUG, INFO, NOTICE, ERROR
-    stats: str | None = "1m0s"
-    progress: bool = False
+    merged_c_d = copy_flags_c.merge(copy_flags_d)
+    print("B:", merged_c_d)
+    merged_d_c = copy_flags_d.merge(copy_flags_c)
+    print("C:", merged_d_c)
 
 
-@dataclass
-class Flags:
-    copy: CopyFlags | None = None
-    sync: SyncFlags | None = None
-    important: ImportantFlags | None = None
-    networking: NetworkingFlags | None = None
-    performance: PerformanceFlags | None = None
-    config: ConfigFlags | None = None
-    debugging: DebuggingFlags | None = None
-    filter: FilterFlags | None = None
-    listing: ListingFlags | None = None
-    logging: LoggingFlags | None = None
+if __name__ == "__main__":
+    unit_test()

rclone_api/rclone.py

@@ -919,6 +919,7 @@ class Rclone:
                 if outdir.exists():
                     print(f"{outdir} mount still exists, attempting to remove")
                     if not _IS_WINDOWS:
+
                         def exec(cmd: str) -> int:
                             if verbose:
                                 print(f"Executing: {cmd}")

rclone_api/s3/chunk_uploader.py

@@ -1,3 +1,4 @@
+import _thread
 import json
 import os
 import time
@@ -260,6 +261,20 @@ def _get_chunk_tmpdir() -> Path:
         return out
 
 
+def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+    sleep_time = timeout / 60 if timeout > 0 else 1
+    start = time.time()
+    while True:
+        try:
+            if file_path.exists():
+                return file_path.stat().st_size
+        except FileNotFoundError:
+            pass
+        if time.time() - start > timeout:
+            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+        time.sleep(sleep_time)
+
+
 def file_chunker(
     upload_state: UploadState, max_chunks: int | None, output: Queue[FileChunk | None]
 ) -> None:
@@ -279,7 +294,8 @@ def file_chunker(
     file_path = upload_info.src_file_path
     chunk_size = upload_info.chunk_size
     src = Path(file_path)
-    file_size = os.path.getsize(file_path)
+    # Mounted files may take a while to appear, so keep retrying.
+    file_size = _get_file_size(src, timeout=60)
     part_number = 1
     done_part_numbers: set[int] = {
         p.part_number for p in upload_state.parts if p is not None
@@ -418,6 +434,7 @@ def upload_file_multipart(
     chunk_size: int = 16 * 1024 * 1024,  # Default chunk size is 16MB; can be overridden
     retries: int = 20,
     max_chunks_before_suspension: int | None = None,
+    abort_transfer_on_failure: bool = False,
 ) -> MultiUploadResult:
     """Upload a file to the bucket using multipart upload with customizable chunk size."""
     file_size = os.path.getsize(str(file_path))
@@ -479,7 +496,13 @@ def upload_file_multipart(
         output=filechunks,
         max_chunks=max_chunks_before_suspension,
     ) -> None:
-        file_chunker(upload_state=upload_state, output=output, max_chunks=max_chunks)
+        try:
+            file_chunker(
+                upload_state=upload_state, output=output, max_chunks=max_chunks
+            )
+        except Exception:
+            _thread.interrupt_main()
+            raise
 
     try:
         thread_chunker = Thread(target=chunker_task, daemon=True)
@@ -492,7 +515,11 @@ def upload_file_multipart(
                     break
 
                 def task(upload_info=upload_info, file_chunk=file_chunk):
-                    return handle_upload(upload_info, file_chunk)
+                    try:
+                        return handle_upload(upload_info, file_chunk)
+                    except Exception:
+                        _thread.interrupt_main()
+                        raise
 
                 fut = executor.submit(task)
 
@@ -525,7 +552,7 @@ def upload_file_multipart(
             f"Multipart upload completed: {file_path} to {bucket_name}/{object_name}"
         )
     except Exception:
-        if upload_info.upload_id:
+        if upload_info.upload_id and abort_transfer_on_failure:
             try:
                 s3_client.abort_multipart_upload(
                     Bucket=bucket_name, Key=object_name, UploadId=upload_info.upload_id

{rclone_api-1.0.98.dist-info → rclone_api-1.0.100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.98
+Version: 1.0.100
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.98.dist-info → rclone_api-1.0.100.dist-info}/RECORD

@@ -12,7 +12,7 @@ rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=eSQNNKS8tO0WHvbMKT_Ov1IIhUxsUy8L0KTOoPpVW-Y,43539
+rclone_api/rclone.py,sha256=_CejQQ8jF-VYc_rO2u5i9qr8jmyksxyuoO4ZdHLW6tg,43540
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -22,15 +22,15 @@ rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/copy_large_s3.py,sha256=33KFvCrh5uk-rdRtkREdEs2WNwxGgTdCAWDLCE4dm0A,2855
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api/experimental/flags.py,sha256=7sJyRU1vh5DgSdpFiv0gom2Aaw1-OP2v4PL5OXEruUo,2083
+rclone_api/experimental/flags.py,sha256=AHbTaFHuyYFm3pjdvbQ100jztOXOdNuxalMr8UjXnV4,4097
 rclone_api/s3/api.py,sha256=VstlaEnBjO2JDQuCRLdTfUGvQLbfshlXXhAzimFv4Vc,3763
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_uploader.py,sha256=Wcm4h_A6ORaJvT9P9qSeknbJYch22RKZ4FGo1iePlAw,17385
+rclone_api/s3/chunk_uploader.py,sha256=-GzafZ93Mm9go7BqOaq8svPsjAwtFbVuzVpFdtw8cVA,18283
 rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
 rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
-rclone_api-1.0.98.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.98.dist-info/METADATA,sha256=N3rINQp548v5kGCXCnQnS36fdiQME8ifJk4gAv2GaGU,4479
-rclone_api-1.0.98.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.0.98.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.0.98.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.98.dist-info/RECORD,,
+rclone_api-1.0.100.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.100.dist-info/METADATA,sha256=ZuM5x0cJJyMDGIi2HApzr4yqo9JdoMLCSOLyUiWkD9w,4480
+rclone_api-1.0.100.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.0.100.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.0.100.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.100.dist-info/RECORD,,