rclone-api 1.1.0__py2.py3-none-any.whl → 1.1.2__py2.py3-none-any.whl

rclone_api/experimental/flags.py

@@ -1,66 +1,9 @@
-from dataclasses import dataclass, fields, is_dataclass
-from typing import Type, TypeVar
+from dataclasses import dataclass
 
-T = TypeVar("T")
-
-
-def _merge(cls: Type[T], dataclass_a: T, dataclass_b: T) -> T:
-    if not is_dataclass(dataclass_a) or not is_dataclass(dataclass_b):
-        raise ValueError("Both inputs must be dataclass instances")
-    if type(dataclass_a) is not type(dataclass_b):
-        raise ValueError("Dataclass instances must be of the same type")
-
-    merged_kwargs = {}
-    for field in fields(dataclass_a):
-        a_value = getattr(dataclass_a, field.name)
-        b_value = getattr(dataclass_b, field.name)
-
-        if is_dataclass(a_value) and is_dataclass(b_value):
-            merged_kwargs[field.name] = _merge(type(a_value), a_value, b_value)
-        else:
-            merged_kwargs[field.name] = b_value if b_value is not None else a_value
-
-    return cls(**merged_kwargs)
-
-
-def _field_name_to_flag(field_name: str) -> str:
-    return f"--{field_name.replace('_', '-')}"
+from rclone_api.experimental.flags_base import BaseFlags, merge_flags
 
 
 @dataclass
-class BaseFlags:
-    def to_args(self) -> list[str]:
-        args = []
-        for field in fields(self):
-            value = getattr(self, field.name)
-            if value is None:
-                continue
-            # If the field value is a nested dataclass that supports to_args, use it.
-            if is_dataclass(value) and hasattr(value, "to_args"):
-                to_args = getattr(value, "to_args")
-                args.extend(to_args())
-            elif isinstance(value, bool):
-                # Only include the flag if the boolean is True.
-                if value:
-                    args.append(_field_name_to_flag(field.name))
-            else:
-                args.append(_field_name_to_flag(field.name))
-                if isinstance(value, list):
-                    # Join list values with a comma.
-                    args.append(",".join(map(str, value)))
-                else:
-                    args.append(str(value))
-        return args
-
-    def merge(self, other: "BaseFlags") -> "BaseFlags":
-        # Use the type of self, so merging CopyFlags returns a CopyFlags instance.
-        return _merge(type(self), self, other)
-
-    def __repr__(self):
-        return str(self.to_args())
-
-
-@dataclass(repr=False)
 class CopyFlags(BaseFlags):
     check_first: bool | None = None
     checksum: bool | None = False
@@ -96,11 +39,29 @@ class CopyFlags(BaseFlags):
     streaming_upload_cutoff: str | None = None
     update: bool | None = None
 
+    def to_args(self) -> list[str]:
+        return super().to_args()
+
+    def merge(self, other: "CopyFlags") -> "CopyFlags":
+        return merge_flags(CopyFlags, self, other)
+
+    def __repr__(self):
+        return super().__repr__()
+
 
-@dataclass(repr=False)
+@dataclass
 class Flags(BaseFlags):
     copy: CopyFlags | None = None
 
+    def to_args(self) -> list[str]:
+        return super().to_args()
+
+    def merge(self, other: "Flags") -> "Flags":
+        return merge_flags(Flags, self, other)
+
+    def __repr__(self):
+        return super().__repr__()
+
 
 def unit_test() -> None:
     copy_flags_a = CopyFlags(compare_dest=["a", "b"])

rclone_api/experimental/flags_base.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass, fields, is_dataclass
+from typing import Type, TypeVar
+
+T = TypeVar("T")
+
+
+def merge_flags(cls: Type[T], dataclass_a: T, dataclass_b: T) -> T:
+    if not is_dataclass(dataclass_a) or not is_dataclass(dataclass_b):
+        raise ValueError("Both inputs must be dataclass instances")
+    if type(dataclass_a) is not type(dataclass_b):
+        raise ValueError("Dataclass instances must be of the same type")
+
+    merged_kwargs = {}
+    for field in fields(dataclass_a):
+        a_value = getattr(dataclass_a, field.name)
+        b_value = getattr(dataclass_b, field.name)
+
+        if is_dataclass(a_value) and is_dataclass(b_value):
+            merged_kwargs[field.name] = merge_flags(type(a_value), a_value, b_value)
+        else:
+            merged_kwargs[field.name] = b_value if b_value is not None else a_value
+
+    return cls(**merged_kwargs)
+
+
+def _field_name_to_flag(field_name: str) -> str:
+    return f"--{field_name.replace('_', '-')}"
+
+
+@dataclass
+class BaseFlags:
+    """provides to_args(), merge() and __repr__ methods for flags dataclasses"""
+
+    def to_args(self) -> list[str]:
+        args = []
+        for field in fields(self):
+            value = getattr(self, field.name)
+            if value is None:
+                continue
+            # If the field value is a nested dataclass that supports to_args, use it.
+            if is_dataclass(value) and hasattr(value, "to_args"):
+                to_args = getattr(value, "to_args")
+                args.extend(to_args())
+            elif isinstance(value, bool):
+                # Only include the flag if the boolean is True.
+                if value:
+                    args.append(_field_name_to_flag(field.name))
+            else:
+                args.append(_field_name_to_flag(field.name))
+                if isinstance(value, list):
+                    # Join list values with a comma.
+                    args.append(",".join(map(str, value)))
+                else:
+                    args.append(str(value))
+        return args
+
+    def __repr__(self):
+        return str(self.to_args())

rclone_api/s3/chunk_uploader.py

@@ -264,13 +264,20 @@ def _get_chunk_tmpdir() -> Path:
 def _get_file_size(file_path: Path, timeout: int = 60) -> int:
     sleep_time = timeout / 60 if timeout > 0 else 1
     start = time.time()
+    not_windows = os.name != "nt"
     while True:
+        expired = time.time() - start > timeout
         try:
+            if not_windows:
+                # Force a refresh of the directory cache
+                os.system(f"ls -l {file_path.parent}")
             if file_path.exists():
                 return file_path.stat().st_size
-        except FileNotFoundError:
-            pass
-        if time.time() - start > timeout:
+        except FileNotFoundError as e:
+            if expired:
+                print(f"File not found: {file_path}, exception is {e}")
+                raise
+        if expired:
             raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
         time.sleep(sleep_time)
 
@@ -492,16 +499,20 @@ def upload_file_multipart(
     upload_info = upload_state.upload_info
     max_workers = 8
 
+    chunker_errors: Queue[Exception] = Queue()
+
     def chunker_task(
         upload_state=upload_state,
         output=filechunks,
         max_chunks=max_chunks_before_suspension,
+        queue_errors=chunker_errors,
     ) -> None:
         try:
             file_chunker(
                 upload_state=upload_state, output=output, max_chunks=max_chunks
             )
-        except Exception:
+        except Exception as e:
+            queue_errors.put(e)
             _thread.interrupt_main()
             raise
 
@@ -533,6 +544,9 @@ def upload_file_multipart(
         # upload_state.finished_parts.put(None)  # Signal the end of the queue
         upload_state.add_finished(None)
         thread_chunker.join()
+
+        if not chunker_errors.empty():
+            raise chunker_errors.get()
         if not upload_state.is_done():
             upload_state.save()
             return MultiUploadResult.SUSPENDED

{rclone_api-1.1.0.dist-info → rclone_api-1.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.0
+Version: 1.1.2
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.0.dist-info → rclone_api-1.1.2.dist-info}/RECORD

@@ -22,15 +22,16 @@ rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/copy_large_s3.py,sha256=33KFvCrh5uk-rdRtkREdEs2WNwxGgTdCAWDLCE4dm0A,2855
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api/experimental/flags.py,sha256=AHbTaFHuyYFm3pjdvbQ100jztOXOdNuxalMr8UjXnV4,4097
+rclone_api/experimental/flags.py,sha256=0-mtXg9J4MoMm2uBKbsMLj4pSGRLQUAqNRDJWGttnAQ,2443
+rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
 rclone_api/s3/api.py,sha256=VstlaEnBjO2JDQuCRLdTfUGvQLbfshlXXhAzimFv4Vc,3763
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_uploader.py,sha256=txnNtugKcTb1UDmPbqWp3K-yLUDEkeW72nEnQnDcA8o,18336
+rclone_api/s3/chunk_uploader.py,sha256=6EB7FH4HftkQRyHijOU8O8CWShFE0qQY-lDo5oU-xIw,18843
 rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
 rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
-rclone_api-1.1.0.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.0.dist-info/METADATA,sha256=lWksanXlrw5bUyZxWDjFpntBvxsBWd2IBImny4OZRZs,4478
-rclone_api-1.1.0.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.0.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.1.0.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.0.dist-info/RECORD,,
+rclone_api-1.1.2.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.2.dist-info/METADATA,sha256=gcug-ENlJLW8DD1HdmG1eeyb6ajIVVVO_luxxZqk2pc,4478
+rclone_api-1.1.2.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.2.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.1.2.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.2.dist-info/RECORD,,