rclone-api 1.1.1__tar.gz → 1.1.3__tar.gz

{rclone_api-1.1.1 → rclone_api-1.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.1
+Version: 1.1.3
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.1 → rclone_api-1.1.3}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.1"
+version = "1.1.3"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.1.3/src/rclone_api/experimental/flags.py

@@ -0,0 +1,82 @@
+from dataclasses import dataclass
+
+from rclone_api.experimental.flags_base import BaseFlags, merge_flags
+
+
+@dataclass
+class CopyFlags(BaseFlags):
+    check_first: bool | None = None
+    checksum: bool | None = False
+    compare_dest: list[str] | None = None
+    copy_dest: list[str] | None = None
+    cutoff_mode: str | None = None
+    ignore_case_sync: bool | None = None
+    ignore_checksum: bool | None = None
+    ignore_existing: bool | None = None
+    ignore_size: bool | None = None
+    ignore_times: bool | None = None
+    immutable: bool | None = None
+    inplace: bool | None = None
+    links: bool | None = None
+    max_backlog: int | None = None
+    max_duration: str | None = None
+    max_transfer: str | None = None
+    metadata: bool | None = None
+    modify_window: str | None = None
+    multi_thread_chunk_size: str | None = None
+    multi_thread_cutoff: str | None = None
+    multi_thread_streams: int | None = None
+    multi_thread_write_buffer_size: str | None = None
+    no_check_dest: bool | None = None
+    no_traverse: bool | None = None
+    no_update_dir_modtime: bool | None = None
+    no_update_modtime: bool | None = None
+    order_by: str | None = None
+    partial_suffix: str | None = None
+    refresh_times: bool | None = None
+    server_side_across_configs: bool | None = None
+    size_only: bool | None = None
+    streaming_upload_cutoff: str | None = None
+    update: bool | None = None
+
+    def to_args(self) -> list[str]:
+        return super().to_args()
+
+    def merge(self, other: "CopyFlags") -> "CopyFlags":
+        return merge_flags(CopyFlags, self, other)
+
+    def __repr__(self):
+        return super().__repr__()
+
+
+@dataclass
+class Flags(BaseFlags):
+    copy: CopyFlags | None = None
+
+    def to_args(self) -> list[str]:
+        return super().to_args()
+
+    def merge(self, other: "Flags") -> "Flags":
+        return merge_flags(Flags, self, other)
+
+    def __repr__(self):
+        return super().__repr__()
+
+
+def unit_test() -> None:
+    copy_flags_a = CopyFlags(compare_dest=["a", "b"])
+    copy_flags_b = CopyFlags(checksum=False)
+    flags_a = copy_flags_a.merge(copy_flags_b)
+    print("A:", flags_a)
+
+    copy_flags_c = CopyFlags(checksum=True)
+    copy_flags_d = CopyFlags(checksum=False)
+
+    merged_c_d = copy_flags_c.merge(copy_flags_d)
+    print("B:", merged_c_d)
+    merged_d_c = copy_flags_d.merge(copy_flags_c)
+    print("C:", merged_d_c)
+
+
+if __name__ == "__main__":
+    unit_test()

rclone_api-1.1.3/src/rclone_api/experimental/flags_base.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass, fields, is_dataclass
+from typing import Type, TypeVar
+
+T = TypeVar("T")
+
+
+def merge_flags(cls: Type[T], dataclass_a: T, dataclass_b: T) -> T:
+    if not is_dataclass(dataclass_a) or not is_dataclass(dataclass_b):
+        raise ValueError("Both inputs must be dataclass instances")
+    if type(dataclass_a) is not type(dataclass_b):
+        raise ValueError("Dataclass instances must be of the same type")
+
+    merged_kwargs = {}
+    for field in fields(dataclass_a):
+        a_value = getattr(dataclass_a, field.name)
+        b_value = getattr(dataclass_b, field.name)
+
+        if is_dataclass(a_value) and is_dataclass(b_value):
+            merged_kwargs[field.name] = merge_flags(type(a_value), a_value, b_value)
+        else:
+            merged_kwargs[field.name] = b_value if b_value is not None else a_value
+
+    return cls(**merged_kwargs)
+
+
+def _field_name_to_flag(field_name: str) -> str:
+    return f"--{field_name.replace('_', '-')}"
+
+
+@dataclass
+class BaseFlags:
+    """provides to_args(), merge() and __repr__ methods for flags dataclasses"""
+
+    def to_args(self) -> list[str]:
+        args = []
+        for field in fields(self):
+            value = getattr(self, field.name)
+            if value is None:
+                continue
+            # If the field value is a nested dataclass that supports to_args, use it.
+            if is_dataclass(value) and hasattr(value, "to_args"):
+                to_args = getattr(value, "to_args")
+                args.extend(to_args())
+            elif isinstance(value, bool):
+                # Only include the flag if the boolean is True.
+                if value:
+                    args.append(_field_name_to_flag(field.name))
+            else:
+                args.append(_field_name_to_flag(field.name))
+                if isinstance(value, list):
+                    # Join list values with a comma.
+                    args.append(",".join(map(str, value)))
+                else:
+                    args.append(str(value))
+        return args
+
+    def __repr__(self):
+        return str(self.to_args())

{rclone_api-1.1.1 → rclone_api-1.1.3}/src/rclone_api/s3/chunk_uploader.py

@@ -127,6 +127,13 @@ class UploadState:
     lock: Lock = Lock()
     parts: list[FinishedPiece | None] = field(default_factory=list)
 
+    def update_source_file(self, src_file: Path) -> None:
+        new_file_size = os.path.getsize(src_file)
+        if new_file_size != self.upload_info.file_size:
+            raise ValueError("File size changed, cannot resume")
+        self.upload_info.src_file_path = src_file
+        self.save()
+
     def is_done(self) -> bool:
         return self.remaining() == 0
 
@@ -264,15 +271,22 @@ def _get_chunk_tmpdir() -> Path:
 def _get_file_size(file_path: Path, timeout: int = 60) -> int:
     sleep_time = timeout / 60 if timeout > 0 else 1
     start = time.time()
+    not_windows = os.name != "nt"
     while True:
+        expired = time.time() - start > timeout
         try:
+            time.sleep(sleep_time)
+            if not_windows:
+                # Force a refresh of the directory cache
+                os.system(f"ls -l {file_path.parent}")
             if file_path.exists():
                 return file_path.stat().st_size
-        except FileNotFoundError:
-            pass
-        if time.time() - start > timeout:
+        except FileNotFoundError as e:
+            if expired:
+                print(f"File not found: {file_path}, exception is {e}")
+                raise
+        if expired:
             raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
-        time.sleep(sleep_time)
 
 
 def file_chunker(
@@ -426,6 +440,18 @@ def prepare_upload_file_multipart(
     return upload_info
 
 
+def _abort_previous_upload(upload_state: UploadState) -> None:
+    if upload_state.upload_info.upload_id:
+        try:
+            upload_state.upload_info.s3_client.abort_multipart_upload(
+                Bucket=upload_state.upload_info.bucket_name,
+                Key=upload_state.upload_info.object_name,
+                UploadId=upload_state.upload_info.upload_id,
+            )
+        except Exception as e:
+            locked_print(f"Error aborting previous upload: {e}")
+
+
 def upload_file_multipart(
     s3_client: BaseClient,
     bucket_name: str,
@@ -481,6 +507,13 @@ def upload_file_multipart(
 
     filechunks: Queue[FileChunk | None] = Queue(10)
     upload_state = get_upload_state() or make_new_state()
+    try:
+        upload_state.update_source_file(file_path)
+    except ValueError as e:
+        locked_print(f"Cannot resume upload: {e}, size changed, starting over")
+        _abort_previous_upload(upload_state)
+        upload_state = make_new_state()
+        upload_state.save()
     if upload_state.is_done():
         return MultiUploadResult.ALREADY_DONE
     finished = upload_state.finished()

{rclone_api-1.1.1 → rclone_api-1.1.3}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.1
+Version: 1.1.3
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.1 → rclone_api-1.1.3}/src/rclone_api.egg-info/SOURCES.txt

@@ -51,6 +51,7 @@ src/rclone_api/assets/example.txt
 src/rclone_api/cmd/copy_large_s3.py
 src/rclone_api/cmd/list_files.py
 src/rclone_api/experimental/flags.py
+src/rclone_api/experimental/flags_base.py
 src/rclone_api/s3/api.py
 src/rclone_api/s3/basic_ops.py
 src/rclone_api/s3/chunk_uploader.py

rclone_api-1.1.1/src/rclone_api/experimental/flags.py

@@ -1,121 +0,0 @@
-from dataclasses import dataclass, fields, is_dataclass
-from typing import Type, TypeVar
-
-T = TypeVar("T")
-
-
-def _merge(cls: Type[T], dataclass_a: T, dataclass_b: T) -> T:
-    if not is_dataclass(dataclass_a) or not is_dataclass(dataclass_b):
-        raise ValueError("Both inputs must be dataclass instances")
-    if type(dataclass_a) is not type(dataclass_b):
-        raise ValueError("Dataclass instances must be of the same type")
-
-    merged_kwargs = {}
-    for field in fields(dataclass_a):
-        a_value = getattr(dataclass_a, field.name)
-        b_value = getattr(dataclass_b, field.name)
-
-        if is_dataclass(a_value) and is_dataclass(b_value):
-            merged_kwargs[field.name] = _merge(type(a_value), a_value, b_value)
-        else:
-            merged_kwargs[field.name] = b_value if b_value is not None else a_value
-
-    return cls(**merged_kwargs)
-
-
-def _field_name_to_flag(field_name: str) -> str:
-    return f"--{field_name.replace('_', '-')}"
-
-
-@dataclass
-class BaseFlags:
-    def to_args(self) -> list[str]:
-        args = []
-        for field in fields(self):
-            value = getattr(self, field.name)
-            if value is None:
-                continue
-            # If the field value is a nested dataclass that supports to_args, use it.
-            if is_dataclass(value) and hasattr(value, "to_args"):
-                to_args = getattr(value, "to_args")
-                args.extend(to_args())
-            elif isinstance(value, bool):
-                # Only include the flag if the boolean is True.
-                if value:
-                    args.append(_field_name_to_flag(field.name))
-            else:
-                args.append(_field_name_to_flag(field.name))
-                if isinstance(value, list):
-                    # Join list values with a comma.
-                    args.append(",".join(map(str, value)))
-                else:
-                    args.append(str(value))
-        return args
-
-    def merge(self, other: "BaseFlags") -> "BaseFlags":
-        # Use the type of self, so merging CopyFlags returns a CopyFlags instance.
-        return _merge(type(self), self, other)
-
-    def __repr__(self):
-        return str(self.to_args())
-
-
-@dataclass(repr=False)
-class CopyFlags(BaseFlags):
-    check_first: bool | None = None
-    checksum: bool | None = False
-    compare_dest: list[str] | None = None
-    copy_dest: list[str] | None = None
-    cutoff_mode: str | None = None
-    ignore_case_sync: bool | None = None
-    ignore_checksum: bool | None = None
-    ignore_existing: bool | None = None
-    ignore_size: bool | None = None
-    ignore_times: bool | None = None
-    immutable: bool | None = None
-    inplace: bool | None = None
-    links: bool | None = None
-    max_backlog: int | None = None
-    max_duration: str | None = None
-    max_transfer: str | None = None
-    metadata: bool | None = None
-    modify_window: str | None = None
-    multi_thread_chunk_size: str | None = None
-    multi_thread_cutoff: str | None = None
-    multi_thread_streams: int | None = None
-    multi_thread_write_buffer_size: str | None = None
-    no_check_dest: bool | None = None
-    no_traverse: bool | None = None
-    no_update_dir_modtime: bool | None = None
-    no_update_modtime: bool | None = None
-    order_by: str | None = None
-    partial_suffix: str | None = None
-    refresh_times: bool | None = None
-    server_side_across_configs: bool | None = None
-    size_only: bool | None = None
-    streaming_upload_cutoff: str | None = None
-    update: bool | None = None
-
-
-@dataclass(repr=False)
-class Flags(BaseFlags):
-    copy: CopyFlags | None = None
-
-
-def unit_test() -> None:
-    copy_flags_a = CopyFlags(compare_dest=["a", "b"])
-    copy_flags_b = CopyFlags(checksum=False)
-    flags_a = copy_flags_a.merge(copy_flags_b)
-    print("A:", flags_a)
-
-    copy_flags_c = CopyFlags(checksum=True)
-    copy_flags_d = CopyFlags(checksum=False)
-
-    merged_c_d = copy_flags_c.merge(copy_flags_d)
-    print("B:", merged_c_d)
-    merged_d_c = copy_flags_d.merge(copy_flags_c)
-    print("C:", merged_d_c)
-
-
-if __name__ == "__main__":
-    unit_test()