rclone-api 1.5.63__py3-none-any.whl → 1.5.65__py3-none-any.whl

rclone_api/__init__.py

@@ -7,6 +7,8 @@ The API wraps the rclone command-line tool, providing a Pythonic interface
 for common operations like copying, listing, and managing remote storage.
 """
 
+import os
+
 # Import core components and utilities
 from datetime import datetime
 from pathlib import Path
@@ -64,6 +66,18 @@ def rclone_verbose(val: bool | None) -> bool:
     return _rclone_verbose(val)
 
 
+class Logging:
+    @staticmethod
+    def enable_upload_parts_logging(value: bool) -> None:
+        """
+        Enable or disable logging of upload parts.
+
+        Args:
+            value: If True, enables upload parts logging; otherwise disables it.
+        """
+        os.environ["LOG_UPLOAD_S3_RESUMABLE"] = "1" if value else "0"
+
+
 class Rclone:
     """
     Main interface for interacting with Rclone.

rclone_api/s3/multipart/upload_parts_resumable.py

@@ -17,11 +17,28 @@ from rclone_api.types import (
     SizeSuffix,
 )
 
+_LOCK = threading.Lock()
+
+
+def _maybe_log(msg: str) -> None:
+    print(msg)
+    if os.getenv("LOG_UPLOAD_S3_RESUMABLE") == "1":
+        log_path = Path("log") / "s3_resumable_upload.log"
+        with _LOCK:
+            log_path.parent.mkdir(parents=True, exist_ok=True)
+            # log_path.write_text(msg, append=True)
+            with open(log_path, mode="a", encoding="utf-8") as f:
+                f.write(msg)
+                f.write("\n")
+
 
 @dataclass
 class UploadPart:
     chunk: Path
     dst_part: str
+    part_num: int
+    total_parts: int
+    total_size: SizeSuffix
     exception: Exception | None = None
     finished: bool = False
 
@@ -46,10 +63,18 @@ def upload_task(self: RcloneImpl, upload_part: UploadPart) -> UploadPart:
         if upload_part.exception is not None:
             return upload_part
         # print(f"Uploading {upload_part.chunk} to {upload_part.dst_part}")
-        msg = "\n#########################################\n"
+        num_parts = upload_part.total_parts
+        total_size = upload_part.total_size
+        part_num = upload_part.part_num
+        msg = "\n#############################################################\n"
         msg += f"# Uploading {upload_part.chunk} to {upload_part.dst_part}\n"
-        msg += "#########################################\n"
-        print(msg)
+        msg += f"# Part number: {part_num} / {num_parts}\n"
+        msg += f"# Total parts: {num_parts}\n"
+        msg += f"# Total size: {total_size.as_int()} bytes\n"
+        msg += f"# Chunk size: {upload_part.chunk.stat().st_size} bytes\n"
+        msg += f"# Range: {upload_part.chunk.name}\n"
+        msg += "##############################################################\n"
+        _maybe_log(msg)
         self.copy_to(upload_part.chunk.as_posix(), upload_part.dst_part)
         return upload_part
     except Exception as e:
@@ -66,6 +91,9 @@ def read_task(
     offset: SizeSuffix,
     length: SizeSuffix,
     part_dst: str,
+    part_number: int,
+    total_parts: int,
+    total_size: SizeSuffix,
 ) -> UploadPart:
     outchunk: Path = tmpdir / f"{offset.as_int()}-{(offset + length).as_int()}.chunk"
     range = Range(offset.as_int(), (offset + length).as_int())
@@ -77,10 +105,23 @@ def read_task(
             dst=outchunk,
         )
         if isinstance(err, Exception):
-            out = UploadPart(chunk=outchunk, dst_part="", exception=err)
+            out = UploadPart(
+                chunk=outchunk,
+                dst_part="",
+                part_num=-1,
+                total_parts=total_parts,
+                total_size=SizeSuffix(0),
+                exception=err,
+            )
             out.dispose()
             return out
-        return UploadPart(chunk=outchunk, dst_part=part_dst)
+        return UploadPart(
+            chunk=outchunk,
+            dst_part=part_dst,
+            part_num=part_number,
+            total_parts=total_parts,
+            total_size=total_size,
+        )
     except KeyboardInterrupt as ke:
         _thread.interrupt_main()
         raise ke
@@ -88,7 +129,14 @@ def read_task(
         _thread.interrupt_main()
         raise se
     except Exception as e:
-        return UploadPart(chunk=outchunk, dst_part=part_dst, exception=e)
+        return UploadPart(
+            chunk=outchunk,
+            dst_part=part_dst,
+            part_num=part_number,
+            total_parts=total_parts,
+            total_size=total_size,
+            exception=e,
+        )
 
 
 def collapse_runs(numbers: list[int]) -> list[str]:
@@ -199,6 +247,7 @@ def upload_parts_resumable(
         f"all_numbers_already_done: {collapse_runs(sorted(list(all_numbers_already_done)))}"
     )
 
+    total_parts = len(part_infos)
     filtered_part_infos: list[PartInfo] = []
     for part_info in part_infos:
         if part_info.part_number not in all_numbers_already_done:
@@ -261,6 +310,9 @@ def upload_parts_resumable(
                             offset=offset,
                             length=length,
                             part_dst=part_dst,
+                            part_number=part_number,
+                            total_parts=total_parts,
+                            total_size=src_size,
                         )
 
                     read_fut: Future[UploadPart] = read_executor.submit(_read_task)
@@ -293,12 +345,17 @@ def upload_parts_resumable(
     shutil.rmtree(tmp_dir, ignore_errors=True)
 
     if len(exceptions) > 0:
-        return Exception(f"Failed to copy parts: {exceptions}", exceptions)
+        msg = f"Failed to copy parts: {exceptions}"
+        _maybe_log(msg)
+        return Exception(msg, exceptions)
 
     finished_parts: list[int] = info_json.fetch_all_finished_part_numbers()
     print(f"finished_names: {finished_parts}")
 
     diff_set = set(all_part_numbers).symmetric_difference(set(finished_parts))
     all_part_numbers_done = len(diff_set) == 0
-    print(f"all_part_numbers_done: {all_part_numbers_done}")
+    # print(f"all_part_numbers_done: {all_part_numbers_done}")
+    msg = f"all_part_numbers_done: {all_part_numbers_done}"
+    _maybe_log(msg)
+
     return None

{rclone_api-1.5.63.dist-info → rclone_api-1.5.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rclone_api
-Version: 1.5.63
+Version: 1.5.65
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.5.63.dist-info → rclone_api-1.5.65.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=gIONLifrNkReIlQkpJOk3TTueJ9jrJ2ZdM2scQSUvSY,34885
+rclone_api/__init__.py,sha256=AH0_iCsWfPCphaXgN0eV7zeZ5WvinUYpyizwPIuUqmQ,35244
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=URZwMME01f0EZymprCESuZ5dk4IuUSKbHhwIeTHrn7A,6131
@@ -55,12 +55,12 @@ rclone_api/s3/multipart/info_json.py,sha256=-e8UCwrqjAP64U8PmH-o2ciJ6TN48DwHktJf
 rclone_api/s3/multipart/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
-rclone_api/s3/multipart/upload_parts_resumable.py,sha256=6-nlMclS8jyVvMvFbQDcZOX9MY1WbCcKA_s9bwuYxnk,9793
+rclone_api/s3/multipart/upload_parts_resumable.py,sha256=Th6m3owHS6Z-ieNhhDtU0_A6nDgKMo9y_Qv6e7lRia8,11634
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.5.63.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.5.63.dist-info/METADATA,sha256=9XSnDbl63rF0hMxexBm7t65FmtxbjMyNiIURw5ixrxo,37305
-rclone_api-1.5.63.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rclone_api-1.5.63.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
-rclone_api-1.5.63.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.5.63.dist-info/RECORD,,
+rclone_api-1.5.65.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.5.65.dist-info/METADATA,sha256=tx0O8RMD9s9oAQYXW79wpP9HupCOuh0fb13px0uy9b0,37305
+rclone_api-1.5.65.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+rclone_api-1.5.65.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
+rclone_api-1.5.65.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.5.65.dist-info/RECORD,,

{rclone_api-1.5.63.dist-info → rclone_api-1.5.65.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any