rclone-api 1.5.61__py3-none-any.whl → 1.5.64__py3-none-any.whl

rclone_api/__init__.py

@@ -7,6 +7,8 @@ The API wraps the rclone command-line tool, providing a Pythonic interface
 for common operations like copying, listing, and managing remote storage.
 """
 
+import os
+
 # Import core components and utilities
 from datetime import datetime
 from pathlib import Path
@@ -64,6 +66,18 @@ def rclone_verbose(val: bool | None) -> bool:
     return _rclone_verbose(val)
 
 
+class Logging:
+    @staticmethod
+    def enable_upload_parts_logging(value: bool) -> None:
+        """
+        Enable or disable logging of upload parts.
+
+        Args:
+            value: If True, enables upload parts logging; otherwise disables it.
+        """
+        os.environ["LOG_UPLOAD_S3_RESUMABLE"] = "1" if value else "0"
+
+
 class Rclone:
     """
     Main interface for interacting with Rclone.
@@ -469,6 +483,41 @@ class Rclone:
         """
         return self.impl.is_s3(dst=dst)
 
+    def copy_file_s3_resumable(
+        self,
+        src: str,  # src:/Bucket/path/myfile.large.zst
+        dst: str,  # dst:/Bucket/path/myfile.large.zst
+        part_infos: list[PartInfo] | None = None,
+        upload_threads: int = 8,  # Number of reader and writer threads to use
+        merge_threads: int = 4,  # Number of threads to use for merging the parts
+    ) -> Exception | None:
+        """
+        Copy a large file to S3 with resumable upload capability.
+
+        This method splits the file into parts for parallel upload and can
+        resume interrupted transfers using a custom algorithm in python.
+
+        Particularly useful for very large files where network interruptions
+        are likely.
+
+        Args:
+            src: Source file path (format: remote:bucket/path/file)
+            dst: Destination file path (format: remote:bucket/path/file)
+            part_infos: Optional list of part information for resuming uploads
+            upload_threads: Number of parallel upload threads
+            merge_threads: Number of threads for merging uploaded parts
+
+        Returns:
+            None if successful, Exception if an error occurred
+        """
+        return self.impl.copy_file_s3_resumable(
+            src=src,
+            dst=dst,
+            part_infos=part_infos,
+            upload_threads=upload_threads,
+            merge_threads=merge_threads,
+        )
+
     def copy_to(
         self,
         src: File | str,
@@ -831,41 +880,6 @@ class Rclone:
         """
         return self.impl.copy_remote(src=src, dst=dst, args=args)
 
-    def copy_file_s3_resumable(
-        self,
-        src: str,  # src:/Bucket/path/myfile.large.zst
-        dst: str,  # dst:/Bucket/path/myfile.large.zst
-        part_infos: list[PartInfo] | None = None,
-        upload_threads: int = 8,  # Number of reader and writer threads to use
-        merge_threads: int = 4,  # Number of threads to use for merging the parts
-    ) -> Exception | None:
-        """
-        Copy a large file to S3 with resumable upload capability.
-
-        This method splits the file into parts for parallel upload and can
-        resume interrupted transfers using a custom algorithm in python.
-
-        Particularly useful for very large files where network interruptions
-        are likely.
-
-        Args:
-            src: Source file path (format: remote:bucket/path/file)
-            dst: Destination file path (format: remote:bucket/path/file)
-            part_infos: Optional list of part information for resuming uploads
-            upload_threads: Number of parallel upload threads
-            merge_threads: Number of threads for merging uploaded parts
-
-        Returns:
-            None if successful, Exception if an error occurred
-        """
-        return self.impl.copy_file_s3_resumable(
-            src=src,
-            dst=dst,
-            part_infos=part_infos,
-            upload_threads=upload_threads,
-            merge_threads=merge_threads,
-        )
-
     def mount(
         self,
         src: Remote | Dir | str,

rclone_api/rclone_impl.py

@@ -1363,7 +1363,8 @@ class RcloneImpl:
         verbose = get_verbose(verbose)
         check = get_check(check)
         if len(files) < 2:
-            tmp = self.size_file(files[0])
+            full_path = f"{src}/{files[0]}"
+            tmp = self.size_file(full_path)
             if isinstance(tmp, Exception):
                 return tmp
             assert isinstance(tmp, SizeSuffix)

rclone_api/s3/multipart/upload_parts_resumable.py

@@ -17,11 +17,28 @@ from rclone_api.types import (
     SizeSuffix,
 )
 
+_LOCK = threading.Lock()
+
+
+def _maybe_log(msg: str) -> None:
+    print(msg)
+    if os.getenv("LOG_UPLOAD_S3_RESUMABLE") == "1":
+        log_path = Path("log") / "upload.log"
+        with _LOCK:
+            log_path.parent.mkdir(parents=True, exist_ok=True)
+            # log_path.write_text(msg, append=True)
+            with open(log_path, mode="a", encoding="utf-8") as f:
+                f.write(msg)
+                f.write("\n")
+
 
 @dataclass
 class UploadPart:
     chunk: Path
     dst_part: str
+    part_num: int
+    total_parts: int
+    total_size: SizeSuffix
     exception: Exception | None = None
     finished: bool = False
 
@@ -46,10 +63,18 @@ def upload_task(self: RcloneImpl, upload_part: UploadPart) -> UploadPart:
         if upload_part.exception is not None:
             return upload_part
         # print(f"Uploading {upload_part.chunk} to {upload_part.dst_part}")
-        msg = "\n#########################################\n"
+        num_parts = upload_part.total_parts
+        total_size = upload_part.total_size
+        part_num = upload_part.part_num
+        msg = "\n#############################################################\n"
         msg += f"# Uploading {upload_part.chunk} to {upload_part.dst_part}\n"
-        msg += "#########################################\n"
-        print(msg)
+        msg += f"# Part number: {part_num} / {num_parts}\n"
+        msg += f"# Total parts: {num_parts}\n"
+        msg += f"# Total size: {total_size.as_int()} bytes\n"
+        msg += f"# Chunk size: {upload_part.chunk.stat().st_size} bytes\n"
+        msg += f"# Range: {upload_part.chunk.name}\n"
+        msg += "##############################################################\n"
+        _maybe_log(msg)
         self.copy_to(upload_part.chunk.as_posix(), upload_part.dst_part)
         return upload_part
     except Exception as e:
@@ -66,6 +91,9 @@ def read_task(
     offset: SizeSuffix,
     length: SizeSuffix,
     part_dst: str,
+    part_number: int,
+    total_parts: int,
+    total_size: SizeSuffix,
 ) -> UploadPart:
     outchunk: Path = tmpdir / f"{offset.as_int()}-{(offset + length).as_int()}.chunk"
     range = Range(offset.as_int(), (offset + length).as_int())
@@ -77,10 +105,23 @@ def read_task(
             dst=outchunk,
         )
         if isinstance(err, Exception):
-            out = UploadPart(chunk=outchunk, dst_part="", exception=err)
+            out = UploadPart(
+                chunk=outchunk,
+                dst_part="",
+                part_num=-1,
+                total_parts=total_parts,
+                total_size=SizeSuffix(0),
+                exception=err,
+            )
             out.dispose()
             return out
-        return UploadPart(chunk=outchunk, dst_part=part_dst)
+        return UploadPart(
+            chunk=outchunk,
+            dst_part=part_dst,
+            part_num=part_number,
+            total_parts=total_parts,
+            total_size=total_size,
+        )
     except KeyboardInterrupt as ke:
         _thread.interrupt_main()
         raise ke
@@ -88,7 +129,14 @@ def read_task(
         _thread.interrupt_main()
         raise se
     except Exception as e:
-        return UploadPart(chunk=outchunk, dst_part=part_dst, exception=e)
+        return UploadPart(
+            chunk=outchunk,
+            dst_part=part_dst,
+            part_num=part_number,
+            total_parts=total_parts,
+            total_size=total_size,
+            exception=e,
+        )
 
 
 def collapse_runs(numbers: list[int]) -> list[str]:
@@ -199,6 +247,7 @@ def upload_parts_resumable(
         f"all_numbers_already_done: {collapse_runs(sorted(list(all_numbers_already_done)))}"
     )
 
+    total_parts = len(part_infos)
     filtered_part_infos: list[PartInfo] = []
     for part_info in part_infos:
         if part_info.part_number not in all_numbers_already_done:
@@ -261,6 +310,9 @@ def upload_parts_resumable(
                             offset=offset,
                             length=length,
                             part_dst=part_dst,
+                            part_number=part_number,
+                            total_parts=total_parts,
+                            total_size=src_size,
                         )
 
                     read_fut: Future[UploadPart] = read_executor.submit(_read_task)
@@ -293,12 +345,17 @@ def upload_parts_resumable(
     shutil.rmtree(tmp_dir, ignore_errors=True)
 
     if len(exceptions) > 0:
-        return Exception(f"Failed to copy parts: {exceptions}", exceptions)
+        msg = f"Failed to copy parts: {exceptions}"
+        _maybe_log(msg)
+        return Exception(msg, exceptions)
 
     finished_parts: list[int] = info_json.fetch_all_finished_part_numbers()
     print(f"finished_names: {finished_parts}")
 
     diff_set = set(all_part_numbers).symmetric_difference(set(finished_parts))
     all_part_numbers_done = len(diff_set) == 0
-    print(f"all_part_numbers_done: {all_part_numbers_done}")
+    # print(f"all_part_numbers_done: {all_part_numbers_done}")
+    msg = f"all_part_numbers_done: {all_part_numbers_done}"
+    _maybe_log(msg)
+
     return None

{rclone_api-1.5.61.dist-info → rclone_api-1.5.64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rclone_api
-Version: 1.5.61
+Version: 1.5.64
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.5.61.dist-info → rclone_api-1.5.64.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=M3-jlyxhO7Md3FGMO8qBJ1Z7oLcbOjWshCHh2-CqKG4,34885
+rclone_api/__init__.py,sha256=AH0_iCsWfPCphaXgN0eV7zeZ5WvinUYpyizwPIuUqmQ,35244
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=URZwMME01f0EZymprCESuZ5dk4IuUSKbHhwIeTHrn7A,6131
@@ -20,7 +20,7 @@ rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=LZqEhuKZunbWVqmsOIqkkCotaxWJpdFRS1InXveoU5E,1428
 rclone_api/mount_util.py,sha256=jqhJEVTHV6c6lOOzUYb4FLMbqDMHdz7-QRcdH-IobFc,10154
 rclone_api/process.py,sha256=V4Ax9AyNdC3m4O6gjWSbIJyCQCMhT-t0f-K8z6xux7Q,5946
-rclone_api/rclone_impl.py,sha256=FXIkDOGnqp7zN9pks3WXoruIJki9M2sJd0-QX5cMQgg,52603
+rclone_api/rclone_impl.py,sha256=PppFWk4qo7gg6p_k3K7am3_6pytsLG6qTAKo6dBT-tI,52649
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
@@ -55,12 +55,12 @@ rclone_api/s3/multipart/info_json.py,sha256=-e8UCwrqjAP64U8PmH-o2ciJ6TN48DwHktJf
 rclone_api/s3/multipart/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
-rclone_api/s3/multipart/upload_parts_resumable.py,sha256=6-nlMclS8jyVvMvFbQDcZOX9MY1WbCcKA_s9bwuYxnk,9793
+rclone_api/s3/multipart/upload_parts_resumable.py,sha256=-KKrgXhCeDWp80JHBsFMEauQyxy6_X6LK0dWS7XSoYE,11621
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.5.61.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.5.61.dist-info/METADATA,sha256=6euBxBpkAYSnNXNCu_JDaYOiTIDW6vzyzHxpx09u-0g,37305
-rclone_api-1.5.61.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rclone_api-1.5.61.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
-rclone_api-1.5.61.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.5.61.dist-info/RECORD,,
+rclone_api-1.5.64.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.5.64.dist-info/METADATA,sha256=vFhk8ddv9OgccApXBXfIPQYesY2lnNE3cThIdl1H91s,37305
+rclone_api-1.5.64.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+rclone_api-1.5.64.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
+rclone_api-1.5.64.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.5.64.dist-info/RECORD,,

{rclone_api-1.5.61.dist-info → rclone_api-1.5.64.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any