rclone-api 1.2.9__py2.py3-none-any.whl → 1.2.12__py2.py3-none-any.whl

rclone_api/s3/chunk_file.py

@@ -9,7 +9,7 @@ from typing import Any, Callable
 
 from rclone_api.mount import FilePart
 from rclone_api.s3.chunk_types import UploadState
-from rclone_api.types import Finished
+from rclone_api.types import EndOfStream
 from rclone_api.util import locked_print
 
 
@@ -41,7 +41,7 @@ def file_chunker(
     fetcher: Callable[[int, int, Any], Future[FilePart]],
     max_chunks: int | None,
     cancel_signal: Event,
-    queue_upload: Queue[FilePart | Finished],
+    queue_upload: Queue[FilePart | EndOfStream],
 ) -> None:
     count = 0
 
@@ -51,6 +51,9 @@ def file_chunker(
         if max_chunks is None:
             return False
         if count >= max_chunks:
+            print(
+                f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
+            )
             return True
         count += 1
         return False
@@ -63,7 +66,7 @@ def file_chunker(
     try:
         part_number = 1
         done_part_numbers: set[int] = {
-            p.part_number for p in upload_state.parts if p is not None
+            p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
         }
         num_parts = upload_info.total_chunks()
 
@@ -133,4 +136,7 @@ def file_chunker(
 
         warnings.warn(f"Error reading file: {e}")
     finally:
-        queue_upload.put(Finished())
+        print("#############################################################")
+        print(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
+        print("#############################################################")
+        queue_upload.put(EndOfStream())

rclone_api/s3/chunk_types.py

@@ -7,7 +7,7 @@ from threading import Lock
 
 from botocore.client import BaseClient
 
-from rclone_api.types import SizeSuffix
+from rclone_api.types import EndOfStream, SizeSuffix
 from rclone_api.util import locked_print
 
 # _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
@@ -83,27 +83,34 @@ class FinishedPiece:
         return json.dumps(self.to_json(), indent=0)
 
     @staticmethod
-    def to_json_array(parts: list["FinishedPiece | None"]) -> list[dict | None]:
-        non_none: list[FinishedPiece] = [p for p in parts if p is not None]
+    def to_json_array(parts: list["FinishedPiece | EndOfStream"]) -> list[dict]:
+        non_none: list[FinishedPiece] = []
+        for p in parts:
+            if not isinstance(p, EndOfStream):
+                non_none.append(p)
         non_none.sort(key=lambda x: x.part_number)
-        all_nones: list[None] = [None for p in parts if p is None]
-        assert len(all_nones) <= 1, "Only one None should be present"
+        # all_nones: list[None] = [None for p in parts if p is None]
+        # assert len(all_nones) <= 1, "Only one None should be present"
+        count_eos = 0
+        for p in parts:
+            if p is EndOfStream:
+                count_eos += 1
+        assert count_eos <= 1, "Only one EndOfStream should be present"
         return [p.to_json() for p in non_none]
 
     @staticmethod
-    def from_json(json: dict | None) -> "FinishedPiece | None":
+    def from_json(json: dict | None) -> "FinishedPiece | EndOfStream":
         if json is None:
-            return None
+            return EndOfStream()
         return FinishedPiece(**json)
 
 
 @dataclass
 class UploadState:
     upload_info: UploadInfo
-    # finished_parts: Queue[FinishedPiece | None]
     peristant: Path | None
     lock: Lock = Lock()
-    parts: list[FinishedPiece | None] = field(default_factory=list)
+    parts: list[FinishedPiece | EndOfStream] = field(default_factory=list)
 
     def update_source_file(self, src_file: Path, known_file_size: int | None) -> None:
         new_file_size = (
@@ -126,7 +133,7 @@ class UploadState:
         num_chunks = self.upload_info.total_chunks()
         count = 0
         for p in self.parts:
-            if p is not None:
+            if not isinstance(p, EndOfStream):
                 count += 1
         return count, num_chunks
 
@@ -141,7 +148,7 @@ class UploadState:
         ), f"Count {count} is greater than num_chunks {num_chunks}"
         return num_chunks - count
 
-    def add_finished(self, part: FinishedPiece | None) -> None:
+    def add_finished(self, part: FinishedPiece | EndOfStream) -> None:
         if part is None:
             return
         with self.lock:
@@ -195,13 +202,13 @@ class UploadState:
     def to_json(self) -> dict:
         # queue -> list
         # parts: list[dict] = [f.to_json() for f in self.parts]
-        parts: list[FinishedPiece | None] = list(self.parts)
+        parts: list[FinishedPiece | EndOfStream] = list(self.parts)
 
         parts_json = FinishedPiece.to_json_array(parts)
         is_done = self.is_done()
         count_non_none: int = 0
         for p in parts:
-            if p is not None:
+            if p is not EndOfStream:
                 count_non_none += 1
 
         file_size_bytes = self.upload_info.file_size

rclone_api/s3/upload_file_multipart.py

@@ -18,7 +18,7 @@ from rclone_api.s3.chunk_types import (
     UploadState,
 )
 from rclone_api.s3.types import MultiUploadResult
-from rclone_api.types import Finished
+from rclone_api.types import EndOfStream
 from rclone_api.util import locked_print
 
 _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
@@ -58,10 +58,10 @@ def upload_task(
 
 
 def handle_upload(
-    upload_info: UploadInfo, fp: FilePart
-) -> FinishedPiece | Exception | None:
-    if fp is None:
-        return None
+    upload_info: UploadInfo, fp: FilePart | EndOfStream
+) -> FinishedPiece | Exception | EndOfStream:
+    if isinstance(fp, EndOfStream):
+        return fp
     assert isinstance(fp.extra, S3FileInfo)
     extra: S3FileInfo = fp.extra
     part_number = extra.part_number
@@ -222,7 +222,7 @@ def upload_file_multipart(
     started_new_upload = finished == 0
     upload_info = upload_state.upload_info
 
-    queue_upload: Queue[FilePart | Finished] = Queue(work_que_max)
+    queue_upload: Queue[FilePart | EndOfStream] = Queue(work_que_max)
     chunker_errors: Queue[Exception] = Queue()
     cancel_chunker_event = Event()
 
@@ -246,6 +246,9 @@ def upload_file_multipart(
             queue_errors.put(e)
             _thread.interrupt_main()
             raise
+        print("#########################################")
+        print("# CHUNKER TASK COMPLETED")
+        print("#########################################")
 
     try:
         thread_chunker = Thread(target=chunker_task, daemon=True)
@@ -254,11 +257,11 @@ def upload_file_multipart(
         with ThreadPoolExecutor(max_workers=upload_threads) as executor:
             try:
                 while True:
-                    file_chunk: FilePart | Finished = queue_upload.get()
-                    if file_chunk is Finished:
+                    file_chunk: FilePart | EndOfStream = queue_upload.get()
+                    if file_chunk is EndOfStream:
                         break
 
-                    if isinstance(file_chunk, Finished):
+                    if isinstance(file_chunk, EndOfStream):
                         break
 
                     def task(upload_info=upload_info, file_chunk=file_chunk):
@@ -280,7 +283,7 @@ def upload_file_multipart(
                 executor.shutdown(wait=False, cancel_futures=True)
                 raise
         # upload_state.finished_parts.put(None)  # Signal the end of the queue
-        upload_state.add_finished(None)
+        upload_state.add_finished(EndOfStream())
         thread_chunker.join()
 
         if not chunker_errors.empty():
@@ -288,7 +291,9 @@ def upload_file_multipart(
         if not upload_state.is_done():
             upload_state.save()
             return MultiUploadResult.SUSPENDED
-        parts: list[FinishedPiece] = [p for p in upload_state.parts if p is not None]
+        parts: list[FinishedPiece] = [
+            p for p in upload_state.parts if not isinstance(p, EndOfStream)
+        ]
         locked_print(f"Upload complete, sorting {len(parts)} parts to complete upload")
         parts.sort(key=lambda x: x.part_number)  # Some backends need this.
         parts_s3: list[dict] = [

rclone_api/types.py

@@ -263,7 +263,7 @@ def get_chunk_tmpdir() -> Path:
         return out
 
 
-class Finished:
+class EndOfStream:
     pass
 
 

{rclone_api-1.2.9.dist-info → rclone_api-1.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.9
+Version: 1.2.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.9.dist-info → rclone_api-1.2.12.dist-info}/RECORD

@@ -17,7 +17,7 @@ rclone_api/rclone.py,sha256=4_eoKkC0eZDT2VnpMlrF5EkDZam5WWaiiWFxNntqs2k,49323
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=bmLu9EnX5Q2DyUkoyi6ExgywkfOkb_YNJdTQfj8YYrk,9923
+rclone_api/types.py,sha256=XiiWoGXAzNxTEfRcszw3BeRF7ZATXHIAPFg2-aJzUfo,9926
 rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
@@ -28,14 +28,14 @@ rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ
 rclone_api/profile/mount_copy_bytes.py,sha256=HK11knsykKO1WNi8LYOkQbipQBpZn7uhV5CVEQDRCJs,8558
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_file.py,sha256=lZh4oCEo87jn5oEWHzK786_G9Y2RMmJz6cuL-ypAIT0,4402
-rclone_api/s3/chunk_types.py,sha256=NOdMz9lvoOi2DzNIaIZVY4-nGa-00P6eLSsPMak0gh8,8498
+rclone_api/s3/chunk_file.py,sha256=mk0koNiohOTJIp3nor2Lg-KlfbHeetC6GM4HVzkmEGY,4805
+rclone_api/s3/chunk_types.py,sha256=I0YCWFgxCvmt8cp4tMabiiwiD2yKTcbA6ZL2D3xnn5w,8781
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=FkUNNAk8fjSbLhDA45YgNQk9BTjmJZHT3fExiCKcDt0,1591
-rclone_api/s3/upload_file_multipart.py,sha256=scsGOxMpg2j9qK2oX_Xe4vHI6u0IHX0xtp9mHnt7Ksc,11113
-rclone_api-1.2.9.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.2.9.dist-info/METADATA,sha256=mSCgGU2Hwr87w4wlc8ZxlqG4oJ1F0t49olLS6I51wDc,4536
-rclone_api-1.2.9.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.2.9.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.2.9.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.2.9.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=TrhrYeRkpQRzmn8x_RR59ww3GXA3Wd95pCKI-Bhu9_0,11372
+rclone_api-1.2.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.2.12.dist-info/METADATA,sha256=47ATkIGrPGHwqK9hqVVJifxeli3iwGFnvusvIbS0_ms,4537
+rclone_api-1.2.12.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.2.12.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.2.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.2.12.dist-info/RECORD,,