rclone-api 1.3.18__py2.py3-none-any.whl → 1.3.20__py2.py3-none-any.whl

rclone_api/s3/upload_file_multipart.py

@@ -25,31 +25,39 @@ _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
 
 
 def upload_task(
-    info: UploadInfo, chunk: bytes, part_number: int, retries: int
+    info: UploadInfo, chunk: FilePart, part_number: int, retries: int
 ) -> FinishedPiece:
-    assert len(chunk) > 0
+    file_or_err: Path | Exception = chunk.get_file()
+    if isinstance(file_or_err, Exception):
+        raise file_or_err
+    file: Path = file_or_err
+    size = os.path.getsize(file)
     retries = retries + 1  # Add one for the initial attempt
     for retry in range(retries):
         try:
             if retry > 0:
                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
             locked_print(
-                f"Uploading part {part_number} for {info.src_file_path} of size {len(chunk)}"
-            )
-            part = info.s3_client.upload_part(
-                Bucket=info.bucket_name,
-                Key=info.object_name,
-                PartNumber=part_number,
-                UploadId=info.upload_id,
-                Body=chunk,
-            )
-            out: FinishedPiece = FinishedPiece(
-                etag=part["ETag"], part_number=part_number
+                f"Uploading part {part_number} for {info.src_file_path} of size {size}"
             )
+
+            with open(file, "rb") as f:
+                part = info.s3_client.upload_part(
+                    Bucket=info.bucket_name,
+                    Key=info.object_name,
+                    PartNumber=part_number,
+                    UploadId=info.upload_id,
+                    Body=f,
+                )
+                out: FinishedPiece = FinishedPiece(
+                    etag=part["ETag"], part_number=part_number
+                )
+            chunk.dispose()
             return out
         except Exception as e:
             if retry == retries - 1:
                 locked_print(f"Error uploading part {part_number}: {e}")
+                chunk.dispose()
                 raise e
             else:
                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
@@ -61,25 +69,29 @@ def handle_upload(
     upload_info: UploadInfo, fp: FilePart | EndOfStream
 ) -> FinishedPiece | Exception | EndOfStream:
     if isinstance(fp, EndOfStream):
-        return fp
-    assert isinstance(fp.extra, S3FileInfo)
-    extra: S3FileInfo = fp.extra
-    part_number = extra.part_number
-    print(f"Handling upload for {part_number}, size {fp.size}")
+        eos: EndOfStream = fp
+        return eos
+    part_number: int | None = None
     try:
+        assert isinstance(fp.extra, S3FileInfo)
+        extra: S3FileInfo = fp.extra
+        part_number = extra.part_number
+        print(f"Handling upload for {part_number}, size {fp.size}")
+
         part: FinishedPiece = upload_task(
             info=upload_info,
-            chunk=fp.load(),
+            chunk=fp,
             part_number=part_number,
             retries=upload_info.retries,
         )
-        fp.close()
         return part
     except Exception as e:
         stacktrace = traceback.format_exc()
         msg = f"Error uploading part {part_number}: {e}\n{stacktrace}"
         warnings.warn(msg)
         return e
+    finally:
+        fp.dispose()
 
 
 def prepare_upload_file_multipart(

rclone_api/types.py

@@ -1,3 +1,4 @@
+import atexit
 import os
 import re
 import time
@@ -267,8 +268,28 @@ class EndOfStream:
     pass
 
 
+_CLEANUP_LIST: list[Path] = []
+
+
+def _add_for_cleanup(path: Path) -> None:
+    _CLEANUP_LIST.append(path)
+
+
+def _on_exit_cleanup() -> None:
+    paths = list(_CLEANUP_LIST)
+    for path in paths:
+        try:
+            if path.exists():
+                path.unlink()
+        except Exception as e:
+            warnings.warn(f"Cannot cleanup {path}: {e}")
+
+
+atexit.register(_on_exit_cleanup)
+
+
 class FilePart:
-    def __init__(self, payload: bytes | Exception, extra: Any) -> None:
+    def __init__(self, payload: Path | bytes | Exception, extra: Any) -> None:
         from rclone_api.util import random_str
 
         self.extra = extra
@@ -277,11 +298,18 @@ class FilePart:
         if isinstance(payload, Exception):
             self.payload = payload
             return
-        self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
-        with _TMP_DIR_ACCESS_LOCK:
-            if not self.payload.parent.exists():
-                self.payload.parent.mkdir(parents=True)
-            self.payload.write_bytes(payload)
+        if isinstance(payload, bytes):
+            self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+            with _TMP_DIR_ACCESS_LOCK:
+                if not self.payload.parent.exists():
+                    self.payload.parent.mkdir(parents=True, exist_ok=True)
+                self.payload.write_bytes(payload)
+            _add_for_cleanup(self.payload)
+        if isinstance(payload, Path):
+            self.payload = payload
+
+    def get_file(self) -> Path | Exception:
+        return self.payload
 
     @property
     def size(self) -> int:
@@ -315,7 +343,7 @@ class FilePart:
     def is_error(self) -> bool:
         return isinstance(self.payload, Exception)
 
-    def close(self) -> None:
+    def dispose(self) -> None:
         with self._lock:
             if isinstance(self.payload, Exception):
                 warnings.warn(
@@ -330,4 +358,4 @@ class FilePart:
                     warnings.warn(f"Cannot close file part because of error: {e}")
 
     def __del__(self):
-        self.close()
+        self.dispose()

{rclone_api-1.3.18.dist-info → rclone_api-1.3.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.18
+Version: 1.3.20
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.18.dist-info → rclone_api-1.3.20.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=bJ6x-7ySj1kC7xjQJqEEA-0cr46RUh_tvIZsebGcyu4,1224
+rclone_api/__init__.py,sha256=gOQJgOs0_oaV_pOwlY00LXRYAHk1_MDwN59od1VpoC0,1334
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -8,19 +8,20 @@ rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
 rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
 rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
 rclone_api/exec.py,sha256=Bq0gkyZ10mEY0FRyzNZgdN4FaWP9vpeCk1kjpg-gN_8,1083
-rclone_api/file.py,sha256=aBHzC-sqwPJ5fSI8bj9XOXJY_I6aCevnxXqiLscl4oo,5393
+rclone_api/file.py,sha256=cz-7_nJArkVdJ9z2QaC_XZYpihXe3IPBC90Z5_3g2aw,5419
 rclone_api/file_item.py,sha256=cH-AQYsxedhNPp4c8NHY1ad4Z7St4yf_VGbmiGD59no,1770
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
+rclone_api/http_server.py,sha256=SmeUDDKaMpJGDqRNkHoImHTRNkvHEtGFzu_8jYfoeZU,8113
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
-rclone_api/mount_read_chunker.py,sha256=bi4N-VkyPdM4RWUYhIqJ71lQrXanck4PpF3pY8k2xnQ,4722
+rclone_api/mount_read_chunker.py,sha256=7jaF1Rsjr-kXIZW--Ol1QuG7WArBgdIcpQ0AJMYn7bI,4764
 rclone_api/process.py,sha256=BGXJTZVT__jeaDyjN8_kRycliOhkBErMPdHO1hKRvJE,5271
-rclone_api/rclone.py,sha256=ogWjSt--Ph2dpeq31mWsBRBvKhpf1EF0jJD2HxgQ8T0,50903
-rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
+rclone_api/rclone.py,sha256=lLUHeG11brzhODCTlOhjy6rqcJeC_LhQBGTBcRTkaw8,54099
+rclone_api/remote.py,sha256=jq3dPbAGvYZFW5cTviqxT2w6_jG2LLfS1RIcYSmMsQQ,503
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=XiiWoGXAzNxTEfRcszw3BeRF7ZATXHIAPFg2-aJzUfo,9926
+rclone_api/types.py,sha256=aj3usJrIDKC4MKuObBo9WjlR1isyJColdOQWXo3Repo,10608
 rclone_api/util.py,sha256=F9Q3zbWRsgPF4NG6OWB63cZ7GVq82lsraP47gmmDohU,5416
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
@@ -29,21 +30,21 @@ rclone_api/cmd/copy_large_s3.py,sha256=nOpAUAQN1mJnf4EIZCh4OVCW7Q4_EXJeLFVe6r_9r
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
-rclone_api/db/db.py,sha256=1yoom28TGwCKVkb9WFbgDTTYE4SxmYfALDEUTW8_Pms,10351
-rclone_api/db/models.py,sha256=unKEiu8l4R4UAEoncEbOHEda227DpXm-cWwRV6vwJXE,1657
+rclone_api/db/db.py,sha256=ZpYfeCUe8MKg_fdJucRSe6-fwGY_rWqUn7WkHCNFH_4,10074
+rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=nZtqMukLhSzHq64Pn1I8pXwjoraqWjCKey3WLAeubx0,9069
+rclone_api/profile/mount_copy_bytes.py,sha256=M1vZn-Mrga14Ik7MHGZHbnwYli41Ep6Tyll7hQc7Wmo,9071
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_task.py,sha256=plrQyjuHFdJfV55kKH4wqo6QGll9n4_7BO4B1A7j6kY,7121
+rclone_api/s3/chunk_task.py,sha256=kA6_5fLNdtT3QdTFrfBY6y8sH9Og8nM2mrjgAz_g1Rc,7196
 rclone_api/s3/chunk_types.py,sha256=oSWv8No9V3BeM7IcGnowyR2a7YrszdAXzEJlxaeZcp0,8852
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
-rclone_api/s3/upload_file_multipart.py,sha256=d8ZWqO8n9wsqRF6JjmvAFmG1aCkFqdSB1L8yxe_5qiY,11669
-rclone_api-1.3.18.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.3.18.dist-info/METADATA,sha256=ow95lJm4jG8h05LuHtbeagZgL8R_qb7dqtwYU9s0cV4,4598
-rclone_api-1.3.18.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.3.18.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.3.18.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.3.18.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=UlrUl8fB0oK8_r0w8ZwH79jlOCHQrMOKWZeNCmHrT7M,12052
+rclone_api-1.3.20.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.3.20.dist-info/METADATA,sha256=RFYBoioCW6Gs8lnUrpIJKl02vRFXTYdOD3YCIt7zJXw,4598
+rclone_api-1.3.20.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.3.20.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.3.20.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.3.20.dist-info/RECORD,,