rclone-api 1.2.8__tar.gz → 1.2.9__tar.gz

{rclone_api-1.2.8 → rclone_api-1.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.8
+Version: 1.2.9
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.8 → rclone_api-1.2.9}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.2.8"
+version = "1.2.9"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/mount.py

@@ -14,7 +14,7 @@ from threading import Lock, Semaphore
 from typing import Any
 
 from rclone_api.process import Process
-from rclone_api.types import SizeSuffix
+from rclone_api.types import FilePart
 
 _SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
 
@@ -34,7 +34,7 @@ def _cleanup_mounts() -> None:
     with ThreadPoolExecutor() as executor:
         mount: Mount
         for mount in _MOUNTS_FOR_GC:
-            executor.submit(mount.close, wait=False)
+            executor.submit(mount.close)
 
 
 atexit.register(_cleanup_mounts)
@@ -296,14 +296,13 @@ def _read_from_mount_task(
     try:
         with path.open("rb") as f:
             f.seek(offset)
-            sz = f.read(size)
-            assert len(sz) == size, f"Invalid read size: {len(sz)}"
-            return sz
+            payload = f.read(size)
+            assert len(payload) == size, f"Invalid read size: {len(payload)}"
+            return payload
+
     except KeyboardInterrupt as e:
         import _thread
 
-        warnings.warn(f"Error fetching file chunk: {e}")
-
         _thread.interrupt_main()
         return Exception(e)
     except Exception as e:
@@ -319,7 +318,6 @@ class MultiMountFileChunker:
         self,
         filename: str,
         filesize: int,
-        chunk_size: SizeSuffix,
         mounts: list[Mount],
         executor: ThreadPoolExecutor,
         verbose: bool | None,
@@ -328,7 +326,6 @@ class MultiMountFileChunker:
 
         self.filename = filename
         self.filesize = filesize
-        self.chunk_size = chunk_size
         self.executor = executor
         self.mounts_processing: list[Mount] = []
         self.mounts_availabe: list[Mount] = mounts
@@ -336,13 +333,26 @@ class MultiMountFileChunker:
         self.lock = Lock()
         self.verbose = get_verbose(verbose)
 
-    def close(self) -> None:
+    def shutdown(self) -> None:
         self.executor.shutdown(wait=True, cancel_futures=True)
         with ThreadPoolExecutor() as executor:
             for mount in self.mounts_processing:
                 executor.submit(lambda: mount.close())
 
-    def fetch(self, offset: int, size: int) -> Future[bytes | Exception]:
+    def _acquire_mount(self) -> Mount:
+        self.semaphore.acquire()
+        with self.lock:
+            mount = self.mounts_availabe.pop()
+            self.mounts_processing.append(mount)
+        return mount
+
+    def _release_mount(self, mount: Mount) -> None:
+        with self.lock:
+            self.mounts_processing.remove(mount)
+            self.mounts_availabe.append(mount)
+            self.semaphore.release()
+
+    def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
         if self.verbose:
             print(f"Fetching data range: offset={offset}, size={size}")
 
@@ -353,28 +363,25 @@ class MultiMountFileChunker:
         ), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
 
         try:
-            self.semaphore.acquire()
-            with self.lock:
-                mount = self.mounts_availabe.pop()
-                self.mounts_processing.append(mount)
-
+            mount = self._acquire_mount()
             path = mount.mount_path / self.filename
 
-            def task(
-                offset=offset, size=size, path=path, mount=mount, verbose=self.verbose
-            ) -> bytes | Exception:
-                out = _read_from_mount_task(
+            def task_fetch_file_range(
+                size=size, path=path, mount=mount, verbose=self.verbose
+            ) -> FilePart:
+                bytes_or_err = _read_from_mount_task(
                     offset=offset, size=size, path=path, verbose=verbose
                 )
-                with self.lock:
-                    self.mounts_processing.remove(mount)
-                    self.mounts_availabe.append(mount)
-                    self.semaphore.release()
+                self._release_mount(mount)
+
+                if isinstance(bytes_or_err, Exception):
+                    return FilePart(payload=bytes_or_err, extra=extra)
+                out = FilePart(payload=bytes_or_err, extra=extra)
                 return out
 
-            fut = self.executor.submit(task)
+            fut = self.executor.submit(task_fetch_file_range)
             return fut
         except Exception as e:
             warnings.warn(f"Error fetching file chunk: {e}")
-            err = Exception(e)
-            return self.executor.submit(lambda: err)
+            fp = FilePart(payload=e, extra=extra)
+            return self.executor.submit(lambda: fp)

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/profile/mount_copy_bytes.py

@@ -15,6 +15,7 @@ from dotenv import load_dotenv
 
 from rclone_api import Config, Rclone, SizeSuffix
 from rclone_api.mount import MultiMountFileChunker
+from rclone_api.types import FilePart
 
 os.environ["RCLONE_API_VERBOSE"] = "1"
 
@@ -130,16 +131,18 @@ def _run_profile(
     )
     bytes_count = 0
 
-    futures: list[Future[bytes | Exception]] = []
+    futures: list[Future[FilePart]] = []
     for i in range(num):
         offset = SizeSuffix(i * chunk_size.as_int()) + offset
-        future = filechunker.fetch(offset.as_int(), size.as_int())
+        future = filechunker.fetch(offset.as_int(), size.as_int(), "TEST OBJECT")
         futures.append(future)
 
     for future in futures:
-        bytes_or_err = future.result()
-        if isinstance(bytes_or_err, Exception):
-            assert False, f"Error: {bytes_or_err}"
+        filepart_or_err = future.result()
+        if isinstance(filepart_or_err, Exception):
+            assert False, f"Error: {filepart_or_err}"
+        bytes_count += filepart_or_err.n_bytes()
+        filepart_or_err.close()
     futures.clear()
 
     start = time.time()
@@ -149,14 +152,14 @@ def _run_profile(
 
     for i in range(num):
         offset = SizeSuffix(i * chunk_size.as_int()) + offset
-        future = filechunker.fetch(offset.as_int(), size.as_int())
+        future = filechunker.fetch(offset.as_int(), size.as_int(), "TEST OBJECT")
         futures.append(future)
 
     for future in futures:
         bytes_or_err = future.result()
         if isinstance(bytes_or_err, Exception):
             assert False, f"Error: {bytes_or_err}"
-        bytes_count += len(bytes_or_err)
+        bytes_count += bytes_or_err.n_bytes()
 
     diff = (time.time() - start) / num
     net_io_end = psutil.net_io_counters()

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/rclone.py

@@ -4,6 +4,7 @@ Unit test file.
 
 import os
 import random
+import shutil
 import subprocess
 import time
 import traceback
@@ -825,7 +826,7 @@ class Rclone:
             traceback.print_exc()
             raise
         finally:
-            chunk_fetcher.close()
+            chunk_fetcher.shutdown()
 
     def _copy_bytes(
         self,
@@ -984,7 +985,6 @@ class Rclone:
         filechunker: MultiMountFileChunker = MultiMountFileChunker(
             filename=filename,
             filesize=filesize,
-            chunk_size=chunk_size,
             mounts=mounts,
             executor=executor,
             verbose=mount_log is not None,
@@ -1004,6 +1004,8 @@ class Rclone:
         direct_io: bool = True,
     ) -> bytes | Exception:
         """Copy bytes from a file to another file."""
+        from rclone_api.types import FilePart
+
         # determine number of threads from chunk size
         threads = max(1, min(max_threads, length // chunk_size.as_int()))
         filechunker = self.get_multi_mount_file_chunker(
@@ -1014,23 +1016,24 @@ class Rclone:
             direct_io=direct_io,
         )
         try:
-            fut = filechunker.fetch(offset, length)
-            data = fut.result()
-            if isinstance(data, Exception):
-                warnings.warn(f"Error copying bytes: {data}")
-                raise data
+            fut = filechunker.fetch(offset, length, extra=None)
+            fp: FilePart = fut.result()
+            payload = fp.payload
+            if isinstance(payload, Exception):
+                return payload
             if outfile is None:
-                return data
-            with open(outfile, "wb") as out:
-                out.write(data)
-                del data
+                out = payload.read_bytes()
+                payload.unlink()
+                return out
+            shutil.move(payload, outfile)
             return bytes(0)
+
         except Exception as e:
             warnings.warn(f"Error copying bytes: {e}")
             return e
         finally:
             try:
-                filechunker.close()
+                filechunker.shutdown()
             except Exception as e:
                 warnings.warn(f"Error closing filechunker: {e}")
 

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/s3/chunk_file.py

@@ -1,12 +1,15 @@
 import time
 import warnings
 from concurrent.futures import Future
+from dataclasses import dataclass
 from pathlib import Path
 from queue import Queue
 from threading import Event
-from typing import Callable
+from typing import Any, Callable
 
-from rclone_api.s3.chunk_types import FileChunk, UploadState
+from rclone_api.mount import FilePart
+from rclone_api.s3.chunk_types import UploadState
+from rclone_api.types import Finished
 from rclone_api.util import locked_print
 
 
@@ -27,12 +30,18 @@ def _get_file_size(file_path: Path, timeout: int = 60) -> int:
             raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
 
 
+@dataclass
+class S3FileInfo:
+    upload_id: str
+    part_number: int
+
+
 def file_chunker(
     upload_state: UploadState,
-    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
+    fetcher: Callable[[int, int, Any], Future[FilePart]],
     max_chunks: int | None,
     cancel_signal: Event,
-    output: Queue[FileChunk | None],
+    queue_upload: Queue[FilePart | Finished],
 ) -> None:
     count = 0
 
@@ -49,7 +58,7 @@ def file_chunker(
     upload_info = upload_state.upload_info
     file_path = upload_info.src_file_path
     chunk_size = upload_info.chunk_size
-    src = Path(file_path)
+    # src = Path(file_path)
 
     try:
         part_number = 1
@@ -73,12 +82,12 @@ def file_chunker(
             return
 
         while not should_stop():
-            curr_parth_num = next_part_number()
-            if curr_parth_num is None:
+            curr_part_number = next_part_number()
+            if curr_part_number is None:
                 locked_print(f"File {file_path} has completed chunking all parts")
                 break
-            assert curr_parth_num is not None
-            offset = (curr_parth_num - 1) * chunk_size
+            assert curr_part_number is not None
+            offset = (curr_part_number - 1) * chunk_size
             file_size = upload_info.file_size
 
             assert offset < file_size, f"Offset {offset} is greater than file size"
@@ -90,44 +99,38 @@ def file_chunker(
 
             # data = chunk_fetcher(offset, chunk_size).result()
 
-            assert curr_parth_num is not None
-            cpn: int = curr_parth_num
+            assert curr_part_number is not None
+            cpn: int = curr_part_number
 
-            def on_complete(
-                fut: Future[bytes | Exception],
-                part_number: int = cpn,
-                file_path: Path = file_path,
-            ) -> None:
-                data: bytes | Exception = fut.result()
-                if isinstance(data, Exception):
+            def on_complete(fut: Future[FilePart]) -> None:
+                fp: FilePart = fut.result()
+                if fp.is_error():
                     warnings.warn(
-                        f"Error reading file: {data}, skipping part {part_number}"
+                        f"Error reading file: {fp}, skipping part {part_number}"
                     )
                     return
 
-                if not data or len(data) == 0:
+                if fp.n_bytes() == 0:
                     warnings.warn(f"Empty data for part {part_number} of {file_path}")
                     raise ValueError(
                         f"Empty data for part {part_number} of {file_path}"
                     )
 
-                file_chunk = FileChunk(
-                    src,
-                    upload_id=upload_info.upload_id,
-                    part_number=part_number,
-                    data=data,  # After this, data should not be reused.
-                )
+                if isinstance(fp.payload, Exception):
+                    warnings.warn(f"Error reading file because of error: {fp.payload}")
+                    return
+
                 done_part_numbers.add(part_number)
-                output.put(file_chunk)
+                queue_upload.put(fp)
 
-            offset = (curr_parth_num - 1) * chunk_size
-            fut = chunk_fetcher(offset, chunk_size)
+            offset = (curr_part_number - 1) * chunk_size
+            fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
             fut.add_done_callback(on_complete)
-            # wait until the output queue can accept the next chunk
-            while output.full():
+            # wait until the queue_upload queue can accept the next chunk
+            while queue_upload.full():
                 time.sleep(0.1)
     except Exception as e:
 
         warnings.warn(f"Error reading file: {e}")
     finally:
-        output.put(None)
+        queue_upload.put(Finished())

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/s3/chunk_types.py

@@ -1,7 +1,6 @@
 import hashlib
 import json
 import os
-import time
 from dataclasses import dataclass, field, fields
 from pathlib import Path
 from threading import Lock
@@ -11,79 +10,9 @@ from botocore.client import BaseClient
 from rclone_api.types import SizeSuffix
 from rclone_api.util import locked_print
 
-_MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
+# _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
 _SAVE_STATE_LOCK = Lock()
 
-_TMP_DIR_ACCESS_LOCK = Lock()
-
-
-def _clean_old_files(out: Path) -> None:
-    # clean up files older than 1 day
-
-    now = time.time()
-    # Erase all stale files and then purge empty directories.
-    for root, dirs, files in os.walk(out):
-        for name in files:
-            f = Path(root) / name
-            filemod = f.stat().st_mtime
-            diff_secs = now - filemod
-            diff_days = diff_secs / (60 * 60 * 24)
-            if diff_days > 1:
-                locked_print(f"Removing old file: {f}")
-                f.unlink()
-
-    for root, dirs, _ in os.walk(out):
-        for dir in dirs:
-            d = Path(root) / dir
-            if not list(d.iterdir()):
-                locked_print(f"Removing empty directory: {d}")
-                d.rmdir()
-
-
-def _get_chunk_tmpdir() -> Path:
-    with _TMP_DIR_ACCESS_LOCK:
-        dat = _get_chunk_tmpdir.__dict__
-        if "out" in dat:
-            return dat["out"]  # Folder already validated.
-        out = Path("chunk_store")
-        if out.exists():
-            # first access, clean up directory
-            _clean_old_files(out)
-        out.mkdir(exist_ok=True, parents=True)
-        dat["out"] = out
-        return out
-
-
-class FileChunk:
-    def __init__(self, src: Path, upload_id: str, part_number: int, data: bytes):
-        assert data is not None, f"{src}: Data must not be None"
-        self.upload_id = upload_id
-        self.src = src
-        self.part_number = part_number
-        name = src.name
-        self.tmpdir = _get_chunk_tmpdir()
-        self.filepart = self.tmpdir / f"{name}_{upload_id}.part_{part_number}.tmp"
-        self.filepart.write_bytes(data)
-        del data  # free up memory
-
-    @property
-    def data(self) -> bytes:
-        assert self.filepart is not None
-        with open(self.filepart, "rb") as f:
-            return f.read()
-        return b""
-
-    def close(self):
-        import traceback
-
-        stacktrace = traceback.format_stack()
-        locked_print(f"Closing file chunk: {self.filepart}\n{stacktrace}")
-        if self.filepart.exists():
-            self.filepart.unlink()
-
-    def __del__(self):
-        self.close()
-
 
 @dataclass
 class UploadInfo:
@@ -220,11 +149,13 @@ class UploadState:
             self._save_no_lock()
 
     def __post_init__(self):
+        from rclone_api.types import get_chunk_tmpdir
+
         if self.peristant is None:
             # upload_id = self.upload_info.upload_id
             object_name = self.upload_info.object_name
             chunk_size = self.upload_info.chunk_size
-            parent = _get_chunk_tmpdir()
+            parent = get_chunk_tmpdir()
             self.peristant = parent / f"{object_name}_chunk_size_{chunk_size}_.json"
 
     def save(self) -> None:

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/s3/types.py

@@ -2,7 +2,9 @@ from concurrent.futures import Future
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
-from typing import Callable
+from typing import Any, Callable
+
+from rclone_api.mount import FilePart
 
 
 class S3Provider(Enum):
@@ -48,7 +50,7 @@ class S3MutliPartUploadConfig:
 
     chunk_size: int
     retries: int
-    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]]
+    chunk_fetcher: Callable[[int, int, Any], Future[FilePart]]
     resume_path_json: Path
     max_write_threads: int
     max_chunks_before_suspension: int | None = None

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/s3/upload_file_multipart.py

@@ -6,13 +6,19 @@ from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
 from queue import Queue
 from threading import Event, Thread
-from typing import Callable
+from typing import Any, Callable
 
 from botocore.client import BaseClient
 
-from rclone_api.s3.chunk_file import file_chunker
-from rclone_api.s3.chunk_types import FileChunk, FinishedPiece, UploadInfo, UploadState
+from rclone_api.mount import FilePart
+from rclone_api.s3.chunk_file import S3FileInfo, file_chunker
+from rclone_api.s3.chunk_types import (
+    FinishedPiece,
+    UploadInfo,
+    UploadState,
+)
 from rclone_api.s3.types import MultiUploadResult
+from rclone_api.types import Finished
 from rclone_api.util import locked_print
 
 _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
@@ -52,20 +58,22 @@ def upload_task(
 
 
 def handle_upload(
-    upload_info: UploadInfo, file_chunk: FileChunk | None
+    upload_info: UploadInfo, fp: FilePart
 ) -> FinishedPiece | Exception | None:
-    if file_chunk is None:
+    if fp is None:
         return None
-    print(f"Handling upload for {file_chunk.part_number}, size {len(file_chunk.data)}")
-    chunk, part_number = file_chunk.data, file_chunk.part_number
+    assert isinstance(fp.extra, S3FileInfo)
+    extra: S3FileInfo = fp.extra
+    part_number = extra.part_number
+    print(f"Handling upload for {part_number}, size {fp.size}")
     try:
         part: FinishedPiece = upload_task(
             info=upload_info,
-            chunk=chunk,
+            chunk=fp.load(),
             part_number=part_number,
             retries=upload_info.retries,
         )
-        file_chunk.close()
+        fp.close()
         return part
     except Exception as e:
         stacktrace = traceback.format_exc()
@@ -121,7 +129,7 @@ def _abort_previous_upload(upload_state: UploadState) -> None:
 
 def upload_file_multipart(
     s3_client: BaseClient,
-    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
+    chunk_fetcher: Callable[[int, int, Any], Future[FilePart]],
     bucket_name: str,
     file_path: Path,
     file_size: int | None,
@@ -178,7 +186,6 @@ def upload_file_multipart(
 
     work_que_max = upload_threads // 2 + 2
 
-    filechunks: Queue[FileChunk | None] = Queue(work_que_max)
     new_state = make_new_state()
     loaded_state = get_upload_state()
 
@@ -215,13 +222,14 @@ def upload_file_multipart(
     started_new_upload = finished == 0
     upload_info = upload_state.upload_info
 
+    queue_upload: Queue[FilePart | Finished] = Queue(work_que_max)
     chunker_errors: Queue[Exception] = Queue()
     cancel_chunker_event = Event()
 
     def chunker_task(
         upload_state=upload_state,
         chunk_fetcher=chunk_fetcher,
-        output=filechunks,
+        queue_upload=queue_upload,
         max_chunks=max_chunks_before_suspension,
         cancel_signal=cancel_chunker_event,
         queue_errors=chunker_errors,
@@ -229,8 +237,8 @@ def upload_file_multipart(
         try:
             file_chunker(
                 upload_state=upload_state,
-                chunk_fetcher=chunk_fetcher,
-                output=output,
+                fetcher=chunk_fetcher,
+                queue_upload=queue_upload,
                 max_chunks=max_chunks,
                 cancel_signal=cancel_signal,
             )
@@ -246,8 +254,11 @@ def upload_file_multipart(
         with ThreadPoolExecutor(max_workers=upload_threads) as executor:
             try:
                 while True:
-                    file_chunk: FileChunk | None = filechunks.get()
-                    if file_chunk is None:
+                    file_chunk: FilePart | Finished = queue_upload.get()
+                    if file_chunk is Finished:
+                        break
+
+                    if isinstance(file_chunk, Finished):
                         break
 
                     def task(upload_info=upload_info, file_chunk=file_chunk):

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api/types.py

@@ -1,6 +1,12 @@
+import os
 import re
+import time
+import warnings
 from dataclasses import dataclass
 from enum import Enum
+from pathlib import Path
+from threading import Lock
+from typing import Any
 
 
 class ModTimeStrategy(Enum):
@@ -214,3 +220,114 @@ class SizeSuffix:
 
     def __int__(self) -> int:
         return self._size
+
+
+_TMP_DIR_ACCESS_LOCK = Lock()
+
+
+def _clean_old_files(out: Path) -> None:
+    # clean up files older than 1 day
+    from rclone_api.util import locked_print
+
+    now = time.time()
+    # Erase all stale files and then purge empty directories.
+    for root, dirs, files in os.walk(out):
+        for name in files:
+            f = Path(root) / name
+            filemod = f.stat().st_mtime
+            diff_secs = now - filemod
+            diff_days = diff_secs / (60 * 60 * 24)
+            if diff_days > 1:
+                locked_print(f"Removing old file: {f}")
+                f.unlink()
+
+    for root, dirs, _ in os.walk(out):
+        for dir in dirs:
+            d = Path(root) / dir
+            if not list(d.iterdir()):
+                locked_print(f"Removing empty directory: {d}")
+                d.rmdir()
+
+
+def get_chunk_tmpdir() -> Path:
+    with _TMP_DIR_ACCESS_LOCK:
+        dat = get_chunk_tmpdir.__dict__
+        if "out" in dat:
+            return dat["out"]  # Folder already validated.
+        out = Path("chunk_store")
+        if out.exists():
+            # first access, clean up directory
+            _clean_old_files(out)
+        out.mkdir(exist_ok=True, parents=True)
+        dat["out"] = out
+        return out
+
+
+class Finished:
+    pass
+
+
+class FilePart:
+    def __init__(self, payload: bytes | Exception, extra: Any) -> None:
+        from rclone_api.util import random_str
+
+        self.extra = extra
+        self._lock = Lock()
+        self.payload: Path | Exception
+        if isinstance(payload, Exception):
+            self.payload = payload
+            return
+        self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+        with _TMP_DIR_ACCESS_LOCK:
+            if not self.payload.parent.exists():
+                self.payload.parent.mkdir(parents=True)
+            self.payload.write_bytes(payload)
+
+    @property
+    def size(self) -> int:
+        with self._lock:
+            if isinstance(self.payload, Path):
+                return self.payload.stat().st_size
+            return -1
+
+    def n_bytes(self) -> int:
+        with self._lock:
+            if isinstance(self.payload, Path):
+                return self.payload.stat().st_size
+            return -1
+
+    def load(self) -> bytes:
+        with self._lock:
+            if isinstance(self.payload, Path):
+                with open(self.payload, "rb") as f:
+                    return f.read()
+            raise ValueError("Cannot load from error")
+
+    def __post_init__(self):
+        if isinstance(self.payload, Path):
+            assert self.payload.exists(), f"File part {self.payload} does not exist"
+            assert self.payload.is_file(), f"File part {self.payload} is not a file"
+            assert self.payload.stat().st_size > 0, f"File part {self.payload} is empty"
+        elif isinstance(self.payload, Exception):
+            warnings.warn(f"File part error: {self.payload}")
+        print(f"File part created with payload: {self.payload}")
+
+    def is_error(self) -> bool:
+        return isinstance(self.payload, Exception)
+
+    def close(self) -> None:
+        with self._lock:
+            if isinstance(self.payload, Exception):
+                warnings.warn(
+                    f"Cannot close file part because the payload represents an error: {self.payload}"
+                )
+                return
+            if self.payload.exists():
+                try:
+                    self.payload.unlink()
+                    print(f"File part {self.payload} deleted")
+                except Exception as e:
+                    warnings.warn(f"Cannot close file part because of error: {e}")
+
+    def __del__(self):
+        self.close()

{rclone_api-1.2.8 → rclone_api-1.2.9}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.8
+Version: 1.2.9
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.8 → rclone_api-1.2.9}/tests/test_s3.py

@@ -3,6 +3,7 @@ import tempfile
 import unittest
 from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
+from typing import Any
 
 from dotenv import load_dotenv
 
@@ -10,6 +11,7 @@ from rclone_api.s3.api import S3Client
 from rclone_api.s3.create import S3Provider
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
 from rclone_api.s3.upload_file_multipart import MultiUploadResult
+from rclone_api.types import FilePart
 
 load_dotenv()
 
@@ -67,14 +69,18 @@ class RcloneS3Tester(unittest.TestCase):
             state_json = Path(tempdir) / "state.json"
 
             def simple_fetcher(
-                offset: int, chunk_size: int
-            ) -> Future[bytes | Exception]:
+                offset: int, chunk_size: int, extra: Any
+            ) -> Future[FilePart]:
                 with ThreadPoolExecutor() as executor:
 
-                    def task() -> bytes | Exception:
+                    def task(
+                        tmpfile=tmpfile, offset=offset, chunk_size=chunk_size
+                    ) -> FilePart:
                         with open(str(tmpfile), "rb") as f:
                             f.seek(offset)
-                            return f.read(chunk_size)
+                            data = f.read(chunk_size)
+                            fp = FilePart(payload=data, extra=extra)
+                            return fp
 
                     fut = executor.submit(task)
                 return fut