rclone-api 1.1.95__tar.gz → 1.1.97__tar.gz

{rclone_api-1.1.95 → rclone_api-1.1.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.95
+Version: 1.1.97
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.95 → rclone_api-1.1.97}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.95"
+version = "1.1.97"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api/profile/mount_copy_bytes.py

@@ -142,27 +142,11 @@ def _run_profile(
             assert False, f"Error: {bytes_or_err}"
     futures.clear()
 
-
     start = time.time()
     net_io_start = psutil.net_io_counters()
 
-    # bytes_or_err: bytes | Exception = rclone.copy_bytes(
-    #     src=src_file,
-    #     offset=offset.as_int(),
-    #     length=size.as_int(),
-    #     chunk_size=chunk_size,
-    #     direct_io=direct_io,
-    #     max_threads=transfers,
-    #     mount_log=mount_log,
-    # )
-
-    # bytes_or_err = filechunker.fetch(
-    #     (offset + SizeSuffix("1G")).as_int(), size.as_int()
-    # )
-
     offset = SizeSuffix("1G")
 
-    futures: list[Future[bytes | Exception]] = []
     for i in range(num):
         offset = SizeSuffix(i * chunk_size.as_int()) + offset
         future = filechunker.fetch(offset.as_int(), size.as_int())
@@ -177,7 +161,9 @@ def _run_profile(
     diff = (time.time() - start) / num
     net_io_end = psutil.net_io_counters()
     # self.assertEqual(len(bytes_or_err), size)
-    assert bytes_count  == SizeSuffix(size* num).as_int(), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
+    assert (
+        bytes_count == SizeSuffix(size * num).as_int()
+    ), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
 
     # print io stats
     bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api/rclone.py

@@ -777,9 +777,18 @@ class Rclone:
                 endpoint_url=section.endpoint(),
             )
 
+            chunk_fetcher: MultiMountFileChunker = self.get_multi_mount_file_chunker(
+                src=src_path.parent.as_posix(),
+                chunk_size=chunk_size,
+                threads=read_threads,
+                mount_log=mount_log,
+                direct_io=True,
+            )
+
             client = S3Client(s3_creds)
-            config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
+            upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
+                chunk_fetcher=chunk_fetcher.fetch,
                 max_write_threads=write_threads,
                 retries=retries,
                 resume_path_json=save_state_json,
@@ -791,10 +800,7 @@ class Rclone:
             print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
             print(f"Source: {src_path}")
             print(f"bucket_name: {bucket_name}")
-            print(f"upload_config: {config}")
-
-            upload_target: S3UploadTarget
-            upload_config: S3MutliPartUploadConfig
+            print(f"upload_config: {upload_config}")
 
             upload_target = S3UploadTarget(
                 bucket_name=bucket_name,
@@ -802,18 +808,14 @@ class Rclone:
                 s3_key=s3_key,
             )
 
-            upload_config = S3MutliPartUploadConfig(
-                chunk_size=chunk_size.as_int(),
-                retries=retries,
-                max_write_threads=write_threads,
-                resume_path_json=save_state_json,
-                max_chunks_before_suspension=max_chunks_before_suspension,
-            )
-
-            out: MultiUploadResult = client.upload_file_multipart(
-                upload_target=upload_target, upload_config=upload_config
-            )
-            return out
+            try:
+                out: MultiUploadResult = client.upload_file_multipart(
+                    upload_target=upload_target,
+                    upload_config=upload_config,
+                )
+                return out
+            finally:
+                chunk_fetcher.close()
 
     def _copy_bytes(
         self,

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api/s3/api.py

@@ -11,7 +11,10 @@ from rclone_api.s3.basic_ops import (
 )
 from rclone_api.s3.create import create_s3_client
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
-from rclone_api.s3.upload_file_multipart import MultiUploadResult, upload_file_multipart
+from rclone_api.s3.upload_file_multipart import (
+    MultiUploadResult,
+    upload_file_multipart,
+)
 
 _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 
@@ -67,6 +70,7 @@ class S3Client:
 
             out = upload_file_multipart(
                 s3_client=self.client,
+                chunk_fetcher=upload_config.chunk_fetcher,
                 bucket_name=bucket_name,
                 file_path=upload_target.src_file,
                 object_name=upload_target.s3_key,

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api/s3/chunk_file.py

@@ -1,8 +1,10 @@
 import time
 import warnings
+from concurrent.futures import Future
 from pathlib import Path
 from queue import Queue
 from threading import Event
+from typing import Callable
 
 from rclone_api.s3.chunk_types import FileChunk, UploadState
 from rclone_api.util import locked_print
@@ -27,12 +29,16 @@ def _get_file_size(file_path: Path, timeout: int = 60) -> int:
 
 def file_chunker(
     upload_state: UploadState,
+    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
     max_chunks: int | None,
     cancel_signal: Event,
     output: Queue[FileChunk | None],
 ) -> None:
     count = 0
 
+    if False:
+        print(chunk_fetcher)
+
     def should_stop() -> bool:
         nonlocal count
 
@@ -41,14 +47,6 @@ def file_chunker(
         if count >= max_chunks:
             return True
         count += 1
-        if count > 10 and count % 10 == 0:
-            # recheck that the file size has not changed
-            file_size = _get_file_size(upload_state.upload_info.src_file_path)
-            if file_size != upload_state.upload_info.file_size:
-                locked_print(
-                    f"File size changed, cannot resume, expected {upload_state.upload_info.file_size}, got {file_size}"
-                )
-                raise ValueError("File size changed, cannot resume")
         return False
 
     upload_info = upload_state.upload_info
@@ -58,7 +56,6 @@ def file_chunker(
     # Mounted files may take a while to appear, so keep retrying.
 
     try:
-        file_size = _get_file_size(src, timeout=60)
         part_number = 1
         done_part_numbers: set[int] = {
             p.part_number for p in upload_state.parts if p is not None
@@ -86,25 +83,56 @@ def file_chunker(
                 break
             assert curr_parth_num is not None
             offset = (curr_parth_num - 1) * chunk_size
+            file_size = upload_info.file_size
 
             assert offset < file_size, f"Offset {offset} is greater than file size"
 
             # Open the file, seek, read the chunk, and close immediately.
-            with open(file_path, "rb") as f:
-                f.seek(offset)
-                data = f.read(chunk_size)
-
-            if not data:
-                warnings.warn(f"Empty data for part {part_number} of {file_path}")
-
-            file_chunk = FileChunk(
-                src,
-                upload_id=upload_info.upload_id,
-                part_number=part_number,
-                data=data,  # After this, data should not be reused.
-            )
-            done_part_numbers.add(part_number)
-            output.put(file_chunk)
+            # with open(file_path, "rb") as f:
+            #     f.seek(offset)
+            #     data = f.read(chunk_size)
+
+            # data = chunk_fetcher(offset, chunk_size).result()
+
+            assert curr_parth_num is not None
+            cpn: int = curr_parth_num
+
+            def on_complete(
+                fut: Future[bytes | Exception],
+                part_number: int = cpn,
+                file_path: Path = file_path,
+            ) -> None:
+                data: bytes | Exception = fut.result()
+                if isinstance(data, Exception):
+                    warnings.warn(
+                        f"Error reading file: {data}, skipping part {part_number}"
+                    )
+                    return
+
+                if isinstance(data, Exception):
+                    err: Exception = data
+                    warnings.warn(
+                        f"Error reading file: {err}, skipping part {part_number}"
+                    )
+                    return
+
+                if not data:
+                    warnings.warn(f"Empty data for part {part_number} of {file_path}")
+
+                file_chunk = FileChunk(
+                    src,
+                    upload_id=upload_info.upload_id,
+                    part_number=part_number,
+                    data=data,  # After this, data should not be reused.
+                )
+                done_part_numbers.add(part_number)
+                output.put(file_chunk)
+
+            fut = chunk_fetcher(curr_parth_num, chunk_size)
+            fut.add_done_callback(on_complete)
+            # wait until the output queue can accept the next chunk
+            while output.full():
+                time.sleep(0.1)
     except Exception as e:
 
         warnings.warn(f"Error reading file: {e}")

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api/s3/types.py

@@ -1,6 +1,8 @@
+from concurrent.futures import Future
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
+from typing import Callable
 
 
 class S3Provider(Enum):
@@ -45,6 +47,7 @@ class S3MutliPartUploadConfig:
 
     chunk_size: int
     retries: int
+    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]]
     resume_path_json: Path
     max_write_threads: int
     max_chunks_before_suspension: int | None = None

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api/s3/upload_file_multipart.py

@@ -2,10 +2,11 @@ import _thread
 import os
 import traceback
 import warnings
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
 from queue import Queue
 from threading import Event, Thread
+from typing import Callable
 
 from botocore.client import BaseClient
 
@@ -117,6 +118,7 @@ def _abort_previous_upload(upload_state: UploadState) -> None:
 
 def upload_file_multipart(
     s3_client: BaseClient,
+    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
     bucket_name: str,
     file_path: Path,
     object_name: str,
@@ -169,7 +171,7 @@ def upload_file_multipart(
         )
         return upload_state
 
-    filechunks: Queue[FileChunk | None] = Queue(10)
+    filechunks: Queue[FileChunk | None] = Queue(upload_threads)
     new_state = make_new_state()
     loaded_state = get_upload_state()
 
@@ -211,6 +213,7 @@ def upload_file_multipart(
 
     def chunker_task(
         upload_state=upload_state,
+        chunk_fetcher=chunk_fetcher,
         output=filechunks,
         max_chunks=max_chunks_before_suspension,
         cancel_signal=cancel_chunker_event,
@@ -219,6 +222,7 @@ def upload_file_multipart(
         try:
             file_chunker(
                 upload_state=upload_state,
+                chunk_fetcher=chunk_fetcher,
                 output=output,
                 max_chunks=max_chunks,
                 cancel_signal=cancel_signal,

{rclone_api-1.1.95 → rclone_api-1.1.97}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.95
+Version: 1.1.97
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.95 → rclone_api-1.1.97}/tests/test_copy_file_resumable_s3.py

@@ -42,7 +42,6 @@ from rclone_api import Rclone, SizeSuffix
 _HERE = Path(__file__).parent
 _PROJECT_ROOT = _HERE.parent
 _CONFIG_PATH = _PROJECT_ROOT / "rclone-mounted-ranged-download.conf"
-_IS_WINDOWS = os.name == "nt"
 
 _CHUNK_SIZE = 1024 * 1024 * 16
 

{rclone_api-1.1.95 → rclone_api-1.1.97}/tests/test_s3.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 import unittest
+from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -65,6 +66,20 @@ class RcloneS3Tester(unittest.TestCase):
             filesize = tmpfile.stat().st_size
             state_json = Path(tempdir) / "state.json"
 
+            def simple_fetcher(
+                chunk_id: int, chunk_size: int
+            ) -> Future[bytes | Exception]:
+                with ThreadPoolExecutor() as executor:
+
+                    def task() -> bytes | Exception:
+                        with open(str(tmpfile), "rb") as f:
+                            idx = (chunk_id - 1) * chunk_size
+                            f.seek(idx * chunk_size)
+                            return f.read(chunk_size)
+
+                    fut = executor.submit(task)
+                return fut
+
             upload_target: S3UploadTarget = S3UploadTarget(
                 src_file=Path(tmpfile),
                 bucket_name=BUCKET_NAME,
@@ -74,6 +89,7 @@ class RcloneS3Tester(unittest.TestCase):
             # Uploads one chunk then stops.
             upload_config_partial: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
+                chunk_fetcher=simple_fetcher,
                 max_write_threads=16,
                 retries=0,
                 resume_path_json=state_json,
@@ -83,6 +99,7 @@ class RcloneS3Tester(unittest.TestCase):
             # Finishes the upload.
             upload_config_all: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
+                chunk_fetcher=simple_fetcher,
                 max_write_threads=16,
                 retries=0,
                 resume_path_json=state_json,
@@ -96,7 +113,8 @@ class RcloneS3Tester(unittest.TestCase):
             )
             self.assertEqual(rslt, MultiUploadResult.SUSPENDED)
             rslt = s3_client.upload_file_multipart(
-                upload_target=upload_target, upload_config=upload_config_all
+                upload_target=upload_target,
+                upload_config=upload_config_all,
             )
             self.assertEqual(rslt, MultiUploadResult.UPLOADED_RESUME)
             print(f"Uploading file {f.name} to {BUCKET_NAME}")
@@ -105,7 +123,8 @@ class RcloneS3Tester(unittest.TestCase):
             print("Upload successful.")
             print(f"Uploading file {f.name} to {BUCKET_NAME}/{dst_path}")
             rslt = s3_client.upload_file_multipart(
-                upload_target=upload_target, upload_config=upload_config_all
+                upload_target=upload_target,
+                upload_config=upload_config_all,
             )
             self.assertEqual(rslt, MultiUploadResult.ALREADY_DONE)
             print("done")