rclone-api 1.1.94__py2.py3-none-any.whl → 1.1.96__py2.py3-none-any.whl

rclone_api/profile/mount_copy_bytes.py

@@ -130,27 +130,23 @@ def _run_profile(
     )
     bytes_count = 0
 
+    futures: list[Future[bytes | Exception]] = []
+    for i in range(num):
+        offset = SizeSuffix(i * chunk_size.as_int()) + offset
+        future = filechunker.fetch(offset.as_int(), size.as_int())
+        futures.append(future)
+
+    for future in futures:
+        bytes_or_err = future.result()
+        if isinstance(bytes_or_err, Exception):
+            assert False, f"Error: {bytes_or_err}"
+    futures.clear()
 
     start = time.time()
     net_io_start = psutil.net_io_counters()
 
-    # bytes_or_err: bytes | Exception = rclone.copy_bytes(
-    #     src=src_file,
-    #     offset=offset.as_int(),
-    #     length=size.as_int(),
-    #     chunk_size=chunk_size,
-    #     direct_io=direct_io,
-    #     max_threads=transfers,
-    #     mount_log=mount_log,
-    # )
-
-    # bytes_or_err = filechunker.fetch(
-    #     (offset + SizeSuffix("1G")).as_int(), size.as_int()
-    # )
-
     offset = SizeSuffix("1G")
 
-    futures: list[Future[bytes | Exception]] = []
     for i in range(num):
         offset = SizeSuffix(i * chunk_size.as_int()) + offset
         future = filechunker.fetch(offset.as_int(), size.as_int())
@@ -165,7 +161,9 @@ def _run_profile(
     diff = (time.time() - start) / num
     net_io_end = psutil.net_io_counters()
     # self.assertEqual(len(bytes_or_err), size)
-    assert bytes_count  == SizeSuffix(size* num).as_int(), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
+    assert (
+        bytes_count == SizeSuffix(size * num).as_int()
+    ), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
 
     # print io stats
     bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num

rclone_api/rclone.py

@@ -777,9 +777,18 @@ class Rclone:
                 endpoint_url=section.endpoint(),
             )
 
+            chunk_fetcher: MultiMountFileChunker = self.get_multi_mount_file_chunker(
+                src=src_path.parent.as_posix(),
+                chunk_size=chunk_size,
+                threads=read_threads,
+                mount_log=mount_log,
+                direct_io=True,
+            )
+
             client = S3Client(s3_creds)
-            config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
+            upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
+                chunk_fetcher=chunk_fetcher.fetch,
                 max_write_threads=write_threads,
                 retries=retries,
                 resume_path_json=save_state_json,
@@ -791,10 +800,7 @@ class Rclone:
             print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
             print(f"Source: {src_path}")
             print(f"bucket_name: {bucket_name}")
-            print(f"upload_config: {config}")
-
-            upload_target: S3UploadTarget
-            upload_config: S3MutliPartUploadConfig
+            print(f"upload_config: {upload_config}")
 
             upload_target = S3UploadTarget(
                 bucket_name=bucket_name,
@@ -802,18 +808,14 @@ class Rclone:
                 s3_key=s3_key,
             )
 
-            upload_config = S3MutliPartUploadConfig(
-                chunk_size=chunk_size.as_int(),
-                retries=retries,
-                max_write_threads=write_threads,
-                resume_path_json=save_state_json,
-                max_chunks_before_suspension=max_chunks_before_suspension,
-            )
-
-            out: MultiUploadResult = client.upload_file_multipart(
-                upload_target=upload_target, upload_config=upload_config
-            )
-            return out
+            try:
+                out: MultiUploadResult = client.upload_file_multipart(
+                    upload_target=upload_target,
+                    upload_config=upload_config,
+                )
+                return out
+            finally:
+                chunk_fetcher.close()
 
     def _copy_bytes(
         self,

rclone_api/s3/api.py

@@ -11,7 +11,10 @@ from rclone_api.s3.basic_ops import (
 )
 from rclone_api.s3.create import create_s3_client
 from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
-from rclone_api.s3.upload_file_multipart import MultiUploadResult, upload_file_multipart
+from rclone_api.s3.upload_file_multipart import (
+    MultiUploadResult,
+    upload_file_multipart,
+)
 
 _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 
@@ -67,6 +70,7 @@ class S3Client:
 
             out = upload_file_multipart(
                 s3_client=self.client,
+                chunk_fetcher=upload_config.chunk_fetcher,
                 bucket_name=bucket_name,
                 file_path=upload_target.src_file,
                 object_name=upload_target.s3_key,

rclone_api/s3/chunk_file.py

@@ -1,8 +1,10 @@
 import time
 import warnings
+from concurrent.futures import Future
 from pathlib import Path
 from queue import Queue
 from threading import Event
+from typing import Callable
 
 from rclone_api.s3.chunk_types import FileChunk, UploadState
 from rclone_api.util import locked_print
@@ -27,12 +29,16 @@ def _get_file_size(file_path: Path, timeout: int = 60) -> int:
 
 def file_chunker(
     upload_state: UploadState,
+    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
     max_chunks: int | None,
     cancel_signal: Event,
     output: Queue[FileChunk | None],
 ) -> None:
     count = 0
 
+    if False:
+        print(chunk_fetcher)
+
     def should_stop() -> bool:
         nonlocal count
 
@@ -41,14 +47,6 @@ def file_chunker(
         if count >= max_chunks:
             return True
         count += 1
-        if count > 10 and count % 10 == 0:
-            # recheck that the file size has not changed
-            file_size = _get_file_size(upload_state.upload_info.src_file_path)
-            if file_size != upload_state.upload_info.file_size:
-                locked_print(
-                    f"File size changed, cannot resume, expected {upload_state.upload_info.file_size}, got {file_size}"
-                )
-                raise ValueError("File size changed, cannot resume")
         return False
 
     upload_info = upload_state.upload_info
@@ -58,7 +56,6 @@ def file_chunker(
     # Mounted files may take a while to appear, so keep retrying.
 
     try:
-        file_size = _get_file_size(src, timeout=60)
         part_number = 1
         done_part_numbers: set[int] = {
             p.part_number for p in upload_state.parts if p is not None
@@ -86,25 +83,56 @@ def file_chunker(
                 break
             assert curr_parth_num is not None
             offset = (curr_parth_num - 1) * chunk_size
+            file_size = upload_info.file_size
 
             assert offset < file_size, f"Offset {offset} is greater than file size"
 
             # Open the file, seek, read the chunk, and close immediately.
-            with open(file_path, "rb") as f:
-                f.seek(offset)
-                data = f.read(chunk_size)
-
-            if not data:
-                warnings.warn(f"Empty data for part {part_number} of {file_path}")
-
-            file_chunk = FileChunk(
-                src,
-                upload_id=upload_info.upload_id,
-                part_number=part_number,
-                data=data,  # After this, data should not be reused.
-            )
-            done_part_numbers.add(part_number)
-            output.put(file_chunk)
+            # with open(file_path, "rb") as f:
+            #     f.seek(offset)
+            #     data = f.read(chunk_size)
+
+            # data = chunk_fetcher(offset, chunk_size).result()
+
+            assert curr_parth_num is not None
+            cpn: int = curr_parth_num
+
+            def on_complete(
+                fut: Future[bytes | Exception],
+                part_number: int = cpn,
+                file_path: Path = file_path,
+            ) -> None:
+                data: bytes | Exception = fut.result()
+                if isinstance(data, Exception):
+                    warnings.warn(
+                        f"Error reading file: {data}, skipping part {part_number}"
+                    )
+                    return
+
+                if isinstance(data, Exception):
+                    err: Exception = data
+                    warnings.warn(
+                        f"Error reading file: {err}, skipping part {part_number}"
+                    )
+                    return
+
+                if not data:
+                    warnings.warn(f"Empty data for part {part_number} of {file_path}")
+
+                file_chunk = FileChunk(
+                    src,
+                    upload_id=upload_info.upload_id,
+                    part_number=part_number,
+                    data=data,  # After this, data should not be reused.
+                )
+                done_part_numbers.add(part_number)
+                output.put(file_chunk)
+
+            fut = chunk_fetcher(curr_parth_num, chunk_size)
+            fut.add_done_callback(on_complete)
+            # wait until the output queue can accept the next chunk
+            while output.full():
+                time.sleep(0.1)
     except Exception as e:
 
         warnings.warn(f"Error reading file: {e}")

rclone_api/s3/types.py

@@ -1,6 +1,8 @@
+from concurrent.futures import Future
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
+from typing import Callable
 
 
 class S3Provider(Enum):
@@ -45,6 +47,7 @@ class S3MutliPartUploadConfig:
 
     chunk_size: int
     retries: int
+    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]]
     resume_path_json: Path
     max_write_threads: int
     max_chunks_before_suspension: int | None = None

rclone_api/s3/upload_file_multipart.py

@@ -2,10 +2,11 @@ import _thread
 import os
 import traceback
 import warnings
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
 from queue import Queue
 from threading import Event, Thread
+from typing import Callable
 
 from botocore.client import BaseClient
 
@@ -117,6 +118,7 @@ def _abort_previous_upload(upload_state: UploadState) -> None:
 
 def upload_file_multipart(
     s3_client: BaseClient,
+    chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
     bucket_name: str,
     file_path: Path,
     object_name: str,
@@ -211,6 +213,7 @@ def upload_file_multipart(
 
     def chunker_task(
         upload_state=upload_state,
+        chunk_fetcher=chunk_fetcher,
         output=filechunks,
         max_chunks=max_chunks_before_suspension,
         cancel_signal=cancel_chunker_event,
@@ -219,6 +222,7 @@ def upload_file_multipart(
         try:
             file_chunker(
                 upload_state=upload_state,
+                chunk_fetcher=chunk_fetcher,
                 output=output,
                 max_chunks=max_chunks,
                 cancel_signal=cancel_signal,

{rclone_api-1.1.94.dist-info → rclone_api-1.1.96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.94
+Version: 1.1.96
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.94.dist-info → rclone_api-1.1.96.dist-info}/RECORD

@@ -13,7 +13,7 @@ rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/mount.py,sha256=pU-YTHTLYMXuNHVkT9ACS1wdZKazXXAhVWQUD6ZvP40,14535
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
-rclone_api/rclone.py,sha256=5sF9t6IRzn4GMoSynW4mfFIYweve2Aomm3J5mCPASvE,48837
+rclone_api/rclone.py,sha256=0UW3gQ93DgZjvNspxyoi_ae_PfbNq32tHb3wVpa6Eq4,48893
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -25,17 +25,17 @@ rclone_api/cmd/copy_large_s3.py,sha256=a-NtpIhy3gj1umIyS0JG1qQMCbsLQoi6VP6u2NjBG
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=-WfEv4K21Juaxqkz3O3aDJmdWM-0ZdlaFIeIJ0Tju2Y,8414
-rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
+rclone_api/profile/mount_copy_bytes.py,sha256=_bd9oergTuCdj1P6AVh_pC6GmtCpLFQYwiTdhwXeYZE,8404
+rclone_api/s3/api.py,sha256=Plhtw_R9wrhN1fNzMI14ZFkHlvKdehUpdJ07P5cw9HI,3842
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_file.py,sha256=D6wM9Nuu73LNGC8JCCfevqjF3qdZ21mQxYQClFLZLMU,3726
+rclone_api/s3/chunk_file.py,sha256=Gro77c-mHQUvW41fw3aFtSrc6ByOcdGt0_padnip6yc,4568
 rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,10316
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
-rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
-rclone_api/s3/upload_file_multipart.py,sha256=y9azNAU8QH5Ovwz33V2HZwNmJdlFjJg-jrXLZ1gtMds,10364
-rclone_api-1.1.94.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.94.dist-info/METADATA,sha256=9ghyMbSCUTVKJUJ4Qrrdipdn37g-wnXxfa5nToR_gyM,4537
-rclone_api-1.1.94.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.94.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.1.94.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.94.dist-info/RECORD,,
+rclone_api/s3/types.py,sha256=MFZkVXB-rUHa-d3b5gAt5qKPP1tv3FrUmy7Wq4HQgjc,1521
+rclone_api/s3/upload_file_multipart.py,sha256=frfz8CDW2x2DCPKW0fcZCaABq-vA8gcv26bO8mYNEf8,10550
+rclone_api-1.1.96.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.96.dist-info/METADATA,sha256=CbTS1pfpa01wYwzt-47lFLHlEaz7R92EGIbE6YsXkc8,4537
+rclone_api-1.1.96.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.96.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.1.96.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.96.dist-info/RECORD,,