rclone-api 1.1.11__tar.gz → 1.1.13__tar.gz

{rclone_api-1.1.11 → rclone_api-1.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.11
+Version: 1.1.13
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.11 → rclone_api-1.1.13}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.11"
+version = "1.1.13"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api/cmd/copy_large_s3.py

@@ -14,6 +14,7 @@ class Args:
     dst: str
     chunk_size: SizeSuffix
     threads: int
+    write_threads: int
     retries: int
     save_state_json: Path
     verbose: bool
@@ -38,7 +39,7 @@ def _parse_args() -> Args:
         "--chunk-size",
         help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
         type=str,
-        default="1G",
+        default="512MB",
     )
     parser.add_argument(
         "--threads",
@@ -46,6 +47,12 @@ def _parse_args() -> Args:
         type=int,
         default=64,
     )
+    parser.add_argument(
+        "--write-threads",
+        help="Max number of chunks to upload in parallel to the destination",
+        type=int,
+        default=64,
+    )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
         "--resume-json",
@@ -61,6 +68,7 @@ def _parse_args() -> Args:
         dst=args.dst,
         chunk_size=SizeSuffix(args.chunk_size),
         threads=args.threads,
+        write_threads=args.write_threads,
         retries=args.retries,
         save_state_json=args.resume_json,
         verbose=args.verbose,
@@ -77,7 +85,8 @@ def main() -> int:
         src=args.src,
         dst=args.dst,
         chunk_size=args.chunk_size,
-        threads=args.threads,
+        read_threads=args.threads,
+        write_threads=args.write_threads,
         # vfs_read_chunk_size=unit_chunk,
         # vfs_read_chunk_size_limit=args.chunk_size,
         # vfs_read_chunk_streams=args.threads,

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api/rclone.py

@@ -678,7 +678,8 @@ class Rclone:
         dst: str,
         save_state_json: Path,
         chunk_size: SizeSuffix | None = None,
-        threads: int = 16,
+        read_threads: int = 16,
+        write_threads: int = 16,
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -691,10 +692,10 @@ class Rclone:
 
         other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
         chunk_size = chunk_size or SizeSuffix("128M")
-        unit_chunk_size = chunk_size / threads
+        unit_chunk_size = chunk_size / read_threads
         vfs_read_chunk_size = unit_chunk_size
         vfs_read_chunk_size_limit = chunk_size
-        vfs_read_chunk_streams = threads
+        vfs_read_chunk_streams = read_threads
         vfs_disk_space_total_size = chunk_size
         assert (
             chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
@@ -773,6 +774,7 @@ class Rclone:
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
+                max_write_threads=write_threads,
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -797,6 +799,7 @@ class Rclone:
             upload_config = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
                 retries=retries,
+                max_write_threads=write_threads,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
             )

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api/s3/chunk_types.py

@@ -184,6 +184,9 @@ class UploadState:
     def is_done(self) -> bool:
         return self.remaining() == 0
 
+    def fingerprint(self) -> str:
+        return self.upload_info.fingerprint()
+
     def count(self) -> tuple[int, int]:  # count, num_chunks
         num_chunks = self.upload_info.total_chunks()
         count = 0
@@ -218,8 +221,31 @@ class UploadState:
 
     def save(self) -> None:
         with _SAVE_STATE_LOCK:
+
+            self._check_fingerprint_no_lock()
+
             self._save_no_lock()
 
+    def _check_fingerprint_no_lock(self) -> None:
+        if self.peristant is None:
+            raise ValueError("No path to save to")
+        s3_client = self.upload_info.s3_client
+        path = self.peristant
+        last_upload_state: UploadState | None = None
+        if path.exists():
+            try:
+                last_upload_state = UploadState.from_json(s3_client, path)
+            except Exception as e:
+                locked_print(f"Error loading state: {e}")
+                last_upload_state = None
+            # now check that the fingerprint is the same
+            if last_upload_state is not None:
+                curr_fingerprint = self.fingerprint()
+                if curr_fingerprint != last_upload_state.fingerprint():
+                    raise ValueError(
+                        f"Cannot save state, fingerprint changed from {curr_fingerprint} to {self.upload_info.fingerprint()}"
+                    )
+
     def _save_no_lock(self) -> None:
         assert self.peristant is not None, "No path to save to"
         self.peristant.write_text(self.to_json_str(), encoding="utf-8")

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api/s3/chunk_uploader.py

@@ -115,6 +115,7 @@ def upload_file_multipart(
     object_name: str,
     resumable_info_path: Path | None,
     chunk_size: int = 16 * 1024 * 1024,  # Default chunk size is 16MB; can be overridden
+    upload_threads: int = 16,
     retries: int = 20,
     max_chunks_before_suspension: int | None = None,
     abort_transfer_on_failure: bool = False,
@@ -197,7 +198,6 @@ def upload_file_multipart(
         )
     started_new_upload = finished == 0
     upload_info = upload_state.upload_info
-    max_workers = 8
 
     chunker_errors: Queue[Exception] = Queue()
 
@@ -220,7 +220,7 @@ def upload_file_multipart(
         thread_chunker = Thread(target=chunker_task, daemon=True)
         thread_chunker.start()
 
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with ThreadPoolExecutor(max_workers=upload_threads) as executor:
             while True:
                 file_chunk: FileChunk | None = filechunks.get()
                 if file_chunk is None:

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api/s3/types.py

@@ -46,6 +46,7 @@ class S3MutliPartUploadConfig:
     chunk_size: int
     retries: int
     resume_path_json: Path
+    max_write_threads: int
     max_chunks_before_suspension: int | None = None
     mount_path: Path | None = (
         None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.11
+Version: 1.1.13
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.11 → rclone_api-1.1.13}/src/rclone_api.egg-info/SOURCES.txt

@@ -8,7 +8,6 @@ clean
 install
 lint
 pyproject.toml
-rclone-mounted-ranged-download.conf.old3393025366
 requirements.testing.txt
 setup.py
 test

{rclone_api-1.1.11 → rclone_api-1.1.13}/tests/test_s3.py

@@ -74,6 +74,7 @@ class RcloneS3Tester(unittest.TestCase):
             # Uploads one chunk then stops.
             upload_config_partial: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
+                max_write_threads=16,
                 retries=0,
                 resume_path_json=state_json,
                 max_chunks_before_suspension=1,
@@ -82,6 +83,7 @@ class RcloneS3Tester(unittest.TestCase):
             # Finishes the upload.
             upload_config_all: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
+                max_write_threads=16,
                 retries=0,
                 resume_path_json=state_json,
                 max_chunks_before_suspension=None,