rclone-api 1.1.10__tar.gz → 1.1.12__tar.gz

{rclone_api-1.1.10 → rclone_api-1.1.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.10
+Version: 1.1.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.10 → rclone_api-1.1.12}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.10"
+version = "1.1.12"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api/cmd/copy_large_s3.py

@@ -14,6 +14,7 @@ class Args:
     dst: str
     chunk_size: SizeSuffix
     threads: int
+    write_threads: int
     retries: int
     save_state_json: Path
     verbose: bool
@@ -38,7 +39,7 @@ def _parse_args() -> Args:
         "--chunk-size",
         help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
         type=str,
-        default="1G",
+        default="512MB",
     )
     parser.add_argument(
         "--threads",
@@ -46,6 +47,12 @@ def _parse_args() -> Args:
         type=int,
         default=64,
     )
+    parser.add_argument(
+        "--write-threads",
+        help="Max number of chunks to upload in parallel to the destination",
+        type=int,
+        default=64,
+    )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
         "--resume-json",
@@ -61,6 +68,7 @@ def _parse_args() -> Args:
         dst=args.dst,
         chunk_size=SizeSuffix(args.chunk_size),
         threads=args.threads,
+        write_threads=args.write_threads,
         retries=args.retries,
         save_state_json=args.resume_json,
         verbose=args.verbose,
@@ -77,7 +85,8 @@ def main() -> int:
         src=args.src,
         dst=args.dst,
         chunk_size=args.chunk_size,
-        threads=args.threads,
+        read_threads=args.threads,
+        write_threads=args.write_threads,
         # vfs_read_chunk_size=unit_chunk,
         # vfs_read_chunk_size_limit=args.chunk_size,
         # vfs_read_chunk_streams=args.threads,

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api/rclone.py

@@ -678,7 +678,8 @@ class Rclone:
         dst: str,
         save_state_json: Path,
         chunk_size: SizeSuffix | None = None,
-        threads: int = 16,
+        read_threads: int = 16,
+        write_threads: int = 16,
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -691,10 +692,10 @@ class Rclone:
 
         other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
         chunk_size = chunk_size or SizeSuffix("128M")
-        unit_chunk_size = chunk_size / threads
+        unit_chunk_size = chunk_size / read_threads
         vfs_read_chunk_size = unit_chunk_size
         vfs_read_chunk_size_limit = chunk_size
-        vfs_read_chunk_streams = threads
+        vfs_read_chunk_streams = read_threads
         vfs_disk_space_total_size = chunk_size
         assert (
             chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
@@ -773,6 +774,7 @@ class Rclone:
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
+                max_write_threads=write_threads,
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -797,6 +799,7 @@ class Rclone:
             upload_config = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
                 retries=retries,
+                max_write_threads=write_threads,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
             )

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api/s3/chunk_types.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import time
@@ -103,6 +104,19 @@ class UploadInfo:
             return
         self._total_chunks = self.total_chunks()
 
+    def fingerprint(self) -> str:
+        # hash the attributes that are used to identify the upload
+        hasher = hashlib.sha256()
+        # first is file size
+        hasher.update(str(self.file_size).encode("utf-8"))
+        # second is the file path
+        hasher.update(str(self.src_file_path).encode("utf-8"))
+        # next is chunk size
+        hasher.update(str(self.chunk_size).encode("utf-8"))
+        # next is the number of parts
+        hasher.update(str(self._total_chunks).encode("utf-8"))
+        return hasher.hexdigest()
+
     def to_json(self) -> dict:
         json_dict = {}
         for f in fields(self):

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api/s3/chunk_uploader.py

@@ -115,6 +115,7 @@ def upload_file_multipart(
     object_name: str,
     resumable_info_path: Path | None,
     chunk_size: int = 16 * 1024 * 1024,  # Default chunk size is 16MB; can be overridden
+    upload_threads: int = 16,
     retries: int = 20,
     max_chunks_before_suspension: int | None = None,
     abort_transfer_on_failure: bool = False,
@@ -162,7 +163,25 @@ def upload_file_multipart(
         return upload_state
 
     filechunks: Queue[FileChunk | None] = Queue(10)
-    upload_state = get_upload_state() or make_new_state()
+    new_state = make_new_state()
+    loaded_state = get_upload_state()
+
+    if loaded_state is None:
+        upload_state = new_state
+    else:
+        # if the file size has changed, we cannot resume
+        if (
+            loaded_state.upload_info.fingerprint()
+            != new_state.upload_info.fingerprint()
+        ):
+            locked_print(
+                f"Cannot resume upload: file size changed, starting over for {file_path}"
+            )
+            _abort_previous_upload(loaded_state)
+            upload_state = new_state
+        else:
+            upload_state = loaded_state
+
     try:
         upload_state.update_source_file(file_path)
     except ValueError as e:
@@ -179,7 +198,6 @@ def upload_file_multipart(
         )
     started_new_upload = finished == 0
     upload_info = upload_state.upload_info
-    max_workers = 8
 
     chunker_errors: Queue[Exception] = Queue()
 
@@ -202,7 +220,7 @@ def upload_file_multipart(
         thread_chunker = Thread(target=chunker_task, daemon=True)
         thread_chunker.start()
 
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with ThreadPoolExecutor(max_workers=upload_threads) as executor:
             while True:
                 file_chunk: FileChunk | None = filechunks.get()
                 if file_chunk is None:

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api/s3/types.py

@@ -46,6 +46,7 @@ class S3MutliPartUploadConfig:
     chunk_size: int
     retries: int
     resume_path_json: Path
+    max_write_threads: int
     max_chunks_before_suspension: int | None = None
     mount_path: Path | None = (
         None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.10
+Version: 1.1.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.10 → rclone_api-1.1.12}/src/rclone_api.egg-info/SOURCES.txt

@@ -8,7 +8,6 @@ clean
 install
 lint
 pyproject.toml
-rclone-mounted-ranged-download.conf.old3393025366
 requirements.testing.txt
 setup.py
 test

{rclone_api-1.1.10 → rclone_api-1.1.12}/tests/test_s3.py

@@ -74,6 +74,7 @@ class RcloneS3Tester(unittest.TestCase):
             # Uploads one chunk then stops.
             upload_config_partial: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
+                max_write_threads=16,
                 retries=0,
                 resume_path_json=state_json,
                 max_chunks_before_suspension=1,
@@ -82,6 +83,7 @@ class RcloneS3Tester(unittest.TestCase):
             # Finishes the upload.
             upload_config_all: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
+                max_write_threads=16,
                 retries=0,
                 resume_path_json=state_json,
                 max_chunks_before_suspension=None,