rclone-api 1.1.10__py2.py3-none-any.whl → 1.1.12__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3.py

@@ -14,6 +14,7 @@ class Args:
     dst: str
     chunk_size: SizeSuffix
     threads: int
+    write_threads: int
     retries: int
     save_state_json: Path
     verbose: bool
@@ -38,7 +39,7 @@ def _parse_args() -> Args:
         "--chunk-size",
         help="Chunk size that will be read and uploaded in in SizeSuffix (i.e. 128M = 128 megabytes) form",
         type=str,
-        default="1G",
+        default="512MB",
     )
     parser.add_argument(
         "--threads",
@@ -46,6 +47,12 @@ def _parse_args() -> Args:
         type=int,
         default=64,
     )
+    parser.add_argument(
+        "--write-threads",
+        help="Max number of chunks to upload in parallel to the destination",
+        type=int,
+        default=64,
+    )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
         "--resume-json",
@@ -61,6 +68,7 @@ def _parse_args() -> Args:
         dst=args.dst,
         chunk_size=SizeSuffix(args.chunk_size),
         threads=args.threads,
+        write_threads=args.write_threads,
         retries=args.retries,
         save_state_json=args.resume_json,
         verbose=args.verbose,
@@ -77,7 +85,8 @@ def main() -> int:
         src=args.src,
         dst=args.dst,
         chunk_size=args.chunk_size,
-        threads=args.threads,
+        read_threads=args.threads,
+        write_threads=args.write_threads,
         # vfs_read_chunk_size=unit_chunk,
         # vfs_read_chunk_size_limit=args.chunk_size,
         # vfs_read_chunk_streams=args.threads,

rclone_api/rclone.py

@@ -678,7 +678,8 @@ class Rclone:
         dst: str,
         save_state_json: Path,
         chunk_size: SizeSuffix | None = None,
-        threads: int = 16,
+        read_threads: int = 16,
+        write_threads: int = 16,
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -691,10 +692,10 @@ class Rclone:
 
         other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
         chunk_size = chunk_size or SizeSuffix("128M")
-        unit_chunk_size = chunk_size / threads
+        unit_chunk_size = chunk_size / read_threads
         vfs_read_chunk_size = unit_chunk_size
         vfs_read_chunk_size_limit = chunk_size
-        vfs_read_chunk_streams = threads
+        vfs_read_chunk_streams = read_threads
         vfs_disk_space_total_size = chunk_size
         assert (
             chunk_size.as_int() % vfs_read_chunk_size.as_int() == 0
@@ -773,6 +774,7 @@ class Rclone:
             client = S3Client(s3_creds)
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
+                max_write_threads=write_threads,
                 retries=retries,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
@@ -797,6 +799,7 @@ class Rclone:
             upload_config = S3MutliPartUploadConfig(
                 chunk_size=chunk_size.as_int(),
                 retries=retries,
+                max_write_threads=write_threads,
                 resume_path_json=save_state_json,
                 max_chunks_before_suspension=max_chunks_before_suspension,
             )

rclone_api/s3/chunk_types.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import os
 import time
@@ -103,6 +104,19 @@ class UploadInfo:
             return
         self._total_chunks = self.total_chunks()
 
+    def fingerprint(self) -> str:
+        # hash the attributes that are used to identify the upload
+        hasher = hashlib.sha256()
+        # first is file size
+        hasher.update(str(self.file_size).encode("utf-8"))
+        # second is the file path
+        hasher.update(str(self.src_file_path).encode("utf-8"))
+        # next is chunk size
+        hasher.update(str(self.chunk_size).encode("utf-8"))
+        # next is the number of parts
+        hasher.update(str(self._total_chunks).encode("utf-8"))
+        return hasher.hexdigest()
+
     def to_json(self) -> dict:
         json_dict = {}
         for f in fields(self):

rclone_api/s3/chunk_uploader.py

@@ -115,6 +115,7 @@ def upload_file_multipart(
     object_name: str,
     resumable_info_path: Path | None,
     chunk_size: int = 16 * 1024 * 1024,  # Default chunk size is 16MB; can be overridden
+    upload_threads: int = 16,
     retries: int = 20,
     max_chunks_before_suspension: int | None = None,
     abort_transfer_on_failure: bool = False,
@@ -162,7 +163,25 @@ def upload_file_multipart(
         return upload_state
 
     filechunks: Queue[FileChunk | None] = Queue(10)
-    upload_state = get_upload_state() or make_new_state()
+    new_state = make_new_state()
+    loaded_state = get_upload_state()
+
+    if loaded_state is None:
+        upload_state = new_state
+    else:
+        # if the file size has changed, we cannot resume
+        if (
+            loaded_state.upload_info.fingerprint()
+            != new_state.upload_info.fingerprint()
+        ):
+            locked_print(
+                f"Cannot resume upload: file size changed, starting over for {file_path}"
+            )
+            _abort_previous_upload(loaded_state)
+            upload_state = new_state
+        else:
+            upload_state = loaded_state
+
     try:
         upload_state.update_source_file(file_path)
     except ValueError as e:
@@ -179,7 +198,6 @@ def upload_file_multipart(
         )
     started_new_upload = finished == 0
     upload_info = upload_state.upload_info
-    max_workers = 8
 
     chunker_errors: Queue[Exception] = Queue()
 
@@ -202,7 +220,7 @@ def upload_file_multipart(
         thread_chunker = Thread(target=chunker_task, daemon=True)
         thread_chunker.start()
 
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with ThreadPoolExecutor(max_workers=upload_threads) as executor:
             while True:
                 file_chunk: FileChunk | None = filechunks.get()
                 if file_chunk is None:

rclone_api/s3/types.py

@@ -46,6 +46,7 @@ class S3MutliPartUploadConfig:
     chunk_size: int
     retries: int
     resume_path_json: Path
+    max_write_threads: int
     max_chunks_before_suspension: int | None = None
     mount_path: Path | None = (
         None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically

{rclone_api-1.1.10.dist-info → rclone_api-1.1.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.10
+Version: 1.1.12
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.10.dist-info → rclone_api-1.1.12.dist-info}/RECORD

@@ -13,7 +13,7 @@ rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/mount.py,sha256=Xj3BMVSDEwUbtMi8ycn5mUom-fAf-F9lJOjR_BzVllw,6073
 rclone_api/process.py,sha256=xYUgU17txkZfZdr4vtRfvD8YjvSfdrbjM7PYW1npAMI,4264
-rclone_api/rclone.py,sha256=BA3DJvCraLIrglDGSi1mF-wCH8rsqpBzc-5O-IWQpq8,39809
+rclone_api/rclone.py,sha256=MelTVgBhV-JQdDUMLYusEuNSOq-RyrZCcKkkkSpsRfA,39955
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -21,20 +21,20 @@ rclone_api/types.py,sha256=zfTb0iM6mhfqgaYS6j6T0NIOA4e9GymNOXLPhVELe4A,3853
 rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/copy_large_s3.py,sha256=UR5qkW9vLtGLC951Om8UWFcQKApZUF-SzUhlIo3YIGU,3069
+rclone_api/cmd/copy_large_s3.py,sha256=3GVr7ZCLyMnVhDysCRGzsd6zlsK_5MUF0kHFZ_i6YbQ,3358
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
 rclone_api/s3/api.py,sha256=VstlaEnBjO2JDQuCRLdTfUGvQLbfshlXXhAzimFv4Vc,3763
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=XPoDl7DJMJIGBMRoPO2wqwqCMT7ZrIsEkDqlbMH8jzs,3506
-rclone_api/s3/chunk_types.py,sha256=6_ythhT7EcchJ9MI8qBKVKh-SVOe1Rf15t_JepHPilA,8587
-rclone_api/s3/chunk_uploader.py,sha256=KO8834Gery9HKWSqjQTNW0pbBbVoGrza9gj-1OaNLQQ,9130
+rclone_api/s3/chunk_types.py,sha256=pZUKc3L418XTwET1rLm4eRo_2swKQckSqlc4XjYmhDo,9147
+rclone_api/s3/chunk_uploader.py,sha256=7Qumc2emQ-SpbnTL8PBN4ElpghnFWJfozVHPhm6LrQ4,9683
 rclone_api/s3/create.py,sha256=SK3IGHZwsSkoG4Zb4NCphcVg9_f7VifDKng-tExMS2s,3088
-rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
-rclone_api-1.1.10.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.10.dist-info/METADATA,sha256=3djgVtUHz682-1HmO5v6MNSCA7-2Iw2-qD6KXlVbUcc,4479
-rclone_api-1.1.10.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.10.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.1.10.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.10.dist-info/RECORD,,
+rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
+rclone_api-1.1.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.12.dist-info/METADATA,sha256=Gs7HUdcg-9dw534zO3eyi5hCNyMXBD0Js5P1fe94mnM,4479
+rclone_api-1.1.12.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.12.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.1.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.12.dist-info/RECORD,,