rclone-api 1.1.97__tar.gz → 1.1.99__tar.gz

{rclone_api-1.1.97 → rclone_api-1.1.99}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.97
+Version: 1.1.99
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.97 → rclone_api-1.1.99}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.1.97"
+version = "1.1.99"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.1.97 → rclone_api-1.1.99}/src/rclone_api/cmd/copy_large_s3.py

@@ -45,13 +45,13 @@ def _parse_args() -> Args:
         "--read-threads",
         help="Number of concurrent read threads per chunk, only one chunk will be read at a time",
         type=int,
-        default=1,
+        default=16,
     )
     parser.add_argument(
         "--write-threads",
         help="Max number of chunks to upload in parallel to the destination, each chunk is uploaded in a separate thread",
         type=int,
-        default=4,
+        default=16,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(

{rclone_api-1.1.97 → rclone_api-1.1.99}/src/rclone_api/s3/chunk_file.py

@@ -36,9 +36,6 @@ def file_chunker(
 ) -> None:
     count = 0
 
-    if False:
-        print(chunk_fetcher)
-
     def should_stop() -> bool:
         nonlocal count
 
@@ -128,7 +125,8 @@ def file_chunker(
                 done_part_numbers.add(part_number)
                 output.put(file_chunk)
 
-            fut = chunk_fetcher(curr_parth_num, chunk_size)
+            offset = (curr_parth_num - 1) * chunk_size
+            fut = chunk_fetcher(offset, chunk_size)
             fut.add_done_callback(on_complete)
             # wait until the output queue can accept the next chunk
             while output.full():

{rclone_api-1.1.97 → rclone_api-1.1.99}/src/rclone_api/s3/upload_file_multipart.py

@@ -171,7 +171,9 @@ def upload_file_multipart(
         )
         return upload_state
 
-    filechunks: Queue[FileChunk | None] = Queue(upload_threads)
+    work_que_max = upload_threads // 2 + 2
+
+    filechunks: Queue[FileChunk | None] = Queue(work_que_max)
     new_state = make_new_state()
     loaded_state = get_upload_state()
 

{rclone_api-1.1.97 → rclone_api-1.1.99}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.97
+Version: 1.1.99
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.1.97 → rclone_api-1.1.99}/tests/test_s3.py

@@ -18,7 +18,7 @@ class RcloneS3Tester(unittest.TestCase):
     """Test rclone functionality."""
 
     # @unittest.skipIf(not _ENABLED, "Test not enabled")
-    @unittest.skipIf(True, "Test not enabled")
+    # @unittest.skipIf(True, "Test not enabled")
     def test_upload_chunks(self) -> None:
         """Test basic Webdav serve functionality."""
 
@@ -67,14 +67,13 @@ class RcloneS3Tester(unittest.TestCase):
             state_json = Path(tempdir) / "state.json"
 
             def simple_fetcher(
-                chunk_id: int, chunk_size: int
+                offset: int, chunk_size: int
             ) -> Future[bytes | Exception]:
                 with ThreadPoolExecutor() as executor:
 
                     def task() -> bytes | Exception:
                         with open(str(tmpfile), "rb") as f:
-                            idx = (chunk_id - 1) * chunk_size
-                            f.seek(idx * chunk_size)
+                            f.seek(offset)
                             return f.read(chunk_size)
 
                     fut = executor.submit(task)