rclone-api 1.4.2__py2.py3-none-any.whl → 1.4.4__py2.py3-none-any.whl

rclone_api/detail/copy_file_parts.py

@@ -8,7 +8,6 @@ from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from tempfile import TemporaryDirectory
 
 from rclone_api.dir_listing import DirListing
 from rclone_api.http_server import HttpServer
@@ -249,6 +248,8 @@ def copy_file_parts(
     threads: int = 1,
 ) -> Exception | None:
     """Copy parts of a file from source to destination."""
+    from rclone_api.util import random_str
+
     if dst_dir.endswith("/"):
         dst_dir = dst_dir[:-1]
     src_size = self.size_file(src)
@@ -311,65 +312,72 @@ def copy_file_parts(
     print(info_json)
 
     finished_tasks: list[UploadPart] = []
+    tmp_dir = str(Path("chunks") / random_str(12))
+    import atexit
+    import shutil
+
+    atexit.register(lambda: shutil.rmtree(tmp_dir, ignore_errors=True))
 
     with self.serve_http(src_dir) as http_server:
-        with TemporaryDirectory() as tmp_dir:
-            tmpdir: Path = Path(tmp_dir)
-            write_semaphore = threading.Semaphore(threads)
-            with ThreadPoolExecutor(max_workers=threads) as upload_executor:
-                with ThreadPoolExecutor(max_workers=threads) as read_executor:
-                    for part_info in part_infos:
-                        part_number: int = part_info.part_number
-                        range: Range = part_info.range
-                        offset: SizeSuffix = SizeSuffix(range.start)
-                        length: SizeSuffix = SizeSuffix(range.end - range.start)
-                        end = offset + length
-                        suffix = _gen_name(part_number, offset, end)
-                        part_dst = f"{dst_dir}/{suffix}"
-
-                        def _read_task(
+        tmpdir: Path = Path(tmp_dir)
+        write_semaphore = threading.Semaphore(threads)
+        with ThreadPoolExecutor(max_workers=threads) as upload_executor:
+            with ThreadPoolExecutor(max_workers=threads) as read_executor:
+                for part_info in part_infos:
+                    part_number: int = part_info.part_number
+                    range: Range = part_info.range
+                    offset: SizeSuffix = SizeSuffix(range.start)
+                    length: SizeSuffix = SizeSuffix(range.end - range.start)
+                    end = offset + length
+                    suffix = _gen_name(part_number, offset, end)
+                    part_dst = f"{dst_dir}/{suffix}"
+
+                    def _read_task(
+                        src_name=src_name,
+                        http_server=http_server,
+                        tmpdir=tmpdir,
+                        offset=offset,
+                        length=length,
+                        part_dst=part_dst,
+                    ) -> UploadPart:
+                        return read_task(
                             src_name=src_name,
                             http_server=http_server,
                             tmpdir=tmpdir,
                             offset=offset,
                             length=length,
                             part_dst=part_dst,
-                        ) -> UploadPart:
-                            return read_task(
-                                src_name=src_name,
-                                http_server=http_server,
-                                tmpdir=tmpdir,
-                                offset=offset,
-                                length=length,
-                                part_dst=part_dst,
-                            )
-
-                        read_fut: Future[UploadPart] = read_executor.submit(_read_task)
-
-                        # Releases the semaphore when the write task is done
-                        def queue_upload_task(
-                            read_fut=read_fut,
-                        ) -> None:
-                            upload_part = read_fut.result()
-                            upload_fut: Future[UploadPart] = upload_executor.submit(
-                                upload_task, self, upload_part
-                            )
-                            # SEMAPHORE RELEASE!!!
-                            upload_fut.add_done_callback(
-                                lambda _: write_semaphore.release()
-                            )
-                            upload_fut.add_done_callback(
-                                lambda fut: finished_tasks.append(fut.result())
-                            )
-
-                        read_fut.add_done_callback(queue_upload_task)
-                        # SEMAPHORE ACQUIRE!!!
-                        # If we are back filled on the writers, then we stall.
-                        write_semaphore.acquire()
+                        )
+
+                    read_fut: Future[UploadPart] = read_executor.submit(_read_task)
+
+                    # Releases the semaphore when the write task is done
+                    def queue_upload_task(
+                        read_fut=read_fut,
+                    ) -> None:
+                        upload_part = read_fut.result()
+                        upload_fut: Future[UploadPart] = upload_executor.submit(
+                            upload_task, self, upload_part
+                        )
+                        # SEMAPHORE RELEASE!!!
+                        upload_fut.add_done_callback(
+                            lambda _: write_semaphore.release()
+                        )
+                        upload_fut.add_done_callback(
+                            lambda fut: finished_tasks.append(fut.result())
+                        )
+
+                    read_fut.add_done_callback(queue_upload_task)
+                    # SEMAPHORE ACQUIRE!!!
+                    # If we are back filled on the writers, then we stall.
+                    write_semaphore.acquire()
 
     exceptions: list[Exception] = [
         t.exception for t in finished_tasks if t.exception is not None
     ]
+
+    shutil.rmtree(tmp_dir, ignore_errors=True)
+
     if len(exceptions) > 0:
         return Exception(f"Failed to copy parts: {exceptions}", exceptions)
 

rclone_api/rclone_impl.py

@@ -1234,7 +1234,19 @@ class RcloneImpl:
         """
         addr = addr or f"localhost:{find_free_port()}"
         _, subpath = src.split(":", 1)  # might not work on local paths.
-        cmd_list: list[str] = ["serve", "http", "--addr", addr, src]
+        cmd_list: list[str] = [
+            "serve",
+            "http",
+            "--addr",
+            addr,
+            src,
+            "--vfs-disk-space-total-size",
+            "0",
+            "--vfs-read-chunk-size-limit",
+            "512M",
+            "--vfs-cache-mode",
+            "off",
+        ]
         if serve_http_log:
             cmd_list += ["--log-file", str(serve_http_log)]
             cmd_list += ["-vvvv"]

{rclone_api-1.4.2.dist-info → rclone_api-1.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.2
+Version: 1.4.4
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.2.dist-info → rclone_api-1.4.4.dist-info}/RECORD

@@ -18,7 +18,7 @@ rclone_api/http_server.py,sha256=YrILbxK0Xpnpkm8l9uGEXH_AOCGPUX8Cx33jKVtu7ZM,820
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
 rclone_api/process.py,sha256=BGXJTZVT__jeaDyjN8_kRycliOhkBErMPdHO1hKRvJE,5271
-rclone_api/rclone_impl.py,sha256=BAnRi4jB_TLRHwZE_mQfX7b1L1SODn3XhdHL7hozTAs,47960
+rclone_api/rclone_impl.py,sha256=HTvWbD0cWhqrIfUPzqCDp3Av1uCEBOoVO0BQrBYmjQM,48205
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
@@ -32,7 +32,7 @@ rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
 rclone_api/db/db.py,sha256=YRnYrCaXHwytQt07uEZ_mMpvPHo9-0IWcOb95fVOOfs,10086
 rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
-rclone_api/detail/copy_file_parts.py,sha256=KgLl5vVF67RLqZ1aUmRR98BvoqTrbcFzPY9oZd0yyI0,12316
+rclone_api/detail/copy_file_parts.py,sha256=2XA2jLCdxkenqEVCFOo3yNrYEkziNTU_Mlsp0Ra-vkg,12291
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
@@ -47,9 +47,9 @@ rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaR
 rclone_api/s3/multipart/finished_piece.py,sha256=TcwA58-qgKBiskfHrePoCWaSSep6Za9psZEpzrLUUhE,1199
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.2.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.2.dist-info/METADATA,sha256=oRC3WBMs6KdWFK8NOJOaa2LjKC4ze4tKy7fQlU63uIA,4627
-rclone_api-1.4.2.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.2.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.2.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.2.dist-info/RECORD,,
+rclone_api-1.4.4.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.4.dist-info/METADATA,sha256=qUjt3zZ0FKik4AqUkdvWOGhIrShXrgFOwEsfROcVRSE,4627
+rclone_api-1.4.4.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.4.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.4.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.4.dist-info/RECORD,,