rclone-api 1.4.3__py2.py3-none-any.whl → 1.4.5__py2.py3-none-any.whl

rclone_api/detail/copy_file_parts.py

@@ -8,8 +8,8 @@ from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from tempfile import TemporaryDirectory
 
+from rclone_api import rclone_verbose
 from rclone_api.dir_listing import DirListing
 from rclone_api.http_server import HttpServer
 from rclone_api.rclone_impl import RcloneImpl
@@ -19,6 +19,8 @@ from rclone_api.types import (
     SizeSuffix,
 )
 
+rclone_verbose(True)
+
 
 @dataclass
 class UploadPart:
@@ -47,6 +49,11 @@ def upload_task(self: RcloneImpl, upload_part: UploadPart) -> UploadPart:
     try:
         if upload_part.exception is not None:
             return upload_part
+        # print(f"Uploading {upload_part.chunk} to {upload_part.dst_part}")
+        msg = "\n#########################################\n"
+        msg += f"# Uploading {upload_part.chunk} to {upload_part.dst_part}\n"
+        msg += "#########################################\n"
+        print
         self.copy_to(upload_part.chunk.as_posix(), upload_part.dst_part)
         return upload_part
     except Exception as e:
@@ -249,6 +256,8 @@ def copy_file_parts(
     threads: int = 1,
 ) -> Exception | None:
     """Copy parts of a file from source to destination."""
+    from rclone_api.util import random_str
+
     if dst_dir.endswith("/"):
         dst_dir = dst_dir[:-1]
     src_size = self.size_file(src)
@@ -311,65 +320,72 @@ def copy_file_parts(
     print(info_json)
 
     finished_tasks: list[UploadPart] = []
+    tmp_dir = str(Path("chunks") / random_str(12))
+    import atexit
+    import shutil
+
+    atexit.register(lambda: shutil.rmtree(tmp_dir, ignore_errors=True))
 
     with self.serve_http(src_dir) as http_server:
-        with TemporaryDirectory() as tmp_dir:
-            tmpdir: Path = Path(tmp_dir)
-            write_semaphore = threading.Semaphore(threads)
-            with ThreadPoolExecutor(max_workers=threads) as upload_executor:
-                with ThreadPoolExecutor(max_workers=threads) as read_executor:
-                    for part_info in part_infos:
-                        part_number: int = part_info.part_number
-                        range: Range = part_info.range
-                        offset: SizeSuffix = SizeSuffix(range.start)
-                        length: SizeSuffix = SizeSuffix(range.end - range.start)
-                        end = offset + length
-                        suffix = _gen_name(part_number, offset, end)
-                        part_dst = f"{dst_dir}/{suffix}"
-
-                        def _read_task(
+        tmpdir: Path = Path(tmp_dir)
+        write_semaphore = threading.Semaphore(threads)
+        with ThreadPoolExecutor(max_workers=threads) as upload_executor:
+            with ThreadPoolExecutor(max_workers=threads) as read_executor:
+                for part_info in part_infos:
+                    part_number: int = part_info.part_number
+                    range: Range = part_info.range
+                    offset: SizeSuffix = SizeSuffix(range.start)
+                    length: SizeSuffix = SizeSuffix(range.end - range.start)
+                    end = offset + length
+                    suffix = _gen_name(part_number, offset, end)
+                    part_dst = f"{dst_dir}/{suffix}"
+
+                    def _read_task(
+                        src_name=src_name,
+                        http_server=http_server,
+                        tmpdir=tmpdir,
+                        offset=offset,
+                        length=length,
+                        part_dst=part_dst,
+                    ) -> UploadPart:
+                        return read_task(
                             src_name=src_name,
                             http_server=http_server,
                             tmpdir=tmpdir,
                             offset=offset,
                             length=length,
                             part_dst=part_dst,
-                        ) -> UploadPart:
-                            return read_task(
-                                src_name=src_name,
-                                http_server=http_server,
-                                tmpdir=tmpdir,
-                                offset=offset,
-                                length=length,
-                                part_dst=part_dst,
-                            )
-
-                        read_fut: Future[UploadPart] = read_executor.submit(_read_task)
-
-                        # Releases the semaphore when the write task is done
-                        def queue_upload_task(
-                            read_fut=read_fut,
-                        ) -> None:
-                            upload_part = read_fut.result()
-                            upload_fut: Future[UploadPart] = upload_executor.submit(
-                                upload_task, self, upload_part
-                            )
-                            # SEMAPHORE RELEASE!!!
-                            upload_fut.add_done_callback(
-                                lambda _: write_semaphore.release()
-                            )
-                            upload_fut.add_done_callback(
-                                lambda fut: finished_tasks.append(fut.result())
-                            )
-
-                        read_fut.add_done_callback(queue_upload_task)
-                        # SEMAPHORE ACQUIRE!!!
-                        # If we are back filled on the writers, then we stall.
-                        write_semaphore.acquire()
+                        )
+
+                    read_fut: Future[UploadPart] = read_executor.submit(_read_task)
+
+                    # Releases the semaphore when the write task is done
+                    def queue_upload_task(
+                        read_fut=read_fut,
+                    ) -> None:
+                        upload_part = read_fut.result()
+                        upload_fut: Future[UploadPart] = upload_executor.submit(
+                            upload_task, self, upload_part
+                        )
+                        # SEMAPHORE RELEASE!!!
+                        upload_fut.add_done_callback(
+                            lambda _: write_semaphore.release()
+                        )
+                        upload_fut.add_done_callback(
+                            lambda fut: finished_tasks.append(fut.result())
+                        )
+
+                    read_fut.add_done_callback(queue_upload_task)
+                    # SEMAPHORE ACQUIRE!!!
+                    # If we are back filled on the writers, then we stall.
+                    write_semaphore.acquire()
 
     exceptions: list[Exception] = [
         t.exception for t in finished_tasks if t.exception is not None
     ]
+
+    shutil.rmtree(tmp_dir, ignore_errors=True)
+
     if len(exceptions) > 0:
         return Exception(f"Failed to copy parts: {exceptions}", exceptions)
 

rclone_api/http_server.py

@@ -3,6 +3,7 @@ Unit test file for testing rclone mount functionality.
 """
 
 import tempfile
+import time
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from pathlib import Path
@@ -63,33 +64,46 @@ class HttpServer:
         self, path: str, dst: Path, range: Range | None = None
     ) -> Path | Exception:
         """Get bytes from the server."""
-        if not dst.parent.exists():
-            dst.parent.mkdir(parents=True, exist_ok=True)
-        headers: dict[str, str] = {}
-        if range:
-            headers.update(range.to_header())
-        url = self._get_file_url(path)
-        try:
-            with httpx.stream(
-                "GET", url, headers=headers, timeout=_TIMEOUT
-            ) as response:
-                response.raise_for_status()
-                with open(dst, "wb") as file:
-                    for chunk in response.iter_bytes(chunk_size=8192):
-                        if chunk:
-                            file.write(chunk)
-                        else:
-                            assert response.is_closed
-                # print(f"Downloaded bytes {start}-{end} to {dst}")
-                if range:
-                    print(f"Downloaded bytes {range.start}-{range.end} to {dst}")
-                else:
-                    size = dst.stat().st_size
-                    print(f"Downloaded {size} bytes to {dst}")
-                return dst
-        except Exception as e:
-            warnings.warn(f"Failed to download {url} to {dst}: {e}")
-            return e
+
+        def task() -> Path | Exception:
+
+            if not dst.parent.exists():
+                dst.parent.mkdir(parents=True, exist_ok=True)
+            headers: dict[str, str] = {}
+            if range:
+                headers.update(range.to_header())
+            url = self._get_file_url(path)
+            try:
+                with httpx.stream(
+                    "GET", url, headers=headers, timeout=_TIMEOUT
+                ) as response:
+                    response.raise_for_status()
+                    with open(dst, "wb") as file:
+                        for chunk in response.iter_bytes(chunk_size=8192):
+                            if chunk:
+                                file.write(chunk)
+                            else:
+                                assert response.is_closed
+                    # print(f"Downloaded bytes {start}-{end} to {dst}")
+                    if range:
+                        print(f"Downloaded bytes {range.start}-{range.end} to {dst}")
+                    else:
+                        size = dst.stat().st_size
+                        print(f"Downloaded {size} bytes to {dst}")
+                    return dst
+            except Exception as e:
+                warnings.warn(f"Failed to download {url} to {dst}: {e}")
+                return e
+
+        retries = 3
+        for i in _range(retries):
+            out = task()
+            if not isinstance(out, Exception):
+                return out
+            warnings.warn(f"Failed to download {path} to {dst}: {out}, retrying ({i})")
+            time.sleep(10)
+        else:
+            return Exception(f"Failed to download {path} to {dst}")
 
     def download_multi_threaded(
         self,

{rclone_api-1.4.3.dist-info → rclone_api-1.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.3
+Version: 1.4.5
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.3.dist-info → rclone_api-1.4.5.dist-info}/RECORD

@@ -14,7 +14,7 @@ rclone_api/file_part.py,sha256=i6ByS5_sae8Eba-4imBVTxd-xKC8ExWy7NR8QGr0ors,6155
 rclone_api/file_stream.py,sha256=_W3qnwCuigqA0hzXl2q5pAxSZDRaUSwet4BkT0lpnzs,1431
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/http_server.py,sha256=YrILbxK0Xpnpkm8l9uGEXH_AOCGPUX8Cx33jKVtu7ZM,8207
+rclone_api/http_server.py,sha256=3fPBV6l50erTe32DyeJBNmsDrn5KuujsbmEAbx13T-c,8720
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
 rclone_api/process.py,sha256=BGXJTZVT__jeaDyjN8_kRycliOhkBErMPdHO1hKRvJE,5271
@@ -32,7 +32,7 @@ rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
 rclone_api/db/db.py,sha256=YRnYrCaXHwytQt07uEZ_mMpvPHo9-0IWcOb95fVOOfs,10086
 rclone_api/db/models.py,sha256=v7qaXUehvsDvU51uk69JI23fSIs9JFGcOa-Tv1c_wVs,1600
-rclone_api/detail/copy_file_parts.py,sha256=KgLl5vVF67RLqZ1aUmRR98BvoqTrbcFzPY9oZd0yyI0,12316
+rclone_api/detail/copy_file_parts.py,sha256=UQ9nZQftZtUVLrofbG6Y1-rBppI8F4qE5lKAlAJC570,12642
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
@@ -47,9 +47,9 @@ rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaR
 rclone_api/s3/multipart/finished_piece.py,sha256=TcwA58-qgKBiskfHrePoCWaSSep6Za9psZEpzrLUUhE,1199
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.3.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.3.dist-info/METADATA,sha256=7YDI5CVaVfOl9Wd-vKdPGRSIXCaX6nLR1yI9LTYsWc4,4627
-rclone_api-1.4.3.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.3.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.3.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.3.dist-info/RECORD,,
+rclone_api-1.4.5.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.5.dist-info/METADATA,sha256=6U0dTVTIP3w6wYGOeDTMJDjPWOUjEe-3kft24IlVjYc,4627
+rclone_api-1.4.5.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.5.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.5.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.5.dist-info/RECORD,,