rclone-api 1.0.56__py2.py3-none-any.whl → 1.0.57__py2.py3-none-any.whl

rclone_api/rclone.py

@@ -34,8 +34,6 @@ from rclone_api.util import (
 )
 from rclone_api.walk import walk
 
-EXECUTOR = ThreadPoolExecutor(16)
-
 
 def rclone_verbose(verbose: bool | None) -> bool:
     if verbose is not None:
@@ -276,6 +274,9 @@ class Rclone:
         verbose: bool | None = None,
         checkers: int | None = None,
         transfers: int | None = None,
+        low_level_retries: int | None = None,
+        retries: int | None = None,
+        max_partition_workers: int | None = None,
         other_args: list[str] | None = None,
     ) -> list[CompletedProcess]:
         """Copy multiple files from source to destination.
@@ -283,6 +284,9 @@ class Rclone:
         Args:
             payload: Dictionary of source and destination file paths
         """
+        max_partition_workers = max_partition_workers or 8
+        low_level_retries = low_level_retries or 10
+        retries = retries or 3
         other_args = other_args or []
         checkers = checkers or 1000
         transfers = transfers or 32
@@ -303,65 +307,69 @@ class Rclone:
 
         futures: list[Future] = []
 
-        for common_prefix, files in datalists.items():
-
-            def _task(files=files) -> subprocess.CompletedProcess:
-
-                with TemporaryDirectory() as tmpdir:
-                    include_files_txt = Path(tmpdir) / "include_files.txt"
-                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
-                    if common_prefix:
-                        src_path = f"{src}/{common_prefix}"
-                        dst_path = f"{dst}/{common_prefix}"
+        with ThreadPoolExecutor(max_workers=max_partition_workers) as executor:
+            for common_prefix, files in datalists.items():
+
+                def _task(files=files) -> subprocess.CompletedProcess:
+                    with TemporaryDirectory() as tmpdir:
+                        include_files_txt = Path(tmpdir) / "include_files.txt"
+                        include_files_txt.write_text("\n".join(files), encoding="utf-8")
+                        if common_prefix:
+                            src_path = f"{src}/{common_prefix}"
+                            dst_path = f"{dst}/{common_prefix}"
+                        else:
+                            src_path = src
+                            dst_path = dst
+
+                        if verbose:
+                            nfiles = len(files)
+                            files_fqdn = [f"  {src_path}/{f}" for f in files]
+                            print(f"Copying {nfiles} files:")
+                            chunk_size = 100
+                            for i in range(0, nfiles, chunk_size):
+                                chunk = files_fqdn[i : i + chunk_size]
+                                files_str = "\n".join(chunk)
+                                print(f"{files_str}")
+                            # files_str = "\n".join(files_fqdn)
+                            # print(f"Copying {nfiles} files: \n{files_str}")
+
+                        # print(include_files_txt)
+                        cmd_list: list[str] = [
+                            "copy",
+                            src_path,
+                            dst_path,
+                            "--files-from",
+                            str(include_files_txt),
+                            "--checkers",
+                            str(checkers),
+                            "--transfers",
+                            str(transfers),
+                            "--low-level-retries",
+                            str(low_level_retries),
+                            "--retries",
+                            str(retries),
+                        ]
+                        if verbose:
+                            if not any(["-v" in x for x in other_args]):
+                                cmd_list.append("-vvvv")
+                            if not any(["--progress" in x for x in other_args]):
+                                cmd_list.append("--progress")
+                        if other_args:
+                            cmd_list += other_args
+                        out = self._run(cmd_list, capture=not verbose)
+                        return out
+
+                fut: Future = executor.submit(_task)
+                futures.append(fut)
+            for fut in futures:
+                cp: subprocess.CompletedProcess = fut.result()
+                assert cp is not None
+                out.append(CompletedProcess.from_subprocess(cp))
+                if cp.returncode != 0:
+                    if check:
+                        raise ValueError(f"Error deleting files: {cp.stderr}")
                     else:
-                        src_path = src
-                        dst_path = dst
-
-                    if verbose:
-                        nfiles = len(files)
-                        files_fqdn = [f"  {src_path}/{f}" for f in files]
-                        print(f"Copying {nfiles} files:")
-                        chunk_size = 100
-                        for i in range(0, nfiles, chunk_size):
-                            chunk = files_fqdn[i : i + chunk_size]
-                            files_str = "\n".join(chunk)
-                            print(f"{files_str}")
-                        # files_str = "\n".join(files_fqdn)
-                        # print(f"Copying {nfiles} files: \n{files_str}")
-
-                    # print(include_files_txt)
-                    cmd_list: list[str] = [
-                        "copy",
-                        src_path,
-                        dst_path,
-                        "--files-from",
-                        str(include_files_txt),
-                        "--checkers",
-                        str(checkers),
-                        "--transfers",
-                        str(transfers),
-                    ]
-                    if verbose:
-                        if not any(["-v" in x for x in other_args]):
-                            cmd_list.append("-vvvv")
-                        if not any(["--progress" in x for x in other_args]):
-                            cmd_list.append("--progress")
-                    if other_args:
-                        cmd_list += other_args
-                    out = self._run(cmd_list, capture=not verbose)
-                    return out
-
-            fut: Future = EXECUTOR.submit(_task)
-            futures.append(fut)
-        for fut in futures:
-            cp: subprocess.CompletedProcess = fut.result()
-            assert cp is not None
-            out.append(CompletedProcess.from_subprocess(cp))
-            if cp.returncode != 0:
-                if check:
-                    raise ValueError(f"Error deleting files: {cp.stderr}")
-                else:
-                    warnings.warn(f"Error deleting files: {cp.stderr}")
+                        warnings.warn(f"Error deleting files: {cp.stderr}")
         return out
 
     def copy(self, src: Dir | str, dst: Dir | str) -> CompletedProcess:
@@ -393,6 +401,7 @@ class Rclone:
         check=True,
         rmdirs=False,
         verbose: bool | None = None,
+        max_partition_workers: int | None = None,
         other_args: list[str] | None = None,
     ) -> CompletedProcess:
         """Delete a directory"""
@@ -414,48 +423,50 @@ class Rclone:
 
         futures: list[Future] = []
 
-        for remote, files in datalists.items():
-
-            def _task(
-                files=files, check=check, remote=remote
-            ) -> subprocess.CompletedProcess:
-                with TemporaryDirectory() as tmpdir:
-                    include_files_txt = Path(tmpdir) / "include_files.txt"
-                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                    # print(include_files_txt)
-                    cmd_list: list[str] = [
-                        "delete",
-                        remote,
-                        "--files-from",
-                        str(include_files_txt),
-                        "--checkers",
-                        "1000",
-                        "--transfers",
-                        "1000",
-                    ]
-                    if verbose:
-                        cmd_list.append("-vvvv")
-                    if rmdirs:
-                        cmd_list.append("--rmdirs")
-                    if other_args:
-                        cmd_list += other_args
-                    out = self._run(cmd_list, check=check)
-                if out.returncode != 0:
-                    if check:
-                        completed_processes.append(out)
-                        raise ValueError(f"Error deleting files: {out}")
-                    else:
-                        warnings.warn(f"Error deleting files: {out}")
-                return out
+        with ThreadPoolExecutor(max_workers=max_partition_workers) as executor:
+
+            for remote, files in datalists.items():
+
+                def _task(
+                    files=files, check=check, remote=remote
+                ) -> subprocess.CompletedProcess:
+                    with TemporaryDirectory() as tmpdir:
+                        include_files_txt = Path(tmpdir) / "include_files.txt"
+                        include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                        # print(include_files_txt)
+                        cmd_list: list[str] = [
+                            "delete",
+                            remote,
+                            "--files-from",
+                            str(include_files_txt),
+                            "--checkers",
+                            "1000",
+                            "--transfers",
+                            "1000",
+                        ]
+                        if verbose:
+                            cmd_list.append("-vvvv")
+                        if rmdirs:
+                            cmd_list.append("--rmdirs")
+                        if other_args:
+                            cmd_list += other_args
+                        out = self._run(cmd_list, check=check)
+                    if out.returncode != 0:
+                        if check:
+                            completed_processes.append(out)
+                            raise ValueError(f"Error deleting files: {out}")
+                        else:
+                            warnings.warn(f"Error deleting files: {out}")
+                    return out
 
-            fut: Future = EXECUTOR.submit(_task)
-            futures.append(fut)
+                fut: Future = executor.submit(_task)
+                futures.append(fut)
 
-        for fut in futures:
-            out = fut.result()
-            assert out is not None
-            completed_processes.append(out)
+            for fut in futures:
+                out = fut.result()
+                assert out is not None
+                completed_processes.append(out)
 
         return CompletedProcess(completed_processes)
 

{rclone_api-1.0.56.dist-info → rclone_api-1.0.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.56
+Version: 1.0.57
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.56.dist-info → rclone_api-1.0.57.dist-info}/RECORD

@@ -12,16 +12,16 @@ rclone_api/file.py,sha256=YtR5Y6c0YfXTS-sReOy2UgiSnafcAeO6b2hnbojBQD4,1423
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=kOHh6ysFDkxjldSwvW6KqmiADUC1yFCdrZRY57TvbGY,5328
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=AiRFXHVS9mylqT2395UHVzDVYC7jybiVAGwxr-keOlk,24463
+rclone_api/rclone.py,sha256=gfXFdGNJ_najxH_xgSX_xk5Pv9i4DslNSlXJzf1i9N8,25466
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/util.py,sha256=IWNOOcPE0xdKvehzXQ9okIppGDBYWJPIfdbUME8BFVM,4015
 rclone_api/walk.py,sha256=kca0t1GAnF6FLclN01G8NG__Qe-ggodLtAbQSHyVPng,2968
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.56.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.56.dist-info/METADATA,sha256=lFNPrMz55v0OUaYWtVHeuJo00qeKSag9e2YiK1feUPs,4489
-rclone_api-1.0.56.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.56.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.56.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.56.dist-info/RECORD,,
+rclone_api-1.0.57.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.57.dist-info/METADATA,sha256=IdoQc7o_OI7jGGTZOeFnRB_bxVYWI89XpIEAOqHSSkc,4489
+rclone_api-1.0.57.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.57.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.57.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.57.dist-info/RECORD,,