rclone-api 1.0.39__py2.py3-none-any.whl → 1.0.41__py2.py3-none-any.whl

rclone_api/rclone.py

@@ -5,6 +5,7 @@ Unit test file.
 import subprocess
 import time
 import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
 from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
@@ -23,9 +24,17 @@ from rclone_api.file import File
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.util import get_rclone_exe, partition_files, to_path, wait_for_mount
+from rclone_api.util import (
+    get_rclone_exe,
+    get_verbose,
+    partition_files,
+    to_path,
+    wait_for_mount,
+)
 from rclone_api.walk import walk
 
+EXECUTOR = ThreadPoolExecutor(16)
+
 
 class ModTimeStrategy(Enum):
     USE_SERVER_MODTIME = "use-server-modtime"
@@ -197,7 +206,7 @@ class Rclone:
         cmd_list: list[str] = ["copyto", src, dst]
         self._run(cmd_list)
 
-    def copyfiles(self, files: str | File | list[str] | list[File]) -> None:
+    def copyfiles(self, files: str | File | list[str] | list[File], check=True) -> None:
         """Copy multiple files from source to destination.
 
         Warning - slow.
@@ -212,28 +221,39 @@ class Rclone:
         datalists: dict[str, list[str]] = partition_files(payload)
         out: subprocess.CompletedProcess | None = None
 
+        futures: list[Future] = []
+
         for remote, files in datalists.items():
-            with TemporaryDirectory() as tmpdir:
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                # print(include_files_txt)
-                cmd_list: list[str] = [
-                    "delete",
-                    remote,
-                    "--files-from",
-                    str(include_files_txt),
-                    "--checkers",
-                    "1000",
-                    "--transfers",
-                    "1000",
-                ]
-                out = self._run(cmd_list)
-                if out.returncode != 0:
-                    print(out)
-                    raise ValueError(f"Error deleting files: {out.stderr}")
 
-        assert out is not None
+            def _task(files=files) -> subprocess.CompletedProcess:
+                with TemporaryDirectory() as tmpdir:
+                    include_files_txt = Path(tmpdir) / "include_files.txt"
+                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                    # print(include_files_txt)
+                    cmd_list: list[str] = [
+                        "delete",
+                        remote,
+                        "--files-from",
+                        str(include_files_txt),
+                        "--checkers",
+                        "1000",
+                        "--transfers",
+                        "1000",
+                    ]
+                    out = self._run(cmd_list)
+                    return out
+
+            fut: Future = EXECUTOR.submit(_task)
+            futures.append(fut)
+        for fut in futures:
+            out = fut.result()
+            assert out is not None
+            if out.returncode != 0:
+                if check:
+                    raise ValueError(f"Error deleting files: {out.stderr}")
+                else:
+                    warnings.warn(f"Error deleting files: {out.stderr}")
 
     def copy(self, src: Dir | str, dst: Dir | str) -> CompletedProcess:
         """Copy files from source to destination.
@@ -263,6 +283,7 @@ class Rclone:
         files: str | File | list[str] | list[File],
         check=True,
         rmdirs=False,
+        verbose: bool | None = None,
         other_args: list[str] | None = None,
     ) -> CompletedProcess:
         """Delete a directory"""
@@ -277,40 +298,52 @@ class Rclone:
             return CompletedProcess.from_subprocess(cp)
 
         datalists: dict[str, list[str]] = partition_files(payload)
-        out: subprocess.CompletedProcess | None = None
-
         completed_processes: list[subprocess.CompletedProcess] = []
+        verbose = get_verbose(verbose)
+
+        futures: list[Future] = []
 
         for remote, files in datalists.items():
-            with TemporaryDirectory() as tmpdir:
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                # print(include_files_txt)
-                cmd_list: list[str] = [
-                    "delete",
-                    remote,
-                    "--files-from",
-                    str(include_files_txt),
-                    "--checkers",
-                    "1000",
-                    "--transfers",
-                    "1000",
-                ]
-                if rmdirs:
-                    cmd_list.append("--rmdirs")
-                if other_args:
-                    cmd_list += other_args
-                out = self._run(cmd_list)
-                completed_processes.append(out)
+
+            def _task(files=files, check=check) -> subprocess.CompletedProcess:
+                with TemporaryDirectory() as tmpdir:
+                    include_files_txt = Path(tmpdir) / "include_files.txt"
+                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                    # print(include_files_txt)
+                    cmd_list: list[str] = [
+                        "delete",
+                        remote,
+                        "--files-from",
+                        str(include_files_txt),
+                        "--checkers",
+                        "1000",
+                        "--transfers",
+                        "1000",
+                    ]
+                    if verbose:
+                        cmd_list.append("-vvvv")
+                    if rmdirs:
+                        cmd_list.append("--rmdirs")
+                    if other_args:
+                        cmd_list += other_args
+                    out = self._run(cmd_list, check=check)
                 if out.returncode != 0:
                     if check:
                         completed_processes.append(out)
                         raise ValueError(f"Error deleting files: {out}")
                     else:
                         warnings.warn(f"Error deleting files: {out}")
+                return out
+
+            fut: Future = EXECUTOR.submit(_task)
+            futures.append(fut)
+
+        for fut in futures:
+            out = fut.result()
+            assert out is not None
+            completed_processes.append(out)
 
-        assert out is not None
         return CompletedProcess(completed_processes)
 
     @deprecated("delete_files")

rclone_api/util.py

@@ -132,15 +132,12 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
 
 
 def partition_files(files: list[str]) -> dict[str, list[str]]:
+    """split between filename and parent directory path"""
     datalists: dict[str, list[str]] = {}
     for f in files:
-        remote, path = f.split(":", 1)
-        if "/" in path:
-            bucket, path = path.split("/", 1)
-            remote = f"{remote}:{bucket}"
-        else:
-            remote = f"{remote}:"
-        if remote not in datalists:
-            datalists[remote] = []
-        datalists[remote].append(path)
+        base = os.path.basename(f)
+        parent_path = os.path.dirname(f)
+        if parent_path not in datalists:
+            datalists[parent_path] = []
+        datalists[parent_path].append(base)
     return datalists

{rclone_api-1.0.39.dist-info → rclone_api-1.0.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.39
+Version: 1.0.41
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.39.dist-info → rclone_api-1.0.41.dist-info}/RECORD

@@ -11,16 +11,16 @@ rclone_api/exec.py,sha256=HWmnU2Jwb-3EttSbAJSaLloYA7YI2mHTzRJ5VEri9aM,941
 rclone_api/file.py,sha256=D02iHJW1LhfOiM_R_yPHP8_ApnDiYrkuraVcrV8-qkw,1246
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=dU2MEiMXC6l_QrvnWkSFpJ0sC7xyyNXCIlhEAzw2puA,19909
+rclone_api/rclone.py,sha256=hPR1dV4s-ekruVBbLHpojNvNLreJVdm3tWVOHsSDh2c,20986
 rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
-rclone_api/util.py,sha256=N1n2Fxt-pU_xKwQdwXM3zAThBHG0zwDQY30s0iQao-0,4374
+rclone_api/util.py,sha256=yUN7afH4PuuUNcrXwdtyDTIluvkNOw-c5v5VPKDXoBM,4325
 rclone_api/walk.py,sha256=kca0t1GAnF6FLclN01G8NG__Qe-ggodLtAbQSHyVPng,2968
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.39.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.39.dist-info/METADATA,sha256=2qM0byVfg7Cj6Rh5uSfYPUyQiv8QNUWMj2j44tS5GX4,4489
-rclone_api-1.0.39.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.39.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.39.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.39.dist-info/RECORD,,
+rclone_api-1.0.41.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.41.dist-info/METADATA,sha256=GnbL4D29d2oXbfexuymEQT9P1-PVEQJtoxxAd8YCa3A,4489
+rclone_api-1.0.41.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+rclone_api-1.0.41.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
+rclone_api-1.0.41.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.41.dist-info/RECORD,,