rclone-api 1.0.39__tar.gz → 1.0.41__tar.gz

{rclone_api-1.0.39 → rclone_api-1.0.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.39
+Version: 1.0.41
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.39 → rclone_api-1.0.41}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.39"
+version = "1.0.41"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.39 → rclone_api-1.0.41}/src/rclone_api/rclone.py

@@ -5,6 +5,7 @@ Unit test file.
 import subprocess
 import time
 import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
 from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
@@ -23,9 +24,17 @@ from rclone_api.file import File
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.util import get_rclone_exe, partition_files, to_path, wait_for_mount
+from rclone_api.util import (
+    get_rclone_exe,
+    get_verbose,
+    partition_files,
+    to_path,
+    wait_for_mount,
+)
 from rclone_api.walk import walk
 
+EXECUTOR = ThreadPoolExecutor(16)
+
 
 class ModTimeStrategy(Enum):
     USE_SERVER_MODTIME = "use-server-modtime"
@@ -197,7 +206,7 @@ class Rclone:
         cmd_list: list[str] = ["copyto", src, dst]
         self._run(cmd_list)
 
-    def copyfiles(self, files: str | File | list[str] | list[File]) -> None:
+    def copyfiles(self, files: str | File | list[str] | list[File], check=True) -> None:
         """Copy multiple files from source to destination.
 
         Warning - slow.
@@ -212,28 +221,39 @@ class Rclone:
         datalists: dict[str, list[str]] = partition_files(payload)
         out: subprocess.CompletedProcess | None = None
 
+        futures: list[Future] = []
+
         for remote, files in datalists.items():
-            with TemporaryDirectory() as tmpdir:
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                # print(include_files_txt)
-                cmd_list: list[str] = [
-                    "delete",
-                    remote,
-                    "--files-from",
-                    str(include_files_txt),
-                    "--checkers",
-                    "1000",
-                    "--transfers",
-                    "1000",
-                ]
-                out = self._run(cmd_list)
-                if out.returncode != 0:
-                    print(out)
-                    raise ValueError(f"Error deleting files: {out.stderr}")
 
-        assert out is not None
+            def _task(files=files) -> subprocess.CompletedProcess:
+                with TemporaryDirectory() as tmpdir:
+                    include_files_txt = Path(tmpdir) / "include_files.txt"
+                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                    # print(include_files_txt)
+                    cmd_list: list[str] = [
+                        "delete",
+                        remote,
+                        "--files-from",
+                        str(include_files_txt),
+                        "--checkers",
+                        "1000",
+                        "--transfers",
+                        "1000",
+                    ]
+                    out = self._run(cmd_list)
+                    return out
+
+            fut: Future = EXECUTOR.submit(_task)
+            futures.append(fut)
+        for fut in futures:
+            out = fut.result()
+            assert out is not None
+            if out.returncode != 0:
+                if check:
+                    raise ValueError(f"Error deleting files: {out.stderr}")
+                else:
+                    warnings.warn(f"Error deleting files: {out.stderr}")
 
     def copy(self, src: Dir | str, dst: Dir | str) -> CompletedProcess:
         """Copy files from source to destination.
@@ -263,6 +283,7 @@ class Rclone:
         files: str | File | list[str] | list[File],
         check=True,
         rmdirs=False,
+        verbose: bool | None = None,
         other_args: list[str] | None = None,
     ) -> CompletedProcess:
         """Delete a directory"""
@@ -277,40 +298,52 @@ class Rclone:
             return CompletedProcess.from_subprocess(cp)
 
         datalists: dict[str, list[str]] = partition_files(payload)
-        out: subprocess.CompletedProcess | None = None
-
         completed_processes: list[subprocess.CompletedProcess] = []
+        verbose = get_verbose(verbose)
+
+        futures: list[Future] = []
 
         for remote, files in datalists.items():
-            with TemporaryDirectory() as tmpdir:
-                include_files_txt = Path(tmpdir) / "include_files.txt"
-                include_files_txt.write_text("\n".join(files), encoding="utf-8")
-
-                # print(include_files_txt)
-                cmd_list: list[str] = [
-                    "delete",
-                    remote,
-                    "--files-from",
-                    str(include_files_txt),
-                    "--checkers",
-                    "1000",
-                    "--transfers",
-                    "1000",
-                ]
-                if rmdirs:
-                    cmd_list.append("--rmdirs")
-                if other_args:
-                    cmd_list += other_args
-                out = self._run(cmd_list)
-                completed_processes.append(out)
+
+            def _task(files=files, check=check) -> subprocess.CompletedProcess:
+                with TemporaryDirectory() as tmpdir:
+                    include_files_txt = Path(tmpdir) / "include_files.txt"
+                    include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                    # print(include_files_txt)
+                    cmd_list: list[str] = [
+                        "delete",
+                        remote,
+                        "--files-from",
+                        str(include_files_txt),
+                        "--checkers",
+                        "1000",
+                        "--transfers",
+                        "1000",
+                    ]
+                    if verbose:
+                        cmd_list.append("-vvvv")
+                    if rmdirs:
+                        cmd_list.append("--rmdirs")
+                    if other_args:
+                        cmd_list += other_args
+                    out = self._run(cmd_list, check=check)
                 if out.returncode != 0:
                     if check:
                         completed_processes.append(out)
                         raise ValueError(f"Error deleting files: {out}")
                     else:
                         warnings.warn(f"Error deleting files: {out}")
+                return out
+
+            fut: Future = EXECUTOR.submit(_task)
+            futures.append(fut)
+
+        for fut in futures:
+            out = fut.result()
+            assert out is not None
+            completed_processes.append(out)
 
-        assert out is not None
         return CompletedProcess(completed_processes)
 
     @deprecated("delete_files")

{rclone_api-1.0.39 → rclone_api-1.0.41}/src/rclone_api/util.py

@@ -132,15 +132,12 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
 
 
 def partition_files(files: list[str]) -> dict[str, list[str]]:
+    """split between filename and parent directory path"""
     datalists: dict[str, list[str]] = {}
     for f in files:
-        remote, path = f.split(":", 1)
-        if "/" in path:
-            bucket, path = path.split("/", 1)
-            remote = f"{remote}:{bucket}"
-        else:
-            remote = f"{remote}:"
-        if remote not in datalists:
-            datalists[remote] = []
-        datalists[remote].append(path)
+        base = os.path.basename(f)
+        parent_path = os.path.dirname(f)
+        if parent_path not in datalists:
+            datalists[parent_path] = []
+        datalists[parent_path].append(base)
     return datalists

{rclone_api-1.0.39 → rclone_api-1.0.41}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.39
+Version: 1.0.41
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies