rclone-api 1.0.35__tar.gz → 1.0.37__tar.gz

{rclone_api-1.0.35 → rclone_api-1.0.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.35
+Version: 1.0.37
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.35 → rclone_api-1.0.37}/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
 ]
 # Change this with the version number bump.
-version = "1.0.35"
+version = "1.0.37"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.0.35 → rclone_api-1.0.37}/src/rclone_api/diff.py

@@ -94,7 +94,6 @@ def _async_diff_stream_from_running_process(
                 print("UnicodeDecodeError")
                 continue
         output.put(None)
-        print("done")
     except KeyboardInterrupt:
         import _thread
 

{rclone_api-1.0.35 → rclone_api-1.0.37}/src/rclone_api/rclone.py

@@ -5,7 +5,6 @@ Unit test file.
 import subprocess
 import time
 import warnings
-from concurrent.futures import ThreadPoolExecutor
 from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
@@ -23,7 +22,7 @@ from rclone_api.file import File
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
-from rclone_api.util import get_rclone_exe, to_path, wait_for_mount
+from rclone_api.util import get_rclone_exe, partition_files, to_path, wait_for_mount
 from rclone_api.walk import walk
 
 
@@ -184,7 +183,7 @@ class Rclone:
         cmd_list: list[str] = ["copyto", src, dst]
         self._run(cmd_list)
 
-    def copyfiles(self, filelist: dict[File, File] | dict[str, str]) -> None:
+    def copy_to(self, src: File | str, dst: File | str) -> None:
         """Copy multiple files from source to destination.
 
         Warning - slow.
@@ -192,17 +191,48 @@ class Rclone:
         Args:
             payload: Dictionary of source and destination file paths
         """
-        str_dict: dict[str, str] = {}
-        for src, dst in filelist.items():
-            src = src if isinstance(src, str) else str(src.path)
-            dst = dst if isinstance(dst, str) else str(dst.path)
-            str_dict[src] = dst
-
-        with ThreadPoolExecutor(max_workers=64) as executor:
-            for src, dst in str_dict.items():  # warning - slow
-                cmd_list: list[str] = ["copyto", src, dst]
-                # self._run(cmd_list)
-                executor.submit(self._run, cmd_list)
+        src = str(src)
+        dst = str(dst)
+        cmd_list: list[str] = ["copyto", src, dst]
+        self._run(cmd_list)
+
+    def copyfiles(self, files: str | File | list[str] | list[File]) -> None:
+        """Copy multiple files from source to destination.
+
+        Warning - slow.
+
+        Args:
+            payload: Dictionary of source and destination file paths
+        """
+        payload: list[str] = convert_to_filestr_list(files)
+        if len(payload) == 0:
+            return
+
+        datalists: dict[str, list[str]] = partition_files(payload)
+        out: subprocess.CompletedProcess | None = None
+
+        for remote, files in datalists.items():
+            with TemporaryDirectory() as tmpdir:
+                include_files_txt = Path(tmpdir) / "include_files.txt"
+                include_files_txt.write_text("\n".join(files), encoding="utf-8")
+
+                # print(include_files_txt)
+                cmd_list: list[str] = [
+                    "delete",
+                    remote,
+                    "--files-from",
+                    str(include_files_txt),
+                    "--checkers",
+                    "1000",
+                    "--transfers",
+                    "1000",
+                ]
+                out = self._run(cmd_list)
+                if out.returncode != 0:
+                    print(out)
+                    raise ValueError(f"Error deleting files: {out.stderr}")
+
+        assert out is not None
 
     def copy(self, src: Dir | str, dst: Dir | str) -> subprocess.CompletedProcess:
         """Copy files from source to destination.
@@ -226,7 +256,7 @@ class Rclone:
         return self._run(cmd_list)
 
     def delete_files(
-        self, files: str | File | list[str] | list[File]
+        self, files: str | File | list[str] | list[File], check=True
     ) -> subprocess.CompletedProcess:
         """Delete a directory"""
         payload: list[str] = convert_to_filestr_list(files)
@@ -238,27 +268,17 @@ class Rclone:
                 stderr="",
             )
 
-        datalists: dict[str, list[str]] = {}
-
-        for f in payload:
-            remote, path = f.split(":", 1)
-            if "/" in path:
-                bucket, path = path.split("/", 1)
-                remote = f"{remote}:{bucket}"
-            else:
-                remote = f"{remote}:"
-            if remote not in datalists:
-                datalists[remote] = []
-            datalists[remote].append(path)
-
+        datalists: dict[str, list[str]] = partition_files(payload)
         out: subprocess.CompletedProcess | None = None
 
+        completed_processes: list[subprocess.CompletedProcess] = []
+
         for remote, files in datalists.items():
             with TemporaryDirectory() as tmpdir:
                 include_files_txt = Path(tmpdir) / "include_files.txt"
                 include_files_txt.write_text("\n".join(files), encoding="utf-8")
 
-                print(include_files_txt)
+                # print(include_files_txt)
                 cmd_list: list[str] = [
                     "delete",
                     remote,
@@ -270,9 +290,13 @@ class Rclone:
                     "1000",
                 ]
                 out = self._run(cmd_list)
+                completed_processes.append(out)
                 if out.returncode != 0:
-                    print(out)
-                    raise ValueError(f"Error deleting files: {out.stderr}")
+                    if check:
+                        completed_processes.append(out)
+                        raise ValueError(f"Error deleting files: {out}")
+                    else:
+                        warnings.warn(f"Error deleting files: {out}")
 
         assert out is not None
         return out
@@ -288,8 +312,9 @@ class Rclone:
         arg: str = convert_to_str(path)
         assert isinstance(arg, str)
         try:
-            self.ls(arg)
-            return True
+            dir_listing = self.ls(arg)
+            # print(dir_listing)
+            return len(dir_listing.dirs) > 0 or len(dir_listing.files) > 0
         except subprocess.CalledProcessError:
             return False
 

{rclone_api-1.0.35 → rclone_api-1.0.37}/src/rclone_api/util.py

@@ -129,3 +129,18 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
         if path.exists():
             return
     raise TimeoutError(f"Path {path} did not exist after {timeout} seconds")
+
+
+def partition_files(files: list[str]) -> dict[str, list[str]]:
+    datalists: dict[str, list[str]] = {}
+    for f in files:
+        remote, path = f.split(":", 1)
+        if "/" in path:
+            bucket, path = path.split("/", 1)
+            remote = f"{remote}:{bucket}"
+        else:
+            remote = f"{remote}:"
+        if remote not in datalists:
+            datalists[remote] = []
+        datalists[remote].append(path)
+    return datalists

{rclone_api-1.0.35 → rclone_api-1.0.37}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.35
+Version: 1.0.37
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 Maintainer: Zachary Vorhies

{rclone_api-1.0.35 → rclone_api-1.0.37}/tests/test_copy.py

@@ -83,22 +83,17 @@ class RcloneCopyTests(unittest.TestCase):
         first_file = str(listing.files[0])
         dest_file = first_file + "_copy"
 
-        # Copy the files to the same location with different names
-        filelist: dict[str, str] = {
-            first_file: dest_file,
-        }
-        # warning slow.
-        rclone.copyfiles(filelist)
-
-        delete_list: list[str] = []
-        for _, dst in filelist.items():
-            delete_list.append(dst)
+        # Copy the file to the same location with different names
+        rclone.copy_to(first_file, dest_file)
 
         # now test that the new file exists
         exists = rclone.exists(dest_file)
         self.assertTrue(exists)
 
-        rclone.delete_files(delete_list)
+        rclone.delete_files(dest_file)
+        print(f"Checking that {dest_file} was deleted")
+        deleted = rclone.exists(dest_file)
+        self.assertFalse(deleted)
         print("done")