rclone-api 1.4.15__tar.gz → 1.4.19__tar.gz

{rclone_api-1.4.15 → rclone_api-1.4.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.15
+Version: 1.4.19
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.15 → rclone_api-1.4.19}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.15"
+version = "1.4.19"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.4.19/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -0,0 +1,73 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+from rclone_api.s3.s3_multipart_uploader_by_copy import (
+    s3_server_side_multi_part_merge,
+)
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
+    verbose: bool
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="Directory that holds the info.json file")
+    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=False
+    )
+    args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
+    out = Args(
+        config_path=config,
+        src=args.src,
+        verbose=args.verbose,
+    )
+    return out
+
+
+def _get_info_path(src: str) -> str:
+    if src.endswith("/"):
+        src = src[:-1]
+    info_path = f"{src}/info.json"
+    return info_path
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    info_path = _get_info_path(src=args.src)
+    s3_server_side_multi_part_merge(
+        rclone=rclone.impl, info_path=info_path, max_workers=1
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
+    )
+    main()

{rclone_api-1.4.15 → rclone_api-1.4.19}/src/rclone_api/detail/copy_file_parts.py

@@ -209,7 +209,10 @@ class InfoJson:
 
     @property
     def parts_dir(self) -> str:
-        return os.path.dirname(self.src_info)
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
 
     @property
     def dst(self) -> str:

{rclone_api-1.4.15 → rclone_api-1.4.19}/src/rclone_api/process.py

@@ -1,12 +1,13 @@
 import atexit
 import subprocess
 import threading
-import time
 import weakref
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
+import psutil
+
 from rclone_api.config import Config
 from rclone_api.util import clear_temp_config_file, get_verbose, make_temp_config_file
 
@@ -24,20 +25,25 @@ class ProcessArgs:
 
 class Process:
     def __init__(self, args: ProcessArgs) -> None:
-        assert args.rclone_exe.exists()
+        assert (
+            args.rclone_exe.exists()
+        ), f"rclone executable not found: {args.rclone_exe}"
         self.args = args
         self.log = args.log
         self.tempfile: Path | None = None
+
         verbose = get_verbose(args.verbose)
+        # Create a temporary config file if needed.
         if isinstance(args.rclone_conf, Config):
-            self.tmpfile = make_temp_config_file()
-            self.tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
-            rclone_conf = self.tmpfile
+            self.tempfile = make_temp_config_file()
+            self.tempfile.write_text(args.rclone_conf.text, encoding="utf-8")
+            rclone_conf = self.tempfile
         else:
             rclone_conf = args.rclone_conf
 
-        assert rclone_conf.exists()
+        assert rclone_conf.exists(), f"rclone config not found: {rclone_conf}"
 
+        # Build the command.
         self.cmd = (
             [str(args.rclone_exe.resolve())]
             + ["--config", str(rclone_conf.resolve())]
@@ -49,16 +55,14 @@ class Process:
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")
-        kwargs: dict = {}
-        kwargs["shell"] = False
+        kwargs: dict = {"shell": False}
         if args.capture_stdout:
             kwargs["stdout"] = subprocess.PIPE
             kwargs["stderr"] = subprocess.STDOUT
 
         self.process = subprocess.Popen(self.cmd, **kwargs)  # type: ignore
 
-        # Register an atexit callback using a weak reference to avoid
-        # keeping the Process instance alive solely due to the callback.
+        # Register an atexit callback using a weak reference to avoid keeping the Process instance alive.
         self_ref = weakref.ref(self)
 
         def exit_cleanup():
@@ -77,39 +81,60 @@ class Process:
         self.cleanup()
 
     def cleanup(self) -> None:
-        clear_temp_config_file(self.tempfile)
+        if self.tempfile:
+            clear_temp_config_file(self.tempfile)
 
-    def _atexit_terminate(self) -> None:
+    def _kill_process_tree(self) -> None:
         """
-        Registered via atexit, this method attempts to gracefully terminate the process.
-        If the process does not exit within a short timeout, it is aggressively killed.
+        Use psutil to recursively terminate the main process and all its child processes.
         """
-        if self.process.poll() is None:  # Process is still running
-
-            def terminate_sequence():
+        try:
+            parent = psutil.Process(self.process.pid)
+        except psutil.NoSuchProcess:
+            return
+
+        # Terminate child processes.
+        children = parent.children(recursive=True)
+        if children:
+            print(f"Terminating {len(children)} child processes...")
+            for child in children:
                 try:
-                    # Try to terminate gracefully.
-                    self.process.terminate()
+                    child.terminate()
                 except Exception as e:
-                    print(f"Error calling terminate on process {self.process.pid}: {e}")
-                # Allow time for graceful shutdown.
-                timeout = 2  # seconds
-                start = time.time()
-                while self.process.poll() is None and (time.time() - start) < timeout:
-                    time.sleep(0.1)
-                # If still running, kill aggressively.
-                if self.process.poll() is None:
+                    print(f"Error terminating child process {child.pid}: {e}")
+            psutil.wait_procs(children, timeout=2)
+            # Kill any that remain.
+            for child in children:
+                if child.is_running():
                     try:
-                        self.process.kill()
+                        child.kill()
                     except Exception as e:
-                        print(f"Error calling kill on process {self.process.pid}: {e}")
-                # Optionally wait briefly for termination.
+                        print(f"Error killing child process {child.pid}: {e}")
+
+        # Terminate the parent process.
+        if parent.is_running():
+            try:
+                parent.terminate()
+            except Exception as e:
+                print(f"Error terminating process {parent.pid}: {e}")
+            try:
+                parent.wait(timeout=3)
+            except psutil.TimeoutExpired:
                 try:
-                    self.process.wait(timeout=1)
-                except Exception:
-                    pass
+                    parent.kill()
+                except Exception as e:
+                    print(f"Error killing process {parent.pid}: {e}")
+
+    def _atexit_terminate(self) -> None:
+        """
+        This method is registered via atexit and uses psutil to clean up the process tree.
+        It runs in a daemon thread so that termination happens without blocking interpreter shutdown.
+        """
+        if self.process.poll() is None:  # Process is still running.
+
+            def terminate_sequence():
+                self._kill_process_tree()
 
-            # Run the termination sequence in a separate daemon thread.
             t = threading.Thread(target=terminate_sequence, daemon=True)
             t.start()
             t.join(timeout=3)
@@ -122,12 +147,12 @@ class Process:
         self.cleanup()
 
     def kill(self) -> None:
-        self.cleanup()
-        return self.process.kill()
+        """Forcefully kill the process tree."""
+        self._kill_process_tree()
 
     def terminate(self) -> None:
-        self.cleanup()
-        return self.process.terminate()
+        """Gracefully terminate the process tree."""
+        self._kill_process_tree()
 
     @property
     def returncode(self) -> int | None:
@@ -147,8 +172,8 @@ class Process:
     def wait(self) -> int:
         return self.process.wait()
 
-    def send_signal(self, signal: int) -> None:
-        return self.process.send_signal(signal)
+    def send_signal(self, sig: int) -> None:
+        self.process.send_signal(sig)
 
     def __str__(self) -> str:
         state = ""

rclone_api-1.4.19/src/rclone_api/s3/merge_state.py

@@ -0,0 +1,147 @@
+"""
+https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
+  *  client.upload_part_copy
+
+This module provides functionality for S3 multipart uploads, including copying parts
+from existing S3 objects using upload_part_copy.
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.s3.multipart.finished_piece import FinishedPiece
+
+
+@dataclass
+class Part:
+    part_number: int
+    s3_key: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "s3_key": self.s3_key}
+
+    @staticmethod
+    def from_json(json_dict: dict) -> "Part | Exception":
+        part_number = json_dict.get("part_number")
+        s3_key = json_dict.get("s3_key")
+        if part_number is None or s3_key is None:
+            return Exception(f"Invalid JSON: {json_dict}")
+        return Part(part_number=part_number, s3_key=s3_key)
+
+    @staticmethod
+    def from_json_array(json_array: list[dict]) -> list["Part"] | Exception:
+        try:
+            out: list[Part] = []
+            for j in json_array:
+                ok_or_err = Part.from_json(j)
+                if isinstance(ok_or_err, Exception):
+                    return ok_or_err
+                else:
+                    out.append(ok_or_err)
+            return out
+        except Exception as e:
+            return e
+
+
+class MergeState:
+
+    def __init__(
+        self,
+        rclone_impl: RcloneImpl,
+        merge_path: str,
+        upload_id: str,
+        bucket: str,
+        dst_key: str,
+        finished: list[FinishedPiece],
+        all_parts: list[Part],
+    ) -> None:
+        self.rclone_impl: RcloneImpl = rclone_impl
+        self.merge_path: str = merge_path
+        self.merge_parts_path: str = f"{merge_path}/merge"  # future use?
+        self.upload_id: str = upload_id
+        self.bucket: str = bucket
+        self.dst_key: str = dst_key
+        self.finished: list[FinishedPiece] = list(finished)
+        self.all_parts: list[Part] = list(all_parts)
+
+    def on_finished(self, finished_piece: FinishedPiece) -> None:
+        self.finished.append(finished_piece)
+
+    def remaining_parts(self) -> list[Part]:
+        finished_parts: set[int] = set([p.part_number for p in self.finished])
+        remaining = [p for p in self.all_parts if p.part_number not in finished_parts]
+        return remaining
+
+    @staticmethod
+    def from_json(rclone_impl: RcloneImpl, json: dict) -> "MergeState | Exception":
+        try:
+            merge_path = json["merge_path"]
+            bucket = json["bucket"]
+            dst_key = json["dst_key"]
+            finished: list[FinishedPiece] = FinishedPiece.from_json_array(
+                json["finished"]
+            )
+            all_parts: list[Part | Exception] = [Part.from_json(j) for j in json["all"]]
+            all_parts_no_err: list[Part] = [
+                p for p in all_parts if not isinstance(p, Exception)
+            ]
+            upload_id: str = json["upload_id"]
+            errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
+            if len(errs):
+                return Exception(f"Errors in parts: {errs}")
+            return MergeState(
+                rclone_impl=rclone_impl,
+                merge_path=merge_path,
+                upload_id=upload_id,
+                bucket=bucket,
+                dst_key=dst_key,
+                finished=finished,
+                all_parts=all_parts_no_err,
+            )
+        except Exception as e:
+            return e
+
+    def to_json(self) -> dict:
+        finished = self.finished.copy()
+        all_parts = self.all_parts.copy()
+        return {
+            "merge_path": self.merge_path,
+            "bucket": self.bucket,
+            "dst_key": self.dst_key,
+            "upload_id": self.upload_id,
+            "finished": FinishedPiece.to_json_array(finished),
+            "all": [part.to_json() for part in all_parts],
+        }
+
+    def to_json_str(self) -> str:
+        data = self.to_json()
+        out = json.dumps(data, indent=2)
+        return out
+
+    def __str__(self):
+        return self.to_json_str()
+
+    def __repr__(self):
+        return self.to_json_str()
+
+    def write(self, rclone_impl: Any, dst: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = self.to_json_str()
+        rclone_impl.write_text(dst, json_str)
+
+    def read(self, rclone_impl: Any, src: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = rclone_impl.read_text(src)
+        if isinstance(json_str, Exception):
+            raise json_str
+        json_dict = json.loads(json_str)
+        ok_or_err = FinishedPiece.from_json_array(json_dict["finished"])
+        if isinstance(ok_or_err, Exception):
+            raise ok_or_err
+        self.finished = ok_or_err

{rclone_api-1.4.15 → rclone_api-1.4.19}/src/rclone_api/s3/multipart/finished_piece.py

@@ -1,4 +1,3 @@
-import json
 import warnings
 from dataclasses import dataclass
 
@@ -13,11 +12,10 @@ class FinishedPiece:
     def to_json(self) -> dict:
         return {"part_number": self.part_number, "etag": self.etag}
 
-    def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=0)
-
     @staticmethod
-    def to_json_array(parts: list["FinishedPiece | EndOfStream"]) -> list[dict]:
+    def to_json_array(
+        parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],
+    ) -> list[dict]:
         non_none: list[FinishedPiece] = []
         for p in parts:
             if not isinstance(p, EndOfStream):
@@ -39,3 +37,15 @@ class FinishedPiece:
         if json is None:
             return EndOfStream()
         return FinishedPiece(**json)
+
+    @staticmethod
+    def from_json_array(json: dict) -> list["FinishedPiece"]:
+        tmp = [FinishedPiece.from_json(j) for j in json]
+        out: list[FinishedPiece] = []
+        for t in tmp:
+            if isinstance(t, FinishedPiece):
+                out.append(t)
+        return out
+
+    def __hash__(self) -> int:
+        return hash(self.part_number)