rclone-api 1.4.15__py2.py3-none-any.whl → 1.4.19__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3_finish.py

@@ -1,24 +1,17 @@
 import argparse
-import os
 from dataclasses import dataclass
 from pathlib import Path
 
 from rclone_api import Rclone
-from rclone_api.detail.copy_file_parts import InfoJson
 from rclone_api.s3.s3_multipart_uploader_by_copy import (
-    finish_multipart_upload_from_keys,
+    s3_server_side_multi_part_merge,
 )
-from rclone_api.types import SizeSuffix
-
-_TIMEOUT_READ = 900
-_TIMEOUT_CONNECTION = 900
 
 
 @dataclass
 class Args:
     config_path: Path
     src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
-    dst: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst
     verbose: bool
 
 
@@ -31,8 +24,7 @@ def list_files(rclone: Rclone, path: str):
 
 def _parse_args() -> Args:
     parser = argparse.ArgumentParser(description="List files in a remote path.")
-    parser.add_argument("src", help="File to copy")
-    parser.add_argument("dst", help="Destination file")
+    parser.add_argument("src", help="Directory that holds the info.json file")
     parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
     parser.add_argument(
         "--config", help="Path to rclone config file", type=Path, required=False
@@ -47,112 +39,26 @@ def _parse_args() -> Args:
     out = Args(
         config_path=config,
         src=args.src,
-        dst=args.dst,
         verbose=args.verbose,
     )
     return out
 
 
-def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
-    from rclone_api.s3.create import (
-        BaseClient,
-        S3Config,
-        S3Credentials,
-        create_s3_client,
-    )
-
-    s3_config = S3Config(
-        verbose=False,
-        timeout_read=_TIMEOUT_READ,
-        timeout_connection=_TIMEOUT_CONNECTION,
-    )
-
-    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(s3_creds=s3_creds, s3_config=s3_config)
-    s3_bucket = s3_creds.bucket_name
-    is_done = info.fetch_is_done()
-    assert is_done, f"Upload is not done: {info}"
-
-    parts_dir = info.parts_dir
-    if parts_dir.endswith("/"):
-        parts_dir = parts_dir[:-1]
-    source_keys = info.fetch_all_finished()
-    # print(parts_dir)
-    # print(source_keys)
-
-    parts_path = parts_dir.split(s3_bucket)[1]
-    if parts_path.startswith("/"):
-        parts_path = parts_path[1:]
-
-    first_part: int | None = info.first_part
-    last_part: int | None = info.last_part
-    size: SizeSuffix | None = info.size
-
-    assert first_part is not None
-    assert last_part is not None
-    assert size is not None
-
-    def _to_s3_key(name: str | None) -> str:
-        if name:
-            out = f"{parts_path}/{name}"
-            return out
-        out = f"{parts_path}"
-        return out
-
-    parts: list[tuple[int, str]] = []
-    part_num = 1
-    for part_key in source_keys:
-        s3_key = _to_s3_key(name=part_key)
-        parts.append((part_num, s3_key))
-        part_num += 1
-
-    chunksize = info.chunksize
-    assert chunksize is not None
-
-    dst_name = info.dst_name
-    dst_dir = os.path.dirname(parts_path)
-    # dst_key =
-    dst_key = f"{dst_dir}/{dst_name}"
-
-    finish_multipart_upload_from_keys(
-        s3_client=s3_client,
-        source_bucket=s3_creds.bucket_name,
-        parts=parts,
-        destination_bucket=s3_creds.bucket_name,
-        destination_key=dst_key,
-        chunk_size=chunksize.as_int(),
-        final_size=size.as_int(),
-        max_workers=10,
-    )
-
-    # now check if the dst now exists, if so, delete the parts folder.
-    # if rclone.exists(dst):
-    #     rclone.purge(parts_dir)
-
-    if not rclone.exists(dst):
-        raise FileNotFoundError(f"Destination file not found: {dst}")
-
-    write_size = rclone.size_file(dst)
-    if write_size != size:
-        raise ValueError(f"Size mismatch: {write_size} != {size}")
-
-    print(f"Upload complete: {dst}")
-    rclone.purge(parts_dir)
+def _get_info_path(src: str) -> str:
+    if src.endswith("/"):
+        src = src[:-1]
+    info_path = f"{src}/info.json"
+    return info_path
 
 
 def main() -> int:
     """Main entry point."""
     args = _parse_args()
     rclone = Rclone(rclone_conf=args.config_path)
-    info_json = f"{args.src}/info.json".replace("//", "/")
-    info = InfoJson(rclone.impl, src=None, src_info=info_json)
-    loaded = info.load()
-    if not loaded:
-        raise FileNotFoundError(
-            f"Info file not found, has the upload finished? {info_json}"
-        )
-    print(info)
-    do_finish_part(rclone=rclone, info=info, dst=args.dst)
+    info_path = _get_info_path(src=args.src)
+    s3_server_side_multi_part_merge(
+        rclone=rclone.impl, info_path=info_path, max_workers=1
+    )
     return 0
 
 
@@ -164,7 +70,4 @@ if __name__ == "__main__":
     sys.argv.append(
         "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
     )
-    sys.argv.append(
-        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
-    )
     main()

rclone_api/detail/copy_file_parts.py

@@ -209,7 +209,10 @@ class InfoJson:
 
     @property
     def parts_dir(self) -> str:
-        return os.path.dirname(self.src_info)
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
 
     @property
     def dst(self) -> str:

rclone_api/process.py

@@ -1,12 +1,13 @@
 import atexit
 import subprocess
 import threading
-import time
 import weakref
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
+import psutil
+
 from rclone_api.config import Config
 from rclone_api.util import clear_temp_config_file, get_verbose, make_temp_config_file
 
@@ -24,20 +25,25 @@ class ProcessArgs:
 
 class Process:
     def __init__(self, args: ProcessArgs) -> None:
-        assert args.rclone_exe.exists()
+        assert (
+            args.rclone_exe.exists()
+        ), f"rclone executable not found: {args.rclone_exe}"
         self.args = args
         self.log = args.log
         self.tempfile: Path | None = None
+
         verbose = get_verbose(args.verbose)
+        # Create a temporary config file if needed.
         if isinstance(args.rclone_conf, Config):
-            self.tmpfile = make_temp_config_file()
-            self.tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
-            rclone_conf = self.tmpfile
+            self.tempfile = make_temp_config_file()
+            self.tempfile.write_text(args.rclone_conf.text, encoding="utf-8")
+            rclone_conf = self.tempfile
         else:
             rclone_conf = args.rclone_conf
 
-        assert rclone_conf.exists()
+        assert rclone_conf.exists(), f"rclone config not found: {rclone_conf}"
 
+        # Build the command.
         self.cmd = (
             [str(args.rclone_exe.resolve())]
             + ["--config", str(rclone_conf.resolve())]
@@ -49,16 +55,14 @@ class Process:
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")
-        kwargs: dict = {}
-        kwargs["shell"] = False
+        kwargs: dict = {"shell": False}
         if args.capture_stdout:
             kwargs["stdout"] = subprocess.PIPE
             kwargs["stderr"] = subprocess.STDOUT
 
         self.process = subprocess.Popen(self.cmd, **kwargs)  # type: ignore
 
-        # Register an atexit callback using a weak reference to avoid
-        # keeping the Process instance alive solely due to the callback.
+        # Register an atexit callback using a weak reference to avoid keeping the Process instance alive.
         self_ref = weakref.ref(self)
 
         def exit_cleanup():
@@ -77,39 +81,60 @@ class Process:
         self.cleanup()
 
     def cleanup(self) -> None:
-        clear_temp_config_file(self.tempfile)
+        if self.tempfile:
+            clear_temp_config_file(self.tempfile)
 
-    def _atexit_terminate(self) -> None:
+    def _kill_process_tree(self) -> None:
         """
-        Registered via atexit, this method attempts to gracefully terminate the process.
-        If the process does not exit within a short timeout, it is aggressively killed.
+        Use psutil to recursively terminate the main process and all its child processes.
         """
-        if self.process.poll() is None:  # Process is still running
-
-            def terminate_sequence():
+        try:
+            parent = psutil.Process(self.process.pid)
+        except psutil.NoSuchProcess:
+            return
+
+        # Terminate child processes.
+        children = parent.children(recursive=True)
+        if children:
+            print(f"Terminating {len(children)} child processes...")
+            for child in children:
                 try:
-                    # Try to terminate gracefully.
-                    self.process.terminate()
+                    child.terminate()
                 except Exception as e:
-                    print(f"Error calling terminate on process {self.process.pid}: {e}")
-                # Allow time for graceful shutdown.
-                timeout = 2  # seconds
-                start = time.time()
-                while self.process.poll() is None and (time.time() - start) < timeout:
-                    time.sleep(0.1)
-                # If still running, kill aggressively.
-                if self.process.poll() is None:
+                    print(f"Error terminating child process {child.pid}: {e}")
+            psutil.wait_procs(children, timeout=2)
+            # Kill any that remain.
+            for child in children:
+                if child.is_running():
                     try:
-                        self.process.kill()
+                        child.kill()
                     except Exception as e:
-                        print(f"Error calling kill on process {self.process.pid}: {e}")
-                # Optionally wait briefly for termination.
+                        print(f"Error killing child process {child.pid}: {e}")
+
+        # Terminate the parent process.
+        if parent.is_running():
+            try:
+                parent.terminate()
+            except Exception as e:
+                print(f"Error terminating process {parent.pid}: {e}")
+            try:
+                parent.wait(timeout=3)
+            except psutil.TimeoutExpired:
                 try:
-                    self.process.wait(timeout=1)
-                except Exception:
-                    pass
+                    parent.kill()
+                except Exception as e:
+                    print(f"Error killing process {parent.pid}: {e}")
+
+    def _atexit_terminate(self) -> None:
+        """
+        This method is registered via atexit and uses psutil to clean up the process tree.
+        It runs in a daemon thread so that termination happens without blocking interpreter shutdown.
+        """
+        if self.process.poll() is None:  # Process is still running.
+
+            def terminate_sequence():
+                self._kill_process_tree()
 
-            # Run the termination sequence in a separate daemon thread.
             t = threading.Thread(target=terminate_sequence, daemon=True)
             t.start()
             t.join(timeout=3)
@@ -122,12 +147,12 @@ class Process:
         self.cleanup()
 
     def kill(self) -> None:
-        self.cleanup()
-        return self.process.kill()
+        """Forcefully kill the process tree."""
+        self._kill_process_tree()
 
     def terminate(self) -> None:
-        self.cleanup()
-        return self.process.terminate()
+        """Gracefully terminate the process tree."""
+        self._kill_process_tree()
 
     @property
     def returncode(self) -> int | None:
@@ -147,8 +172,8 @@ class Process:
     def wait(self) -> int:
         return self.process.wait()
 
-    def send_signal(self, signal: int) -> None:
-        return self.process.send_signal(signal)
+    def send_signal(self, sig: int) -> None:
+        self.process.send_signal(sig)
 
     def __str__(self) -> str:
         state = ""

rclone_api/s3/merge_state.py

@@ -0,0 +1,147 @@
+"""
+https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
+  *  client.upload_part_copy
+
+This module provides functionality for S3 multipart uploads, including copying parts
+from existing S3 objects using upload_part_copy.
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.s3.multipart.finished_piece import FinishedPiece
+
+
+@dataclass
+class Part:
+    part_number: int
+    s3_key: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "s3_key": self.s3_key}
+
+    @staticmethod
+    def from_json(json_dict: dict) -> "Part | Exception":
+        part_number = json_dict.get("part_number")
+        s3_key = json_dict.get("s3_key")
+        if part_number is None or s3_key is None:
+            return Exception(f"Invalid JSON: {json_dict}")
+        return Part(part_number=part_number, s3_key=s3_key)
+
+    @staticmethod
+    def from_json_array(json_array: list[dict]) -> list["Part"] | Exception:
+        try:
+            out: list[Part] = []
+            for j in json_array:
+                ok_or_err = Part.from_json(j)
+                if isinstance(ok_or_err, Exception):
+                    return ok_or_err
+                else:
+                    out.append(ok_or_err)
+            return out
+        except Exception as e:
+            return e
+
+
+class MergeState:
+
+    def __init__(
+        self,
+        rclone_impl: RcloneImpl,
+        merge_path: str,
+        upload_id: str,
+        bucket: str,
+        dst_key: str,
+        finished: list[FinishedPiece],
+        all_parts: list[Part],
+    ) -> None:
+        self.rclone_impl: RcloneImpl = rclone_impl
+        self.merge_path: str = merge_path
+        self.merge_parts_path: str = f"{merge_path}/merge"  # future use?
+        self.upload_id: str = upload_id
+        self.bucket: str = bucket
+        self.dst_key: str = dst_key
+        self.finished: list[FinishedPiece] = list(finished)
+        self.all_parts: list[Part] = list(all_parts)
+
+    def on_finished(self, finished_piece: FinishedPiece) -> None:
+        self.finished.append(finished_piece)
+
+    def remaining_parts(self) -> list[Part]:
+        finished_parts: set[int] = set([p.part_number for p in self.finished])
+        remaining = [p for p in self.all_parts if p.part_number not in finished_parts]
+        return remaining
+
+    @staticmethod
+    def from_json(rclone_impl: RcloneImpl, json: dict) -> "MergeState | Exception":
+        try:
+            merge_path = json["merge_path"]
+            bucket = json["bucket"]
+            dst_key = json["dst_key"]
+            finished: list[FinishedPiece] = FinishedPiece.from_json_array(
+                json["finished"]
+            )
+            all_parts: list[Part | Exception] = [Part.from_json(j) for j in json["all"]]
+            all_parts_no_err: list[Part] = [
+                p for p in all_parts if not isinstance(p, Exception)
+            ]
+            upload_id: str = json["upload_id"]
+            errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
+            if len(errs):
+                return Exception(f"Errors in parts: {errs}")
+            return MergeState(
+                rclone_impl=rclone_impl,
+                merge_path=merge_path,
+                upload_id=upload_id,
+                bucket=bucket,
+                dst_key=dst_key,
+                finished=finished,
+                all_parts=all_parts_no_err,
+            )
+        except Exception as e:
+            return e
+
+    def to_json(self) -> dict:
+        finished = self.finished.copy()
+        all_parts = self.all_parts.copy()
+        return {
+            "merge_path": self.merge_path,
+            "bucket": self.bucket,
+            "dst_key": self.dst_key,
+            "upload_id": self.upload_id,
+            "finished": FinishedPiece.to_json_array(finished),
+            "all": [part.to_json() for part in all_parts],
+        }
+
+    def to_json_str(self) -> str:
+        data = self.to_json()
+        out = json.dumps(data, indent=2)
+        return out
+
+    def __str__(self):
+        return self.to_json_str()
+
+    def __repr__(self):
+        return self.to_json_str()
+
+    def write(self, rclone_impl: Any, dst: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = self.to_json_str()
+        rclone_impl.write_text(dst, json_str)
+
+    def read(self, rclone_impl: Any, src: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = rclone_impl.read_text(src)
+        if isinstance(json_str, Exception):
+            raise json_str
+        json_dict = json.loads(json_str)
+        ok_or_err = FinishedPiece.from_json_array(json_dict["finished"])
+        if isinstance(ok_or_err, Exception):
+            raise ok_or_err
+        self.finished = ok_or_err

rclone_api/s3/multipart/finished_piece.py

@@ -1,4 +1,3 @@
-import json
 import warnings
 from dataclasses import dataclass
 
@@ -13,11 +12,10 @@ class FinishedPiece:
     def to_json(self) -> dict:
         return {"part_number": self.part_number, "etag": self.etag}
 
-    def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=0)
-
     @staticmethod
-    def to_json_array(parts: list["FinishedPiece | EndOfStream"]) -> list[dict]:
+    def to_json_array(
+        parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],
+    ) -> list[dict]:
         non_none: list[FinishedPiece] = []
         for p in parts:
             if not isinstance(p, EndOfStream):
@@ -39,3 +37,15 @@ class FinishedPiece:
         if json is None:
             return EndOfStream()
         return FinishedPiece(**json)
+
+    @staticmethod
+    def from_json_array(json: dict) -> list["FinishedPiece"]:
+        tmp = [FinishedPiece.from_json(j) for j in json]
+        out: list[FinishedPiece] = []
+        for t in tmp:
+            if isinstance(t, FinishedPiece):
+                out.append(t)
+        return out
+
+    def __hash__(self) -> int:
+        return hash(self.part_number)