rclone-api 1.4.14__py2.py3-none-any.whl → 1.4.18__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3_finish.py

@@ -6,9 +6,13 @@ from pathlib import Path
 from rclone_api import Rclone
 from rclone_api.detail.copy_file_parts import InfoJson
 from rclone_api.s3.s3_multipart_uploader_by_copy import (
-    finish_multipart_upload_from_keys,
+    Part,
+    S3MultiPartUploader,
 )
-from rclone_api.types import SizeSuffix
+
+_TIMEOUT_READ = 900
+_TIMEOUT_CONNECTION = 900
+_MAX_WORKERS = 10
 
 
 @dataclass
@@ -50,7 +54,55 @@ def _parse_args() -> Args:
     return out
 
 
-def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
+# def finish_multipart_upload_from_keys(
+#     s3_client: BaseClient,
+#     source_bucket: str,
+#     parts: list[Part],
+#     destination_bucket: str,
+#     destination_key: str,
+#     chunk_size: int,  # 5MB default
+#     max_workers: int = 100,
+#     retries: int = 3,
+# ) -> str | Exception:
+#     """
+#     Finish a multipart upload by copying parts from existing S3 objects.
+
+#     Args:
+#         s3_client: Boto3 S3 client
+#         source_bucket: Source bucket name
+#         source_keys: List of source object keys to copy from
+#         destination_bucket: Destination bucket name
+#         destination_key: Destination object key
+#         chunk_size: Size of each part in bytes
+#         retries: Number of retry attempts
+#         byte_ranges: Optional list of byte ranges corresponding to source_keys
+
+#     Returns:
+#         The URL of the completed object
+#     """
+
+#     # Create upload info
+#     info = begin_upload(
+#         s3_client=s3_client,
+#         parts=parts,
+#         destination_bucket=destination_bucket,
+#         destination_key=destination_key,
+#         chunk_size=chunk_size,
+#         retries=retries,
+#     )
+
+#     out = do_body_work(
+#         info=info,
+#         source_bucket=source_bucket,
+#         parts=parts,
+#         max_workers=max_workers,
+#         retries=retries,
+#     )
+
+#     return out
+
+
+def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> Exception | None:
     from rclone_api.s3.create import (
         BaseClient,
         S3Config,
@@ -58,12 +110,17 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         create_s3_client,
     )
 
-    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(
-        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    s3_config = S3Config(
+        verbose=False,
+        timeout_read=_TIMEOUT_READ,
+        timeout_connection=_TIMEOUT_CONNECTION,
     )
+
+    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
+    s3_client: BaseClient = create_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
+    size = info.size
     assert is_done, f"Upload is not done: {info}"
 
     parts_dir = info.parts_dir
@@ -79,11 +136,9 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
 
     first_part: int | None = info.first_part
     last_part: int | None = info.last_part
-    size: SizeSuffix | None = info.size
 
     assert first_part is not None
     assert last_part is not None
-    assert size is not None
 
     def _to_s3_key(name: str | None) -> str:
         if name:
@@ -92,11 +147,12 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         out = f"{parts_path}"
         return out
 
-    parts: list[tuple[int, str]] = []
+    parts: list[Part] = []
     part_num = 1
     for part_key in source_keys:
         s3_key = _to_s3_key(name=part_key)
-        parts.append((part_num, s3_key))
+        part = Part(part_number=part_num, s3_key=s3_key)
+        parts.append(part)
         part_num += 1
 
     chunksize = info.chunksize
@@ -107,31 +163,36 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     # dst_key =
     dst_key = f"{dst_dir}/{dst_name}"
 
-    finish_multipart_upload_from_keys(
+    uploader: S3MultiPartUploader = S3MultiPartUploader(
         s3_client=s3_client,
-        source_bucket=s3_creds.bucket_name,
+        verbose=True,
+    )
+
+    from rclone_api.s3.s3_multipart_uploader_by_copy import MultipartUploadInfo
+
+    state: MultipartUploadInfo = uploader.begin_new_upload(
         parts=parts,
         destination_bucket=s3_creds.bucket_name,
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
-        final_size=size.as_int(),
-        max_workers=50,
-        retries=3,
     )
 
+    uploader.start_upload(info=state, parts=parts, max_workers=_MAX_WORKERS)
+
     # now check if the dst now exists, if so, delete the parts folder.
     # if rclone.exists(dst):
     #     rclone.purge(parts_dir)
 
     if not rclone.exists(dst):
-        raise FileNotFoundError(f"Destination file not found: {dst}")
+        return FileNotFoundError(f"Destination file not found: {dst}")
 
     write_size = rclone.size_file(dst)
     if write_size != size:
-        raise ValueError(f"Size mismatch: {write_size} != {size}")
+        return ValueError(f"Size mismatch: {write_size} != {size}")
 
     print(f"Upload complete: {dst}")
     rclone.purge(parts_dir)
+    return None
 
 
 def main() -> int:

rclone_api/process.py

@@ -1,12 +1,13 @@
 import atexit
 import subprocess
 import threading
-import time
 import weakref
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
+import psutil
+
 from rclone_api.config import Config
 from rclone_api.util import clear_temp_config_file, get_verbose, make_temp_config_file
 
@@ -24,20 +25,25 @@ class ProcessArgs:
 
 class Process:
     def __init__(self, args: ProcessArgs) -> None:
-        assert args.rclone_exe.exists()
+        assert (
+            args.rclone_exe.exists()
+        ), f"rclone executable not found: {args.rclone_exe}"
         self.args = args
         self.log = args.log
         self.tempfile: Path | None = None
+
         verbose = get_verbose(args.verbose)
+        # Create a temporary config file if needed.
         if isinstance(args.rclone_conf, Config):
-            self.tmpfile = make_temp_config_file()
-            self.tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
-            rclone_conf = self.tmpfile
+            self.tempfile = make_temp_config_file()
+            self.tempfile.write_text(args.rclone_conf.text, encoding="utf-8")
+            rclone_conf = self.tempfile
         else:
             rclone_conf = args.rclone_conf
 
-        assert rclone_conf.exists()
+        assert rclone_conf.exists(), f"rclone config not found: {rclone_conf}"
 
+        # Build the command.
         self.cmd = (
             [str(args.rclone_exe.resolve())]
             + ["--config", str(rclone_conf.resolve())]
@@ -49,16 +55,14 @@ class Process:
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")
-        kwargs: dict = {}
-        kwargs["shell"] = False
+        kwargs: dict = {"shell": False}
         if args.capture_stdout:
             kwargs["stdout"] = subprocess.PIPE
             kwargs["stderr"] = subprocess.STDOUT
 
         self.process = subprocess.Popen(self.cmd, **kwargs)  # type: ignore
 
-        # Register an atexit callback using a weak reference to avoid
-        # keeping the Process instance alive solely due to the callback.
+        # Register an atexit callback using a weak reference to avoid keeping the Process instance alive.
         self_ref = weakref.ref(self)
 
         def exit_cleanup():
@@ -77,39 +81,60 @@ class Process:
         self.cleanup()
 
     def cleanup(self) -> None:
-        clear_temp_config_file(self.tempfile)
+        if self.tempfile:
+            clear_temp_config_file(self.tempfile)
 
-    def _atexit_terminate(self) -> None:
+    def _kill_process_tree(self) -> None:
         """
-        Registered via atexit, this method attempts to gracefully terminate the process.
-        If the process does not exit within a short timeout, it is aggressively killed.
+        Use psutil to recursively terminate the main process and all its child processes.
         """
-        if self.process.poll() is None:  # Process is still running
-
-            def terminate_sequence():
+        try:
+            parent = psutil.Process(self.process.pid)
+        except psutil.NoSuchProcess:
+            return
+
+        # Terminate child processes.
+        children = parent.children(recursive=True)
+        if children:
+            print(f"Terminating {len(children)} child processes...")
+            for child in children:
                 try:
-                    # Try to terminate gracefully.
-                    self.process.terminate()
+                    child.terminate()
                 except Exception as e:
-                    print(f"Error calling terminate on process {self.process.pid}: {e}")
-                # Allow time for graceful shutdown.
-                timeout = 2  # seconds
-                start = time.time()
-                while self.process.poll() is None and (time.time() - start) < timeout:
-                    time.sleep(0.1)
-                # If still running, kill aggressively.
-                if self.process.poll() is None:
+                    print(f"Error terminating child process {child.pid}: {e}")
+            psutil.wait_procs(children, timeout=2)
+            # Kill any that remain.
+            for child in children:
+                if child.is_running():
                     try:
-                        self.process.kill()
+                        child.kill()
                     except Exception as e:
-                        print(f"Error calling kill on process {self.process.pid}: {e}")
-                # Optionally wait briefly for termination.
+                        print(f"Error killing child process {child.pid}: {e}")
+
+        # Terminate the parent process.
+        if parent.is_running():
+            try:
+                parent.terminate()
+            except Exception as e:
+                print(f"Error terminating process {parent.pid}: {e}")
+            try:
+                parent.wait(timeout=3)
+            except psutil.TimeoutExpired:
                 try:
-                    self.process.wait(timeout=1)
-                except Exception:
-                    pass
+                    parent.kill()
+                except Exception as e:
+                    print(f"Error killing process {parent.pid}: {e}")
+
+    def _atexit_terminate(self) -> None:
+        """
+        This method is registered via atexit and uses psutil to clean up the process tree.
+        It runs in a daemon thread so that termination happens without blocking interpreter shutdown.
+        """
+        if self.process.poll() is None:  # Process is still running.
+
+            def terminate_sequence():
+                self._kill_process_tree()
 
-            # Run the termination sequence in a separate daemon thread.
             t = threading.Thread(target=terminate_sequence, daemon=True)
             t.start()
             t.join(timeout=3)
@@ -122,12 +147,12 @@ class Process:
         self.cleanup()
 
     def kill(self) -> None:
-        self.cleanup()
-        return self.process.kill()
+        """Forcefully kill the process tree."""
+        self._kill_process_tree()
 
     def terminate(self) -> None:
-        self.cleanup()
-        return self.process.terminate()
+        """Gracefully terminate the process tree."""
+        self._kill_process_tree()
 
     @property
     def returncode(self) -> int | None:
@@ -147,8 +172,8 @@ class Process:
     def wait(self) -> int:
         return self.process.wait()
 
-    def send_signal(self, signal: int) -> None:
-        return self.process.send_signal(signal)
+    def send_signal(self, sig: int) -> None:
+        self.process.send_signal(sig)
 
     def __str__(self) -> str:
         state = ""

rclone_api/s3/api.py

@@ -20,11 +20,11 @@ _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 
 
 class S3Client:
-    def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
+    def __init__(self, s3_creds: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
-        self.credentials: S3Credentials = credentials
+        self.credentials: S3Credentials = s3_creds
         self.client: BaseClient = create_s3_client(
-            credentials, config=S3Config(verbose=verbose)
+            s3_creds=s3_creds, s3_config=S3Config(verbose=verbose)
         )
 
     def list_bucket_contents(self, bucket_name: str) -> None:

rclone_api/s3/create.py

@@ -8,7 +8,7 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 50
+_MAX_CONNECTIONS = 10
 _TIMEOUT_READ = 120
 _TIMEOUT_CONNECT = 60
 
@@ -30,14 +30,16 @@ class S3Config:
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_backblaze_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    region_name = creds.region_name
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    region_name = s3_creds.region_name
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -48,9 +50,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=config.max_pool_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -59,18 +61,20 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_unknown_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if config.verbose:
+        if s3_config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -79,25 +83,25 @@ def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseCli
         endpoint_url=endpoint_url,
         config=Config(
             signature_version="s3v4",
-            region_name=creds.region_name,
-            max_pool_connections=config.max_pool_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            region_name=s3_creds.region_name,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
         ),
     )
 
 
 def create_s3_client(
-    credentials: S3Credentials, config: S3Config | None = None
+    s3_creds: S3Credentials, s3_config: S3Config | None = None
 ) -> BaseClient:
     """Create and return an S3 client."""
-    config = config or S3Config()
-    provider = credentials.provider
+    s3_config = s3_config or S3Config()
+    provider = s3_creds.provider
     if provider == S3Provider.BACKBLAZE:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, config=config)
+        return _create_backblaze_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     else:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, config=config)
+        return _create_unknown_s3_client(s3_creds=s3_creds, s3_config=s3_config)

rclone_api/s3/merge_state.py

@@ -0,0 +1,115 @@
+"""
+https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
+  *  client.upload_part_copy
+
+This module provides functionality for S3 multipart uploads, including copying parts
+from existing S3 objects using upload_part_copy.
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from rclone_api.s3.multipart.finished_piece import FinishedPiece
+
+
+@dataclass
+class Part:
+    part_number: int
+    s3_key: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "s3_key": self.s3_key}
+
+    @staticmethod
+    def from_json(json_dict: dict) -> "Part | Exception":
+        part_number = json_dict.get("part_number")
+        s3_key = json_dict.get("s3_key")
+        if part_number is None or s3_key is None:
+            return Exception(f"Invalid JSON: {json_dict}")
+        return Part(part_number=part_number, s3_key=s3_key)
+
+    @staticmethod
+    def from_json_array(json_array: list[dict]) -> list["Part"] | Exception:
+        try:
+            out: list[Part] = []
+            for j in json_array:
+                ok_or_err = Part.from_json(j)
+                if isinstance(ok_or_err, Exception):
+                    return ok_or_err
+                else:
+                    out.append(ok_or_err)
+            return out
+        except Exception as e:
+            return e
+
+
+class MergeState:
+
+    def __init__(self, finished: list[FinishedPiece], all_parts: list[Part]) -> None:
+        self.finished: list[FinishedPiece] = list(finished)
+        self.all_parts: list[Part] = list(all_parts)
+        self.callbacks: list[Callable[[FinishedPiece], None]] = []
+
+    def add_callback(self, callback: Callable[[FinishedPiece], None]) -> None:
+        self.callbacks.append(callback)
+
+    def on_finished(self, finished_piece: FinishedPiece) -> None:
+        for callback in list(self.callbacks):
+            callback(finished_piece)
+
+    @staticmethod
+    def from_json_array(json_array: dict) -> "MergeState | Exception":
+        try:
+            finished: list[FinishedPiece] = FinishedPiece.from_json_array(
+                json_array["finished"]
+            )
+            all_parts: list[Part | Exception] = [
+                Part.from_json(j) for j in json_array["all"]
+            ]
+            all_parts_no_err: list[Part] = [
+                p for p in all_parts if not isinstance(p, Exception)
+            ]
+            errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
+            if len(errs):
+                return Exception(f"Errors in parts: {errs}")
+            return MergeState(finished=finished, all_parts=all_parts_no_err)
+        except Exception as e:
+            return e
+
+    def to_json(self) -> dict:
+        finished = self.finished.copy()
+        all_parts = self.all_parts.copy()
+        return {
+            "finished": FinishedPiece.to_json_array(finished),
+            "all": [part.to_json() for part in all_parts],
+        }
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_json(), indent=1)
+
+    def __str__(self):
+        return self.to_json_str()
+
+    def __repr__(self):
+        return self.to_json_str()
+
+    def write(self, rclone_impl: Any, dst: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = self.to_json_str()
+        rclone_impl.write_text(dst, json_str)
+
+    def read(self, rclone_impl: Any, src: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = rclone_impl.read_text(src)
+        if isinstance(json_str, Exception):
+            raise json_str
+        json_dict = json.loads(json_str)
+        ok_or_err = FinishedPiece.from_json_array(json_dict["finished"])
+        if isinstance(ok_or_err, Exception):
+            raise ok_or_err
+        self.finished = ok_or_err

rclone_api/s3/multipart/finished_piece.py

@@ -17,7 +17,9 @@ class FinishedPiece:
         return json.dumps(self.to_json(), indent=0)
 
     @staticmethod
-    def to_json_array(parts: list["FinishedPiece | EndOfStream"]) -> list[dict]:
+    def to_json_array(
+        parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],
+    ) -> list[dict]:
         non_none: list[FinishedPiece] = []
         for p in parts:
             if not isinstance(p, EndOfStream):
@@ -39,3 +41,15 @@ class FinishedPiece:
         if json is None:
             return EndOfStream()
         return FinishedPiece(**json)
+
+    @staticmethod
+    def from_json_array(json: dict) -> list["FinishedPiece"]:
+        tmp = [FinishedPiece.from_json(j) for j in json]
+        out: list[FinishedPiece] = []
+        for t in tmp:
+            if isinstance(t, FinishedPiece):
+                out.append(t)
+        return out
+
+    def __hash__(self) -> int:
+        return hash(self.part_number)

rclone_api/s3/s3_multipart_uploader_by_copy.py

@@ -14,6 +14,7 @@ from typing import Optional
 
 from botocore.client import BaseClient
 
+from rclone_api.s3.merge_state import MergeState, Part
 from rclone_api.s3.multipart.finished_piece import FinishedPiece
 from rclone_api.util import locked_print
 
@@ -27,8 +28,6 @@ class MultipartUploadInfo:
     object_name: str
     upload_id: str
     chunk_size: int
-    retries: int
-    file_size: Optional[int] = None
     src_file_path: Optional[Path] = None
 
 
@@ -37,7 +36,6 @@ def upload_part_copy_task(
     source_bucket: str,
     source_key: str,
     part_number: int,
-    retries: int = 3,
 ) -> FinishedPiece | Exception:
     """
     Upload a part by copying from an existing S3 object.
@@ -56,8 +54,8 @@ def upload_part_copy_task(
     copy_source = {"Bucket": source_bucket, "Key": source_key}
 
     # from botocore.exceptions import NoSuchKey
-
-    retries = retries + 1  # Add one for the initial attempt
+    default_retries = 9
+    retries = default_retries + 1  # Add one for the initial attempt
     for retry in range(retries):
         params: dict = {}
         try:
@@ -82,18 +80,24 @@ def upload_part_copy_task(
 
             # Extract ETag from the response
             etag = part["CopyPartResult"]["ETag"]
-            return FinishedPiece(etag=etag, part_number=part_number)
+            out = FinishedPiece(etag=etag, part_number=part_number)
+            locked_print(f"Finished part {part_number} for {info.object_name}")
+            return out
 
         except Exception as e:
             msg = f"Error copying {copy_source} -> {info.object_name}: {e}, params={params}"
-            if "NoSuchKey" in str(e):
+            if "An error occurred (InternalError)" in str(e):
+                locked_print(msg)
+            elif "NoSuchKey" in str(e):
                 locked_print(msg)
-                return e
             if retry == retries - 1:
                 locked_print(msg)
                 return e
             else:
                 locked_print(f"{msg}, retrying")
+                # sleep
+                sleep_time = 2**retry
+                locked_print(f"Sleeping for {sleep_time} seconds")
                 continue
 
     return Exception("Should not reach here")
@@ -132,17 +136,64 @@ def complete_multipart_upload_from_parts(
     return response.get("Location", f"s3://{info.bucket_name}/{info.object_name}")
 
 
-def finish_multipart_upload_from_keys(
-    s3_client: BaseClient,
+def do_body_work(
+    info: MultipartUploadInfo,
     source_bucket: str,
-    parts: list[tuple[int, str]],
-    final_size: int,
+    max_workers: int,
+    merge_state: MergeState,
+) -> str | Exception:
+
+    futures: list[Future[FinishedPiece | Exception]] = []
+    parts = list(merge_state.all_parts)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        semaphore = Semaphore(max_workers)
+        for part in parts:
+            part_number, s3_key = part.part_number, part.s3_key
+
+            def task(
+                info=info,
+                source_bucket=source_bucket,
+                s3_key=s3_key,
+                part_number=part_number,
+            ):
+                out = upload_part_copy_task(
+                    info=info,
+                    source_bucket=source_bucket,
+                    source_key=s3_key,
+                    part_number=part_number,
+                )
+                if isinstance(out, Exception):
+                    return out
+                merge_state.on_finished(out)
+                return out
+
+            fut = executor.submit(task)
+            fut.add_done_callback(lambda x: semaphore.release())
+            futures.append(fut)
+            semaphore.acquire()
+
+        # Upload parts by copying from source objects
+        finished_parts: list[FinishedPiece] = []
+
+        for fut in futures:
+            finished_part = fut.result()
+            if isinstance(finished_part, Exception):
+                executor.shutdown(wait=True, cancel_futures=True)
+                return finished_part
+            finished_parts.append(finished_part)
+
+        # Complete the multipart upload
+        return complete_multipart_upload_from_parts(info, finished_parts)
+
+
+def begin_upload(
+    s3_client: BaseClient,
+    parts: list[Part],
     destination_bucket: str,
     destination_key: str,
-    chunk_size: int,  # 5MB default
-    max_workers: int = 100,
-    retries: int = 3,
-) -> str:
+    chunk_size: int,
+) -> MultipartUploadInfo:
     """
     Finish a multipart upload by copying parts from existing S3 objects.
 
@@ -164,7 +215,6 @@ def finish_multipart_upload_from_keys(
     locked_print(
         f"Creating multipart upload for {destination_bucket}/{destination_key} from {len(parts)} source objects"
     )
-
     create_params: dict[str, str] = {
         "Bucket": destination_bucket,
         "Key": destination_key,
@@ -175,95 +225,103 @@ def finish_multipart_upload_from_keys(
     upload_id = mpu["UploadId"]
 
     # Create upload info
-    upload_info = MultipartUploadInfo(
+    info = MultipartUploadInfo(
         s3_client=s3_client,
         bucket_name=destination_bucket,
         object_name=destination_key,
         upload_id=upload_id,
-        retries=retries,
         chunk_size=chunk_size,
-        file_size=final_size,
     )
+    return info
 
-    futures: list[Future[FinishedPiece | Exception]] = []
 
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        # semaphore
+def finish_multipart_upload_from_keys(
+    s3_client: BaseClient,
+    source_bucket: str,
+    parts: list[Part],
+    destination_bucket: str,
+    destination_key: str,
+    chunk_size: int,  # 5MB default
+    max_workers: int = 100,
+) -> str | Exception:
+    """
+    Finish a multipart upload by copying parts from existing S3 objects.
 
-        semaphore = Semaphore(max_workers * 2)
-        for part_number, source_key in parts:
+    Args:
+        s3_client: Boto3 S3 client
+        source_bucket: Source bucket name
+        source_keys: List of source object keys to copy from
+        destination_bucket: Destination bucket name
+        destination_key: Destination object key
+        chunk_size: Size of each part in bytes
+        retries: Number of retry attempts
+        byte_ranges: Optional list of byte ranges corresponding to source_keys
 
-            def task(
-                info=upload_info,
-                source_bucket=source_bucket,
-                source_key=source_key,
-                part_number=part_number,
-                retries=retries,
-            ):
-                return upload_part_copy_task(
-                    info=info,
-                    source_bucket=source_bucket,
-                    source_key=source_key,
-                    part_number=part_number,
-                    retries=retries,
-                )
+    Returns:
+        The URL of the completed object
+    """
 
-            fut = executor.submit(task)
-            fut.add_done_callback(lambda x: semaphore.release())
-            futures.append(fut)
-            semaphore.acquire()
+    merge_state = MergeState(finished=[], all_parts=parts)
 
-        # Upload parts by copying from source objects
-        finished_parts: list[FinishedPiece] = []
+    # Create upload info
+    info = begin_upload(
+        s3_client=s3_client,
+        parts=merge_state.all_parts,
+        destination_bucket=destination_bucket,
+        destination_key=destination_key,
+        chunk_size=chunk_size,
+    )
 
-        for fut in futures:
-            finished_part = fut.result()
-            if isinstance(finished_part, Exception):
-                executor.shutdown(wait=True, cancel_futures=True)
-                raise finished_part
-            finished_parts.append(finished_part)
+    out = do_body_work(
+        info=info,
+        source_bucket=source_bucket,
+        max_workers=max_workers,
+        merge_state=merge_state,
+    )
+
+    return out
 
-        # Complete the multipart upload
-        return complete_multipart_upload_from_parts(upload_info, finished_parts)
+
+_DEFAULT_MAX_WORKERS = 10
 
 
 class S3MultiPartUploader:
-    def __init__(self, s3_client: BaseClient, verbose: bool) -> None:
-        self.s3_client = s3_client
+    def __init__(self, s3_client: BaseClient, verbose: bool = False) -> None:
         self.verbose = verbose
+        self.client: BaseClient = s3_client
 
-    def finish_from_keys(
+    def begin_new_upload(
         self,
-        source_bucket: str,
-        parts: list[tuple[int, str]],
+        parts: list[Part],
         destination_bucket: str,
         destination_key: str,
         chunk_size: int,
-        final_size: int,
-        retries: int = 3,
-    ) -> str:
-        """
-        Finish a multipart upload by copying parts from existing S3 objects.
-
-        Args:
-            source_bucket: Source bucket name
-            source_keys: List of source object keys to copy from
-            destination_bucket: Destination bucket name
-            destination_key: Destination object key
-            chunk_size: Size of each part in bytes
-            retries: Number of retry attempts
-            byte_ranges: Optional list of byte ranges corresponding to source_keys
-
-        Returns:
-            The URL of the completed object
-        """
-        return finish_multipart_upload_from_keys(
-            s3_client=self.s3_client,
-            source_bucket=source_bucket,
+    ) -> MultipartUploadInfo:
+        return begin_upload(
+            s3_client=self.client,
             parts=parts,
             destination_bucket=destination_bucket,
             destination_key=destination_key,
             chunk_size=chunk_size,
-            final_size=final_size,
-            retries=retries,
+        )
+
+    def start_upload_resume(
+        self,
+        info: MultipartUploadInfo,
+        state: MergeState,
+        max_workers: int = _DEFAULT_MAX_WORKERS,
+    ) -> MultipartUploadInfo | Exception:
+        return Exception("Not implemented")
+
+    def start_upload(
+        self,
+        info: MultipartUploadInfo,
+        parts: list[Part],
+        max_workers: int = _DEFAULT_MAX_WORKERS,
+    ) -> str | Exception:
+        return do_body_work(
+            info=info,
+            source_bucket=info.bucket_name,
+            max_workers=max_workers,
+            merge_state=MergeState(finished=[], all_parts=parts),
         )

rclone_api/util.py

@@ -5,10 +5,13 @@ import shutil
 import signal
 import subprocess
 import warnings
+import weakref
 from pathlib import Path
 from threading import Lock
 from typing import Any
 
+import psutil
+
 from rclone_api.config import Config
 from rclone_api.dir import Dir
 from rclone_api.remote import Remote
@@ -167,51 +170,109 @@ def rclone_execute(
     tmpfile: Path | None = None
     verbose = get_verbose(verbose)
 
-    # Handle the Path case for capture
-    output_file = None
+    # Handle the Path case for capture.
+    output_file: Path | None = None
     if isinstance(capture, Path):
         output_file = capture
-        capture = False  # Don't capture to memory when redirecting to file
+        capture = False  # When redirecting to file, don't capture to memory.
     else:
         capture = capture if isinstance(capture, bool) else True
 
-    assert verbose is not None
-
     try:
+        # Create a temporary config file if needed.
         if isinstance(rclone_conf, Config):
             tmpfile = make_temp_config_file()
             tmpfile.write_text(rclone_conf.text, encoding="utf-8")
             rclone_conf = tmpfile
-        cmd = (
+
+        # Build the command line.
+        full_cmd = (
             [str(rclone_exe.resolve())] + ["--config", str(rclone_conf.resolve())] + cmd
         )
         if verbose:
-            cmd_str = subprocess.list2cmdline(cmd)
+            cmd_str = subprocess.list2cmdline(full_cmd)
             print(f"\nRunning: {cmd_str}")
 
-        # If output_file is set, redirect output to that file
+        # Prepare subprocess parameters.
+        proc_kwargs: dict[str, Any] = {
+            "encoding": "utf-8",
+            "shell": False,
+            "stderr": subprocess.PIPE,
+        }
+        file_handle = None
         if output_file:
-            with open(output_file, "w", encoding="utf-8") as f:
-                cp = subprocess.run(
-                    cmd,
-                    stdout=f,
-                    stderr=subprocess.PIPE,
-                    encoding="utf-8",
-                    check=False,
-                    shell=False,
-                )
+            # Open the file for writing.
+            file_handle = open(output_file, "w", encoding="utf-8")
+            proc_kwargs["stdout"] = file_handle
         else:
-            cp = subprocess.run(
-                cmd, capture_output=capture, encoding="utf-8", check=False, shell=False
-            )
+            proc_kwargs["stdout"] = subprocess.PIPE if capture else None
+
+        # Start the process.
+        process = subprocess.Popen(full_cmd, **proc_kwargs)
+
+        # Register an atexit callback that uses psutil to kill the process tree.
+        proc_ref = weakref.ref(process)
+
+        def cleanup():
+            proc = proc_ref()
+            if proc is None:
+                return
+            try:
+                parent = psutil.Process(proc.pid)
+            except psutil.NoSuchProcess:
+                return
+            # Terminate all child processes first.
+            children = parent.children(recursive=True)
+            if children:
+                print(f"Terminating {len(children)} child process(es)...")
+                for child in children:
+                    try:
+                        child.terminate()
+                    except Exception as e:
+                        print(f"Error terminating child {child.pid}: {e}")
+                psutil.wait_procs(children, timeout=2)
+                for child in children:
+                    if child.is_running():
+                        try:
+                            child.kill()
+                        except Exception as e:
+                            print(f"Error killing child {child.pid}: {e}")
+            # Now terminate the parent process.
+            if parent.is_running():
+                try:
+                    parent.terminate()
+                    parent.wait(timeout=3)
+                except (psutil.TimeoutExpired, Exception):
+                    try:
+                        parent.kill()
+                    except Exception as e:
+                        print(f"Error killing process {parent.pid}: {e}")
+
+        atexit.register(cleanup)
+
+        # Wait for the process to complete.
+        out, err = process.communicate()
+        # Close the file handle if used.
+        if file_handle:
+            file_handle.close()
+
+        cp: subprocess.CompletedProcess = subprocess.CompletedProcess(
+            args=full_cmd,
+            returncode=process.returncode,
+            stdout=out,
+            stderr=err,
+        )
+
+        # Warn or raise if return code is non-zero.
         if cp.returncode != 0:
-            cmd_str = subprocess.list2cmdline(cmd)
+            cmd_str = subprocess.list2cmdline(full_cmd)
             warnings.warn(
-                f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
+                f"Error running: {cmd_str}, returncode: {cp.returncode}\n"
+                f"{cp.stdout}\n{cp.stderr}"
             )
             if check:
                 raise subprocess.CalledProcessError(
-                    cp.returncode, cmd, cp.stdout, cp.stderr
+                    cp.returncode, full_cmd, cp.stdout, cp.stderr
                 )
         return cp
     finally:

{rclone_api-1.4.14.dist-info → rclone_api-1.4.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.14
+Version: 1.4.18
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.14.dist-info → rclone_api-1.4.18.dist-info}/RECORD

@@ -17,17 +17,17 @@ rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,803
 rclone_api/http_server.py,sha256=3fPBV6l50erTe32DyeJBNmsDrn5KuujsbmEAbx13T-c,8720
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
-rclone_api/process.py,sha256=I7B4arAIbFcTBsek27cZ0t-l5YRWVHJJPji7G6ZLGjQ,4989
+rclone_api/process.py,sha256=tGooS5NLdPuqHh7hCH8SfK44A6LGftPQCPQUNgSo0a0,5714
 rclone_api/rclone_impl.py,sha256=xTTriz6-zn_aSrkY8B7wzT-zRXax7Og7ns6xu6-7O6g,48769
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
 rclone_api/types.py,sha256=HkpEZgZWhr5Gb04iHq5NxMRXxieWoN-PKmOfJFrg5Qg,12155
-rclone_api/util.py,sha256=9w_m6W62l_X42Jw5q8p_p30h-QoxAqufvnCLI4PTMOE,7056
+rclone_api/util.py,sha256=yY72YKpmpT_ZM7AleVtPpl0YZZYQPTwTdqKn9qPwm8Y,9290
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
 rclone_api/cmd/copy_large_s3.py,sha256=B17GliDQyAauNglJCpsey0d3eArT2DAcT9g684TMQk8,3514
-rclone_api/cmd/copy_large_s3_finish.py,sha256=7bDaUZ0forOqR4JOT1eoMBF_2qtifM9GcLK_he53cw4,4877
+rclone_api/cmd/copy_large_s3_finish.py,sha256=FeeFZsuc1Gh7Xpso3alvirTX1_eXvHoAIsddMrVdCyw,6521
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
@@ -37,20 +37,21 @@ rclone_api/detail/copy_file_parts.py,sha256=dpqZ0d7l195dZg6Vob2Ty43Uah1v0ozQu5kM
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/s3/api.py,sha256=nIjOskTgTlbksbBHgF27ExFEYheCT_OvcdYl6FqRAto,4058
+rclone_api/s3/api.py,sha256=owoQ1H-R0hXcUozxC6sl53D7NmMOewHk2pUxK-ye8ms,4061
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
-rclone_api/s3/create.py,sha256=NinodMsVPtNXWOM_hlMJWWl-tOj6gV4BcOxEnXFGRWE,3902
-rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=KNEWG3CrM8lVWd2b0dA6WT5LO3gEEXNk2tA6Uk9Vi7I,8701
+rclone_api/s3/create.py,sha256=_Q-faQ4Zl8XKTB28gireRxVXWP-YNxoAK4bligxDtiI,3998
+rclone_api/s3/merge_state.py,sha256=FJOQvOVLvAe9E_M_UgPffrTSN2YzRYfBMnzkOtBTqwg,3939
+rclone_api/s3/s3_multipart_uploader_by_copy.py,sha256=Rww9S81pbCUUu_f72xkxU4HQ_xOf69_C5MyMmmNchcw,10339
 rclone_api/s3/types.py,sha256=cYI5MbXRNdT-ps5kGIRQaYrseHyx_ozT4AcwBABTKwk,1616
 rclone_api/s3/upload_file_multipart.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
 rclone_api/s3/multipart/file_info.py,sha256=8v_07_eADo0K-Nsv7F0Ac1wcv3lkIsrR3MaRCmkYLTQ,105
-rclone_api/s3/multipart/finished_piece.py,sha256=9nMWnVZ8S99wi2VFQsm1h1ZHqmebkhMGgd2s56wNj9w,1331
+rclone_api/s3/multipart/finished_piece.py,sha256=6ev7MFOV3dWqylJFEttOIeoaEA74RMqNWh258L_ENnY,1732
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.14.dist-info/METADATA,sha256=w_EeXXSG4Z-yWrXBFz3kKPRtXh4oMZH2kTrrfko7eos,4628
-rclone_api-1.4.14.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.14.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.14.dist-info/RECORD,,
+rclone_api-1.4.18.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.18.dist-info/METADATA,sha256=Ym9QwhV3G0YNmRao9cluDShuY7AEwclfGTDEz-B2xio,4628
+rclone_api-1.4.18.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.18.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.18.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.18.dist-info/RECORD,,