rclone-api 1.4.14__tar.gz → 1.4.18__tar.gz

{rclone_api-1.4.14 → rclone_api-1.4.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.14
+Version: 1.4.18
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.14 → rclone_api-1.4.18}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.14"
+version = "1.4.18"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.14 → rclone_api-1.4.18}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -6,9 +6,13 @@ from pathlib import Path
 from rclone_api import Rclone
 from rclone_api.detail.copy_file_parts import InfoJson
 from rclone_api.s3.s3_multipart_uploader_by_copy import (
-    finish_multipart_upload_from_keys,
+    Part,
+    S3MultiPartUploader,
 )
-from rclone_api.types import SizeSuffix
+
+_TIMEOUT_READ = 900
+_TIMEOUT_CONNECTION = 900
+_MAX_WORKERS = 10
 
 
 @dataclass
@@ -50,7 +54,55 @@ def _parse_args() -> Args:
     return out
 
 
-def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
+# def finish_multipart_upload_from_keys(
+#     s3_client: BaseClient,
+#     source_bucket: str,
+#     parts: list[Part],
+#     destination_bucket: str,
+#     destination_key: str,
+#     chunk_size: int,  # 5MB default
+#     max_workers: int = 100,
+#     retries: int = 3,
+# ) -> str | Exception:
+#     """
+#     Finish a multipart upload by copying parts from existing S3 objects.
+
+#     Args:
+#         s3_client: Boto3 S3 client
+#         source_bucket: Source bucket name
+#         source_keys: List of source object keys to copy from
+#         destination_bucket: Destination bucket name
+#         destination_key: Destination object key
+#         chunk_size: Size of each part in bytes
+#         retries: Number of retry attempts
+#         byte_ranges: Optional list of byte ranges corresponding to source_keys
+
+#     Returns:
+#         The URL of the completed object
+#     """
+
+#     # Create upload info
+#     info = begin_upload(
+#         s3_client=s3_client,
+#         parts=parts,
+#         destination_bucket=destination_bucket,
+#         destination_key=destination_key,
+#         chunk_size=chunk_size,
+#         retries=retries,
+#     )
+
+#     out = do_body_work(
+#         info=info,
+#         source_bucket=source_bucket,
+#         parts=parts,
+#         max_workers=max_workers,
+#         retries=retries,
+#     )
+
+#     return out
+
+
+def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> Exception | None:
     from rclone_api.s3.create import (
         BaseClient,
         S3Config,
@@ -58,12 +110,17 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         create_s3_client,
     )
 
-    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
-    s3_client: BaseClient = create_s3_client(
-        s3_creds, S3Config(verbose=False, timeout_read=5 * 60)
+    s3_config = S3Config(
+        verbose=False,
+        timeout_read=_TIMEOUT_READ,
+        timeout_connection=_TIMEOUT_CONNECTION,
     )
+
+    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
+    s3_client: BaseClient = create_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     s3_bucket = s3_creds.bucket_name
     is_done = info.fetch_is_done()
+    size = info.size
     assert is_done, f"Upload is not done: {info}"
 
     parts_dir = info.parts_dir
@@ -79,11 +136,9 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
 
     first_part: int | None = info.first_part
     last_part: int | None = info.last_part
-    size: SizeSuffix | None = info.size
 
     assert first_part is not None
     assert last_part is not None
-    assert size is not None
 
     def _to_s3_key(name: str | None) -> str:
         if name:
@@ -92,11 +147,12 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
         out = f"{parts_path}"
         return out
 
-    parts: list[tuple[int, str]] = []
+    parts: list[Part] = []
     part_num = 1
     for part_key in source_keys:
         s3_key = _to_s3_key(name=part_key)
-        parts.append((part_num, s3_key))
+        part = Part(part_number=part_num, s3_key=s3_key)
+        parts.append(part)
         part_num += 1
 
     chunksize = info.chunksize
@@ -107,31 +163,36 @@ def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
     # dst_key =
     dst_key = f"{dst_dir}/{dst_name}"
 
-    finish_multipart_upload_from_keys(
+    uploader: S3MultiPartUploader = S3MultiPartUploader(
         s3_client=s3_client,
-        source_bucket=s3_creds.bucket_name,
+        verbose=True,
+    )
+
+    from rclone_api.s3.s3_multipart_uploader_by_copy import MultipartUploadInfo
+
+    state: MultipartUploadInfo = uploader.begin_new_upload(
         parts=parts,
         destination_bucket=s3_creds.bucket_name,
         destination_key=dst_key,
         chunk_size=chunksize.as_int(),
-        final_size=size.as_int(),
-        max_workers=50,
-        retries=3,
     )
 
+    uploader.start_upload(info=state, parts=parts, max_workers=_MAX_WORKERS)
+
     # now check if the dst now exists, if so, delete the parts folder.
     # if rclone.exists(dst):
     #     rclone.purge(parts_dir)
 
     if not rclone.exists(dst):
-        raise FileNotFoundError(f"Destination file not found: {dst}")
+        return FileNotFoundError(f"Destination file not found: {dst}")
 
     write_size = rclone.size_file(dst)
     if write_size != size:
-        raise ValueError(f"Size mismatch: {write_size} != {size}")
+        return ValueError(f"Size mismatch: {write_size} != {size}")
 
     print(f"Upload complete: {dst}")
     rclone.purge(parts_dir)
+    return None
 
 
 def main() -> int:

{rclone_api-1.4.14 → rclone_api-1.4.18}/src/rclone_api/process.py

@@ -1,12 +1,13 @@
 import atexit
 import subprocess
 import threading
-import time
 import weakref
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
+import psutil
+
 from rclone_api.config import Config
 from rclone_api.util import clear_temp_config_file, get_verbose, make_temp_config_file
 
@@ -24,20 +25,25 @@ class ProcessArgs:
 
 class Process:
     def __init__(self, args: ProcessArgs) -> None:
-        assert args.rclone_exe.exists()
+        assert (
+            args.rclone_exe.exists()
+        ), f"rclone executable not found: {args.rclone_exe}"
         self.args = args
         self.log = args.log
         self.tempfile: Path | None = None
+
         verbose = get_verbose(args.verbose)
+        # Create a temporary config file if needed.
         if isinstance(args.rclone_conf, Config):
-            self.tmpfile = make_temp_config_file()
-            self.tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
-            rclone_conf = self.tmpfile
+            self.tempfile = make_temp_config_file()
+            self.tempfile.write_text(args.rclone_conf.text, encoding="utf-8")
+            rclone_conf = self.tempfile
         else:
             rclone_conf = args.rclone_conf
 
-        assert rclone_conf.exists()
+        assert rclone_conf.exists(), f"rclone config not found: {rclone_conf}"
 
+        # Build the command.
         self.cmd = (
             [str(args.rclone_exe.resolve())]
             + ["--config", str(rclone_conf.resolve())]
@@ -49,16 +55,14 @@ class Process:
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")
-        kwargs: dict = {}
-        kwargs["shell"] = False
+        kwargs: dict = {"shell": False}
         if args.capture_stdout:
             kwargs["stdout"] = subprocess.PIPE
             kwargs["stderr"] = subprocess.STDOUT
 
         self.process = subprocess.Popen(self.cmd, **kwargs)  # type: ignore
 
-        # Register an atexit callback using a weak reference to avoid
-        # keeping the Process instance alive solely due to the callback.
+        # Register an atexit callback using a weak reference to avoid keeping the Process instance alive.
         self_ref = weakref.ref(self)
 
         def exit_cleanup():
@@ -77,39 +81,60 @@ class Process:
         self.cleanup()
 
     def cleanup(self) -> None:
-        clear_temp_config_file(self.tempfile)
+        if self.tempfile:
+            clear_temp_config_file(self.tempfile)
 
-    def _atexit_terminate(self) -> None:
+    def _kill_process_tree(self) -> None:
         """
-        Registered via atexit, this method attempts to gracefully terminate the process.
-        If the process does not exit within a short timeout, it is aggressively killed.
+        Use psutil to recursively terminate the main process and all its child processes.
         """
-        if self.process.poll() is None:  # Process is still running
-
-            def terminate_sequence():
+        try:
+            parent = psutil.Process(self.process.pid)
+        except psutil.NoSuchProcess:
+            return
+
+        # Terminate child processes.
+        children = parent.children(recursive=True)
+        if children:
+            print(f"Terminating {len(children)} child processes...")
+            for child in children:
                 try:
-                    # Try to terminate gracefully.
-                    self.process.terminate()
+                    child.terminate()
                 except Exception as e:
-                    print(f"Error calling terminate on process {self.process.pid}: {e}")
-                # Allow time for graceful shutdown.
-                timeout = 2  # seconds
-                start = time.time()
-                while self.process.poll() is None and (time.time() - start) < timeout:
-                    time.sleep(0.1)
-                # If still running, kill aggressively.
-                if self.process.poll() is None:
+                    print(f"Error terminating child process {child.pid}: {e}")
+            psutil.wait_procs(children, timeout=2)
+            # Kill any that remain.
+            for child in children:
+                if child.is_running():
                     try:
-                        self.process.kill()
+                        child.kill()
                     except Exception as e:
-                        print(f"Error calling kill on process {self.process.pid}: {e}")
-                # Optionally wait briefly for termination.
+                        print(f"Error killing child process {child.pid}: {e}")
+
+        # Terminate the parent process.
+        if parent.is_running():
+            try:
+                parent.terminate()
+            except Exception as e:
+                print(f"Error terminating process {parent.pid}: {e}")
+            try:
+                parent.wait(timeout=3)
+            except psutil.TimeoutExpired:
                 try:
-                    self.process.wait(timeout=1)
-                except Exception:
-                    pass
+                    parent.kill()
+                except Exception as e:
+                    print(f"Error killing process {parent.pid}: {e}")
+
+    def _atexit_terminate(self) -> None:
+        """
+        This method is registered via atexit and uses psutil to clean up the process tree.
+        It runs in a daemon thread so that termination happens without blocking interpreter shutdown.
+        """
+        if self.process.poll() is None:  # Process is still running.
+
+            def terminate_sequence():
+                self._kill_process_tree()
 
-            # Run the termination sequence in a separate daemon thread.
             t = threading.Thread(target=terminate_sequence, daemon=True)
             t.start()
             t.join(timeout=3)
@@ -122,12 +147,12 @@ class Process:
         self.cleanup()
 
     def kill(self) -> None:
-        self.cleanup()
-        return self.process.kill()
+        """Forcefully kill the process tree."""
+        self._kill_process_tree()
 
     def terminate(self) -> None:
-        self.cleanup()
-        return self.process.terminate()
+        """Gracefully terminate the process tree."""
+        self._kill_process_tree()
 
     @property
     def returncode(self) -> int | None:
@@ -147,8 +172,8 @@ class Process:
     def wait(self) -> int:
         return self.process.wait()
 
-    def send_signal(self, signal: int) -> None:
-        return self.process.send_signal(signal)
+    def send_signal(self, sig: int) -> None:
+        self.process.send_signal(sig)
 
     def __str__(self) -> str:
         state = ""

{rclone_api-1.4.14 → rclone_api-1.4.18}/src/rclone_api/s3/api.py

@@ -20,11 +20,11 @@ _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 
 
 class S3Client:
-    def __init__(self, credentials: S3Credentials, verbose: bool = False) -> None:
+    def __init__(self, s3_creds: S3Credentials, verbose: bool = False) -> None:
         self.verbose = verbose
-        self.credentials: S3Credentials = credentials
+        self.credentials: S3Credentials = s3_creds
         self.client: BaseClient = create_s3_client(
-            credentials, config=S3Config(verbose=verbose)
+            s3_creds=s3_creds, s3_config=S3Config(verbose=verbose)
         )
 
     def list_bucket_contents(self, bucket_name: str) -> None:

{rclone_api-1.4.14 → rclone_api-1.4.18}/src/rclone_api/s3/create.py

@@ -8,7 +8,7 @@ from botocore.config import Config
 from rclone_api.s3.types import S3Credentials, S3Provider
 
 _DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
-_MAX_CONNECTIONS = 50
+_MAX_CONNECTIONS = 10
 _TIMEOUT_READ = 120
 _TIMEOUT_CONNECT = 60
 
@@ -30,14 +30,16 @@ class S3Config:
 # Create a Boto3 session and S3 client, this is back blaze specific.
 # Add a function if you want to use a different S3 provider.
 # If AWS support is added in a fork then please merge it back here.
-def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_backblaze_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    region_name = creds.region_name
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    region_name = s3_creds.region_name
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -48,9 +50,9 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
         config=Config(
             signature_version="s3v4",
             region_name=region_name,
-            max_pool_connections=config.max_pool_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
             # Note that BackBlase has a boko3 bug where it doesn't support the new
             # checksum header, the following line was an attempt of fix it on the newest
             # version of boto3, but it didn't work.
@@ -59,18 +61,20 @@ def _create_backblaze_s3_client(creds: S3Credentials, config: S3Config) -> BaseC
     )
 
 
-def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseClient:
+def _create_unknown_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
     """Create and return an S3 client."""
-    access_key = creds.access_key_id
-    secret_key = creds.secret_access_key
-    endpoint_url = creds.endpoint_url
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
     if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
-        if config.verbose:
+        if s3_config.verbose:
             warnings.warn(
                 f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
             )
         endpoint_url = f"https://{endpoint_url}"
-    config.resolve_defaults()
+    s3_config.resolve_defaults()
     session = boto3.session.Session()  # type: ignore
     return session.client(
         service_name="s3",
@@ -79,25 +83,25 @@ def _create_unknown_s3_client(creds: S3Credentials, config: S3Config) -> BaseCli
         endpoint_url=endpoint_url,
         config=Config(
             signature_version="s3v4",
-            region_name=creds.region_name,
-            max_pool_connections=config.max_pool_connections,
-            read_timeout=config.timeout_read,
-            connect_timeout=config.timeout_connection,
+            region_name=s3_creds.region_name,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
         ),
     )
 
 
 def create_s3_client(
-    credentials: S3Credentials, config: S3Config | None = None
+    s3_creds: S3Credentials, s3_config: S3Config | None = None
 ) -> BaseClient:
     """Create and return an S3 client."""
-    config = config or S3Config()
-    provider = credentials.provider
+    s3_config = s3_config or S3Config()
+    provider = s3_creds.provider
     if provider == S3Provider.BACKBLAZE:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating BackBlaze S3 client")
-        return _create_backblaze_s3_client(creds=credentials, config=config)
+        return _create_backblaze_s3_client(s3_creds=s3_creds, s3_config=s3_config)
     else:
-        if config.verbose:
+        if s3_config.verbose:
             print("Creating generic/unknown S3 client")
-        return _create_unknown_s3_client(creds=credentials, config=config)
+        return _create_unknown_s3_client(s3_creds=s3_creds, s3_config=s3_config)

rclone_api-1.4.18/src/rclone_api/s3/merge_state.py

@@ -0,0 +1,115 @@
+"""
+https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
+  *  client.upload_part_copy
+
+This module provides functionality for S3 multipart uploads, including copying parts
+from existing S3 objects using upload_part_copy.
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from rclone_api.s3.multipart.finished_piece import FinishedPiece
+
+
+@dataclass
+class Part:
+    part_number: int
+    s3_key: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "s3_key": self.s3_key}
+
+    @staticmethod
+    def from_json(json_dict: dict) -> "Part | Exception":
+        part_number = json_dict.get("part_number")
+        s3_key = json_dict.get("s3_key")
+        if part_number is None or s3_key is None:
+            return Exception(f"Invalid JSON: {json_dict}")
+        return Part(part_number=part_number, s3_key=s3_key)
+
+    @staticmethod
+    def from_json_array(json_array: list[dict]) -> list["Part"] | Exception:
+        try:
+            out: list[Part] = []
+            for j in json_array:
+                ok_or_err = Part.from_json(j)
+                if isinstance(ok_or_err, Exception):
+                    return ok_or_err
+                else:
+                    out.append(ok_or_err)
+            return out
+        except Exception as e:
+            return e
+
+
+class MergeState:
+
+    def __init__(self, finished: list[FinishedPiece], all_parts: list[Part]) -> None:
+        self.finished: list[FinishedPiece] = list(finished)
+        self.all_parts: list[Part] = list(all_parts)
+        self.callbacks: list[Callable[[FinishedPiece], None]] = []
+
+    def add_callback(self, callback: Callable[[FinishedPiece], None]) -> None:
+        self.callbacks.append(callback)
+
+    def on_finished(self, finished_piece: FinishedPiece) -> None:
+        for callback in list(self.callbacks):
+            callback(finished_piece)
+
+    @staticmethod
+    def from_json_array(json_array: dict) -> "MergeState | Exception":
+        try:
+            finished: list[FinishedPiece] = FinishedPiece.from_json_array(
+                json_array["finished"]
+            )
+            all_parts: list[Part | Exception] = [
+                Part.from_json(j) for j in json_array["all"]
+            ]
+            all_parts_no_err: list[Part] = [
+                p for p in all_parts if not isinstance(p, Exception)
+            ]
+            errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
+            if len(errs):
+                return Exception(f"Errors in parts: {errs}")
+            return MergeState(finished=finished, all_parts=all_parts_no_err)
+        except Exception as e:
+            return e
+
+    def to_json(self) -> dict:
+        finished = self.finished.copy()
+        all_parts = self.all_parts.copy()
+        return {
+            "finished": FinishedPiece.to_json_array(finished),
+            "all": [part.to_json() for part in all_parts],
+        }
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.to_json(), indent=1)
+
+    def __str__(self):
+        return self.to_json_str()
+
+    def __repr__(self):
+        return self.to_json_str()
+
+    def write(self, rclone_impl: Any, dst: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = self.to_json_str()
+        rclone_impl.write_text(dst, json_str)
+
+    def read(self, rclone_impl: Any, src: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = rclone_impl.read_text(src)
+        if isinstance(json_str, Exception):
+            raise json_str
+        json_dict = json.loads(json_str)
+        ok_or_err = FinishedPiece.from_json_array(json_dict["finished"])
+        if isinstance(ok_or_err, Exception):
+            raise ok_or_err
+        self.finished = ok_or_err

{rclone_api-1.4.14 → rclone_api-1.4.18}/src/rclone_api/s3/multipart/finished_piece.py

@@ -17,7 +17,9 @@ class FinishedPiece:
         return json.dumps(self.to_json(), indent=0)
 
     @staticmethod
-    def to_json_array(parts: list["FinishedPiece | EndOfStream"]) -> list[dict]:
+    def to_json_array(
+        parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],
+    ) -> list[dict]:
         non_none: list[FinishedPiece] = []
         for p in parts:
             if not isinstance(p, EndOfStream):
@@ -39,3 +41,15 @@ class FinishedPiece:
         if json is None:
             return EndOfStream()
         return FinishedPiece(**json)
+
+    @staticmethod
+    def from_json_array(json: dict) -> list["FinishedPiece"]:
+        tmp = [FinishedPiece.from_json(j) for j in json]
+        out: list[FinishedPiece] = []
+        for t in tmp:
+            if isinstance(t, FinishedPiece):
+                out.append(t)
+        return out
+
+    def __hash__(self) -> int:
+        return hash(self.part_number)