rclone-api 1.4.18__tar.gz → 1.4.19__tar.gz

{rclone_api-1.4.18 → rclone_api-1.4.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.18
+Version: 1.4.19
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.18 → rclone_api-1.4.19}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.18"
+version = "1.4.19"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

rclone_api-1.4.19/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -0,0 +1,73 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+from rclone_api.s3.s3_multipart_uploader_by_copy import (
+    s3_server_side_multi_part_merge,
+)
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
+    verbose: bool
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="Directory that holds the info.json file")
+    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=False
+    )
+    args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
+    out = Args(
+        config_path=config,
+        src=args.src,
+        verbose=args.verbose,
+    )
+    return out
+
+
+def _get_info_path(src: str) -> str:
+    if src.endswith("/"):
+        src = src[:-1]
+    info_path = f"{src}/info.json"
+    return info_path
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    info_path = _get_info_path(src=args.src)
+    s3_server_side_multi_part_merge(
+        rclone=rclone.impl, info_path=info_path, max_workers=1
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
+    )
+    main()

{rclone_api-1.4.18 → rclone_api-1.4.19}/src/rclone_api/detail/copy_file_parts.py

@@ -209,7 +209,10 @@ class InfoJson:
 
     @property
     def parts_dir(self) -> str:
-        return os.path.dirname(self.src_info)
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
 
     @property
     def dst(self) -> str:

{rclone_api-1.4.18 → rclone_api-1.4.19}/src/rclone_api/s3/merge_state.py

@@ -8,8 +8,9 @@ from existing S3 objects using upload_part_copy.
 
 import json
 from dataclasses import dataclass
-from typing import Any, Callable
+from typing import Any
 
+from rclone_api.rclone_impl import RcloneImpl
 from rclone_api.s3.multipart.finished_piece import FinishedPiece
 
 
@@ -46,34 +47,59 @@ class Part:
 
 class MergeState:
 
-    def __init__(self, finished: list[FinishedPiece], all_parts: list[Part]) -> None:
+    def __init__(
+        self,
+        rclone_impl: RcloneImpl,
+        merge_path: str,
+        upload_id: str,
+        bucket: str,
+        dst_key: str,
+        finished: list[FinishedPiece],
+        all_parts: list[Part],
+    ) -> None:
+        self.rclone_impl: RcloneImpl = rclone_impl
+        self.merge_path: str = merge_path
+        self.merge_parts_path: str = f"{merge_path}/merge"  # future use?
+        self.upload_id: str = upload_id
+        self.bucket: str = bucket
+        self.dst_key: str = dst_key
         self.finished: list[FinishedPiece] = list(finished)
         self.all_parts: list[Part] = list(all_parts)
-        self.callbacks: list[Callable[[FinishedPiece], None]] = []
-
-    def add_callback(self, callback: Callable[[FinishedPiece], None]) -> None:
-        self.callbacks.append(callback)
 
     def on_finished(self, finished_piece: FinishedPiece) -> None:
-        for callback in list(self.callbacks):
-            callback(finished_piece)
+        self.finished.append(finished_piece)
+
+    def remaining_parts(self) -> list[Part]:
+        finished_parts: set[int] = set([p.part_number for p in self.finished])
+        remaining = [p for p in self.all_parts if p.part_number not in finished_parts]
+        return remaining
 
     @staticmethod
-    def from_json_array(json_array: dict) -> "MergeState | Exception":
+    def from_json(rclone_impl: RcloneImpl, json: dict) -> "MergeState | Exception":
         try:
+            merge_path = json["merge_path"]
+            bucket = json["bucket"]
+            dst_key = json["dst_key"]
             finished: list[FinishedPiece] = FinishedPiece.from_json_array(
-                json_array["finished"]
+                json["finished"]
             )
-            all_parts: list[Part | Exception] = [
-                Part.from_json(j) for j in json_array["all"]
-            ]
+            all_parts: list[Part | Exception] = [Part.from_json(j) for j in json["all"]]
             all_parts_no_err: list[Part] = [
                 p for p in all_parts if not isinstance(p, Exception)
             ]
+            upload_id: str = json["upload_id"]
             errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
             if len(errs):
                 return Exception(f"Errors in parts: {errs}")
-            return MergeState(finished=finished, all_parts=all_parts_no_err)
+            return MergeState(
+                rclone_impl=rclone_impl,
+                merge_path=merge_path,
+                upload_id=upload_id,
+                bucket=bucket,
+                dst_key=dst_key,
+                finished=finished,
+                all_parts=all_parts_no_err,
+            )
         except Exception as e:
             return e
 
@@ -81,12 +107,18 @@ class MergeState:
         finished = self.finished.copy()
         all_parts = self.all_parts.copy()
         return {
+            "merge_path": self.merge_path,
+            "bucket": self.bucket,
+            "dst_key": self.dst_key,
+            "upload_id": self.upload_id,
             "finished": FinishedPiece.to_json_array(finished),
             "all": [part.to_json() for part in all_parts],
         }
 
     def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=1)
+        data = self.to_json()
+        out = json.dumps(data, indent=2)
+        return out
 
     def __str__(self):
         return self.to_json_str()

{rclone_api-1.4.18 → rclone_api-1.4.19}/src/rclone_api/s3/multipart/finished_piece.py

@@ -1,4 +1,3 @@
-import json
 import warnings
 from dataclasses import dataclass
 
@@ -13,9 +12,6 @@ class FinishedPiece:
     def to_json(self) -> dict:
         return {"part_number": self.part_number, "etag": self.etag}
 
-    def to_json_str(self) -> str:
-        return json.dumps(self.to_json(), indent=0)
-
     @staticmethod
     def to_json_array(
         parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],