rclone-api 1.4.19__tar.gz → 1.4.22__tar.gz

{rclone_api-1.4.19 → rclone_api-1.4.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.19
+Version: 1.4.22
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.19 → rclone_api-1.4.22}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.19"
+version = "1.4.22"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/__init__.py

@@ -428,11 +428,16 @@ class Rclone:
         src: str,  # src:/Bucket/path/myfile.large.zst
         dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/part.{part_number:05d}.start-end
         part_infos: list[PartInfo] | None = None,
-        threads: int = 1,  # Number of reader and writer threads to use
+        upload_threads: int = 8,  # Number of reader and writer threads to use
+        merge_threads: int = 4,  # Number of threads to use for merging the parts
     ) -> Exception | None:
         """Copy a file in parts."""
         return self.impl.copy_file_parts(
-            src=src, dst_dir=dst_dir, part_infos=part_infos, threads=threads
+            src=src,
+            dst_dir=dst_dir,
+            part_infos=part_infos,
+            upload_threads=upload_threads,
+            merge_threads=merge_threads,
         )
 
     def mount(
@@ -496,7 +501,7 @@ class Rclone:
         other_args: list[str] | None = None,
         check: bool | None = False,
         verbose: bool | None = None,
-    ) -> SizeResult:
+    ) -> SizeResult | Exception:
         """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
         return self.impl.size_files(
             src=src,

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/cmd/copy_large_s3.py

@@ -90,8 +90,6 @@ def main() -> int:
     err: Exception | None = rclone.copy_file_parts(
         src=args.src,
         dst_dir=args.dst,
-        threads=8,
-        # verbose=args.verbose,
     )
     if err is not None:
         print(f"Error: {err}")

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from rclone_api import Rclone
-from rclone_api.s3.s3_multipart_uploader_by_copy import (
+from rclone_api.s3.multipart.upload_parts_server_side_merge import (
     s3_server_side_multi_part_merge,
 )
 
@@ -57,7 +57,7 @@ def main() -> int:
     rclone = Rclone(rclone_conf=args.config_path)
     info_path = _get_info_path(src=args.src)
     s3_server_side_multi_part_merge(
-        rclone=rclone.impl, info_path=info_path, max_workers=1
+        rclone=rclone.impl, info_path=info_path, max_workers=5
     )
     return 0
 

rclone_api-1.4.22/src/rclone_api/detail/copy_file_parts_resumable.py

@@ -0,0 +1,42 @@
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.types import (
+    PartInfo,
+)
+
+
+def copy_file_parts_resumable(
+    self: RcloneImpl,
+    src: str,  # src:/Bucket/path/myfile.large.zst
+    dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
+    part_infos: list[PartInfo] | None = None,
+    upload_threads: int = 10,
+    merge_threads: int = 5,
+    verbose: bool | None = None,
+) -> Exception | None:
+    # _upload_parts
+    from rclone_api.s3.multipart.upload_parts_resumable import upload_parts_resumable
+    from rclone_api.s3.multipart.upload_parts_server_side_merge import (
+        s3_server_side_multi_part_merge,
+    )
+
+    if verbose is None:
+        verbose = self.get_verbose()
+
+    err: Exception | None = upload_parts_resumable(
+        self=self,
+        src=src,
+        dst_dir=dst_dir,
+        part_infos=part_infos,
+        threads=upload_threads,
+    )
+    if isinstance(err, Exception):
+        return err
+    if dst_dir.endswith("/"):
+        dst_dir = dst_dir[:-1]
+    dst_info = f"{dst_dir}/info.json"
+    err = s3_server_side_multi_part_merge(
+        rclone=self, info_path=dst_info, max_workers=merge_threads, verbose=verbose
+    )
+    if isinstance(err, Exception):
+        return err
+    return None

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/http_server.py

@@ -86,7 +86,10 @@ class HttpServer:
                                 assert response.is_closed
                     # print(f"Downloaded bytes {start}-{end} to {dst}")
                     if range:
-                        print(f"Downloaded bytes {range.start}-{range.end} to {dst}")
+                        length = range.end - range.start
+                        print(
+                            f"Downloaded bytes starting at {range.start} with size {length} to {dst}"
+                        )
                     else:
                         size = dst.stat().st_size
                         print(f"Downloaded {size} bytes to {dst}")

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/rclone_impl.py

@@ -455,6 +455,9 @@ class RcloneImpl:
         out = self._run(cmd)
         return CompletedProcess.from_subprocess(out)
 
+    def get_verbose(self) -> bool:
+        return get_verbose(None)
+
     def copy_to(
         self,
         src: File | str,
@@ -789,17 +792,21 @@ class RcloneImpl:
         src: str,  # src:/Bucket/path/myfile.large.zst
         dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
         part_infos: list[PartInfo] | None = None,
-        threads: int = 1,
+        upload_threads: int = 8,
+        merge_threads: int = 4,
     ) -> Exception | None:
         """Copy parts of a file from source to destination."""
-        from rclone_api.detail.copy_file_parts import copy_file_parts
+        from rclone_api.detail.copy_file_parts_resumable import (
+            copy_file_parts_resumable,
+        )
 
-        out = copy_file_parts(
+        out = copy_file_parts_resumable(
             self=self,
             src=src,
             dst_dir=dst_dir,
             part_infos=part_infos,
-            threads=threads,
+            upload_threads=upload_threads,
+            merge_threads=merge_threads,
         )
         return out
 
@@ -853,15 +860,25 @@ class RcloneImpl:
 
     def size_file(self, src: str) -> SizeSuffix | Exception:
         """Get the size of a file or directory."""
-        src_parent = os.path.dirname(src)
-        src_name = os.path.basename(src)
-        out: SizeResult = self.size_files(src_parent, [src_name])
-        one_file = len(out.file_sizes) == 1
-        if not one_file:
-            return Exception(
-                f"More than one result returned, is this is a directory? {out}"
-            )
-        return SizeSuffix(out.total_size)
+        # src_parent = os.path.dirname(src)
+        # src_name = os.path.basename(src)
+        # can't use this because it's only one file.
+        # out: SizeResult = self.size_files(src_parent, [src_name])
+        # one_file = len(out.file_sizes) == 1
+        # if not one_file:
+        #     return Exception(
+        #         f"More than one result returned, is this is a directory? {out}"
+        #     )
+        # return SizeSuffix(out.total_size)
+        dirlist: DirListing = self.ls(
+            src, listing_option=ListingOption.FILES_ONLY, max_depth=0
+        )
+        if len(dirlist.files) == 0:
+            return FileNotFoundError(f"File not found: {src}")
+        if len(dirlist.files) > 1:
+            return Exception(f"More than one file found: {src}")
+        file: File = dirlist.files[0]
+        return SizeSuffix(file.size)
 
     def get_s3_credentials(
         self, remote: str, verbose: bool | None = None
@@ -943,7 +960,9 @@ class RcloneImpl:
         name = src_path.name
         src_parent_path = Path(src).parent.as_posix()
 
-        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        size_result: SizeResult | Exception = self.size_files(src_parent_path, [name])
+        if isinstance(size_result, Exception):
+            raise size_result
         target_size = SizeSuffix(size_result.total_size)
 
         chunk_size = chunk_size or SizeSuffix("64M")
@@ -1286,10 +1305,18 @@ class RcloneImpl:
         other_args: list[str] | None = None,
         check: bool | None = False,
         verbose: bool | None = None,
-    ) -> SizeResult:
+    ) -> SizeResult | Exception:
         """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
         verbose = get_verbose(verbose)
         check = get_check(check)
+        if len(files) < 2:
+            tmp = self.size_file(files[0])
+            if isinstance(tmp, Exception):
+                return tmp
+            assert isinstance(tmp, SizeSuffix)
+            return SizeResult(
+                prefix=src, total_size=tmp.as_int(), file_sizes={files[0]: tmp.as_int()}
+            )
         if fast_list or (other_args and "--fast-list" in other_args):
             warnings.warn(
                 "It's not recommended to use --fast-list with size_files as this will perform poorly on large repositories since the entire repository has to be scanned."

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/s3/api.py

@@ -10,11 +10,11 @@ from rclone_api.s3.basic_ops import (
     upload_file,
 )
 from rclone_api.s3.create import S3Config, create_s3_client
-from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
-from rclone_api.s3.upload_file_multipart import (
+from rclone_api.s3.multipart.upload_parts_inline import (
     MultiUploadResult,
     upload_file_multipart,
 )
+from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
 
 _MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
 

{rclone_api-1.4.19 → rclone_api-1.4.22}/src/rclone_api/s3/multipart/finished_piece.py

@@ -10,7 +10,17 @@ class FinishedPiece:
     etag: str
 
     def to_json(self) -> dict:
-        return {"part_number": self.part_number, "etag": self.etag}
+        # return {"part_number": self.part_number, "etag": self.etag}
+        # amazon s3 style dict
+        tag = self.etag
+        if not tag.startswith('"'):
+            tag = f'"{tag}"'
+        out = {"PartNumber": self.part_number, "ETag": self.etag}
+        return out
+
+    def __post_init__(self):
+        assert isinstance(self.part_number, int)
+        assert isinstance(self.etag, str)
 
     @staticmethod
     def to_json_array(
@@ -30,13 +40,21 @@ class FinishedPiece:
         # assert count_eos <= 1, "Only one EndOfStream should be present"
         if count_eos > 1:
             warnings.warn(f"Only one EndOfStream should be present, found {count_eos}")
-        return [p.to_json() for p in non_none]
+        out = [p.to_json() for p in non_none]
+        return out
 
     @staticmethod
     def from_json(json: dict | None) -> "FinishedPiece | EndOfStream":
         if json is None:
             return EndOfStream()
-        return FinishedPiece(**json)
+        part_number = json.get("PartNumber")
+        etag = json.get("ETag")
+        assert isinstance(etag, str)
+        # handle the double quotes around the etag
+        etag = etag.replace('"', "")
+        assert isinstance(part_number, int)
+        assert isinstance(etag, str)
+        return FinishedPiece(part_number=part_number, etag=etag)
 
     @staticmethod
     def from_json_array(json: dict) -> list["FinishedPiece"]:

rclone_api-1.4.22/src/rclone_api/s3/multipart/info_json.py

@@ -0,0 +1,239 @@
+import hashlib
+import json
+import os
+import warnings
+from datetime import datetime
+
+from rclone_api.dir_listing import DirListing
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.types import (
+    PartInfo,
+    SizeSuffix,
+)
+
+
+def _fetch_all_names(
+    self: RcloneImpl,
+    src: str,
+) -> list[str]:
+    dl: DirListing = self.ls(src)
+    files = dl.files
+    filenames: list[str] = [f.name for f in files]
+    filtered: list[str] = [f for f in filenames if f.startswith("part.")]
+    return filtered
+
+
+def _get_info_json(self: RcloneImpl, src: str | None, src_info: str) -> dict:
+    from rclone_api.file import File
+
+    data: dict
+    text: str
+    if src is None:
+        # just try to load the file
+        text_or_err = self.read_text(src_info)
+        if isinstance(text_or_err, Exception):
+            raise FileNotFoundError(f"Could not load {src_info}: {text_or_err}")
+        assert isinstance(text_or_err, str)
+        text = text_or_err
+        data = json.loads(text)
+        return data
+
+    src_stat: File | Exception = self.stat(src)
+    if isinstance(src_stat, Exception):
+        # just try to load the file
+        raise FileNotFoundError(f"Failed to stat {src}: {src_stat}")
+
+    now: datetime = datetime.now()
+    new_data = {
+        "new": True,
+        "created": now.isoformat(),
+        "src": src,
+        "src_modtime": src_stat.mod_time(),
+        "size": src_stat.size,
+        "chunksize": None,
+        "chunksize_int": None,
+        "first_part": None,
+        "last_part": None,
+        "hash": None,
+    }
+
+    text_or_err = self.read_text(src_info)
+    err: Exception | None = text_or_err if isinstance(text_or_err, Exception) else None
+    if isinstance(text_or_err, Exception):
+        warnings.warn(f"Failed to read {src_info}: {text_or_err}")
+        return new_data
+    assert isinstance(text_or_err, str)
+    text = text_or_err
+
+    if err is not None:
+        return new_data
+
+    try:
+        data = json.loads(text)
+        return data
+    except Exception as e:
+        warnings.warn(f"Failed to parse JSON: {e} at {src_info}")
+        return new_data
+
+
+def _save_info_json(self: RcloneImpl, src: str, data: dict) -> None:
+    data = data.copy()
+    data["new"] = False
+    # hash
+
+    h = hashlib.md5()
+    tmp = [
+        data.get("src"),
+        data.get("src_modtime"),
+        data.get("size"),
+        data.get("chunksize_int"),
+    ]
+    data_vals: list[str] = [str(v) for v in tmp]
+    str_data = "".join(data_vals)
+    h.update(str_data.encode("utf-8"))
+    data["hash"] = h.hexdigest()
+    json_str = json.dumps(data, indent=0)
+    self.write_text(dst=src, text=json_str)
+
+
+class InfoJson:
+    def __init__(self, rclone: RcloneImpl, src: str | None, src_info: str) -> None:
+        self.rclone = rclone
+        self.src = src
+        self.src_info = src_info
+        self.data: dict = {}
+
+    def load(self) -> bool:
+        """Returns true if the file exist and is now loaded."""
+        self.data = _get_info_json(self.rclone, self.src, self.src_info)
+        return not self.data.get("new", False)
+
+    def save(self) -> None:
+        _save_info_json(self.rclone, self.src_info, self.data)
+
+    def print(self) -> None:
+        self.rclone.print(self.src_info)
+
+    def fetch_all_finished(self) -> list[str]:
+        parent_path = os.path.dirname(self.src_info)
+        out = _fetch_all_names(self.rclone, parent_path)
+        return out
+
+    def fetch_all_finished_part_numbers(self) -> list[int]:
+        names = self.fetch_all_finished()
+        part_numbers = [int(name.split("_")[0].split(".")[1]) for name in names]
+        return part_numbers
+
+    @property
+    def parts_dir(self) -> str:
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
+
+    @property
+    def dst(self) -> str:
+        parts_dir = self.parts_dir
+        assert parts_dir.endswith("-parts")
+        out = parts_dir[:-6]
+        return out
+
+    @property
+    def dst_name(self) -> str:
+        return os.path.basename(self.dst)
+
+    def compute_all_parts(self) -> list[PartInfo] | Exception:
+        # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
+        # src_size, SizeSuffix("96MB")
+        try:
+
+            src_size = self.size
+            chunk_size = self.chunksize
+            assert isinstance(src_size, SizeSuffix)
+            assert isinstance(chunk_size, SizeSuffix)
+            first_part = self.data["first_part"]
+            last_part = self.data["last_part"]
+            full_part_infos: list[PartInfo] = PartInfo.split_parts(src_size, chunk_size)
+            return full_part_infos[first_part : last_part + 1]
+        except Exception as e:
+            return e
+
+    def compute_all_part_numbers(self) -> list[int] | Exception:
+        all_parts: list[PartInfo] | Exception = self.compute_all_parts()
+        if isinstance(all_parts, Exception):
+            raise all_parts
+
+        all_part_nums: list[int] = [p.part_number for p in all_parts]
+        return all_part_nums
+
+    def fetch_remaining_part_numbers(self) -> list[int] | Exception:
+        all_part_nums: list[int] | Exception = self.compute_all_part_numbers()
+        if isinstance(all_part_nums, Exception):
+            return all_part_nums
+        finished_part_nums: list[int] = self.fetch_all_finished_part_numbers()
+        remaining_part_nums: list[int] = list(
+            set(all_part_nums) - set(finished_part_nums)
+        )
+        return sorted(remaining_part_nums)
+
+    def fetch_is_done(self) -> bool:
+        remaining_part_nums: list[int] | Exception = self.fetch_remaining_part_numbers()
+        if isinstance(remaining_part_nums, Exception):
+            return False
+        return len(remaining_part_nums) == 0
+
+    @property
+    def new(self) -> bool:
+        return self.data.get("new", False)
+
+    @property
+    def chunksize(self) -> SizeSuffix | None:
+        chunksize_int: int | None = self.data.get("chunksize_int")
+        if chunksize_int is None:
+            return None
+        return SizeSuffix(chunksize_int)
+
+    @chunksize.setter
+    def chunksize(self, value: SizeSuffix) -> None:
+        self.data["chunksize"] = str(value)
+        self.data["chunksize_int"] = value.as_int()
+
+    @property
+    def src_modtime(self) -> datetime:
+        return datetime.fromisoformat(self.data["src_modtime"])
+
+    @src_modtime.setter
+    def src_modtime(self, value: datetime) -> None:
+        self.data["src_modtime"] = value.isoformat()
+
+    @property
+    def size(self) -> SizeSuffix:
+        return SizeSuffix(self.data["size"])
+
+    def _get_first_part(self) -> int | None:
+        return self.data.get("first_part")
+
+    def _set_first_part(self, value: int) -> None:
+        self.data["first_part"] = value
+
+    def _get_last_part(self) -> int | None:
+        return self.data.get("last_part")
+
+    def _set_last_part(self, value: int) -> None:
+        self.data["last_part"] = value
+
+    first_part: int | None = property(_get_first_part, _set_first_part)  # type: ignore
+    last_part: int | None = property(_get_last_part, _set_last_part)  # type: ignore
+
+    @property
+    def hash(self) -> str | None:
+        return self.data.get("hash")
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.data)
+
+    def __repr__(self):
+        return f"InfoJson({self.src}, {self.src_info}, {self.data})"
+
+    def __str__(self):
+        return self.to_json_str()