rclone-api 1.5.8__py3-none-any.whl

rclone_api/s3/multipart/info_json.py

@@ -0,0 +1,239 @@
+import hashlib
+import json
+import os
+import warnings
+from datetime import datetime
+
+from rclone_api.dir_listing import DirListing
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.types import (
+    PartInfo,
+    SizeSuffix,
+)
+
+
+def _fetch_all_names(
+    self: RcloneImpl,
+    src: str,
+) -> list[str]:
+    dl: DirListing = self.ls(src)
+    files = dl.files
+    filenames: list[str] = [f.name for f in files]
+    filtered: list[str] = [f for f in filenames if f.startswith("part.")]
+    return filtered
+
+
+def _get_info_json(self: RcloneImpl, src: str | None, src_info: str) -> dict:
+    from rclone_api.file import File
+
+    data: dict
+    text: str
+    if src is None:
+        # just try to load the file
+        text_or_err = self.read_text(src_info)
+        if isinstance(text_or_err, Exception):
+            raise FileNotFoundError(f"Could not load {src_info}: {text_or_err}")
+        assert isinstance(text_or_err, str)
+        text = text_or_err
+        data = json.loads(text)
+        return data
+
+    src_stat: File | Exception = self.stat(src)
+    if isinstance(src_stat, Exception):
+        # just try to load the file
+        raise FileNotFoundError(f"Failed to stat {src}: {src_stat}")
+
+    now: datetime = datetime.now()
+    new_data = {
+        "new": True,
+        "created": now.isoformat(),
+        "src": src,
+        "src_modtime": src_stat.mod_time(),
+        "size": src_stat.size,
+        "chunksize": None,
+        "chunksize_int": None,
+        "first_part": None,
+        "last_part": None,
+        "hash": None,
+    }
+
+    text_or_err = self.read_text(src_info)
+    err: Exception | None = text_or_err if isinstance(text_or_err, Exception) else None
+    if isinstance(text_or_err, Exception):
+        warnings.warn(f"Failed to read {src_info}: {text_or_err}")
+        return new_data
+    assert isinstance(text_or_err, str)
+    text = text_or_err
+
+    if err is not None:
+        return new_data
+
+    try:
+        data = json.loads(text)
+        return data
+    except Exception as e:
+        warnings.warn(f"Failed to parse JSON: {e} at {src_info}")
+        return new_data
+
+
+def _save_info_json(self: RcloneImpl, src: str, data: dict) -> None:
+    data = data.copy()
+    data["new"] = False
+    # hash
+
+    h = hashlib.md5()
+    tmp = [
+        data.get("src"),
+        data.get("src_modtime"),
+        data.get("size"),
+        data.get("chunksize_int"),
+    ]
+    data_vals: list[str] = [str(v) for v in tmp]
+    str_data = "".join(data_vals)
+    h.update(str_data.encode("utf-8"))
+    data["hash"] = h.hexdigest()
+    json_str = json.dumps(data, indent=0)
+    self.write_text(dst=src, text=json_str)
+
+
+class InfoJson:
+    def __init__(self, rclone: RcloneImpl, src: str | None, src_info: str) -> None:
+        self.rclone = rclone
+        self.src = src
+        self.src_info = src_info
+        self.data: dict = {}
+
+    def load(self) -> bool:
+        """Returns true if the file exist and is now loaded."""
+        self.data = _get_info_json(self.rclone, self.src, self.src_info)
+        return not self.data.get("new", False)
+
+    def save(self) -> None:
+        _save_info_json(self.rclone, self.src_info, self.data)
+
+    def print(self) -> None:
+        self.rclone.print(self.src_info)
+
+    def fetch_all_finished(self) -> list[str]:
+        parent_path = os.path.dirname(self.src_info)
+        out = _fetch_all_names(self.rclone, parent_path)
+        return out
+
+    def fetch_all_finished_part_numbers(self) -> list[int]:
+        names = self.fetch_all_finished()
+        part_numbers = [int(name.split("_")[0].split(".")[1]) for name in names]
+        return part_numbers
+
+    @property
+    def parts_dir(self) -> str:
+        parts_dir = os.path.dirname(self.src_info)
+        if parts_dir.endswith("/"):
+            parts_dir = parts_dir[:-1]
+        return parts_dir
+
+    @property
+    def dst(self) -> str:
+        parts_dir = self.parts_dir
+        assert parts_dir.endswith("-parts")
+        out = parts_dir[:-6]
+        return out
+
+    @property
+    def dst_name(self) -> str:
+        return os.path.basename(self.dst)
+
+    def compute_all_parts(self) -> list[PartInfo] | Exception:
+        # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
+        # src_size, SizeSuffix("96MB")
+        try:
+
+            src_size = self.size
+            chunk_size = self.chunksize
+            assert isinstance(src_size, SizeSuffix)
+            assert isinstance(chunk_size, SizeSuffix)
+            first_part = self.data["first_part"]
+            last_part = self.data["last_part"]
+            full_part_infos: list[PartInfo] = PartInfo.split_parts(src_size, chunk_size)
+            return full_part_infos[first_part : last_part + 1]
+        except Exception as e:
+            return e
+
+    def compute_all_part_numbers(self) -> list[int] | Exception:
+        all_parts: list[PartInfo] | Exception = self.compute_all_parts()
+        if isinstance(all_parts, Exception):
+            raise all_parts
+
+        all_part_nums: list[int] = [p.part_number for p in all_parts]
+        return all_part_nums
+
+    def fetch_remaining_part_numbers(self) -> list[int] | Exception:
+        all_part_nums: list[int] | Exception = self.compute_all_part_numbers()
+        if isinstance(all_part_nums, Exception):
+            return all_part_nums
+        finished_part_nums: list[int] = self.fetch_all_finished_part_numbers()
+        remaining_part_nums: list[int] = list(
+            set(all_part_nums) - set(finished_part_nums)
+        )
+        return sorted(remaining_part_nums)
+
+    def fetch_is_done(self) -> bool:
+        remaining_part_nums: list[int] | Exception = self.fetch_remaining_part_numbers()
+        if isinstance(remaining_part_nums, Exception):
+            return False
+        return len(remaining_part_nums) == 0
+
+    @property
+    def new(self) -> bool:
+        return self.data.get("new", False)
+
+    @property
+    def chunksize(self) -> SizeSuffix | None:
+        chunksize_int: int | None = self.data.get("chunksize_int")
+        if chunksize_int is None:
+            return None
+        return SizeSuffix(chunksize_int)
+
+    @chunksize.setter
+    def chunksize(self, value: SizeSuffix) -> None:
+        self.data["chunksize"] = str(value)
+        self.data["chunksize_int"] = value.as_int()
+
+    @property
+    def src_modtime(self) -> datetime:
+        return datetime.fromisoformat(self.data["src_modtime"])
+
+    @src_modtime.setter
+    def src_modtime(self, value: datetime) -> None:
+        self.data["src_modtime"] = value.isoformat()
+
+    @property
+    def size(self) -> SizeSuffix:
+        return SizeSuffix(self.data["size"])
+
+    def _get_first_part(self) -> int | None:
+        return self.data.get("first_part")
+
+    def _set_first_part(self, value: int) -> None:
+        self.data["first_part"] = value
+
+    def _get_last_part(self) -> int | None:
+        return self.data.get("last_part")
+
+    def _set_last_part(self, value: int) -> None:
+        self.data["last_part"] = value
+
+    first_part: int | None = property(_get_first_part, _set_first_part)  # type: ignore
+    last_part: int | None = property(_get_last_part, _set_last_part)  # type: ignore
+
+    @property
+    def hash(self) -> str | None:
+        return self.data.get("hash")
+
+    def to_json_str(self) -> str:
+        return json.dumps(self.data)
+
+    def __repr__(self):
+        return f"InfoJson({self.src}, {self.src_info}, {self.data})"
+
+    def __str__(self):
+        return self.to_json_str()

rclone_api/s3/multipart/merge_state.py

@@ -0,0 +1,147 @@
+"""
+https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_part_copy.html
+  *  client.upload_part_copy
+
+This module provides functionality for S3 multipart uploads, including copying parts
+from existing S3 objects using upload_part_copy.
+"""
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from rclone_api.rclone_impl import RcloneImpl
+from rclone_api.s3.multipart.finished_piece import FinishedPiece
+
+
+@dataclass
+class Part:
+    part_number: int
+    s3_key: str
+
+    def to_json(self) -> dict:
+        return {"part_number": self.part_number, "s3_key": self.s3_key}
+
+    @staticmethod
+    def from_json(json_dict: dict) -> "Part | Exception":
+        part_number = json_dict.get("part_number")
+        s3_key = json_dict.get("s3_key")
+        if part_number is None or s3_key is None:
+            return Exception(f"Invalid JSON: {json_dict}")
+        return Part(part_number=part_number, s3_key=s3_key)
+
+    @staticmethod
+    def from_json_array(json_array: list[dict]) -> list["Part"] | Exception:
+        try:
+            out: list[Part] = []
+            for j in json_array:
+                ok_or_err = Part.from_json(j)
+                if isinstance(ok_or_err, Exception):
+                    return ok_or_err
+                else:
+                    out.append(ok_or_err)
+            return out
+        except Exception as e:
+            return e
+
+
+class MergeState:
+
+    def __init__(
+        self,
+        rclone_impl: RcloneImpl,
+        merge_path: str,
+        upload_id: str,
+        bucket: str,
+        dst_key: str,
+        finished: list[FinishedPiece],
+        all_parts: list[Part],
+    ) -> None:
+        self.rclone_impl: RcloneImpl = rclone_impl
+        self.merge_path: str = merge_path
+        self.merge_parts_path: str = f"{merge_path}/merge"  # future use?
+        self.upload_id: str = upload_id
+        self.bucket: str = bucket
+        self.dst_key: str = dst_key
+        self.finished: list[FinishedPiece] = list(finished)
+        self.all_parts: list[Part] = list(all_parts)
+
+    def on_finished(self, finished_piece: FinishedPiece) -> None:
+        self.finished.append(finished_piece)
+
+    def remaining_parts(self) -> list[Part]:
+        finished_parts: set[int] = set([p.part_number for p in self.finished])
+        remaining = [p for p in self.all_parts if p.part_number not in finished_parts]
+        return remaining
+
+    @staticmethod
+    def from_json(rclone_impl: RcloneImpl, json: dict) -> "MergeState | Exception":
+        try:
+            merge_path = json["merge_path"]
+            bucket = json["bucket"]
+            dst_key = json["dst_key"]
+            finished: list[FinishedPiece] = FinishedPiece.from_json_array(
+                json["finished"]
+            )
+            all_parts: list[Part | Exception] = [Part.from_json(j) for j in json["all"]]
+            all_parts_no_err: list[Part] = [
+                p for p in all_parts if not isinstance(p, Exception)
+            ]
+            upload_id: str = json["upload_id"]
+            errs: list[Exception] = [p for p in all_parts if isinstance(p, Exception)]
+            if len(errs):
+                return Exception(f"Errors in parts: {errs}")
+            return MergeState(
+                rclone_impl=rclone_impl,
+                merge_path=merge_path,
+                upload_id=upload_id,
+                bucket=bucket,
+                dst_key=dst_key,
+                finished=finished,
+                all_parts=all_parts_no_err,
+            )
+        except Exception as e:
+            return e
+
+    def to_json(self) -> dict:
+        finished = self.finished.copy()
+        all_parts = self.all_parts.copy()
+        return {
+            "merge_path": self.merge_path,
+            "bucket": self.bucket,
+            "dst_key": self.dst_key,
+            "upload_id": self.upload_id,
+            "finished": FinishedPiece.to_json_array(finished),
+            "all": [part.to_json() for part in all_parts],
+        }
+
+    def to_json_str(self) -> str:
+        data = self.to_json()
+        out = json.dumps(data, indent=2)
+        return out
+
+    def __str__(self):
+        return self.to_json_str()
+
+    def __repr__(self):
+        return self.to_json_str()
+
+    def write(self, rclone_impl: Any, dst: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = self.to_json_str()
+        rclone_impl.write_text(dst, json_str)
+
+    def read(self, rclone_impl: Any, src: str) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone_impl, RcloneImpl)
+        json_str = rclone_impl.read_text(src)
+        if isinstance(json_str, Exception):
+            raise json_str
+        json_dict = json.loads(json_str)
+        ok_or_err = FinishedPiece.from_json_array(json_dict["finished"])
+        if isinstance(ok_or_err, Exception):
+            raise ok_or_err
+        self.finished = ok_or_err

rclone_api/s3/multipart/upload_info.py

@@ -0,0 +1,62 @@
+import hashlib
+from dataclasses import dataclass, fields
+from pathlib import Path
+
+from botocore.client import BaseClient
+
+
+@dataclass
+class UploadInfo:
+    s3_client: BaseClient
+    bucket_name: str
+    object_name: str
+    src_file_path: Path
+    upload_id: str
+    retries: int
+    chunk_size: int
+    file_size: int
+    _total_chunks: int | None = None
+
+    def total_chunks(self) -> int:
+        out = self.file_size // self.chunk_size
+        if self.file_size % self.chunk_size:
+            return out + 1
+        return out
+
+    def __post_init__(self):
+        if self._total_chunks is not None:
+            return
+        self._total_chunks = self.total_chunks()
+
+    def fingerprint(self) -> str:
+        # hash the attributes that are used to identify the upload
+        hasher = hashlib.sha256()
+        # first is file size
+        hasher.update(str(self.file_size).encode("utf-8"))
+        # next is chunk size
+        hasher.update(str(self.chunk_size).encode("utf-8"))
+        # next is the number of parts
+        hasher.update(str(self._total_chunks).encode("utf-8"))
+        return hasher.hexdigest()
+
+    def to_json(self) -> dict:
+        json_dict = {}
+        for f in fields(self):
+            value = getattr(self, f.name)
+            # Convert non-serializable objects (like s3_client) to a string representation.
+            if f.name == "s3_client":
+                continue
+            else:
+                if isinstance(value, Path):
+                    value = str(value)
+                json_dict[f.name] = value
+
+        return json_dict
+
+    @staticmethod
+    def from_json(s3_client: BaseClient, json_dict: dict) -> "UploadInfo":
+        # json_dict.pop("s3_client")  # Remove the placeholder string
+        if "s3_client" in json_dict:
+            json_dict.pop("s3_client")
+
+        return UploadInfo(s3_client=s3_client, **json_dict)