rclone-api 1.4.7__tar.gz → 1.4.9__tar.gz

{rclone_api-1.4.7 → rclone_api-1.4.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.4.7
+Version: 1.4.9
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.4.7 → rclone_api-1.4.9}/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.4.7"
+version = "1.4.9"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.4.7 → rclone_api-1.4.9}/src/rclone_api/cmd/copy_large_s3.py

@@ -11,8 +11,7 @@ class Args:
     src: str
     dst: str
     chunk_size: SizeSuffix
-    read_threads: int
-    write_threads: int
+    threads: int
     retries: int
     save_state_json: Path
     verbose: bool
@@ -40,16 +39,10 @@ def _parse_args() -> Args:
         default="128MB",  # if this is too low or too high an s3 service
     )
     parser.add_argument(
-        "--read-threads",
-        help="Number of concurrent read threads per chunk, only one chunk will be read at a time",
-        type=int,
-        default=8,
-    )
-    parser.add_argument(
-        "--write-threads",
+        "--threads",
         help="Max number of chunks to upload in parallel to the destination, each chunk is uploaded in a separate thread",
         type=int,
-        default=16,
+        default=8,
     )
     parser.add_argument("--retries", help="Number of retries", type=int, default=3)
     parser.add_argument(
@@ -70,9 +63,8 @@ def _parse_args() -> Args:
         config_path=config,
         src=args.src,
         dst=args.dst,
+        threads=args.threads,
         chunk_size=SizeSuffix(args.chunk_size),
-        read_threads=args.read_threads,
-        write_threads=args.write_threads,
         retries=args.retries,
         save_state_json=args.resume_json,
         verbose=args.verbose,
@@ -122,6 +114,6 @@ if __name__ == "__main__":
         "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
     )
     sys.argv.append(
-        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts"
     )
     main()

rclone_api-1.4.9/src/rclone_api/cmd/copy_large_s3_finish.py

@@ -0,0 +1,228 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+from rclone_api.detail.copy_file_parts import InfoJson
+from rclone_api.s3.s3_multipart_uploader_by_copy import (
+    finish_multipart_upload_from_keys,
+)
+from rclone_api.types import SizeSuffix
+
+DATA_SOURCE = (
+    "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+)
+
+
+# response = client.upload_part_copy(
+#     Bucket='string',
+#     CopySource='string' or {'Bucket': 'string', 'Key': 'string', 'VersionId': 'string'},
+#     CopySourceIfMatch='string',
+#     CopySourceIfModifiedSince=datetime(2015, 1, 1),
+#     CopySourceIfNoneMatch='string',
+#     CopySourceIfUnmodifiedSince=datetime(2015, 1, 1),
+#     CopySourceRange='string',
+#     Key='string',
+#     PartNumber=123,
+#     UploadId='string',
+#     SSECustomerAlgorithm='string',
+#     SSECustomerKey='string',
+#     CopySourceSSECustomerAlgorithm='string',
+#     CopySourceSSECustomerKey='string',
+#     RequestPayer='requester',
+#     ExpectedBucketOwner='string',
+#     ExpectedSourceBucketOwner='string'
+# )
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
+    dst: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst
+    verbose: bool
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=False
+    )
+    parser.add_argument(
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
+        type=str,
+        default="128MB",  # if this is too low or too high an s3 service
+    )
+
+    args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
+    out = Args(
+        config_path=config,
+        src=args.src,
+        dst=args.dst,
+        verbose=args.verbose,
+    )
+    return out
+
+
+# from dataclasses import dataclass
+
+# def parse_info_json(text: str) -> UploadInfo:
+#     import json
+#     data = json.loads(text)
+#     chunk_size = data["chunksize_int"]
+#     first_part = data["first_part"]
+#     last_part = data["last_part"]
+#     assert isinstance(chunk_size, int)
+#     assert isinstance(first_part, int)
+#     assert isinstance(last_part, int)
+#     assert first_part <= last_part
+#     parts: list[str] = []
+#     fmt = "part.{:05d}_{}-{}"
+#     for i in range(first_part, last_part + 1):
+#         offset: int = i * chunk_size
+#         end: int = (i + 1) * chunk_size
+#         part = fmt.format(i, offset, end)
+#         parts.append(part)
+#     return UploadInfo(chunk_size=chunk_size, parts=parts)
+
+
+def do_finish_part(rclone: Rclone, info: InfoJson, dst: str) -> None:
+    from rclone_api.s3.create import BaseClient, S3Credentials, create_s3_client
+
+    s3_creds: S3Credentials = rclone.impl.get_s3_credentials(remote=dst)
+    s3_client: BaseClient = create_s3_client(s3_creds)
+    s3_bucket = s3_creds.bucket_name
+    is_done = info.fetch_is_done()
+    assert is_done, f"Upload is not done: {info}"
+
+    parts_dir = info.parts_dir
+    if parts_dir.endswith("/"):
+        parts_dir = parts_dir[:-1]
+    source_keys = info.fetch_all_finished()
+    # print(parts_dir)
+    # print(source_keys)
+
+    parts_path = parts_dir.split(s3_bucket)[1]
+    if parts_path.startswith("/"):
+        parts_path = parts_path[1:]
+
+    first_part: int | None = info.first_part
+    last_part: int | None = info.last_part
+    size: SizeSuffix | None = info.size
+
+    assert first_part is not None
+    assert last_part is not None
+    assert size is not None
+
+    def _to_s3_key(name: str | None) -> str:
+        if name:
+            out = f"{parts_path}/{name}"
+            return out
+        out = f"{parts_path}"
+        return out
+
+    # s3_keys: list[str] = [_to_s3_key(name=p) for p in source_keys]
+    parts: list[tuple[int, str]] = []
+    part_num = 1
+    for part_key in source_keys:
+        s3_key = _to_s3_key(name=part_key)
+        parts.append((part_num, s3_key))
+        part_num += 1
+
+    # for key in parts:
+    #     print(key)
+
+    chunksize = info.chunksize
+    assert chunksize is not None
+
+    import os
+
+    dst_name = info.dst_name
+    dst_dir = os.path.dirname(parts_path)
+    # dst_key =
+    dst_key = f"{dst_dir}/{dst_name}"
+
+    finish_multipart_upload_from_keys(
+        s3_client=s3_client,
+        source_bucket=s3_creds.bucket_name,
+        parts=parts,
+        destination_bucket=s3_creds.bucket_name,
+        destination_key=dst_key,
+        chunk_size=chunksize.as_int(),
+        final_size=size.as_int(),
+        max_workers=100,
+        retries=3,
+    )
+
+    if False:
+        print(finish_multipart_upload_from_keys)
+        print(s3_client)
+    print("done")
+
+    # def finish_multipart_upload_from_keys(
+    # s3_client: BaseClient,
+    # source_bucket: str,
+    # source_keys: list[str],
+    # destination_bucket: str,
+    # destination_key: str,
+    # chunk_size: int = 5 * 1024 * 1024,  # 5MB default
+    # retries: int = 3,
+    # byte_ranges: list[str] | None = None,
+
+    # if False:
+    #     finish_multipart_upload_from_keys(
+    #         s3_client=s3_client,
+    #         source_bucket="TODO",
+    #         source_keys=[p.key for p in all_parts],
+    #         destination_bucket=info.dst_bucket,
+    #         destination_key=info.dst_key,
+    #         chunk_size=5 * 1024 * 1024,
+    #         retries=3,
+    #         byte_ranges=None,
+    #     )
+
+    # print(all_parts)
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    info_json = f"{args.src}/info.json".replace("//", "/")
+    info = InfoJson(rclone.impl, src=None, src_info=info_json)
+    loaded = info.load()
+    assert loaded
+    print(info)
+    do_finish_part(rclone=rclone, info=info, dst=args.dst)
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+    )
+    main()

{rclone_api-1.4.7 → rclone_api-1.4.9}/src/rclone_api/detail/copy_file_parts.py

@@ -53,7 +53,7 @@ def upload_task(self: RcloneImpl, upload_part: UploadPart) -> UploadPart:
         msg = "\n#########################################\n"
         msg += f"# Uploading {upload_part.chunk} to {upload_part.dst_part}\n"
         msg += "#########################################\n"
-        print
+        print(msg)
         self.copy_to(upload_part.chunk.as_posix(), upload_part.dst_part)
         return upload_part
     except Exception as e:
@@ -106,11 +106,24 @@ def _fetch_all_names(
     return filtered
 
 
-def _get_info_json(self: RcloneImpl, src: str, src_info: str) -> dict:
+def _get_info_json(self: RcloneImpl, src: str | None, src_info: str) -> dict:
     from rclone_api.file import File
 
+    data: dict
+    text: str
+    if src is None:
+        # just try to load the file
+        text_or_err = self.read_text(src_info)
+        if isinstance(text_or_err, Exception):
+            raise FileNotFoundError(f"Could not load {src_info}: {text_or_err}")
+        assert isinstance(text_or_err, str)
+        text = text_or_err
+        data = json.loads(text)
+        return data
+
     src_stat: File | Exception = self.stat(src)
     if isinstance(src_stat, Exception):
+        # just try to load the file
         raise FileNotFoundError(f"Failed to stat {src}: {src_stat}")
 
     now: datetime = datetime.now()
@@ -133,12 +146,11 @@ def _get_info_json(self: RcloneImpl, src: str, src_info: str) -> dict:
         warnings.warn(f"Failed to read {src_info}: {text_or_err}")
         return new_data
     assert isinstance(text_or_err, str)
-    text: str = text_or_err
+    text = text_or_err
 
     if err is not None:
         return new_data
 
-    data: dict = {}
     try:
         data = json.loads(text)
         return data
@@ -168,13 +180,14 @@ def _save_info_json(self: RcloneImpl, src: str, data: dict) -> None:
 
 
 class InfoJson:
-    def __init__(self, rclone: RcloneImpl, src: str, src_info: str) -> None:
+    def __init__(self, rclone: RcloneImpl, src: str | None, src_info: str) -> None:
         self.rclone = rclone
         self.src = src
         self.src_info = src_info
         self.data: dict = {}
 
     def load(self) -> bool:
+        """Returns true if the file exist and is now loaded."""
         self.data = _get_info_json(self.rclone, self.src, self.src_info)
         return not self.data.get("new", False)
 
@@ -194,16 +207,71 @@ class InfoJson:
         part_numbers = [int(name.split("_")[0].split(".")[1]) for name in names]
         return part_numbers
 
+    @property
+    def parts_dir(self) -> str:
+        return os.path.dirname(self.src_info)
+
+    @property
+    def dst(self) -> str:
+        parts_dir = self.parts_dir
+        assert parts_dir.endswith("-parts")
+        out = parts_dir[:-6]
+        return out
+
+    @property
+    def dst_name(self) -> str:
+        return os.path.basename(self.dst)
+
+    def compute_all_parts(self) -> list[PartInfo] | Exception:
+        # full_part_infos: list[PartInfo] | Exception = PartInfo.split_parts(
+        # src_size, SizeSuffix("96MB")
+        try:
+
+            src_size = self.size
+            chunk_size = self.chunksize
+            assert isinstance(src_size, SizeSuffix)
+            assert isinstance(chunk_size, SizeSuffix)
+            first_part = self.data["first_part"]
+            last_part = self.data["last_part"]
+            full_part_infos: list[PartInfo] = PartInfo.split_parts(src_size, chunk_size)
+            return full_part_infos[first_part : last_part + 1]
+        except Exception as e:
+            return e
+
+    def compute_all_part_numbers(self) -> list[int] | Exception:
+        all_parts: list[PartInfo] | Exception = self.compute_all_parts()
+        if isinstance(all_parts, Exception):
+            raise all_parts
+
+        all_part_nums: list[int] = [p.part_number for p in all_parts]
+        return all_part_nums
+
+    def fetch_remaining_part_numbers(self) -> list[int] | Exception:
+        all_part_nums: list[int] | Exception = self.compute_all_part_numbers()
+        if isinstance(all_part_nums, Exception):
+            return all_part_nums
+        finished_part_nums: list[int] = self.fetch_all_finished_part_numbers()
+        remaining_part_nums: list[int] = list(
+            set(all_part_nums) - set(finished_part_nums)
+        )
+        return sorted(remaining_part_nums)
+
+    def fetch_is_done(self) -> bool:
+        remaining_part_nums: list[int] | Exception = self.fetch_remaining_part_numbers()
+        if isinstance(remaining_part_nums, Exception):
+            return False
+        return len(remaining_part_nums) == 0
+
     @property
     def new(self) -> bool:
         return self.data.get("new", False)
 
     @property
     def chunksize(self) -> SizeSuffix | None:
-        chunksize: str | None = self.data.get("chunksize")
-        if chunksize is None:
+        chunksize_int: int | None = self.data.get("chunksize_int")
+        if chunksize_int is None:
             return None
-        return SizeSuffix(chunksize)
+        return SizeSuffix(chunksize_int)
 
     @chunksize.setter
     def chunksize(self, value: SizeSuffix) -> None:
@@ -219,21 +287,24 @@ class InfoJson:
         self.data["src_modtime"] = value.isoformat()
 
     @property
-    def first_part(self) -> int | None:
+    def size(self) -> SizeSuffix:
+        return SizeSuffix(self.data["size"])
+
+    def _get_first_part(self) -> int | None:
         return self.data.get("first_part")
 
-    @first_part.setter
-    def first_part(self, value: int) -> None:
+    def _set_first_part(self, value: int) -> None:
         self.data["first_part"] = value
 
-    @property
-    def last_part(self) -> int | None:
+    def _get_last_part(self) -> int | None:
         return self.data.get("last_part")
 
-    @last_part.setter
-    def last_part(self, value: int) -> None:
+    def _set_last_part(self, value: int) -> None:
         self.data["last_part"] = value
 
+    first_part: int | None = property(_get_first_part, _set_first_part)  # type: ignore
+    last_part: int | None = property(_get_last_part, _set_last_part)  # type: ignore
+
     @property
     def hash(self) -> str | None:
         return self.data.get("hash")

{rclone_api-1.4.7 → rclone_api-1.4.9}/src/rclone_api/process.py

@@ -5,11 +5,10 @@ import time
 import weakref
 from dataclasses import dataclass
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from typing import Any
 
 from rclone_api.config import Config
-from rclone_api.util import get_verbose
+from rclone_api.util import clear_temp_config_file, get_verbose, make_temp_config_file
 
 
 @dataclass
@@ -28,17 +27,14 @@ class Process:
         assert args.rclone_exe.exists()
         self.args = args
         self.log = args.log
-        self.tempdir: TemporaryDirectory | None = None
+        self.tempfile: Path | None = None
         verbose = get_verbose(args.verbose)
         if isinstance(args.rclone_conf, Config):
-            self.tempdir = TemporaryDirectory()
-            tmpfile = Path(self.tempdir.name) / "rclone.conf"
-            tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
-            rclone_conf = tmpfile
-            self.needs_cleanup = True
+            self.tmpfile = make_temp_config_file()
+            self.tmpfile.write_text(args.rclone_conf.text, encoding="utf-8")
+            rclone_conf = self.tmpfile
         else:
             rclone_conf = args.rclone_conf
-            self.needs_cleanup = False
 
         assert rclone_conf.exists()
 
@@ -81,11 +77,7 @@ class Process:
         self.cleanup()
 
     def cleanup(self) -> None:
-        if self.tempdir and self.needs_cleanup:
-            try:
-                self.tempdir.cleanup()
-            except Exception as e:
-                print(f"Error cleaning up tempdir: {e}")
+        clear_temp_config_file(self.tempfile)
 
     def _atexit_terminate(self) -> None:
         """

{rclone_api-1.4.7 → rclone_api-1.4.9}/src/rclone_api/rclone_impl.py

@@ -32,6 +32,7 @@ from rclone_api.mount import Mount, clean_mount, prepare_mount
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.s3.create import S3Credentials
 from rclone_api.s3.types import (
     MultiUploadResult,
     S3MutliPartUploadConfig,
@@ -862,6 +863,64 @@ class RcloneImpl:
             )
         return SizeSuffix(out.total_size)
 
+    def get_s3_credentials(
+        self, remote: str, verbose: bool | None = None
+    ) -> S3Credentials:
+        from rclone_api.util import S3PathInfo, split_s3_path
+
+        verbose = get_verbose(verbose)
+        path_info: S3PathInfo = split_s3_path(remote)
+
+        # path_info: S3PathInfo = split_s3_path(remote)
+        remote = path_info.remote
+        bucket_name = path_info.bucket
+
+        remote = path_info.remote
+        parsed: Parsed = self.config.parse()
+        sections: dict[str, Section] = parsed.sections
+        if remote not in sections:
+            raise ValueError(
+                f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
+            )
+
+        section: Section = sections[remote]
+        dst_type = section.type()
+        if dst_type != "s3" and dst_type != "b2":
+            raise ValueError(
+                f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
+            )
+
+        def get_provider_str(section=section) -> str | None:
+            type: str = section.type()
+            provider: str | None = section.provider()
+            if provider is not None:
+                return provider
+            if type == "b2":
+                return S3Provider.BACKBLAZE.value
+            if type != "s3":
+                raise ValueError(f"Remote {remote} is not an S3 remote")
+            return S3Provider.S3.value
+
+        provider: str
+        if provided_provider_str := get_provider_str():
+            if verbose:
+                print(f"Using provided provider: {provided_provider_str}")
+            provider = provided_provider_str
+        else:
+            if verbose:
+                print(f"Using default provider: {S3Provider.S3.value}")
+            provider = S3Provider.S3.value
+        provider_enum = S3Provider.from_str(provider)
+
+        s3_creds: S3Credentials = S3Credentials(
+            bucket_name=bucket_name,
+            provider=provider_enum,
+            access_key_id=section.access_key_id(),
+            secret_access_key=section.secret_access_key(),
+            endpoint_url=section.endpoint(),
+        )
+        return s3_creds
+
     def copy_file_resumable_s3(
         self,
         src: str,
@@ -878,7 +937,6 @@ class RcloneImpl:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.http_server import HttpFetcher, HttpServer
         from rclone_api.s3.api import S3Client
-        from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, split_s3_path
 
         src_path = Path(src)
@@ -909,51 +967,10 @@ class RcloneImpl:
             )
 
         path_info: S3PathInfo = split_s3_path(dst)
-        remote = path_info.remote
+        # remote = path_info.remote
         bucket_name = path_info.bucket
         s3_key = path_info.key
-        parsed: Parsed = self.config.parse()
-        sections: dict[str, Section] = parsed.sections
-        if remote not in sections:
-            raise ValueError(
-                f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
-            )
-
-        section: Section = sections[remote]
-        dst_type = section.type()
-        if dst_type != "s3" and dst_type != "b2":
-            raise ValueError(
-                f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
-            )
-
-        def get_provider_str(section=section) -> str | None:
-            type: str = section.type()
-            provider: str | None = section.provider()
-            if provider is not None:
-                return provider
-            if type == "b2":
-                return S3Provider.BACKBLAZE.value
-            if type != "s3":
-                raise ValueError(f"Remote {remote} is not an S3 remote")
-            return S3Provider.S3.value
-
-        provider: str
-        if provided_provider_str := get_provider_str():
-            if verbose:
-                print(f"Using provided provider: {provided_provider_str}")
-            provider = provided_provider_str
-        else:
-            if verbose:
-                print(f"Using default provider: {S3Provider.S3.value}")
-            provider = S3Provider.S3.value
-        provider_enum = S3Provider.from_str(provider)
-
-        s3_creds: S3Credentials = S3Credentials(
-            provider=provider_enum,
-            access_key_id=section.access_key_id(),
-            secret_access_key=section.secret_access_key(),
-            endpoint_url=section.endpoint(),
-        )
+        s3_creds: S3Credentials = self.get_s3_credentials(dst, verbose=verbose)
 
         port = random.randint(10000, 20000)
         http_server: HttpServer = self.serve_http(

{rclone_api-1.4.7 → rclone_api-1.4.9}/src/rclone_api/s3/create.py

@@ -25,7 +25,7 @@ def _create_backblaze_s3_client(creds: S3Credentials, verbose: bool) -> BaseClie
         aws_access_key_id=access_key,
         aws_secret_access_key=secret_key,
         endpoint_url=endpoint_url,
-        verify=False,  # Disables SSL certificate verification
+        # verify=False,  # Disables SSL certificate verification
         config=Config(
             signature_version="s3v4",
             region_name=region_name,

{rclone_api-1.4.7 → rclone_api-1.4.9}/src/rclone_api/s3/multipart/finished_piece.py

@@ -1,4 +1,5 @@
 import json
+import warnings
 from dataclasses import dataclass
 
 from rclone_api.types import EndOfStream
@@ -28,7 +29,9 @@ class FinishedPiece:
         for p in parts:
             if p is EndOfStream:
                 count_eos += 1
-        assert count_eos <= 1, "Only one EndOfStream should be present"
+        # assert count_eos <= 1, "Only one EndOfStream should be present"
+        if count_eos > 1:
+            warnings.warn(f"Only one EndOfStream should be present, found {count_eos}")
         return [p.to_json() for p in non_none]
 
     @staticmethod