rclone-api 1.0.89__py2.py3-none-any.whl → 1.0.92__py2.py3-none-any.whl

rclone_api/cmd/copy_large_s3.py

@@ -0,0 +1,99 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import MultiUploadResult, Rclone
+
+_1MB = 1024 * 1024
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str
+    dst: str
+    chunk_size_mb: int
+    read_concurrent_chunks: int
+    retries: int
+    save_state_json: Path
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=True
+    )
+    parser.add_argument(
+        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+    )
+    parser.add_argument(
+        "--read-concurrent-chunks",
+        help="Maximum number of chunks to read",
+        type=int,
+        default=4,
+    )
+    parser.add_argument("--retries", help="Number of retries", type=int, default=3)
+    parser.add_argument(
+        "--resumable-json",
+        help="Path to resumable JSON file",
+        type=Path,
+        default="resume.json",
+    )
+
+    args = parser.parse_args()
+    out = Args(
+        config_path=Path(args.config),
+        src=args.src,
+        dst=args.dst,
+        chunk_size_mb=args.chunk_size_mb,
+        read_concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.resumable_json,
+    )
+    return out
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
+        src=args.src,
+        dst=args.dst,
+        chunk_size=args.chunk_size_mb * _1MB,
+        concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.save_state_json,
+    )
+    print(rslt)
+    return 0
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+
+    here = Path(__file__).parent
+    project_root = here.parent.parent.parent
+    print(f"project_root: {project_root}")
+    os.chdir(str(project_root))
+    cwd = Path(__file__).parent
+    print(f"cwd: {cwd}")
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    main()

rclone_api/config.py

@@ -10,14 +10,26 @@ class Section:
     def add(self, key: str, value: str) -> None:
         self.data[key] = value
 
-    def provider(self) -> str:
-        return self.data["provider"]
+    def type(self) -> str:
+        return self.data["type"]
+
+    def provider(self) -> str | None:
+        return self.data.get("provider")
 
     def access_key_id(self) -> str:
-        return self.data["access_key_id"]
+        if "access_key_id" in self.data:
+            return self.data["access_key_id"]
+        elif "account" in self.data:
+            return self.data["account"]
+        raise KeyError("No access key found")
 
     def secret_access_key(self) -> str:
-        return self.data["secret_access_key"]
+        # return self.data["secret_access_key"]
+        if "secret_access_key" in self.data:
+            return self.data["secret_access_key"]
+        elif "key" in self.data:
+            return self.data["key"]
+        raise KeyError("No secret access key found")
 
     def endpoint(self) -> str | None:
         return self.data.get("endpoint")

rclone_api/rclone.py

@@ -679,8 +679,13 @@ class Rclone:
         concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
         retries: int = 3,
         max_chunks_before_suspension: int | None = None,
+        mount_path: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        from rclone_api.s3.api import S3Client
+        from rclone_api.s3.create import S3Credentials
+        from rclone_api.util import S3PathInfo, random_str, split_s3_path
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
@@ -695,7 +700,7 @@ class Rclone:
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
         ]
-        mount_path = Path("rclone_api_upload_mount")
+        mount_path = mount_path or Path("tmp_mnts") / random_str(12)
         src_path = Path(src)
         name = src_path.name
 
@@ -708,8 +713,6 @@ class Rclone:
             other_args=other_args,
         ):
             # raise NotImplementedError("Not implemented yet")
-            from rclone_api.s3.create import S3Credentials
-            from rclone_api.util import S3PathInfo, split_s3_path
 
             path_info: S3PathInfo = split_s3_path(dst)
             remote = path_info.remote
@@ -723,7 +726,24 @@ class Rclone:
                 )
 
             section: Section = sections[remote]
-            provider: str = section.provider()
+            dst_type = section.type()
+            if dst_type != "s3" and dst_type != "b2":
+                raise ValueError(
+                    f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
+                )
+
+            def get_provider_str(section=section) -> str | None:
+                type: str = section.type()
+                provider: str | None = section.provider()
+                if provider is not None:
+                    return provider
+                if type == "b2":
+                    return S3Provider.BACKBLAZE.value
+                if type != "s3":
+                    raise ValueError(f"Remote {remote} is not an S3 remote")
+                return S3Provider.S3.value
+
+            provider: str = get_provider_str() or S3Provider.S3.value
             provider_enum = S3Provider.from_str(provider)
 
             s3_creds: S3Credentials = S3Credentials(
@@ -732,15 +752,8 @@ class Rclone:
                 secret_access_key=section.secret_access_key(),
                 endpoint_url=section.endpoint(),
             )
-            print(s3_creds)
-            # create_s3_client
-
-            print(f"Info: {section}")
-            from rclone_api.s3.api import S3Client
 
             client = S3Client(s3_creds)
-            print(f"Client: {client}")
-
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
                 retries=retries,
@@ -864,6 +877,7 @@ class Rclone:
         other_args: list[str] | None = None,
     ) -> Generator[Process, None, None]:
         """Like mount, but can be used in a context manager."""
+        error_happened = False
         proc = self.mount(
             src,
             outdir,
@@ -875,6 +889,7 @@ class Rclone:
         try:
             yield proc
         except Exception as e:
+            error_happened = True
             stack_trace = traceback.format_exc()
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
@@ -882,6 +897,24 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
+            if not error_happened and outdir.exists():
+                time.sleep(2)
+                if outdir.exists():
+                    print(f"{outdir} mount still exists, attempting to remove")
+                    if not _IS_WINDOWS:
+                        # attempt
+                        os.system(f"fusermount -u {outdir}")
+                        os.system(f"umount {outdir}")
+                        time.sleep(2)
+                        if outdir.exists():
+                            is_empty = not list(outdir.iterdir())
+                            if not is_empty:
+                                warnings.warn(f"Failed to unmount {outdir}")
+                            else:
+                                try:
+                                    outdir.rmdir()
+                                except Exception as e:
+                                    warnings.warn(f"Failed to remove {outdir}: {e}")
 
     @deprecated("mount")
     def mount_webdav(

rclone_api/s3/api.py

@@ -45,28 +45,40 @@ class S3Client:
         upload_target: S3UploadTarget,
         upload_config: S3MutliPartUploadConfig,
     ) -> MultiUploadResult:
-        filesize = upload_target.src_file.stat().st_size
-        if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
-            warnings.warn(
-                f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
-            )
-            err = self.upload_file(upload_target)
-            if err:
-                raise err
-            return MultiUploadResult.UPLOADED_FRESH
+
         chunk_size = upload_config.chunk_size
         retries = upload_config.retries
         resume_path_json = upload_config.resume_path_json
         max_chunks_before_suspension = upload_config.max_chunks_before_suspension
         bucket_name = upload_target.bucket_name
-        out = upload_file_multipart(
-            s3_client=self.client,
-            bucket_name=bucket_name,
-            file_path=upload_target.src_file,
-            object_name=upload_target.s3_key,
-            resumable_info_path=resume_path_json,
-            chunk_size=chunk_size,
-            retries=retries,
-            max_chunks_before_suspension=max_chunks_before_suspension,
-        )
-        return out
+
+        try:
+            filesize = upload_target.src_file.stat().st_size
+            if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
+                warnings.warn(
+                    f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
+                )
+                err = self.upload_file(upload_target)
+                if err:
+                    raise err
+                return MultiUploadResult.UPLOADED_FRESH
+
+            out = upload_file_multipart(
+                s3_client=self.client,
+                bucket_name=bucket_name,
+                file_path=upload_target.src_file,
+                object_name=upload_target.s3_key,
+                resumable_info_path=resume_path_json,
+                chunk_size=chunk_size,
+                retries=retries,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+            return out
+        except Exception:
+            key = upload_target.s3_key
+            access_key_id = self.credentials.access_key_id[:4] + "..."
+            secret = self.credentials.secret_access_key[:4] + "..."
+            warnings.warn(
+                f"Error uploading {key} to {bucket_name} with\n  access_key_id: {access_key_id}\n  secret: {secret}\n"
+            )
+            raise

rclone_api/s3/create.py

@@ -16,9 +16,7 @@ def _create_backblaze_s3_client(creds: S3Credentials) -> BaseClient:
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
-
-    if region_name is not None:
-        warnings.warn(f"Region name is not used for provider: {creds.provider}")
+    region_name = region_name or "https://s3.us-west-002.backblazeb2.com"
 
     session = boto3.session.Session()  # type: ignore
     return session.client(

rclone_api/s3/types.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 
 class S3Provider(Enum):
+    S3 = "s3"  # generic S3
     BACKBLAZE = "b2"
     DIGITAL_OCEAN = "DigitalOcean"
 
@@ -46,6 +47,9 @@ class S3MutliPartUploadConfig:
     retries: int
     resume_path_json: Path
     max_chunks_before_suspension: int | None = None
+    mount_path: Path | None = (
+        None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
+    )
 
 
 class MultiUploadResult(Enum):

rclone_api/util.py

@@ -141,6 +141,8 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
             # how many files?
             dircontents = os.listdir(str(path))
             if len(dircontents) > 0:
+                print(f"Mount point {path}, waiting 5 seconds for files to appear.")
+                time.sleep(5)
                 return
         time.sleep(1)
 
@@ -164,3 +166,10 @@ def split_s3_path(path: str) -> S3PathInfo:
     assert bucket
     assert key
     return S3PathInfo(remote=remote, bucket=bucket, key=key)
+
+
+def random_str(length: int) -> str:
+    import random
+    import string
+
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))

{rclone_api-1.0.89.dist-info → rclone_api-1.0.92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.89
+Version: 1.0.92
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.89.dist-info → rclone_api-1.0.92.dist-info}/RECORD

@@ -1,7 +1,7 @@
 rclone_api/__init__.py,sha256=m7hQkaIsn9grAlziu9nvDlOIQY6VflYFwC1c_g0qCJo,833
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
-rclone_api/config.py,sha256=okqhnkV2fAAwm9nGc92-uDL6g69MlAWN3P6OKmRCQRY,2099
+rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
 rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
 rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
 rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
@@ -12,23 +12,24 @@ rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=qKOgK544zWCmD4Uowc35X0gq1BNGLwcAjIHFgOF9mH8,40603
+rclone_api/rclone.py,sha256=tXVZv9N83DVVxVwevOo7FyzHCuUGjWx_KP51VwQ0QIY,42246
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
 rclone_api/types.py,sha256=NC3e78aXCx-sEQ-FqEaC9KzaJDdJhJrKa4Nwum_-Db0,563
-rclone_api/util.py,sha256=HwNqr_HmJvbK8uqHQRF6ockT9Nsgw4G1lUGEkEUkjEs,4910
+rclone_api/util.py,sha256=efck9W0rw5wfeRI35iiEz4dy2cMkNpVXrQ9zzynkBks,5185
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
+rclone_api/cmd/copy_large_s3.py,sha256=FPU0S1Y9pApVLmWcdMT_3llywjEtOtwobXTnMXE-FhY,2690
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api/s3/api.py,sha256=rFiUSQuW5q37CIfuN2mPq1aekFDMQ9lwH0qH9AQ9blg,2704
+rclone_api/s3/api.py,sha256=7elz5U5su4rdPHEMHHMBZ_vESQobJDTK_MR7sA_a1YU,3182
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_uploader.py,sha256=Wcm4h_A6ORaJvT9P9qSeknbJYch22RKZ4FGo1iePlAw,17385
-rclone_api/s3/create.py,sha256=M9_c30XYGUfqMY8o4Vl1OrVL7Fi416C7JqUEjDmb3_k,2741
-rclone_api/s3/types.py,sha256=hGzmWx040X4Dmk_C7hugNk-DT1Zqs4staJy1b8z6PlY,1188
-rclone_api-1.0.89.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.89.dist-info/METADATA,sha256=i1u6MZ5Go_-oYnV64Rbdm2zh8oOB6EPsPv3dGjEYQaI,4479
-rclone_api-1.0.89.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.0.89.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.89.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.89.dist-info/RECORD,,
+rclone_api/s3/create.py,sha256=X4mgjekgAoyHNeXnznHNS63DFViAhI0p7-fXDxml0y4,2701
+rclone_api/s3/types.py,sha256=81_3jwg6MGIxC-GxL-6zANzKO6au9C0BWvAqRyODxOM,1361
+rclone_api-1.0.92.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.92.dist-info/METADATA,sha256=rCGGQg-8Xgf7fRatzVH0cmaNtwVhXsFkc8NO7rPYVY8,4479
+rclone_api-1.0.92.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.0.92.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.0.92.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.92.dist-info/RECORD,,

{rclone_api-1.0.89.dist-info → rclone_api-1.0.92.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+rclone-api-copylarge-s3 = rclone_api.cmd.copy_large_s3:main
 rclone-api-listfiles = rclone_api.cmd.list_files:main