rclone-api 1.0.89__tar.gz → 1.0.92__tar.gz

{rclone_api-1.0.89 → rclone_api-1.0.92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.89
+Version: 1.0.92
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.89 → rclone_api-1.0.92}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.0.89"
+version = "1.0.92"
 
 [tool.setuptools]
 package-dir = {"" = "src"}
@@ -51,4 +51,5 @@ ignore_missing_imports = true
 disable_error_code = ["import-untyped"]
 
 [project.scripts]
-rclone-api-listfiles = "rclone_api.cmd.list_files:main"
+rclone-api-listfiles = "rclone_api.cmd.list_files:main"
+rclone-api-copylarge-s3 = "rclone_api.cmd.copy_large_s3:main"

rclone_api-1.0.92/src/rclone_api/cmd/copy_large_s3.py

@@ -0,0 +1,99 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import MultiUploadResult, Rclone
+
+_1MB = 1024 * 1024
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str
+    dst: str
+    chunk_size_mb: int
+    read_concurrent_chunks: int
+    retries: int
+    save_state_json: Path
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=True
+    )
+    parser.add_argument(
+        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+    )
+    parser.add_argument(
+        "--read-concurrent-chunks",
+        help="Maximum number of chunks to read",
+        type=int,
+        default=4,
+    )
+    parser.add_argument("--retries", help="Number of retries", type=int, default=3)
+    parser.add_argument(
+        "--resumable-json",
+        help="Path to resumable JSON file",
+        type=Path,
+        default="resume.json",
+    )
+
+    args = parser.parse_args()
+    out = Args(
+        config_path=Path(args.config),
+        src=args.src,
+        dst=args.dst,
+        chunk_size_mb=args.chunk_size_mb,
+        read_concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.resumable_json,
+    )
+    return out
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
+        src=args.src,
+        dst=args.dst,
+        chunk_size=args.chunk_size_mb * _1MB,
+        concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.save_state_json,
+    )
+    print(rslt)
+    return 0
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+
+    here = Path(__file__).parent
+    project_root = here.parent.parent.parent
+    print(f"project_root: {project_root}")
+    os.chdir(str(project_root))
+    cwd = Path(__file__).parent
+    print(f"cwd: {cwd}")
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    main()

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api/config.py

@@ -10,14 +10,26 @@ class Section:
     def add(self, key: str, value: str) -> None:
         self.data[key] = value
 
-    def provider(self) -> str:
-        return self.data["provider"]
+    def type(self) -> str:
+        return self.data["type"]
+
+    def provider(self) -> str | None:
+        return self.data.get("provider")
 
     def access_key_id(self) -> str:
-        return self.data["access_key_id"]
+        if "access_key_id" in self.data:
+            return self.data["access_key_id"]
+        elif "account" in self.data:
+            return self.data["account"]
+        raise KeyError("No access key found")
 
     def secret_access_key(self) -> str:
-        return self.data["secret_access_key"]
+        # return self.data["secret_access_key"]
+        if "secret_access_key" in self.data:
+            return self.data["secret_access_key"]
+        elif "key" in self.data:
+            return self.data["key"]
+        raise KeyError("No secret access key found")
 
     def endpoint(self) -> str | None:
         return self.data.get("endpoint")

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api/rclone.py

@@ -679,8 +679,13 @@ class Rclone:
         concurrent_chunks: int = 4,  # This setting will scale the performance of the upload
         retries: int = 3,
         max_chunks_before_suspension: int | None = None,
+        mount_path: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        from rclone_api.s3.api import S3Client
+        from rclone_api.s3.create import S3Credentials
+        from rclone_api.util import S3PathInfo, random_str, split_s3_path
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
@@ -695,7 +700,7 @@ class Rclone:
             str(concurrent_chunks),
             "--vfs-fast-fingerprint",
         ]
-        mount_path = Path("rclone_api_upload_mount")
+        mount_path = mount_path or Path("tmp_mnts") / random_str(12)
         src_path = Path(src)
         name = src_path.name
 
@@ -708,8 +713,6 @@ class Rclone:
             other_args=other_args,
         ):
             # raise NotImplementedError("Not implemented yet")
-            from rclone_api.s3.create import S3Credentials
-            from rclone_api.util import S3PathInfo, split_s3_path
 
             path_info: S3PathInfo = split_s3_path(dst)
             remote = path_info.remote
@@ -723,7 +726,24 @@ class Rclone:
                 )
 
             section: Section = sections[remote]
-            provider: str = section.provider()
+            dst_type = section.type()
+            if dst_type != "s3" and dst_type != "b2":
+                raise ValueError(
+                    f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
+                )
+
+            def get_provider_str(section=section) -> str | None:
+                type: str = section.type()
+                provider: str | None = section.provider()
+                if provider is not None:
+                    return provider
+                if type == "b2":
+                    return S3Provider.BACKBLAZE.value
+                if type != "s3":
+                    raise ValueError(f"Remote {remote} is not an S3 remote")
+                return S3Provider.S3.value
+
+            provider: str = get_provider_str() or S3Provider.S3.value
             provider_enum = S3Provider.from_str(provider)
 
             s3_creds: S3Credentials = S3Credentials(
@@ -732,15 +752,8 @@ class Rclone:
                 secret_access_key=section.secret_access_key(),
                 endpoint_url=section.endpoint(),
             )
-            print(s3_creds)
-            # create_s3_client
-
-            print(f"Info: {section}")
-            from rclone_api.s3.api import S3Client
 
             client = S3Client(s3_creds)
-            print(f"Client: {client}")
-
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
                 retries=retries,
@@ -864,6 +877,7 @@ class Rclone:
         other_args: list[str] | None = None,
     ) -> Generator[Process, None, None]:
         """Like mount, but can be used in a context manager."""
+        error_happened = False
         proc = self.mount(
             src,
             outdir,
@@ -875,6 +889,7 @@ class Rclone:
         try:
             yield proc
         except Exception as e:
+            error_happened = True
             stack_trace = traceback.format_exc()
             warnings.warn(f"Error in scoped_mount: {e}\n\nStack Trace:\n{stack_trace}")
             raise
@@ -882,6 +897,24 @@ class Rclone:
             if proc.poll() is None:
                 proc.terminate()
             proc.wait()
+            if not error_happened and outdir.exists():
+                time.sleep(2)
+                if outdir.exists():
+                    print(f"{outdir} mount still exists, attempting to remove")
+                    if not _IS_WINDOWS:
+                        # attempt
+                        os.system(f"fusermount -u {outdir}")
+                        os.system(f"umount {outdir}")
+                        time.sleep(2)
+                        if outdir.exists():
+                            is_empty = not list(outdir.iterdir())
+                            if not is_empty:
+                                warnings.warn(f"Failed to unmount {outdir}")
+                            else:
+                                try:
+                                    outdir.rmdir()
+                                except Exception as e:
+                                    warnings.warn(f"Failed to remove {outdir}: {e}")
 
     @deprecated("mount")
     def mount_webdav(

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api/s3/api.py

@@ -45,28 +45,40 @@ class S3Client:
         upload_target: S3UploadTarget,
         upload_config: S3MutliPartUploadConfig,
     ) -> MultiUploadResult:
-        filesize = upload_target.src_file.stat().st_size
-        if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
-            warnings.warn(
-                f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
-            )
-            err = self.upload_file(upload_target)
-            if err:
-                raise err
-            return MultiUploadResult.UPLOADED_FRESH
+
         chunk_size = upload_config.chunk_size
         retries = upload_config.retries
         resume_path_json = upload_config.resume_path_json
         max_chunks_before_suspension = upload_config.max_chunks_before_suspension
         bucket_name = upload_target.bucket_name
-        out = upload_file_multipart(
-            s3_client=self.client,
-            bucket_name=bucket_name,
-            file_path=upload_target.src_file,
-            object_name=upload_target.s3_key,
-            resumable_info_path=resume_path_json,
-            chunk_size=chunk_size,
-            retries=retries,
-            max_chunks_before_suspension=max_chunks_before_suspension,
-        )
-        return out
+
+        try:
+            filesize = upload_target.src_file.stat().st_size
+            if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
+                warnings.warn(
+                    f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
+                )
+                err = self.upload_file(upload_target)
+                if err:
+                    raise err
+                return MultiUploadResult.UPLOADED_FRESH
+
+            out = upload_file_multipart(
+                s3_client=self.client,
+                bucket_name=bucket_name,
+                file_path=upload_target.src_file,
+                object_name=upload_target.s3_key,
+                resumable_info_path=resume_path_json,
+                chunk_size=chunk_size,
+                retries=retries,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+            return out
+        except Exception:
+            key = upload_target.s3_key
+            access_key_id = self.credentials.access_key_id[:4] + "..."
+            secret = self.credentials.secret_access_key[:4] + "..."
+            warnings.warn(
+                f"Error uploading {key} to {bucket_name} with\n  access_key_id: {access_key_id}\n  secret: {secret}\n"
+            )
+            raise

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api/s3/create.py

@@ -16,9 +16,7 @@ def _create_backblaze_s3_client(creds: S3Credentials) -> BaseClient:
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
-
-    if region_name is not None:
-        warnings.warn(f"Region name is not used for provider: {creds.provider}")
+    region_name = region_name or "https://s3.us-west-002.backblazeb2.com"
 
     session = boto3.session.Session()  # type: ignore
     return session.client(

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api/s3/types.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 
 class S3Provider(Enum):
+    S3 = "s3"  # generic S3
     BACKBLAZE = "b2"
     DIGITAL_OCEAN = "DigitalOcean"
 
@@ -46,6 +47,9 @@ class S3MutliPartUploadConfig:
     retries: int
     resume_path_json: Path
     max_chunks_before_suspension: int | None = None
+    mount_path: Path | None = (
+        None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
+    )
 
 
 class MultiUploadResult(Enum):

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api/util.py

@@ -141,6 +141,8 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
             # how many files?
             dircontents = os.listdir(str(path))
             if len(dircontents) > 0:
+                print(f"Mount point {path}, waiting 5 seconds for files to appear.")
+                time.sleep(5)
                 return
         time.sleep(1)
 
@@ -164,3 +166,10 @@ def split_s3_path(path: str) -> S3PathInfo:
     assert bucket
     assert key
     return S3PathInfo(remote=remote, bucket=bucket, key=key)
+
+
+def random_str(length: int) -> str:
+    import random
+    import string
+
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.89
+Version: 1.0.92
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api.egg-info/SOURCES.txt

@@ -48,6 +48,7 @@ src/rclone_api.egg-info/entry_points.txt
 src/rclone_api.egg-info/requires.txt
 src/rclone_api.egg-info/top_level.txt
 src/rclone_api/assets/example.txt
+src/rclone_api/cmd/copy_large_s3.py
 src/rclone_api/cmd/list_files.py
 src/rclone_api/s3/api.py
 src/rclone_api/s3/basic_ops.py

{rclone_api-1.0.89 → rclone_api-1.0.92}/src/rclone_api.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+rclone-api-copylarge-s3 = rclone_api.cmd.copy_large_s3:main
 rclone-api-listfiles = rclone_api.cmd.list_files:main

{rclone_api-1.0.89 → rclone_api-1.0.92}/tests/test_mounted_ranged_download.py

@@ -118,7 +118,7 @@ class RcloneCopyResumableFileToS3(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
-    @unittest.skipIf(_IS_WINDOWS, "Test not enabled on Windows")
+    # @unittest.skipIf(_IS_WINDOWS, "Test not enabled on Windows")
     def test_upload_chunks(self) -> None:
         """Test basic Webdav serve functionality."""
         # config = _generate_rclone_config(PORT)