rclone-api 1.0.89__tar.gz → 1.0.90__tar.gz

{rclone_api-1.0.89 → rclone_api-1.0.90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.89
+Version: 1.0.90
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.89 → rclone_api-1.0.90}/pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.0.89"
+version = "1.0.90"
 
 [tool.setuptools]
 package-dir = {"" = "src"}
@@ -51,4 +51,5 @@ ignore_missing_imports = true
 disable_error_code = ["import-untyped"]
 
 [project.scripts]
-rclone-api-listfiles = "rclone_api.cmd.list_files:main"
+rclone-api-listfiles = "rclone_api.cmd.list_files:main"
+rclone-api-copylarge-s3 = "rclone_api.cmd.copy_large_s3:main"

rclone_api-1.0.90/src/rclone_api/cmd/copy_large_s3.py

@@ -0,0 +1,99 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import MultiUploadResult, Rclone
+
+_1MB = 1024 * 1024
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str
+    dst: str
+    chunk_size_mb: int
+    read_concurrent_chunks: int
+    retries: int
+    save_state_json: Path
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=True
+    )
+    parser.add_argument(
+        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+    )
+    parser.add_argument(
+        "--read-concurrent-chunks",
+        help="Maximum number of chunks to read",
+        type=int,
+        default=4,
+    )
+    parser.add_argument("--retries", help="Number of retries", type=int, default=3)
+    parser.add_argument(
+        "--resumable-json",
+        help="Path to resumable JSON file",
+        type=Path,
+        default="resume.json",
+    )
+
+    args = parser.parse_args()
+    out = Args(
+        config_path=Path(args.config),
+        src=args.src,
+        dst=args.dst,
+        chunk_size_mb=args.chunk_size_mb,
+        read_concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.resumable_json,
+    )
+    return out
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
+        src=args.src,
+        dst=args.dst,
+        chunk_size=args.chunk_size_mb * _1MB,
+        concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.save_state_json,
+    )
+    print(rslt)
+    return 0
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+
+    here = Path(__file__).parent
+    project_root = here.parent.parent.parent
+    print(f"project_root: {project_root}")
+    os.chdir(str(project_root))
+    cwd = Path(__file__).parent
+    print(f"cwd: {cwd}")
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    main()

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api/config.py

@@ -10,14 +10,26 @@ class Section:
     def add(self, key: str, value: str) -> None:
         self.data[key] = value
 
-    def provider(self) -> str:
-        return self.data["provider"]
+    def type(self) -> str:
+        return self.data["type"]
+
+    def provider(self) -> str | None:
+        return self.data.get("provider")
 
     def access_key_id(self) -> str:
-        return self.data["access_key_id"]
+        if "access_key_id" in self.data:
+            return self.data["access_key_id"]
+        elif "account" in self.data:
+            return self.data["account"]
+        raise KeyError("No access key found")
 
     def secret_access_key(self) -> str:
-        return self.data["secret_access_key"]
+        # return self.data["secret_access_key"]
+        if "secret_access_key" in self.data:
+            return self.data["secret_access_key"]
+        elif "key" in self.data:
+            return self.data["key"]
+        raise KeyError("No secret access key found")
 
     def endpoint(self) -> str | None:
         return self.data.get("endpoint")

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api/rclone.py

@@ -681,6 +681,10 @@ class Rclone:
         max_chunks_before_suspension: int | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        from rclone_api.s3.api import S3Client
+        from rclone_api.s3.create import S3Credentials
+        from rclone_api.util import S3PathInfo, split_s3_path
+
         other_args: list[str] = [
             "--no-modtime",
             "--vfs-read-wait",
@@ -708,8 +712,6 @@ class Rclone:
             other_args=other_args,
         ):
             # raise NotImplementedError("Not implemented yet")
-            from rclone_api.s3.create import S3Credentials
-            from rclone_api.util import S3PathInfo, split_s3_path
 
             path_info: S3PathInfo = split_s3_path(dst)
             remote = path_info.remote
@@ -723,7 +725,24 @@ class Rclone:
                 )
 
             section: Section = sections[remote]
-            provider: str = section.provider()
+            dst_type = section.type()
+            if dst_type != "s3" and dst_type != "b2":
+                raise ValueError(
+                    f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
+                )
+
+            def get_provider_str(section=section) -> str | None:
+                type: str = section.type()
+                provider: str | None = section.provider()
+                if provider is not None:
+                    return provider
+                if type == "b2":
+                    return S3Provider.BACKBLAZE.value
+                if type != "s3":
+                    raise ValueError(f"Remote {remote} is not an S3 remote")
+                return S3Provider.S3.value
+
+            provider: str = get_provider_str() or S3Provider.S3.value
             provider_enum = S3Provider.from_str(provider)
 
             s3_creds: S3Credentials = S3Credentials(
@@ -732,15 +751,8 @@ class Rclone:
                 secret_access_key=section.secret_access_key(),
                 endpoint_url=section.endpoint(),
             )
-            print(s3_creds)
-            # create_s3_client
-
-            print(f"Info: {section}")
-            from rclone_api.s3.api import S3Client
 
             client = S3Client(s3_creds)
-            print(f"Client: {client}")
-
             config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
                 chunk_size=chunk_size,
                 retries=retries,
@@ -772,7 +784,8 @@ class Rclone:
             )
 
             out: MultiUploadResult = client.upload_file_multipart(
-                upload_target=upload_target, upload_config=upload_config
+                upload_target=upload_target,
+                upload_config=upload_config
             )
             return out
 

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api/s3/api.py

@@ -1,4 +1,5 @@
 import warnings
+from pathlib import Path
 
 from botocore.client import BaseClient
 

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api/s3/create.py

@@ -16,9 +16,7 @@ def _create_backblaze_s3_client(creds: S3Credentials) -> BaseClient:
     access_key = creds.access_key_id
     secret_key = creds.secret_access_key
     endpoint_url = creds.endpoint_url
-
-    if region_name is not None:
-        warnings.warn(f"Region name is not used for provider: {creds.provider}")
+    region_name = region_name or "https://s3.us-west-002.backblazeb2.com"
 
     session = boto3.session.Session()  # type: ignore
     return session.client(

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api/s3/types.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 
 class S3Provider(Enum):
+    S3 = "s3"  # generic S3
     BACKBLAZE = "b2"
     DIGITAL_OCEAN = "DigitalOcean"
 
@@ -46,6 +47,8 @@ class S3MutliPartUploadConfig:
     retries: int
     resume_path_json: Path
     max_chunks_before_suspension: int | None = None
+    mount_path: Path | None = None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
+
 
 
 class MultiUploadResult(Enum):

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api/util.py

@@ -141,6 +141,8 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
             # how many files?
             dircontents = os.listdir(str(path))
             if len(dircontents) > 0:
+                print(f"Mount point {path}, waiting 5 seconds for files to appear.")
+                time.sleep(5)
                 return
         time.sleep(1)
 

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.89
+Version: 1.0.90
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api.egg-info/SOURCES.txt

@@ -48,6 +48,7 @@ src/rclone_api.egg-info/entry_points.txt
 src/rclone_api.egg-info/requires.txt
 src/rclone_api.egg-info/top_level.txt
 src/rclone_api/assets/example.txt
+src/rclone_api/cmd/copy_large_s3.py
 src/rclone_api/cmd/list_files.py
 src/rclone_api/s3/api.py
 src/rclone_api/s3/basic_ops.py

{rclone_api-1.0.89 → rclone_api-1.0.90}/src/rclone_api.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+rclone-api-copylarge-s3 = rclone_api.cmd.copy_large_s3:main
 rclone-api-listfiles = rclone_api.cmd.list_files:main

{rclone_api-1.0.89 → rclone_api-1.0.90}/tests/test_mounted_ranged_download.py

@@ -118,7 +118,7 @@ class RcloneCopyResumableFileToS3(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
-    @unittest.skipIf(_IS_WINDOWS, "Test not enabled on Windows")
+    # @unittest.skipIf(_IS_WINDOWS, "Test not enabled on Windows")
     def test_upload_chunks(self) -> None:
         """Test basic Webdav serve functionality."""
         # config = _generate_rclone_config(PORT)