rclone-api 1.0.88__tar.gz → 1.0.90__tar.gz

{rclone_api-1.0.88 → rclone_api-1.0.90}/.gitignore

@@ -145,3 +145,8 @@ uv.lock
 !.aiderignore
 
 rclone*.conf
+test_mount2
+t.py
+mount
+t2.py
+chunk_store

{rclone_api-1.0.88 → rclone_api-1.0.90}/PKG-INFO

@@ -1,9 +1,8 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.88
+Version: 1.0.90
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
-Maintainer: Zachary Vorhies
 License: BSD 3-Clause License
 Keywords: template-python-cmd
 Classifier: Programming Language :: Python :: 3
@@ -12,8 +11,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyright>=1.1.393
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: boto3<=1.35.99,>=1.20.1
 Dynamic: home-page
-Dynamic: maintainer
 
 # rclone-api
 

{rclone_api-1.0.88 → rclone_api-1.0.90}/pyproject.toml

@@ -13,9 +13,15 @@ classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
     "pyright>=1.1.393",
     "python-dotenv>=1.0.0",
+
+    # BOTO3 Library needs to be pinned to a specific version
+    # BackBlaze S3 fails with checksum header which it doesn't support after 1.35.99
+    # The 1.20.1 was the earliest one I checked that worked and is not the true lower bound.
+    "boto3>=1.20.1,<=1.35.99",
 ]
+
 # Change this with the version number bump.
-version = "1.0.88"
+version = "1.0.90"
 
 [tool.setuptools]
 package-dir = {"" = "src"}
@@ -45,4 +51,5 @@ ignore_missing_imports = true
 disable_error_code = ["import-untyped"]
 
 [project.scripts]
-rclone-api-listfiles = "rclone_api.cmd.list_files:main"
+rclone-api-listfiles = "rclone_api.cmd.list_files:main"
+rclone-api-copylarge-s3 = "rclone_api.cmd.copy_large_s3:main"

{rclone_api-1.0.88 → rclone_api-1.0.90}/setup.py

@@ -15,7 +15,6 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 
 if __name__ == "__main__":
     setup(
-        maintainer="Zachary Vorhies",
         keywords=KEYWORDS,
         url=URL,
         package_data={"": ["assets/example.txt"]},

{rclone_api-1.0.88 → rclone_api-1.0.90}/src/rclone_api/__init__.py

@@ -1,5 +1,5 @@
 from .completed_process import CompletedProcess
-from .config import Config
+from .config import Config, Parsed, Section
 from .diff import DiffItem, DiffOption, DiffType
 from .dir import Dir
 from .dir_listing import DirListing
@@ -9,6 +9,7 @@ from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
 from .rpath import RPath
+from .s3.types import MultiUploadResult
 from .types import ListingOption, Order, SizeResult
 
 __all__ = [
@@ -30,4 +31,7 @@ __all__ = [
     "Order",
     "ListingOption",
     "SizeResult",
+    "Parsed",
+    "Section",
+    "MultiUploadResult",
 ]

rclone_api-1.0.90/src/rclone_api/cmd/copy_large_s3.py

@@ -0,0 +1,99 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import MultiUploadResult, Rclone
+
+_1MB = 1024 * 1024
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str
+    dst: str
+    chunk_size_mb: int
+    read_concurrent_chunks: int
+    retries: int
+    save_state_json: Path
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=True
+    )
+    parser.add_argument(
+        "--chunk-size-mb", help="Chunk size in MB", type=int, default=256
+    )
+    parser.add_argument(
+        "--read-concurrent-chunks",
+        help="Maximum number of chunks to read",
+        type=int,
+        default=4,
+    )
+    parser.add_argument("--retries", help="Number of retries", type=int, default=3)
+    parser.add_argument(
+        "--resumable-json",
+        help="Path to resumable JSON file",
+        type=Path,
+        default="resume.json",
+    )
+
+    args = parser.parse_args()
+    out = Args(
+        config_path=Path(args.config),
+        src=args.src,
+        dst=args.dst,
+        chunk_size_mb=args.chunk_size_mb,
+        read_concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.resumable_json,
+    )
+    return out
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
+        src=args.src,
+        dst=args.dst,
+        chunk_size=args.chunk_size_mb * _1MB,
+        concurrent_chunks=args.read_concurrent_chunks,
+        retries=args.retries,
+        save_state_json=args.save_state_json,
+    )
+    print(rslt)
+    return 0
+
+
+if __name__ == "__main__":
+    import os
+    import sys
+
+    here = Path(__file__).parent
+    project_root = here.parent.parent.parent
+    print(f"project_root: {project_root}")
+    os.chdir(str(project_root))
+    cwd = Path(__file__).parent
+    print(f"cwd: {cwd}")
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent"
+    )
+    main()

rclone_api-1.0.90/src/rclone_api/config.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+
+@dataclass
+class Section:
+    name: str
+    data: Dict[str, str] = field(default_factory=dict)
+
+    def add(self, key: str, value: str) -> None:
+        self.data[key] = value
+
+    def type(self) -> str:
+        return self.data["type"]
+
+    def provider(self) -> str | None:
+        return self.data.get("provider")
+
+    def access_key_id(self) -> str:
+        if "access_key_id" in self.data:
+            return self.data["access_key_id"]
+        elif "account" in self.data:
+            return self.data["account"]
+        raise KeyError("No access key found")
+
+    def secret_access_key(self) -> str:
+        # return self.data["secret_access_key"]
+        if "secret_access_key" in self.data:
+            return self.data["secret_access_key"]
+        elif "key" in self.data:
+            return self.data["key"]
+        raise KeyError("No secret access key found")
+
+    def endpoint(self) -> str | None:
+        return self.data.get("endpoint")
+
+
+@dataclass
+class Parsed:
+    # sections: List[ParsedSection]
+    sections: dict[str, Section]
+
+    @staticmethod
+    def parse(content: str) -> "Parsed":
+        return parse_rclone_config(content)
+
+
+@dataclass
+class Config:
+    """Rclone configuration dataclass."""
+
+    text: str
+
+    def parse(self) -> Parsed:
+        return Parsed.parse(self.text)
+
+
+def parse_rclone_config(content: str) -> Parsed:
+    """
+    Parses an rclone configuration file and returns a list of RcloneConfigSection objects.
+
+    Each section in the file starts with a line like [section_name]
+    followed by key=value pairs.
+    """
+    sections: List[Section] = []
+    current_section: Section | None = None
+
+    lines = content.splitlines()
+    for line in lines:
+        line = line.strip()
+        # Skip empty lines and comments (assumed to start with '#' or ';')
+        if not line or line.startswith(("#", ";")):
+            continue
+        # New section header detected
+        if line.startswith("[") and line.endswith("]"):
+            section_name = line[1:-1].strip()
+            current_section = Section(name=section_name)
+            sections.append(current_section)
+        elif "=" in line and current_section is not None:
+            # Parse key and value, splitting only on the first '=' found
+            key, value = line.split("=", 1)
+            current_section.add(key.strip(), value.strip())
+
+    data: dict[str, Section] = {}
+    for section in sections:
+        data[section.name] = section
+    return Parsed(sections=data)

{rclone_api-1.0.88 → rclone_api-1.0.90}/src/rclone_api/group_files.py

@@ -68,7 +68,10 @@ class TreeNode:
         paths_reversed: list[str] = [self.name]
         node: TreeNode | None = self
         assert node is not None
-        while node := node.parent:
+        while True:
+            node = node.parent
+            if node is None:
+                break
             paths_reversed.append(node.name)
         return "/".join(reversed(paths_reversed))