rclone-api 1.5.8__py3-none-any.whl

rclone_api/assets/example.txt

@@ -0,0 +1 @@
+Example assets that will be deployed with python code.

rclone_api/cli.py

@@ -0,0 +1,15 @@
+"""
+Main entry point.
+"""
+
+import sys
+
+
+def main() -> int:
+    """Main entry point for the template_python_cmd package."""
+    print("Replace with a CLI entry point.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rclone_api/cmd/analyze.py

@@ -0,0 +1,51 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+
+
+@dataclass
+class Args:
+    config: Path
+    path: str
+
+    def __post_init__(self):
+        if not self.config.exists():
+            raise FileNotFoundError(f"Config file not found: {self.config}")
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+
+    with rclone.ls_stream(path, fast_list=True) as files:
+        for file_item in files:
+            print(file_item.path, "", file_item.size, file_item.mod_time)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, default="rclone.conf"
+    )
+    parser.add_argument("path", help="Remote path to list")
+    tmp = parser.parse_args()
+    return Args(config=tmp.config, path=tmp.path)
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    path = args.path
+    rclone = Rclone(Path(args.config))
+    list_files(rclone, path)
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    cwd = Path(".").absolute()
+    print(f"cwd: {cwd}")
+    sys.argv.append("dst:TorrentBooks")
+    main()

rclone_api/cmd/copy_large_s3.py

@@ -0,0 +1,111 @@
+import argparse
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone, SizeSuffix
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str
+    dst: str
+    chunk_size: SizeSuffix
+    threads: int
+    retries: int
+    save_state_json: Path
+    verbose: bool
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="File to copy")
+    parser.add_argument("dst", help="Destination file")
+    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=False
+    )
+    parser.add_argument(
+        "--chunk-size",
+        help="Chunk size that will be read and uploaded in SizeSuffix form, too low or too high will cause issues",
+        type=str,
+        default="128MB",  # if this is too low or too high an s3 service
+    )
+    parser.add_argument(
+        "--threads",
+        help="Max number of chunks to upload in parallel to the destination, each chunk is uploaded in a separate thread",
+        type=int,
+        default=8,
+    )
+    parser.add_argument("--retries", help="Number of retries", type=int, default=3)
+    parser.add_argument(
+        "--resume-json",
+        help="Path to resumable JSON file",
+        type=Path,
+        default="resume.json",
+    )
+
+    args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
+    out = Args(
+        config_path=config,
+        src=args.src,
+        dst=args.dst,
+        threads=args.threads,
+        chunk_size=SizeSuffix(args.chunk_size),
+        retries=args.retries,
+        save_state_json=args.resume_json,
+        verbose=args.verbose,
+    )
+    return out
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    rclone = Rclone(rclone_conf=args.config_path)
+    # unit_chunk = args.chunk_size / args.threads
+    # rslt: MultiUploadResult = rclone.copy_file_resumable_s3(
+    #     src=args.src,
+    #     dst=args.dst,
+    #     chunk_size=args.chunk_size,
+    #     read_threads=args.read_threads,
+    #     write_threads=args.write_threads,
+    #     retries=args.retries,
+    #     save_state_json=args.save_state_json,
+    #     verbose=args.verbose,
+    # )
+    err: Exception | None = rclone.copy_file_s3_resumable(
+        src=args.src,
+        dst=args.dst,
+    )
+    if err is not None:
+        print(f"Error: {err}")
+        raise err
+    return 0
+
+
+if __name__ == "__main__":
+
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "45061:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+    )
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+    )
+    main()

rclone_api/cmd/copy_large_s3_finish.py

@@ -0,0 +1,81 @@
+import argparse
+from dataclasses import dataclass
+from pathlib import Path
+
+from rclone_api import Rclone
+from rclone_api.s3.multipart.upload_parts_server_side_merge import (
+    s3_server_side_multi_part_merge,
+)
+
+
+@dataclass
+class Args:
+    config_path: Path
+    src: str  # like dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/ (info.json will be located here)
+    verbose: bool
+
+    def __repr__(self):
+        return f"Args(config_path={self.config_path}, src={self.src}, verbose={self.verbose})"
+
+    def __str__(self):
+        return repr(self)
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("src", help="Directory that holds the info.json file")
+    parser.add_argument("--no-verbose", help="Verbose output", action="store_true")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, required=False
+    )
+    args = parser.parse_args()
+    config: Path | None = args.config
+    if config is None:
+        config = Path("rclone.conf")
+        if not config.exists():
+            raise FileNotFoundError(f"Config file not found: {config}")
+    assert config is not None
+    out = Args(
+        config_path=config,
+        src=args.src,
+        verbose=not args.no_verbose,
+    )
+    return out
+
+
+def _get_info_path(src: str) -> str:
+    if src.endswith("/"):
+        src = src[:-1]
+    info_path = f"{src}/info.json"
+    return info_path
+
+
+def main() -> int:
+    """Main entry point."""
+    print("Starting...")
+    args = _parse_args()
+    print(f"args: {args}")
+    rclone = Rclone(rclone_conf=args.config_path)
+    info_path = _get_info_path(src=args.src)
+    s3_server_side_multi_part_merge(
+        rclone=rclone.impl, info_path=info_path, max_workers=5, verbose=args.verbose
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv.append("--config")
+    sys.argv.append("rclone.conf")
+    sys.argv.append(
+        "dst:TorrentBooks/aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst-parts/"
+    )
+    main()

rclone_api/cmd/list_files.py

@@ -0,0 +1,27 @@
+import argparse
+from pathlib import Path
+
+from rclone_api import Rclone
+
+
+def list_files(rclone: Rclone, path: str):
+    """List files in a remote path."""
+    for dirlisting in rclone.walk(path):
+        for file in dirlisting.files:
+            print(file.path)
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument("--config", help="Path to rclone config file", required=True)
+    parser.add_argument("path", help="Remote path to list")
+    return parser.parse_args()
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    path = args.path
+    rclone = Rclone(Path(args.config))
+    list_files(rclone, path)
+    return 0

rclone_api/cmd/save_to_db.py

@@ -0,0 +1,77 @@
+import argparse
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from rclone_api import Rclone
+
+# load_dotenv()
+
+
+# DB_URL = "sqlite:///data.db"
+
+# os.environ["DB_URL"] = "sqlite:///data.db"
+
+
+def _db_url_from_env_or_raise() -> str:
+    load_dotenv(Path(".env"))
+    db_url = os.getenv("DB_URL")
+    if db_url is None:
+        raise ValueError("DB_URL not set")
+    return db_url
+
+
+@dataclass
+class Args:
+    config: Path
+    path: str
+    db_url: str
+    fast_list: bool
+
+    def __post_init__(self):
+        if not self.config.exists():
+            raise FileNotFoundError(f"Config file not found: {self.config}")
+
+
+def fill_db(rclone: Rclone, path: str, fast_list: bool) -> None:
+    """List files in a remote path."""
+    # db = DB(_db_url_from_env_or_raise())
+    db_url = _db_url_from_env_or_raise()
+    rclone.save_to_db(src=path, db_url=db_url, fast_list=fast_list)
+
+
+def _parse_args() -> Args:
+    parser = argparse.ArgumentParser(description="List files in a remote path.")
+    parser.add_argument(
+        "--config", help="Path to rclone config file", type=Path, default="rclone.conf"
+    )
+    parser.add_argument("--db", help="Database URL", type=str, default=None)
+    parser.add_argument("--fast-list", help="Use fast list", action="store_true")
+    parser.add_argument("path", help="Remote path to list")
+    tmp = parser.parse_args()
+    return Args(
+        config=tmp.config,
+        path=tmp.path,
+        db_url=tmp.db if tmp.db is not None else _db_url_from_env_or_raise(),
+        fast_list=tmp.fast_list,
+    )
+
+
+def main() -> int:
+    """Main entry point."""
+    args = _parse_args()
+    path = args.path
+    rclone = Rclone(Path(args.config))
+    fill_db(rclone=rclone, path=path, fast_list=args.fast_list)
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    cwd = Path(".").absolute()
+    print(f"cwd: {cwd}")
+    sys.argv.append("dst:TorrentBooks/meta")
+    main()

rclone_api/completed_process.py

@@ -0,0 +1,60 @@
+import subprocess
+from dataclasses import dataclass
+
+
+@dataclass
+class CompletedProcess:
+    completed: list[subprocess.CompletedProcess]
+
+    @property
+    def ok(self) -> bool:
+        return all([p.returncode == 0 for p in self.completed])
+
+    @staticmethod
+    def from_subprocess(process: subprocess.CompletedProcess) -> "CompletedProcess":
+        return CompletedProcess(completed=[process])
+
+    def failed(self) -> list[subprocess.CompletedProcess]:
+        return [p for p in self.completed if p.returncode != 0]
+
+    def successes(self) -> list[subprocess.CompletedProcess]:
+        return [p for p in self.completed if p.returncode == 0]
+
+    @property
+    def stdout(self) -> str:
+        tmp: list[str] = []
+        for cp in self.completed:
+            stdout = cp.stdout
+            if stdout is not None:
+                tmp.append(stdout)
+        return "\n".join(tmp)
+
+    @property
+    def stderr(self) -> str:
+        tmp: list[str] = []
+        for cp in self.completed:
+            stderr = cp.stderr
+            if stderr is not None:
+                tmp.append(stderr)
+        return "\n".join(tmp)
+
+    @property
+    def returncode(self) -> int | None:
+        for cp in self.completed:
+            rtn = cp.returncode
+            if rtn is None:
+                return None
+            if rtn != 0:
+                return rtn
+        return 0
+
+    def __str__(self) -> str:
+
+        cmd_strs: list[str] = []
+        rtn_cods: list[int] = []
+        for cp in self.completed:
+            cmd_strs.append(subprocess.list2cmdline(cp.args))
+            rtn_cods.append(cp.returncode)
+        msg = f"CompletedProcess: {len(cmd_strs)} commands\n"
+        msg += "\n".join([f"{cmd} -> {rtn}" for cmd, rtn in zip(cmd_strs, rtn_cods)])
+        return msg

rclone_api/config.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+
+@dataclass
+class Section:
+    name: str
+    data: Dict[str, str] = field(default_factory=dict)
+
+    def add(self, key: str, value: str) -> None:
+        self.data[key] = value
+
+    def type(self) -> str:
+        return self.data["type"]
+
+    def provider(self) -> str | None:
+        return self.data.get("provider")
+
+    def access_key_id(self) -> str:
+        if "access_key_id" in self.data:
+            return self.data["access_key_id"]
+        elif "account" in self.data:
+            return self.data["account"]
+        raise KeyError("No access key found")
+
+    def secret_access_key(self) -> str:
+        # return self.data["secret_access_key"]
+        if "secret_access_key" in self.data:
+            return self.data["secret_access_key"]
+        elif "key" in self.data:
+            return self.data["key"]
+        raise KeyError("No secret access key found")
+
+    def endpoint(self) -> str | None:
+        return self.data.get("endpoint")
+
+
+@dataclass
+class Parsed:
+    # sections: List[ParsedSection]
+    sections: dict[str, Section]
+
+    @staticmethod
+    def parse(content: str) -> "Parsed":
+        return parse_rclone_config(content)
+
+
+@dataclass
+class Config:
+    """Rclone configuration dataclass."""
+
+    text: str
+
+    def parse(self) -> Parsed:
+        return Parsed.parse(self.text)
+
+
+def parse_rclone_config(content: str) -> Parsed:
+    """
+    Parses an rclone configuration file and returns a list of RcloneConfigSection objects.
+
+    Each section in the file starts with a line like [section_name]
+    followed by key=value pairs.
+    """
+    sections: List[Section] = []
+    current_section: Section | None = None
+
+    lines = content.splitlines()
+    for line in lines:
+        line = line.strip()
+        # Skip empty lines and comments (assumed to start with '#' or ';')
+        if not line or line.startswith(("#", ";")):
+            continue
+        # New section header detected
+        if line.startswith("[") and line.endswith("]"):
+            section_name = line[1:-1].strip()
+            current_section = Section(name=section_name)
+            sections.append(current_section)
+        elif "=" in line and current_section is not None:
+            # Parse key and value, splitting only on the first '=' found
+            key, value = line.split("=", 1)
+            current_section.add(key.strip(), value.strip())
+
+    data: dict[str, Section] = {}
+    for section in sections:
+        data[section.name] = section
+    return Parsed(sections=data)

rclone_api/convert.py

@@ -0,0 +1,31 @@
+from rclone_api.dir import Dir
+from rclone_api.file import File
+from rclone_api.remote import Remote
+
+
+def convert_to_filestr_list(files: str | File | list[str] | list[File]) -> list[str]:
+    out: list[str] = []
+    if isinstance(files, str):
+        out.append(files)
+    elif isinstance(files, File):
+        out.append(str(files.path))
+    elif isinstance(files, list):
+        for f in files:
+            if isinstance(f, File):
+                f = str(f.path)
+            out.append(f)
+    else:
+        raise ValueError(f"Invalid type for file: {type(files)}")
+    return out
+
+
+def convert_to_str(file_or_dir: str | File | Dir | Remote) -> str:
+    if isinstance(file_or_dir, str):
+        return file_or_dir
+    if isinstance(file_or_dir, File):
+        return str(file_or_dir.path)
+    if isinstance(file_or_dir, Dir):
+        return str(file_or_dir.path)
+    if isinstance(file_or_dir, Remote):
+        return str(file_or_dir)
+    raise ValueError(f"Invalid type for file_or_dir: {type(file_or_dir)}")

rclone_api/db/__init__.py

@@ -0,0 +1,3 @@
+from .db import DB
+
+__all__ = ["DB"]