easyclone 0.1.0__py3-none-any.whl

easyclone/__main__.py

@@ -0,0 +1,4 @@
+from .main import app
+
+if __name__ == "__main__":
+    app()

easyclone/config.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+from threading import Lock
+from os import getenv
+from pathlib import Path
+from easyclone.utypes.enums import LogLevel
+from easyclone.utypes.config import BackupConfigModel, ConfigModel, RcloneConfigModel
+import toml
+
+class Config:
+    _instance: Config | None = None
+    _lock: Lock = Lock()
+    _path: Path = Path.home() / '.config' / "easyclone" / "config.toml"
+    _config: ConfigModel | None = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            with cls._lock:
+                instance = super().__new__(cls)
+                instance._get_config_path()
+                instance._config = instance._load_config()
+                cls._instance = instance
+
+        return cls._instance
+
+    def _get_config_path(self):
+        xdg_config_home = getenv("XDG_CONFIG_HOME")
+
+        empty_config = ConfigModel(
+            backup=BackupConfigModel(
+                sync_paths=[],
+                copy_paths=[],
+                remote_name="GoogleDrive",
+                root_dir="Backups/PC",
+                verbose_log=False
+            ),
+            rclone=RcloneConfigModel(
+                args=[
+                    "--update",
+                    "--verbose",
+                    "--transfers 30",
+                    "--checkers 8",
+                    "--contimeout 60s",
+                    "--timeout 300s",
+                    "--retries 3",
+                    "--low-level-retries 10",
+                    "--stats 1s"
+                ],
+                concurrent_limit=50
+            )
+        )
+    
+        if xdg_config_home:
+            config_dir = Path(xdg_config_home) / 'easyclone'
+        else: 
+            config_dir = Path.home() / '.config' / "easyclone"
+    
+        config_file = config_dir / "config.toml"
+
+        if not config_dir.exists():
+            config_dir.mkdir(parents=True, exist_ok=True)
+
+        if not config_file.exists():
+            config_file.touch()
+            with open(config_file, "w") as f:
+                _ = toml.dump(empty_config.model_dump(), f)
+    
+        self._path = config_file
+
+    def _load_config(self):
+        from easyclone.utils.essentials import log
+        self._get_config_path()
+
+
+        try:
+            with open(self._path) as f:
+                parsed_string = f.read()
+        except FileNotFoundError:
+            log(f"Config file not found at {self._path}.", LogLevel.ERROR)
+            exit(1)
+        except Exception as e:
+            log(f"Error while opening the config file {self._path}: {e}", LogLevel.ERROR)
+            exit(1)
+
+        try:
+            parsed_toml = toml.loads(parsed_string)
+            validated_config = ConfigModel.model_validate(parsed_toml)
+            return validated_config
+        except Exception as e:
+            log(f"Invalid config: {e}", LogLevel.ERROR)
+            exit(1)
+
+    def config(self) -> ConfigModel:
+        if self._config is None:
+            raise RuntimeError("Config is not loaded yet")
+        return self._config
+
+cfg = Config().config()

easyclone/ipc/__init__.py

@@ -0,0 +1,2 @@
+from .server import start_status_server
+from .client import listen_ipc

easyclone/ipc/client.py

@@ -0,0 +1,26 @@
+import asyncio
+import json
+
+from easyclone.utils.essentials import log
+from easyclone.utypes.enums import LogLevel
+
+SOCKET_PATH = "/tmp/easyclone.sock"
+
+async def listen_ipc():
+    try:
+        reader, writer = await asyncio.open_unix_connection(SOCKET_PATH)
+    except (FileNotFoundError, ConnectionRefusedError):
+        log("No tasks are running at the moment.", LogLevel.ERROR)
+        exit(1)
+
+    try:
+        line = await reader.readline()
+        if not line:
+            log("No tasks are running at the moment", LogLevel.ERROR)
+            exit(1)
+
+        data = json.loads(line.decode())
+        return data
+    finally:
+        writer.close()
+        await writer.wait_closed()

easyclone/ipc/server.py

@@ -0,0 +1,50 @@
+import asyncio
+import json
+from pathlib import Path
+from easyclone.utypes.enums import LogLevel
+from easyclone.shared import sync_status
+from easyclone.utils.essentials import log
+
+SOCKET_PATH = "/tmp/easyclone.sock"
+
+async def handle_client(_reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
+    try:
+        while True:
+            message = json.dumps({
+                "total_path_count": await sync_status.get_total_path(),
+                "finished_path_count": await sync_status.get_currently_finished(),
+                "operation_count": await sync_status.get_operation_count(),
+                "operations": await sync_status.get_operations()
+            }).encode() + b"\n"
+
+            writer.write(message)
+
+            await writer.drain()
+            await asyncio.sleep(0.5)
+    except (ConnectionResetError, BrokenPipeError, asyncio.IncompleteReadError):
+        pass
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+    finally:
+        try:
+            writer.close()
+            await writer.wait_closed()
+        except BrokenPipeError:
+            pass
+
+async def start_status_server():
+    try:
+        Path(SOCKET_PATH).unlink()
+    except FileNotFoundError as e:
+        pass
+    except Exception as e:
+        log(f"Something happened while connecting to the socket for status server: {e}", LogLevel.ERROR)
+        raise
+
+    try:
+        server = await asyncio.start_unix_server(handle_client, path=SOCKET_PATH)
+    except Exception as e:
+        log(f"Couldn't create the UNIX server: {e}", LogLevel.ERROR)
+        raise
+
+    return server

easyclone/main.py

@@ -0,0 +1,59 @@
+import asyncio
+from typing import Annotated, Any
+from easyclone.config import cfg
+from easyclone.ipc.client import listen_ipc
+from easyclone.ipc.server import start_status_server
+from easyclone.rclone.operations import backup_copy_operation, backup_sync_operation
+import typer
+import json
+
+from easyclone.shared import sync_status
+
+app = typer.Typer(
+    help="Very convenient Rclone bulk backup wrapper",
+    context_settings={"help_option_names": ["-h", "--help"]}
+)
+
+async def ipc():
+    server = await start_status_server()
+    async with server:
+        await server.serve_forever()
+
+@app.command(help="Starts the backup process using the details in the config file.")
+def start_backup(verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Enables the rclone logging (overrides config).")] = False):
+    async def start():
+        await sync_status.set_total_path_count(len(cfg.backup.sync_paths) + len(cfg.backup.copy_paths))
+
+        _ipc_task = asyncio.create_task(ipc()) 
+
+        verbose_state = verbose or cfg.backup.verbose_log
+
+        await backup_copy_operation(verbose_state)
+        await backup_sync_operation(verbose_state)
+
+    asyncio.run(start())
+
+@app.command(help="Gets status information about the backup process.")
+def get_status(
+        all: Annotated[bool, typer.Option("--all", "-a", help="Show all the backup status information.")] = False,
+        show_total: Annotated[bool, typer.Option("--show-total", "-t", help="Show the total amount of paths.")] = False,
+        show_current: Annotated[bool, typer.Option("--show-current", "-c", help="Show the total amount of pending paths.")] = False,
+        show_operations: Annotated[bool, typer.Option("--show-operations", "-o", help="Show currently running operations.")] = False
+):
+    data: Any = asyncio.run(listen_ipc())
+
+    if (not show_total and not show_current and not show_operations or all) or (show_total and show_current and show_operations):
+        print(json.dumps(data, indent=2))
+        return
+
+    if show_total:
+        print(data["total_path_count"])
+
+    if show_current:
+        print(data["operation_count"])
+
+    if show_operations:
+        print(json.dumps(data["operations"], indent=2))
+
+if __name__ == "__main__":
+    app()

easyclone/rclone/__init__.py

@@ -0,0 +1,3 @@
+from .backup import backup
+from .create_dirs import create_dirs_array
+from .operations import backup_sync_operation, backup_copy_operation

easyclone/rclone/backup.py

@@ -0,0 +1,59 @@
+import asyncio
+import shlex
+from easyclone.shared import sync_status
+from easyclone.utypes.enums import BackupLog, BackupStatus, CommandType, LogLevel, RcloneOperationType
+from easyclone.utypes.models import PathItem
+from easyclone.utils.essentials import log
+from easyclone.utils.path_manipulation import collapseuser
+
+async def backup_command(rclone_command: list[str], source: str, dest: str, path_type: str, command_type: CommandType, verbose: bool = False):
+    cmd = rclone_command + [source, dest]
+    operation_name = command_type.value.capitalize() + "ing"
+
+    log(f"{operation_name} {collapseuser(source)}", BackupLog.WAIT)
+    process = await asyncio.create_subprocess_exec(*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+    process_id = await sync_status.add_operation(source=source, dest=dest, path_type=path_type, status=BackupStatus.IN_PROGRESS, operation_type=RcloneOperationType.BACKUP)
+
+    stdout, stderr = await process.communicate()
+    stdout = stdout.decode(errors="ignore").strip()
+    stderr = stderr.decode(errors="ignore").strip()
+
+    collapsed_source = collapseuser(source)
+    match process.returncode:
+        case 0:
+            log(f"Backed up successfully: {collapsed_source}", BackupLog.OK)
+        case 1:
+            log(f"Back up operation failed: {collapsed_source}", BackupLog.ERR)
+        case _:
+            log(f"Back up operation failed with {process.returncode} exit code: {collapsed_source}", BackupLog.ERR)
+
+    await sync_status.delete_operation(process_id)
+    await sync_status.add_currently_finished()
+
+    if verbose:
+        if stderr: 
+            log(f"{stderr}", LogLevel.WARN)
+        if stdout:
+            log(f"{stdout}", LogLevel.LOG)
+
+async def backup(paths: list[PathItem], command_type: CommandType, rclone_args: list[str], semaphore: asyncio.Semaphore, verbose: bool = False):
+    cmd = ["rclone", command_type.value]
+
+
+    for arg in rclone_args:
+        parts = shlex.split(arg)
+        cmd += parts
+
+    # I have to do this because python doesn't have anon coroutines
+    async def backup_task(source: str, dest: str, path_type: str):
+        async with semaphore:
+            await backup_command(cmd, source, dest, path_type, command_type, verbose)
+
+
+    tasks = [
+        backup_task(path["source"], path["dest"], path["path_type"])
+        for path in paths
+    ]
+    
+    return await asyncio.gather(*tasks)
+

easyclone/rclone/create_dirs.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+import asyncio
+from easyclone.shared import sync_status
+from easyclone.utils.essentials import log
+from easyclone.utypes.models import PathItem
+from easyclone.utypes.enums import BackupLog, BackupStatus, LogLevel, PathType, RcloneOperationType
+import os
+from pathlib import Path
+from collections import deque
+from easyclone.config import cfg
+
+class DirNode:
+    def __init__(self, name: str, details: PathItem):
+        self.name: str = name
+        self.details: PathItem = details
+        self.children: list[DirNode] = []
+
+    def find_child(self, name: str) -> "DirNode | None":
+        for child in self.children:
+            if child.name == name:
+                return child
+        return None
+
+    def add_child(self, name: str, details: PathItem) -> "DirNode":
+        existing = self.find_child(name)
+        if existing:
+            return existing
+        new_child = DirNode(name, details)
+        self.children.append(new_child)
+        return new_child
+
+    def print_tree(self, level: int=0):
+        indent = "  " * level
+        print(f"{indent}- {self.name} ({self.details.get('dest')})")
+        for child in sorted(self.children, key=lambda c: c.name):
+            child.print_tree(level + 1)
+
+def create_dirs_array(path_list: list[PathItem]):
+    only_dirs: list[PathItem] = []
+
+    for path in path_list:
+        new_dir_path: PathItem
+
+        if path.get("type") == "dir":
+            new_dir_path = path
+        else:
+            new_dir_path = {
+                "source": os.path.dirname(path.get("source")),
+                "dest": path.get("dest"),
+                "path_type": PathType.DIR.value
+            }
+
+        if new_dir_path not in only_dirs:
+            only_dirs.append(new_dir_path)
+
+    return only_dirs
+
+def create_dir_tree(path_list: list[PathItem]):
+    root = DirNode("Root", {"source": "/", "dest": f"{cfg.backup.remote_name}:{str(cfg.backup.root_dir).rstrip("/")}", "path_type": "dir",})
+
+    for path_item in path_list:
+        source_str = path_item.get("source")
+        dest_str = path_item.get("dest")
+    
+        source_parts = Path(source_str).parts
+        dest_main, _, dest_path = dest_str.partition(":")
+
+        if dest_path:
+            dest_parts = Path(dest_path).parts
+        else:
+            dest_parts = []
+    
+        current = root
+    
+        for i in range(1, len(source_parts)):
+            source_sub_path = str(Path(*source_parts[:i+1]))
+
+            if dest_path:
+                dest_sub_path = f"{dest_main}:{Path(*dest_parts[:i+2])}"
+            else:
+                dest_sub_path = ""
+    
+            node_details: PathItem = {
+                "source": source_sub_path,
+                "dest": dest_sub_path,
+                "path_type": PathType.DIR.value
+            }
+    
+            part_name = source_parts[i]
+            current = current.add_child(part_name, node_details)
+
+    return root
+
+async def create_folder_command(source: str, dest: str, verbose: bool):
+    lsd_cmd = ["rclone", "lsd", dest]
+    log(f"Checking if directory exist at {dest}", BackupLog.WAIT)
+
+    process = await asyncio.create_subprocess_exec(*lsd_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+    stdout, stderr = await process.communicate()
+
+    if process.returncode == 0:
+        log(f"Directory exist at {dest}", BackupLog.OK)
+        return
+
+    mkdir_cmd = ["rclone", "mkdir", dest]
+
+    log(f"Creating a directory at {dest}", BackupLog.WAIT)
+    process = await asyncio.create_subprocess_exec(*mkdir_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+    process_id = await sync_status.add_operation(source=source, dest=dest, path_type=PathType.DIR.value, status=BackupStatus.IN_PROGRESS, operation_type=RcloneOperationType.MKDIR)
+
+    stdout, stderr = await process.communicate()
+    stdout = stdout.decode(errors="ignore").strip()
+    stderr = stderr.decode(errors="ignore").strip()
+
+    match process.returncode:
+        case 0:
+            log(f"Directory created successfully: {dest}", BackupLog.OK)
+        case 1:
+            log(f"Couldn't create the directory: {dest}", BackupLog.ERR)
+        case _:
+            log(f"Operation failed with {process.returncode} exit code: {dest}", BackupLog.ERR)
+
+    await sync_status.delete_operation(process_id)
+
+    if verbose:
+        if stderr: 
+            log(f"{stderr}", LogLevel.WARN)
+        if stdout:
+            log(f"{stdout}", LogLevel.LOG)
+
+async def create_folders_on_remote(nodes: list[DirNode], semaphore: asyncio.Semaphore, verbose: bool):
+    async def mkdir_task(source: str, dest: str):
+        async with semaphore:
+            await create_folder_command(source, dest, verbose)
+
+    tasks = [mkdir_task(node.details.get("source"), node.details.get("dest")) for node in nodes]
+
+    _ = await asyncio.gather(*tasks)
+
+async def traverse_and_create_folders_by_depth(root: DirNode, verbose: bool, semaphore: asyncio.Semaphore):
+    queue = deque([(root, 0)])
+    current_depth = 0
+    current_level_nodes: list[DirNode] = []
+
+    while queue:
+        node, depth = queue.popleft()
+
+        if depth != current_depth:
+            await create_folders_on_remote(current_level_nodes, semaphore, verbose)
+            current_level_nodes = []
+            current_depth = depth
+
+        current_level_nodes.append(node)
+
+        for child in node.children:
+            queue.append((child, depth + 1))
+
+    if current_level_nodes:
+        await create_folders_on_remote(current_level_nodes, semaphore, verbose)

easyclone/rclone/operations.py

@@ -0,0 +1,48 @@
+import asyncio
+from easyclone.config import cfg
+from easyclone.rclone.backup import backup
+from easyclone.rclone.create_dirs import create_dir_tree, create_dirs_array, traverse_and_create_folders_by_depth
+from easyclone.utils.path_manipulation import organize_paths
+from easyclone.utypes.enums import CommandType
+
+async def backup_copy_operation(verbose: bool):
+    copy_paths = organize_paths(cfg.backup.copy_paths, cfg.backup.remote_name)
+    copy_task_semaphore = asyncio.Semaphore(cfg.rclone.concurrent_limit)
+    copy_dirs_task_semaphore = asyncio.Semaphore(cfg.rclone.concurrent_limit)
+    copy_dirs_array = create_dirs_array(copy_paths)
+    copy_dirs_root = create_dir_tree(copy_dirs_array)
+
+    _copy_folders_create_operation = await traverse_and_create_folders_by_depth(
+        root=copy_dirs_root,
+        verbose=verbose,
+        semaphore=copy_dirs_task_semaphore
+    )
+    _copy_operation = await backup(
+        paths=copy_paths,
+        command_type=CommandType.COPY,
+        rclone_args=cfg.rclone.args,
+        semaphore=copy_task_semaphore,
+        verbose=verbose
+    )
+
+
+async def backup_sync_operation(verbose: bool):
+    sync_paths = organize_paths(cfg.backup.sync_paths, cfg.backup.remote_name)
+    sync_task_semaphore = asyncio.Semaphore(cfg.rclone.concurrent_limit)
+    sync_dirs_task_semaphore = asyncio.Semaphore(cfg.rclone.concurrent_limit)
+    sync_dirs_array = create_dirs_array(sync_paths)
+    sync_dirs_root = create_dir_tree(sync_dirs_array)
+
+    _sync_folders_create_operation = await traverse_and_create_folders_by_depth(
+        root=sync_dirs_root,
+        verbose=verbose,
+        semaphore=sync_dirs_task_semaphore
+    )
+
+    _sync_operation = await backup(
+        paths=sync_paths,
+        command_type=CommandType.COPY,
+        rclone_args=cfg.rclone.args,
+        semaphore=sync_task_semaphore,
+        verbose=verbose
+    )

easyclone/shared/__init__.py

@@ -0,0 +1,3 @@
+from .sync_status import SyncStatus
+
+sync_status = SyncStatus()

easyclone/shared/sync_status.py

@@ -0,0 +1,61 @@
+import asyncio
+from easyclone.utypes.enums import BackupStatus, RcloneOperationType
+from easyclone.utypes.models import SyncStatusItem
+import uuid
+
+class SyncStatus:
+    def __init__(self):
+        self.total_path_count: int = 0
+        self.operations: list[SyncStatusItem]  = []
+        self.lock: asyncio.Lock = asyncio.Lock()
+        self.finished_path_count: int = 0
+
+    async def add_operation(self, source: str, dest: str, path_type: str, status: BackupStatus, operation_type: RcloneOperationType):
+        random_id = str(uuid.uuid4())
+
+        async with self.lock:
+            self.operations.append({
+                "id": random_id,
+                "source": source,
+                "dest": dest,
+                "status": status.value,
+                "path_type": path_type,
+                "operation_type": operation_type.value
+            })
+
+        return random_id
+
+    async def delete_operation(self, target_id: str):
+        async with self.lock:
+            for index, item in enumerate(self.operations):
+                if item.get("id") == target_id:
+                    del self.operations[index]
+                    break;
+
+    async def get_operations(self):
+        async with self.lock:
+            return self.operations
+    
+    async def get_operation_count(self):
+        async with self.lock:
+            return len(self.operations)
+
+    async def set_total_path_count(self, count: int):
+        async with self.lock:
+            self.total_path_count = count
+
+    async def reset_total_paths(self):
+        async with self.lock:
+            self.total_path_count = 0
+
+    async def get_total_path(self):
+        async with self.lock:
+            return self.total_path_count
+
+    async def add_currently_finished(self):
+        async with self.lock:
+            self.finished_path_count += 1
+
+    async def get_currently_finished(self):
+        async with self.lock:
+            return self.finished_path_count

easyclone/utils/__init__.py

@@ -0,0 +1,2 @@
+from .essentials import log
+from .path_manipulation import organize_paths, collapseuser

easyclone/utils/essentials.py

@@ -0,0 +1,24 @@
+from easyclone.utypes.enums import LogLevel, BackupLog
+
+def log(message: str, logtype: LogLevel | BackupLog) -> None:
+    color = "\033[32;1m"
+
+    match logtype:
+        case LogLevel.ERROR | BackupLog.ERR:
+            # RED
+            color = "\033[31;1m"
+        case LogLevel.LOG:
+            # BLUE
+            color = "\033[34;1m"
+        case LogLevel.INFO | BackupLog.OK:
+            # GREEN
+            color = "\033[32;1m"
+        case LogLevel.WARN | BackupLog.WAIT:
+            # YELLOW
+            color = "\033[33;1m"
+
+    full_msg = f"{color}{logtype.value}"
+    if(isinstance(logtype, LogLevel)):
+        full_msg = full_msg + ":"
+
+    print(f"{full_msg}\033[0m {message}")

easyclone/utils/path_manipulation.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+import os
+from easyclone.utypes.enums import PathType
+from easyclone.utypes.models import PathItem
+
+def organize_paths(paths: list[str], remote_name: str) -> list[PathItem]:
+    from easyclone.config import cfg
+    source_dest_array: list[PathItem] = []
+    root_dir = cfg.backup.root_dir
+
+    for path in paths:
+        p = Path(path).expanduser()
+
+        if p.is_dir():
+            source_dest_array.append({
+                "source": f"{p}",
+                "dest": f"{remote_name}:{root_dir}{p}",
+                "path_type": PathType.DIR.value
+            })
+        elif p.is_file():
+            dest_dir = p.parent
+            source_dest_array.append({
+                "source": f"{p}",
+                "dest": f"{remote_name}:{root_dir}{dest_dir}",
+                "path_type": PathType.FILE.value
+            })
+
+    return source_dest_array
+
+def collapseuser(path: str) -> str:
+    """
+    Opposite of path.expanduser()
+    """
+    home = os.path.expanduser("~")
+    if path.startswith(home):
+        return path.replace(home, "~", 1)
+    return path

easyclone/utypes/__init__.py

@@ -0,0 +1,3 @@
+from .config import *
+from .enums import *
+from .models import *

easyclone/utypes/config.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+
+class BackupConfigModel(BaseModel):
+    sync_paths: list[str]
+    copy_paths: list[str]
+    remote_name: str
+    root_dir: str
+    verbose_log: bool
+
+class RcloneConfigModel(BaseModel):
+    args: list[str]
+    concurrent_limit: int
+
+class ConfigModel(BaseModel):
+    backup: BackupConfigModel
+    rclone: RcloneConfigModel