db-sync-tool-kmi 2.11.6__py3-none-any.whl → 3.0.2__py3-none-any.whl

db_sync_tool/remote/client.py

@@ -1,39 +1,181 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
-Client script
+SSH client management for remote connections.
+
+Provides both legacy global access and modern context manager patterns:
+
+Legacy (backwards compatible):
+    from db_sync_tool.remote import client
+    client.load_ssh_client_origin()
+    # use client.ssh_client_origin
+    client.close_ssh_clients()
+
+Modern (recommended):
+    from db_sync_tool.remote.client import SSHClientManager
+    with SSHClientManager() as mgr:
+        mgr.load_origin()
+        # use mgr.origin
 """
 
-import sys
+import warnings
 import paramiko
 from db_sync_tool.utility import mode, system, helper, output
+from db_sync_tool.utility.exceptions import DbSyncError
 
-ssh_client_origin = None
-ssh_client_target = None
-additional_ssh_clients = []
+# Suppress paramiko warnings about unknown host keys
+warnings.filterwarnings("ignore", message="Unknown.*host key", module="paramiko")
 
 default_timeout = 600
 
 
+class SSHClientManager:
+    """
+    Manages SSH client connections with automatic cleanup.
+
+    Can be used as a context manager for guaranteed resource cleanup,
+    or instantiated directly for more control.
+    """
+
+    _instance: 'SSHClientManager | None' = None
+
+    def __init__(self):
+        self._origin: paramiko.SSHClient | None = None
+        self._target: paramiko.SSHClient | None = None
+        self._additional: list[paramiko.SSHClient] = []
+
+    @classmethod
+    def get_instance(cls) -> 'SSHClientManager':
+        """Get or create the singleton instance."""
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @property
+    def origin(self) -> paramiko.SSHClient | None:
+        """Get origin SSH client."""
+        return self._origin
+
+    @property
+    def target(self) -> paramiko.SSHClient | None:
+        """Get target SSH client."""
+        return self._target
+
+    def _update_legacy_globals(self) -> None:
+        """Sync internal state to legacy global variables."""
+        global ssh_client_origin, ssh_client_target, additional_ssh_clients
+        ssh_client_origin = self._origin
+        ssh_client_target = self._target
+        additional_ssh_clients = list(self._additional)
+
+    def load_origin(self) -> paramiko.SSHClient:
+        """Load and return the origin SSH client."""
+        self._origin = load_ssh_client(mode.Client.ORIGIN)
+        self._update_legacy_globals()
+        helper.run_script(mode.Client.ORIGIN, 'before')
+        return self._origin
+
+    def load_target(self) -> paramiko.SSHClient:
+        """Load and return the target SSH client."""
+        self._target = load_ssh_client(mode.Client.TARGET)
+        self._update_legacy_globals()
+        helper.run_script(mode.Client.TARGET, 'before')
+        return self._target
+
+    def add_additional(self, client: paramiko.SSHClient) -> None:
+        """Register an additional SSH client for cleanup."""
+        self._additional.append(client)
+        self._update_legacy_globals()
+
+    def close_all(self) -> None:
+        """
+        Close all managed SSH connections.
+
+        Uses exception handling to ensure all cleanup steps complete,
+        even if individual steps fail. This is important since close_all()
+        is called from __exit__.
+        """
+        errors = []
+
+        # Origin cleanup
+        try:
+            helper.run_script(mode.Client.ORIGIN, 'after')
+        except Exception as e:
+            errors.append(f"origin after-script: {e}")
+        try:
+            if self._origin is not None:
+                self._origin.close()
+        except Exception as e:
+            errors.append(f"origin close: {e}")
+        finally:
+            self._origin = None
+
+        # Target cleanup
+        try:
+            helper.run_script(mode.Client.TARGET, 'after')
+        except Exception as e:
+            errors.append(f"target after-script: {e}")
+        try:
+            if self._target is not None:
+                self._target.close()
+        except Exception as e:
+            errors.append(f"target close: {e}")
+        finally:
+            self._target = None
+
+        # Additional clients cleanup
+        for i, client in enumerate(self._additional):
+            try:
+                client.close()
+            except Exception as e:
+                errors.append(f"additional client {i}: {e}")
+        self._additional.clear()
+
+        # Global after-script
+        try:
+            helper.run_script(script='after')
+        except Exception as e:
+            errors.append(f"global after-script: {e}")
+
+        # Always sync globals
+        self._update_legacy_globals()
+
+        # Log errors if any occurred (but don't raise - cleanup should be silent)
+        if errors:
+            for err in errors:
+                output.message(output.Subject.WARNING, f"Cleanup error: {err}", verbose_only=True)
+
+    def __enter__(self) -> 'SSHClientManager':
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """Context manager exit - ensures cleanup even on errors."""
+        self.close_all()
+
+
+# Legacy global variables for backwards compatibility
+ssh_client_origin: paramiko.SSHClient | None = None
+ssh_client_target: paramiko.SSHClient | None = None
+additional_ssh_clients: list[paramiko.SSHClient] = []
+
+
 def load_ssh_client_origin():
     """
-    Loading the origin ssh client
-    :return:
+    Loading the origin ssh client (legacy function).
+
+    Prefer using SSHClientManager for new code.
     """
-    global ssh_client_origin
-    ssh_client_origin = load_ssh_client(mode.Client.ORIGIN)
-    helper.run_script(mode.Client.ORIGIN, 'before')
+    SSHClientManager.get_instance().load_origin()
 
 
 def load_ssh_client_target():
     """
-    Loading the target ssh client
-    :return:
+    Loading the target ssh client (legacy function).
+
+    Prefer using SSHClientManager for new code.
     """
-    global ssh_client_target
-    ssh_client_target = load_ssh_client(mode.Client.TARGET)
-    helper.run_script(mode.Client.TARGET, 'before')
+    SSHClientManager.get_instance().load_target()
 
 
 def load_ssh_client(ssh):
@@ -42,39 +184,42 @@ def load_ssh_client(ssh):
     :param ssh: String
     :return:
     """
+    cfg = system.get_typed_config()
+    client_cfg = cfg.get_client(ssh)
+
     _host_name = helper.get_ssh_host_name(ssh, True)
     _ssh_client = paramiko.SSHClient()
-    _ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    # Load known hosts from system for security (prevents MITM attacks)
+    _ssh_client.load_system_host_keys()
+    # Log warning for unknown hosts but still accept connection (convenience over strict security)
+    # Note: This does NOT prevent MITM - use RejectPolicy() for strict host verification
+    _ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy())
 
-    _ssh_port = system.config[ssh]['port'] if 'port' in system.config[ssh] else 22
+    _ssh_port = client_cfg.port
     _ssh_key = None
     _ssh_password = None
 
     # Check authentication
-    if 'ssh_key' in system.config[ssh]:
+    if client_cfg.ssh_key:
         _authentication_method = f'{output.CliFormat.BLACK} - ' \
                                  f'(authentication: key){output.CliFormat.ENDC}'
-        _ssh_key = system.config[ssh]['ssh_key']
-    elif 'password' in system.config[ssh]:
+        _ssh_key = client_cfg.ssh_key
+    elif client_cfg.password:
         _authentication_method = f'{output.CliFormat.BLACK} - ' \
-                                 f'authentication: password){output.CliFormat.ENDC}'
-        _ssh_password = system.config[ssh]['password']
-    elif 'ssh_agent' in system.config:
+                                 f'(authentication: password){output.CliFormat.ENDC}'
+        _ssh_password = client_cfg.password
+    elif cfg.ssh_agent:
         _authentication_method = f'{output.CliFormat.BLACK} - ' \
                                  f'(authentication: key){output.CliFormat.ENDC}'
     else:
-        sys.exit(
-            output.message(
-                output.Subject.ERROR,
-                'Missing SSH authentication. Neither ssh key nor ssh password given.',
-                False
-            )
+        raise DbSyncError(
+            'Missing SSH authentication. Neither ssh key nor ssh password given.'
         )
 
     # Try to connect to remote client via paramiko
     try:
-        _ssh_client.connect(hostname=system.config[ssh]['host'],
-                            username=system.config[ssh]['user'],
+        _ssh_client.connect(hostname=client_cfg.host,
+                            username=client_cfg.user,
                             key_filename=_ssh_key,
                             password=_ssh_password,
                             port=_ssh_port,
@@ -88,13 +233,7 @@ def load_ssh_client(ssh):
         _ssh_client.get_transport().set_keepalive(60)
 
     except paramiko.ssh_exception.AuthenticationException:
-        sys.exit(
-            output.message(
-                output.Subject.ERROR,
-                f'SSH authentication for {_host_name} failed',
-                False
-            )
-        )
+        raise DbSyncError(f'SSH authentication for {_host_name} failed') from None
 
     output.message(
         output.host_to_subject(ssh),
@@ -107,21 +246,11 @@ def load_ssh_client(ssh):
 
 def close_ssh_clients():
     """
-    Closing ssh client sessions
-    :return:
-    """
-    helper.run_script(mode.Client.ORIGIN, 'after')
-    if not ssh_client_origin is None:
-        ssh_client_origin.close()
+    Closing ssh client sessions (legacy function).
 
-    helper.run_script(mode.Client.TARGET, 'after')
-    if not ssh_client_target is None:
-        ssh_client_target.close()
-
-    for additional_ssh_client in additional_ssh_clients:
-        additional_ssh_client.close()
-
-    helper.run_script(script='after')
+    Prefer using SSHClientManager as context manager for new code.
+    """
+    SSHClientManager.get_instance().close_all()
 
 
 def get_jump_host_channel(client):
@@ -131,26 +260,32 @@ def get_jump_host_channel(client):
     :param client:
     :return:
     """
+    cfg = system.get_typed_config()
+    client_cfg = cfg.get_client(client)
+
     _jump_host_channel = None
-    if 'jump_host' in system.config[client]:
+    if client_cfg.jump_host is not None:
+        jump_host = client_cfg.jump_host
+
         # prepare jump host config
         _jump_host_client = paramiko.SSHClient()
-        _jump_host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        _jump_host_client.load_system_host_keys()
+        _jump_host_client.set_missing_host_key_policy(paramiko.WarningPolicy())
 
-        _jump_host_host = system.config[client]['jump_host']['host']
-        _jump_host_user = system.config[client]['jump_host']['user'] if 'user' in system.config[client]['jump_host'] else system.config[client]['user']
+        _jump_host_host = jump_host.host
+        _jump_host_user = jump_host.user if jump_host.user else client_cfg.user
 
-        if 'ssh_key' in system.config[client]['jump_host']:
-            _jump_host_ssh_key = system.config[client]['jump_host']['ssh_key']
-        elif 'ssh_key' in system.config[client]:
-            _jump_host_ssh_key = system.config[client]['ssh_key']
+        if jump_host.ssh_key:
+            _jump_host_ssh_key = jump_host.ssh_key
+        elif client_cfg.ssh_key:
+            _jump_host_ssh_key = client_cfg.ssh_key
         else:
             _jump_host_ssh_key = None
 
-        if 'port' in system.config[client]['jump_host']:
-            _jump_host_port = system.config[client]['jump_host']['port']
-        elif 'port' in system.config[client]:
-            _jump_host_port = system.config[client]['port']
+        if jump_host.port:
+            _jump_host_port = jump_host.port
+        elif client_cfg.port:
+            _jump_host_port = client_cfg.port
         else:
             _jump_host_port = 22
 
@@ -159,26 +294,26 @@ def get_jump_host_channel(client):
             hostname=_jump_host_host,
             username=_jump_host_user,
             key_filename=_jump_host_ssh_key,
-            password=system.config[client]['jump_host']['password'] if 'password' in system.config[client]['jump_host'] else None,
+            password=jump_host.password,
             port=_jump_host_port,
             compress=True,
             timeout=default_timeout
         )
 
-        global additional_ssh_clients
-        additional_ssh_clients.append(_jump_host_client)
+        # Register for cleanup via manager (also updates legacy global)
+        SSHClientManager.get_instance().add_additional(_jump_host_client)
 
         # open the necessary channel
         _jump_host_transport = _jump_host_client.get_transport()
         _jump_host_channel = _jump_host_transport.open_channel(
             'direct-tcpip',
-            dest_addr=(system.config[client]['host'], 22),
-            src_addr=(system.config[client]['jump_host']['private'] if 'private' in system.config[client]['jump_host'] else system.config[client]['jump_host']['host'], 22)
+            dest_addr=(client_cfg.host, 22),
+            src_addr=(jump_host.private if jump_host.private else jump_host.host, 22)
         )
 
         # print information
         _destination_client = helper.get_ssh_host_name(client, minimal=True)
-        _jump_host_name = system.config[client]['jump_host']['name'] if 'name' in system.config[client]['jump_host'] else _jump_host_host
+        _jump_host_name = jump_host.name if jump_host.name else _jump_host_host
         output.message(
             output.host_to_subject(client),
             f'Initialize remote SSH jump host {output.CliFormat.BLACK}local ➔ {output.CliFormat.BOLD}{_jump_host_name}{output.CliFormat.ENDC}{output.CliFormat.BLACK} ➔ {_destination_client}{output.CliFormat.ENDC}',

db_sync_tool/remote/file_transfer.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+
+"""
+File transfer module using rsync.
+
+Provides file synchronization functionality integrated with db-sync-tool's
+existing infrastructure for SSH authentication and rsync handling.
+"""
+
+import shutil
+from pathlib import Path
+
+from db_sync_tool.utility import mode, system, output, helper
+from db_sync_tool.utility.config import FileTransferConfig
+from db_sync_tool.remote import rsync
+
+# Temporary directory for PROXY mode file transfers
+_temp_file_dir: str | None = None
+
+
+def transfer_files() -> None:
+    """
+    Transfer configured files between clients.
+
+    Iterates through the files configuration and synchronizes each
+    origin/target pair using rsync.
+    """
+    cfg = system.get_typed_config()
+
+    # Check if file transfer is enabled
+    if not cfg.with_files and not cfg.files_only:
+        return
+
+    # Skip if no files configured
+    if not cfg.files:
+        output.message(
+            output.Subject.WARNING,
+            'File transfer enabled but no files configured',
+            True
+        )
+        return
+
+    # Skip file transfer for dump/import modes
+    if mode.is_dump() or mode.is_import():
+        output.message(
+            output.Subject.WARNING,
+            'File transfer not available in dump/import mode',
+            True
+        )
+        return
+
+    output.message(
+        output.Subject.INFO,
+        f'Starting file transfer ({len(cfg.files)} configured)',
+        True
+    )
+
+    for i, file_config in enumerate(cfg.files, 1):
+        output.message(
+            output.Subject.INFO,
+            f'[{i}/{len(cfg.files)}] {file_config.origin} -> {file_config.target}',
+            True
+        )
+        _transfer_single(file_config)
+
+    output.message(
+        output.Subject.INFO,
+        'File transfer completed',
+        True
+    )
+
+
+def _transfer_single(file_config: FileTransferConfig) -> None:
+    """
+    Transfer a single file configuration entry.
+
+    :param file_config: FileTransferConfig instance
+    """
+    cfg = system.get_typed_config()
+
+    origin_path = _resolve_path(file_config.origin, mode.Client.ORIGIN)
+    target_path = _resolve_path(file_config.target, mode.Client.TARGET)
+
+    if cfg.dry_run:
+        output.message(
+            output.Subject.INFO,
+            f'[DRY RUN] Would sync {origin_path} -> {target_path}',
+            True
+        )
+        return
+
+    sync_mode_val = mode.get_sync_mode()
+
+    if sync_mode_val == mode.SyncMode.PROXY:
+        _transfer_proxy(origin_path, target_path, file_config)
+    elif sync_mode_val == mode.SyncMode.SYNC_REMOTE:
+        _transfer_remote_to_remote(origin_path, target_path, file_config)
+    else:
+        _transfer_standard(origin_path, target_path, file_config)
+
+
+def _resolve_path(path: str, client: str) -> str:
+    """
+    Resolve file path, making relative paths absolute.
+
+    :param path: Path (relative or absolute)
+    :param client: Client identifier (origin/target)
+    :return: Resolved absolute path
+    """
+    if path.startswith('/'):
+        return path
+
+    base_path = system.get_typed_config().get_client(client).path
+    if not base_path:
+        return path
+
+    # Preserve trailing slash for rsync (important for directory sync behavior)
+    resolved = str(Path(base_path).parent / path)
+    if path.endswith('/') and not resolved.endswith('/'):
+        resolved += '/'
+    return resolved
+
+
+def _get_excludes(excludes: list[str]) -> str:
+    """
+    Build rsync exclude arguments.
+
+    :param excludes: List of patterns to exclude
+    :return: String with --exclude flags
+    """
+    if not excludes:
+        return ''
+    return ' '.join(f'--exclude={helper.quote_shell_arg(e)}' for e in excludes)
+
+
+def _get_file_options(file_config: FileTransferConfig) -> str:
+    """
+    Get rsync options for file transfer.
+
+    :param file_config: FileTransferConfig instance
+    :return: Combined options string
+    """
+    cfg = system.get_typed_config()
+    base_options = rsync.get_options()
+
+    # Add global file options
+    if cfg.files_options:
+        base_options += f' {cfg.files_options}'
+
+    # Add per-transfer options
+    if file_config.options:
+        base_options += f' {file_config.options}'
+
+    return base_options
+
+
+def _transfer_standard(origin_path: str, target_path: str,
+                       file_config: FileTransferConfig) -> None:
+    """
+    Standard file transfer (RECEIVER, SENDER, SYNC_LOCAL modes).
+
+    :param origin_path: Source path
+    :param target_path: Destination path
+    :param file_config: FileTransferConfig instance
+    """
+    # Determine which client is remote
+    if mode.is_origin_remote():
+        remote_client = mode.Client.ORIGIN
+    elif mode.is_target_remote():
+        remote_client = mode.Client.TARGET
+    else:
+        remote_client = None
+
+    _run_rsync(
+        remote_client=remote_client,
+        origin_path=origin_path,
+        target_path=target_path,
+        file_config=file_config
+    )
+
+
+def _transfer_proxy(origin_path: str, target_path: str,
+                    file_config: FileTransferConfig) -> None:
+    """
+    Proxy mode file transfer (remote -> local -> remote).
+
+    :param origin_path: Source path
+    :param target_path: Destination path
+    :param file_config: FileTransferConfig instance
+    """
+    global _temp_file_dir
+
+    # Create local temp directory
+    _temp_file_dir = system.default_local_sync_path + 'files/'
+    helper.check_and_create_dump_dir(mode.Client.LOCAL, _temp_file_dir)
+
+    output.message(
+        output.Subject.INFO,
+        'Proxy mode: origin -> local',
+        True
+    )
+
+    # Step 1: Origin -> Local
+    _run_rsync(
+        remote_client=mode.Client.ORIGIN,
+        origin_path=origin_path,
+        target_path=_temp_file_dir,
+        file_config=file_config
+    )
+
+    output.message(
+        output.Subject.INFO,
+        'Proxy mode: local -> target',
+        True
+    )
+
+    # Step 2: Local -> Target
+    _run_rsync(
+        remote_client=mode.Client.TARGET,
+        origin_path=_temp_file_dir,
+        target_path=target_path,
+        file_config=file_config
+    )
+
+
+def _transfer_remote_to_remote(origin_path: str, target_path: str,
+                                file_config: FileTransferConfig) -> None:
+    """
+    Remote-to-remote file transfer (SYNC_REMOTE mode).
+
+    Executes rsync on origin to sync directly to target.
+
+    :param origin_path: Source path
+    :param target_path: Destination path
+    :param file_config: FileTransferConfig instance
+    """
+    cfg = system.get_typed_config()
+
+    # Build rsync command to run on origin
+    excludes = _get_excludes(file_config.exclude)
+    options = _get_file_options(file_config)
+    password_env = rsync.get_password_environment(mode.Client.TARGET)
+    auth = rsync.get_authorization(mode.Client.TARGET)
+    target_host = f'{cfg.target.user}@{cfg.target.host}:'
+
+    command = (
+        f'{password_env}rsync {options} {auth} '
+        f'{excludes} {origin_path} {target_host}{target_path}'
+    ).strip()
+
+    # Clean up multiple spaces
+    command = ' '.join(command.split())
+
+    mode.run_command(command, mode.Client.ORIGIN, True)
+
+
+def _run_rsync(remote_client: str | None, origin_path: str, target_path: str,
+               file_config: FileTransferConfig) -> None:
+    """
+    Execute rsync command for file transfer.
+
+    :param remote_client: Client identifier for SSH authentication (or None for local)
+    :param origin_path: Source path
+    :param target_path: Destination path
+    :param file_config: FileTransferConfig instance
+    """
+    excludes = _get_excludes(file_config.exclude)
+    options = _get_file_options(file_config)
+
+    # Build origin and target host prefixes
+    if remote_client == mode.Client.ORIGIN:
+        origin_host = rsync.get_host(mode.Client.ORIGIN)
+        target_host = ''
+    elif remote_client == mode.Client.TARGET:
+        origin_host = ''
+        target_host = rsync.get_host(mode.Client.TARGET)
+    else:
+        origin_host = ''
+        target_host = ''
+
+    # Build the rsync command
+    password_env = rsync.get_password_environment(remote_client) if remote_client else ''
+    auth = rsync.get_authorization(remote_client) if remote_client else ''
+
+    command = (
+        f'{password_env}rsync {options} {auth} {excludes} '
+        f'{origin_host}{origin_path} {target_host}{target_path}'
+    ).strip()
+
+    # Clean up multiple spaces
+    command = ' '.join(command.split())
+
+    output_str = mode.run_command(command, mode.Client.LOCAL, True)
+    if output_str:
+        rsync.read_stats(output_str)
+
+
+def cleanup() -> None:
+    """Clean up temporary file transfer directory."""
+    global _temp_file_dir
+    if _temp_file_dir and Path(_temp_file_dir).exists():
+        shutil.rmtree(_temp_file_dir, ignore_errors=True)
+        _temp_file_dir = None