db-sync-tool-kmi 2.11.6__py3-none-any.whl → 3.0.2__py3-none-any.whl

db_sync_tool/utility/system.py

@@ -1,24 +1,30 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
 System module
 """
 
-import sys
 import json
 import os
 import getpass
+import secrets
+from pathlib import Path
+from typing import Any, TYPE_CHECKING
 import yaml
 from db_sync_tool.utility import log, parser, mode, helper, output, validation
+from db_sync_tool.utility.console import get_output_manager
+from db_sync_tool.utility.exceptions import ConfigError
 from db_sync_tool.remote import utility as remote_utility
 
+if TYPE_CHECKING:
+    from db_sync_tool.utility.config import SyncConfig
+
 #
 # GLOBALS
 #
 
-config = {
-    'verbose': False,
+config: dict[str, Any] = {
+    'verbose': 0,  # 0=compact, 1=verbose, 2=debug
     'mute': False,
     'dry_run': False,
     'keep_dump': False,
@@ -32,9 +38,11 @@ config = {
     'config_file_path': None,
     'clear_database': False,
     'force_password': False,
-    'use_rsync': False,
+    'use_rsync': True,  # rsync is 5-10x faster than Paramiko SFTP
     'use_rsync_options': None,
     'use_sshpass': False,
+    'with_files': False,  # Enable file synchronization (opt-in)
+    'files_only': False,  # Sync only files, skip database
     'ssh_agent': False,
     'ssh_password': {
         mode.Client.ORIGIN: None,
@@ -47,11 +55,94 @@ config = {
     'additional_mysqldump_options': ''
 }
 
+# Typed configuration (single source of truth after migration)
+_typed_config: 'SyncConfig | None' = None
+
+
+def get_typed_config() -> 'SyncConfig':
+    """
+    Get current configuration as typed SyncConfig object.
+
+    :return: SyncConfig instance
+    """
+    global _typed_config
+    if _typed_config is None:
+        from db_sync_tool.utility.config import SyncConfig
+        _typed_config = SyncConfig.from_dict(config)
+    return _typed_config
+
+
+def refresh_typed_config() -> None:
+    """
+    Refresh typed config after dict changes.
+
+    Call this after modifying system.config to keep _typed_config in sync.
+    This is needed during the migration phase where both dict and dataclass
+    are used.
+    """
+    global _typed_config
+    from db_sync_tool.utility.config import SyncConfig
+    _typed_config = SyncConfig.from_dict(config)
+
+
+def _set_config_value(key: str, value, client: str | None = None) -> None:
+    """
+    Set a configuration value and refresh typed config.
+
+    :param key: Configuration key
+    :param value: Value to set
+    :param client: Optional client identifier for nested config
+    """
+    if client:
+        config.setdefault(client, {})[key] = value
+    else:
+        config[key] = value
+    refresh_typed_config()
+
+
+def set_database_config(client: str, db_config: dict) -> None:
+    """Set database configuration for a client."""
+    _set_config_value('db', db_config, client)
+
+
+def set_framework_type(type_name: str) -> None:
+    """Set the framework type."""
+    _set_config_value('type', type_name)
+
+
+def set_is_same_client(value: bool) -> None:
+    """Set is_same_client flag."""
+    _set_config_value('is_same_client', value)
+
+
+def set_use_sshpass(value: bool) -> None:
+    """Set use_sshpass flag."""
+    _set_config_value('use_sshpass', value)
+
+
 #
 # DEFAULTS
 #
 
-default_local_sync_path = '/tmp/db_sync_tool/'
+# Generate a secure random suffix to prevent predictable temp paths
+_temp_suffix = secrets.token_hex(8)
+default_local_sync_path = f'/tmp/db_sync_tool_{_temp_suffix}/'
+
+
+def create_secure_temp_dir(path):
+    """
+    Create a temporary directory with secure permissions (0700).
+    Prevents other users from accessing sensitive database dumps.
+
+    :param path: String path to create
+    :return: String path
+    """
+    if not os.path.exists(path):
+        os.makedirs(path, mode=0o700)
+    else:
+        # Ensure secure permissions even if directory exists
+        os.chmod(path, 0o700)
+    return path
 
 
 #
@@ -70,7 +161,7 @@ def get_configuration(host_config, args = {}):
     """
     Checking configuration information by file or dictionary
     :param host_config: Dictionary
-    :param args: Dictionary
+    :param args: Dictionary (or argparse.Namespace with resolved_config attribute)
     :return:
     """
     global config
@@ -92,13 +183,9 @@ def get_configuration(host_config, args = {}):
                 elif _config_file_path.endswith('.yaml') or _config_file_path.endswith('.yml'):
                     config.update(yaml.safe_load(read_file))
                 else:
-                    sys.exit(
-                        output.message(
-                            output.Subject.ERROR,
-                            f'Unsupported configuration file type [json,yml,yaml]: '
-                            f'{config["config_file_path"]}',
-                            False
-                        )
+                    raise ConfigError(
+                        f'Unsupported configuration file type [json,yml,yaml]: '
+                        f'{config["config_file_path"]}'
                     )
                 output.message(
                     output.Subject.LOCAL,
@@ -107,14 +194,13 @@ def get_configuration(host_config, args = {}):
                     True
                 )
         else:
-            sys.exit(
-                output.message(
-                    output.Subject.ERROR,
-                    f'Local configuration not found: {config["config_file_path"]}',
-                    False
-                )
+            raise ConfigError(
+                f'Local configuration not found: {config["config_file_path"]}'
             )
 
+    # Apply resolved config from ConfigResolver (if present)
+    _apply_resolved_config(args)
+
     # workaround for argument order handling respecting the linking feature
     build_config(args, True)
     link_configuration_with_hosts()
@@ -124,138 +210,137 @@ def get_configuration(host_config, args = {}):
     check_options()
 
     if not config[mode.Client.TARGET] and not config[mode.Client.ORIGIN]:
-        sys.exit(
-            output.message(
-                output.Subject.ERROR,
-                f'Configuration is missing, use a separate file or provide host parameter',
-                False
-            )
+        raise ConfigError(
+            'Configuration is missing, use a separate file or provide host parameter'
         )
+
+    # Refresh typed config after all configuration is loaded
+    refresh_typed_config()
+
     helper.run_script(script='before')
     log.get_logger().info('Starting db_sync_tool')
 
 
-def build_config(args, pre_run = False):
+def _apply_resolved_config(args) -> None:
     """
-    ADding the provided arguments
-    :param args:
-    :param pre_run:
-    :return:
-    """
-    if args is None or not args:
-        return {}
-
-    if not args.type is None:
-        config['type'] = args.type
-
-    if not args.tables is None:
-        config['tables'] = args.tables
-
-    if not args.origin is None:
-        config['link_origin'] = args.origin
-
-    if not args.target is None:
-        config['link_target'] = args.target
-
-    # for order reasons check just the link arguments
-    if pre_run: return
-
-    if not args.target_path is None:
-        config[mode.Client.TARGET]['path'] = args.target_path
-
-    if not args.target_name is None:
-        config[mode.Client.TARGET]['name'] = args.target_name
-
-    if not args.target_host is None:
-        config[mode.Client.TARGET]['host'] = args.target_host
-
-    if not args.target_user is None:
-        config[mode.Client.TARGET]['user'] = args.target_user
+    Apply resolved config from ConfigResolver to the global config.
 
-    if not args.target_password is None:
-        config[mode.Client.TARGET]['password'] = args.target_password
-
-    if not args.target_key is None:
-        config[mode.Client.TARGET]['ssh_key'] = args.target_key
-
-    if not args.target_port is None:
-        config[mode.Client.TARGET]['port'] = args.target_port
-
-    if not args.target_dump_dir is None:
-        config[mode.Client.TARGET]['dump_dir'] = args.target_dump_dir
-
-    if not args.target_db_name is None:
-        check_config_dict_key(mode.Client.TARGET, 'db')
-        config[mode.Client.TARGET]['db']['name'] = args.target_db_name
-
-    if not args.target_db_host is None:
-        check_config_dict_key(mode.Client.TARGET, 'db')
-        config[mode.Client.TARGET]['db']['host'] = args.target_db_host
-
-    if not args.target_db_user is None:
-        check_config_dict_key(mode.Client.TARGET, 'db')
-        config[mode.Client.TARGET]['db']['user'] = args.target_db_user
-
-    if not args.target_db_password is None:
-        check_config_dict_key(mode.Client.TARGET, 'db')
-        config[mode.Client.TARGET]['db']['password'] = args.target_db_password
-
-    if not args.target_db_port is None:
-        check_config_dict_key(mode.Client.TARGET, 'db')
-        config[mode.Client.TARGET]['db']['port'] = args.target_db_port
-
-    if not args.target_after_dump is None:
-        config[mode.Client.TARGET]['after_dump'] = args.target_after_dump
-
-    if not args.origin_path is None:
-        config[mode.Client.ORIGIN]['path'] = args.origin_path
-
-    if not args.origin_name is None:
-        config[mode.Client.ORIGIN]['name'] = args.origin_name
-
-    if not args.origin_host is None:
-        config[mode.Client.ORIGIN]['host'] = args.origin_host
-
-    if not args.origin_user is None:
-        config[mode.Client.ORIGIN]['user'] = args.origin_user
+    :param args: argparse.Namespace or dict that may contain resolved_config
+    """
+    global config
 
-    if not args.origin_password is None:
-        config[mode.Client.ORIGIN]['password'] = args.origin_password
+    # Get resolved_config from args (if present)
+    resolved_config = getattr(args, 'resolved_config', None)
+    if resolved_config is None:
+        return
 
-    if not args.origin_key is None:
-        config[mode.Client.ORIGIN]['ssh_key'] = args.origin_key
+    # Apply merged config first (global defaults + project defaults)
+    if resolved_config.merged_config:
+        for key, value in resolved_config.merged_config.items():
+            if key not in ('origin', 'target'):
+                config[key] = value
 
-    if not args.origin_port is None:
-        config[mode.Client.ORIGIN]['port'] = args.origin_port
+    # Apply origin config
+    if resolved_config.origin_config:
+        config[mode.Client.ORIGIN] = {**config[mode.Client.ORIGIN], **resolved_config.origin_config}
 
-    if not args.origin_dump_dir is None:
-        config[mode.Client.ORIGIN]['dump_dir'] = args.origin_dump_dir
+    # Apply target config
+    if resolved_config.target_config:
+        config[mode.Client.TARGET] = {**config[mode.Client.TARGET], **resolved_config.target_config}
 
-    if not args.origin_db_name is None:
-        check_config_dict_key(mode.Client.ORIGIN, 'db')
-        config[mode.Client.ORIGIN]['db']['name'] = args.origin_db_name
+    # Log the source
+    if resolved_config.source:
+        output.message(
+            output.Subject.INFO,
+            f'Configuration resolved from {resolved_config.source}',
+            True
+        )
 
-    if not args.origin_db_host is None:
-        check_config_dict_key(mode.Client.ORIGIN, 'db')
-        config[mode.Client.ORIGIN]['db']['host'] = args.origin_db_host
 
-    if not args.origin_db_user is None:
-        check_config_dict_key(mode.Client.ORIGIN, 'db')
-        config[mode.Client.ORIGIN]['db']['user'] = args.origin_db_user
+# Argument mapping: (arg_name, config_path)
+# config_path is a tuple: (client, key) or (client, nested_key, key) or just (key,) for top-level
+_ARG_MAPPINGS_PRE_RUN = [
+    ('type', ('type',)),
+    ('tables', ('tables',)),
+    ('origin', ('link_origin',)),
+    ('target', ('link_target',)),
+]
+
+_ARG_MAPPINGS_MAIN = [
+    # Target client mappings
+    ('target_path', (mode.Client.TARGET, 'path')),
+    ('target_name', (mode.Client.TARGET, 'name')),
+    ('target_host', (mode.Client.TARGET, 'host')),
+    ('target_user', (mode.Client.TARGET, 'user')),
+    ('target_password', (mode.Client.TARGET, 'password')),
+    ('target_key', (mode.Client.TARGET, 'ssh_key')),
+    ('target_port', (mode.Client.TARGET, 'port')),
+    ('target_dump_dir', (mode.Client.TARGET, 'dump_dir')),
+    ('target_after_dump', (mode.Client.TARGET, 'after_dump')),
+    ('target_db_name', (mode.Client.TARGET, 'db', 'name')),
+    ('target_db_host', (mode.Client.TARGET, 'db', 'host')),
+    ('target_db_user', (mode.Client.TARGET, 'db', 'user')),
+    ('target_db_password', (mode.Client.TARGET, 'db', 'password')),
+    ('target_db_port', (mode.Client.TARGET, 'db', 'port')),
+    # Origin client mappings
+    ('origin_path', (mode.Client.ORIGIN, 'path')),
+    ('origin_name', (mode.Client.ORIGIN, 'name')),
+    ('origin_host', (mode.Client.ORIGIN, 'host')),
+    ('origin_user', (mode.Client.ORIGIN, 'user')),
+    ('origin_password', (mode.Client.ORIGIN, 'password')),
+    ('origin_key', (mode.Client.ORIGIN, 'ssh_key')),
+    ('origin_port', (mode.Client.ORIGIN, 'port')),
+    ('origin_dump_dir', (mode.Client.ORIGIN, 'dump_dir')),
+    ('origin_db_name', (mode.Client.ORIGIN, 'db', 'name')),
+    ('origin_db_host', (mode.Client.ORIGIN, 'db', 'host')),
+    ('origin_db_user', (mode.Client.ORIGIN, 'db', 'user')),
+    ('origin_db_password', (mode.Client.ORIGIN, 'db', 'password')),
+    ('origin_db_port', (mode.Client.ORIGIN, 'db', 'port')),
+    # Top-level config mappings
+    ('where', ('where',)),
+    ('additional_mysqldump_options', ('additional_mysqldump_options',)),
+]
+
+
+def _apply_arg_mapping(args, mapping):
+    """
+    Apply argument mappings to config dict.
 
-    if not args.origin_db_password is None:
-        check_config_dict_key(mode.Client.ORIGIN, 'db')
-        config[mode.Client.ORIGIN]['db']['password'] = args.origin_db_password
+    :param args: Argument namespace
+    :param mapping: List of (arg_name, config_path) tuples
+    """
+    for arg_name, config_path in mapping:
+        value = getattr(args, arg_name, None)
+        if value is None:
+            continue
+
+        if len(config_path) == 1:
+            # Top-level config key
+            config[config_path[0]] = value
+        elif len(config_path) == 2:
+            # Client-level key: (client, key)
+            config[config_path[0]][config_path[1]] = value
+        elif len(config_path) == 3:
+            # Nested client key: (client, nested_key, key)
+            check_config_dict_key(config_path[0], config_path[1])
+            config[config_path[0]][config_path[1]][config_path[2]] = value
+
+
+def build_config(args, pre_run=False):
+    """
+    Apply provided CLI arguments to config dict.
 
-    if not args.origin_db_port is None:
-        check_config_dict_key(mode.Client.ORIGIN, 'db')
-        config[mode.Client.ORIGIN]['db']['port'] = args.origin_db_port
+    :param args: Argument namespace from argparse
+    :param pre_run: Boolean, if True only apply link-related args
+    :return: config dict
+    """
+    if args is None or not args:
+        return {}
 
-    if not args.where is None:
-        config['where'] = args.where
+    _apply_arg_mapping(args, _ARG_MAPPINGS_PRE_RUN)
 
-    if not args.additional_mysqldump_options is None:
-        config['additional_mysqldump_options'] = args.additional_mysqldump_options
+    if not pre_run:
+        _apply_arg_mapping(args, _ARG_MAPPINGS_MAIN)
 
     return config
 
@@ -275,6 +360,19 @@ def check_options():
     if 'check_dump' in config:
         config['check_dump'] = config['check_dump']
 
+    # Check rsync availability if enabled (default: True)
+    # Falls back to Paramiko SFTP if rsync is not available
+    if config['use_rsync']:
+        if helper.check_rsync_version():
+            helper.check_sshpass_version()
+        else:
+            config['use_rsync'] = False
+            output.message(
+                output.Subject.WARNING,
+                'rsync not found, falling back to SFTP transfer',
+                True
+            )
+
     reverse_hosts()
     mode.check_sync_mode()
 
@@ -286,6 +384,8 @@ def check_authorizations():
     """
     check_authorization(mode.Client.ORIGIN)
     check_authorization(mode.Client.TARGET)
+    # Refresh typed config after authorization changes (ssh_agent, password)
+    refresh_typed_config()
 
 
 def check_authorization(client):
@@ -311,13 +411,7 @@ def check_authorization(client):
         elif 'ssh_key' in config[client]:
             _ssh_key = config[client]['ssh_key']
             if not os.path.isfile(_ssh_key):
-                sys.exit(
-                    output.message(
-                        output.Subject.ERROR,
-                        f'SSH {client} private key not found: {_ssh_key}',
-                        False
-                    )
-                )
+                raise ConfigError(f'SSH {client} private key not found: {_ssh_key}')
         elif 'password' in config[client]:
             config[client]['password'] = config[client]['password']
         elif remote_utility.check_keys_from_ssh_agent():
@@ -338,13 +432,11 @@ def get_password_by_user(client):
     :param client: String
     :return: String password
     """
-    _password = getpass.getpass(
-        output.message(
-            output.Subject.INFO,
-            'SSH password ' + helper.get_ssh_host_name(client, True) + ': ',
-            False
-        )
+    _prompt = get_output_manager().build_prompt(
+        f'SSH password {helper.get_ssh_host_name(client, True)}: ',
+        subject='INFO'
     )
+    _password = getpass.getpass(_prompt)
 
     while _password.strip() == '':
         output.message(
@@ -353,13 +445,11 @@ def get_password_by_user(client):
             True
         )
 
-        _password = getpass.getpass(
-            output.message(
-                output.Subject.INFO,
-                'SSH password ' + helper.get_ssh_host_name(client, True) + ': ',
-                False
-            )
+        _prompt = get_output_manager().build_prompt(
+            f'SSH password {helper.get_ssh_host_name(client, True)}: ',
+            subject='INFO'
         )
+        _password = getpass.getpass(_prompt)
 
     return _password
 
@@ -377,7 +467,9 @@ def check_args_options(config_file=None,
                        force_password=False,
                        use_rsync=False,
                        use_rsync_options=None,
-                       reverse=False):
+                       reverse=False,
+                       with_files=False,
+                       files_only=False):
     """
     Checking arguments and fill options array
     :param config_file:
@@ -394,6 +486,8 @@ def check_args_options(config_file=None,
     :param use_rsync:
     :param use_rsync_options:
     :param reverse:
+    :param with_files:
+    :param files_only:
     :return:
     """
     global config
@@ -436,15 +530,11 @@ def check_args_options(config_file=None,
     if not force_password is None:
         config['force_password'] = force_password
 
-    if not use_rsync is None:
+    if use_rsync is not None:
         config['use_rsync'] = use_rsync
 
-        if use_rsync is True:
-            helper.check_rsync_version()
-            helper.check_sshpass_version()
-
-        if not use_rsync_options is None:
-            config['use_rsync_options'] = use_rsync_options
+    if use_rsync_options is not None:
+        config['use_rsync_options'] = use_rsync_options
 
     if not reverse is None:
         config['reverse'] = reverse
@@ -463,10 +553,19 @@ def check_args_options(config_file=None,
             True
         )
 
+    if with_files is not None:
+        config['with_files'] = with_files
+
+    if files_only is not None:
+        config['files_only'] = files_only
+        if files_only:
+            # files_only implies with_files
+            config['with_files'] = True
+
 
 def reverse_hosts():
     """
-    Checking authorization for clients
+    Reverse origin and target hosts if --reverse flag is set.
     :return:
     """
     if config['reverse']:
@@ -476,6 +575,9 @@ def reverse_hosts():
         config[mode.Client.ORIGIN] = _target
         config[mode.Client.TARGET] = _origin
 
+        # Refresh typed config after swapping
+        refresh_typed_config()
+
         output.message(
             output.Subject.INFO,
             'Reverse origin and target hosts',
@@ -500,22 +602,19 @@ def link_configuration_with_hosts():
 
         if config['link_hosts'] == '':
             # Try to find default hosts.json file in same directory
-            sys.exit(
-                output.message(
-                    output.Subject.ERROR,
-                    f'Missing hosts file for linking hosts with configuration. '
-                    f'Use the "-o" / "--hosts" argument to define the filepath for the hosts file, '
-                    f'when using a link parameter within the configuration or define the the '
-                    f'filepath direct in the link entry e.g. "host.yaml@entry1".',
-                    False
-                )
+            raise ConfigError(
+                'Missing hosts file for linking hosts with configuration. '
+                'Use the "-o" / "--hosts" argument to define the filepath for the hosts file, '
+                'when using a link parameter within the configuration or define the '
+                'filepath direct in the link entry e.g. "host.yaml@entry1".'
             )
 
     if config['link_hosts'] != '':
 
         # Adjust filepath from relative to absolute
-        if config['link_hosts'][0] != '/':
-            config['link_hosts'] = os.path.dirname(os.path.abspath(config['config_file_path'])) + '/' + config['link_hosts']
+        if not config['link_hosts'].startswith('/'):
+            base_path = Path(config['config_file_path']).resolve().parent if config['config_file_path'] else Path.cwd()
+            config['link_hosts'] = str(base_path / config['link_hosts'])
 
         if os.path.isfile(config['link_hosts']):
             with open(config['link_hosts'], 'r') as read_file:
@@ -545,30 +644,14 @@ def link_configuration_with_hosts():
                             config[mode.Client.TARGET] = _hosts[config['link_target']]
                             config[mode.Client.ORIGIN] = _hosts[config['link_origin']]
                         else:
-                            sys.exit(
-                                output.message(
-                                    output.Subject.ERROR,
-                                    f'Misconfiguration of link hosts {config["link_origin"]}, '
-                                    f'{config["link_target"]} in {config["link_hosts"]}',
-                                    False
-                                )
+                            raise ConfigError(
+                                f'Misconfiguration of link hosts {config["link_origin"]}, '
+                                f'{config["link_target"]} in {config["link_hosts"]}'
                             )
                     else:
-                        sys.exit(
-                            output.message(
-                                output.Subject.ERROR,
-                                f'Missing link hosts for {config["link_hosts"]}',
-                                False
-                            )
-                        )
+                        raise ConfigError(f'Missing link hosts for {config["link_hosts"]}')
         else:
-            sys.exit(
-                output.message(
-                    output.Subject.ERROR,
-                    f'Local host file not found: {config["link_hosts"]}',
-                    False
-                )
-            )
+            raise ConfigError(f'Local host file not found: {config["link_hosts"]}')
 
 
 def check_config_dict_key(client, key):

db_sync_tool/utility/validation.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
 Validation script
 """
 
-import sys
 from jsonschema import validators
 from db_sync_tool.utility import output
+from db_sync_tool.utility.exceptions import ValidationError
 
 #
 # GLOBALS
@@ -107,10 +106,4 @@ def check(config):
             True
         )
     if errors:
-        sys.exit(
-            output.message(
-                output.Subject.ERROR,
-                'Validation error(s)',
-                do_print=False
-            )
-        )
+        raise ValidationError('Configuration validation failed')